diff --git a/database/csvx/blank_test.go b/database/csvx/blank_test.go new file mode 100644 index 0000000..8846921 --- /dev/null +++ b/database/csvx/blank_test.go @@ -0,0 +1,84 @@ +package csvx + +import ( + "encoding/csv" + "strings" + "testing" +) + +type blankRec struct { + A string `json:"a"` + B string `json:"b"` +} + +func parseBlank[T any](in string) ([]T, error) { + c := csv.NewReader(strings.NewReader(in)) + c.FieldsPerRecord = -1 // ragged-tolerant, as sheet readers configure + return ParseCSV[T](c) +} + +// Leading blank rows must not be taken as the header (Google Sheets +// exports with an empty row 1). +func TestParseCSVBlankHeaderRow(t *testing.T) { + items, err := parseBlank[blankRec](",,,,\r\n\r\na,b\r\n1,x\r\n2,y\r\n") + if err != nil { + t.Fatalf("parse: %v", err) + } + want := []blankRec{{A: "1", B: "x"}, {A: "2", B: "y"}} + if len(items) != len(want) { + t.Fatalf("rows: %d, want %d (%+v)", len(items), len(want), items) + } + for i := range want { + if items[i] != want[i] { + t.Errorf("row %d: %+v, want %+v", i, items[i], want[i]) + } + } +} + +// Blank and whitespace-only data rows are skipped. +func TestParseCSVBlankDataRows(t *testing.T) { + items, err := parseBlank[blankRec]("a,b\r\n1,x\r\n\r\n ,\r\n2,y\r\n") + if err != nil { + t.Fatalf("parse: %v", err) + } + if len(items) != 2 || items[0] != (blankRec{A: "1", B: "x"}) || items[1] != (blankRec{A: "2", B: "y"}) { + t.Fatalf("rows: %+v", items) + } +} + +// Normal input is unchanged. +func TestParseCSVNoBlanks(t *testing.T) { + items, err := parseBlank[blankRec]("a,b\r\n1,x\r\n2,y\r\n") + if err != nil { + t.Fatalf("parse: %v", err) + } + if len(items) != 2 || items[1] != (blankRec{A: "2", B: "y"}) { + t.Fatalf("rows: %+v", items) + } +} + +// TSV: blank header row with tab delimiter. +func TestParseCSVBlankHeaderTSV(t *testing.T) { + in := "\t\t\r\na\tb\r\n1\tx\r\n" + c := csv.NewReader(strings.NewReader(in)) + c.Comma = '\t' + c.FieldsPerRecord = -1 + items, err := ParseCSV[blankRec](c) + if err != nil { + t.Fatalf("parse: %v", err) + } + if len(items) != 1 || items[0] != (blankRec{A: "1", B: "x"}) { + t.Fatalf("rows: %+v", items) + } +} + +// All-blank input: no header, no rows. +func TestParseCSVAllBlank(t *testing.T) { + items, err := parseBlank[blankRec]("\r\n,\r\n") + if err != nil { + t.Fatalf("parse: %v", err) + } + if items != nil { + t.Fatalf("rows: %+v, want nil", items) + } +} diff --git a/database/csvx/encoding.go b/database/csvx/encoding.go index b7743c7..aa36de4 100644 --- a/database/csvx/encoding.go +++ b/database/csvx/encoding.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "strings" "github.com/jszwec/csvutil" ) @@ -34,7 +35,63 @@ func Parse[T any](r io.Reader, format string) ([]T, error) { // ParseCSV reads CSV/TSV from a csvutil Reader using csvutil streaming. // The reader must already have its delimiter configured. // Uses json struct tags and csvx marshalers for type handling. +// +// Blank records — rows where every cell is empty or whitespace-only — +// are skipped everywhere, including before the header. Human-maintained +// sources routinely carry stray blank lines (e.g. a Google Sheets export +// with an empty row above the header); without skipping, the first blank +// row would be taken as the header and every data row would decode to +// zero values. func ParseCSV[T any](r csvutil.Reader) ([]T, error) { + records := make([][]string, 0, 16) + for { + rec, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if isBlankRecord(rec) { + continue + } + records = append(records, rec) + } + if len(records) == 0 { + return nil, nil + } + return parseDecoded[T](&recordReader{records: records}) +} + +// isBlankRecord reports whether every cell is empty or whitespace-only. +func isBlankRecord(rec []string) bool { + for _, f := range rec { + if strings.TrimSpace(f) != "" { + return false + } + } + return true +} + +// recordReader replays in-memory records through the csvutil.Reader +// interface so the csvutil decoder sees a blank-free stream; the first +// record is the header. +type recordReader struct { + records [][]string + i int +} + +func (rr *recordReader) Read() ([]string, error) { + if rr.i >= len(rr.records) { + return nil, io.EOF + } + rec := rr.records[rr.i] + rr.i++ + return rec, nil +} + +// parseDecoded runs the csvutil decode loop over a csvutil.Reader. +func parseDecoded[T any](r csvutil.Reader) ([]T, error) { dec, err := csvutil.NewDecoder(r) if err != nil { return nil, err