mirror of
https://github.com/therootcompany/golib.git
synced 2026-11-04 06:55:00 +00:00
fix(csvx): skip blank records in ParseCSV, including before the header
Human-maintained sources (Google Sheets exports) often carry stray blank rows. A blank row above the header was taken as the header, so every data row decoded to zero values; blank data rows decoded as empty records. ParseCSV now consumes the stream, drops records whose cells are all empty or whitespace-only, and replays the remaining records (first = header) through the csvutil decoder via a small recordReader. No delimiter or field-count handling changes; ragged-row behavior of the underlying csv.Reader is untouched. Adds 5 tests: blank header row (csv + tsv), blank/whitespace data rows, normal-input regression, all-blank input.
This commit is contained in:
parent
466978c8ca
commit
622f9c2efd
84
database/csvx/blank_test.go
Normal file
84
database/csvx/blank_test.go
Normal file
@ -0,0 +1,84 @@
|
||||
package csvx
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type blankRec struct {
|
||||
A string `json:"a"`
|
||||
B string `json:"b"`
|
||||
}
|
||||
|
||||
func parseBlank[T any](in string) ([]T, error) {
|
||||
c := csv.NewReader(strings.NewReader(in))
|
||||
c.FieldsPerRecord = -1 // ragged-tolerant, as sheet readers configure
|
||||
return ParseCSV[T](c)
|
||||
}
|
||||
|
||||
// Leading blank rows must not be taken as the header (Google Sheets
|
||||
// exports with an empty row 1).
|
||||
func TestParseCSVBlankHeaderRow(t *testing.T) {
|
||||
items, err := parseBlank[blankRec](",,,,\r\n\r\na,b\r\n1,x\r\n2,y\r\n")
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
want := []blankRec{{A: "1", B: "x"}, {A: "2", B: "y"}}
|
||||
if len(items) != len(want) {
|
||||
t.Fatalf("rows: %d, want %d (%+v)", len(items), len(want), items)
|
||||
}
|
||||
for i := range want {
|
||||
if items[i] != want[i] {
|
||||
t.Errorf("row %d: %+v, want %+v", i, items[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Blank and whitespace-only data rows are skipped.
|
||||
func TestParseCSVBlankDataRows(t *testing.T) {
|
||||
items, err := parseBlank[blankRec]("a,b\r\n1,x\r\n\r\n ,\r\n2,y\r\n")
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if len(items) != 2 || items[0] != (blankRec{A: "1", B: "x"}) || items[1] != (blankRec{A: "2", B: "y"}) {
|
||||
t.Fatalf("rows: %+v", items)
|
||||
}
|
||||
}
|
||||
|
||||
// Normal input is unchanged.
|
||||
func TestParseCSVNoBlanks(t *testing.T) {
|
||||
items, err := parseBlank[blankRec]("a,b\r\n1,x\r\n2,y\r\n")
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if len(items) != 2 || items[1] != (blankRec{A: "2", B: "y"}) {
|
||||
t.Fatalf("rows: %+v", items)
|
||||
}
|
||||
}
|
||||
|
||||
// TSV: blank header row with tab delimiter.
|
||||
func TestParseCSVBlankHeaderTSV(t *testing.T) {
|
||||
in := "\t\t\r\na\tb\r\n1\tx\r\n"
|
||||
c := csv.NewReader(strings.NewReader(in))
|
||||
c.Comma = '\t'
|
||||
c.FieldsPerRecord = -1
|
||||
items, err := ParseCSV[blankRec](c)
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if len(items) != 1 || items[0] != (blankRec{A: "1", B: "x"}) {
|
||||
t.Fatalf("rows: %+v", items)
|
||||
}
|
||||
}
|
||||
|
||||
// All-blank input: no header, no rows.
|
||||
func TestParseCSVAllBlank(t *testing.T) {
|
||||
items, err := parseBlank[blankRec]("\r\n,\r\n")
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if items != nil {
|
||||
t.Fatalf("rows: %+v, want nil", items)
|
||||
}
|
||||
}
|
||||
@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/jszwec/csvutil"
|
||||
)
|
||||
@ -34,7 +35,63 @@ func Parse[T any](r io.Reader, format string) ([]T, error) {
|
||||
// ParseCSV reads CSV/TSV from a csvutil Reader using csvutil streaming.
|
||||
// The reader must already have its delimiter configured.
|
||||
// Uses json struct tags and csvx marshalers for type handling.
|
||||
//
|
||||
// Blank records — rows where every cell is empty or whitespace-only —
|
||||
// are skipped everywhere, including before the header. Human-maintained
|
||||
// sources routinely carry stray blank lines (e.g. a Google Sheets export
|
||||
// with an empty row above the header); without skipping, the first blank
|
||||
// row would be taken as the header and every data row would decode to
|
||||
// zero values.
|
||||
func ParseCSV[T any](r csvutil.Reader) ([]T, error) {
|
||||
records := make([][]string, 0, 16)
|
||||
for {
|
||||
rec, err := r.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if isBlankRecord(rec) {
|
||||
continue
|
||||
}
|
||||
records = append(records, rec)
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return parseDecoded[T](&recordReader{records: records})
|
||||
}
|
||||
|
||||
// isBlankRecord reports whether every cell is empty or whitespace-only.
|
||||
func isBlankRecord(rec []string) bool {
|
||||
for _, f := range rec {
|
||||
if strings.TrimSpace(f) != "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// recordReader replays in-memory records through the csvutil.Reader
|
||||
// interface so the csvutil decoder sees a blank-free stream; the first
|
||||
// record is the header.
|
||||
type recordReader struct {
|
||||
records [][]string
|
||||
i int
|
||||
}
|
||||
|
||||
func (rr *recordReader) Read() ([]string, error) {
|
||||
if rr.i >= len(rr.records) {
|
||||
return nil, io.EOF
|
||||
}
|
||||
rec := rr.records[rr.i]
|
||||
rr.i++
|
||||
return rec, nil
|
||||
}
|
||||
|
||||
// parseDecoded runs the csvutil decode loop over a csvutil.Reader.
|
||||
func parseDecoded[T any](r csvutil.Reader) ([]T, error) {
|
||||
dec, err := csvutil.NewDecoder(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user