diff --git a/io/transform/gsheet2csv/gsheet2csv.go b/io/transform/gsheet2csv/gsheet2csv.go index b48ab5c..a1b9a14 100644 --- a/io/transform/gsheet2csv/gsheet2csv.go +++ b/io/transform/gsheet2csv/gsheet2csv.go @@ -12,6 +12,7 @@ package gsheet2csv import ( + "context" "encoding/csv" "errors" "fmt" @@ -48,6 +49,14 @@ type Reader struct { err error } +// NewReaderFromContext is NewReaderFrom with request cancellation. +func NewReaderFromContext(ctx context.Context, urlOrPath string) *Reader { + if strings.HasPrefix(urlOrPath, "https://") || strings.HasPrefix(urlOrPath, "http://") { + return NewReaderFromURLContext(ctx, urlOrPath) + } + return NewReaderFrom(urlOrPath) +} + func NewReaderFrom(urlOrPath string) *Reader { if strings.HasPrefix(urlOrPath, "https://") || strings.HasPrefix(urlOrPath, "http://") { return NewReaderFromURL(urlOrPath) @@ -64,14 +73,29 @@ func NewReaderFrom(urlOrPath string) *Reader { return r } +func NewReaderFromURLContext(ctx context.Context, url string) *Reader { + docid, gid := ParseIDs(url) + return NewReaderFromIDsContext(ctx, httpClient, docid, gid) +} + func NewReaderFromURL(url string) *Reader { docid, gid := ParseIDs(url) - return NewReaderFromIDs(docid, gid) } +// NewReaderFromIDsContext uses client and ctx for the sheet request. +// It leaves NewReaderFromIDs unchanged for compatibility. +func NewReaderFromIDsContext(ctx context.Context, client *http.Client, docid, gid string) *Reader { + resp, err := GetSheetContext(ctx, client, docid, gid) + return newReaderFromResponse(resp, err, docid, gid) +} + func NewReaderFromIDs(docid, gid string) *Reader { resp, err := GetSheet(docid, gid) + return newReaderFromResponse(resp, err, docid, gid) +} + +func newReaderFromResponse(resp *http.Response, err error, docid, gid string) *Reader { if err != nil { r := NewReader(nil) r.err = err @@ -91,10 +115,16 @@ func ToCSVURL(docid, gid string) string { return fmt.Sprintf("https://docs.google.com/spreadsheets/d/%s/export?format=csv&usp=sharing&gid=%s", docid, gid) } -func GetSheet(docid, gid string) (*http.Response, error) { +func GetSheetContext(ctx context.Context, client *http.Client, docid, gid string) (*http.Response, error) { downloadURL := ToCSVURL(docid, gid) - - resp, err := httpGet(downloadURL) + request, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) + if err != nil { + return nil, err + } + if client == nil { + client = httpClient + } + resp, err := client.Do(request) if err != nil { return nil, err } @@ -107,6 +137,22 @@ func GetSheet(docid, gid string) (*http.Response, error) { return resp, nil } +func GetSheet(docid, gid string) (*http.Response, error) { + return getSheetWithGet(ToCSVURL(docid, gid), httpGet) +} + +func getSheetWithGet(downloadURL string, get func(string) (*http.Response, error)) (*http.Response, error) { + resp, err := get(downloadURL) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + _ = resp.Body.Close() + return nil, ErrHTTPGet + } + return resp, nil +} + func NewReader(r io.Reader) *Reader { csvr := csv.NewReader(r) csvr.Comma = ',' diff --git a/net/iplist/go.mod b/net/iplist/go.mod index d93ff05..02a3996 100644 --- a/net/iplist/go.mod +++ b/net/iplist/go.mod @@ -4,6 +4,7 @@ go 1.26.5 require ( github.com/therootcompany/golib/https v0.9.0 + github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0 github.com/therootcompany/golib/net/httpcache v0.5.1 ) diff --git a/net/iplist/go.sum b/net/iplist/go.sum index 9f9d3ba..3304d22 100644 --- a/net/iplist/go.sum +++ b/net/iplist/go.sum @@ -1,5 +1,7 @@ github.com/therootcompany/golib/https v0.9.0 h1:D9gh0K6ljkvF805JUbylITn8SHPMo08q5PglQGXsK8c= github.com/therootcompany/golib/https v0.9.0/go.mod h1:ilExqR/Uh7oB36oy9mwuNhANcFh+zdCfD3uJiZ11YAY= +github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0 h1:R5zAPun5p3JPlNfcMD9es4LD9sqh6nWDNTflZBeeUJo= +github.com/therootcompany/golib/io/transform/gsheet2csv v1.1.0/go.mod h1:CsntOAhhIMdzSAYuwuRZtfkdoiER9+TfSgYOIDWN71o= github.com/therootcompany/golib/net/httpcache v0.5.1 h1:vQM758tBwPEqCo7ojpSeL3fHqgq29LYmvD2hvDTBfKY= github.com/therootcompany/golib/net/httpcache v0.5.1/go.mod h1:mKSQJWpGSCDBzWkTCwT7dO1SCVCsFmdkRgKr39fNS1U= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= diff --git a/net/iplist/iplist.go b/net/iplist/iplist.go index 07b2371..b106a9a 100644 --- a/net/iplist/iplist.go +++ b/net/iplist/iplist.go @@ -17,6 +17,7 @@ import ( "time" "github.com/therootcompany/golib/https" + "github.com/therootcompany/golib/io/transform/gsheet2csv" "github.com/therootcompany/golib/net/httpcache" ) @@ -64,6 +65,12 @@ func load(ctx context.Context, source, cacheDir string, seen map[string]struct{} func readSource(ctx context.Context, source, cacheDir string, seen map[string]struct{}) ([]string, error) { parsed, err := url.Parse(source) + if err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") { + if docid, gid := gsheet2csv.ParseIDs(source); docid != "" { + source = gsheet2csv.ToCSVURL(docid, gid) + parsed, err = url.Parse(source) + } + } if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") { body, err := os.ReadFile(source) if err != nil {