From aaa7942bef35e5e4779ee8829365379d07de39ae Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Fri, 10 Oct 2025 16:38:38 -0600 Subject: [PATCH] feat(transform): maintenance fork of github.com/tidwall/transform --- 3p/transform/LICENSE | 15 ++ 3p/transform/README.md | 226 +++++++++++++++++++++++++++++ 3p/transform/transform.go | 54 +++++++ 3p/transform/transform_test.go | 250 +++++++++++++++++++++++++++++++++ 4 files changed, 545 insertions(+) create mode 100644 3p/transform/LICENSE create mode 100644 3p/transform/README.md create mode 100644 3p/transform/transform.go create mode 100644 3p/transform/transform_test.go diff --git a/3p/transform/LICENSE b/3p/transform/LICENSE new file mode 100644 index 0000000..3df8052 --- /dev/null +++ b/3p/transform/LICENSE @@ -0,0 +1,15 @@ +ISC License + +Copyright (c) 2017, Joshua J Baker + +Permission to use, copy, modify, and/or distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright notice +and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. diff --git a/3p/transform/README.md b/3p/transform/README.md new file mode 100644 index 0000000..20619bd --- /dev/null +++ b/3p/transform/README.md @@ -0,0 +1,226 @@ +# Transform + +[![GoDoc](https://img.shields.io/badge/api-reference-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/therootcompany/golib/3p/transform) + +Transform is a Go package that provides a simple pattern for performing [chainable](#chaining) data transformations on streams of bytes. It conforms to the [io.Reader](https://golang.org/pkg/io/#Reader) interface and is useful for operations such as converting data formats, audio/video resampling, image transforms, log filters, regex line matching, etc. + +The [transutil package](#transutil-package) provides few examples that work with JSON such as `JSONToMsgPack`, `MsgPackToJSON`, `JSONToPrettyJSON`, `JSONToUglyJSON`, `JSONToProtoBuf`, and `ProtoBufToJSON`. It also includes a handy `Gzipper` and `Gunzipper`. + +# Getting Started + +## Installing + +To start using Transform, install Go and run `go get`: + +```sh +go get -u github.com/therootcompany/golib/3p/transform +``` + +## Using + +Below are a few very simple examples of custom transformers. + +### ToUpper + +Convert a string to uppper case. Unicode aware. In this example +we only process one rune at a time. + +```go +func ToUpper(r io.Reader) io.Reader { + br := bufio.NewReader(r) + return transform.NewTransformer(func() ([]byte, error) { + c, _, err := br.ReadRune() + if err != nil { + return nil, err + } + return []byte(strings.ToUpper(string([]rune{c}))), nil + }) +} +``` + +```go +msg := "Hello World" +data, err := ioutil.ReadAll(ToUpper(bytes.NewBufferString(msg))) +if err != nil { + log.Fatal(err) +} +fmt.Println(string(data)) +``` + +Output: + +```text +HELLO WORLD +``` + +### Rot13 + +The [Rot13](https://en.wikipedia.org/wiki/ROT13) cipher. + +```go +func Rot13(r io.Reader) io.Reader { + buf := make([]byte, 256) + return transform.NewTransformer(func() ([]byte, error) { + n, err := r.Read(buf) + if err != nil { + return nil, err + } + for i := 0; i < n; i++ { + if buf[i] >= 'a' && buf[i] <= 'z' { + buf[i] = ((buf[i] - 'a' + 13) % 26) + 'a' + } else if buf[i] >= 'A' && buf[i] <= 'Z' { + buf[i] = ((buf[i] - 'A' + 13) % 26) + 'A' + } + } + return buf[:n], nil + }) +} +``` + +```go +msg := "Hello World" +data, err := ioutil.ReadAll(Rot13(bytes.NewBufferString(msg))) +if err != nil { + log.Fatal(err) +} +fmt.Println(string(data)) +``` + +Output: + +```text +Uryyb Jbeyq +``` + +### RegExp Line Matcher + +A line reader that filters lines that match on a RegExp pattern. + +```go +func LineMatch(r io.Reader, pattern string) io.Reader { + br := bufio.NewReader(r) + return NewTransformer(func() ([]byte, error) { + for { + line, err := br.ReadBytes('\n') + matched, _ := regexp.Match(pattern, line) + if matched { + return line, err + } + if err != nil { + return nil, err + } + } + }) +} +``` + +```go +logs := ` +23 Apr 17:32:23.604 [INFO] DB loaded in 0.551 seconds +23 Apr 17:32:23.605 [WARN] Disk space is low +23 Apr 17:32:23.054 [INFO] Server started on port 7812 +23 Apr 17:32:23.141 [INFO] Ready for connections +` +data, err := ioutil.ReadAll(LineMatch(bytes.NewBufferString(logs), "WARN")) +if err != nil { + log.Fatal(err) +} +fmt.Println(string(data)) +``` + +Output: + +```text +23 Apr 17:32:23.605 [WARN] Disk space is low +``` + +### LineTrimSpace + +A line reader that trims the spaces from all lines. + +```go +func LineTrimSpace(r io.Reader, pattern string) io.Reader { + br := bufio.NewReader(r) + return transform.NewTransformer(func() ([]byte, error) { + for { + line, err := br.ReadBytes('\n') + if len(line) > 0 { + line = append(bytes.TrimSpace(line), '\n') + } + return line, err + } + }) +} +``` + +```go +phrases := " lacy timber \n" +phrases += "\t\thybrid gossiping\t\n" +phrases += " coy radioactivity\n" +phrases += "rocky arrow \n" +out, err := ioutil.ReadAll(LineTrimSpace(bytes.NewBufferString(phrases))) +if err != nil { + log.Fatal(err) +} +fmt.Printf("%s\n", out) +``` + +Output: + +```text +lacy timber +hybrid gossiping +coy radioactivity +rocky arrow +``` + +### Chaining + +A reader that matches lines on the letter 'o', trims the +space from the lines, and transforms everything to upper case. + +```go +phrases := " lacy timber \n" +phrases += "\t\thybrid gossiping\t\n" +phrases += " coy radioactivity\n" +phrases += "rocky arrow \n" + +r := ToUpper(LineTrimSpace(LineMatch(bytes.NewBufferString(phrases), "o"))) + +// Pass the string though the transformer. +out, err := ioutil.ReadAll(r) +if err != nil { + log.Fatal(err) +} + +fmt.Printf("%s\n", out) +``` + +Output: + +```text +HYBRID GOSSIPING +COY RADIOACTIVITY +ROCKY ARROW +``` + +## Transutil package + +[![GoDoc](https://img.shields.io/badge/api-reference-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/tidwall/transform/transutil) + +The `github.com/tidwall/transform/transutil` package includes additional examples. + +```go +func Gunzipper(r io.Reader) io.Reader +func Gzipper(r io.Reader) io.Reader +func JSONToMsgPack(r io.Reader) io.Reader +func JSONToPrettyJSON(r io.Reader) io.Reader +func JSONToProtoBuf(r io.Reader, pb proto.Message, multimessage bool) io.Reader +func JSONToUglyJSON(r io.Reader) io.Reader +func MsgPackToJSON(r io.Reader) io.Reader +func ProtoBufToJSON(r io.Reader, pb proto.Message, multimessage bool) io.Reader +``` + +## License + +This is a maintenance fork of . diff --git a/3p/transform/transform.go b/3p/transform/transform.go new file mode 100644 index 0000000..42ed298 --- /dev/null +++ b/3p/transform/transform.go @@ -0,0 +1,54 @@ +// Package transform provides a convenient utility for transforming one data +// format to another. +package transform + +// Transformer represents a transform reader. +type Transformer struct { + tfn func() ([]byte, error) // user-defined transform function + buf []byte // read buffer + idx int // read buffer index + err error // last error +} + +// NewTransformer returns an object that can be used for transforming one +// data formant to another. The param is a function that performs the +// conversion and returns the transformed data in chunks/messages. +func NewTransformer(fn func() ([]byte, error)) *Transformer { + return &Transformer{tfn: fn} +} + +// ReadMessage allows for reading a one transformed message at a time. +func (r *Transformer) ReadMessage() ([]byte, error) { + return r.tfn() +} + +// Read conforms to io.Reader +func (r *Transformer) Read(p []byte) (n int, err error) { + if len(r.buf)-r.idx > 0 { + // There's data in the read buffer, return it prior to returning errors + // or reading more messages. + if len(r.buf)-r.idx > len(p) { + // The input slice is smaller than the read buffer, copy a subslice + // of the read buffer and increase the read index. + copy(p, r.buf[r.idx:r.idx+len(p)]) + r.idx += len(p) + return len(p), nil + } + // Copy the entire read buffer to the input slice. + n = len(r.buf) - r.idx + copy(p[:n], r.buf[r.idx:]) + r.buf = r.buf[:0] // reset the read buffer, keeping it's capacity + r.idx = 0 // rewind the read buffer index + return n, nil + } + if r.err != nil { + return 0, r.err + } + var msg []byte + msg, r.err = r.ReadMessage() + // We should immediately append the incoming message to the read + // buffer to allow for the implemented transformer to repurpose + // it's own message space if needed. + r.buf = append(r.buf, msg...) + return r.Read(p) +} diff --git a/3p/transform/transform_test.go b/3p/transform/transform_test.go new file mode 100644 index 0000000..3e537fc --- /dev/null +++ b/3p/transform/transform_test.go @@ -0,0 +1,250 @@ +package transform + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "math/rand" + "regexp" + "strings" + "testing" + "time" +) + +func Rot13(r io.Reader) *Transformer { + buf := make([]byte, rand.Int()%256+1) // used to test varying slice sizes + //buf := make([]byte, 256) + return NewTransformer(func() ([]byte, error) { + n, err := r.Read(buf) + if err != nil { + return nil, err + } + for i := 0; i < n; i++ { + if buf[i] >= 'a' && buf[i] <= 'z' { + buf[i] = ((buf[i] - 'a' + 13) % 26) + 'a' + } else if buf[i] >= 'A' && buf[i] <= 'Z' { + buf[i] = ((buf[i] - 'A' + 13) % 26) + 'A' + } + } + return buf[:n], nil + }) +} + +func TestTransformer(t *testing.T) { + // simple + msg := "Hello\n13th Floor" + data, err := ioutil.ReadAll(Rot13(Rot13(bytes.NewBufferString(msg)))) + if err != nil { + t.Fatal(err) + } + if msg != string(data) { + t.Fatalf("expected '%v', got '%v'\n", msg, string(data)) + } + // random + rand.Seed(time.Now().UnixNano()) + buf := make([]byte, 10000) + for i := 0; i < 1000; i++ { + _, err := rand.Read(buf) + if err != nil { + t.Fatal(err) + } + data, err := ioutil.ReadAll(Rot13(Rot13(bytes.NewBuffer(buf)))) + if err != nil { + t.Fatal(err) + } + if string(buf) != string(data) { + t.Fatalf("expected '%v', got '%v'\n", string(buf), string(data)) + } + } +} + +func ExampleTransformer_rot13() { + // Rot13 transformation + rot13 := func(r io.Reader) *Transformer { + buf := make([]byte, 256) + return NewTransformer(func() ([]byte, error) { + n, err := r.Read(buf) + if err != nil { + return nil, err + } + for i := 0; i < n; i++ { + if buf[i] >= 'a' && buf[i] <= 'z' { + buf[i] = ((buf[i] - 'a' + 13) % 26) + 'a' + } else if buf[i] >= 'A' && buf[i] <= 'Z' { + buf[i] = ((buf[i] - 'A' + 13) % 26) + 'A' + } + } + return buf[:n], nil + }) + } + // Pass the string though the transformer. + out, err := ioutil.ReadAll(rot13(bytes.NewBufferString("Hello World"))) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s\n", out) + // Output: + // Uryyb Jbeyq +} + +func ExampleTransformer_toUpper() { + // Convert a string to uppper case. Unicode aware. + toUpper := func(r io.Reader) *Transformer { + br := bufio.NewReader(r) + return NewTransformer(func() ([]byte, error) { + c, _, err := br.ReadRune() + if err != nil { + return nil, err + } + return []byte(strings.ToUpper(string([]rune{c}))), nil + }) + } + // Pass the string though the transformer. + out, err := ioutil.ReadAll(toUpper(bytes.NewBufferString("Hello World"))) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s\n", out) + // Output: + // HELLO WORLD +} + +func ExampleTransformer_lineMatcherRegExp() { + // Filter lines matching a pattern + matcher := func(r io.Reader, pattern string) *Transformer { + br := bufio.NewReader(r) + return NewTransformer(func() ([]byte, error) { + for { + line, err := br.ReadBytes('\n') + matched, _ := regexp.Match(pattern, line) + if matched { + return line, err + } + if err != nil { + return nil, err + } + } + }) + } + + logs := ` +23 Apr 17:32:23.604 [INFO] DB loaded in 0.551 seconds +23 Apr 17:32:23.605 [WARN] Disk space is low +23 Apr 17:32:23.054 [INFO] Server started on port 7812 +23 Apr 17:32:23.141 [INFO] Ready for connections + ` + // Pass the string though the transformer. + out, err := ioutil.ReadAll(matcher(bytes.NewBufferString(logs), "WARN")) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s\n", out) + // Output: + // 23 Apr 17:32:23.605 [WARN] Disk space is low +} + +func ExampleTransformer_trimmer() { + // Trim space from all lines + trimmer := func(r io.Reader) *Transformer { + br := bufio.NewReader(r) + return NewTransformer(func() ([]byte, error) { + for { + line, err := br.ReadBytes('\n') + if len(line) > 0 { + line = append(bytes.TrimSpace(line), '\n') + } + return line, err + } + }) + } + + phrases := " lacy timber \n" + phrases += "\t\thybrid gossiping\t\n" + phrases += " coy radioactivity\n" + phrases += "rocky arrow \n" + // Pass the string though the transformer. + out, err := ioutil.ReadAll(trimmer(bytes.NewBufferString(phrases))) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s\n", out) + // Output: + // lacy timber + // hybrid gossiping + // coy radioactivity + // rocky arrow +} + +func ExampleTransformer_pipeline() { + // Filter lines matching a pattern + matcher := func(r io.Reader, pattern string) *Transformer { + br := bufio.NewReader(r) + return NewTransformer(func() ([]byte, error) { + for { + line, err := br.ReadBytes('\n') + matched, _ := regexp.Match(pattern, line) + if matched { + return line, err + } + if err != nil { + return nil, err + } + } + }) + } + + // Trim space from all lines + trimmer := func(r io.Reader) *Transformer { + br := bufio.NewReader(r) + return NewTransformer(func() ([]byte, error) { + for { + line, err := br.ReadBytes('\n') + if len(line) > 0 { + line = append(bytes.TrimSpace(line), '\n') + } + return line, err + } + }) + } + + // Convert a string to uppper case. Unicode aware. In this example + // we only process one rune at a time. It works but it's not ideal + // for production. + toUpper := func(r io.Reader) *Transformer { + br := bufio.NewReader(r) + return NewTransformer(func() ([]byte, error) { + c, _, err := br.ReadRune() + if err != nil { + return nil, err + } + return []byte(strings.ToUpper(string([]rune{c}))), nil + }) + } + phrases := " lacy timber \n" + phrases += "\t\thybrid gossiping\t\n" + phrases += " coy radioactivity\n" + phrases += "rocky arrow \n" + + // create a transformer that matches lines on the letter 'o', trims the + // space from the lines, and transforms to upper case. + r := toUpper(trimmer(matcher(bytes.NewBufferString(phrases), "o"))) + + // Pass the string though the transformer. + out, err := ioutil.ReadAll(r) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s\n", out) + // Output: + // HYBRID GOSSIPING + // COY RADIOACTIVITY + // ROCKY ARROW +}