root/golib
1
0
Fork 0
mirror of https://github.com/therootcompany/golib.git synced 2025-10-12 20:18:16 +00:00
Code Issues Projects Releases Wiki Activity

feat(transform): maintenance fork of github.com/tidwall/transform

Browse Source
This commit is contained in:
AJ ONeal 2025-10-10 16:38:38 -06:00
parent f3828d2037
commit aaa7942bef
No known key found for this signature in database
4 changed files with 545 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
3p/transform/LICENSE Normal file
View File

@ -0,0 +1,15 @@
ISC License
Copyright (c) 2017, Joshua J Baker
Permission to use, copy, modify, and/or distribute this software for any purpose
with or without fee is hereby granted, provided that the above copyright notice
and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
THIS SOFTWARE.

226
3p/transform/README.md Normal file
View File

@ -0,0 +1,226 @@
# Transform
[![GoDoc](https://img.shields.io/badge/api-reference-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/therootcompany/golib/3p/transform)
Transform is a Go package that provides a simple pattern for performing [chainable](#chaining) data transformations on streams of bytes. It conforms to the [io.Reader](https://golang.org/pkg/io/#Reader) interface and is useful for operations such as converting data formats, audio/video resampling, image transforms, log filters, regex line matching, etc.
The [transutil package](#transutil-package) provides few examples that work with JSON such as `JSONToMsgPack`, `MsgPackToJSON`, `JSONToPrettyJSON`, `JSONToUglyJSON`, `JSONToProtoBuf`, and `ProtoBufToJSON`. It also includes a handy `Gzipper` and `Gunzipper`.
# Getting Started
## Installing
To start using Transform, install Go and run `go get`:
```sh
go get -u github.com/therootcompany/golib/3p/transform
```
## Using
Below are a few very simple examples of custom transformers.
### ToUpper
Convert a string to uppper case. Unicode aware. In this example
we only process one rune at a time.
```go
func ToUpper(r io.Reader) io.Reader {
br := bufio.NewReader(r)
return transform.NewTransformer(func() ([]byte, error) {
c, _, err := br.ReadRune()
if err != nil {
return nil, err
}
return []byte(strings.ToUpper(string([]rune{c}))), nil
})
}
```
```go
msg := "Hello World"
data, err := ioutil.ReadAll(ToUpper(bytes.NewBufferString(msg)))
if err != nil {
log.Fatal(err)
}
fmt.Println(string(data))
```
Output:
```text
HELLO WORLD
```
### Rot13
The [Rot13](https://en.wikipedia.org/wiki/ROT13) cipher.
```go
func Rot13(r io.Reader) io.Reader {
buf := make([]byte, 256)
return transform.NewTransformer(func() ([]byte, error) {
n, err := r.Read(buf)
if err != nil {
return nil, err
}
for i := 0; i < n; i++ {
if buf[i] >= 'a' && buf[i] <= 'z' {
buf[i] = ((buf[i] - 'a' + 13) % 26) + 'a'
} else if buf[i] >= 'A' && buf[i] <= 'Z' {
buf[i] = ((buf[i] - 'A' + 13) % 26) + 'A'
}
}
return buf[:n], nil
})
}
```
```go
msg := "Hello World"
data, err := ioutil.ReadAll(Rot13(bytes.NewBufferString(msg)))
if err != nil {
log.Fatal(err)
}
fmt.Println(string(data))
```
Output:
```text
Uryyb Jbeyq
```
### RegExp Line Matcher
A line reader that filters lines that match on a RegExp pattern.
```go
func LineMatch(r io.Reader, pattern string) io.Reader {
br := bufio.NewReader(r)
return NewTransformer(func() ([]byte, error) {
for {
line, err := br.ReadBytes('\n')
matched, _ := regexp.Match(pattern, line)
if matched {
return line, err
}
if err != nil {
return nil, err
}
}
})
}
```
```go
logs := `
23 Apr 17:32:23.604 [INFO] DB loaded in 0.551 seconds
23 Apr 17:32:23.605 [WARN] Disk space is low
23 Apr 17:32:23.054 [INFO] Server started on port 7812
23 Apr 17:32:23.141 [INFO] Ready for connections
`
data, err := ioutil.ReadAll(LineMatch(bytes.NewBufferString(logs), "WARN"))
if err != nil {
log.Fatal(err)
}
fmt.Println(string(data))
```
Output:
```text
23 Apr 17:32:23.605 [WARN] Disk space is low
```
### LineTrimSpace
A line reader that trims the spaces from all lines.
```go
func LineTrimSpace(r io.Reader, pattern string) io.Reader {
br := bufio.NewReader(r)
return transform.NewTransformer(func() ([]byte, error) {
for {
line, err := br.ReadBytes('\n')
if len(line) > 0 {
line = append(bytes.TrimSpace(line), '\n')
}
return line, err
}
})
}
```
```go
phrases := " lacy timber \n"
phrases += "\t\thybrid gossiping\t\n"
phrases += " coy radioactivity\n"
phrases += "rocky arrow \n"
out, err := ioutil.ReadAll(LineTrimSpace(bytes.NewBufferString(phrases)))
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s\n", out)
```
Output:
```text
lacy timber
hybrid gossiping
coy radioactivity
rocky arrow
```
### Chaining
A reader that matches lines on the letter 'o', trims the
space from the lines, and transforms everything to upper case.
```go
phrases := " lacy timber \n"
phrases += "\t\thybrid gossiping\t\n"
phrases += " coy radioactivity\n"
phrases += "rocky arrow \n"
r := ToUpper(LineTrimSpace(LineMatch(bytes.NewBufferString(phrases), "o")))
// Pass the string though the transformer.
out, err := ioutil.ReadAll(r)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s\n", out)
```
Output:
```text
HYBRID GOSSIPING
COY RADIOACTIVITY
ROCKY ARROW
```
## Transutil package
[![GoDoc](https://img.shields.io/badge/api-reference-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/tidwall/transform/transutil)
The `github.com/tidwall/transform/transutil` package includes additional examples.
```go
func Gunzipper(r io.Reader) io.Reader
func Gzipper(r io.Reader) io.Reader
func JSONToMsgPack(r io.Reader) io.Reader
func JSONToPrettyJSON(r io.Reader) io.Reader
func JSONToProtoBuf(r io.Reader, pb proto.Message, multimessage bool) io.Reader
func JSONToUglyJSON(r io.Reader) io.Reader
func MsgPackToJSON(r io.Reader) io.Reader
func ProtoBufToJSON(r io.Reader, pb proto.Message, multimessage bool) io.Reader
```
## License
This is a maintenance fork of <https://github.com/tidwall/transform>.

54
3p/transform/transform.go Normal file
View File

@ -0,0 +1,54 @@
// Package transform provides a convenient utility for transforming one data
// format to another.
package transform
// Transformer represents a transform reader.
type Transformer struct {
tfn func() ([]byte, error) // user-defined transform function
buf []byte // read buffer
idx int // read buffer index
err error // last error
}
// NewTransformer returns an object that can be used for transforming one
// data formant to another. The param is a function that performs the
// conversion and returns the transformed data in chunks/messages.
func NewTransformer(fn func() ([]byte, error)) *Transformer {
return &Transformer{tfn: fn}
}
// ReadMessage allows for reading a one transformed message at a time.
func (r *Transformer) ReadMessage() ([]byte, error) {
return r.tfn()
}
// Read conforms to io.Reader
func (r *Transformer) Read(p []byte) (n int, err error) {
if len(r.buf)-r.idx > 0 {
// There's data in the read buffer, return it prior to returning errors
// or reading more messages.
if len(r.buf)-r.idx > len(p) {
// The input slice is smaller than the read buffer, copy a subslice
// of the read buffer and increase the read index.
copy(p, r.buf[r.idx:r.idx+len(p)])
r.idx += len(p)
return len(p), nil
}
// Copy the entire read buffer to the input slice.
n = len(r.buf) - r.idx
copy(p[:n], r.buf[r.idx:])
r.buf = r.buf[:0] // reset the read buffer, keeping it's capacity
r.idx = 0 // rewind the read buffer index
return n, nil
}
if r.err != nil {
return 0, r.err
}
var msg []byte
msg, r.err = r.ReadMessage()
// We should immediately append the incoming message to the read
// buffer to allow for the implemented transformer to repurpose
// it's own message space if needed.
r.buf = append(r.buf, msg...)
return r.Read(p)
}

250
3p/transform/transform_test.go Normal file
View File

@ -0,0 +1,250 @@
package transform
import (
"bufio"
"bytes"
"fmt"
"io"
"io/ioutil"
"log"
"math/rand"
"regexp"
"strings"
"testing"
"time"
)
func Rot13(r io.Reader) *Transformer {
buf := make([]byte, rand.Int()%256+1) // used to test varying slice sizes
//buf := make([]byte, 256)
return NewTransformer(func() ([]byte, error) {
n, err := r.Read(buf)
if err != nil {
return nil, err
}
for i := 0; i < n; i++ {
if buf[i] >= 'a' && buf[i] <= 'z' {
buf[i] = ((buf[i] - 'a' + 13) % 26) + 'a'
} else if buf[i] >= 'A' && buf[i] <= 'Z' {
buf[i] = ((buf[i] - 'A' + 13) % 26) + 'A'
}
}
return buf[:n], nil
})
}
func TestTransformer(t *testing.T) {
// simple
msg := "Hello\n13th Floor"
data, err := ioutil.ReadAll(Rot13(Rot13(bytes.NewBufferString(msg))))
if err != nil {
t.Fatal(err)
}
if msg != string(data) {
t.Fatalf("expected '%v', got '%v'\n", msg, string(data))
}
// random
rand.Seed(time.Now().UnixNano())
buf := make([]byte, 10000)
for i := 0; i < 1000; i++ {
_, err := rand.Read(buf)
if err != nil {
t.Fatal(err)
}
data, err := ioutil.ReadAll(Rot13(Rot13(bytes.NewBuffer(buf))))
if err != nil {
t.Fatal(err)
}
if string(buf) != string(data) {
t.Fatalf("expected '%v', got '%v'\n", string(buf), string(data))
}
}
}
func ExampleTransformer_rot13() {
// Rot13 transformation
rot13 := func(r io.Reader) *Transformer {
buf := make([]byte, 256)
return NewTransformer(func() ([]byte, error) {
n, err := r.Read(buf)
if err != nil {
return nil, err
}
for i := 0; i < n; i++ {
if buf[i] >= 'a' && buf[i] <= 'z' {
buf[i] = ((buf[i] - 'a' + 13) % 26) + 'a'
} else if buf[i] >= 'A' && buf[i] <= 'Z' {
buf[i] = ((buf[i] - 'A' + 13) % 26) + 'A'
}
}
return buf[:n], nil
})
}
// Pass the string though the transformer.
out, err := ioutil.ReadAll(rot13(bytes.NewBufferString("Hello World")))
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s\n", out)
// Output:
// Uryyb Jbeyq
}
func ExampleTransformer_toUpper() {
// Convert a string to uppper case. Unicode aware.
toUpper := func(r io.Reader) *Transformer {
br := bufio.NewReader(r)
return NewTransformer(func() ([]byte, error) {
c, _, err := br.ReadRune()
if err != nil {
return nil, err
}
return []byte(strings.ToUpper(string([]rune{c}))), nil
})
}
// Pass the string though the transformer.
out, err := ioutil.ReadAll(toUpper(bytes.NewBufferString("Hello World")))
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s\n", out)
// Output:
// HELLO WORLD
}
func ExampleTransformer_lineMatcherRegExp() {
// Filter lines matching a pattern
matcher := func(r io.Reader, pattern string) *Transformer {
br := bufio.NewReader(r)
return NewTransformer(func() ([]byte, error) {
for {
line, err := br.ReadBytes('\n')
matched, _ := regexp.Match(pattern, line)
if matched {
return line, err
}
if err != nil {
return nil, err
}
}
})
}
logs := `
23 Apr 17:32:23.604 [INFO] DB loaded in 0.551 seconds
23 Apr 17:32:23.605 [WARN] Disk space is low
23 Apr 17:32:23.054 [INFO] Server started on port 7812
23 Apr 17:32:23.141 [INFO] Ready for connections
`
// Pass the string though the transformer.
out, err := ioutil.ReadAll(matcher(bytes.NewBufferString(logs), "WARN"))
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s\n", out)
// Output:
// 23 Apr 17:32:23.605 [WARN] Disk space is low
}
func ExampleTransformer_trimmer() {
// Trim space from all lines
trimmer := func(r io.Reader) *Transformer {
br := bufio.NewReader(r)
return NewTransformer(func() ([]byte, error) {
for {
line, err := br.ReadBytes('\n')
if len(line) > 0 {
line = append(bytes.TrimSpace(line), '\n')
}
return line, err
}
})
}
phrases := " lacy timber \n"
phrases += "\t\thybrid gossiping\t\n"
phrases += " coy radioactivity\n"
phrases += "rocky arrow \n"
// Pass the string though the transformer.
out, err := ioutil.ReadAll(trimmer(bytes.NewBufferString(phrases)))
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s\n", out)
// Output:
// lacy timber
// hybrid gossiping
// coy radioactivity
// rocky arrow
}
func ExampleTransformer_pipeline() {
// Filter lines matching a pattern
matcher := func(r io.Reader, pattern string) *Transformer {
br := bufio.NewReader(r)
return NewTransformer(func() ([]byte, error) {
for {
line, err := br.ReadBytes('\n')
matched, _ := regexp.Match(pattern, line)
if matched {
return line, err
}
if err != nil {
return nil, err
}
}
})
}
// Trim space from all lines
trimmer := func(r io.Reader) *Transformer {
br := bufio.NewReader(r)
return NewTransformer(func() ([]byte, error) {
for {
line, err := br.ReadBytes('\n')
if len(line) > 0 {
line = append(bytes.TrimSpace(line), '\n')
}
return line, err
}
})
}
// Convert a string to uppper case. Unicode aware. In this example
// we only process one rune at a time. It works but it's not ideal
// for production.
toUpper := func(r io.Reader) *Transformer {
br := bufio.NewReader(r)
return NewTransformer(func() ([]byte, error) {
c, _, err := br.ReadRune()
if err != nil {
return nil, err
}
return []byte(strings.ToUpper(string([]rune{c}))), nil
})
}
phrases := " lacy timber \n"
phrases += "\t\thybrid gossiping\t\n"
phrases += " coy radioactivity\n"
phrases += "rocky arrow \n"
// create a transformer that matches lines on the letter 'o', trims the
// space from the lines, and transforms to upper case.
r := toUpper(trimmer(matcher(bytes.NewBufferString(phrases), "o")))
// Pass the string though the transformer.
out, err := ioutil.ReadAll(r)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s\n", out)
// Output:
// HYBRID GOSSIPING
// COY RADIOACTIVITY
// ROCKY ARROW
}