diff --git a/tools/jsontypes/cmd/jsonsample/main.go b/tools/jsontypes/cmd/jsonsample/main.go new file mode 100644 index 0000000..0e0b100 --- /dev/null +++ b/tools/jsontypes/cmd/jsonsample/main.go @@ -0,0 +1,41 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + + "github.com/therootcompany/golib/tools/jsontypes" +) + +func main() { + maxStr := flag.Int("maxstr", 40, "max string length before truncation (0 = no limit)") + flag.Parse() + + var input *os.File + if flag.NArg() > 0 && flag.Arg(0) != "-" { + f, err := os.Open(flag.Arg(0)) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + defer f.Close() + input = f + } else { + input = os.Stdin + } + + var data any + dec := json.NewDecoder(input) + dec.UseNumber() + if err := dec.Decode(&data); err != nil { + fmt.Fprintf(os.Stderr, "error parsing JSON: %v\n", err) + os.Exit(1) + } + + cfg := jsontypes.SampleConfig{MaxStringLen: *maxStr} + for _, line := range jsontypes.SampleWithConfig(data, cfg) { + fmt.Println(line) + } +} diff --git a/tools/jsontypes/coalesce.go b/tools/jsontypes/coalesce.go index 22d33da..7b71d31 100644 --- a/tools/jsontypes/coalesce.go +++ b/tools/jsontypes/coalesce.go @@ -40,7 +40,10 @@ func parseRawLines(lines []string) []rawLine { } func parseRawLine(line string) rawLine { - idx := strings.LastIndex(line, "{") + // Use the first { since paths never contain braces. + // This correctly handles sample format where the type value + // contains nested JSON braces: {Root0:{"name":"pikachu"}} + idx := strings.Index(line, "{") if idx < 0 || !strings.HasSuffix(line, "}") { return rawLine{path: line} } @@ -52,8 +55,22 @@ func parseRawLine(line string) rawLine { var namedTypePattern = regexp.MustCompile(`^[A-Z][a-zA-Z0-9-]*\d+$`) -func isNamedType(typeVal string) bool { - return namedTypePattern.MatchString(typeVal) +// extractTypeName returns the type name from a typeVal. +// Handles both plain format ("Root0") and sample format ("Root0:{...}"). +// Returns ("", false) if the typeVal is not a named type. +func extractTypeName(typeVal string) (name string, ok bool) { + // Check for sample format: "TypeName:..." + if idx := strings.Index(typeVal, ":"); idx > 0 { + candidate := typeVal[:idx] + if namedTypePattern.MatchString(candidate) { + return candidate, true + } + } + // Check plain format. + if namedTypePattern.MatchString(typeVal) { + return typeVal, true + } + return "", false } // coalesceType is a named type from the streamer output. @@ -68,14 +85,15 @@ func buildCoalesceTypes(lines []rawLine) []*coalesceType { var types []*coalesceType for i, line := range lines { - if !isNamedType(line.typeVal) { + typeName, ok := extractTypeName(line.typeVal) + if !ok { continue } // Find the extent: until next type intro at the same path. end := len(lines) for j := i + 1; j < len(lines); j++ { - if isNamedType(lines[j].typeVal) && lines[j].path == line.path { + if _, jOk := extractTypeName(lines[j].typeVal); jOk && lines[j].path == line.path { end = j break } @@ -114,7 +132,7 @@ func buildCoalesceTypes(lines []rawLine) []*coalesceType { sort.Strings(keys) types = append(types, &coalesceType{ - name: line.typeVal, + name: typeName, path: line.path, keys: keys, keysSig: strings.Join(keys, ","), @@ -294,8 +312,8 @@ func emitCoalesced(lines []rawLine, nameMap map[string]string, nullables map[str continue } - if isNamedType(line.typeVal) { - canonicalName := nameMap[line.typeVal] + if typeName, ok := extractTypeName(line.typeVal); ok { + canonicalName := nameMap[typeName] nullable := "" if nullables[line.path] { nullable = "?" @@ -312,9 +330,14 @@ func emitCoalesced(lines []rawLine, nameMap map[string]string, nullables map[str continue } - // First occurrence — emit type intro and let fields be emitted normally. + // First occurrence — emit type intro with sample data if present. emittedTypes[canonicalName] = true - result = append(result, line.path+"{"+canonicalName+nullable+"}") + sample := extractSampleData(line.typeVal) + if sample != "" { + result = append(result, line.path+"{"+canonicalName+":"+sample+nullable+"}") + } else { + result = append(result, line.path+"{"+canonicalName+nullable+"}") + } i++ continue } @@ -333,6 +356,18 @@ func emitCoalesced(lines []rawLine, nameMap map[string]string, nullables map[str return result } +// extractSampleData returns the sample JSON portion from a type value +// like "Root0:{...}". Returns "" for plain type values like "Root0". +func extractSampleData(typeVal string) string { + if idx := strings.Index(typeVal, ":"); idx > 0 { + candidate := typeVal[:idx] + if namedTypePattern.MatchString(candidate) { + return typeVal[idx+1:] + } + } + return "" +} + func coalesceFieldPrefix(path string) string { if path == "" { return "." diff --git a/tools/jsontypes/sample.go b/tools/jsontypes/sample.go new file mode 100644 index 0000000..87ed305 --- /dev/null +++ b/tools/jsontypes/sample.go @@ -0,0 +1,249 @@ +package jsontypes + +import ( + "encoding/json" + "fmt" + "strings" +) + +// Sample extracts one representative JSON sample for each distinct type +// (by key set) from a decoded JSON value. The output is a flat list of +// lines in the same path format as RawPaths, but with actual JSON values +// at leaf nodes and compact JSON samples at type intro lines. +// +// The value must have been decoded with json.Decoder.UseNumber(). +func Sample(v any) []string { + return SampleWithConfig(v, SampleConfig{}) +} + +// SampleConfig controls sample output. +type SampleConfig struct { + // MaxStringLen truncates string values. 0 = no truncation. + MaxStringLen int +} + +// SampleWithConfig is like Sample but accepts configuration. +func SampleWithConfig(v any, cfg SampleConfig) []string { + s := &sampler{maxStr: cfg.MaxStringLen} + s.walk("", v) + return Coalesce(s.paths) +} + +type sampler struct { + counter int + paths []string + maxStr int +} + +func (s *sampler) emit(path string) { + s.paths = append(s.paths, path) +} + +func (s *sampler) nextName(path string) string { + name := inferRawTypeName(path) + id := s.counter + s.counter++ + return fmt.Sprintf("%s%d", name, id) +} + +func (s *sampler) walk(prefix string, v any) { + switch val := v.(type) { + case nil: + s.emit(prefix + "{null}") + case bool: + s.emit(prefix + "{" + fmt.Sprintf("%v", val) + "}") + case json.Number: + s.emit(prefix + "{" + val.String() + "}") + case string: + s.emit(prefix + "{" + s.quotedString(val) + "}") + case map[string]any: + s.walkObject(prefix, val) + case []any: + s.walkArray(prefix, val) + } +} + +func (s *sampler) quotedString(val string) string { + if s.maxStr > 0 && len(val) > s.maxStr { + return fmt.Sprintf("%q", val[:s.maxStr]) + "..." + } + return fmt.Sprintf("%q", val) +} + +func (s *sampler) walkObject(prefix string, obj map[string]any) { + if len(obj) == 0 { + s.emit(prefix + "{empty}") + return + } + + isMap, _ := looksLikeMap(obj) + if isMap { + s.walkMap(prefix, obj) + return + } + + s.walkStruct(prefix, []map[string]any{obj}) +} + +func (s *sampler) walkMap(prefix string, obj map[string]any) { + keyName := inferKeyName(obj) + mapPrefix := prefix + "[" + keyName + "]" + values := make([]any, 0, len(obj)) + for _, v := range obj { + values = append(values, v) + } + s.walkCollection(mapPrefix, values) +} + +func (s *sampler) walkStruct(prefix string, instances []map[string]any) { + merged := mergeObjects(instances) + typeName := s.nextName(prefix) + + // Emit type intro with a compact JSON sample of the first instance. + sample := s.compactSample(instances[0]) + s.emit(prefix + "{" + typeName + ":" + sample + "}") + + keys := sortedKeys(merged) + for _, k := range keys { + var fieldVals []any + for _, inst := range instances { + if v, ok := inst[k]; ok { + fieldVals = append(fieldVals, v) + } else { + fieldVals = append(fieldVals, absentValue) + } + } + s.walkCollection(joinPath(prefix, k), fieldVals) + } +} + +// compactSample creates a one-line JSON of an object with truncated values. +func (s *sampler) compactSample(obj map[string]any) string { + keys := sortedKeys(obj) + var parts []string + for _, k := range keys { + parts = append(parts, fmt.Sprintf("%q:%s", k, s.compactValue(obj[k]))) + } + return "{" + strings.Join(parts, ",") + "}" +} + +func (s *sampler) compactValue(v any) string { + switch val := v.(type) { + case nil: + return "null" + case bool: + return fmt.Sprintf("%v", val) + case json.Number: + return val.String() + case string: + return s.quotedString(val) + case map[string]any: + return "{...}" + case []any: + if len(val) == 0 { + return "[]" + } + return "[...]" + default: + return "?" + } +} + +func (s *sampler) walkArray(prefix string, arr []any) { + if len(arr) == 0 { + s.emit(prefix + "[]{empty}") + return + } + + if isTupleCandidate(arr) { + for i, v := range arr { + s.walk(fmt.Sprintf("%s[%d]", prefix, i), v) + } + return + } + + s.walkCollection(prefix+"[]", arr) +} + +func (s *sampler) walkCollection(prefix string, values []any) { + type shapeInstances struct { + sig string + instances []map[string]any + } + var shapes []shapeInstances + shapeIndex := make(map[string]int) + hasNull := false + hasUndefined := false + var nonObjects []any + + for _, v := range values { + if v == absentValue { + hasUndefined = true + continue + } + switch val := v.(type) { + case nil: + hasNull = true + case map[string]any: + sig := shapeSignature(val) + if idx, ok := shapeIndex[sig]; ok { + shapes[idx].instances = append(shapes[idx].instances, val) + } else { + shapeIndex[sig] = len(shapes) + shapes = append(shapes, shapeInstances{sig, []map[string]any{val}}) + } + default: + nonObjects = append(nonObjects, val) + } + } + + if hasUndefined { + s.emit(prefix + "{undefined}") + } + if hasNull { + s.emit(prefix + "{null}") + } + + for _, shape := range shapes { + if len(shape.instances) == 1 { + obj := shape.instances[0] + isMap, _ := looksLikeMap(obj) + if isMap { + s.walkMap(prefix, obj) + continue + } + } + s.walkStruct(prefix, shape.instances) + } + + // Emit one sample per primitive type (not per value). + seen := make(map[string]bool) // keyed by kind: "number", "string", "bool" + for _, v := range nonObjects { + switch val := v.(type) { + case json.Number: + if !seen["number"] { + seen["number"] = true + s.emit(prefix + "{" + val.String() + "}") + } + case string: + if !seen["string"] { + seen["string"] = true + s.emit(prefix + "{" + s.quotedString(val) + "}") + } + case bool: + if !seen["bool"] { + seen["bool"] = true + s.emit(prefix + "{" + fmt.Sprintf("%v", val) + "}") + } + case []any: + if !seen["array"] { + seen["array"] = true + s.walkArray(prefix, val) + } + } + } + + if !hasNull && !hasUndefined && len(shapes) == 0 && len(nonObjects) == 0 { + s.emit(prefix + "{empty}") + } +} diff --git a/tools/jsontypes/sample_test.go b/tools/jsontypes/sample_test.go new file mode 100644 index 0000000..7cd25a8 --- /dev/null +++ b/tools/jsontypes/sample_test.go @@ -0,0 +1,125 @@ +package jsontypes + +import ( + "encoding/json" + "os" + "strings" + "testing" +) + +func TestSampleBasic(t *testing.T) { + input := `{"name":"Alice","age":30,"email":null}` + var data any + dec := json.NewDecoder(strings.NewReader(input)) + dec.UseNumber() + if err := dec.Decode(&data); err != nil { + t.Fatal(err) + } + + got := Sample(data) + + // Type intro should have sample JSON. + assertLineContains(t, got, "{Root:") + // Leaf nodes should have actual values. + assertLineContains(t, got, `.name{"Alice"}`) + assertLineContains(t, got, ".age{30}") + // Null should be preserved (email has both null and no other type → stays {null}). + assertLineContains(t, got, ".email{null}") + + t.Logf("output (%d lines):\n%s", len(got), strings.Join(got, "\n")) +} + +func TestSampleTruncation(t *testing.T) { + input := `{"bio":"This is a very long biography that should be truncated"}` + var data any + dec := json.NewDecoder(strings.NewReader(input)) + dec.UseNumber() + if err := dec.Decode(&data); err != nil { + t.Fatal(err) + } + + got := SampleWithConfig(data, SampleConfig{MaxStringLen: 10}) + + // Should be truncated with "..." + assertLineContains(t, got, `"This is a "...`) + + t.Logf("output (%d lines):\n%s", len(got), strings.Join(got, "\n")) +} + +func TestSampleDedup(t *testing.T) { + input := `{ + "author": {"name": "Alice", "url": "https://example.com/alice"}, + "reviewer": {"name": "Bob", "url": "https://example.com/bob"}, + "editor": {"name": "Carol", "url": "https://example.com/carol"} + }` + var data any + dec := json.NewDecoder(strings.NewReader(input)) + dec.UseNumber() + if err := dec.Decode(&data); err != nil { + t.Fatal(err) + } + + got := Sample(data) + + // Three types with same keys {name, url} should be deduplicated. + // Only one should have field expansion. + nameCount := 0 + for _, line := range got { + if strings.Contains(line, `.name{"`) { + nameCount++ + } + } + if nameCount != 1 { + t.Errorf("expected 1 name field (deduped), got %d", nameCount) + } + + t.Logf("output (%d lines):\n%s", len(got), strings.Join(got, "\n")) +} + +func TestSamplePokemon(t *testing.T) { + data := loadJSONFixture(t, "testdata/pokemon-pikachu.json") + got := SampleWithConfig(data, SampleConfig{MaxStringLen: 30}) + + // Should have compact JSON in type intros. + assertLineContains(t, got, "{Root:") + assertLineContains(t, got, "{AbilitiesItem:") + assertLineContains(t, got, "{NameUrl:") + + // Should have actual values at leaves. + assertLineContains(t, got, `.name{"pikachu"}`) + assertLineContains(t, got, ".id{25}") + + t.Logf("sample output: %d lines", len(got)) +} + +func TestSampleCoalesceRoundTrip(t *testing.T) { + // Sample output should be valid input for Coalesce. + data := loadJSONFixture(t, "testdata/swapi-planets.json") + sample := SampleWithConfig(data, SampleConfig{MaxStringLen: 20}) + + // Feed sample output back through Coalesce — should be idempotent. + coalesced := Coalesce(sample) + + if len(coalesced) != len(sample) { + t.Errorf("re-coalescing changed line count: %d → %d", len(sample), len(coalesced)) + } + + t.Logf("sample: %d lines, re-coalesced: %d lines", len(sample), len(coalesced)) +} + +func loadJSONFixture(t *testing.T, path string) any { + t.Helper() + f, err := os.Open(path) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + defer f.Close() + + var data any + dec := json.NewDecoder(f) + dec.UseNumber() + if err := dec.Decode(&data); err != nil { + t.Fatalf("decode %s: %v", path, err) + } + return data +}