diff --git a/tools/jsontypes/cmd/jsoncoalesce/main.go b/tools/jsontypes/cmd/jsoncoalesce/main.go new file mode 100644 index 0000000..684abc3 --- /dev/null +++ b/tools/jsontypes/cmd/jsoncoalesce/main.go @@ -0,0 +1,38 @@ +package main + +import ( + "bufio" + "fmt" + "os" + + "github.com/therootcompany/golib/tools/jsontypes" +) + +func main() { + var input *os.File + if len(os.Args) > 1 && os.Args[1] != "-" { + f, err := os.Open(os.Args[1]) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + defer f.Close() + input = f + } else { + input = os.Stdin + } + + var lines []string + scanner := bufio.NewScanner(input) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + if err := scanner.Err(); err != nil { + fmt.Fprintf(os.Stderr, "error reading input: %v\n", err) + os.Exit(1) + } + + for _, line := range jsontypes.Coalesce(lines) { + fmt.Println(line) + } +} diff --git a/tools/jsontypes/coalesce.go b/tools/jsontypes/coalesce.go new file mode 100644 index 0000000..22d33da --- /dev/null +++ b/tools/jsontypes/coalesce.go @@ -0,0 +1,341 @@ +package jsontypes + +import ( + "fmt" + "regexp" + "sort" + "strings" +) + +// Coalesce takes streamer output lines and returns coalesced lines with: +// - Types deduplicated by key set (same keys = same type) +// - Nullable annotation ({string?} when {null} + concrete type at same path) +// - Cleaner type names (no number when unique, per-base-name numbering for conflicts) +// - Duplicate type definitions elided (referenced by name only) +func Coalesce(lines []string) []string { + parsed := parseRawLines(lines) + types := buildCoalesceTypes(parsed) + groups := groupTypesByKeySet(types) + nameMap := assignCanonicalNames(groups) + nullables := detectNullablePaths(parsed) + return emitCoalesced(parsed, nameMap, nullables) +} + +// rawLine is a parsed streamer output line. +type rawLine struct { + path string // everything before the last { + typeVal string // everything inside the last {} +} + +func parseRawLines(lines []string) []rawLine { + result := make([]rawLine, 0, len(lines)) + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + result = append(result, parseRawLine(line)) + } + return result +} + +func parseRawLine(line string) rawLine { + idx := strings.LastIndex(line, "{") + if idx < 0 || !strings.HasSuffix(line, "}") { + return rawLine{path: line} + } + return rawLine{ + path: line[:idx], + typeVal: line[idx+1 : len(line)-1], + } +} + +var namedTypePattern = regexp.MustCompile(`^[A-Z][a-zA-Z0-9-]*\d+$`) + +func isNamedType(typeVal string) bool { + return namedTypePattern.MatchString(typeVal) +} + +// coalesceType is a named type from the streamer output. +type coalesceType struct { + name string // original name, e.g., "Ability2" + path string // path where introduced + keys []string // sorted direct field names + keysSig string // comma-joined keys for comparison +} + +func buildCoalesceTypes(lines []rawLine) []*coalesceType { + var types []*coalesceType + + for i, line := range lines { + if !isNamedType(line.typeVal) { + continue + } + + // Find the extent: until next type intro at the same path. + end := len(lines) + for j := i + 1; j < len(lines); j++ { + if isNamedType(lines[j].typeVal) && lines[j].path == line.path { + end = j + break + } + } + + // Extract direct field names from lines[i+1:end]. + prefix := coalesceFieldPrefix(line.path) + fieldNames := make(map[string]bool) + for j := i + 1; j < end; j++ { + p := lines[j].path + if !strings.HasPrefix(p, prefix) { + continue + } + remainder := p[len(prefix):] + // Direct field: no "." in the remainder. + if strings.Contains(remainder, ".") { + continue + } + // Strip array brackets to get field name. + bracketIdx := strings.Index(remainder, "[") + var fieldName string + if bracketIdx < 0 { + fieldName = remainder + } else { + fieldName = remainder[:bracketIdx] + } + if fieldName != "" { + fieldNames[fieldName] = true + } + } + + keys := make([]string, 0, len(fieldNames)) + for k := range fieldNames { + keys = append(keys, k) + } + sort.Strings(keys) + + types = append(types, &coalesceType{ + name: line.typeVal, + path: line.path, + keys: keys, + keysSig: strings.Join(keys, ","), + }) + } + + return types +} + +// keySetGroup holds all types that share the same key set. +type keySetGroup struct { + keysSig string + keys []string + members []*coalesceType +} + +func groupTypesByKeySet(types []*coalesceType) []*keySetGroup { + groupMap := make(map[string]*keySetGroup) + var groups []*keySetGroup + + for _, t := range types { + if g, ok := groupMap[t.keysSig]; ok { + g.members = append(g.members, t) + } else { + g = &keySetGroup{ + keysSig: t.keysSig, + keys: t.keys, + members: []*coalesceType{t}, + } + groupMap[t.keysSig] = g + groups = append(groups, g) + } + } + + return groups +} + +func assignCanonicalNames(groups []*keySetGroup) map[string]string { + nameMap := make(map[string]string) // old name → new name + + // Pick a base name for each group. + type groupName struct { + group *keySetGroup + baseName string + } + var gns []groupName + baseNameCount := make(map[string]int) + for _, g := range groups { + name := pickGroupName(g) + gns = append(gns, groupName{g, name}) + baseNameCount[name]++ + } + + // Assign final names, numbering when base names conflict. + baseNameCounter := make(map[string]int) + for _, gn := range gns { + var finalName string + if baseNameCount[gn.baseName] == 1 { + finalName = gn.baseName + } else { + idx := baseNameCounter[gn.baseName] + baseNameCounter[gn.baseName]++ + finalName = fmt.Sprintf("%s%d", gn.baseName, idx) + } + for _, t := range gn.group.members { + nameMap[t.name] = finalName + } + } + + return nameMap +} + +// pickGroupName selects the best canonical name for a key-set group. +func pickGroupName(g *keySetGroup) string { + // Count distinct base names among members. + baseNames := make(map[string]int) + var firstBase string + for _, t := range g.members { + base := stripTrailingDigits(t.name) + baseNames[base]++ + if firstBase == "" { + firstBase = base + } + } + + // If many distinct names share this shape, derive from keys + // (e.g., {name, url} → "NameUrl"). + if len(baseNames) > 2 { + return deriveNameFromKeys(g.keys) + } + + // If one base name dominates, use it. + if len(baseNames) == 1 { + return firstBase + } + + // Two distinct names — pick the most common, then shortest. + var best string + var bestCount int + for name, count := range baseNames { + if count > bestCount { + best = name + bestCount = count + } else if count == bestCount && len(name) < len(best) { + best = name + } + } + return best +} + +func deriveNameFromKeys(keys []string) string { + if len(keys) == 0 { + return "Unknown" + } + // Use all keys (up to 3) to form the name. + n := len(keys) + if n > 3 { + n = 3 + } + var parts []string + for _, k := range keys[:n] { + parts = append(parts, snakeToPascal(k)) + } + return strings.Join(parts, "") +} + +// stripTrailingDigits removes trailing digits from a type name. +func stripTrailingDigits(name string) string { + i := len(name) - 1 + for i >= 0 && name[i] >= '0' && name[i] <= '9' { + i-- + } + if i < 0 { + return name + } + return name[:i+1] +} + +// detectNullablePaths returns the set of paths where {null} appears +// alongside at least one concrete type. +func detectNullablePaths(lines []rawLine) map[string]bool { + pathTypes := make(map[string][]string) + for _, line := range lines { + pathTypes[line.path] = append(pathTypes[line.path], line.typeVal) + } + + nullables := make(map[string]bool) + for path, typeVals := range pathTypes { + hasNull := false + hasOther := false + for _, tv := range typeVals { + if tv == "null" { + hasNull = true + } else if tv != "undefined" && tv != "empty" { + hasOther = true + } + } + if hasNull && hasOther { + nullables[path] = true + } + } + + return nullables +} + +func emitCoalesced(lines []rawLine, nameMap map[string]string, nullables map[string]bool) []string { + emittedTypes := make(map[string]bool) // canonical name → already expanded + var result []string + + i := 0 + for i < len(lines) { + line := lines[i] + + // Skip standalone {null} lines at nullable paths (merged into ?). + if line.typeVal == "null" && nullables[line.path] { + i++ + continue + } + + if isNamedType(line.typeVal) { + canonicalName := nameMap[line.typeVal] + nullable := "" + if nullables[line.path] { + nullable = "?" + } + + if emittedTypes[canonicalName] { + // Emit reference only, skip field lines. + result = append(result, line.path+"{"+canonicalName+nullable+"}") + i++ + prefix := coalesceFieldPrefix(line.path) + for i < len(lines) && strings.HasPrefix(lines[i].path, prefix) { + i++ + } + continue + } + + // First occurrence — emit type intro and let fields be emitted normally. + emittedTypes[canonicalName] = true + result = append(result, line.path+"{"+canonicalName+nullable+"}") + i++ + continue + } + + // Regular line (primitive, null, undefined, empty, etc.). + tv := line.typeVal + nullable := "" + if nullables[line.path] && tv != "undefined" && tv != "empty" { + nullable = "?" + } + + result = append(result, line.path+"{"+tv+nullable+"}") + i++ + } + + return result +} + +func coalesceFieldPrefix(path string) string { + if path == "" { + return "." + } + return path + "." +} diff --git a/tools/jsontypes/coalesce_test.go b/tools/jsontypes/coalesce_test.go new file mode 100644 index 0000000..765ffb3 --- /dev/null +++ b/tools/jsontypes/coalesce_test.go @@ -0,0 +1,248 @@ +package jsontypes + +import ( + "encoding/json" + "os" + "strings" + "testing" +) + +func TestCoalesceBasic(t *testing.T) { + input := []string{ + "{Root0}", + ".name{string}", + ".email{string}", + ".email{null}", + ".friends[]{FriendsItem1}", + ".friends[].name{string}", + ".friends[].age{int}", + } + + got := Coalesce(input) + + // Root0 → Root (unique) + assertLineContains(t, got, "{Root}") + // email should be {string?} (nullable) + assertLineContains(t, got, ".email{string?}") + // standalone {null} should be removed + assertLineNotContains(t, got, ".email{null}") + // FriendsItem1 → FriendsItem (unique) + assertLineContains(t, got, ".friends[]{FriendsItem}") + + t.Logf("output (%d lines):\n%s", len(got), strings.Join(got, "\n")) +} + +func TestCoalesceDedup(t *testing.T) { + // Two types with the same key set at different paths. + input := []string{ + "{Root0}", + ".author{Author1}", + ".author.name{string}", + ".author.url{string}", + ".reviewer{Reviewer2}", + ".reviewer.name{string}", + ".reviewer.url{string}", + } + + got := Coalesce(input) + + // Both Author and Reviewer have {name, url} → merged into one type. + // Only one should have fields expanded. + nameUrlCount := 0 + fieldCount := 0 + for _, line := range got { + if strings.Contains(line, ".name{string}") { + fieldCount++ + } + if strings.Contains(line, "{Author}") || strings.Contains(line, "{Reviewer}") { + nameUrlCount++ + } + } + + // The canonical name should appear at both paths. + if nameUrlCount != 2 { + t.Errorf("expected 2 references to canonical type, got %d", nameUrlCount) + } + + // But fields should only appear once (for the first occurrence). + if fieldCount != 1 { + t.Errorf("expected 1 field expansion, got %d", fieldCount) + } + + t.Logf("output (%d lines):\n%s", len(got), strings.Join(got, "\n")) +} + +func TestCoalesceDedupManyNames(t *testing.T) { + // >2 distinct base names with same key set → derive name from keys. + input := []string{ + "{Root0}", + ".ability{Ability1}", + ".ability.name{string}", + ".ability.url{string}", + ".version{Version2}", + ".version.name{string}", + ".version.url{string}", + ".species{Species3}", + ".species.name{string}", + ".species.url{string}", + } + + got := Coalesce(input) + + // 3 distinct base names → derived name "NameUrl". + assertLineContains(t, got, "{NameUrl}") + // Fields expanded only once. + nameCount := 0 + for _, line := range got { + if strings.HasSuffix(line, ".name{string}") { + nameCount++ + } + } + if nameCount != 1 { + t.Errorf("expected 1 .name{string} (deduped), got %d", nameCount) + } + + t.Logf("output (%d lines):\n%s", len(got), strings.Join(got, "\n")) +} + +func TestCoalesceUndefinedPreserved(t *testing.T) { + input := []string{ + "[]{RootItem0}", + "[].name{string}", + "[].email{undefined}", + "[].email{string}", + } + + got := Coalesce(input) + + // {undefined} should be preserved (not collapsed into ?). + assertLineContains(t, got, "[].email{undefined}") + // {string} should NOT be nullable (undefined ≠ null). + assertLineContains(t, got, "[].email{string}") + assertLineNotContains(t, got, "[].email{string?}") + + t.Logf("output (%d lines):\n%s", len(got), strings.Join(got, "\n")) +} + +func TestCoalesceNullAndUndefined(t *testing.T) { + input := []string{ + "[]{RootItem0}", + "[].email{undefined}", + "[].email{null}", + "[].email{string}", + } + + got := Coalesce(input) + + // {undefined} preserved + assertLineContains(t, got, "[].email{undefined}") + // {null} collapsed into {string?} + assertLineContains(t, got, "[].email{string?}") + assertLineNotContains(t, got, "[].email{null}") + + t.Logf("output (%d lines):\n%s", len(got), strings.Join(got, "\n")) +} + +func TestCoalescePokemonFixture(t *testing.T) { + lines := rawPathsFromFixture(t, "testdata/pokemon-pikachu.json") + got := Coalesce(lines) + + // Should be significantly fewer lines than the raw output. + if len(got) >= len(lines) { + t.Errorf("coalesced output (%d lines) should be fewer than raw (%d lines)", + len(got), len(lines)) + } + + // {name, url} types should be merged (many types share this shape). + assertLineContains(t, got, "{NameUrl}") + + // Root should lose its number. + assertLineContains(t, got, "{Root}") + + // Core structure preserved. + assertLineContains(t, got, ".abilities[]{AbilitiesItem}") + assertLineContains(t, got, ".stats[]{StatsItem}") + assertLineContains(t, got, ".types[]{TypesItem}") + + // Nullable fields: dream_world.front_female is null in some contexts. + // (In single-document mode, null-only fields stay as {null}.) + + t.Logf("raw: %d lines → coalesced: %d lines", len(lines), len(got)) + t.Logf("output:\n%s", strings.Join(got, "\n")) +} + +func TestCoalesceSWAPIPlanets(t *testing.T) { + lines := rawPathsFromFixture(t, "testdata/swapi-planets.json") + got := Coalesce(lines) + + // Root should be clean. + assertLineContains(t, got, "{Root}") + assertLineContains(t, got, ".results[]{ResultsItem}") + + // No nullable fields in this fixture (previous is null but standalone). + assertLineContains(t, got, ".previous{null}") + + t.Logf("raw: %d lines → coalesced: %d lines", len(lines), len(got)) + t.Logf("output:\n%s", strings.Join(got, "\n")) +} + +func TestCoalescePerBaseNameNumbering(t *testing.T) { + // Two types with the same base name but different key sets. + input := []string{ + "{Root0}", + ".config{Settings1}", + ".config.host{string}", + ".config.port{int}", + ".prefs{Settings2}", + ".prefs.theme{string}", + ".prefs.lang{string}", + ".prefs.debug{bool}", + } + + got := Coalesce(input) + + // Both are "Settings" base but different keys → numbered. + assertLineContains(t, got, ".config{Settings0}") + assertLineContains(t, got, ".prefs{Settings1}") + + t.Logf("output (%d lines):\n%s", len(got), strings.Join(got, "\n")) +} + +// Helper: run RawPaths on a fixture file and return the lines. +func rawPathsFromFixture(t *testing.T, path string) []string { + t.Helper() + f, err := os.Open(path) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + defer f.Close() + + var data any + dec := json.NewDecoder(f) + dec.UseNumber() + if err := dec.Decode(&data); err != nil { + t.Fatalf("decode: %v", err) + } + + return RawPaths(data) +} + +func assertLineContains(t *testing.T, lines []string, want string) { + t.Helper() + for _, line := range lines { + if strings.Contains(line, want) { + return + } + } + t.Errorf("missing expected line containing %q", want) +} + +func assertLineNotContains(t *testing.T, lines []string, want string) { + t.Helper() + for _, line := range lines { + if line == want { + t.Errorf("unexpected line %q", want) + return + } + } +}