feat(jsontypes): add RawPaths and jsonrawpaths CLI

RawPaths walks JSON depth-first and emits flat paths with monotonically
numbered type names (Root0, People2, etc.). Each type intro immediately
precedes its fields, making the output streamable and order-stable.

Also improves map detection heuristics:
- All-numeric keys are always maps (even with 1-2 keys)
- Keys containing digits detected as IDs (e.g., emp_101)
- New isKnownFieldName with broad list of common field names
  (timestamps, flags, relations, etc.) as strong struct signal
This commit is contained in:
AJ ONeal 2026-03-07 22:18:41 -07:00
parent 1fbaef9caf
commit b58c09350b
No known key found for this signature in database
4 changed files with 393 additions and 32 deletions

View File

@ -517,9 +517,11 @@ func TestDecideMapOrStructDefault(t *testing.T) {
},
})
// Use 2 keys that aren't known field names — ambiguous enough to
// trigger the resolver (not confident either way).
obj := map[string]any{
"errors": []any{},
"rooms": []any{map[string]any{"name": "foo"}},
"foo": []any{},
"bar": []any{map[string]any{"x": "y"}},
}
a.Analyze(".", obj)

View File

@ -0,0 +1,36 @@
package main
import (
"encoding/json"
"fmt"
"os"
"github.com/therootcompany/golib/tools/jsontypes"
)
func main() {
var input *os.File
if len(os.Args) > 1 && os.Args[1] != "-" {
f, err := os.Open(os.Args[1])
if err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)
}
defer f.Close()
input = f
} else {
input = os.Stdin
}
var data any
dec := json.NewDecoder(input)
dec.UseNumber()
if err := dec.Decode(&data); err != nil {
fmt.Fprintf(os.Stderr, "error parsing JSON: %v\n", err)
os.Exit(1)
}
for _, path := range jsontypes.RawPaths(data) {
fmt.Println(path)
}
}

View File

@ -14,12 +14,8 @@ import (
func looksLikeMap(obj map[string]any) (isMap bool, confident bool) {
keys := sortedKeys(obj)
n := len(keys)
if n < 3 {
// Too few keys to be confident about anything
return false, false
}
// All keys are integers?
// All keys are integers → always a map, even with 1 key.
allInts := true
for _, k := range keys {
if _, err := strconv.ParseInt(k, 10, 64); err != nil {
@ -27,28 +23,39 @@ func looksLikeMap(obj map[string]any) (isMap bool, confident bool) {
break
}
}
if allInts {
if allInts && n > 0 {
return true, true
}
// All keys same length and contain mixed letters+digits → likely IDs
if allSameLength(keys) && allAlphanumericWithDigits(keys) {
// If any key is a well-known struct field name, it's definitely a struct.
// This catches 1-2 key objects like {"name": ..., "age": ...}.
if hasKnownFieldName(keys) {
return false, true
}
// All keys contain mixed letters+digits → likely IDs (e.g., "emp_101").
if allContainDigits(keys) {
return true, true
}
// All keys same length and look like base64/hex IDs
// All keys same length and look like base64/hex IDs.
if allSameLength(keys) && allLookLikeIDs(keys) {
return true, true
}
// Keys look like typical struct field names (camelCase, snake_case, short words)
if n < 3 {
// Too few keys and no strong signal either way.
return false, false
}
// Keys look like typical struct field names (camelCase, snake_case, short words).
// This must be checked before value-shape heuristics: a struct with many
// fields whose values happen to share a shape is still a struct.
if allLookLikeFieldNames(keys) {
return false, true
}
// Large number of keys where most values have the same shape — likely a map
// Large number of keys where most values have the same shape → likely a map.
if n > 20 && valuesHaveSimilarShape(obj) {
return true, true
}
@ -56,6 +63,122 @@ func looksLikeMap(obj map[string]any) (isMap bool, confident bool) {
return false, false
}
// hasKnownFieldName returns true if any key matches a well-known struct field
// name. A single match is a strong signal — real maps don't have keys named
// "created_at" or "email".
func hasKnownFieldName(keys []string) bool {
for _, k := range keys {
if isKnownFieldName(k) {
return true
}
}
return false
}
// isKnownFieldName checks whether a key is a common struct/object field name.
// This is deliberately broad — false positives (calling a map key a field)
// are less harmful than false negatives (treating a struct as a map).
func isKnownFieldName(k string) bool {
lower := strings.ToLower(k)
// Suffix patterns: timestamps, flags, relations
suffixes := []string{
"_at", "_on", "_by", "_id", "_ids", "_url", "_uri",
"_name", "_type", "_kind", "_key", "_code", "_date",
"_count", "_size", "_path", "_hash", "_token",
"_enabled", "_disabled", "_active", "_status",
}
for _, s := range suffixes {
if strings.HasSuffix(lower, s) {
return true
}
}
// camelCase suffixes
camelSuffixes := []string{
"At", "On", "By", "Id", "Ids", "Url", "Uri",
"Name", "Type", "Kind", "Key", "Code", "Date",
"Count", "Size", "Path", "Hash", "Token",
}
for _, s := range camelSuffixes {
if strings.HasSuffix(k, s) && len(k) > len(s) {
return true
}
}
// Exact matches: common field names across APIs
switch lower {
case
// identity
"id", "uid", "uuid", "guid", "slug",
// naming
"name", "title", "label", "description", "summary",
// typing/classification
"type", "kind", "category", "class", "role", "status", "state",
// content
"value", "data", "content", "body", "text", "message", "comment",
"url", "uri", "href", "link", "path",
"email", "phone", "address",
// flags
"active", "enabled", "disabled", "visible", "hidden",
"required", "optional", "readonly", "deleted", "archived",
"public", "private", "verified", "approved", "published",
// numbers
"count", "total", "size", "length", "width", "height",
"amount", "price", "cost", "quantity", "weight",
"score", "rating", "level", "priority", "order", "index", "position",
"version", "revision",
"min", "max", "limit", "offset", "page",
"latitude", "longitude", "lat", "lng", "lon",
// structure
"parent", "children", "items", "entries", "results", "records",
"tags", "labels", "groups", "members", "users", "roles",
"permissions", "scopes", "features", "options", "settings",
"config", "configuration", "preferences", "metadata", "meta",
"errors", "warnings",
// media
"format", "encoding", "charset", "locale", "language", "currency",
"color", "icon", "image", "avatar", "thumbnail", "logo",
"filename", "extension", "mimetype",
// auth
"username", "password", "token", "secret", "credential",
// time
"date", "time", "timestamp", "duration", "interval",
"start", "end", "expires":
return true
}
// Prefix patterns: is_*, has_*, can_*, should_*, allow_*
prefixes := []string{"is_", "has_", "can_", "should_", "allow_", "num_"}
for _, p := range prefixes {
if strings.HasPrefix(lower, p) {
return true
}
}
return false
}
// allContainDigits checks if every key contains at least one digit,
// suggesting they are IDs rather than field names (e.g., "emp_101", "abc123").
func allContainDigits(keys []string) bool {
if len(keys) == 0 {
return false
}
for _, k := range keys {
hasDigit := false
for _, r := range k {
if unicode.IsDigit(r) {
hasDigit = true
break
}
}
if !hasDigit {
return false
}
}
return true
}
func allSameLength(keys []string) bool {
if len(keys) == 0 {
return true
@ -99,25 +222,7 @@ func isAlphanumeric(s string) bool {
return true
}
// allAlphanumericWithDigits checks if all keys are alphanumeric and each
// contains at least one digit (distinguishing IDs like "abc123" from field
// names like "name").
func allAlphanumericWithDigits(keys []string) bool {
for _, k := range keys {
hasDigit := false
for _, r := range k {
if unicode.IsDigit(r) {
hasDigit = true
} else if !unicode.IsLetter(r) {
return false
}
}
if !hasDigit {
return false
}
}
return true
}
func isBase64(s string) bool {
// Try standard and URL-safe base64

218
tools/jsontypes/rawpaths.go Normal file
View File

@ -0,0 +1,218 @@
package jsontypes
import (
"encoding/json"
"fmt"
)
// RawPaths walks a decoded JSON value depth-first and emits flat paths with
// monotonically numbered type names. Every struct shape gets a unique name
// derived from its parent path segment and a global counter (e.g., Root0,
// People2, People3). Type intros appear immediately before their fields,
// making the output streamable and order-stable.
//
// The value must have been decoded with json.Decoder.UseNumber() so that
// integers and floats are distinguishable.
func RawPaths(v any) []string {
w := &rawWalker{}
w.walk(".", v)
return w.paths
}
type rawWalker struct {
counter int
paths []string
}
func (w *rawWalker) emit(path string) {
w.paths = append(w.paths, path)
}
func (w *rawWalker) nextName(path string) string {
name := inferTypeName(path)
id := w.counter
w.counter++
return fmt.Sprintf("%s%d", name, id)
}
func (w *rawWalker) walk(prefix string, v any) {
switch val := v.(type) {
case nil:
w.emit(prefix + "{null}")
case bool:
w.emit(prefix + "{bool}")
case json.Number:
if _, err := val.Int64(); err == nil {
w.emit(prefix + "{int}")
} else {
w.emit(prefix + "{float}")
}
case string:
w.emit(prefix + "{string}")
case map[string]any:
w.walkObject(prefix, val)
case []any:
w.walkArray(prefix, val)
}
}
func (w *rawWalker) walkObject(prefix string, obj map[string]any) {
if len(obj) == 0 {
w.emit(prefix + "{any}")
return
}
isMap, _ := looksLikeMap(obj)
if isMap {
w.walkMap(prefix, obj)
return
}
w.walkStruct(prefix, []map[string]any{obj})
}
func (w *rawWalker) walkMap(prefix string, obj map[string]any) {
keyName := inferKeyName(obj)
mapPrefix := prefix + "[" + keyName + "]"
// Collect all values and walk as a collection.
values := make([]any, 0, len(obj))
for _, v := range obj {
values = append(values, v)
}
w.walkCollection(mapPrefix, values)
}
func (w *rawWalker) walkStruct(prefix string, instances []map[string]any) {
merged := mergeObjects(instances)
typeName := w.nextName(prefix)
typePrefix := prefix + "{" + typeName + "}"
// Emit type intro.
w.emit(typePrefix)
keys := sortedKeys(merged)
for _, k := range keys {
// Collect all values for this field across instances.
var fieldVals []any
for _, inst := range instances {
if v, ok := inst[k]; ok {
fieldVals = append(fieldVals, v)
} else {
fieldVals = append(fieldVals, nil)
}
}
w.walkCollection(typePrefix+"."+k, fieldVals)
}
}
func (w *rawWalker) walkArray(prefix string, arr []any) {
if len(arr) == 0 {
w.emit(prefix + "[]{any}")
return
}
if isTupleCandidate(arr) {
w.walkTuple(prefix, arr)
return
}
w.walkCollection(prefix+"[]", arr)
}
func (w *rawWalker) walkTuple(prefix string, arr []any) {
for i, v := range arr {
w.walk(fmt.Sprintf("%s[%d]", prefix, i), v)
}
}
// walkCollection handles multiple values at the same path position (array
// elements, map values, or field values across struct instances). It groups
// objects by shape and walks each shape separately.
func (w *rawWalker) walkCollection(prefix string, values []any) {
// Separate by kind: objects, primitives/arrays, nulls.
type shapeInstances struct {
sig string
instances []map[string]any
}
var shapes []shapeInstances
shapeIndex := make(map[string]int)
hasNull := false
var nonObjects []any
for _, v := range values {
switch val := v.(type) {
case nil:
hasNull = true
case map[string]any:
sig := shapeSignature(val)
if idx, ok := shapeIndex[sig]; ok {
shapes[idx].instances = append(shapes[idx].instances, val)
} else {
shapeIndex[sig] = len(shapes)
shapes = append(shapes, shapeInstances{sig, []map[string]any{val}})
}
default:
nonObjects = append(nonObjects, val)
}
}
// Emit null if seen.
if hasNull {
w.emit(prefix + "{null}")
}
// Emit each object shape. If a shape has exactly one instance and it
// looks like a map, walk it as a map instead of a struct.
for _, shape := range shapes {
if len(shape.instances) == 1 {
obj := shape.instances[0]
isMap, _ := looksLikeMap(obj)
if isMap {
w.walkMap(prefix, obj)
continue
}
}
w.walkStruct(prefix, shape.instances)
}
// Emit primitive/array types (deduplicated).
seen := make(map[string]bool)
for _, v := range nonObjects {
switch val := v.(type) {
case json.Number:
var typ string
if _, err := val.Int64(); err == nil {
typ = "int"
} else {
typ = "float"
}
if !seen[typ] {
seen[typ] = true
w.emit(prefix + "{" + typ + "}")
}
case string:
if !seen["string"] {
seen["string"] = true
w.emit(prefix + "{string}")
}
case bool:
if !seen["bool"] {
seen["bool"] = true
w.emit(prefix + "{bool}")
}
case []any:
// Nested arrays — walk once.
if !seen["array"] {
seen["array"] = true
w.walkArray(prefix, val)
}
}
}
// If nothing was emitted (all nils, empty), emit any.
if !hasNull && len(shapes) == 0 && len(nonObjects) == 0 {
w.emit(prefix + "{any}")
}
}