mirror of
https://github.com/therootcompany/golib.git
synced 2026-11-04 06:55:00 +00:00
feat(jsontypes): add RawPaths and jsonrawpaths CLI
RawPaths walks JSON depth-first and emits flat paths with monotonically numbered type names (Root0, People2, etc.). Each type intro immediately precedes its fields, making the output streamable and order-stable. Also improves map detection heuristics: - All-numeric keys are always maps (even with 1-2 keys) - Keys containing digits detected as IDs (e.g., emp_101) - New isKnownFieldName with broad list of common field names (timestamps, flags, relations, etc.) as strong struct signal
This commit is contained in:
parent
1fbaef9caf
commit
b58c09350b
@ -517,9 +517,11 @@ func TestDecideMapOrStructDefault(t *testing.T) {
|
||||
},
|
||||
})
|
||||
|
||||
// Use 2 keys that aren't known field names — ambiguous enough to
|
||||
// trigger the resolver (not confident either way).
|
||||
obj := map[string]any{
|
||||
"errors": []any{},
|
||||
"rooms": []any{map[string]any{"name": "foo"}},
|
||||
"foo": []any{},
|
||||
"bar": []any{map[string]any{"x": "y"}},
|
||||
}
|
||||
a.Analyze(".", obj)
|
||||
|
||||
|
||||
36
tools/jsontypes/cmd/jsonrawpaths/main.go
Normal file
36
tools/jsontypes/cmd/jsonrawpaths/main.go
Normal file
@ -0,0 +1,36 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/therootcompany/golib/tools/jsontypes"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var input *os.File
|
||||
if len(os.Args) > 1 && os.Args[1] != "-" {
|
||||
f, err := os.Open(os.Args[1])
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer f.Close()
|
||||
input = f
|
||||
} else {
|
||||
input = os.Stdin
|
||||
}
|
||||
|
||||
var data any
|
||||
dec := json.NewDecoder(input)
|
||||
dec.UseNumber()
|
||||
if err := dec.Decode(&data); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error parsing JSON: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
for _, path := range jsontypes.RawPaths(data) {
|
||||
fmt.Println(path)
|
||||
}
|
||||
}
|
||||
@ -14,12 +14,8 @@ import (
|
||||
func looksLikeMap(obj map[string]any) (isMap bool, confident bool) {
|
||||
keys := sortedKeys(obj)
|
||||
n := len(keys)
|
||||
if n < 3 {
|
||||
// Too few keys to be confident about anything
|
||||
return false, false
|
||||
}
|
||||
|
||||
// All keys are integers?
|
||||
// All keys are integers → always a map, even with 1 key.
|
||||
allInts := true
|
||||
for _, k := range keys {
|
||||
if _, err := strconv.ParseInt(k, 10, 64); err != nil {
|
||||
@ -27,28 +23,39 @@ func looksLikeMap(obj map[string]any) (isMap bool, confident bool) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if allInts {
|
||||
if allInts && n > 0 {
|
||||
return true, true
|
||||
}
|
||||
|
||||
// All keys same length and contain mixed letters+digits → likely IDs
|
||||
if allSameLength(keys) && allAlphanumericWithDigits(keys) {
|
||||
// If any key is a well-known struct field name, it's definitely a struct.
|
||||
// This catches 1-2 key objects like {"name": ..., "age": ...}.
|
||||
if hasKnownFieldName(keys) {
|
||||
return false, true
|
||||
}
|
||||
|
||||
// All keys contain mixed letters+digits → likely IDs (e.g., "emp_101").
|
||||
if allContainDigits(keys) {
|
||||
return true, true
|
||||
}
|
||||
|
||||
// All keys same length and look like base64/hex IDs
|
||||
// All keys same length and look like base64/hex IDs.
|
||||
if allSameLength(keys) && allLookLikeIDs(keys) {
|
||||
return true, true
|
||||
}
|
||||
|
||||
// Keys look like typical struct field names (camelCase, snake_case, short words)
|
||||
if n < 3 {
|
||||
// Too few keys and no strong signal either way.
|
||||
return false, false
|
||||
}
|
||||
|
||||
// Keys look like typical struct field names (camelCase, snake_case, short words).
|
||||
// This must be checked before value-shape heuristics: a struct with many
|
||||
// fields whose values happen to share a shape is still a struct.
|
||||
if allLookLikeFieldNames(keys) {
|
||||
return false, true
|
||||
}
|
||||
|
||||
// Large number of keys where most values have the same shape — likely a map
|
||||
// Large number of keys where most values have the same shape → likely a map.
|
||||
if n > 20 && valuesHaveSimilarShape(obj) {
|
||||
return true, true
|
||||
}
|
||||
@ -56,6 +63,122 @@ func looksLikeMap(obj map[string]any) (isMap bool, confident bool) {
|
||||
return false, false
|
||||
}
|
||||
|
||||
// hasKnownFieldName returns true if any key matches a well-known struct field
|
||||
// name. A single match is a strong signal — real maps don't have keys named
|
||||
// "created_at" or "email".
|
||||
func hasKnownFieldName(keys []string) bool {
|
||||
for _, k := range keys {
|
||||
if isKnownFieldName(k) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isKnownFieldName checks whether a key is a common struct/object field name.
|
||||
// This is deliberately broad — false positives (calling a map key a field)
|
||||
// are less harmful than false negatives (treating a struct as a map).
|
||||
func isKnownFieldName(k string) bool {
|
||||
lower := strings.ToLower(k)
|
||||
|
||||
// Suffix patterns: timestamps, flags, relations
|
||||
suffixes := []string{
|
||||
"_at", "_on", "_by", "_id", "_ids", "_url", "_uri",
|
||||
"_name", "_type", "_kind", "_key", "_code", "_date",
|
||||
"_count", "_size", "_path", "_hash", "_token",
|
||||
"_enabled", "_disabled", "_active", "_status",
|
||||
}
|
||||
for _, s := range suffixes {
|
||||
if strings.HasSuffix(lower, s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// camelCase suffixes
|
||||
camelSuffixes := []string{
|
||||
"At", "On", "By", "Id", "Ids", "Url", "Uri",
|
||||
"Name", "Type", "Kind", "Key", "Code", "Date",
|
||||
"Count", "Size", "Path", "Hash", "Token",
|
||||
}
|
||||
for _, s := range camelSuffixes {
|
||||
if strings.HasSuffix(k, s) && len(k) > len(s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// Exact matches: common field names across APIs
|
||||
switch lower {
|
||||
case
|
||||
// identity
|
||||
"id", "uid", "uuid", "guid", "slug",
|
||||
// naming
|
||||
"name", "title", "label", "description", "summary",
|
||||
// typing/classification
|
||||
"type", "kind", "category", "class", "role", "status", "state",
|
||||
// content
|
||||
"value", "data", "content", "body", "text", "message", "comment",
|
||||
"url", "uri", "href", "link", "path",
|
||||
"email", "phone", "address",
|
||||
// flags
|
||||
"active", "enabled", "disabled", "visible", "hidden",
|
||||
"required", "optional", "readonly", "deleted", "archived",
|
||||
"public", "private", "verified", "approved", "published",
|
||||
// numbers
|
||||
"count", "total", "size", "length", "width", "height",
|
||||
"amount", "price", "cost", "quantity", "weight",
|
||||
"score", "rating", "level", "priority", "order", "index", "position",
|
||||
"version", "revision",
|
||||
"min", "max", "limit", "offset", "page",
|
||||
"latitude", "longitude", "lat", "lng", "lon",
|
||||
// structure
|
||||
"parent", "children", "items", "entries", "results", "records",
|
||||
"tags", "labels", "groups", "members", "users", "roles",
|
||||
"permissions", "scopes", "features", "options", "settings",
|
||||
"config", "configuration", "preferences", "metadata", "meta",
|
||||
"errors", "warnings",
|
||||
// media
|
||||
"format", "encoding", "charset", "locale", "language", "currency",
|
||||
"color", "icon", "image", "avatar", "thumbnail", "logo",
|
||||
"filename", "extension", "mimetype",
|
||||
// auth
|
||||
"username", "password", "token", "secret", "credential",
|
||||
// time
|
||||
"date", "time", "timestamp", "duration", "interval",
|
||||
"start", "end", "expires":
|
||||
return true
|
||||
}
|
||||
|
||||
// Prefix patterns: is_*, has_*, can_*, should_*, allow_*
|
||||
prefixes := []string{"is_", "has_", "can_", "should_", "allow_", "num_"}
|
||||
for _, p := range prefixes {
|
||||
if strings.HasPrefix(lower, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// allContainDigits checks if every key contains at least one digit,
|
||||
// suggesting they are IDs rather than field names (e.g., "emp_101", "abc123").
|
||||
func allContainDigits(keys []string) bool {
|
||||
if len(keys) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, k := range keys {
|
||||
hasDigit := false
|
||||
for _, r := range k {
|
||||
if unicode.IsDigit(r) {
|
||||
hasDigit = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasDigit {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func allSameLength(keys []string) bool {
|
||||
if len(keys) == 0 {
|
||||
return true
|
||||
@ -99,25 +222,7 @@ func isAlphanumeric(s string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// allAlphanumericWithDigits checks if all keys are alphanumeric and each
|
||||
// contains at least one digit (distinguishing IDs like "abc123" from field
|
||||
// names like "name").
|
||||
func allAlphanumericWithDigits(keys []string) bool {
|
||||
for _, k := range keys {
|
||||
hasDigit := false
|
||||
for _, r := range k {
|
||||
if unicode.IsDigit(r) {
|
||||
hasDigit = true
|
||||
} else if !unicode.IsLetter(r) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if !hasDigit {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
func isBase64(s string) bool {
|
||||
// Try standard and URL-safe base64
|
||||
|
||||
218
tools/jsontypes/rawpaths.go
Normal file
218
tools/jsontypes/rawpaths.go
Normal file
@ -0,0 +1,218 @@
|
||||
package jsontypes
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// RawPaths walks a decoded JSON value depth-first and emits flat paths with
|
||||
// monotonically numbered type names. Every struct shape gets a unique name
|
||||
// derived from its parent path segment and a global counter (e.g., Root0,
|
||||
// People2, People3). Type intros appear immediately before their fields,
|
||||
// making the output streamable and order-stable.
|
||||
//
|
||||
// The value must have been decoded with json.Decoder.UseNumber() so that
|
||||
// integers and floats are distinguishable.
|
||||
func RawPaths(v any) []string {
|
||||
w := &rawWalker{}
|
||||
w.walk(".", v)
|
||||
return w.paths
|
||||
}
|
||||
|
||||
type rawWalker struct {
|
||||
counter int
|
||||
paths []string
|
||||
}
|
||||
|
||||
func (w *rawWalker) emit(path string) {
|
||||
w.paths = append(w.paths, path)
|
||||
}
|
||||
|
||||
func (w *rawWalker) nextName(path string) string {
|
||||
name := inferTypeName(path)
|
||||
id := w.counter
|
||||
w.counter++
|
||||
return fmt.Sprintf("%s%d", name, id)
|
||||
}
|
||||
|
||||
func (w *rawWalker) walk(prefix string, v any) {
|
||||
switch val := v.(type) {
|
||||
case nil:
|
||||
w.emit(prefix + "{null}")
|
||||
case bool:
|
||||
w.emit(prefix + "{bool}")
|
||||
case json.Number:
|
||||
if _, err := val.Int64(); err == nil {
|
||||
w.emit(prefix + "{int}")
|
||||
} else {
|
||||
w.emit(prefix + "{float}")
|
||||
}
|
||||
case string:
|
||||
w.emit(prefix + "{string}")
|
||||
case map[string]any:
|
||||
w.walkObject(prefix, val)
|
||||
case []any:
|
||||
w.walkArray(prefix, val)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *rawWalker) walkObject(prefix string, obj map[string]any) {
|
||||
if len(obj) == 0 {
|
||||
w.emit(prefix + "{any}")
|
||||
return
|
||||
}
|
||||
|
||||
isMap, _ := looksLikeMap(obj)
|
||||
if isMap {
|
||||
w.walkMap(prefix, obj)
|
||||
return
|
||||
}
|
||||
|
||||
w.walkStruct(prefix, []map[string]any{obj})
|
||||
}
|
||||
|
||||
func (w *rawWalker) walkMap(prefix string, obj map[string]any) {
|
||||
keyName := inferKeyName(obj)
|
||||
mapPrefix := prefix + "[" + keyName + "]"
|
||||
|
||||
// Collect all values and walk as a collection.
|
||||
values := make([]any, 0, len(obj))
|
||||
for _, v := range obj {
|
||||
values = append(values, v)
|
||||
}
|
||||
w.walkCollection(mapPrefix, values)
|
||||
}
|
||||
|
||||
func (w *rawWalker) walkStruct(prefix string, instances []map[string]any) {
|
||||
merged := mergeObjects(instances)
|
||||
typeName := w.nextName(prefix)
|
||||
typePrefix := prefix + "{" + typeName + "}"
|
||||
|
||||
// Emit type intro.
|
||||
w.emit(typePrefix)
|
||||
|
||||
keys := sortedKeys(merged)
|
||||
for _, k := range keys {
|
||||
// Collect all values for this field across instances.
|
||||
var fieldVals []any
|
||||
for _, inst := range instances {
|
||||
if v, ok := inst[k]; ok {
|
||||
fieldVals = append(fieldVals, v)
|
||||
} else {
|
||||
fieldVals = append(fieldVals, nil)
|
||||
}
|
||||
}
|
||||
w.walkCollection(typePrefix+"."+k, fieldVals)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *rawWalker) walkArray(prefix string, arr []any) {
|
||||
if len(arr) == 0 {
|
||||
w.emit(prefix + "[]{any}")
|
||||
return
|
||||
}
|
||||
|
||||
if isTupleCandidate(arr) {
|
||||
w.walkTuple(prefix, arr)
|
||||
return
|
||||
}
|
||||
|
||||
w.walkCollection(prefix+"[]", arr)
|
||||
}
|
||||
|
||||
func (w *rawWalker) walkTuple(prefix string, arr []any) {
|
||||
for i, v := range arr {
|
||||
w.walk(fmt.Sprintf("%s[%d]", prefix, i), v)
|
||||
}
|
||||
}
|
||||
|
||||
// walkCollection handles multiple values at the same path position (array
|
||||
// elements, map values, or field values across struct instances). It groups
|
||||
// objects by shape and walks each shape separately.
|
||||
func (w *rawWalker) walkCollection(prefix string, values []any) {
|
||||
// Separate by kind: objects, primitives/arrays, nulls.
|
||||
type shapeInstances struct {
|
||||
sig string
|
||||
instances []map[string]any
|
||||
}
|
||||
var shapes []shapeInstances
|
||||
shapeIndex := make(map[string]int)
|
||||
hasNull := false
|
||||
var nonObjects []any
|
||||
|
||||
for _, v := range values {
|
||||
switch val := v.(type) {
|
||||
case nil:
|
||||
hasNull = true
|
||||
case map[string]any:
|
||||
sig := shapeSignature(val)
|
||||
if idx, ok := shapeIndex[sig]; ok {
|
||||
shapes[idx].instances = append(shapes[idx].instances, val)
|
||||
} else {
|
||||
shapeIndex[sig] = len(shapes)
|
||||
shapes = append(shapes, shapeInstances{sig, []map[string]any{val}})
|
||||
}
|
||||
default:
|
||||
nonObjects = append(nonObjects, val)
|
||||
}
|
||||
}
|
||||
|
||||
// Emit null if seen.
|
||||
if hasNull {
|
||||
w.emit(prefix + "{null}")
|
||||
}
|
||||
|
||||
// Emit each object shape. If a shape has exactly one instance and it
|
||||
// looks like a map, walk it as a map instead of a struct.
|
||||
for _, shape := range shapes {
|
||||
if len(shape.instances) == 1 {
|
||||
obj := shape.instances[0]
|
||||
isMap, _ := looksLikeMap(obj)
|
||||
if isMap {
|
||||
w.walkMap(prefix, obj)
|
||||
continue
|
||||
}
|
||||
}
|
||||
w.walkStruct(prefix, shape.instances)
|
||||
}
|
||||
|
||||
// Emit primitive/array types (deduplicated).
|
||||
seen := make(map[string]bool)
|
||||
for _, v := range nonObjects {
|
||||
switch val := v.(type) {
|
||||
case json.Number:
|
||||
var typ string
|
||||
if _, err := val.Int64(); err == nil {
|
||||
typ = "int"
|
||||
} else {
|
||||
typ = "float"
|
||||
}
|
||||
if !seen[typ] {
|
||||
seen[typ] = true
|
||||
w.emit(prefix + "{" + typ + "}")
|
||||
}
|
||||
case string:
|
||||
if !seen["string"] {
|
||||
seen["string"] = true
|
||||
w.emit(prefix + "{string}")
|
||||
}
|
||||
case bool:
|
||||
if !seen["bool"] {
|
||||
seen["bool"] = true
|
||||
w.emit(prefix + "{bool}")
|
||||
}
|
||||
case []any:
|
||||
// Nested arrays — walk once.
|
||||
if !seen["array"] {
|
||||
seen["array"] = true
|
||||
w.walkArray(prefix, val)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If nothing was emitted (all nils, empty), emit any.
|
||||
if !hasNull && len(shapes) == 0 && len(nonObjects) == 0 {
|
||||
w.emit(prefix + "{any}")
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user