feat: merge object shapes with optional fields in raw paths

tools/jsontypes/coalesce.go

@@ -271,8 +271,8 @@ func stripTrailingDigits(name string) string {
 	return name[:i+1]
 }
 
-// detectNullablePaths returns the set of paths where {null} appears
-// alongside at least one concrete type.
+// detectNullablePaths returns the set of paths where {null} or {undefined}
+// appears alongside at least one concrete type.
 func detectNullablePaths(lines []rawLine) map[string]bool {
 	pathTypes := make(map[string][]string)
 	for _, line := range lines {
@@ -281,16 +281,16 @@ func detectNullablePaths(lines []rawLine) map[string]bool {
 
 	nullables := make(map[string]bool)
 	for path, typeVals := range pathTypes {
-		hasNull := false
+		hasAbsent := false
 		hasOther := false
 		for _, tv := range typeVals {
-			if tv == "null" {
-				hasNull = true
-			} else if tv != "undefined" && tv != "empty" {
+			if tv == "null" || tv == "undefined" {
+				hasAbsent = true
+			} else if tv != "empty" {
 				hasOther = true
 			}
 		}
-		if hasNull && hasOther {
+		if hasAbsent && hasOther {
 			nullables[path] = true
 		}
 	}
@@ -306,8 +306,9 @@ func emitCoalesced(lines []rawLine, nameMap map[string]string, nullables map[str
 	for i < len(lines) {
 		line := lines[i]
 
-		// Skip standalone {null} lines at nullable paths (merged into ?).
-		if line.typeVal == "null" && nullables[line.path] {
+		// Skip standalone {null} and {undefined} lines at nullable paths
+		// (merged into ? on the concrete type line).
+		if (line.typeVal == "null" || line.typeVal == "undefined") && nullables[line.path] {
 			i++
 			continue
 		}

tools/jsontypes/coalesce_test.go

@@ -105,7 +105,7 @@ func TestCoalesceDedupManyNames(t *testing.T) {
 	t.Logf("output (%d lines):\n%s", len(got), strings.Join(got, "\n"))
 }
 
-func TestCoalesceUndefinedPreserved(t *testing.T) {
+func TestCoalesceUndefinedCollapses(t *testing.T) {
 	input := []string{
 		"[]{RootItem0}",
 		"[].name{string}",
@@ -115,16 +115,15 @@ func TestCoalesceUndefinedPreserved(t *testing.T) {
 
 	got := Coalesce(input)
 
-	// {undefined} should be preserved (not collapsed into ?).
-	assertLineContains(t, got, "[].email{undefined}")
-	// {string} should NOT be nullable (undefined ≠ null).
-	assertLineContains(t, got, "[].email{string}")
-	assertLineNotContains(t, got, "[].email{string?}")
+	// {undefined} collapses into ? on the concrete type (same as null).
+	assertLineContains(t, got, "[].email{string?}")
+	assertLineNotContains(t, got, "[].email{undefined}")
+	assertLineNotContains(t, got, "[].email{string}")
 
 	t.Logf("output (%d lines):\n%s", len(got), strings.Join(got, "\n"))
 }
 
-func TestCoalesceNullAndUndefined(t *testing.T) {
+func TestCoalesceNullAndUndefinedBothCollapse(t *testing.T) {
 	input := []string{
 		"[]{RootItem0}",
 		"[].email{undefined}",
@@ -134,10 +133,9 @@ func TestCoalesceNullAndUndefined(t *testing.T) {
 
 	got := Coalesce(input)
 
-	// {undefined} preserved
-	assertLineContains(t, got, "[].email{undefined}")
-	// {null} collapsed into {string?}
+	// Both {undefined} and {null} collapse into {string?}.
 	assertLineContains(t, got, "[].email{string?}")
+	assertLineNotContains(t, got, "[].email{undefined}")
 	assertLineNotContains(t, got, "[].email{null}")
 
 	t.Logf("output (%d lines):\n%s", len(got), strings.Join(got, "\n"))

tools/jsontypes/rawpaths.go

@@ -208,18 +208,43 @@ func (w *rawWalker) walkCollection(prefix string, values []any) {
 		w.emit(prefix + "{null}")
 	}
 
-	// Emit each object shape. If a shape has exactly one instance and it
-	// looks like a map, walk it as a map instead of a struct.
-	for _, shape := range shapes {
-		if len(shape.instances) == 1 {
-			obj := shape.instances[0]
-			isMap, _ := looksLikeMap(obj)
-			if isMap {
-				w.walkMap(prefix, obj)
-				continue
+	// If multiple shapes share a common core (≥ 2 keys in > 50% of objects),
+	// treat them as one struct with optional fields.
+	if len(shapes) > 1 {
+		var allObjs []map[string]any
+		for _, s := range shapes {
+			allObjs = append(allObjs, s.instances...)
+		}
+		if shouldMergeObjects(allObjs) {
+			// Merge all into one struct — absent fields become optional.
+			w.walkStruct(prefix, allObjs)
+		} else {
+			// No common core — emit each shape separately.
+			for _, shape := range shapes {
+				if len(shape.instances) == 1 {
+					obj := shape.instances[0]
+					isMap, _ := looksLikeMap(obj)
+					if isMap {
+						w.walkMap(prefix, obj)
+						continue
+					}
+				}
+				w.walkStruct(prefix, shape.instances)
 			}
 		}
-		w.walkStruct(prefix, shape.instances)
+	} else {
+		// Single shape (or none) — emit directly.
+		for _, shape := range shapes {
+			if len(shape.instances) == 1 {
+				obj := shape.instances[0]
+				isMap, _ := looksLikeMap(obj)
+				if isMap {
+					w.walkMap(prefix, obj)
+					continue
+				}
+			}
+			w.walkStruct(prefix, shape.instances)
+		}
 	}
 
 	// Emit primitive/array types (deduplicated).
@@ -344,3 +369,37 @@ func joinPath(prefix, field string) string {
 	}
 	return prefix + "." + field
 }
+
+// shouldMergeObjects decides whether a pool of objects should be treated as
+// one struct type with optional fields, rather than multiple distinct types.
+//
+// Heuristic: if ≥ 2 keys appear in more than half the objects, the objects
+// likely share a common shape with optional fields. This catches the common
+// case of API responses where every record has the same core fields but some
+// records have extra ones (e.g. pagination results).
+func shouldMergeObjects(objects []map[string]any) bool {
+	n := len(objects)
+	if n < 2 {
+		return false
+	}
+
+	// Count how many objects contain each key.
+	keyCount := make(map[string]int)
+	for _, obj := range objects {
+		for k := range obj {
+			keyCount[k]++
+		}
+	}
+
+	// Count keys that appear in a majority of objects.
+	threshold := n / 2
+	highFreq := 0
+	for _, count := range keyCount {
+		if count > threshold {
+			highFreq++
+		}
+	}
+
+	// Two or more high-frequency keys means the objects share a common core.
+	return highFreq >= 2
+}

tools/jsontypes/sample.go

@@ -204,16 +204,40 @@ func (s *sampler) walkCollection(prefix string, values []any) {
 		s.emit(prefix + "{null}")
 	}
 
-	for _, shape := range shapes {
-		if len(shape.instances) == 1 {
-			obj := shape.instances[0]
-			isMap, _ := looksLikeMap(obj)
-			if isMap {
-				s.walkMap(prefix, obj)
-				continue
+	// If multiple shapes share a common core (≥ 2 keys in > 50% of objects),
+	// treat them as one struct with optional fields.
+	if len(shapes) > 1 {
+		var allObjs []map[string]any
+		for _, s := range shapes {
+			allObjs = append(allObjs, s.instances...)
+		}
+		if shouldMergeObjects(allObjs) {
+			s.walkStruct(prefix, allObjs)
+		} else {
+			for _, shape := range shapes {
+				if len(shape.instances) == 1 {
+					obj := shape.instances[0]
+					isMap, _ := looksLikeMap(obj)
+					if isMap {
+						s.walkMap(prefix, obj)
+						continue
+					}
+				}
+				s.walkStruct(prefix, shape.instances)
 			}
 		}
-		s.walkStruct(prefix, shape.instances)
+	} else {
+		for _, shape := range shapes {
+			if len(shape.instances) == 1 {
+				obj := shape.instances[0]
+				isMap, _ := looksLikeMap(obj)
+				if isMap {
+					s.walkMap(prefix, obj)
+					continue
+				}
+			}
+			s.walkStruct(prefix, shape.instances)
+		}
 	}
 
 	// Emit one sample per primitive type (not per value).

tools/jsontypes/testdata/sample.paths

@@ -1,14 +1,18 @@
-[string]{Person}
-[string].active{bool}
-[string].age{int}
-[string].friends[]{Friend}
-[string].friends[].identification{Identification?}
-[string].friends[].identification.id{string?}
-[string].friends[].identification.name{string}
-[string].friends[].identification.number{string?}
-[string].friends[].identification.restrictions{null}
-[string].friends[].identification.restrictions[]{string}
-[string].friends[].identification.type{string}
-[string].friends[].name{string}
-[string].name{string}
-[string].score{float?}
+{Root}
+.abc123{Abc}
+.abc123.active{bool}
+.abc123.age{int}
+.abc123.friends[]{FriendsItem}
+.abc123.friends[].identification{Identification0?}
+.abc123.friends[].identification.name{string}
+.abc123.friends[].identification.number{string}
+.abc123.friends[].identification.type{string}
+.abc123.friends[].name{string}
+.abc123.name{string}
+.def456{Abc}
+.ghi789{Ghi}
+.ghi789.active{bool}
+.ghi789.age{int}
+.ghi789.friends[]{FriendsItem}
+.ghi789.name{string}
+.ghi789.score{float}