feat(jsontypes): replace base64 check with pronounceability heuristic

isBase64 matched common field names like "name" — too loose for map
detection. Replace with isPronounceable which checks vowel/consonant
rhythm: real field names have natural syllable structure (15-80% vowels,
no 5+ consonant runs), while tokens and IDs don't.

Also fix allLookLikeIDs to reject keys that don't match any ID pattern
instead of silently accepting them.
This commit is contained in:
AJ ONeal 2026-03-07 22:27:47 -07:00
parent b58c09350b
commit 767115586c
No known key found for this signature in database

View File

@ -1,7 +1,6 @@
package jsontypes
import (
"encoding/base64"
"fmt"
"strconv"
"strings"
@ -193,24 +192,28 @@ func allSameLength(keys []string) bool {
}
// allLookLikeIDs checks if keys look like identifiers/tokens rather than field
// names: no spaces, alphanumeric/base64/hex, and not common English field names.
// names. Uses hex detection, pronounceability, and field name checks.
func allLookLikeIDs(keys []string) bool {
for _, k := range keys {
if strings.ContainsAny(k, " \t\n") {
return false
}
// Hex or base64 strings of any length ≥ 4
if len(k) >= 4 && (isHex(k) || isAlphanumeric(k) || isBase64(k)) {
// Pure hex is always an ID.
if len(k) >= 4 && isHex(k) {
continue
}
// Pronounceable strings are field names, not IDs.
if isPronounceable(k) {
return false
}
// Non-pronounceable alphanumeric of sufficient length → likely ID.
if len(k) >= 4 && isAlphanumeric(k) {
continue
}
// Doesn't match any ID pattern.
return false
}
// Additional check: IDs typically don't look like field names.
// If ALL of them look like field names (e.g., camelCase), not IDs.
if allLookLikeFieldNames(keys) {
return false
}
return true
return len(keys) > 0
}
func isAlphanumeric(s string) bool {
@ -224,20 +227,126 @@ func isAlphanumeric(s string) bool {
func isBase64(s string) bool {
// Try standard and URL-safe base64
if _, err := base64.StdEncoding.DecodeString(s); err == nil {
return true
// isPronounceable checks whether a string has the vowel/consonant rhythm of
// natural language. Field names like "metadata", "created_at", "userName" are
// pronounceable; tokens like "a1b2c3d4", "dGVzdA==", "xK9mP4q" are not.
//
// The check splits on underscores and case boundaries (camelCase), then
// verifies each word-like segment has a reasonable vowel ratio (15-80%) and
// no long consonant runs (>4). Pure-digit segments are ignored.
func isPronounceable(s string) bool {
// Split on underscores, then on camelCase boundaries.
segments := splitWordSegments(s)
if len(segments) == 0 {
return false
}
if _, err := base64.URLEncoding.DecodeString(s); err == nil {
return true
pronounceable := 0
total := 0
for _, seg := range segments {
// Skip pure digits and very short segments.
if len(seg) < 2 || isAllDigits(seg) {
continue
}
total++
if segmentIsPronounceable(seg) {
pronounceable++
}
}
if _, err := base64.RawURLEncoding.DecodeString(s); err == nil {
if total == 0 {
return false
}
// Most segments should be pronounceable.
return pronounceable > total/2
}
func segmentIsPronounceable(s string) bool {
s = strings.ToLower(s)
vowels := 0
consonantRun := 0
maxConsonantRun := 0
letters := 0
for _, r := range s {
if !unicode.IsLetter(r) {
consonantRun = 0
continue
}
letters++
if isVowel(r) {
vowels++
consonantRun = 0
} else {
consonantRun++
if consonantRun > maxConsonantRun {
maxConsonantRun = consonantRun
}
}
}
if letters < 2 {
return false
}
ratio := float64(vowels) / float64(letters)
// English words typically have 30-50% vowels.
// Allow a wide band (15-80%) to cover abbreviations and acronyms.
if ratio < 0.15 || ratio > 0.80 {
return false
}
// No natural word has 5+ consonants in a row.
if maxConsonantRun > 4 {
return false
}
return true
}
func isVowel(r rune) bool {
switch r {
case 'a', 'e', 'i', 'o', 'u', 'y':
return true
}
return false
}
func isAllDigits(s string) bool {
for _, r := range s {
if !unicode.IsDigit(r) {
return false
}
}
return len(s) > 0
}
// splitWordSegments breaks a string into word-like segments by splitting on
// underscores, hyphens, and camelCase boundaries.
func splitWordSegments(s string) []string {
// First split on underscores and hyphens.
var parts []string
for _, part := range strings.FieldsFunc(s, func(r rune) bool {
return r == '_' || r == '-'
}) {
// Then split on camelCase boundaries.
parts = append(parts, splitCamelCase(part)...)
}
return parts
}
func splitCamelCase(s string) []string {
var segments []string
start := 0
runes := []rune(s)
for i := 1; i < len(runes); i++ {
if unicode.IsUpper(runes[i]) && (i > start) {
segments = append(segments, string(runes[start:i]))
start = i
}
}
segments = append(segments, string(runes[start:]))
return segments
}
func isHex(s string) bool {
for _, r := range s {
if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')) {