mirror of
https://github.com/therootcompany/golib.git
synced 2026-11-04 06:55:00 +00:00
feat(jsontypes): replace base64 check with pronounceability heuristic
isBase64 matched common field names like "name" — too loose for map detection. Replace with isPronounceable which checks vowel/consonant rhythm: real field names have natural syllable structure (15-80% vowels, no 5+ consonant runs), while tokens and IDs don't. Also fix allLookLikeIDs to reject keys that don't match any ID pattern instead of silently accepting them.
This commit is contained in:
parent
b58c09350b
commit
767115586c
@ -1,7 +1,6 @@
|
|||||||
package jsontypes
|
package jsontypes
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/base64"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
@ -193,24 +192,28 @@ func allSameLength(keys []string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// allLookLikeIDs checks if keys look like identifiers/tokens rather than field
|
// allLookLikeIDs checks if keys look like identifiers/tokens rather than field
|
||||||
// names: no spaces, alphanumeric/base64/hex, and not common English field names.
|
// names. Uses hex detection, pronounceability, and field name checks.
|
||||||
func allLookLikeIDs(keys []string) bool {
|
func allLookLikeIDs(keys []string) bool {
|
||||||
for _, k := range keys {
|
for _, k := range keys {
|
||||||
if strings.ContainsAny(k, " \t\n") {
|
if strings.ContainsAny(k, " \t\n") {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
// Hex or base64 strings of any length ≥ 4
|
// Pure hex is always an ID.
|
||||||
if len(k) >= 4 && (isHex(k) || isAlphanumeric(k) || isBase64(k)) {
|
if len(k) >= 4 && isHex(k) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
// Pronounceable strings are field names, not IDs.
|
||||||
|
if isPronounceable(k) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// Non-pronounceable alphanumeric of sufficient length → likely ID.
|
||||||
|
if len(k) >= 4 && isAlphanumeric(k) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Doesn't match any ID pattern.
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
// Additional check: IDs typically don't look like field names.
|
return len(keys) > 0
|
||||||
// If ALL of them look like field names (e.g., camelCase), not IDs.
|
|
||||||
if allLookLikeFieldNames(keys) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func isAlphanumeric(s string) bool {
|
func isAlphanumeric(s string) bool {
|
||||||
@ -224,20 +227,126 @@ func isAlphanumeric(s string) bool {
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
func isBase64(s string) bool {
|
// isPronounceable checks whether a string has the vowel/consonant rhythm of
|
||||||
// Try standard and URL-safe base64
|
// natural language. Field names like "metadata", "created_at", "userName" are
|
||||||
if _, err := base64.StdEncoding.DecodeString(s); err == nil {
|
// pronounceable; tokens like "a1b2c3d4", "dGVzdA==", "xK9mP4q" are not.
|
||||||
return true
|
//
|
||||||
|
// The check splits on underscores and case boundaries (camelCase), then
|
||||||
|
// verifies each word-like segment has a reasonable vowel ratio (15-80%) and
|
||||||
|
// no long consonant runs (>4). Pure-digit segments are ignored.
|
||||||
|
func isPronounceable(s string) bool {
|
||||||
|
// Split on underscores, then on camelCase boundaries.
|
||||||
|
segments := splitWordSegments(s)
|
||||||
|
if len(segments) == 0 {
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
if _, err := base64.URLEncoding.DecodeString(s); err == nil {
|
|
||||||
return true
|
pronounceable := 0
|
||||||
|
total := 0
|
||||||
|
for _, seg := range segments {
|
||||||
|
// Skip pure digits and very short segments.
|
||||||
|
if len(seg) < 2 || isAllDigits(seg) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
total++
|
||||||
|
if segmentIsPronounceable(seg) {
|
||||||
|
pronounceable++
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if _, err := base64.RawURLEncoding.DecodeString(s); err == nil {
|
|
||||||
|
if total == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// Most segments should be pronounceable.
|
||||||
|
return pronounceable > total/2
|
||||||
|
}
|
||||||
|
|
||||||
|
func segmentIsPronounceable(s string) bool {
|
||||||
|
s = strings.ToLower(s)
|
||||||
|
vowels := 0
|
||||||
|
consonantRun := 0
|
||||||
|
maxConsonantRun := 0
|
||||||
|
letters := 0
|
||||||
|
|
||||||
|
for _, r := range s {
|
||||||
|
if !unicode.IsLetter(r) {
|
||||||
|
consonantRun = 0
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
letters++
|
||||||
|
if isVowel(r) {
|
||||||
|
vowels++
|
||||||
|
consonantRun = 0
|
||||||
|
} else {
|
||||||
|
consonantRun++
|
||||||
|
if consonantRun > maxConsonantRun {
|
||||||
|
maxConsonantRun = consonantRun
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if letters < 2 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
ratio := float64(vowels) / float64(letters)
|
||||||
|
// English words typically have 30-50% vowels.
|
||||||
|
// Allow a wide band (15-80%) to cover abbreviations and acronyms.
|
||||||
|
if ratio < 0.15 || ratio > 0.80 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// No natural word has 5+ consonants in a row.
|
||||||
|
if maxConsonantRun > 4 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func isVowel(r rune) bool {
|
||||||
|
switch r {
|
||||||
|
case 'a', 'e', 'i', 'o', 'u', 'y':
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isAllDigits(s string) bool {
|
||||||
|
for _, r := range s {
|
||||||
|
if !unicode.IsDigit(r) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(s) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// splitWordSegments breaks a string into word-like segments by splitting on
|
||||||
|
// underscores, hyphens, and camelCase boundaries.
|
||||||
|
func splitWordSegments(s string) []string {
|
||||||
|
// First split on underscores and hyphens.
|
||||||
|
var parts []string
|
||||||
|
for _, part := range strings.FieldsFunc(s, func(r rune) bool {
|
||||||
|
return r == '_' || r == '-'
|
||||||
|
}) {
|
||||||
|
// Then split on camelCase boundaries.
|
||||||
|
parts = append(parts, splitCamelCase(part)...)
|
||||||
|
}
|
||||||
|
return parts
|
||||||
|
}
|
||||||
|
|
||||||
|
func splitCamelCase(s string) []string {
|
||||||
|
var segments []string
|
||||||
|
start := 0
|
||||||
|
runes := []rune(s)
|
||||||
|
for i := 1; i < len(runes); i++ {
|
||||||
|
if unicode.IsUpper(runes[i]) && (i > start) {
|
||||||
|
segments = append(segments, string(runes[start:i]))
|
||||||
|
start = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
segments = append(segments, string(runes[start:]))
|
||||||
|
return segments
|
||||||
|
}
|
||||||
|
|
||||||
func isHex(s string) bool {
|
func isHex(s string) bool {
|
||||||
for _, r := range s {
|
for _, r := range s {
|
||||||
if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')) {
|
if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')) {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user