package classification import ( "fmt" "sort" "strings" "unicode" "finance-duck/internal/domain" ) func normalize(text string) string { return strings.Join(strings.Fields(strings.Map(func(r rune) rune { if unicode.IsLetter(r) || unicode.IsDigit(r) { return unicode.ToLower(r) } return ' ' }, text)), " ") } // Only whole normalized phrases match, so e.g. Shell does not match Seashell. // Equal-length aliases shared by different merchants are ambiguous, not rules. func aliasMatch(description string, merchants []domain.Merchant) *domain.Merchant { text := " " + normalize(description) + " " var best *domain.Merchant score := 0 ambiguous := false for i := range merchants { m := &merchants[i] names := append([]string{m.Name}, m.Aliases...) for _, name := range names { alias := normalize(name) if alias == "" || !strings.Contains(text, " "+alias+" ") { continue } if len(alias) > score { best = m score = len(alias) ambiguous = false } else if len(alias) == score && best != nil && best.ID != m.ID { ambiguous = true } } } if ambiguous { return nil } return best } func duplicateMerchant(name string, merchants []domain.Merchant) *domain.Merchant { key := normalize(name) var best *domain.Merchant for i := range merchants { m := &merchants[i] match := normalize(m.Name) == key for _, alias := range m.Aliases { match = match || normalize(alias) == key } if match && (best == nil || m.ID < best.ID) { best = m } } if best != nil { return best } // A near spelling can reuse an existing merchant only when exactly one // registry entry is similar. Token counts protect e.g. REWE vs REWE To Go. for i := range merchants { m := &merchants[i] match := nearMerchant(key, normalize(m.Name)) for _, alias := range m.Aliases { match = match || nearMerchant(key, normalize(alias)) } if !match { continue } if best != nil && best.ID != m.ID { return nil } best = m } return best } func nearMerchant(a, b string) bool { if a == b { return true } left, right := []rune(a), []rune(b) if len(left) < 8 || len(right) < 8 || len(strings.Fields(a)) != len(strings.Fields(b)) { return false } if len(left)*100 < len(right)*85 || len(right)*100 < len(left)*85 { return false } trigrams := func(runes []rune) map[string]bool { out := map[string]bool{} for i := range len(runes) - 2 { out[string(runes[i:i+3])] = true } return out } x, y := trigrams(left), trigrams(right) shared := 0 for gram := range x { if y[gram] { shared++ } } return shared*200 >= (len(x)+len(y))*92 } type candidate struct { ID string `json:"id"` Name string `json:"name"` } type candidateSet struct { categories, tags, merchants []candidate categoryIDs, tagIDs, merchantIDs map[string]string } type ranked struct { id, name string score int } func similarity(description, name string) int { a, b := normalize(description), normalize(name) if b == "" { return 0 } if strings.Contains(" "+a+" ", " "+b+" ") { return 10000 + len(b) } words := strings.Fields(a) score := 0 for _, word := range strings.Fields(b) { for _, input := range words { if input == word { score += len(word) break } } } return score } func bounded(rows []ranked, prefix string, limit int, clean func(string) string) ([]candidate, map[string]string) { sort.Slice(rows, func(i, j int) bool { if rows[i].score != rows[j].score { return rows[i].score > rows[j].score } return rows[i].id < rows[j].id }) if limit > 0 && len(rows) > limit { rows = rows[:limit] } out := make([]candidate, 0, len(rows)) ids := make(map[string]string, len(rows)) for i, row := range rows { id := fmt.Sprintf("%s%d", prefix, i+1) name := clean(row.name) if name == "" { name = "unnamed" } out = append(out, candidate{ID: id, Name: name}) ids[id] = row.id } return out, ids } func retrieve(description, kind string, data domain.Dataset, clean, merchantClean func(string) string) candidateSet { var categories, tags, merchants []ranked fallback := domain.ExpenseFallback if kind == "income" { fallback = domain.IncomeFallback } parents := map[string]bool{} for _, cat := range data.Categories { parents[cat.ParentID] = true } for _, cat := range data.Categories { if cat.Kind != kind || parents[cat.ID] { continue } name := domain.CategoryPath(data, cat.ID) score := similarity(description, name) if cat.ID == fallback { score = int(^uint(0) >> 1) } categories = append(categories, ranked{id: cat.ID, name: name, score: score}) } for _, tag := range data.Tags { tags = append(tags, ranked{id: tag.ID, name: tag.Name, score: similarity(description, tag.Name)}) } for _, m := range data.Merchants { score := similarity(description, m.Name) for _, alias := range m.Aliases { if s := similarity(description, alias); s > score { score = s } } merchants = append(merchants, ranked{id: m.ID, name: m.Name, score: score}) } var set candidateSet set.categories, set.categoryIDs = bounded(categories, "c", 0, clean) set.tags, set.tagIDs = bounded(tags, "t", 0, clean) set.merchants, set.merchantIDs = bounded(merchants, "m", 20, merchantClean) return set } func candidateEnums(candidates []candidate) []string { ids := make([]string, 0, len(candidates)) for _, c := range candidates { ids = append(ids, c.ID) } return ids } func (c candidateSet) schema() map[string]any { merchantEnums := []any{nil} for _, m := range c.merchants { merchantEnums = append(merchantEnums, m.ID) } tagItems := map[string]any{"type": "string"} if len(c.tags) > 0 { tagItems["enum"] = candidateEnums(c.tags) } tags := map[string]any{"type": "array", "items": tagItems, "maxItems": len(c.tags), "uniqueItems": true} return map[string]any{ "type": "object", "additionalProperties": false, "required": []string{"merchant_id", "new_merchant", "category_id", "tag_ids"}, "properties": map[string]any{ "merchant_id": map[string]any{"type": []string{"string", "null"}, "enum": merchantEnums, "description": "Existing merchant candidate ID, or null."}, "new_merchant": map[string]any{"type": []string{"string", "null"}, "maxLength": 100, "description": "Public business name only when no existing merchant matches, otherwise null."}, "category_id": map[string]any{"type": "string", "enum": candidateEnums(c.categories)}, "tag_ids": tags, }, } }