package classification import ( "slices" "sort" "strings" "unicode" "finance-duck/internal/domain" ) func normalize(text string) string { return strings.Join(strings.Fields(strings.Map(func(r rune) rune { if unicode.IsLetter(r) || unicode.IsDigit(r) { return unicode.ToLower(r) } return ' ' }, text)), " ") } // Only whole normalized phrases match, so e.g. Shell does not match Seashell. // Equal-length aliases shared by different merchants are ambiguous, not rules. func aliasMatch(description string, merchants []domain.Merchant) *domain.Merchant { text := " " + normalize(description) + " " var best *domain.Merchant score := 0 ambiguous := false for i := range merchants { m := &merchants[i] names := append([]string{m.Name}, m.Aliases...) for _, name := range names { alias := normalize(name) if alias == "" || !strings.Contains(text, " "+alias+" ") { continue } if len(alias) > score { best = m score = len(alias) ambiguous = false } else if len(alias) == score && best != nil && best.ID != m.ID { ambiguous = true } } } if ambiguous { return nil } return best } // LearnAlias adds a chosen transaction counterparty only when the real matcher // remains unambiguous after the write-back. func LearnAlias(d *domain.Dataset, facts domain.Facts, merchantID string) bool { alias := strings.Join(strings.Fields(facts.Counterparty), " ") if alias == "" || normalize(alias) == "" || merchantID == "" { return false } index := slices.IndexFunc(d.Merchants, func(m domain.Merchant) bool { return m.ID == merchantID }) if index < 0 || len(d.Merchants[index].Aliases) >= 32 { return false } if matched := aliasMatch(alias, d.Merchants); matched != nil && matched.ID == merchantID { return false } trial := slices.Clone(d.Merchants) trial[index].Aliases = append(slices.Clone(trial[index].Aliases), alias) if matched := aliasMatch(alias, trial); matched == nil || matched.ID != merchantID { return false } d.Merchants[index].Aliases = trial[index].Aliases return true } func duplicateMerchant(name string, merchants []domain.Merchant) *domain.Merchant { key := normalize(name) var best *domain.Merchant for i := range merchants { m := &merchants[i] match := normalize(m.Name) == key for _, alias := range m.Aliases { match = match || normalize(alias) == key } if match && (best == nil || m.ID < best.ID) { best = m } } if best != nil { return best } for i := range merchants { m := &merchants[i] match := nearMerchant(key, normalize(m.Name)) for _, alias := range m.Aliases { match = match || nearMerchant(key, normalize(alias)) } if !match { continue } if best != nil && best.ID != m.ID { return nil } best = m } return best } func nearMerchant(a, b string) bool { if a == b { return true } left, right := []rune(a), []rune(b) if len(left) < 8 || len(right) < 8 || len(strings.Fields(a)) != len(strings.Fields(b)) { return false } if len(left)*100 < len(right)*85 || len(right)*100 < len(left)*85 { return false } trigrams := func(runes []rune) map[string]bool { out := map[string]bool{} for i := range len(runes) - 2 { out[string(runes[i:i+3])] = true } return out } x, y := trigrams(left), trigrams(right) shared := 0 for gram := range x { if y[gram] { shared++ } } return shared*200 >= (len(x)+len(y))*92 } type categoryPrompt struct { ID string `json:"id"` Path string `json:"path"` Kind string `json:"kind"` Hint string `json:"hint,omitempty"` } type tagPrompt struct { ID string `json:"id"` Name string `json:"name"` Hint string `json:"hint,omitempty"` } type merchantPrompt struct { ID string `json:"id"` Name string `json:"name"` Aliases []string `json:"aliases"` UsualCategory string `json:"usual_category,omitempty"` } // candidate is the historical merchant prompt shape used by older callers. type candidate = merchantPrompt type promptHistory struct { Date string `json:"date"` Amount string `json:"amount"` Description string `json:"description"` Counterparty string `json:"counterparty"` CategoryID string `json:"category_id"` MerchantID string `json:"merchant_id,omitempty"` TagIDs []string `json:"tag_ids"` // Source separates the user's own decisions ("user") from earlier model // output ("ai"): without the distinction, precedent feeds the model its // own past answers as evidence and a manual correction never wins. Source string `json:"source"` } type candidateSet struct { categories []categoryPrompt tags []tagPrompt merchants []merchantPrompt categoryIDs map[string]string tagIDs map[string]string merchantIDs map[string]string } func similarity(description, name string) int { a, b := normalize(description), normalize(name) if b == "" { return 0 } if strings.Contains(" "+a+" ", " "+b+" ") { return 10000 + len(b) } score := 0 for _, word := range strings.Fields(b) { for _, input := range strings.Fields(a) { if input == word { score += len(word) break } } } return score } // retrieve emits every registry entry with its real id. The legacy cleaner // arguments remain in the signature because CSV/classification fixtures use // this helper directly; ranking and bounding are intentionally gone. func retrieve(_ string, kind string, data domain.Dataset, clean, merchantClean func(string) string) candidateSet { parents := map[string]bool{} for _, cat := range data.Categories { parents[cat.ParentID] = true } set := candidateSet{ categoryIDs: map[string]string{}, tagIDs: map[string]string{}, merchantIDs: map[string]string{}, } for _, cat := range data.Categories { if cat.Kind != kind || parents[cat.ID] { continue } path := domain.CategoryPath(data, cat.ID) if clean != nil { path = clean(path) } set.categories = append(set.categories, categoryPrompt{ID: cat.ID, Path: path, Kind: cat.Kind, Hint: cleanText(clean, cat.Hint)}) set.categoryIDs[cat.ID] = cat.ID } sort.Slice(set.categories, func(i, j int) bool { return set.categories[i].Path < set.categories[j].Path || set.categories[i].Path == set.categories[j].Path && set.categories[i].ID < set.categories[j].ID }) for _, tag := range data.Tags { name := cleanText(clean, tag.Name) set.tags = append(set.tags, tagPrompt{ID: tag.ID, Name: name, Hint: cleanText(clean, tag.Hint)}) set.tagIDs[tag.ID] = tag.ID } sort.Slice(set.tags, func(i, j int) bool { return set.tags[i].Name < set.tags[j].Name || set.tags[i].Name == set.tags[j].Name && set.tags[i].ID < set.tags[j].ID }) usual := map[string]string{} counts := map[string]map[string]int{} for _, tx := range data.Transactions { merchantID, categoryID := tx.Enrichment.MerchantID, tx.Enrichment.CategoryID if merchantID == "" || categoryID == "" { continue } if counts[merchantID] == nil { counts[merchantID] = map[string]int{} } counts[merchantID][categoryID]++ } for merchantID, values := range counts { for categoryID, count := range values { current := usual[merchantID] if current == "" || count > values[current] || count == values[current] && categoryID < current { usual[merchantID] = categoryID } } } for _, merchant := range data.Merchants { name := cleanText(merchantClean, merchant.Name) aliases := make([]string, 0, len(merchant.Aliases)) for _, alias := range merchant.Aliases { if value := cleanText(merchantClean, alias); value != "" { aliases = append(aliases, value) } } usualCategory := merchant.DefaultCategoryID if categoryID := usual[merchant.ID]; categoryID != "" { usualCategory = categoryID } set.merchants = append(set.merchants, merchantPrompt{ ID: merchant.ID, Name: name, Aliases: aliases, UsualCategory: usualCategory, }) set.merchantIDs[merchant.ID] = merchant.ID } sort.Slice(set.merchants, func(i, j int) bool { return set.merchants[i].Name < set.merchants[j].Name || set.merchants[i].Name == set.merchants[j].Name && set.merchants[i].ID < set.merchants[j].ID }) return set } func cleanText(clean func(string) string, value string) string { if clean == nil { return normalize(value) } return clean(value) } func (c candidateSet) schema() map[string]any { merchantEnums := []any{nil} for _, m := range c.merchants { merchantEnums = append(merchantEnums, m.ID) } tagIDs := make([]any, 0, len(c.tags)) for _, tag := range c.tags { tagIDs = append(tagIDs, tag.ID) } tagItems := map[string]any{"type": "string"} if len(tagIDs) > 0 { tagItems["enum"] = tagIDs } return map[string]any{ "type": "object", "additionalProperties": false, "required": []string{"merchant_id", "new_merchant", "category_id", "tag_ids", "confidence"}, "properties": map[string]any{ "merchant_id": map[string]any{"type": []string{"string", "null"}, "enum": merchantEnums}, "new_merchant": map[string]any{"type": []string{"string", "null"}, "maxLength": 100}, "category_id": map[string]any{"type": "string", "enum": candidateIDs(c.categories)}, "tag_ids": map[string]any{"type": "array", "items": tagItems}, "confidence": map[string]any{"type": "string", "enum": []string{"high", "medium", "low"}}, }, } } func candidateIDs(values []categoryPrompt) []string { ids := make([]string, 0, len(values)) for _, value := range values { ids = append(ids, value.ID) } return ids } func answerSchema(d domain.Dataset, kind string) map[string]any { return retrieve("", kind, d, nil, nil).schema() } // history selects precedent for the prompt: the nearest rows by word overlap, // filled out with the most recent. The user's own decisions — manual edits // and locally applied merchant rules — outrank rows the model classified // itself, so one correction beats any number of uncorrected AI answers for // the same payee. func history(f domain.Facts, d domain.Dataset, clean func(string) string, limit int) []promptHistory { type row struct { tx domain.Transaction score int user bool } rows := []row{} for _, tx := range d.Transactions { e := tx.Enrichment if tx.Facts.ID == f.ID || e.Kind == "transfer" || e.CategoryID == "" || e.CategoryID == domain.ExpenseFallback || e.CategoryID == domain.IncomeFallback { continue } source := tx.Enrichment.Classification.Source rows = append(rows, row{ tx: tx, score: similarity(f.RawDescription+" "+f.Counterparty, tx.Facts.RawDescription+" "+tx.Facts.Counterparty), user: source == "manual" || source == "rule", }) } sort.Slice(rows, func(i, j int) bool { if rows[i].score != rows[j].score { return rows[i].score > rows[j].score } if rows[i].user != rows[j].user { return rows[i].user } if rows[i].tx.Facts.BookingDate != rows[j].tx.Facts.BookingDate { return rows[i].tx.Facts.BookingDate > rows[j].tx.Facts.BookingDate } return rows[i].tx.Facts.ID < rows[j].tx.Facts.ID }) if limit > 0 && len(rows) > limit { // Never let recent AI output crowd every correction out of a full // window: user rows keep their slots ahead of equally similar AI rows. kept := make([]row, 0, limit) users := 0 for _, r := range rows { if r.user { users++ } } userBudget := min(users, limit/2) aiBudget := limit - userBudget for _, r := range rows { if r.user && userBudget > 0 { kept = append(kept, r) userBudget-- } else if !r.user && aiBudget > 0 { kept = append(kept, r) aiBudget-- } else if r.user && aiBudget > 0 { kept = append(kept, r) aiBudget-- } } rows = kept } out := make([]promptHistory, 0, len(rows)) for _, row := range rows { tags := row.tx.Enrichment.TagIDs if tags == nil { tags = []string{} } source := "ai" if row.user { source = "user" } out = append(out, promptHistory{ Date: row.tx.Facts.BookingDate, Amount: string(row.tx.Facts.Amount), Description: clean(row.tx.Facts.RawDescription), Counterparty: clean(row.tx.Facts.Counterparty), CategoryID: row.tx.Enrichment.CategoryID, MerchantID: row.tx.Enrichment.MerchantID, TagIDs: append([]string{}, tags...), Source: source, }) } return out }