357 lines
11 KiB
Go
357 lines
11 KiB
Go
package classification
|
|
|
|
import (
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"unicode"
|
|
|
|
"finance-duck/internal/domain"
|
|
)
|
|
|
|
func normalize(text string) string {
|
|
return strings.Join(strings.Fields(strings.Map(func(r rune) rune {
|
|
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
|
return unicode.ToLower(r)
|
|
}
|
|
return ' '
|
|
}, text)), " ")
|
|
}
|
|
|
|
// Only whole normalized phrases match, so e.g. Shell does not match Seashell.
|
|
// Equal-length aliases shared by different merchants are ambiguous, not rules.
|
|
func aliasMatch(description string, merchants []domain.Merchant) *domain.Merchant {
|
|
text := " " + normalize(description) + " "
|
|
var best *domain.Merchant
|
|
score := 0
|
|
ambiguous := false
|
|
for i := range merchants {
|
|
m := &merchants[i]
|
|
names := append([]string{m.Name}, m.Aliases...)
|
|
for _, name := range names {
|
|
alias := normalize(name)
|
|
if alias == "" || !strings.Contains(text, " "+alias+" ") {
|
|
continue
|
|
}
|
|
if len(alias) > score {
|
|
best = m
|
|
score = len(alias)
|
|
ambiguous = false
|
|
} else if len(alias) == score && best != nil && best.ID != m.ID {
|
|
ambiguous = true
|
|
}
|
|
}
|
|
}
|
|
if ambiguous {
|
|
return nil
|
|
}
|
|
return best
|
|
}
|
|
|
|
// LearnAlias adds a chosen transaction counterparty only when the real matcher
|
|
// remains unambiguous after the write-back.
|
|
func LearnAlias(d *domain.Dataset, facts domain.Facts, merchantID string) bool {
|
|
alias := strings.Join(strings.Fields(facts.Counterparty), " ")
|
|
if alias == "" || normalize(alias) == "" || merchantID == "" {
|
|
return false
|
|
}
|
|
index := slices.IndexFunc(d.Merchants, func(m domain.Merchant) bool { return m.ID == merchantID })
|
|
if index < 0 || len(d.Merchants[index].Aliases) >= 32 {
|
|
return false
|
|
}
|
|
if matched := aliasMatch(alias, d.Merchants); matched != nil && matched.ID == merchantID {
|
|
return false
|
|
}
|
|
trial := slices.Clone(d.Merchants)
|
|
trial[index].Aliases = append(slices.Clone(trial[index].Aliases), alias)
|
|
if matched := aliasMatch(alias, trial); matched == nil || matched.ID != merchantID {
|
|
return false
|
|
}
|
|
d.Merchants[index].Aliases = trial[index].Aliases
|
|
return true
|
|
}
|
|
|
|
func duplicateMerchant(name string, merchants []domain.Merchant) *domain.Merchant {
|
|
key := normalize(name)
|
|
var best *domain.Merchant
|
|
for i := range merchants {
|
|
m := &merchants[i]
|
|
match := normalize(m.Name) == key
|
|
for _, alias := range m.Aliases {
|
|
match = match || normalize(alias) == key
|
|
}
|
|
if match && (best == nil || m.ID < best.ID) {
|
|
best = m
|
|
}
|
|
}
|
|
if best != nil {
|
|
return best
|
|
}
|
|
for i := range merchants {
|
|
m := &merchants[i]
|
|
match := nearMerchant(key, normalize(m.Name))
|
|
for _, alias := range m.Aliases {
|
|
match = match || nearMerchant(key, normalize(alias))
|
|
}
|
|
if !match {
|
|
continue
|
|
}
|
|
if best != nil && best.ID != m.ID {
|
|
return nil
|
|
}
|
|
best = m
|
|
}
|
|
return best
|
|
}
|
|
|
|
func nearMerchant(a, b string) bool {
|
|
if a == b {
|
|
return true
|
|
}
|
|
left, right := []rune(a), []rune(b)
|
|
if len(left) < 8 || len(right) < 8 || len(strings.Fields(a)) != len(strings.Fields(b)) {
|
|
return false
|
|
}
|
|
if len(left)*100 < len(right)*85 || len(right)*100 < len(left)*85 {
|
|
return false
|
|
}
|
|
trigrams := func(runes []rune) map[string]bool {
|
|
out := map[string]bool{}
|
|
for i := range len(runes) - 2 {
|
|
out[string(runes[i:i+3])] = true
|
|
}
|
|
return out
|
|
}
|
|
x, y := trigrams(left), trigrams(right)
|
|
shared := 0
|
|
for gram := range x {
|
|
if y[gram] {
|
|
shared++
|
|
}
|
|
}
|
|
return shared*200 >= (len(x)+len(y))*92
|
|
}
|
|
|
|
type categoryPrompt struct {
|
|
ID string `json:"id"`
|
|
Path string `json:"path"`
|
|
Kind string `json:"kind"`
|
|
Hint string `json:"hint,omitempty"`
|
|
}
|
|
type tagPrompt struct {
|
|
ID string `json:"id"`
|
|
Name string `json:"name"`
|
|
Hint string `json:"hint,omitempty"`
|
|
}
|
|
type merchantPrompt struct {
|
|
ID string `json:"id"`
|
|
Name string `json:"name"`
|
|
Aliases []string `json:"aliases"`
|
|
UsualCategory string `json:"usual_category,omitempty"`
|
|
}
|
|
|
|
// candidate is the historical merchant prompt shape used by older callers.
|
|
type candidate = merchantPrompt
|
|
type promptHistory struct {
|
|
Date string `json:"date"`
|
|
Amount string `json:"amount"`
|
|
Description string `json:"description"`
|
|
Counterparty string `json:"counterparty"`
|
|
CategoryID string `json:"category_id"`
|
|
MerchantID string `json:"merchant_id,omitempty"`
|
|
TagIDs []string `json:"tag_ids"`
|
|
}
|
|
type candidateSet struct {
|
|
categories []categoryPrompt
|
|
tags []tagPrompt
|
|
merchants []merchantPrompt
|
|
categoryIDs map[string]string
|
|
tagIDs map[string]string
|
|
merchantIDs map[string]string
|
|
}
|
|
|
|
func similarity(description, name string) int {
|
|
a, b := normalize(description), normalize(name)
|
|
if b == "" {
|
|
return 0
|
|
}
|
|
if strings.Contains(" "+a+" ", " "+b+" ") {
|
|
return 10000 + len(b)
|
|
}
|
|
score := 0
|
|
for _, word := range strings.Fields(b) {
|
|
for _, input := range strings.Fields(a) {
|
|
if input == word {
|
|
score += len(word)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
return score
|
|
}
|
|
|
|
// retrieve emits every registry entry with its real id. The legacy cleaner
|
|
// arguments remain in the signature because CSV/classification fixtures use
|
|
// this helper directly; ranking and bounding are intentionally gone.
|
|
func retrieve(_ string, kind string, data domain.Dataset, clean, merchantClean func(string) string) candidateSet {
|
|
parents := map[string]bool{}
|
|
for _, cat := range data.Categories {
|
|
parents[cat.ParentID] = true
|
|
}
|
|
set := candidateSet{
|
|
categoryIDs: map[string]string{},
|
|
tagIDs: map[string]string{},
|
|
merchantIDs: map[string]string{},
|
|
}
|
|
for _, cat := range data.Categories {
|
|
if cat.Kind != kind || parents[cat.ID] {
|
|
continue
|
|
}
|
|
path := domain.CategoryPath(data, cat.ID)
|
|
if clean != nil {
|
|
path = clean(path)
|
|
}
|
|
set.categories = append(set.categories, categoryPrompt{ID: cat.ID, Path: path, Kind: cat.Kind, Hint: cleanText(clean, cat.Hint)})
|
|
set.categoryIDs[cat.ID] = cat.ID
|
|
}
|
|
sort.Slice(set.categories, func(i, j int) bool {
|
|
return set.categories[i].Path < set.categories[j].Path || set.categories[i].Path == set.categories[j].Path && set.categories[i].ID < set.categories[j].ID
|
|
})
|
|
for _, tag := range data.Tags {
|
|
name := cleanText(clean, tag.Name)
|
|
set.tags = append(set.tags, tagPrompt{ID: tag.ID, Name: name, Hint: cleanText(clean, tag.Hint)})
|
|
set.tagIDs[tag.ID] = tag.ID
|
|
}
|
|
sort.Slice(set.tags, func(i, j int) bool {
|
|
return set.tags[i].Name < set.tags[j].Name || set.tags[i].Name == set.tags[j].Name && set.tags[i].ID < set.tags[j].ID
|
|
})
|
|
usual := map[string]string{}
|
|
counts := map[string]map[string]int{}
|
|
for _, tx := range data.Transactions {
|
|
merchantID, categoryID := tx.Enrichment.MerchantID, tx.Enrichment.CategoryID
|
|
if merchantID == "" || categoryID == "" {
|
|
continue
|
|
}
|
|
if counts[merchantID] == nil {
|
|
counts[merchantID] = map[string]int{}
|
|
}
|
|
counts[merchantID][categoryID]++
|
|
}
|
|
for merchantID, values := range counts {
|
|
for categoryID, count := range values {
|
|
current := usual[merchantID]
|
|
if current == "" || count > values[current] || count == values[current] && categoryID < current {
|
|
usual[merchantID] = categoryID
|
|
}
|
|
}
|
|
}
|
|
for _, merchant := range data.Merchants {
|
|
name := cleanText(merchantClean, merchant.Name)
|
|
aliases := make([]string, 0, len(merchant.Aliases))
|
|
for _, alias := range merchant.Aliases {
|
|
if value := cleanText(merchantClean, alias); value != "" {
|
|
aliases = append(aliases, value)
|
|
}
|
|
}
|
|
usualCategory := merchant.DefaultCategoryID
|
|
if categoryID := usual[merchant.ID]; categoryID != "" {
|
|
usualCategory = categoryID
|
|
}
|
|
set.merchants = append(set.merchants, merchantPrompt{
|
|
ID: merchant.ID, Name: name, Aliases: aliases,
|
|
UsualCategory: usualCategory,
|
|
})
|
|
set.merchantIDs[merchant.ID] = merchant.ID
|
|
}
|
|
sort.Slice(set.merchants, func(i, j int) bool {
|
|
return set.merchants[i].Name < set.merchants[j].Name || set.merchants[i].Name == set.merchants[j].Name && set.merchants[i].ID < set.merchants[j].ID
|
|
})
|
|
return set
|
|
}
|
|
|
|
func cleanText(clean func(string) string, value string) string {
|
|
if clean == nil {
|
|
return normalize(value)
|
|
}
|
|
return clean(value)
|
|
}
|
|
|
|
func (c candidateSet) schema() map[string]any {
|
|
merchantEnums := []any{nil}
|
|
for _, m := range c.merchants {
|
|
merchantEnums = append(merchantEnums, m.ID)
|
|
}
|
|
tagIDs := make([]any, 0, len(c.tags))
|
|
for _, tag := range c.tags {
|
|
tagIDs = append(tagIDs, tag.ID)
|
|
}
|
|
tagItems := map[string]any{"type": "string"}
|
|
if len(tagIDs) > 0 {
|
|
tagItems["enum"] = tagIDs
|
|
}
|
|
return map[string]any{
|
|
"type": "object", "additionalProperties": false,
|
|
"required": []string{"merchant_id", "new_merchant", "category_id", "tag_ids", "confidence"},
|
|
"properties": map[string]any{
|
|
"merchant_id": map[string]any{"type": []string{"string", "null"}, "enum": merchantEnums},
|
|
"new_merchant": map[string]any{"type": []string{"string", "null"}, "maxLength": 100},
|
|
"category_id": map[string]any{"type": "string", "enum": candidateIDs(c.categories)},
|
|
"tag_ids": map[string]any{"type": "array", "uniqueItems": true, "maxItems": len(tagIDs), "items": tagItems},
|
|
"confidence": map[string]any{"type": "string", "enum": []string{"high", "medium", "low"}},
|
|
},
|
|
}
|
|
}
|
|
|
|
func candidateIDs(values []categoryPrompt) []string {
|
|
ids := make([]string, 0, len(values))
|
|
for _, value := range values {
|
|
ids = append(ids, value.ID)
|
|
}
|
|
return ids
|
|
}
|
|
|
|
func answerSchema(d domain.Dataset, kind string) map[string]any {
|
|
return retrieve("", kind, d, nil, nil).schema()
|
|
}
|
|
|
|
func history(f domain.Facts, d domain.Dataset, clean func(string) string, limit int) []promptHistory {
|
|
type row struct {
|
|
tx domain.Transaction
|
|
score int
|
|
}
|
|
rows := []row{}
|
|
for _, tx := range d.Transactions {
|
|
e := tx.Enrichment
|
|
if tx.Facts.ID == f.ID || e.Kind == "transfer" || e.CategoryID == "" || e.CategoryID == domain.ExpenseFallback || e.CategoryID == domain.IncomeFallback {
|
|
continue
|
|
}
|
|
rows = append(rows, row{tx: tx, score: similarity(f.RawDescription+" "+f.Counterparty, tx.Facts.RawDescription+" "+tx.Facts.Counterparty)})
|
|
}
|
|
sort.Slice(rows, func(i, j int) bool {
|
|
if rows[i].score != rows[j].score {
|
|
return rows[i].score > rows[j].score
|
|
}
|
|
if rows[i].tx.Facts.BookingDate != rows[j].tx.Facts.BookingDate {
|
|
return rows[i].tx.Facts.BookingDate > rows[j].tx.Facts.BookingDate
|
|
}
|
|
return rows[i].tx.Facts.ID < rows[j].tx.Facts.ID
|
|
})
|
|
if limit > 0 && len(rows) > limit {
|
|
rows = rows[:limit]
|
|
}
|
|
out := make([]promptHistory, 0, len(rows))
|
|
for _, row := range rows {
|
|
tags := row.tx.Enrichment.TagIDs
|
|
if tags == nil {
|
|
tags = []string{}
|
|
}
|
|
out = append(out, promptHistory{
|
|
Date: row.tx.Facts.BookingDate, Amount: string(row.tx.Facts.Amount),
|
|
Description: clean(row.tx.Facts.RawDescription), Counterparty: clean(row.tx.Facts.Counterparty),
|
|
CategoryID: row.tx.Enrichment.CategoryID, MerchantID: row.tx.Enrichment.MerchantID,
|
|
TagIDs: append([]string{}, tags...),
|
|
})
|
|
}
|
|
return out
|
|
}
|