Implement classification redesign

2026-09-11 22:46:17 +02:00
parent cc43a2f9a7
commit 87f052a3ea
23 changed files with 1602 additions and 296 deletions
@@ -1,7 +1,7 @@
 package classification

 import (
-	"fmt"
+	"slices"
 	"sort"
 	"strings"
 	"unicode"
@@ -48,6 +48,29 @@ func aliasMatch(description string, merchants []domain.Merchant) *domain.Merchan
 	return best
 }

+// LearnAlias adds a chosen transaction counterparty only when the real matcher
+// remains unambiguous after the write-back.
+func LearnAlias(d *domain.Dataset, facts domain.Facts, merchantID string) bool {
+	alias := strings.Join(strings.Fields(facts.Counterparty), " ")
+	if alias == "" || normalize(alias) == "" || merchantID == "" {
+		return false
+	}
+	index := slices.IndexFunc(d.Merchants, func(m domain.Merchant) bool { return m.ID == merchantID })
+	if index < 0 || len(d.Merchants[index].Aliases) >= 32 {
+		return false
+	}
+	if matched := aliasMatch(alias, d.Merchants); matched != nil && matched.ID == merchantID {
+		return false
+	}
+	trial := slices.Clone(d.Merchants)
+	trial[index].Aliases = append(slices.Clone(trial[index].Aliases), alias)
+	if matched := aliasMatch(alias, trial); matched == nil || matched.ID != merchantID {
+		return false
+	}
+	d.Merchants[index].Aliases = trial[index].Aliases
+	return true
+}
+
 func duplicateMerchant(name string, merchants []domain.Merchant) *domain.Merchant {
 	key := normalize(name)
 	var best *domain.Merchant
@@ -64,8 +87,6 @@ func duplicateMerchant(name string, merchants []domain.Merchant) *domain.Merchan
 	if best != nil {
 		return best
 	}
-	// A near spelling can reuse an existing merchant only when exactly one
-	// registry entry is similar. Token counts protect e.g. REWE vs REWE To Go.
 	for i := range merchants {
 		m := &merchants[i]
 		match := nearMerchant(key, normalize(m.Name))
@@ -111,17 +132,42 @@ func nearMerchant(a, b string) bool {
 	return shared*200 >= (len(x)+len(y))*92
 }

-type candidate struct {
+type categoryPrompt struct {
+	ID   string `json:"id"`
+	Path string `json:"path"`
+	Kind string `json:"kind"`
+	Hint string `json:"hint,omitempty"`
+}
+type tagPrompt struct {
 	ID   string `json:"id"`
 	Name string `json:"name"`
+	Hint string `json:"hint,omitempty"`
+}
+type merchantPrompt struct {
+	ID            string   `json:"id"`
+	Name          string   `json:"name"`
+	Aliases       []string `json:"aliases"`
+	UsualCategory string   `json:"usual_category,omitempty"`
+}
+
+// candidate is the historical merchant prompt shape used by older callers.
+type candidate = merchantPrompt
+type promptHistory struct {
+	Date         string   `json:"date"`
+	Amount       string   `json:"amount"`
+	Description  string   `json:"description"`
+	Counterparty string   `json:"counterparty"`
+	CategoryID   string   `json:"category_id"`
+	MerchantID   string   `json:"merchant_id,omitempty"`
+	TagIDs       []string `json:"tag_ids"`
 }
 type candidateSet struct {
-	categories, tags, merchants      []candidate
-	categoryIDs, tagIDs, merchantIDs map[string]string
-}
-type ranked struct {
-	id, name string
-	score    int
+	categories  []categoryPrompt
+	tags        []tagPrompt
+	merchants   []merchantPrompt
+	categoryIDs map[string]string
+	tagIDs      map[string]string
+	merchantIDs map[string]string
 }

 func similarity(description, name string) int {
@@ -132,10 +178,9 @@ func similarity(description, name string) int {
 	if strings.Contains(" "+a+" ", " "+b+" ") {
 		return 10000 + len(b)
 	}
-	words := strings.Fields(a)
 	score := 0
 	for _, word := range strings.Fields(b) {
-		for _, input := range words {
+		for _, input := range strings.Fields(a) {
 			if input == word {
 				score += len(word)
 				break
@@ -145,76 +190,90 @@ func similarity(description, name string) int {
 	return score
 }

-func bounded(rows []ranked, prefix string, limit int, clean func(string) string) ([]candidate, map[string]string) {
-	sort.Slice(rows, func(i, j int) bool {
-		if rows[i].score != rows[j].score {
-			return rows[i].score > rows[j].score
-		}
-		return rows[i].id < rows[j].id
-	})
-	if limit > 0 && len(rows) > limit {
-		rows = rows[:limit]
-	}
-	out := make([]candidate, 0, len(rows))
-	ids := make(map[string]string, len(rows))
-	for i, row := range rows {
-		id := fmt.Sprintf("%s%d", prefix, i+1)
-		name := clean(row.name)
-		if name == "" {
-			name = "unnamed"
-		}
-		out = append(out, candidate{ID: id, Name: name})
-		ids[id] = row.id
-	}
-	return out, ids
-}
-
-func retrieve(description, kind string, data domain.Dataset, clean, merchantClean func(string) string) candidateSet {
-	var categories, tags, merchants []ranked
-	fallback := domain.ExpenseFallback
-	if kind == "income" {
-		fallback = domain.IncomeFallback
-	}
+// retrieve emits every registry entry with its real id. The legacy cleaner
+// arguments remain in the signature because CSV/classification fixtures use
+// this helper directly; ranking and bounding are intentionally gone.
+func retrieve(_ string, kind string, data domain.Dataset, clean, merchantClean func(string) string) candidateSet {
 	parents := map[string]bool{}
 	for _, cat := range data.Categories {
 		parents[cat.ParentID] = true
 	}
+	set := candidateSet{
+		categoryIDs: map[string]string{},
+		tagIDs:      map[string]string{},
+		merchantIDs: map[string]string{},
+	}
 	for _, cat := range data.Categories {
 		if cat.Kind != kind || parents[cat.ID] {
 			continue
 		}
-		name := domain.CategoryPath(data, cat.ID)
-		score := similarity(description, name)
-		if cat.ID == fallback {
-			score = int(^uint(0) >> 1)
+		path := domain.CategoryPath(data, cat.ID)
+		if clean != nil {
+			path = clean(path)
 		}
-		categories = append(categories, ranked{id: cat.ID, name: name, score: score})
+		set.categories = append(set.categories, categoryPrompt{ID: cat.ID, Path: path, Kind: cat.Kind, Hint: cleanText(clean, cat.Hint)})
+		set.categoryIDs[cat.ID] = cat.ID
 	}
+	sort.Slice(set.categories, func(i, j int) bool {
+		return set.categories[i].Path < set.categories[j].Path || set.categories[i].Path == set.categories[j].Path && set.categories[i].ID < set.categories[j].ID
+	})
 	for _, tag := range data.Tags {
-		tags = append(tags, ranked{id: tag.ID, name: tag.Name, score: similarity(description, tag.Name)})
+		name := cleanText(clean, tag.Name)
+		set.tags = append(set.tags, tagPrompt{ID: tag.ID, Name: name, Hint: cleanText(clean, tag.Hint)})
+		set.tagIDs[tag.ID] = tag.ID
 	}
-	for _, m := range data.Merchants {
-		score := similarity(description, m.Name)
-		for _, alias := range m.Aliases {
-			if s := similarity(description, alias); s > score {
-				score = s
+	sort.Slice(set.tags, func(i, j int) bool {
+		return set.tags[i].Name < set.tags[j].Name || set.tags[i].Name == set.tags[j].Name && set.tags[i].ID < set.tags[j].ID
+	})
+	usual := map[string]string{}
+	counts := map[string]map[string]int{}
+	for _, tx := range data.Transactions {
+		merchantID, categoryID := tx.Enrichment.MerchantID, tx.Enrichment.CategoryID
+		if merchantID == "" || categoryID == "" {
+			continue
+		}
+		if counts[merchantID] == nil {
+			counts[merchantID] = map[string]int{}
+		}
+		counts[merchantID][categoryID]++
+	}
+	for merchantID, values := range counts {
+		for categoryID, count := range values {
+			current := usual[merchantID]
+			if current == "" || count > values[current] || count == values[current] && categoryID < current {
+				usual[merchantID] = categoryID
 			}
 		}
-		merchants = append(merchants, ranked{id: m.ID, name: m.Name, score: score})
 	}
-	var set candidateSet
-	set.categories, set.categoryIDs = bounded(categories, "c", 0, clean)
-	set.tags, set.tagIDs = bounded(tags, "t", 0, clean)
-	set.merchants, set.merchantIDs = bounded(merchants, "m", 20, merchantClean)
+	for _, merchant := range data.Merchants {
+		name := cleanText(merchantClean, merchant.Name)
+		aliases := make([]string, 0, len(merchant.Aliases))
+		for _, alias := range merchant.Aliases {
+			if value := cleanText(merchantClean, alias); value != "" {
+				aliases = append(aliases, value)
+			}
+		}
+		usualCategory := merchant.DefaultCategoryID
+		if categoryID := usual[merchant.ID]; categoryID != "" {
+			usualCategory = categoryID
+		}
+		set.merchants = append(set.merchants, merchantPrompt{
+			ID: merchant.ID, Name: name, Aliases: aliases,
+			UsualCategory: usualCategory,
+		})
+		set.merchantIDs[merchant.ID] = merchant.ID
+	}
+	sort.Slice(set.merchants, func(i, j int) bool {
+		return set.merchants[i].Name < set.merchants[j].Name || set.merchants[i].Name == set.merchants[j].Name && set.merchants[i].ID < set.merchants[j].ID
+	})
 	return set
 }

-func candidateEnums(candidates []candidate) []string {
-	ids := make([]string, 0, len(candidates))
-	for _, c := range candidates {
-		ids = append(ids, c.ID)
+func cleanText(clean func(string) string, value string) string {
+	if clean == nil {
+		return normalize(value)
 	}
-	return ids
+	return clean(value)
 }

 func (c candidateSet) schema() map[string]any {
@@ -222,19 +281,76 @@ func (c candidateSet) schema() map[string]any {
 	for _, m := range c.merchants {
 		merchantEnums = append(merchantEnums, m.ID)
 	}
+	tagIDs := make([]any, 0, len(c.tags))
+	for _, tag := range c.tags {
+		tagIDs = append(tagIDs, tag.ID)
+	}
 	tagItems := map[string]any{"type": "string"}
-	if len(c.tags) > 0 {
-		tagItems["enum"] = candidateEnums(c.tags)
+	if len(tagIDs) > 0 {
+		tagItems["enum"] = tagIDs
 	}
-	tags := map[string]any{"type": "array", "items": tagItems, "maxItems": len(c.tags), "uniqueItems": true}
 	return map[string]any{
 		"type": "object", "additionalProperties": false,
-		"required": []string{"merchant_id", "new_merchant", "category_id", "tag_ids"},
+		"required": []string{"merchant_id", "new_merchant", "category_id", "tag_ids", "confidence"},
 		"properties": map[string]any{
-			"merchant_id":  map[string]any{"type": []string{"string", "null"}, "enum": merchantEnums, "description": "Existing merchant candidate ID, or null."},
-			"new_merchant": map[string]any{"type": []string{"string", "null"}, "maxLength": 100, "description": "Public business name only when no existing merchant matches, otherwise null."},
-			"category_id":  map[string]any{"type": "string", "enum": candidateEnums(c.categories)},
-			"tag_ids":      tags,
+			"merchant_id":  map[string]any{"type": []string{"string", "null"}, "enum": merchantEnums},
+			"new_merchant": map[string]any{"type": []string{"string", "null"}, "maxLength": 100},
+			"category_id":  map[string]any{"type": "string", "enum": candidateIDs(c.categories)},
+			"tag_ids":      map[string]any{"type": "array", "uniqueItems": true, "maxItems": len(tagIDs), "items": tagItems},
+			"confidence":   map[string]any{"type": "string", "enum": []string{"high", "medium", "low"}},
 		},
 	}
 }
+
+func candidateIDs(values []categoryPrompt) []string {
+	ids := make([]string, 0, len(values))
+	for _, value := range values {
+		ids = append(ids, value.ID)
+	}
+	return ids
+}
+
+func answerSchema(d domain.Dataset, kind string) map[string]any {
+	return retrieve("", kind, d, nil, nil).schema()
+}
+
+func history(f domain.Facts, d domain.Dataset, clean func(string) string, limit int) []promptHistory {
+	type row struct {
+		tx    domain.Transaction
+		score int
+	}
+	rows := []row{}
+	for _, tx := range d.Transactions {
+		e := tx.Enrichment
+		if tx.Facts.ID == f.ID || e.Kind == "transfer" || e.CategoryID == "" || e.CategoryID == domain.ExpenseFallback || e.CategoryID == domain.IncomeFallback {
+			continue
+		}
+		rows = append(rows, row{tx: tx, score: similarity(f.RawDescription+" "+f.Counterparty, tx.Facts.RawDescription+" "+tx.Facts.Counterparty)})
+	}
+	sort.Slice(rows, func(i, j int) bool {
+		if rows[i].score != rows[j].score {
+			return rows[i].score > rows[j].score
+		}
+		if rows[i].tx.Facts.BookingDate != rows[j].tx.Facts.BookingDate {
+			return rows[i].tx.Facts.BookingDate > rows[j].tx.Facts.BookingDate
+		}
+		return rows[i].tx.Facts.ID < rows[j].tx.Facts.ID
+	})
+	if limit > 0 && len(rows) > limit {
+		rows = rows[:limit]
+	}
+	out := make([]promptHistory, 0, len(rows))
+	for _, row := range rows {
+		tags := row.tx.Enrichment.TagIDs
+		if tags == nil {
+			tags = []string{}
+		}
+		out = append(out, promptHistory{
+			Date: row.tx.Facts.BookingDate, Amount: string(row.tx.Facts.Amount),
+			Description: clean(row.tx.Facts.RawDescription), Counterparty: clean(row.tx.Facts.Counterparty),
+			CategoryID: row.tx.Enrichment.CategoryID, MerchantID: row.tx.Enrichment.MerchantID,
+			TagIDs: append([]string{}, tags...),
+		})
+	}
+	return out
+}