Use compact classification IDs and extend preview lifetime
This commit is contained in:
Lars Nolden
@@ -3,6 +3,7 @@ package classification
|
||||
import (
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
@@ -150,8 +151,6 @@ type merchantPrompt struct {
|
||||
UsualCategory string `json:"usual_category,omitempty"`
|
||||
}
|
||||
|
||||
// candidate is the historical merchant prompt shape used by older callers.
|
||||
type candidate = merchantPrompt
|
||||
type promptHistory struct {
|
||||
Date string `json:"date"`
|
||||
Amount string `json:"amount"`
|
||||
@@ -166,12 +165,15 @@ type promptHistory struct {
|
||||
Source string `json:"source"`
|
||||
}
|
||||
type candidateSet struct {
|
||||
categories []categoryPrompt
|
||||
tags []tagPrompt
|
||||
merchants []merchantPrompt
|
||||
categoryIDs map[string]string
|
||||
tagIDs map[string]string
|
||||
merchantIDs map[string]string
|
||||
categories []categoryPrompt
|
||||
tags []tagPrompt
|
||||
merchants []merchantPrompt
|
||||
categoryIDs map[string]string
|
||||
tagIDs map[string]string
|
||||
merchantIDs map[string]string
|
||||
categoryRefs map[string]string
|
||||
tagRefs map[string]string
|
||||
merchantRefs map[string]string
|
||||
}
|
||||
|
||||
func similarity(description, name string) int {
|
||||
@@ -194,18 +196,20 @@ func similarity(description, name string) int {
|
||||
return score
|
||||
}
|
||||
|
||||
// retrieve emits every registry entry with its real id. The legacy cleaner
|
||||
// arguments remain in the signature because CSV/classification fixtures use
|
||||
// this helper directly; ranking and bounding are intentionally gone.
|
||||
// retrieve offers every eligible registry entry under a short request-local
|
||||
// reference. Names and paths retain their meaning; canonical IDs stay local.
|
||||
func retrieve(_ string, kind string, data domain.Dataset, clean, merchantClean func(string) string) candidateSet {
|
||||
parents := map[string]bool{}
|
||||
for _, cat := range data.Categories {
|
||||
parents[cat.ParentID] = true
|
||||
}
|
||||
set := candidateSet{
|
||||
categoryIDs: map[string]string{},
|
||||
tagIDs: map[string]string{},
|
||||
merchantIDs: map[string]string{},
|
||||
categoryIDs: map[string]string{},
|
||||
tagIDs: map[string]string{},
|
||||
merchantIDs: map[string]string{},
|
||||
categoryRefs: map[string]string{},
|
||||
tagRefs: map[string]string{},
|
||||
merchantRefs: map[string]string{},
|
||||
}
|
||||
for _, cat := range data.Categories {
|
||||
if cat.Kind != kind || parents[cat.ID] {
|
||||
@@ -216,19 +220,31 @@ func retrieve(_ string, kind string, data domain.Dataset, clean, merchantClean f
|
||||
path = clean(path)
|
||||
}
|
||||
set.categories = append(set.categories, categoryPrompt{ID: cat.ID, Path: path, Kind: cat.Kind, Hint: cleanText(clean, cat.Hint)})
|
||||
set.categoryIDs[cat.ID] = cat.ID
|
||||
}
|
||||
sort.Slice(set.categories, func(i, j int) bool {
|
||||
return set.categories[i].Path < set.categories[j].Path || set.categories[i].Path == set.categories[j].Path && set.categories[i].ID < set.categories[j].ID
|
||||
})
|
||||
for i := range set.categories {
|
||||
category := &set.categories[i]
|
||||
ref := "c" + strconv.Itoa(i+1)
|
||||
set.categoryIDs[ref] = category.ID
|
||||
set.categoryRefs[category.ID] = ref
|
||||
category.ID = ref
|
||||
}
|
||||
for _, tag := range data.Tags {
|
||||
name := cleanText(clean, tag.Name)
|
||||
set.tags = append(set.tags, tagPrompt{ID: tag.ID, Name: name, Hint: cleanText(clean, tag.Hint)})
|
||||
set.tagIDs[tag.ID] = tag.ID
|
||||
}
|
||||
sort.Slice(set.tags, func(i, j int) bool {
|
||||
return set.tags[i].Name < set.tags[j].Name || set.tags[i].Name == set.tags[j].Name && set.tags[i].ID < set.tags[j].ID
|
||||
})
|
||||
for i := range set.tags {
|
||||
tag := &set.tags[i]
|
||||
ref := "t" + strconv.Itoa(i+1)
|
||||
set.tagIDs[ref] = tag.ID
|
||||
set.tagRefs[tag.ID] = ref
|
||||
tag.ID = ref
|
||||
}
|
||||
usual := map[string]string{}
|
||||
counts := map[string]map[string]int{}
|
||||
for _, tx := range data.Transactions {
|
||||
@@ -263,13 +279,19 @@ func retrieve(_ string, kind string, data domain.Dataset, clean, merchantClean f
|
||||
}
|
||||
set.merchants = append(set.merchants, merchantPrompt{
|
||||
ID: merchant.ID, Name: name, Aliases: aliases,
|
||||
UsualCategory: usualCategory,
|
||||
UsualCategory: set.categoryRefs[usualCategory],
|
||||
})
|
||||
set.merchantIDs[merchant.ID] = merchant.ID
|
||||
}
|
||||
sort.Slice(set.merchants, func(i, j int) bool {
|
||||
return set.merchants[i].Name < set.merchants[j].Name || set.merchants[i].Name == set.merchants[j].Name && set.merchants[i].ID < set.merchants[j].ID
|
||||
})
|
||||
for i := range set.merchants {
|
||||
merchant := &set.merchants[i]
|
||||
ref := "m" + strconv.Itoa(i+1)
|
||||
set.merchantIDs[ref] = merchant.ID
|
||||
set.merchantRefs[merchant.ID] = ref
|
||||
merchant.ID = ref
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
@@ -314,16 +336,12 @@ func candidateIDs(values []categoryPrompt) []string {
|
||||
return ids
|
||||
}
|
||||
|
||||
func answerSchema(d domain.Dataset, kind string) map[string]any {
|
||||
return retrieve("", kind, d, nil, nil).schema()
|
||||
}
|
||||
|
||||
// history selects precedent for the prompt: the nearest rows by word overlap,
|
||||
// filled out with the most recent. The user's own decisions — manual edits
|
||||
// and locally applied merchant rules — outrank rows the model classified
|
||||
// itself, so one correction beats any number of uncorrected AI answers for
|
||||
// the same payee.
|
||||
func history(f domain.Facts, d domain.Dataset, clean func(string) string, limit int) []promptHistory {
|
||||
// history selects precedent whose category is offered in this request: the
|
||||
// nearest rows by word overlap, filled out with the most recent. The user's
|
||||
// own decisions — manual edits and locally applied merchant rules — outrank
|
||||
// rows the model classified itself. References use the same mapping as the
|
||||
// candidate lists and response schema.
|
||||
func (c candidateSet) history(f domain.Facts, d domain.Dataset, clean func(string) string, limit int) []promptHistory {
|
||||
type row struct {
|
||||
tx domain.Transaction
|
||||
score int
|
||||
@@ -332,7 +350,7 @@ func history(f domain.Facts, d domain.Dataset, clean func(string) string, limit
|
||||
rows := []row{}
|
||||
for _, tx := range d.Transactions {
|
||||
e := tx.Enrichment
|
||||
if tx.Facts.ID == f.ID || e.Kind == "transfer" || e.CategoryID == "" || e.CategoryID == domain.ExpenseFallback || e.CategoryID == domain.IncomeFallback {
|
||||
if tx.Facts.ID == f.ID || e.Kind == "transfer" || c.categoryRefs[e.CategoryID] == "" || e.CategoryID == domain.ExpenseFallback || e.CategoryID == domain.IncomeFallback {
|
||||
continue
|
||||
}
|
||||
source := tx.Enrichment.Classification.Source
|
||||
@@ -382,9 +400,11 @@ func history(f domain.Facts, d domain.Dataset, clean func(string) string, limit
|
||||
}
|
||||
out := make([]promptHistory, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
tags := row.tx.Enrichment.TagIDs
|
||||
if tags == nil {
|
||||
tags = []string{}
|
||||
tags := make([]string, 0, len(row.tx.Enrichment.TagIDs))
|
||||
for _, id := range row.tx.Enrichment.TagIDs {
|
||||
if ref := c.tagRefs[id]; ref != "" {
|
||||
tags = append(tags, ref)
|
||||
}
|
||||
}
|
||||
source := "ai"
|
||||
if row.user {
|
||||
@@ -393,8 +413,8 @@ func history(f domain.Facts, d domain.Dataset, clean func(string) string, limit
|
||||
out = append(out, promptHistory{
|
||||
Date: row.tx.Facts.BookingDate, Amount: string(row.tx.Facts.Amount),
|
||||
Description: clean(row.tx.Facts.RawDescription), Counterparty: clean(row.tx.Facts.Counterparty),
|
||||
CategoryID: row.tx.Enrichment.CategoryID, MerchantID: row.tx.Enrichment.MerchantID,
|
||||
TagIDs: append([]string{}, tags...),
|
||||
CategoryID: c.categoryRefs[row.tx.Enrichment.CategoryID], MerchantID: c.merchantRefs[row.tx.Enrichment.MerchantID],
|
||||
TagIDs: tags,
|
||||
Source: source,
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user