package classification import ( "context" "fmt" "net/http" "reflect" "strings" "testing" "finance-duck/internal/domain" ) // ledgerFixture mirrors a production ledger that repeatedly broke // classification in the field: a proposed two-level taxonomy (43 expense // leaves), tags, a merchant registry polluted with location-like names, and // German bank rows whose payee text carries reference numbers. Personal // names and IBANs are fabricated. func ledgerFixture() (domain.Dataset, domain.Facts) { d := domain.NewDataset() d.Accounts = []domain.Account{{ID: "acct_kontist", DisplayName: "Business", Institution: "Kontist", Currency: "EUR", Active: true}} tree := map[string][]string{ "housing": {"rent", "utilities", "household", "maintenance"}, "food": {"groceries", "restaurants", "takeaway"}, "transport": {"fuel", "public-transport", "parking", "taxi", "vehicle-maintenance"}, "shopping": {"clothing", "electronics", "household-goods", "other"}, "pets": {"pet-food", "pet-health", "supplies"}, "entertainment": {"games", "events", "ent-media"}, "travel": {"accommodation", "travel-transport", "activities"}, "health": {"medical", "pharmacy", "fitness"}, "education": {"tuition", "books", "courses"}, "subscriptions": {"software", "sub-media", "services"}, "insurance": {"vehicle-insurance", "health-insurance", "other-insurance"}, "financial": {"bank-fees", "interest-paid", "taxes"}, "gifts": nil, "donations": nil, } for parent, children := range tree { d.Categories = append(d.Categories, domain.Category{ID: "cat_" + parent, Name: parent, ParentID: "cat_expenses", Kind: "expense"}) for _, child := range children { d.Categories = append(d.Categories, domain.Category{ID: "cat_" + child, Name: child, ParentID: "cat_" + parent, Kind: "expense"}) } } for _, name := range []string{"personal", "business", "travel", "hobby", "home", "mx5", "education", "gift", "tax-deductible", "subscription", "groceries"} { d.Tags = append(d.Tags, domain.Tag{ID: "tag_" + name, Name: name}) } // Location-like junk from a taxonomy proposal run: it must stay selectable // without breaking the strict schema or the alias matcher. for _, name := range []string{"smart steuerservice", "kranken", "Chittaway Bay", "Toronto", "bruhl", "brunico", "St. Ulrich", "Git Server", "Mobilfunk", "Swopper"} { d.Merchants = append(d.Merchants, domain.Merchant{ID: domain.NewID("mer"), Name: name, Aliases: []string{}, DefaultTagIDs: []string{}, UseDefaults: false}) } facts := domain.Facts{ ID: "tx_finanzamt", Source: "enablebanking", AccountID: "acct_kontist", BookingDate: "2026-08-30", ValueDate: "2026-08-30", Amount: "-849.45", Currency: "EUR", RawDescription: "0904303543105 224/5220/5869", Counterparty: "Finanzamt Bruehl", CounterpartyIBAN: "DE02120300000000202051", Fingerprint: "f1e2d3", } d.Transactions = []domain.Transaction{{Facts: facts, Enrichment: domain.Fallback(facts)}} return d, facts } // strictKeywords is what every targeted provider accepts in strict // structured-output mode. uniqueItems is rejected outright by OpenAI-family // endpoints ("'uniqueItems' is not permitted"); minItems/maxItems make Gemini // expand array item schemas per element and reject real registries with a // bare HTTP 400. Counts and duplicates are enforced server-side instead. var strictKeywords = map[string]bool{ "type": true, "properties": true, "required": true, "additionalProperties": true, "items": true, "enum": true, "maxLength": true, "minLength": true, } func checkStrict(t *testing.T, path string, value any) { t.Helper() switch v := value.(type) { case map[string]any: for key, child := range v { if path == "" || strings.HasSuffix(path, ".properties") { // Property names and the schema root are not keywords. } else if !strictKeywords[key] { t.Errorf("%s uses %q, which strict structured-output mode rejects", path, key) } checkStrict(t, path+"."+key, child) } case []any: for _, child := range v { checkStrict(t, path+"[]", child) } } } func TestWireSchemasUseOnlyStrictModeKeywords(t *testing.T) { d, facts := ledgerFixture() set := retrieve(facts.RawDescription, "expense", d, nil, nil) for name, schema := range map[string]map[string]any{ "classification": set.schema(), "batch": set.batchSchema([]string{"r1", "r2", "r3"}), "taxonomy": taxonomySchema(), "csv": csvMappingSchema(CSVMappingRequest{Headers: []string{"Buchung", "Betrag"}}), } { checkStrict(t, name, map[string]any{"properties": schema["properties"]}) } } // The exact answer a live gpt-5.6-luna-pro returned for this row over a // zero-data-retention route must land as reviewable enrichment: taxes // category, a new public merchant seeded with the counterparty alias, no // tags, recorded confidence. func TestLedgerRowClassifiesThroughStrictSchema(t *testing.T) { d, facts := ledgerFixture() taxes := "" for _, c := range d.Categories { if c.Name == "taxes" { taxes = c.ID } } c := mockClient(t, func(w http.ResponseWriter, r *http.Request) { reply(w, `{"merchant_id":null,"new_merchant":"Finanzamt Bruehl","category_id":"`+taxes+`","tag_ids":[],"confidence":"high"}`) }) p, err := c.Classify(context.Background(), facts, d, true) if err != nil { t.Fatal(err) } if p.Enrichment.CategoryID != taxes || p.Enrichment.Classification.Confidence != "high" { t.Fatalf("classification lost: %+v", p.Enrichment) } if p.NewMerchant == nil || p.NewMerchant.Name != "Finanzamt Bruehl" || !reflect.DeepEqual(p.NewMerchant.Aliases, []string{"Finanzamt Bruehl"}) { t.Fatalf("merchant proposal lost: %+v", p.NewMerchant) } if len(p.Enrichment.TagIDs) != 0 { t.Fatalf("unexpected tags: %+v", p.Enrichment.TagIDs) } } // Identifier redaction must not eat ordinary 8- and 11-letter payee words, // which blinded the model to the merchant it was asked to classify // ("WWW.RACETRACKER.DE" became "WWW. .DE"). A bare bank-code-shaped token is // vocabulary; real BICs still die labeled or trailing their IBAN. func TestBICRedactionKeepsPayeeVocabulary(t *testing.T) { d, facts := ledgerFixture() clean := redactor(d, facts, nil) for _, keep := range []string{"Openbank", "OPENBANK", "Baumarkt", "BAUMARKT", "RACETRACKER", "toom Baumarkt"} { if got := clean(keep); got != normalize(keep) { t.Errorf("payee word %q was redacted to %q", keep, got) } } for name, text := range map[string]string{ "labeled iban": "IBAN DE89370400440532013000 COBADEFFXXX invoice", "trailing bic": "pay DE89370400440532013000 COBADEFFXXX today", "labeled bic": "BIC DEUTDEDBFRA", "labeled swift": "SWIFT GENODED1SPO", } { got := clean(text) if strings.Contains(got, "de8937") || strings.Contains(got, "cobadeff") || strings.Contains(got, "deutdedb") || strings.Contains(got, "genoded1") { t.Errorf("%s: identifier survived redaction: %q", name, got) } } } // One manual correction must outrank any number of the model's own past // answers for the same payee: without source ranking, precedent feeds the // model its uncorrected output as majority evidence and corrections never // stick. func TestManualCorrectionsOutrankAIPrecedent(t *testing.T) { d, _ := ledgerFixture() groceries, events := "", "" for _, c := range d.Categories { if c.Name == "groceries" { groceries = c.ID } if c.Name == "events" { events = c.ID } } add := func(id, date, category, source string) { f := domain.Facts{ID: id, Source: "test", AccountID: "acct_kontist", BookingDate: date, Amount: "-13.00", Currency: "EUR", Counterparty: "LVR Landesmuseum Bonn", Fingerprint: id} d.Transactions = append(d.Transactions, domain.Transaction{Facts: f, Enrichment: domain.Enrichment{ Kind: "expense", CategoryID: category, TagIDs: []string{}, Classification: domain.Provenance{Source: source}, }}) } // Many uncorrected AI answers, one older manual correction. for i := range 30 { add(fmt.Sprintf("tx_ai_%02d", i), "2026-08-20", groceries, "openrouter") } add("tx_corrected", "2026-08-01", events, "manual") target := domain.Facts{ID: "tx_new", AccountID: "acct_kontist", BookingDate: "2026-08-30", Amount: "-13.00", Currency: "EUR", Counterparty: "LVR Landesmuseum Bonn"} rows := history(target, d, func(s string) string { return normalize(s) }, 20) if len(rows) == 0 || rows[0].Source != "user" || rows[0].CategoryID != events { t.Fatalf("manual correction did not lead precedent: %+v", rows[0]) } // The correction keeps its slot even in a window the AI rows could fill. users := 0 for _, row := range rows { if row.Source == "user" { users++ } } if users == 0 { t.Fatal("correction crowded out of the history window") } }