Make OpenAI-family strict mode routable and stop redacting payee words

Strict structured-output mode rejects uniqueItems, so every request to a gpt-5.6-family zero-data-retention endpoint failed with HTTP 400 behind a generic error; duplicates were already rejected server-side, so the keyword leaves the wire schemas, pinned by a strict-keyword allowlist test built from the ledger that hit this. The bare-BIC redaction pattern deleted every 8- and 11-letter word — Openbank, BAUMARKT, RACETRACKER — blinding the model to the payee it was asked to classify and tripping the unsafe-merchant check on honest answers. BICs now die only labeled or attached to their IBAN, account labels join the redaction secrets, an identifier-shaped merchant name degrades to a merchant-less proposal instead of failing the row, and a provider error inside an HTTP 200 envelope is reported as such (numeric code only) instead of as envelope corruption.
2026-09-12 23:17:24 +02:00
parent ec99434002
commit c5999adb1b
8 changed files with 223 additions and 25 deletions
@@ -0,0 +1,156 @@
+package classification
+
+import (
+	"context"
+	"net/http"
+	"reflect"
+	"strings"
+	"testing"
+
+	"finance-duck/internal/domain"
+)
+
+// ledgerFixture mirrors a production ledger that repeatedly broke
+// classification in the field: a proposed two-level taxonomy (43 expense
+// leaves), tags, a merchant registry polluted with location-like names, and
+// German bank rows whose payee text carries reference numbers. Personal
+// names and IBANs are fabricated.
+func ledgerFixture() (domain.Dataset, domain.Facts) {
+	d := domain.NewDataset()
+	d.Accounts = []domain.Account{{ID: "acct_kontist", DisplayName: "Business", Institution: "Kontist", Currency: "EUR", Active: true}}
+	tree := map[string][]string{
+		"housing":       {"rent", "utilities", "household", "maintenance"},
+		"food":          {"groceries", "restaurants", "takeaway"},
+		"transport":     {"fuel", "public-transport", "parking", "taxi", "vehicle-maintenance"},
+		"shopping":      {"clothing", "electronics", "household-goods", "other"},
+		"pets":          {"pet-food", "pet-health", "supplies"},
+		"entertainment": {"games", "events", "ent-media"},
+		"travel":        {"accommodation", "travel-transport", "activities"},
+		"health":        {"medical", "pharmacy", "fitness"},
+		"education":     {"tuition", "books", "courses"},
+		"subscriptions": {"software", "sub-media", "services"},
+		"insurance":     {"vehicle-insurance", "health-insurance", "other-insurance"},
+		"financial":     {"bank-fees", "interest-paid", "taxes"},
+		"gifts":         nil,
+		"donations":     nil,
+	}
+	for parent, children := range tree {
+		d.Categories = append(d.Categories, domain.Category{ID: "cat_" + parent, Name: parent, ParentID: "cat_expenses", Kind: "expense"})
+		for _, child := range children {
+			d.Categories = append(d.Categories, domain.Category{ID: "cat_" + child, Name: child, ParentID: "cat_" + parent, Kind: "expense"})
+		}
+	}
+	for _, name := range []string{"personal", "business", "travel", "hobby", "home", "mx5", "education", "gift", "tax-deductible", "subscription", "groceries"} {
+		d.Tags = append(d.Tags, domain.Tag{ID: "tag_" + name, Name: name})
+	}
+	// Location-like junk from a taxonomy proposal run: it must stay selectable
+	// without breaking the strict schema or the alias matcher.
+	for _, name := range []string{"smart steuerservice", "kranken", "Chittaway Bay", "Toronto", "bruhl", "brunico", "St. Ulrich", "Git Server", "Mobilfunk", "Swopper"} {
+		d.Merchants = append(d.Merchants, domain.Merchant{ID: domain.NewID("mer"), Name: name, Aliases: []string{}, DefaultTagIDs: []string{}, UseDefaults: false})
+	}
+	facts := domain.Facts{
+		ID: "tx_finanzamt", Source: "enablebanking", AccountID: "acct_kontist",
+		BookingDate: "2026-08-30", ValueDate: "2026-08-30", Amount: "-849.45", Currency: "EUR",
+		RawDescription: "0904303543105 224/5220/5869",
+		Counterparty:   "Finanzamt Bruehl", CounterpartyIBAN: "DE02120300000000202051",
+		Fingerprint: "f1e2d3",
+	}
+	d.Transactions = []domain.Transaction{{Facts: facts, Enrichment: domain.Fallback(facts)}}
+	return d, facts
+}
+
+// strictKeywords is what OpenAI-family strict structured-output mode accepts.
+// uniqueItems is specifically rejected ("'uniqueItems' is not permitted") and
+// took every zero-data-retention route for those models down with HTTP 400;
+// duplicates are rejected server-side by decodeAnswer instead.
+var strictKeywords = map[string]bool{
+	"type": true, "properties": true, "required": true, "additionalProperties": true,
+	"items": true, "enum": true, "maxItems": true, "maxLength": true, "minLength": true,
+}
+
+func checkStrict(t *testing.T, path string, value any) {
+	t.Helper()
+	switch v := value.(type) {
+	case map[string]any:
+		for key, child := range v {
+			if path == "" || strings.HasSuffix(path, ".properties") {
+				// Property names and the schema root are not keywords.
+			} else if !strictKeywords[key] {
+				t.Errorf("%s uses %q, which strict structured-output mode rejects", path, key)
+			}
+			checkStrict(t, path+"."+key, child)
+		}
+	case []any:
+		for _, child := range v {
+			checkStrict(t, path+"[]", child)
+		}
+	}
+}
+
+func TestWireSchemasUseOnlyStrictModeKeywords(t *testing.T) {
+	d, facts := ledgerFixture()
+	set := retrieve(facts.RawDescription, "expense", d, nil, nil)
+	for name, schema := range map[string]map[string]any{
+		"classification": set.schema(),
+		"taxonomy":       taxonomySchema(),
+		"csv":            csvMappingSchema(CSVMappingRequest{Headers: []string{"Buchung", "Betrag"}}),
+	} {
+		checkStrict(t, name, map[string]any{"properties": schema["properties"]})
+	}
+}
+
+// The exact answer a live gpt-5.6-luna-pro returned for this row over a
+// zero-data-retention route must land as reviewable enrichment: taxes
+// category, a new public merchant seeded with the counterparty alias, no
+// tags, recorded confidence.
+func TestLedgerRowClassifiesThroughStrictSchema(t *testing.T) {
+	d, facts := ledgerFixture()
+	taxes := ""
+	for _, c := range d.Categories {
+		if c.Name == "taxes" {
+			taxes = c.ID
+		}
+	}
+	c := mockClient(t, func(w http.ResponseWriter, r *http.Request) {
+		reply(w, `{"merchant_id":null,"new_merchant":"Finanzamt Bruehl","category_id":"`+taxes+`","tag_ids":[],"confidence":"high"}`)
+	})
+	p, err := c.Classify(context.Background(), facts, d, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if p.Enrichment.CategoryID != taxes || p.Enrichment.Classification.Confidence != "high" {
+		t.Fatalf("classification lost: %+v", p.Enrichment)
+	}
+	if p.NewMerchant == nil || p.NewMerchant.Name != "Finanzamt Bruehl" ||
+		!reflect.DeepEqual(p.NewMerchant.Aliases, []string{"Finanzamt Bruehl"}) {
+		t.Fatalf("merchant proposal lost: %+v", p.NewMerchant)
+	}
+	if len(p.Enrichment.TagIDs) != 0 {
+		t.Fatalf("unexpected tags: %+v", p.Enrichment.TagIDs)
+	}
+}
+
+// Identifier redaction must not eat ordinary 8- and 11-letter payee words,
+// which blinded the model to the merchant it was asked to classify
+// ("WWW.RACETRACKER.DE" became "WWW. .DE"). A bare bank-code-shaped token is
+// vocabulary; real BICs still die labeled or trailing their IBAN.
+func TestBICRedactionKeepsPayeeVocabulary(t *testing.T) {
+	d, facts := ledgerFixture()
+	clean := redactor(d, facts, nil)
+	for _, keep := range []string{"Openbank", "OPENBANK", "Baumarkt", "BAUMARKT", "RACETRACKER", "toom Baumarkt"} {
+		if got := clean(keep); got != normalize(keep) {
+			t.Errorf("payee word %q was redacted to %q", keep, got)
+		}
+	}
+	for name, text := range map[string]string{
+		"labeled iban":  "IBAN DE89370400440532013000 COBADEFFXXX invoice",
+		"trailing bic":  "pay DE89370400440532013000 COBADEFFXXX today",
+		"labeled bic":   "BIC DEUTDEDBFRA",
+		"labeled swift": "SWIFT GENODED1SPO",
+	} {
+		got := clean(text)
+		if strings.Contains(got, "de8937") || strings.Contains(got, "cobadeff") || strings.Contains(got, "deutdedb") || strings.Contains(got, "genoded1") {
+			t.Errorf("%s: identifier survived redaction: %q", name, got)
+		}
+	}
+}