Make OpenAI-family strict mode routable and stop redacting payee words

Strict structured-output mode rejects uniqueItems, so every request to a
gpt-5.6-family zero-data-retention endpoint failed with HTTP 400 behind
a generic error; duplicates were already rejected server-side, so the
keyword leaves the wire schemas, pinned by a strict-keyword allowlist
test built from the ledger that hit this.

The bare-BIC redaction pattern deleted every 8- and 11-letter word —
Openbank, BAUMARKT, RACETRACKER — blinding the model to the payee it
was asked to classify and tripping the unsafe-merchant check on honest
answers. BICs now die only labeled or attached to their IBAN, account
labels join the redaction secrets, an identifier-shaped merchant name
degrades to a merchant-less proposal instead of failing the row, and a
provider error inside an HTTP 200 envelope is reported as such (numeric
code only) instead of as envelope corruption.

internal/classification/candidates.go

@@ -296,7 +296,7 @@ func (c candidateSet) schema() map[string]any {
 			"merchant_id":  map[string]any{"type": []string{"string", "null"}, "enum": merchantEnums},
 			"new_merchant": map[string]any{"type": []string{"string", "null"}, "maxLength": 100},
 			"category_id":  map[string]any{"type": "string", "enum": candidateIDs(c.categories)},
-			"tag_ids":      map[string]any{"type": "array", "uniqueItems": true, "maxItems": len(tagIDs), "items": tagItems},
+			"tag_ids":      map[string]any{"type": "array", "maxItems": len(tagIDs), "items": tagItems},
 			"confidence":   map[string]any{"type": "string", "enum": []string{"high", "medium", "low"}},
 		},
 	}

internal/classification/client.go

@@ -241,10 +241,13 @@ func (c *Client) Classify(ctx context.Context, facts domain.Facts, data domain.D
 	}
 	if answer.NewMerchant != nil {
 		name := strings.Join(strings.Fields(*answer.NewMerchant), " ")
+		// An identifier-shaped or oversized name is dropped, never stored, but
+		// the row keeps its independently enum-validated category and tags: a
+		// legitimate payee whose spelling trips the redactor (observed in the
+		// field) must not lose its whole classification.
 		if !utf8.ValidString(name) || utf8.RuneCountInString(name) > 100 || normalize(name) == "" || normalize(clean(name)) != normalize(name) {
-			return fail("AI proposed an unsafe merchant name")
-		}
-		if existing := duplicateMerchant(name, data.Merchants); existing != nil {
+			// no merchant
+		} else if existing := duplicateMerchant(name, data.Merchants); existing != nil {
 			e.MerchantID = existing.ID
 		} else {
 			aliases := []string{}
@@ -352,7 +355,22 @@ func (c *Client) complete(ctx context.Context, gate *ratelimit.Controller, r com
 			} `json:"message"`
 		} `json:"choices"`
 	}
-	if json.Unmarshal(raw, &envelope) != nil || (len(envelope.Error) > 0 && string(envelope.Error) != "null") || len(envelope.Choices) != 1 {
+	if json.Unmarshal(raw, &envelope) != nil {
+		return "", errors.New("invalid AI response envelope")
+	}
+	if len(envelope.Error) > 0 && string(envelope.Error) != "null" {
+		// The provider reported a failure inside an HTTP 200 envelope. Only
+		// its numeric code is safe to surface; the message may quote content.
+		var detail struct {
+			Code int `json:"code"`
+		}
+		_ = json.Unmarshal(envelope.Error, &detail)
+		if detail.Code != 0 {
+			return "", fmt.Errorf("AI provider reported an error (code %d)", detail.Code)
+		}
+		return "", errors.New("AI provider reported an error")
+	}
+	if len(envelope.Choices) != 1 {
 		return "", errors.New("invalid AI response envelope")
 	}
 	choice := envelope.Choices[0]

internal/classification/client_test.go

@@ -192,7 +192,7 @@ func TestIdentifierOnlyPromptRedactionAndRouting(t *testing.T) {
 	f.CounterpartyIBAN = "DE89370400440532013000"
 	d.Accounts[0].IBAN = "DE44500105175407324931"
 	d.Accounts[0].ExternalAccountID = "ext_local_secret"
-	f.RawDescription = "Coffee House -918.27 EUR Alice Privateperson DE89 3704 0044 0532 0130 00 private_external private_fingerprint tx_private account_private ext_local_secret private_source Personal Checking Private Bank 550e8400-e29b-41d4-a716-446655440000 COBADEFFXXX ; reference secretpayment ; user@example.com"
+	f.RawDescription = "Coffee House -918.27 EUR Alice Privateperson DE89 3704 0044 0532 0130 00 COBADEFFXXX private_external private_fingerprint tx_private account_private ext_local_secret private_source Personal Checking Private Bank 550e8400-e29b-41d4-a716-446655440000 ; reference secretpayment ; user@example.com"
 	var captured map[string]json.RawMessage
 	c := mockClient(t, func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path != "/chat/completions" || r.Header.Get("Authorization") != "Bearer test-secret" {
@@ -294,16 +294,23 @@ func TestTransactionAmountAndCounterpartyAreSent(t *testing.T) {
 	}
 }
 
-func TestUnsafeMerchantProposalRejected(t *testing.T) {
+func TestUnsafeMerchantProposalDroppedWithoutLosingClassification(t *testing.T) {
 	for _, name := range []string{"Alice Privateperson", "DE89370400440532013000", "Bank 123456789", "reference secretpayment", strings.Repeat("x", 101)} {
 		t.Run(name, func(t *testing.T) {
 			f, d := fixture()
 			f.Counterparty = "Alice Privateperson"
-			answer, _ := json.Marshal(map[string]any{"merchant_id": nil, "new_merchant": name, "category_id": "c1", "tag_ids": []string{}})
+			answer, _ := json.Marshal(map[string]any{"merchant_id": nil, "new_merchant": name, "category_id": "cat_food", "tag_ids": []string{}, "confidence": "high"})
 			c := mockClient(t, func(w http.ResponseWriter, r *http.Request) { reply(w, string(answer)) })
+			c.PrivateNames = []string{"Alice Privateperson"}
 			p, err := c.Classify(context.Background(), f, d, true)
-			if err == nil || p.NewMerchant != nil {
-				t.Fatalf("unsafe merchant accepted: %+v", p)
+			if err != nil {
+				t.Fatalf("unsafe name must degrade, not fail the row: %v", err)
+			}
+			if p.NewMerchant != nil || p.Enrichment.MerchantID != "" {
+				t.Fatalf("unsafe merchant stored: %+v", p)
+			}
+			if p.Enrichment.CategoryID != "cat_food" || p.Enrichment.Classification.Confidence != "high" {
+				t.Fatalf("validated classification lost with the merchant: %+v", p.Enrichment)
 			}
 		})
 	}
@@ -340,9 +347,13 @@ func TestMalformedEnvelopesRejected(t *testing.T) {
 		t.Run(fmt.Sprint(i), func(t *testing.T) {
 			f, d := fixture()
 			c := mockClient(t, func(w http.ResponseWriter, r *http.Request) { _, _ = io.WriteString(w, body) })
-			if p, err := c.Classify(context.Background(), f, d, true); err == nil || p.Enrichment.Classification.Source != "fallback" {
+			p, err := c.Classify(context.Background(), f, d, true)
+			if err == nil || p.Enrichment.Classification.Source != "fallback" {
 				t.Fatalf("bad envelope accepted: %+v %v", p, err)
 			}
+			if strings.Contains(err.Error(), "private") {
+				t.Fatalf("provider text leaked into the error: %v", err)
+			}
 		})
 	}
 }
@@ -435,7 +446,7 @@ func TestConfiguredPrivateNamesAndIdentifiersRedactWithoutRemovingPayee(t *testi
 	f, d := fixture()
 	f.Counterparty = "Coffee House"
 	clean := redactor(d, f, []string{"Alice"})
-	text := clean("Alice Alice Alice Coffee House cobadeffxxx DE89370400440532013000")
+	text := clean("Alice Alice Alice Coffee House DE89370400440532013000 COBADEFFXXX")
 	if strings.Contains(text, "alice") || strings.Contains(text, "cobadeff") || strings.Contains(text, "de893704") || !strings.Contains(text, "coffee house") {
 		t.Fatalf("redaction: %q", text)
 	}

internal/classification/ledger_test.go

@@ -0,0 +1,156 @@
+package classification
+
+import (
+	"context"
+	"net/http"
+	"reflect"
+	"strings"
+	"testing"
+
+	"finance-duck/internal/domain"
+)
+
+// ledgerFixture mirrors a production ledger that repeatedly broke
+// classification in the field: a proposed two-level taxonomy (43 expense
+// leaves), tags, a merchant registry polluted with location-like names, and
+// German bank rows whose payee text carries reference numbers. Personal
+// names and IBANs are fabricated.
+func ledgerFixture() (domain.Dataset, domain.Facts) {
+	d := domain.NewDataset()
+	d.Accounts = []domain.Account{{ID: "acct_kontist", DisplayName: "Business", Institution: "Kontist", Currency: "EUR", Active: true}}
+	tree := map[string][]string{
+		"housing":       {"rent", "utilities", "household", "maintenance"},
+		"food":          {"groceries", "restaurants", "takeaway"},
+		"transport":     {"fuel", "public-transport", "parking", "taxi", "vehicle-maintenance"},
+		"shopping":      {"clothing", "electronics", "household-goods", "other"},
+		"pets":          {"pet-food", "pet-health", "supplies"},
+		"entertainment": {"games", "events", "ent-media"},
+		"travel":        {"accommodation", "travel-transport", "activities"},
+		"health":        {"medical", "pharmacy", "fitness"},
+		"education":     {"tuition", "books", "courses"},
+		"subscriptions": {"software", "sub-media", "services"},
+		"insurance":     {"vehicle-insurance", "health-insurance", "other-insurance"},
+		"financial":     {"bank-fees", "interest-paid", "taxes"},
+		"gifts":         nil,
+		"donations":     nil,
+	}
+	for parent, children := range tree {
+		d.Categories = append(d.Categories, domain.Category{ID: "cat_" + parent, Name: parent, ParentID: "cat_expenses", Kind: "expense"})
+		for _, child := range children {
+			d.Categories = append(d.Categories, domain.Category{ID: "cat_" + child, Name: child, ParentID: "cat_" + parent, Kind: "expense"})
+		}
+	}
+	for _, name := range []string{"personal", "business", "travel", "hobby", "home", "mx5", "education", "gift", "tax-deductible", "subscription", "groceries"} {
+		d.Tags = append(d.Tags, domain.Tag{ID: "tag_" + name, Name: name})
+	}
+	// Location-like junk from a taxonomy proposal run: it must stay selectable
+	// without breaking the strict schema or the alias matcher.
+	for _, name := range []string{"smart steuerservice", "kranken", "Chittaway Bay", "Toronto", "bruhl", "brunico", "St. Ulrich", "Git Server", "Mobilfunk", "Swopper"} {
+		d.Merchants = append(d.Merchants, domain.Merchant{ID: domain.NewID("mer"), Name: name, Aliases: []string{}, DefaultTagIDs: []string{}, UseDefaults: false})
+	}
+	facts := domain.Facts{
+		ID: "tx_finanzamt", Source: "enablebanking", AccountID: "acct_kontist",
+		BookingDate: "2026-08-30", ValueDate: "2026-08-30", Amount: "-849.45", Currency: "EUR",
+		RawDescription: "0904303543105 224/5220/5869",
+		Counterparty:   "Finanzamt Bruehl", CounterpartyIBAN: "DE02120300000000202051",
+		Fingerprint: "f1e2d3",
+	}
+	d.Transactions = []domain.Transaction{{Facts: facts, Enrichment: domain.Fallback(facts)}}
+	return d, facts
+}
+
+// strictKeywords is what OpenAI-family strict structured-output mode accepts.
+// uniqueItems is specifically rejected ("'uniqueItems' is not permitted") and
+// took every zero-data-retention route for those models down with HTTP 400;
+// duplicates are rejected server-side by decodeAnswer instead.
+var strictKeywords = map[string]bool{
+	"type": true, "properties": true, "required": true, "additionalProperties": true,
+	"items": true, "enum": true, "maxItems": true, "maxLength": true, "minLength": true,
+}
+
+func checkStrict(t *testing.T, path string, value any) {
+	t.Helper()
+	switch v := value.(type) {
+	case map[string]any:
+		for key, child := range v {
+			if path == "" || strings.HasSuffix(path, ".properties") {
+				// Property names and the schema root are not keywords.
+			} else if !strictKeywords[key] {
+				t.Errorf("%s uses %q, which strict structured-output mode rejects", path, key)
+			}
+			checkStrict(t, path+"."+key, child)
+		}
+	case []any:
+		for _, child := range v {
+			checkStrict(t, path+"[]", child)
+		}
+	}
+}
+
+func TestWireSchemasUseOnlyStrictModeKeywords(t *testing.T) {
+	d, facts := ledgerFixture()
+	set := retrieve(facts.RawDescription, "expense", d, nil, nil)
+	for name, schema := range map[string]map[string]any{
+		"classification": set.schema(),
+		"taxonomy":       taxonomySchema(),
+		"csv":            csvMappingSchema(CSVMappingRequest{Headers: []string{"Buchung", "Betrag"}}),
+	} {
+		checkStrict(t, name, map[string]any{"properties": schema["properties"]})
+	}
+}
+
+// The exact answer a live gpt-5.6-luna-pro returned for this row over a
+// zero-data-retention route must land as reviewable enrichment: taxes
+// category, a new public merchant seeded with the counterparty alias, no
+// tags, recorded confidence.
+func TestLedgerRowClassifiesThroughStrictSchema(t *testing.T) {
+	d, facts := ledgerFixture()
+	taxes := ""
+	for _, c := range d.Categories {
+		if c.Name == "taxes" {
+			taxes = c.ID
+		}
+	}
+	c := mockClient(t, func(w http.ResponseWriter, r *http.Request) {
+		reply(w, `{"merchant_id":null,"new_merchant":"Finanzamt Bruehl","category_id":"`+taxes+`","tag_ids":[],"confidence":"high"}`)
+	})
+	p, err := c.Classify(context.Background(), facts, d, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if p.Enrichment.CategoryID != taxes || p.Enrichment.Classification.Confidence != "high" {
+		t.Fatalf("classification lost: %+v", p.Enrichment)
+	}
+	if p.NewMerchant == nil || p.NewMerchant.Name != "Finanzamt Bruehl" ||
+		!reflect.DeepEqual(p.NewMerchant.Aliases, []string{"Finanzamt Bruehl"}) {
+		t.Fatalf("merchant proposal lost: %+v", p.NewMerchant)
+	}
+	if len(p.Enrichment.TagIDs) != 0 {
+		t.Fatalf("unexpected tags: %+v", p.Enrichment.TagIDs)
+	}
+}
+
+// Identifier redaction must not eat ordinary 8- and 11-letter payee words,
+// which blinded the model to the merchant it was asked to classify
+// ("WWW.RACETRACKER.DE" became "WWW. .DE"). A bare bank-code-shaped token is
+// vocabulary; real BICs still die labeled or trailing their IBAN.
+func TestBICRedactionKeepsPayeeVocabulary(t *testing.T) {
+	d, facts := ledgerFixture()
+	clean := redactor(d, facts, nil)
+	for _, keep := range []string{"Openbank", "OPENBANK", "Baumarkt", "BAUMARKT", "RACETRACKER", "toom Baumarkt"} {
+		if got := clean(keep); got != normalize(keep) {
+			t.Errorf("payee word %q was redacted to %q", keep, got)
+		}
+	}
+	for name, text := range map[string]string{
+		"labeled iban":  "IBAN DE89370400440532013000 COBADEFFXXX invoice",
+		"trailing bic":  "pay DE89370400440532013000 COBADEFFXXX today",
+		"labeled bic":   "BIC DEUTDEDBFRA",
+		"labeled swift": "SWIFT GENODED1SPO",
+	} {
+		got := clean(text)
+		if strings.Contains(got, "de8937") || strings.Contains(got, "cobadeff") || strings.Contains(got, "deutdedb") || strings.Contains(got, "genoded1") {
+			t.Errorf("%s: identifier survived redaction: %q", name, got)
+		}
+	}
+}

internal/classification/privacy.go

@@ -12,10 +12,15 @@ import (
 
 var bankingPatterns = []*regexp.Regexp{
 	// Apply before tokenization to capture formatted identifiers as a unit.
-	regexp.MustCompile(`(?i)\b[a-z]{2}\s*\d{2}(?:[ -]?[a-z0-9]){11,30}\b`),
+	// An IBAN may carry its BIC as the next token; both go as one unit. A
+	// *bare* BIC-shaped token is deliberately not redacted: the shape matches
+	// every 8- or 11-letter word ("Openbank", "BAUMARKT", "RACETRACKER"),
+	// which blinded the model to the very payee it should classify, and a
+	// bank code reveals nothing the prompt's institution field does not.
+	// Labeled forms ("BIC ...", "SWIFT ...") die with the label below.
+	regexp.MustCompile(`(?i)\b[a-z]{2}\s*\d{2}(?:[ -]?[a-z0-9]){11,30}\b(?:\s+[a-z]{6}[a-z0-9]{2}(?:[a-z0-9]{3})?\b)?`),
 	regexp.MustCompile(`(?i)\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b`),
 	regexp.MustCompile(`(?i)\b(?:iban|bic|swift|account(?:\s*(?:number|no))?|konto(?:nummer)?|reference|ref|payment\s*(?:id|reference)|end\s*to\s*end(?:\s*id)?|e2e|eref|mref|kref|cred|mandate|mandat(?:sreferenz)?|kunden(?:nummer|referenz)|kreditornummer|glaeubiger\s*id|gläubiger\s*id)\b[^;\n|]*`),
-	regexp.MustCompile(`(?i)\b[A-Z]{6}[A-Z0-9]{2}(?:[A-Z0-9]{3})?\b`),
 	regexp.MustCompile(`(?i)\b(?:https?://|www\.)\S+|\b[^\s@]+@[^\s@]+\b`),
 }
 
@@ -54,6 +59,9 @@ func redactor(d domain.Dataset, f domain.Facts, private []string) func(string) s
 		addSecret(secrets, a.ID)
 		addSecret(secrets, a.IBAN)
 		addSecret(secrets, a.ExternalAccountID)
+		// People put their own name in the account label; the label is never
+		// sent as a field and its text is own-identity data, like PrivateNames.
+		addSecret(secrets, a.DisplayName)
 	}
 	for _, value := range []string{f.ID, f.ExternalID, f.Fingerprint, f.CounterpartyIBAN} {
 		addSecret(secrets, value)

internal/classification/propose.go

@@ -65,7 +65,7 @@ func taxonomySchema() map[string]any {
 	merchant := map[string]any{
 		"type": "object", "additionalProperties": false,
 		"required":   []string{"name", "aliases"},
-		"properties": map[string]any{"name": name, "aliases": map[string]any{"type": "array", "maxItems": 32, "uniqueItems": true, "items": name}},
+		"properties": map[string]any{"name": name, "aliases": map[string]any{"type": "array", "maxItems": 32, "items": name}},
 	}
 	return map[string]any{
 		"type": "object", "additionalProperties": false,