lars/finance-duck
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make OpenAI-family strict mode routable and stop redacting payee words

Browse Source 
Strict structured-output mode rejects uniqueItems, so every request to a
gpt-5.6-family zero-data-retention endpoint failed with HTTP 400 behind
a generic error; duplicates were already rejected server-side, so the
keyword leaves the wire schemas, pinned by a strict-keyword allowlist
test built from the ledger that hit this.

The bare-BIC redaction pattern deleted every 8- and 11-letter word —
Openbank, BAUMARKT, RACETRACKER — blinding the model to the payee it
was asked to classify and tripping the unsafe-merchant check on honest
answers. BICs now die only labeled or attached to their IBAN, account
labels join the redaction secrets, an identifier-shaped merchant name
degrades to a merchant-less proposal instead of failing the row, and a
provider error inside an HTTP 200 envelope is reported as such (numeric
code only) instead of as envelope corruption.
This commit is contained in:
Lars Nolden
2026-09-12 23:17:24 +02:00
parent ec99434002
commit c5999adb1b
8 changed files with 223 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/classification/client_test.go
+18 -7
View File
@@ -192,7 +192,7 @@ func TestIdentifierOnlyPromptRedactionAndRouting(t *testing.T) {
f.CounterpartyIBAN = "DE89370400440532013000"
d.Accounts[0].IBAN = "DE44500105175407324931"
d.Accounts[0].ExternalAccountID = "ext_local_secret"
f.RawDescription = "Coffee House -918.27 EUR Alice Privateperson DE89 3704 0044 0532 0130 00 private_external private_fingerprint tx_private account_private ext_local_secret private_source Personal Checking Private Bank 550e8400-e29b-41d4-a716-446655440000 COBADEFFXXX ; reference secretpayment ; user@example.com"
f.RawDescription = "Coffee House -918.27 EUR Alice Privateperson DE89 3704 0044 0532 0130 00 COBADEFFXXX private_external private_fingerprint tx_private account_private ext_local_secret private_source Personal Checking Private Bank 550e8400-e29b-41d4-a716-446655440000 ; reference secretpayment ; user@example.com"
var captured map[string]json.RawMessage
c := mockClient(t, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/chat/completions" || r.Header.Get("Authorization") != "Bearer test-secret" {
@@ -294,16 +294,23 @@ func TestTransactionAmountAndCounterpartyAreSent(t *testing.T) {
}
}
func TestUnsafeMerchantProposalRejected(t *testing.T) {
func TestUnsafeMerchantProposalDroppedWithoutLosingClassification(t *testing.T) {
for _, name := range []string{"Alice Privateperson", "DE89370400440532013000", "Bank 123456789", "reference secretpayment", strings.Repeat("x", 101)} {
t.Run(name, func(t *testing.T) {
f, d := fixture()
f.Counterparty = "Alice Privateperson"
answer, _ := json.Marshal(map[string]any{"merchant_id": nil, "new_merchant": name, "category_id": "c1", "tag_ids": []string{}})
answer, _ := json.Marshal(map[string]any{"merchant_id": nil, "new_merchant": name, "category_id": "cat_food", "tag_ids": []string{}, "confidence": "high"})
c := mockClient(t, func(w http.ResponseWriter, r *http.Request) { reply(w, string(answer)) })
c.PrivateNames = []string{"Alice Privateperson"}
p, err := c.Classify(context.Background(), f, d, true)
if err == nil || p.NewMerchant != nil {
t.Fatalf("unsafe merchant accepted: %+v", p)
if err != nil {
t.Fatalf("unsafe name must degrade, not fail the row: %v", err)
}
if p.NewMerchant != nil || p.Enrichment.MerchantID != "" {
t.Fatalf("unsafe merchant stored: %+v", p)
}
if p.Enrichment.CategoryID != "cat_food" || p.Enrichment.Classification.Confidence != "high" {
t.Fatalf("validated classification lost with the merchant: %+v", p.Enrichment)
}
})
}
@@ -340,9 +347,13 @@ func TestMalformedEnvelopesRejected(t *testing.T) {
t.Run(fmt.Sprint(i), func(t *testing.T) {
f, d := fixture()
c := mockClient(t, func(w http.ResponseWriter, r *http.Request) { _, _ = io.WriteString(w, body) })
if p, err := c.Classify(context.Background(), f, d, true); err == nil || p.Enrichment.Classification.Source != "fallback" {
p, err := c.Classify(context.Background(), f, d, true)
if err == nil || p.Enrichment.Classification.Source != "fallback" {
t.Fatalf("bad envelope accepted: %+v %v", p, err)
}
if strings.Contains(err.Error(), "private") {
t.Fatalf("provider text leaked into the error: %v", err)
}
})
}
}
@@ -435,7 +446,7 @@ func TestConfiguredPrivateNamesAndIdentifiersRedactWithoutRemovingPayee(t *testi
f, d := fixture()
f.Counterparty = "Coffee House"
clean := redactor(d, f, []string{"Alice"})
text := clean("Alice Alice Alice Coffee House cobadeffxxx DE89370400440532013000")
text := clean("Alice Alice Alice Coffee House DE89370400440532013000 COBADEFFXXX")
if strings.Contains(text, "alice") || strings.Contains(text, "cobadeff") || strings.Contains(text, "de893704") || !strings.Contains(text, "coffee house") {
t.Fatalf("redaction: %q", text)
}