Use compact classification IDs and extend preview lifetime
This commit is contained in:
Lars Nolden
@@ -28,7 +28,7 @@ func fixture() (domain.Facts, domain.Dataset) {
|
||||
return f, d
|
||||
}
|
||||
|
||||
const validAnswer = `{"merchant_id":null,"new_merchant":null,"category_id":"cat_food","tag_ids":[],"confidence":"medium"}`
|
||||
const validAnswer = `{"merchant_id":null,"new_merchant":null,"category_id":"c1","tag_ids":[],"confidence":"medium"}`
|
||||
|
||||
func reply(w http.ResponseWriter, content string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
@@ -44,6 +44,39 @@ func mockClient(t *testing.T, handler http.HandlerFunc) *Client {
|
||||
return client
|
||||
}
|
||||
|
||||
type classificationPrompt struct {
|
||||
Categories []categoryPrompt `json:"categories"`
|
||||
Tags []tagPrompt `json:"tags"`
|
||||
Merchants []merchantPrompt `json:"merchants"`
|
||||
History []promptHistory `json:"history"`
|
||||
Transactions []struct {
|
||||
Ref string `json:"ref"`
|
||||
Counterparty string `json:"counterparty"`
|
||||
Amount string `json:"amount"`
|
||||
Currency string `json:"currency"`
|
||||
} `json:"transactions"`
|
||||
}
|
||||
|
||||
func decodeClassificationPrompt(t *testing.T, r *http.Request) classificationPrompt {
|
||||
t.Helper()
|
||||
var req struct {
|
||||
Messages []struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"messages"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(req.Messages) != 2 {
|
||||
t.Fatalf("expected system and user messages, got %d", len(req.Messages))
|
||||
}
|
||||
var prompt classificationPrompt
|
||||
if err := json.Unmarshal([]byte(req.Messages[1].Content), &prompt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return prompt
|
||||
}
|
||||
|
||||
func TestExplicitDefaultsAreOptInAndBypassAI(t *testing.T) {
|
||||
f, d := fixture()
|
||||
d.Merchants[0].UseDefaults = true
|
||||
@@ -72,7 +105,22 @@ func TestForceAIOverridesRuleWithoutChangingKind(t *testing.T) {
|
||||
f, d := fixture()
|
||||
d.Merchants[0].UseDefaults = true
|
||||
calls := 0
|
||||
c := mockClient(t, func(w http.ResponseWriter, r *http.Request) { calls++; reply(w, validAnswer) })
|
||||
c := mockClient(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
prompt := decodeClassificationPrompt(t, r)
|
||||
// Food is an expense-only choice; do not reuse c1 after the request
|
||||
// switches to income, where that reference names a different category.
|
||||
categoryID := "c999"
|
||||
for _, category := range prompt.Categories {
|
||||
if category.Path == normalize(domain.CategoryPath(d, "cat_food")) {
|
||||
categoryID = category.ID
|
||||
}
|
||||
if calls == 2 && category.Kind != "income" {
|
||||
t.Errorf("income request offered an expense category: %+v", category)
|
||||
}
|
||||
}
|
||||
reply(w, fmt.Sprintf(`{"merchant_id":null,"new_merchant":null,"category_id":%q,"tag_ids":[],"confidence":"medium"}`, categoryID))
|
||||
})
|
||||
p, err := c.Classify(context.Background(), f, d, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -82,7 +130,7 @@ func TestForceAIOverridesRuleWithoutChangingKind(t *testing.T) {
|
||||
}
|
||||
f.Amount = "918.27"
|
||||
p, err = c.Classify(context.Background(), f, d, true)
|
||||
if err == nil || p.Enrichment.Kind != "income" || p.Enrichment.CategoryID != domain.IncomeFallback {
|
||||
if err == nil || calls != 2 || p.Enrichment.Kind != "income" || p.Enrichment.CategoryID != domain.IncomeFallback {
|
||||
t.Fatalf("income sign: %+v %v", p, err)
|
||||
}
|
||||
}
|
||||
@@ -117,21 +165,24 @@ func TestTransferNeverCallsAIOrAliases(t *testing.T) {
|
||||
|
||||
func TestInvalidModelOutputsFailClosed(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"unknown key": `{"merchant_id":null,"new_merchant":null,"category_id":"c1","tag_ids":[],"confidence":0.9}`,
|
||||
"change kind": `{"merchant_id":null,"new_merchant":null,"category_id":"c1","tag_ids":[],"kind":"transfer"}`,
|
||||
"missing field": `{"merchant_id":null,"category_id":"c1","tag_ids":[]}`,
|
||||
"duplicate key": `{"merchant_id":null,"new_merchant":null,"category_id":"c1","category_id":"c2","tag_ids":[]}`,
|
||||
"case folded key": `{"Merchant_ID":null,"new_merchant":null,"category_id":"c1","tag_ids":[]}`,
|
||||
"unknown category": `{"merchant_id":null,"new_merchant":null,"category_id":"cat_invented","tag_ids":[]}`,
|
||||
"real ID not offered": `{"merchant_id":null,"new_merchant":null,"category_id":"cat_food","tag_ids":[]}`,
|
||||
"unknown tag": `{"merchant_id":null,"new_merchant":null,"category_id":"c1","tag_ids":["t999"]}`,
|
||||
"duplicate tags": `{"merchant_id":null,"new_merchant":null,"category_id":"c1","tag_ids":["t1","t1"]}`,
|
||||
"null tags": `{"merchant_id":null,"new_merchant":null,"category_id":"c1","tag_ids":null}`,
|
||||
"null tag member": `{"merchant_id":null,"new_merchant":null,"category_id":"c1","tag_ids":[null]}`,
|
||||
"unknown merchant": `{"merchant_id":"m999","new_merchant":null,"category_id":"c1","tag_ids":[]}`,
|
||||
"both merchant modes": `{"merchant_id":"m1","new_merchant":"Coffee","category_id":"c1","tag_ids":[]}`,
|
||||
"blank proposal": `{"merchant_id":null,"new_merchant":" ","category_id":"c1","tag_ids":[]}`,
|
||||
"wrong scalar": `{"merchant_id":23,"new_merchant":null,"category_id":"c1","tag_ids":[]}`,
|
||||
"unknown key": `{"merchant_id":null,"new_merchant":null,"category_id":"c1","tag_ids":[],"confidence":"high","unexpected":true}`,
|
||||
"change kind": `{"merchant_id":null,"new_merchant":null,"category_id":"c1","tag_ids":[],"confidence":"high","kind":"transfer"}`,
|
||||
"missing field": `{"merchant_id":null,"category_id":"c1","tag_ids":[],"confidence":"high"}`,
|
||||
"duplicate key": `{"merchant_id":null,"new_merchant":null,"category_id":"c1","category_id":"c2","tag_ids":[],"confidence":"high"}`,
|
||||
"case folded key": `{"Merchant_ID":null,"new_merchant":null,"category_id":"c1","tag_ids":[],"confidence":"high"}`,
|
||||
"unknown category": `{"merchant_id":null,"new_merchant":null,"category_id":"c999","tag_ids":[],"confidence":"high"}`,
|
||||
"canonical category": `{"merchant_id":null,"new_merchant":null,"category_id":"cat_food","tag_ids":[],"confidence":"high"}`,
|
||||
"unknown tag": `{"merchant_id":null,"new_merchant":null,"category_id":"c1","tag_ids":["t999"],"confidence":"high"}`,
|
||||
"canonical tag": `{"merchant_id":null,"new_merchant":null,"category_id":"c1","tag_ids":["tag_daily"],"confidence":"high"}`,
|
||||
"duplicate tags": `{"merchant_id":null,"new_merchant":null,"category_id":"c1","tag_ids":["t1","t1"],"confidence":"high"}`,
|
||||
"null tags": `{"merchant_id":null,"new_merchant":null,"category_id":"c1","tag_ids":null,"confidence":"high"}`,
|
||||
"null tag member": `{"merchant_id":null,"new_merchant":null,"category_id":"c1","tag_ids":[null],"confidence":"high"}`,
|
||||
"unknown merchant": `{"merchant_id":"m999","new_merchant":null,"category_id":"c1","tag_ids":[],"confidence":"high"}`,
|
||||
"canonical merchant": `{"merchant_id":"mer_coffee","new_merchant":null,"category_id":"c1","tag_ids":[],"confidence":"high"}`,
|
||||
"both merchant modes": `{"merchant_id":"m1","new_merchant":"Coffee","category_id":"c1","tag_ids":[],"confidence":"high"}`,
|
||||
"blank proposal": `{"merchant_id":null,"new_merchant":" ","category_id":"c1","tag_ids":[],"confidence":"high"}`,
|
||||
"wrong scalar": `{"merchant_id":23,"new_merchant":null,"category_id":"c1","tag_ids":[],"confidence":"high"}`,
|
||||
"numeric confidence": `{"merchant_id":null,"new_merchant":null,"category_id":"c1","tag_ids":[],"confidence":0.9}`,
|
||||
"trailing JSON": validAnswer + ` {}`,
|
||||
"markdown": "```json\n" + validAnswer + "\n```",
|
||||
"array": "[" + validAnswer + "]",
|
||||
@@ -157,9 +208,9 @@ func TestMerchantSelectionAndLocalProposal(t *testing.T) {
|
||||
name, content, merchant string
|
||||
new bool
|
||||
}{
|
||||
{"existing", `{"merchant_id":"mer_coffee","new_merchant":null,"category_id":"cat_food","tag_ids":["tag_daily"],"confidence":"high"}`, "mer_coffee", false},
|
||||
{"duplicate alias", `{"merchant_id":null,"new_merchant":"COFFEE-house","category_id":"cat_food","tag_ids":["tag_daily"],"confidence":"high"}`, "mer_coffee", false},
|
||||
{"new", `{"merchant_id":null,"new_merchant":"Bakery Lane","category_id":"cat_food","tag_ids":["tag_daily"],"confidence":"high"}`, "", true},
|
||||
{"existing", `{"merchant_id":"m1","new_merchant":null,"category_id":"c1","tag_ids":["t1"],"confidence":"high"}`, "mer_coffee", false},
|
||||
{"duplicate alias", `{"merchant_id":null,"new_merchant":"COFFEE-house","category_id":"c1","tag_ids":["t1"],"confidence":"high"}`, "mer_coffee", false},
|
||||
{"new", `{"merchant_id":null,"new_merchant":"Bakery Lane","category_id":"c1","tag_ids":["t1"],"confidence":"high"}`, "", true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
@@ -216,18 +267,14 @@ func TestIdentifierOnlyPromptRedactionAndRouting(t *testing.T) {
|
||||
if len(messages) != 2 {
|
||||
t.Fatal("unexpected messages")
|
||||
}
|
||||
var prompt struct {
|
||||
Transaction map[string]any `json:"transaction"`
|
||||
History []any `json:"history"`
|
||||
Categories []any `json:"categories"`
|
||||
Tags []any `json:"tags"`
|
||||
Merchants []any `json:"merchants"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(messages[1].Content), &prompt); err != nil {
|
||||
wire, err := json.Marshal(captured)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(prompt.Transaction) == 0 || len(prompt.Categories) == 0 || len(prompt.Merchants) == 0 {
|
||||
t.Fatal("complete structured prompt missing")
|
||||
for _, canonicalID := range []string{"cat_food", "cat_expenses", "cat_income", "mer_coffee", "tag_daily"} {
|
||||
if strings.Contains(string(wire), canonicalID) {
|
||||
t.Errorf("request or response schema exposed canonical ID %q", canonicalID)
|
||||
}
|
||||
}
|
||||
lower := strings.ToLower(messages[1].Content)
|
||||
for _, secret := range []string{"private_external", "private_fingerprint", "tx_private", "account_private", "ext_local_secret", "private_source", "personal checking", "550e8400", "cobadeff", "secretpayment", "example.com", "alice privateperson", "de89370400440532013000", "de44500105175407324931"} {
|
||||
@@ -300,7 +347,7 @@ func TestUnsafeMerchantProposalDroppedWithoutLosingClassification(t *testing.T)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f, d := fixture()
|
||||
f.Counterparty = "Alice Privateperson"
|
||||
answer, _ := json.Marshal(map[string]any{"merchant_id": nil, "new_merchant": name, "category_id": "cat_food", "tag_ids": []string{}, "confidence": "high"})
|
||||
answer, _ := json.Marshal(map[string]any{"merchant_id": nil, "new_merchant": name, "category_id": "c1", "tag_ids": []string{}, "confidence": "high"})
|
||||
c := mockClient(t, func(w http.ResponseWriter, r *http.Request) { reply(w, string(answer)) })
|
||||
c.PrivateNames = []string{"Alice Privateperson"}
|
||||
p, err := c.Classify(context.Background(), f, d, true)
|
||||
@@ -410,33 +457,67 @@ func TestCompleteRegistryPayloadAndGlobalDuplicateDetection(t *testing.T) {
|
||||
d.Tags = append(d.Tags, domain.Tag{ID: fmt.Sprintf("tag_%02d", i), Name: fmt.Sprintf("Tag %02d", i)})
|
||||
d.Categories = append(d.Categories, domain.Category{ID: fmt.Sprintf("cat_%02d", i), Name: fmt.Sprintf("Category %02d", i), Kind: "expense", ParentID: "cat_expenses"})
|
||||
}
|
||||
d.Merchants[34].Name = "Distant Bakery"
|
||||
set := retrieve(f.RawDescription, "expense", d, redactor(d, f, nil), redactor(d, f, nil))
|
||||
if len(set.merchantIDs) != 35 || len(set.tags) != 36 {
|
||||
t.Fatalf("complete registry omitted entries: merchants=%d tags=%d", len(set.merchantIDs), len(set.tags))
|
||||
}
|
||||
if set.merchantIDs["mer_34"] != "mer_34" ||
|
||||
set.tagIDs["tag_34"] != "tag_34" ||
|
||||
set.categoryIDs["cat_34"] != "cat_34" {
|
||||
t.Fatal("registry omitted real ids")
|
||||
}
|
||||
d.Merchants[34].Name = "Z Distant Bakery"
|
||||
before := domain.Clone(d)
|
||||
c := mockClient(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
content, _ := json.Marshal(map[string]any{
|
||||
"merchant_id": "mer_34",
|
||||
"new_merchant": nil,
|
||||
"category_id": "cat_34",
|
||||
"tag_ids": []string{"tag_34"},
|
||||
"confidence": "high",
|
||||
for _, mode := range []string{"existing", "duplicate name"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
c := mockClient(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
prompt := decodeClassificationPrompt(t, r)
|
||||
if len(prompt.Merchants) != 35 || len(prompt.Tags) != 36 || len(prompt.Categories) != 37 {
|
||||
t.Fatalf("complete candidates missing: merchants=%d tags=%d categories=%d", len(prompt.Merchants), len(prompt.Tags), len(prompt.Categories))
|
||||
}
|
||||
merchants, categories, tags := map[string]string{}, map[string]string{}, map[string]string{}
|
||||
for _, merchant := range prompt.Merchants {
|
||||
merchants[merchant.Name] = merchant.ID
|
||||
}
|
||||
for _, category := range prompt.Categories {
|
||||
if category.Kind != "expense" {
|
||||
t.Errorf("ineligible category candidate: %+v", category)
|
||||
}
|
||||
categories[category.Path] = category.ID
|
||||
}
|
||||
for _, tag := range prompt.Tags {
|
||||
tags[tag.Name] = tag.ID
|
||||
}
|
||||
for _, merchant := range d.Merchants {
|
||||
if merchants[normalize(merchant.Name)] == "" {
|
||||
t.Errorf("merchant omitted: %s", merchant.Name)
|
||||
}
|
||||
}
|
||||
for _, category := range d.Categories {
|
||||
if category.Kind == "expense" && category.ID != "cat_expenses" && categories[normalize(domain.CategoryPath(d, category.ID))] == "" {
|
||||
t.Errorf("eligible category omitted: %s", category.Name)
|
||||
}
|
||||
}
|
||||
for _, tag := range d.Tags {
|
||||
if tags[normalize(tag.Name)] == "" {
|
||||
t.Errorf("tag omitted: %s", tag.Name)
|
||||
}
|
||||
}
|
||||
var merchantID, newMerchant any = merchants["z distant bakery"], nil
|
||||
if mode == "duplicate name" {
|
||||
merchantID, newMerchant = nil, "Z Distant Bakery"
|
||||
}
|
||||
content, err := json.Marshal(map[string]any{
|
||||
"merchant_id": merchantID,
|
||||
"new_merchant": newMerchant,
|
||||
"category_id": categories[normalize(domain.CategoryPath(d, "cat_34"))],
|
||||
"tag_ids": []string{tags["tag 34"]},
|
||||
"confidence": "high",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reply(w, string(content))
|
||||
})
|
||||
p, err := c.Classify(context.Background(), f, d, true)
|
||||
if err != nil || p.NewMerchant != nil || p.Enrichment.MerchantID != "mer_34" || p.Enrichment.CategoryID != "cat_34" || !reflect.DeepEqual(p.Enrichment.TagIDs, []string{"tag_34"}) {
|
||||
t.Fatalf("complete registry selection failed: %+v %v", p, err)
|
||||
}
|
||||
if !reflect.DeepEqual(before, d) {
|
||||
t.Fatal("classification mutated the dataset")
|
||||
}
|
||||
})
|
||||
reply(w, string(content))
|
||||
})
|
||||
p, err := c.Classify(context.Background(), f, d, true)
|
||||
if err != nil || p.Enrichment.MerchantID != "mer_34" || p.Enrichment.CategoryID != "cat_34" || !reflect.DeepEqual(p.Enrichment.TagIDs, []string{"tag_34"}) {
|
||||
t.Fatalf("complete registry selection failed: %+v %v", p, err)
|
||||
}
|
||||
if !reflect.DeepEqual(before, d) {
|
||||
t.Fatal("retrieval mutated registry order")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -480,7 +561,7 @@ func TestConfiguredPrivateNamesAndIdentifiersRedactWithoutRemovingPayee(t *testi
|
||||
func TestLowConfidenceKeepsProposalAndRecordsConfidence(t *testing.T) {
|
||||
f, d := fixture()
|
||||
c := mockClient(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
reply(w, `{"merchant_id":"mer_coffee","new_merchant":null,"category_id":"cat_food","tag_ids":["tag_daily"],"confidence":"low"}`)
|
||||
reply(w, `{"merchant_id":"m1","new_merchant":null,"category_id":"c1","tag_ids":["t1"],"confidence":"low"}`)
|
||||
})
|
||||
p, err := c.Classify(context.Background(), f, d, true)
|
||||
if err != nil {
|
||||
@@ -547,7 +628,7 @@ func TestPayeeAndPublicMerchantAreSentToAI(t *testing.T) {
|
||||
Description string `json:"description"`
|
||||
Counterparty string `json:"counterparty"`
|
||||
} `json:"transaction"`
|
||||
Merchants []candidate `json:"merchants"`
|
||||
Merchants []merchantPrompt `json:"merchants"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(req.Messages[1].Content), &prompt); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -558,7 +639,7 @@ func TestPayeeAndPublicMerchantAreSentToAI(t *testing.T) {
|
||||
if len(prompt.Merchants) != 26 || prompt.Merchants[0].Name != "coffee house" {
|
||||
t.Fatalf("complete merchant registry missing: %d", len(prompt.Merchants))
|
||||
}
|
||||
reply(w, `{"merchant_id":"mer_coffee","new_merchant":null,"category_id":"cat_food","tag_ids":[],"confidence":"high"}`)
|
||||
reply(w, fmt.Sprintf(`{"merchant_id":%q,"new_merchant":null,"category_id":"c1","tag_ids":[],"confidence":"high"}`, prompt.Merchants[0].ID))
|
||||
})
|
||||
p, err := c.Classify(context.Background(), f, d, true)
|
||||
if err != nil || p.Enrichment.MerchantID != "mer_coffee" {
|
||||
|
||||
Reference in New Issue
Block a user