diff --git a/OPERATIONS.txt b/OPERATIONS.txt index da8562f..230f3ff 100644 --- a/OPERATIONS.txt +++ b/OPERATIONS.txt @@ -115,6 +115,14 @@ save preferences. Replace key rotates it; Remove key disables AI. No SSH, Nix rebuild or restart is needed. Changes affect future classification requests; in-flight requests retain their original key. +"Classify newly imported transactions with AI" in Classification preferences +controls whether imports contact the provider at all. It applies to CSV imports +and bank synchronization, defaults to on, and is stored as classify_on_import +in config.toml. With it off, no import makes a provider request: enabled +merchant-default rules still classify, an alias match still attaches its +merchant, and everything else arrives on the editable fallback with no +provenance error. AI classification -> Analyse is unaffected by this preference. + The key is stored as private 0600 plaintext in state/openrouter.json under the data directory, never in config.toml or browser storage. API responses never return the saved key. Backups of the data directory contain this secret. @@ -354,7 +362,7 @@ count as income/spending. Populate local account IBANs to support recognition. Canonical files and recovery ---------------------------- finance/ - config.toml model/amount opt-in only, no API keys + config.toml model/amount/import opt-in only, no API keys accounts.finance categories.finance tags.finance diff --git a/README.md b/README.md index 2243bfd..a28e71f 100644 --- a/README.md +++ b/README.md @@ -375,6 +375,8 @@ For administrator-managed startup configuration, `OPENROUTER_API_KEY` in the ser Bank synchronization and recognized N26, ING, and Kontist CSV imports do **not** require this key; only mapping an unrecognized CSV layout does. Without AI, explicit merchant-default rules still work; unresolved transactions remain unclassified and editable. +**Classify newly imported transactions with AI** under **Classification preferences** controls whether importing contacts the provider at all. It covers CSV imports and bank synchronization, is on by default, and is stored as `classify_on_import` in `config.toml`. With it off, no import makes a provider request: enabled merchant rules still classify, and everything else arrives unclassified and editable without a failure that would suggest the provider was unreachable. **AI classification → Analyse** still works on demand, so you can review a batch deliberately instead of on every import. + Every AI classification requests `provider.data_collection = "deny"`, `provider.zdr = true`, and `provider.require_parameters = true`. Unsupported private routing fails rather than falling back to a less restrictive provider. Amount sharing is off by default. Keep OpenRouter account prompt logging disabled as well. Automatic redaction minimizes data; it is not a guarantee that arbitrary transaction prose is anonymous. Classification spaces request starts by at least **three seconds**, including successful requests, rather than sending a burst between 429s. This is a conservative application policy, not a published quota for every model. On HTTP 429, backoff starts at **15 seconds** and increases across consecutive failures; `Retry-After` seconds or HTTP dates can extend the wait. Successful retries retain the learned spacing (up to **30 seconds**) instead of immediately bursting again. Each operation makes at most **four attempts**, with at most **two minutes of automatic retry waiting**, preserving the same model, sanitized prompt, and privacy controls. Imports and previews share this pacing and cooldown. Long or exhausted limits leave records unclassified with a retry-time error; local merchant rules still work. After the cooldown, run **AI classification → Analyse** again for previously failed records—repeating a bank import does not reclassify existing transactions. diff --git a/internal/app/app.go b/internal/app/app.go index 42ff61c..45ff4c0 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -20,9 +20,13 @@ import ( "finance-duck/internal/journal" ) +// Settings holds preferences only, never credentials. ClassifyOnImport controls +// whether newly imported transactions are sent to the model at all; merchant +// rules always apply. type Settings struct { - Model string `json:"model"` - IncludeAmount bool `json:"include_amount"` + Model string `json:"model"` + IncludeAmount bool `json:"include_amount"` + ClassifyOnImport bool `json:"classify_on_import"` } type Status struct { SyncError string `json:"sync_error"` @@ -74,6 +78,9 @@ func Open(dir string) (*App, error) { return nil, err } a := &App{dir: dir, journal: j, previews: make(map[string]Preview), csvImports: make(map[string]CSVImport), authStates: make(map[string]authorization), syncRequested: make(chan struct{}, 1)} + // Configurations written before this preference existed keep classifying + // imports; only an explicit key switches it off. + a.settings.ClassifyOnImport = true fail := func(e error) (*App, error) { j.Close(); return nil, e } if err = os.MkdirAll(filepath.Join(dir, "state"), 0700); err != nil { return fail(err) @@ -98,6 +105,8 @@ func Open(dir string) (*App, error) { a.settings.Model, err = strconv.Unquote(v) case "include_amount": a.settings.IncludeAmount, err = strconv.ParseBool(v) + case "classify_on_import": + a.settings.ClassifyOnImport, err = strconv.ParseBool(v) default: err = fmt.Errorf("unknown setting %q", k) } @@ -326,7 +335,10 @@ func (a *App) SaveSettings(ctx context.Context, s Settings) (State, error) { if len(s.Model) > 200 { return State{}, errors.New("model name is too long") } - b := []byte("# Preferences only. Manage secrets in Settings or environment variables, never this file.\nclassification_model = " + strconv.Quote(s.Model) + "\ninclude_amount = " + strconv.FormatBool(s.IncludeAmount) + "\n") + b := []byte("# Preferences only. Manage secrets in Settings or environment variables, never this file.\n" + + "classification_model = " + strconv.Quote(s.Model) + "\n" + + "include_amount = " + strconv.FormatBool(s.IncludeAmount) + "\n" + + "classify_on_import = " + strconv.FormatBool(s.ClassifyOnImport) + "\n") if err := atomicFile(filepath.Join(a.dir, "config.toml"), b); err != nil { return State{}, err } diff --git a/internal/app/csv_import_test.go b/internal/app/csv_import_test.go index afa374e..1701284 100644 --- a/internal/app/csv_import_test.go +++ b/internal/app/csv_import_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "reflect" "strings" + "sync/atomic" "testing" "finance-duck/internal/classification" @@ -17,6 +18,9 @@ const n26Statement = "Booking Date,Value Date,Partner Name,Partner IBAN,Type,Pay "2026-09-01,2026-08-31,Cafe,DE02120300000000202051,Card,Lunch,Main,-12.30\n" + "2026-09-02,2026-09-02,Employer,DE89370400440532013000,Transfer,Salary,Main,2400.00\n" +const laterStatement = "Booking Date,Value Date,Partner Name,Partner IBAN,Type,Payment Reference,Account Name,Amount (EUR)\n" + + "2026-10-01,2026-10-01,Bakery,DE02120300000000202051,Card,Breakfast,Main,-4.50\n" + // unmappedStatement uses columns no preset recognizes, so it needs a proposed // mapping: German booking dates, split Soll/Haben money and a free-text note. const unmappedStatement = "Datum;Empfänger;Soll;Haben;Notiz\n" + @@ -300,3 +304,96 @@ func TestConfirmationRequiresTheReviewedJournalRevision(t *testing.T) { t.Fatalf("rejected confirmations imported %d transactions", len(current.Data.Transactions)) } } + +// Switching AI classification off for imports must stop every provider call +// while leaving deterministic merchant rules, and the imported facts, intact. +func TestAIClassificationOnImportCanBeSwitchedOff(t *testing.T) { + a, s := testApp(t) + var calls atomic.Int32 + mockClassifier(t, a, func(*http.Request) { calls.Add(1) }) + ctx := context.Background() + s, err := a.Mutate(ctx, s.Revision, func(d *domain.Dataset) error { + d.Merchants = append(d.Merchants, domain.Merchant{ + ID: "mer_cafe", Name: "Cafe", Aliases: []string{}, + DefaultCategoryID: "groceries", DefaultTagIDs: []string{"home"}, UseDefaults: true, + }) + return nil + }) + if err != nil { + t.Fatal(err) + } + if !s.Settings.ClassifyOnImport { + t.Fatal("imports must classify by default") + } + s, err = a.SaveSettings(ctx, Settings{Model: "test/model", ClassifyOnImport: false}) + if err != nil { + t.Fatal(err) + } + if s.Settings.ClassifyOnImport { + t.Fatal("saved preference did not switch classification off") + } + prepared := prepare(t, a, s, n26Statement) + result, err := a.ConfirmCSVImport(ctx, prepared.ID, prepared.Revision) + if err != nil { + t.Fatal(err) + } + if result.Imported != 2 || calls.Load() != 0 { + t.Fatalf("import contacted the provider %d times with classification off", calls.Load()) + } + for _, tx := range result.State.Data.Transactions { + switch tx.Facts.RawDescription { + case "Lunch": + if tx.Enrichment.MerchantID != "mer_cafe" || tx.Enrichment.CategoryID != "groceries" || + !reflect.DeepEqual(tx.Enrichment.TagIDs, []string{"home"}) || tx.Enrichment.Classification.Source != "rule" { + t.Fatalf("merchant rule did not apply without AI: %+v", tx.Enrichment) + } + default: + // No rule and no AI is not a failure: the record stays editable + // without a provenance error suggesting the provider failed. + if tx.Enrichment.CategoryID != domain.IncomeFallback || tx.Enrichment.Classification.Source != "fallback" || tx.Enrichment.Classification.Error != "" { + t.Fatalf("unclassified import reported a failure: %+v", tx.Enrichment) + } + } + } + + // The preference is a stored preference, not a session flag. + a = reopenBankingApp(t, a) + mockClassifier(t, a, func(*http.Request) { calls.Add(1) }) + restarted, err := a.Snapshot(ctx) + if err != nil { + t.Fatal(err) + } + if restarted.Settings.ClassifyOnImport || restarted.Settings.Model != "test/model" { + t.Fatalf("preferences did not survive restart: %+v", restarted.Settings) + } + prepared = prepare(t, a, restarted, laterStatement) + if _, err = a.ConfirmCSVImport(ctx, prepared.ID, prepared.Revision); err != nil { + t.Fatal(err) + } + if calls.Load() != 0 { + t.Fatalf("restarted import contacted the provider %d times", calls.Load()) + } + + // Switching it back on classifies subsequent imports again. + enabled, err := a.SaveSettings(ctx, Settings{Model: "test/model", ClassifyOnImport: true}) + if err != nil { + t.Fatal(err) + } + prepared = prepare(t, a, enabled, "Booking Date,Partner Name,Type,Payment Reference,Amount (EUR)\n2026-11-02,Market,Card,REWE Groceries,-21.40\n") + final, err := a.ConfirmCSVImport(ctx, prepared.ID, prepared.Revision) + if err != nil { + t.Fatal(err) + } + if calls.Load() != 1 { + t.Fatalf("re-enabled import made %d provider requests", calls.Load()) + } + classified := false + for _, tx := range final.State.Data.Transactions { + if tx.Facts.RawDescription == "REWE Groceries" { + classified = tx.Enrichment.Classification.Source == "openrouter" && tx.Enrichment.CategoryID == "groceries" + } + } + if !classified { + t.Fatal("re-enabled classification did not reach the new transaction") + } +} diff --git a/internal/app/import.go b/internal/app/import.go index f2dcd48..9be2855 100644 --- a/internal/app/import.go +++ b/internal/app/import.go @@ -58,7 +58,12 @@ func (a *App) importFacts(ctx context.Context, s State, facts []domain.Facts) (I if !ids[t.Facts.ID] || t.Enrichment.Kind == "transfer" { continue } - p, e := a.classifier.Classify(ctx, t.Facts, s.Data, false) + // With AI classification off for imports, no provider is contacted at + // all: deterministic merchant rules still apply. + p, e := classification.Rules(t.Facts, s.Data) + if a.settings.ClassifyOnImport { + p, e = a.classifier.Classify(ctx, t.Facts, s.Data, false) + } if e == nil { e = addProposal(&s.Data, p) } diff --git a/internal/classification/client.go b/internal/classification/client.go index 1d8e0a7..16462cb 100644 --- a/internal/classification/client.go +++ b/internal/classification/client.go @@ -76,17 +76,63 @@ type Proposal struct { NewMerchant *domain.Merchant `json:"new_merchant,omitempty"` } -// Classify returns a safe fallback with error provenance on any AI failure. Callers -// must check the error before applying a proposal. No provider response is logged. -func (c *Client) Classify(ctx context.Context, facts domain.Facts, data domain.Dataset, forceAI bool) (Proposal, error) { +// ruleProposal applies deterministic local classification: an existing transfer +// keeps its enrichment, and a matching merchant alias contributes that merchant +// plus, only when the merchant opts in, its default category and tags. done +// reports that no provider call can improve the result. +func ruleProposal(facts domain.Facts, data domain.Dataset, forceAI bool) (Proposal, bool, error) { for _, tx := range data.Transactions { if tx.Facts.ID == facts.ID && tx.Enrichment.Kind == "transfer" { e := tx.Enrichment e.TagIDs = append([]string{}, e.TagIDs...) - return Proposal{Enrichment: e}, nil + return Proposal{Enrichment: e}, true, nil } } p := Proposal{Enrichment: domain.Fallback(facts)} + fail := func(message string) (Proposal, bool, error) { + p.Enrichment = domain.Fallback(facts) + p.Enrichment.Classification = domain.Provenance{Source: "fallback", Timestamp: time.Now().UTC().Format(time.RFC3339), Error: message} + return p, true, errors.New(message) + } + if _, err := facts.Amount.Minor(); err != nil { + return fail("invalid transaction amount") + } + merchant := aliasMatch(facts.RawDescription+" "+facts.Counterparty, data.Merchants) + if merchant == nil || forceAI { + return p, false, nil + } + p.Enrichment.MerchantID = merchant.ID + p.Enrichment.Classification = domain.Provenance{Source: "rule", Timestamp: time.Now().UTC().Format(time.RFC3339)} + if !merchant.UseDefaults { + // The alias identifies the merchant; only an opted-in rule may classify. + return p, false, nil + } + if merchant.DefaultCategoryID != "" { + p.Enrichment.CategoryID = merchant.DefaultCategoryID + } + p.Enrichment.TagIDs = append([]string{}, merchant.DefaultTagIDs...) + if err := domain.ValidateEnrichment(data, facts, p.Enrichment); err != nil { + return fail("merchant defaults are invalid for this transaction") + } + return p, true, nil +} + +// Rules classifies without contacting any provider. Imports use it when AI +// classification on import is switched off: merchant alias rules still apply, +// and everything else stays on the editable fallback without a failure that +// would suggest the provider was unreachable. +func Rules(facts domain.Facts, data domain.Dataset) (Proposal, error) { + p, _, err := ruleProposal(facts, data, false) + return p, err +} + +// Classify returns a safe fallback with error provenance on any AI failure. Callers +// must check the error before applying a proposal. No provider response is logged. +func (c *Client) Classify(ctx context.Context, facts domain.Facts, data domain.Dataset, forceAI bool) (Proposal, error) { + p, done, err := ruleProposal(facts, data, forceAI) + if done || err != nil { + return p, err + } failError := func(err error) (Proposal, error) { p.Enrichment.Classification = domain.Provenance{Source: "fallback", Timestamp: time.Now().UTC().Format(time.RFC3339), Error: err.Error()} return p, err @@ -94,25 +140,7 @@ func (c *Client) Classify(ctx context.Context, facts domain.Facts, data domain.D fail := func(message string) (Proposal, error) { return failError(errors.New(message)) } - if _, err := facts.Amount.Minor(); err != nil { - return fail("invalid transaction amount") - } localDescription := facts.RawDescription + " " + facts.Counterparty - if merchant := aliasMatch(localDescription, data.Merchants); merchant != nil && !forceAI { - p.Enrichment.MerchantID = merchant.ID - if merchant.UseDefaults { - if merchant.DefaultCategoryID != "" { - p.Enrichment.CategoryID = merchant.DefaultCategoryID - } - p.Enrichment.TagIDs = append([]string{}, merchant.DefaultTagIDs...) - p.Enrichment.Classification = domain.Provenance{Source: "rule", Timestamp: time.Now().UTC().Format(time.RFC3339)} - if err := domain.ValidateEnrichment(data, facts, p.Enrichment); err != nil { - p.Enrichment = domain.Fallback(facts) - return fail("merchant defaults are invalid for this transaction") - } - return p, nil - } - } apiKey, model := c.APIKey, c.Model includeAmount := c.IncludeAmount if strings.TrimSpace(apiKey) == "" || strings.TrimSpace(model) == "" { diff --git a/web/src/Accounts.tsx b/web/src/Accounts.tsx index 45425f0..d9bdffd 100644 --- a/web/src/Accounts.tsx +++ b/web/src/Accounts.tsx @@ -630,6 +630,7 @@ function ImportForm({ {prepared && ( { @@ -647,16 +648,20 @@ function ImportForm({ // only obvious against real records. function ImportReview({ prepared, + state, acceptState, onError, close, }: { prepared: PreparedImport; + state: State; acceptState: (state: State, message?: string) => void; onError: (error: string) => void; close: () => void; }) { const [busy, setBusy] = useState(false); + const classifying = + state.settings.classify_on_import && state.status.ai_configured; const discard = () => { // Free the server's prepared statement; an expiring one is harmless. void request("/api/import/cancel", { id: prepared.id }).catch(() => {}); @@ -744,6 +749,11 @@ function ImportReview({ {prepared.samples.length} of {prepared.records} records, including the largest amount and both directions of money.

+

+ {classifying + ? "After importing, these transactions are classified with AI. Turn that off under Settings → Classification preferences." + : "AI classification of imports is off, so these transactions arrive unclassified and editable. Enabled merchant rules still apply."} +