diff --git a/internal/app/backfill_test.go b/internal/app/backfill_test.go index c8085b2..8fb0358 100644 --- a/internal/app/backfill_test.go +++ b/internal/app/backfill_test.go @@ -18,6 +18,7 @@ type backfillBank struct { statusIDs []string accounts []domain.Account toDates []string + longests []bool statusErr error fetchErr error } @@ -30,10 +31,11 @@ func (b *backfillBank) Status(ctx context.Context, id string) (banking.SessionSt return b.historyBank.Status(ctx, id) } -func (b *backfillBank) Transactions(ctx context.Context, account domain.Account, from, to string) ([]domain.Facts, error) { +func (b *backfillBank) Transactions(ctx context.Context, account domain.Account, from, to string, longest bool) ([]domain.Facts, error) { b.accounts = append(b.accounts, account) b.toDates = append(b.toDates, to) - rows, err := b.historyBank.Transactions(ctx, account, from, to) + b.longests = append(b.longests, longest) + rows, err := b.historyBank.Transactions(ctx, account, from, to, longest) if b.fetchErr != nil { // A provider can fail after accumulating a page: none of it is importable. return rows, b.fetchErr @@ -99,7 +101,7 @@ func TestBackfillImportsOlderFactsOnceWithoutChangingSyncState(t *testing.T) { if err != nil { t.Fatal(err) } - b.statusIDs, b.accounts, b.fromDates, b.toDates = nil, nil, nil, nil + b.statusIDs, b.accounts, b.fromDates, b.toDates, b.longests = nil, nil, nil, nil, nil // Provider-local IDs need not match our canonical account ID. providerAccount := s.Data.Accounts[0] providerAccount.ID = "provider_generated_id" @@ -121,6 +123,13 @@ func TestBackfillImportsOlderFactsOnceWithoutChangingSyncState(t *testing.T) { if len(b.fromDates) != 1 || (b.fromDates[0] != fromBefore && b.fromDates[0] != fromAfter) || (b.toDates[0] != toBefore && b.toDates[0] != toAfter) { t.Fatalf("backfill did not request the selected calendar-month range: %v to %v", b.fromDates, b.toDates) } + if !reflect.DeepEqual(b.longests, []bool{true}) { + t.Fatal("manual history import did not request the tolerant longest-period strategy") + } + oldest := time.Now().UTC().AddDate(0, 0, -400).Format("2006-01-02") + if result.RequestedFrom != b.fromDates[0] || result.EarliestFetched != oldest { + t.Fatalf("backfill misreported its history coverage: %+v", result) + } for _, existing := range s.Data.Transactions { found := false for _, tx := range result.State.Data.Transactions { diff --git a/internal/app/consent_test.go b/internal/app/consent_test.go index 84a6876..286cb24 100644 --- a/internal/app/consent_test.go +++ b/internal/app/consent_test.go @@ -23,7 +23,7 @@ func (b *historyBank) Authorize(_ context.Context, _, _, state string) (string, return "https://bank.example/authorize", nil } -func (b *historyBank) Transactions(_ context.Context, account domain.Account, from, to string) ([]domain.Facts, error) { +func (b *historyBank) Transactions(_ context.Context, account domain.Account, from, to string, _ bool) ([]domain.Facts, error) { b.fromDates = append(b.fromDates, from) var rows []domain.Facts for _, days := range []int{400, 300, 1} { diff --git a/internal/app/import.go b/internal/app/import.go index 489deed..6c9745e 100644 --- a/internal/app/import.go +++ b/internal/app/import.go @@ -16,8 +16,13 @@ import ( ) type ImportResult struct { - Imported int `json:"imported"` - State State `json:"state"` + Imported int `json:"imported"` + // RequestedFrom and EarliestFetched describe manual history retrieval: + // the requested window start, and the oldest booking date the bank + // actually returned (empty when it returned nothing). + RequestedFrom string `json:"requested_from,omitempty"` + EarliestFetched string `json:"earliest_fetched,omitempty"` + State State `json:"state"` } func addProposal(d *domain.Dataset, p classification.Proposal) error { @@ -147,13 +152,27 @@ func (a *App) Backfill(ctx context.Context, rev, accountID string, historyMonths return ImportResult{}, banking.ErrReconnect } now := time.Now().UTC() - facts, err := a.bank.Transactions(ctx, account, now.AddDate(0, -historyMonths, 0).Format("2006-01-02"), now.Format("2006-01-02")) + from := now.AddDate(0, -historyMonths, 0).Format("2006-01-02") + // The longest fetching strategy imports whatever period the bank still + // permits: many banks cap history on an established consent instead of + // serving the full requested range. + facts, err := a.bank.Transactions(ctx, account, from, now.Format("2006-01-02"), true) if err != nil { return ImportResult{}, bankFailure(err, "transaction retrieval failed; retry importing history") } // Use normal import processing without changing sync cursors or saved consent // settings, including when the requested range adds no transactions. - return a.importFacts(ctx, s, facts) + result, err := a.importFacts(ctx, s, facts) + if err != nil { + return ImportResult{}, err + } + result.RequestedFrom = from + for _, f := range facts { + if result.EarliestFetched == "" || f.BookingDate < result.EarliestFetched { + result.EarliestFetched = f.BookingDate + } + } + return result, nil } func (a *App) Authorize(ctx context.Context, institution, country string, historyMonths int) (string, error) { a.mu.Lock() @@ -310,6 +329,10 @@ func bankFailure(err error, fallback string) error { if errors.Is(err, banking.ErrReconnect) { return banking.ErrReconnect } + var api *banking.APIError + if errors.As(err, &api) { + return api + } return errors.New(fallback) } @@ -412,7 +435,7 @@ func (a *App) Sync(ctx context.Context) (State, error) { months := a.ops.Consents[accountSession[account.ID]].historyMonths() from = now.AddDate(0, -months, 0).Format("2006-01-02") } - facts, e := a.bank.Transactions(ctx, account, from, to) + facts, e := a.bank.Transactions(ctx, account, from, to, false) if e != nil { meta := a.ops.Consents[sessionID] meta.Error = bankFailure(e, "transaction retrieval failed; retry synchronization").Error() @@ -457,13 +480,24 @@ func (a *App) RunScheduler(ctx context.Context) { a.mu.Lock() configured := a.bank != nil last, err := time.Parse(time.RFC3339, a.ops.LastSync) - due := force || err != nil || time.Since(last) >= 24*time.Hour + failed := a.ops.SyncError != "" a.mu.Unlock() - if configured && due { - a.Sync(ctx) - timer.Reset(24 * time.Hour) - } else { + due := force || err != nil || failed || time.Since(last) >= 24*time.Hour + if !configured || !due { timer.Reset(time.Minute) + continue + } + a.Sync(ctx) + a.mu.Lock() + failed = a.ops.SyncError != "" + a.mu.Unlock() + if failed { + // A failed sync leaves its persisted error banner behind. Retry + // hourly so transient provider failures clear without waiting a + // day, while bounding unattended traffic toward the provider. + timer.Reset(time.Hour) + } else { + timer.Reset(24 * time.Hour) } } } diff --git a/internal/app/sync_test.go b/internal/app/sync_test.go index 6c42500..95207b6 100644 --- a/internal/app/sync_test.go +++ b/internal/app/sync_test.go @@ -40,7 +40,7 @@ func (b *bankScenario) Status(context.Context, string) (banking.SessionStatus, e func (b *bankScenario) Balances(context.Context, string) ([]banking.Balance, error) { return []banking.Balance{{Amount: "100.00", Currency: "EUR", Type: "CLBD"}}, nil } -func (b *bankScenario) Transactions(_ context.Context, a domain.Account, from, to string) ([]domain.Facts, error) { +func (b *bankScenario) Transactions(_ context.Context, a domain.Account, from, to string, _ bool) ([]domain.Facts, error) { if b.fail { return nil, errors.New("offline") } @@ -153,9 +153,9 @@ func (b *sessionBank) Status(_ context.Context, id string) (banking.SessionStatu return b.statuses[id], b.failures[id] } -func (b *sessionBank) Transactions(ctx context.Context, account domain.Account, from, to string) ([]domain.Facts, error) { +func (b *sessionBank) Transactions(ctx context.Context, account domain.Account, from, to string, longest bool) ([]domain.Facts, error) { b.fetched = append(b.fetched, account.ID) - return b.bankScenario.Transactions(ctx, account, from, to) + return b.bankScenario.Transactions(ctx, account, from, to, longest) } func TestSyncSessionRateLimitPreservesBindingsAndRecovers(t *testing.T) { diff --git a/internal/banking/enablebanking.go b/internal/banking/enablebanking.go index 08338e2..f27bc27 100644 --- a/internal/banking/enablebanking.go +++ b/internal/banking/enablebanking.go @@ -55,7 +55,7 @@ type Provider interface { Exchange(context.Context, string) (Session, error) Status(context.Context, string) (SessionStatus, error) Balances(context.Context, string) ([]Balance, error) - Transactions(context.Context, domain.Account, string, string) ([]domain.Facts, error) + Transactions(ctx context.Context, account domain.Account, from, to string, longest bool) ([]domain.Facts, error) } type EnableBanking struct { HTTPClient *http.Client @@ -206,6 +206,75 @@ func (e *BackgroundQuotaError) Unwrap() error { return e.RateLimitError } +// APIError reports a failed Enable Banking call. Its message is built only +// from the HTTP status and, when the response envelope's error code exactly +// matches the documented enumeration, that code with a locally written hint. +// Provider response text is never included. +type APIError struct { + Status int + Code string // documented Enable Banking error code, or empty +} + +func (e *APIError) Error() string { + if hint, known := apiErrorHints[e.Code]; known { + return fmt.Sprintf("Enable Banking returned HTTP %d (%s: %s)", e.Status, e.Code, hint) + } + return fmt.Sprintf("Enable Banking returned HTTP %d", e.Status) +} + +// Unwrap exposes inactive-consent codes as ErrReconnect so callers offer +// reconnection instead of a dead-end provider failure. +func (e *APIError) Unwrap() error { + switch e.Code { + case "CLOSED_SESSION", "EXPIRED_SESSION", "REVOKED_SESSION", "SESSION_DOES_NOT_EXIST": + return ErrReconnect + } + return nil +} + +// apiErrorHints holds locally written descriptions for the documented error +// codes relevant to account information. Only exact matches are ever exposed. +var apiErrorHints = map[string]string{ + "ACCESS_DENIED": "access to this resource is denied for the application", + "ACCOUNT_DOES_NOT_EXIST": "no account matches the stored identifier", + "ASPSP_ACCOUNT_NOT_ACCESSIBLE": "the bank did not grant access to the requested account", + "ASPSP_ERROR": "the bank reported an error", + "ASPSP_PSU_ACTION_REQUIRED": "the bank requires action in your banking app or online banking", + "ASPSP_TIMEOUT": "the bank did not respond in time", + "CLOSED_SESSION": "the bank session is closed", + "DATE_FROM_IN_FUTURE": "the requested start date is in the future", + "EXPIRED_SESSION": "the bank session has expired", + "NO_ACCOUNTS_ADDED": "no allowed accounts are added to the application", + "PSU_HEADER_INVALID": "the forwarded browser metadata was rejected", + "PSU_HEADER_NOT_PROVIDED": "this bank requires a user-initiated request", + "REVOKED_SESSION": "the bank session was revoked", + "SESSION_DOES_NOT_EXIST": "the bank session no longer exists", + "UNAUTHORIZED_ACCESS": "the application is not authorized for this request", + "UNAUTHORIZED_IP": "this network address is not authorized for the request", + "WRONG_CONTINUATION_KEY": "the pagination key was rejected", + "WRONG_DATE_INTERVAL": "the start date must not be after the end date", + "WRONG_REQUEST_PARAMETERS": "the request parameters were rejected", + "WRONG_SESSION_STATUS": "the bank session is in the wrong state for this request", + "WRONG_TRANSACTIONS_PERIOD": "the bank does not provide transactions for the requested period; banks commonly limit history to about 90 days after the initial connection", +} + +// apiError classifies a non-success response by its documented error code +// without retaining or exposing any other provider response content. +func apiError(response *http.Response) *APIError { + failure := &APIError{Status: response.StatusCode} + const envelopeLimit = 16 << 10 + body, err := io.ReadAll(io.LimitReader(response.Body, envelopeLimit+1)) + var envelope struct { + Error string `json:"error"` + } + if err == nil && len(body) <= envelopeLimit && json.Unmarshal(body, &envelope) == nil { + if _, known := apiErrorHints[envelope.Error]; known { + failure.Code = envelope.Error + } + } + return failure +} + // backgroundRetryAt follows the bank's six-hour guidance, never shortening a // longer provider deadline. Zero retains the limiter's unbounded-delay meaning. func backgroundRetryAt(header string, now time.Time) time.Time { @@ -320,7 +389,7 @@ func (p *EnableBanking) request(ctx context.Context, method, path string, input, } defer response.Body.Close() if response.StatusCode < 200 || response.StatusCode >= 300 { - return fmt.Errorf("Enable Banking returned HTTP %d", response.StatusCode) + return apiError(response) } const limit = 16 << 20 b, err := io.ReadAll(io.LimitReader(response.Body, limit+1)) @@ -546,7 +615,11 @@ type transactionDTO struct { DebtorAccount accountIdentificationDTO `json:"debtor_account"` } -func (p *EnableBanking) Transactions(ctx context.Context, account domain.Account, from, to string) ([]domain.Facts, error) { +// Transactions retrieves booked transactions in the requested window. With +// longest, the documented "longest" fetching strategy asks the provider for +// the maximum period the bank permits instead of rejecting an out-of-range +// start date; rows outside the requested window are still filtered out here. +func (p *EnableBanking) Transactions(ctx context.Context, account domain.Account, from, to string, longest bool) ([]domain.Facts, error) { if account.ID == "" || account.ExternalAccountID == "" { return nil, fmt.Errorf("account is not connected to Enable Banking") } @@ -567,6 +640,9 @@ func (p *EnableBanking) Transactions(ctx context.Context, account domain.Account if to != "" { query.Set("date_to", to) } + if longest { + query.Set("strategy", "longest") + } result := make([]domain.Facts, 0) seen := map[string]bool{} for range 1000 { diff --git a/internal/banking/enablebanking_test.go b/internal/banking/enablebanking_test.go index 66ef3dd..db2f5e6 100644 --- a/internal/banking/enablebanking_test.go +++ b/internal/banking/enablebanking_test.go @@ -16,6 +16,7 @@ import ( "io" "net/http" "net/http/httptest" + "reflect" "strings" "sync/atomic" "testing" @@ -193,7 +194,7 @@ func TestEnableBankingDocumentedFlowAndPagination(t *testing.T) { if err != nil || len(balances) != 1 || balances[0].Amount.String() != "1234.5678" || balances[0].Type != "CLBD" { t.Fatalf("balance precision lost: %+v %v", balances, err) } - transactions, err := p.Transactions(context.Background(), session.Accounts[0], "2026-09-01", "2026-09-30") + transactions, err := p.Transactions(context.Background(), session.Accounts[0], "2026-09-01", "2026-09-30", false) if err != nil { t.Fatal(err) } @@ -227,11 +228,58 @@ func TestEnableBankingRejectsPaginationCyclesAndPartialResults(t *testing.T) { }) account := fixtureDataset().Accounts[0] account.ExternalAccountID = "uid" - rows, err := p.Transactions(context.Background(), account, "", "") + rows, err := p.Transactions(context.Background(), account, "", "", false) if err == nil || rows != nil { t.Fatal("pagination cycle returned partial import") } } +func TestEnableBankingLongestStrategyIsOnlySentWhenRequested(t *testing.T) { + var strategies []string + p, _ := testProvider(t, func(w http.ResponseWriter, r *http.Request) { + strategies = append(strategies, r.URL.Query().Get("strategy")) + fmt.Fprint(w, `{"transactions":[]}`) + }) + account := fixtureDataset().Accounts[0] + account.ExternalAccountID = "uid" + if _, err := p.Transactions(context.Background(), account, "2020-01-01", "", true); err != nil { + t.Fatal(err) + } + if _, err := p.Transactions(context.Background(), account, "2026-09-01", "2026-09-30", false); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(strategies, []string{"longest", ""}) { + t.Fatalf("wrong fetching strategies requested: %q", strategies) + } +} +func TestEnableBankingSurfacesOnlyDocumentedErrorCodes(t *testing.T) { + body := "" + p, _ := testProvider(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnprocessableEntity) + fmt.Fprint(w, body) + }) + account := fixtureDataset().Accounts[0] + account.ExternalAccountID = "uid" + fetch := func() error { + _, err := p.Transactions(context.Background(), account, "2020-01-01", "", true) + return err + } + body = `{"code":422,"error":"WRONG_TRANSACTIONS_PERIOD","message":"private-bank-text","detail":"private-account-detail"}` + err := fetch() + if err == nil || !strings.Contains(err.Error(), "WRONG_TRANSACTIONS_PERIOD") || !strings.Contains(err.Error(), "422") { + t.Fatalf("documented period error was hidden: %v", err) + } + if strings.Contains(err.Error(), "private") || errors.Is(err, ErrReconnect) { + t.Fatalf("unsafe or misclassified period error: %v", err) + } + body = `{"code":422,"error":"UNDOCUMENTED_PRIVATE_CODE","detail":"secret"}` + if err = fetch(); err == nil || strings.Contains(err.Error(), "UNDOCUMENTED") || strings.Contains(err.Error(), "secret") || !strings.Contains(err.Error(), "422") { + t.Fatalf("undocumented provider code leaked: %v", err) + } + body = `{"code":404,"error":"SESSION_DOES_NOT_EXIST"}` + if err = fetch(); !errors.Is(err, ErrReconnect) { + t.Fatalf("dead session did not request reconnection: %v", err) + } +} func TestEnableBankingSessionRevocationAndInvalidBookedDates(t *testing.T) { p, _ := testProvider(t, func(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(r.URL.Path, "/sessions/") { @@ -245,7 +293,7 @@ func TestEnableBankingSessionRevocationAndInvalidBookedDates(t *testing.T) { } account := fixtureDataset().Accounts[0] account.ExternalAccountID = "uid" - rows, err := p.Transactions(context.Background(), account, "", "") + rows, err := p.Transactions(context.Background(), account, "", "", false) if err == nil || rows != nil { t.Fatal("invented booking date for missing bank fact") } @@ -352,7 +400,7 @@ func TestEnableBankingGETRecoversAfterRateLimit(t *testing.T) { } else { account := fixtureDataset().Accounts[0] account.ExternalAccountID = "uid" - rows, err := p.Transactions(context.Background(), account, "", "") + rows, err := p.Transactions(context.Background(), account, "", "", false) if err != nil || len(rows) != 1 || rows[0].ExternalID != "entry" || rows[0].Amount.String() != "1.00" { t.Fatalf("transaction retrieval did not recover: %+v %v", rows, err) } @@ -381,7 +429,7 @@ func TestEnableBankingCooldownCoversAllEndpoints(t *testing.T) { for name, request := range map[string]func() error{ "status": func() error { _, err := p.Status(context.Background(), "other-session"); return err }, "balances": func() error { _, err := p.Balances(context.Background(), "uid"); return err }, - "transactions": func() error { _, err := p.Transactions(context.Background(), account, "", ""); return err }, + "transactions": func() error { _, err := p.Transactions(context.Background(), account, "", "", false); return err }, "exchange": func() error { _, err := p.Exchange(context.Background(), "once-only-code"); return err }, "authorize": func() error { _, err := p.Authorize(context.Background(), "N26", "DE", "state"); return err }, } { @@ -509,7 +557,7 @@ func TestEnableBankingPSUSurvivesRetryAndPaginationWithoutLeakingToBackground(t ctx := WithPSU(context.Background(), psu) account := fixtureDataset().Accounts[0] account.ExternalAccountID = "uid" - rows, err := p.Transactions(ctx, account, "2026-01-01", "") + rows, err := p.Transactions(ctx, account, "2026-01-01", "", false) if err != nil || len(rows) != 1 || rows[0].ExternalID != "entry" { t.Fatalf("manual history failed: %+v %v", rows, err) } @@ -565,7 +613,7 @@ func TestEnableBankingBackgroundQuotaIsScopedAndExpires(t *testing.T) { } account := fixtureDataset().Accounts[0] account.ExternalAccountID = "uid" - if _, err := p.Transactions(context.Background(), account, "", ""); err != nil { + if _, err := p.Transactions(context.Background(), account, "", "", false); err != nil { t.Fatalf("balance quota blocked transaction endpoint: %v", err) } // Simulate the stored deadline passing without a six-hour wall-clock wait. diff --git a/web/src/Accounts.tsx b/web/src/Accounts.tsx index c986927..4dc432b 100644 --- a/web/src/Accounts.tsx +++ b/web/src/Accounts.tsx @@ -441,18 +441,28 @@ function BackfillHistory({ setError(""); setSuccess(""); try { - const response = await request<{ imported: number; state: State }>( - "/api/backfill", - { - revision: state.revision, - account_id: account.id, - history_months: months, - }, - ); - const message = + const response = await request<{ + imported: number; + requested_from?: string; + earliest_fetched?: string; + state: State; + }>("/api/backfill", { + revision: state.revision, + account_id: account.id, + history_months: months, + }); + let message = response.imported === 0 ? `No new transactions imported for ${account.display_name}. Existing transactions were not duplicated.` : `Imported ${response.imported} new transactions for ${account.display_name}. Existing transactions were not duplicated.`; + if (!response.earliest_fetched) { + message += " The bank returned no transactions for this range."; + } else if ( + response.requested_from && + response.earliest_fetched > response.requested_from + ) { + message += ` The bank provided history starting ${response.earliest_fetched}, not the requested ${response.requested_from}; banks often limit how far back an existing connection can read. Older transactions can be added via CSV import.`; + } acceptState(response.state, message); setSuccess(message); } catch (err) {