Import the longest bank-permitted history and surface real provider errors

Manual history imports failed opaquely once a bank capped lookback on an
established consent (N26 rejects date_from beyond ~90 days with
WRONG_TRANSACTIONS_PERIOD). Backfill now requests the documented longest
fetching strategy, reports the coverage the bank actually provided, and
non-2xx responses surface allowlisted documented error codes instead of a
generic fallback. Dead-session codes map to reconnection. Failed syncs
retry hourly so a stale sync banner no longer persists for a day.
This commit is contained in:
Lars Nolden
2026-09-10 22:58:03 +02:00
parent 4324660888
commit a8722d58c3
7 changed files with 213 additions and 36 deletions
+12 -3
View File
@@ -18,6 +18,7 @@ type backfillBank struct {
statusIDs []string
accounts []domain.Account
toDates []string
longests []bool
statusErr error
fetchErr error
}
@@ -30,10 +31,11 @@ func (b *backfillBank) Status(ctx context.Context, id string) (banking.SessionSt
return b.historyBank.Status(ctx, id)
}
func (b *backfillBank) Transactions(ctx context.Context, account domain.Account, from, to string) ([]domain.Facts, error) {
func (b *backfillBank) Transactions(ctx context.Context, account domain.Account, from, to string, longest bool) ([]domain.Facts, error) {
b.accounts = append(b.accounts, account)
b.toDates = append(b.toDates, to)
rows, err := b.historyBank.Transactions(ctx, account, from, to)
b.longests = append(b.longests, longest)
rows, err := b.historyBank.Transactions(ctx, account, from, to, longest)
if b.fetchErr != nil {
// A provider can fail after accumulating a page: none of it is importable.
return rows, b.fetchErr
@@ -99,7 +101,7 @@ func TestBackfillImportsOlderFactsOnceWithoutChangingSyncState(t *testing.T) {
if err != nil {
t.Fatal(err)
}
b.statusIDs, b.accounts, b.fromDates, b.toDates = nil, nil, nil, nil
b.statusIDs, b.accounts, b.fromDates, b.toDates, b.longests = nil, nil, nil, nil, nil
// Provider-local IDs need not match our canonical account ID.
providerAccount := s.Data.Accounts[0]
providerAccount.ID = "provider_generated_id"
@@ -121,6 +123,13 @@ func TestBackfillImportsOlderFactsOnceWithoutChangingSyncState(t *testing.T) {
if len(b.fromDates) != 1 || (b.fromDates[0] != fromBefore && b.fromDates[0] != fromAfter) || (b.toDates[0] != toBefore && b.toDates[0] != toAfter) {
t.Fatalf("backfill did not request the selected calendar-month range: %v to %v", b.fromDates, b.toDates)
}
if !reflect.DeepEqual(b.longests, []bool{true}) {
t.Fatal("manual history import did not request the tolerant longest-period strategy")
}
oldest := time.Now().UTC().AddDate(0, 0, -400).Format("2006-01-02")
if result.RequestedFrom != b.fromDates[0] || result.EarliestFetched != oldest {
t.Fatalf("backfill misreported its history coverage: %+v", result)
}
for _, existing := range s.Data.Transactions {
found := false
for _, tx := range result.State.Data.Transactions {
+1 -1
View File
@@ -23,7 +23,7 @@ func (b *historyBank) Authorize(_ context.Context, _, _, state string) (string,
return "https://bank.example/authorize", nil
}
func (b *historyBank) Transactions(_ context.Context, account domain.Account, from, to string) ([]domain.Facts, error) {
func (b *historyBank) Transactions(_ context.Context, account domain.Account, from, to string, _ bool) ([]domain.Facts, error) {
b.fromDates = append(b.fromDates, from)
var rows []domain.Facts
for _, days := range []int{400, 300, 1} {
+42 -8
View File
@@ -17,6 +17,11 @@ import (
type ImportResult struct {
Imported int `json:"imported"`
// RequestedFrom and EarliestFetched describe manual history retrieval:
// the requested window start, and the oldest booking date the bank
// actually returned (empty when it returned nothing).
RequestedFrom string `json:"requested_from,omitempty"`
EarliestFetched string `json:"earliest_fetched,omitempty"`
State State `json:"state"`
}
@@ -147,13 +152,27 @@ func (a *App) Backfill(ctx context.Context, rev, accountID string, historyMonths
return ImportResult{}, banking.ErrReconnect
}
now := time.Now().UTC()
facts, err := a.bank.Transactions(ctx, account, now.AddDate(0, -historyMonths, 0).Format("2006-01-02"), now.Format("2006-01-02"))
from := now.AddDate(0, -historyMonths, 0).Format("2006-01-02")
// The longest fetching strategy imports whatever period the bank still
// permits: many banks cap history on an established consent instead of
// serving the full requested range.
facts, err := a.bank.Transactions(ctx, account, from, now.Format("2006-01-02"), true)
if err != nil {
return ImportResult{}, bankFailure(err, "transaction retrieval failed; retry importing history")
}
// Use normal import processing without changing sync cursors or saved consent
// settings, including when the requested range adds no transactions.
return a.importFacts(ctx, s, facts)
result, err := a.importFacts(ctx, s, facts)
if err != nil {
return ImportResult{}, err
}
result.RequestedFrom = from
for _, f := range facts {
if result.EarliestFetched == "" || f.BookingDate < result.EarliestFetched {
result.EarliestFetched = f.BookingDate
}
}
return result, nil
}
func (a *App) Authorize(ctx context.Context, institution, country string, historyMonths int) (string, error) {
a.mu.Lock()
@@ -310,6 +329,10 @@ func bankFailure(err error, fallback string) error {
if errors.Is(err, banking.ErrReconnect) {
return banking.ErrReconnect
}
var api *banking.APIError
if errors.As(err, &api) {
return api
}
return errors.New(fallback)
}
@@ -412,7 +435,7 @@ func (a *App) Sync(ctx context.Context) (State, error) {
months := a.ops.Consents[accountSession[account.ID]].historyMonths()
from = now.AddDate(0, -months, 0).Format("2006-01-02")
}
facts, e := a.bank.Transactions(ctx, account, from, to)
facts, e := a.bank.Transactions(ctx, account, from, to, false)
if e != nil {
meta := a.ops.Consents[sessionID]
meta.Error = bankFailure(e, "transaction retrieval failed; retry synchronization").Error()
@@ -457,13 +480,24 @@ func (a *App) RunScheduler(ctx context.Context) {
a.mu.Lock()
configured := a.bank != nil
last, err := time.Parse(time.RFC3339, a.ops.LastSync)
due := force || err != nil || time.Since(last) >= 24*time.Hour
failed := a.ops.SyncError != ""
a.mu.Unlock()
if configured && due {
a.Sync(ctx)
timer.Reset(24 * time.Hour)
} else {
due := force || err != nil || failed || time.Since(last) >= 24*time.Hour
if !configured || !due {
timer.Reset(time.Minute)
continue
}
a.Sync(ctx)
a.mu.Lock()
failed = a.ops.SyncError != ""
a.mu.Unlock()
if failed {
// A failed sync leaves its persisted error banner behind. Retry
// hourly so transient provider failures clear without waiting a
// day, while bounding unattended traffic toward the provider.
timer.Reset(time.Hour)
} else {
timer.Reset(24 * time.Hour)
}
}
}
+3 -3
View File
@@ -40,7 +40,7 @@ func (b *bankScenario) Status(context.Context, string) (banking.SessionStatus, e
func (b *bankScenario) Balances(context.Context, string) ([]banking.Balance, error) {
return []banking.Balance{{Amount: "100.00", Currency: "EUR", Type: "CLBD"}}, nil
}
func (b *bankScenario) Transactions(_ context.Context, a domain.Account, from, to string) ([]domain.Facts, error) {
func (b *bankScenario) Transactions(_ context.Context, a domain.Account, from, to string, _ bool) ([]domain.Facts, error) {
if b.fail {
return nil, errors.New("offline")
}
@@ -153,9 +153,9 @@ func (b *sessionBank) Status(_ context.Context, id string) (banking.SessionStatu
return b.statuses[id], b.failures[id]
}
func (b *sessionBank) Transactions(ctx context.Context, account domain.Account, from, to string) ([]domain.Facts, error) {
func (b *sessionBank) Transactions(ctx context.Context, account domain.Account, from, to string, longest bool) ([]domain.Facts, error) {
b.fetched = append(b.fetched, account.ID)
return b.bankScenario.Transactions(ctx, account, from, to)
return b.bankScenario.Transactions(ctx, account, from, to, longest)
}
func TestSyncSessionRateLimitPreservesBindingsAndRecovers(t *testing.T) {
+79 -3
View File
@@ -55,7 +55,7 @@ type Provider interface {
Exchange(context.Context, string) (Session, error)
Status(context.Context, string) (SessionStatus, error)
Balances(context.Context, string) ([]Balance, error)
Transactions(context.Context, domain.Account, string, string) ([]domain.Facts, error)
Transactions(ctx context.Context, account domain.Account, from, to string, longest bool) ([]domain.Facts, error)
}
type EnableBanking struct {
HTTPClient *http.Client
@@ -206,6 +206,75 @@ func (e *BackgroundQuotaError) Unwrap() error {
return e.RateLimitError
}
// APIError reports a failed Enable Banking call. Its message is built only
// from the HTTP status and, when the response envelope's error code exactly
// matches the documented enumeration, that code with a locally written hint.
// Provider response text is never included.
type APIError struct {
Status int
Code string // documented Enable Banking error code, or empty
}
func (e *APIError) Error() string {
if hint, known := apiErrorHints[e.Code]; known {
return fmt.Sprintf("Enable Banking returned HTTP %d (%s: %s)", e.Status, e.Code, hint)
}
return fmt.Sprintf("Enable Banking returned HTTP %d", e.Status)
}
// Unwrap exposes inactive-consent codes as ErrReconnect so callers offer
// reconnection instead of a dead-end provider failure.
func (e *APIError) Unwrap() error {
switch e.Code {
case "CLOSED_SESSION", "EXPIRED_SESSION", "REVOKED_SESSION", "SESSION_DOES_NOT_EXIST":
return ErrReconnect
}
return nil
}
// apiErrorHints holds locally written descriptions for the documented error
// codes relevant to account information. Only exact matches are ever exposed.
var apiErrorHints = map[string]string{
"ACCESS_DENIED": "access to this resource is denied for the application",
"ACCOUNT_DOES_NOT_EXIST": "no account matches the stored identifier",
"ASPSP_ACCOUNT_NOT_ACCESSIBLE": "the bank did not grant access to the requested account",
"ASPSP_ERROR": "the bank reported an error",
"ASPSP_PSU_ACTION_REQUIRED": "the bank requires action in your banking app or online banking",
"ASPSP_TIMEOUT": "the bank did not respond in time",
"CLOSED_SESSION": "the bank session is closed",
"DATE_FROM_IN_FUTURE": "the requested start date is in the future",
"EXPIRED_SESSION": "the bank session has expired",
"NO_ACCOUNTS_ADDED": "no allowed accounts are added to the application",
"PSU_HEADER_INVALID": "the forwarded browser metadata was rejected",
"PSU_HEADER_NOT_PROVIDED": "this bank requires a user-initiated request",
"REVOKED_SESSION": "the bank session was revoked",
"SESSION_DOES_NOT_EXIST": "the bank session no longer exists",
"UNAUTHORIZED_ACCESS": "the application is not authorized for this request",
"UNAUTHORIZED_IP": "this network address is not authorized for the request",
"WRONG_CONTINUATION_KEY": "the pagination key was rejected",
"WRONG_DATE_INTERVAL": "the start date must not be after the end date",
"WRONG_REQUEST_PARAMETERS": "the request parameters were rejected",
"WRONG_SESSION_STATUS": "the bank session is in the wrong state for this request",
"WRONG_TRANSACTIONS_PERIOD": "the bank does not provide transactions for the requested period; banks commonly limit history to about 90 days after the initial connection",
}
// apiError classifies a non-success response by its documented error code
// without retaining or exposing any other provider response content.
func apiError(response *http.Response) *APIError {
failure := &APIError{Status: response.StatusCode}
const envelopeLimit = 16 << 10
body, err := io.ReadAll(io.LimitReader(response.Body, envelopeLimit+1))
var envelope struct {
Error string `json:"error"`
}
if err == nil && len(body) <= envelopeLimit && json.Unmarshal(body, &envelope) == nil {
if _, known := apiErrorHints[envelope.Error]; known {
failure.Code = envelope.Error
}
}
return failure
}
// backgroundRetryAt follows the bank's six-hour guidance, never shortening a
// longer provider deadline. Zero retains the limiter's unbounded-delay meaning.
func backgroundRetryAt(header string, now time.Time) time.Time {
@@ -320,7 +389,7 @@ func (p *EnableBanking) request(ctx context.Context, method, path string, input,
}
defer response.Body.Close()
if response.StatusCode < 200 || response.StatusCode >= 300 {
return fmt.Errorf("Enable Banking returned HTTP %d", response.StatusCode)
return apiError(response)
}
const limit = 16 << 20
b, err := io.ReadAll(io.LimitReader(response.Body, limit+1))
@@ -546,7 +615,11 @@ type transactionDTO struct {
DebtorAccount accountIdentificationDTO `json:"debtor_account"`
}
func (p *EnableBanking) Transactions(ctx context.Context, account domain.Account, from, to string) ([]domain.Facts, error) {
// Transactions retrieves booked transactions in the requested window. With
// longest, the documented "longest" fetching strategy asks the provider for
// the maximum period the bank permits instead of rejecting an out-of-range
// start date; rows outside the requested window are still filtered out here.
func (p *EnableBanking) Transactions(ctx context.Context, account domain.Account, from, to string, longest bool) ([]domain.Facts, error) {
if account.ID == "" || account.ExternalAccountID == "" {
return nil, fmt.Errorf("account is not connected to Enable Banking")
}
@@ -567,6 +640,9 @@ func (p *EnableBanking) Transactions(ctx context.Context, account domain.Account
if to != "" {
query.Set("date_to", to)
}
if longest {
query.Set("strategy", "longest")
}
result := make([]domain.Facts, 0)
seen := map[string]bool{}
for range 1000 {
+55 -7
View File
@@ -16,6 +16,7 @@ import (
"io"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"sync/atomic"
"testing"
@@ -193,7 +194,7 @@ func TestEnableBankingDocumentedFlowAndPagination(t *testing.T) {
if err != nil || len(balances) != 1 || balances[0].Amount.String() != "1234.5678" || balances[0].Type != "CLBD" {
t.Fatalf("balance precision lost: %+v %v", balances, err)
}
transactions, err := p.Transactions(context.Background(), session.Accounts[0], "2026-09-01", "2026-09-30")
transactions, err := p.Transactions(context.Background(), session.Accounts[0], "2026-09-01", "2026-09-30", false)
if err != nil {
t.Fatal(err)
}
@@ -227,11 +228,58 @@ func TestEnableBankingRejectsPaginationCyclesAndPartialResults(t *testing.T) {
})
account := fixtureDataset().Accounts[0]
account.ExternalAccountID = "uid"
rows, err := p.Transactions(context.Background(), account, "", "")
rows, err := p.Transactions(context.Background(), account, "", "", false)
if err == nil || rows != nil {
t.Fatal("pagination cycle returned partial import")
}
}
func TestEnableBankingLongestStrategyIsOnlySentWhenRequested(t *testing.T) {
var strategies []string
p, _ := testProvider(t, func(w http.ResponseWriter, r *http.Request) {
strategies = append(strategies, r.URL.Query().Get("strategy"))
fmt.Fprint(w, `{"transactions":[]}`)
})
account := fixtureDataset().Accounts[0]
account.ExternalAccountID = "uid"
if _, err := p.Transactions(context.Background(), account, "2020-01-01", "", true); err != nil {
t.Fatal(err)
}
if _, err := p.Transactions(context.Background(), account, "2026-09-01", "2026-09-30", false); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(strategies, []string{"longest", ""}) {
t.Fatalf("wrong fetching strategies requested: %q", strategies)
}
}
func TestEnableBankingSurfacesOnlyDocumentedErrorCodes(t *testing.T) {
body := ""
p, _ := testProvider(t, func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusUnprocessableEntity)
fmt.Fprint(w, body)
})
account := fixtureDataset().Accounts[0]
account.ExternalAccountID = "uid"
fetch := func() error {
_, err := p.Transactions(context.Background(), account, "2020-01-01", "", true)
return err
}
body = `{"code":422,"error":"WRONG_TRANSACTIONS_PERIOD","message":"private-bank-text","detail":"private-account-detail"}`
err := fetch()
if err == nil || !strings.Contains(err.Error(), "WRONG_TRANSACTIONS_PERIOD") || !strings.Contains(err.Error(), "422") {
t.Fatalf("documented period error was hidden: %v", err)
}
if strings.Contains(err.Error(), "private") || errors.Is(err, ErrReconnect) {
t.Fatalf("unsafe or misclassified period error: %v", err)
}
body = `{"code":422,"error":"UNDOCUMENTED_PRIVATE_CODE","detail":"secret"}`
if err = fetch(); err == nil || strings.Contains(err.Error(), "UNDOCUMENTED") || strings.Contains(err.Error(), "secret") || !strings.Contains(err.Error(), "422") {
t.Fatalf("undocumented provider code leaked: %v", err)
}
body = `{"code":404,"error":"SESSION_DOES_NOT_EXIST"}`
if err = fetch(); !errors.Is(err, ErrReconnect) {
t.Fatalf("dead session did not request reconnection: %v", err)
}
}
func TestEnableBankingSessionRevocationAndInvalidBookedDates(t *testing.T) {
p, _ := testProvider(t, func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, "/sessions/") {
@@ -245,7 +293,7 @@ func TestEnableBankingSessionRevocationAndInvalidBookedDates(t *testing.T) {
}
account := fixtureDataset().Accounts[0]
account.ExternalAccountID = "uid"
rows, err := p.Transactions(context.Background(), account, "", "")
rows, err := p.Transactions(context.Background(), account, "", "", false)
if err == nil || rows != nil {
t.Fatal("invented booking date for missing bank fact")
}
@@ -352,7 +400,7 @@ func TestEnableBankingGETRecoversAfterRateLimit(t *testing.T) {
} else {
account := fixtureDataset().Accounts[0]
account.ExternalAccountID = "uid"
rows, err := p.Transactions(context.Background(), account, "", "")
rows, err := p.Transactions(context.Background(), account, "", "", false)
if err != nil || len(rows) != 1 || rows[0].ExternalID != "entry" || rows[0].Amount.String() != "1.00" {
t.Fatalf("transaction retrieval did not recover: %+v %v", rows, err)
}
@@ -381,7 +429,7 @@ func TestEnableBankingCooldownCoversAllEndpoints(t *testing.T) {
for name, request := range map[string]func() error{
"status": func() error { _, err := p.Status(context.Background(), "other-session"); return err },
"balances": func() error { _, err := p.Balances(context.Background(), "uid"); return err },
"transactions": func() error { _, err := p.Transactions(context.Background(), account, "", ""); return err },
"transactions": func() error { _, err := p.Transactions(context.Background(), account, "", "", false); return err },
"exchange": func() error { _, err := p.Exchange(context.Background(), "once-only-code"); return err },
"authorize": func() error { _, err := p.Authorize(context.Background(), "N26", "DE", "state"); return err },
} {
@@ -509,7 +557,7 @@ func TestEnableBankingPSUSurvivesRetryAndPaginationWithoutLeakingToBackground(t
ctx := WithPSU(context.Background(), psu)
account := fixtureDataset().Accounts[0]
account.ExternalAccountID = "uid"
rows, err := p.Transactions(ctx, account, "2026-01-01", "")
rows, err := p.Transactions(ctx, account, "2026-01-01", "", false)
if err != nil || len(rows) != 1 || rows[0].ExternalID != "entry" {
t.Fatalf("manual history failed: %+v %v", rows, err)
}
@@ -565,7 +613,7 @@ func TestEnableBankingBackgroundQuotaIsScopedAndExpires(t *testing.T) {
}
account := fixtureDataset().Accounts[0]
account.ExternalAccountID = "uid"
if _, err := p.Transactions(context.Background(), account, "", ""); err != nil {
if _, err := p.Transactions(context.Background(), account, "", "", false); err != nil {
t.Fatalf("balance quota blocked transaction endpoint: %v", err)
}
// Simulate the stored deadline passing without a six-hour wall-clock wait.
+16 -6
View File
@@ -441,18 +441,28 @@ function BackfillHistory({
setError("");
setSuccess("");
try {
const response = await request<{ imported: number; state: State }>(
"/api/backfill",
{
const response = await request<{
imported: number;
requested_from?: string;
earliest_fetched?: string;
state: State;
}>("/api/backfill", {
revision: state.revision,
account_id: account.id,
history_months: months,
},
);
const message =
});
let message =
response.imported === 0
? `No new transactions imported for ${account.display_name}. Existing transactions were not duplicated.`
: `Imported ${response.imported} new transactions for ${account.display_name}. Existing transactions were not duplicated.`;
if (!response.earliest_fetched) {
message += " The bank returned no transactions for this range.";
} else if (
response.requested_from &&
response.earliest_fetched > response.requested_from
) {
message += ` The bank provided history starting ${response.earliest_fetched}, not the requested ${response.requested_from}; banks often limit how far back an existing connection can read. Older transactions can be added via CSV import.`;
}
acceptState(response.state, message);
setSuccess(message);
} catch (err) {