lars/finance-duck
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import the longest bank-permitted history and surface real provider errors

Browse Source 
Manual history imports failed opaquely once a bank capped lookback on an
established consent (N26 rejects date_from beyond ~90 days with
WRONG_TRANSACTIONS_PERIOD). Backfill now requests the documented longest
fetching strategy, reports the coverage the bank actually provided, and
non-2xx responses surface allowlisted documented error codes instead of a
generic fallback. Dead-session codes map to reconnection. Failed syncs
retry hourly so a stale sync banner no longer persists for a day.
This commit is contained in:
Lars Nolden
2026-09-10 22:58:03 +02:00
parent 4324660888
commit a8722d58c3
7 changed files with 213 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/app/backfill_test.go
+12 -3
View File
@@ -18,6 +18,7 @@ type backfillBank struct {
statusIDs []string
accounts []domain.Account
toDates []string
longests []bool
statusErr error
fetchErr error
}
@@ -30,10 +31,11 @@ func (b *backfillBank) Status(ctx context.Context, id string) (banking.SessionSt
return b.historyBank.Status(ctx, id)
}
func (b *backfillBank) Transactions(ctx context.Context, account domain.Account, from, to string) ([]domain.Facts, error) {
func (b *backfillBank) Transactions(ctx context.Context, account domain.Account, from, to string, longest bool) ([]domain.Facts, error) {
b.accounts = append(b.accounts, account)
b.toDates = append(b.toDates, to)
rows, err := b.historyBank.Transactions(ctx, account, from, to)
b.longests = append(b.longests, longest)
rows, err := b.historyBank.Transactions(ctx, account, from, to, longest)
if b.fetchErr != nil {
// A provider can fail after accumulating a page: none of it is importable.
return rows, b.fetchErr
@@ -99,7 +101,7 @@ func TestBackfillImportsOlderFactsOnceWithoutChangingSyncState(t *testing.T) {
if err != nil {
t.Fatal(err)
}
b.statusIDs, b.accounts, b.fromDates, b.toDates = nil, nil, nil, nil
b.statusIDs, b.accounts, b.fromDates, b.toDates, b.longests = nil, nil, nil, nil, nil
// Provider-local IDs need not match our canonical account ID.
providerAccount := s.Data.Accounts[0]
providerAccount.ID = "provider_generated_id"
@@ -121,6 +123,13 @@ func TestBackfillImportsOlderFactsOnceWithoutChangingSyncState(t *testing.T) {
if len(b.fromDates) != 1 || (b.fromDates[0] != fromBefore && b.fromDates[0] != fromAfter) || (b.toDates[0] != toBefore && b.toDates[0] != toAfter) {
t.Fatalf("backfill did not request the selected calendar-month range: %v to %v", b.fromDates, b.toDates)
}
if !reflect.DeepEqual(b.longests, []bool{true}) {
t.Fatal("manual history import did not request the tolerant longest-period strategy")
}
oldest := time.Now().UTC().AddDate(0, 0, -400).Format("2006-01-02")
if result.RequestedFrom != b.fromDates[0] || result.EarliestFetched != oldest {
t.Fatalf("backfill misreported its history coverage: %+v", result)
}
for _, existing := range s.Data.Transactions {
found := false
for _, tx := range result.State.Data.Transactions {
internal/app/consent_test.go
+1 -1
View File
@@ -23,7 +23,7 @@ func (b *historyBank) Authorize(_ context.Context, _, _, state string) (string,
return "https://bank.example/authorize", nil
}
func (b *historyBank) Transactions(_ context.Context, account domain.Account, from, to string) ([]domain.Facts, error) {
func (b *historyBank) Transactions(_ context.Context, account domain.Account, from, to string, _ bool) ([]domain.Facts, error) {
b.fromDates = append(b.fromDates, from)
var rows []domain.Facts
for _, days := range []int{400, 300, 1} {
internal/app/import.go
+44 -10
View File
@@ -16,8 +16,13 @@ import (
)
type ImportResult struct {
Imported int `json:"imported"`
State State `json:"state"`
Imported int `json:"imported"`
// RequestedFrom and EarliestFetched describe manual history retrieval:
// the requested window start, and the oldest booking date the bank
// actually returned (empty when it returned nothing).
RequestedFrom string `json:"requested_from,omitempty"`
EarliestFetched string `json:"earliest_fetched,omitempty"`
State State `json:"state"`
}
func addProposal(d *domain.Dataset, p classification.Proposal) error {
@@ -147,13 +152,27 @@ func (a *App) Backfill(ctx context.Context, rev, accountID string, historyMonths
return ImportResult{}, banking.ErrReconnect
}
now := time.Now().UTC()
facts, err := a.bank.Transactions(ctx, account, now.AddDate(0, -historyMonths, 0).Format("2006-01-02"), now.Format("2006-01-02"))
from := now.AddDate(0, -historyMonths, 0).Format("2006-01-02")
// The longest fetching strategy imports whatever period the bank still
// permits: many banks cap history on an established consent instead of
// serving the full requested range.
facts, err := a.bank.Transactions(ctx, account, from, now.Format("2006-01-02"), true)
if err != nil {
return ImportResult{}, bankFailure(err, "transaction retrieval failed; retry importing history")
}
// Use normal import processing without changing sync cursors or saved consent
// settings, including when the requested range adds no transactions.
return a.importFacts(ctx, s, facts)
result, err := a.importFacts(ctx, s, facts)
if err != nil {
return ImportResult{}, err
}
result.RequestedFrom = from
for _, f := range facts {
if result.EarliestFetched == "" || f.BookingDate < result.EarliestFetched {
result.EarliestFetched = f.BookingDate
}
}
return result, nil
}
func (a *App) Authorize(ctx context.Context, institution, country string, historyMonths int) (string, error) {
a.mu.Lock()
@@ -310,6 +329,10 @@ func bankFailure(err error, fallback string) error {
if errors.Is(err, banking.ErrReconnect) {
return banking.ErrReconnect
}
var api *banking.APIError
if errors.As(err, &api) {
return api
}
return errors.New(fallback)
}
@@ -412,7 +435,7 @@ func (a *App) Sync(ctx context.Context) (State, error) {
months := a.ops.Consents[accountSession[account.ID]].historyMonths()
from = now.AddDate(0, -months, 0).Format("2006-01-02")
}
facts, e := a.bank.Transactions(ctx, account, from, to)
facts, e := a.bank.Transactions(ctx, account, from, to, false)
if e != nil {
meta := a.ops.Consents[sessionID]
meta.Error = bankFailure(e, "transaction retrieval failed; retry synchronization").Error()
@@ -457,13 +480,24 @@ func (a *App) RunScheduler(ctx context.Context) {
a.mu.Lock()
configured := a.bank != nil
last, err := time.Parse(time.RFC3339, a.ops.LastSync)
due := force || err != nil || time.Since(last) >= 24*time.Hour
failed := a.ops.SyncError != ""
a.mu.Unlock()
if configured && due {
a.Sync(ctx)
timer.Reset(24 * time.Hour)
} else {
due := force || err != nil || failed || time.Since(last) >= 24*time.Hour
if !configured || !due {
timer.Reset(time.Minute)
continue
}
a.Sync(ctx)
a.mu.Lock()
failed = a.ops.SyncError != ""
a.mu.Unlock()
if failed {
// A failed sync leaves its persisted error banner behind. Retry
// hourly so transient provider failures clear without waiting a
// day, while bounding unattended traffic toward the provider.
timer.Reset(time.Hour)
} else {
timer.Reset(24 * time.Hour)
}
}
}
internal/app/sync_test.go
+3 -3
View File
@@ -40,7 +40,7 @@ func (b *bankScenario) Status(context.Context, string) (banking.SessionStatus, e
func (b *bankScenario) Balances(context.Context, string) ([]banking.Balance, error) {
return []banking.Balance{{Amount: "100.00", Currency: "EUR", Type: "CLBD"}}, nil
}
func (b *bankScenario) Transactions(_ context.Context, a domain.Account, from, to string) ([]domain.Facts, error) {
func (b *bankScenario) Transactions(_ context.Context, a domain.Account, from, to string, _ bool) ([]domain.Facts, error) {
if b.fail {
return nil, errors.New("offline")
}
@@ -153,9 +153,9 @@ func (b *sessionBank) Status(_ context.Context, id string) (banking.SessionStatu
return b.statuses[id], b.failures[id]
}
func (b *sessionBank) Transactions(ctx context.Context, account domain.Account, from, to string) ([]domain.Facts, error) {
func (b *sessionBank) Transactions(ctx context.Context, account domain.Account, from, to string, longest bool) ([]domain.Facts, error) {
b.fetched = append(b.fetched, account.ID)
return b.bankScenario.Transactions(ctx, account, from, to)
return b.bankScenario.Transactions(ctx, account, from, to, longest)
}
func TestSyncSessionRateLimitPreservesBindingsAndRecovers(t *testing.T) {