Respect provider rate limits and preserve bank connections on throttling

2026-09-10 17:25:09 +02:00
parent 2259db3e85
commit ba3ea6ae5a
14 changed files with 1250 additions and 95 deletions
@@ -3,12 +3,17 @@ package app
 import (
 	"context"
 	"errors"
+	"fmt"
+	"io"
+	"net/http"
 	"reflect"
+	"strings"
 	"testing"
 	"time"

 	"finance-duck/internal/banking"
 	"finance-duck/internal/domain"
+	"finance-duck/internal/ratelimit"
 )

 type bankScenario struct {
@@ -22,11 +27,15 @@ func (b *bankScenario) Authorize(context.Context, string, string, string) (strin
 func (b *bankScenario) Exchange(context.Context, string) (banking.Session, error) {
 	return b.session, nil
 }
-func (b *bankScenario) Status(context.Context, string) (banking.Session, error) {
+func (b *bankScenario) Status(context.Context, string) (banking.SessionStatus, error) {
 	if b.fail {
-		return banking.Session{}, errors.New("expired")
+		return banking.SessionStatus{}, errors.New("expired")
 	}
-	return b.session, nil
+	status := banking.SessionStatus{ValidUntil: b.session.ValidUntil}
+	for _, account := range b.session.Accounts {
+		status.AccountIDs = append(status.AccountIDs, account.ExternalAccountID)
+	}
+	return status, nil
 }
 func (b *bankScenario) Balances(context.Context, string) ([]banking.Balance, error) {
 	return []banking.Balance{{Amount: "100.00", Currency: "EUR", Type: "CLBD"}}, nil
@@ -110,3 +119,196 @@ func TestReconnectReplacesOldConsentWithoutDuplicatingLocalAccount(t *testing.T)
 		t.Fatal("authorization state replay was accepted")
 	}
 }
+
+// Exercise the controller's typed error without contacting a provider or waiting.
+func bankRateError(t *testing.T) error {
+	t.Helper()
+	var controller ratelimit.Controller
+	ctx := context.Background()
+	if err := controller.Acquire(ctx); err != nil {
+		t.Fatal(err)
+	}
+	defer controller.Release()
+	_, err := controller.Do(ctx, func(context.Context) (*http.Response, error) {
+		return &http.Response{
+			StatusCode: http.StatusTooManyRequests,
+			Header:     http.Header{"Retry-After": []string{"300"}},
+			Body:       io.NopCloser(strings.NewReader("private provider response")),
+		}, nil
+	}, false)
+	if err == nil {
+		t.Fatal("rate limit response was accepted")
+	}
+	return fmt.Errorf("private provider response: %w", err)
+}
+
+type sessionBank struct {
+	bankScenario
+	statuses map[string]banking.SessionStatus
+	failures map[string]error
+	fetched  []string
+}
+
+func (b *sessionBank) Status(_ context.Context, id string) (banking.SessionStatus, error) {
+	return b.statuses[id], b.failures[id]
+}
+
+func (b *sessionBank) Transactions(ctx context.Context, account domain.Account, from, to string) ([]domain.Facts, error) {
+	b.fetched = append(b.fetched, account.ID)
+	return b.bankScenario.Transactions(ctx, account, from, to)
+}
+
+func TestSyncSessionRateLimitPreservesBindingsAndRecovers(t *testing.T) {
+	a, s := testApp(t)
+	ctx := context.Background()
+	s, err := a.Mutate(ctx, s.Revision, func(d *domain.Dataset) error {
+		d.Accounts[0].ExternalAccountID = "main_uid"
+		for _, id := range []string{"mwst", "tax", "independent"} {
+			d.Accounts = append(d.Accounts, domain.Account{ID: id, DisplayName: id, Currency: "EUR", Active: true, ExternalAccountID: id + "_uid"})
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	expiry := time.Now().Add(24 * time.Hour).Format(time.RFC3339)
+	limited := banking.Session{ID: "limited", ValidUntil: expiry}
+	healthy := banking.Session{ID: "healthy", ValidUntil: expiry}
+	for _, account := range s.Data.Accounts {
+		if account.ID == "independent" {
+			healthy.Accounts = append(healthy.Accounts, account)
+		} else {
+			limited.Accounts = append(limited.Accounts, account)
+		}
+	}
+	a.ops.Sessions = []banking.Session{limited, healthy}
+	a.ops.Consents["limited"] = Consent{Institution: "N26", Country: "DE", HistoryMonths: 24}
+	a.ops.Consents["healthy"] = Consent{Institution: "Other", Country: "DE", HistoryMonths: 12}
+	b := &sessionBank{
+		statuses: map[string]banking.SessionStatus{
+			"limited": {ValidUntil: expiry, AccountIDs: []string{"main_uid", "mwst_uid", "tax_uid"}},
+			// Another consent must not authorize an account whose own status failed.
+			"healthy": {ValidUntil: expiry, AccountIDs: []string{"independent_uid", "main_uid"}},
+		},
+		failures: map[string]error{},
+	}
+	a.bank = b
+	before, err := a.Sync(ctx)
+	if err != nil || len(before.Data.Transactions) != 4 {
+		t.Fatalf("initial sync: transactions=%d, error=%v, sync error=%s", len(before.Data.Transactions), err, before.Status.SyncError)
+	}
+	old := time.Now().Add(-48 * time.Hour).UTC().Format(time.RFC3339)
+	a.ops.LastSync = old
+	for _, account := range before.Data.Accounts {
+		a.ops.AccountSync[account.ID] = old
+	}
+	b.fetched = nil
+	b.failures["limited"] = bankRateError(t)
+	failed, err := a.Sync(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Count(failed.Status.SyncError, "429") != 1 || strings.Contains(failed.Status.SyncError, "bank connection unavailable") || strings.Contains(failed.Status.SyncError, "private provider response") || !strings.Contains(failed.Status.SyncError, "retry") {
+		t.Fatal("session rate limit was duplicated, obscured, or exposed private data")
+	}
+	if !reflect.DeepEqual(b.fetched, []string{"independent"}) {
+		t.Fatal("failed consent authorized retrieval or independent consent stopped syncing")
+	}
+	if !reflect.DeepEqual(before.Data, failed.Data) || !reflect.DeepEqual(before.Sessions, failed.Sessions) || a.ops.LastSync != old {
+		t.Fatal("rate limit changed existing facts, bindings, metadata, or last successful sync")
+	}
+	for _, account := range before.Data.Accounts {
+		if account.ID != "independent" && a.ops.AccountSync[account.ID] != old {
+			t.Fatal("failed account advanced its cursor")
+		}
+	}
+	if a.ops.AccountSync["independent"] == old || a.ops.Consents["limited"].NeedsReconnect || a.ops.Consents["limited"].HistoryMonths != 24 {
+		t.Fatal("rate limit lost consent settings, required reconnect, or stopped the healthy cursor")
+	}
+	a = reopenBankingApp(t, a)
+	a.bank = b
+	if a.ops.Consents["limited"].NeedsReconnect || a.ops.LastSync != old || a.ops.AccountSync["mwst"] != old {
+		t.Fatal("rate failure state did not survive restart safely")
+	}
+	delete(b.failures, "limited")
+	b.fetched = nil
+	recovered, err := a.Sync(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if recovered.Status.SyncError != "" || a.ops.Consents["limited"].Error != "" || a.ops.Consents["limited"].NeedsReconnect || a.ops.LastSync == old {
+		t.Fatal("successful retry did not clear the transient failure")
+	}
+	if !reflect.DeepEqual(before.Data, recovered.Data) || len(b.fetched) != 4 {
+		t.Fatal("recovery duplicated facts or skipped an account")
+	}
+	for _, account := range recovered.Data.Accounts {
+		if a.ops.AccountSync[account.ID] == old {
+			t.Fatal("recovered account cursor did not advance")
+		}
+	}
+}
+
+func TestSyncMissingMembershipStillRejectsAccount(t *testing.T) {
+	a, s, b := backfillApp(t)
+	b.session.Accounts = []domain.Account{s.Data.Accounts[1]}
+	before := domain.Clone(s.Data)
+	last := a.ops.LastSync
+	after, err := a.Sync(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(after.Status.SyncError, s.Data.Accounts[0].DisplayName+": bank connection unavailable") || !a.ops.Consents["current"].NeedsReconnect {
+		t.Fatal("missing account membership was treated as authorized")
+	}
+	if len(b.accounts) != 1 || b.accounts[0].ID != "other" || a.ops.AccountSync[s.Data.Accounts[0].ID] != last || a.ops.LastSync != last {
+		t.Fatal("missing member was fetched or advanced its cursor, or valid member was skipped")
+	}
+	if !reflect.DeepEqual(before.Accounts, after.Data.Accounts) || len(after.Data.Transactions) != 1 || after.Data.Transactions[0].Facts.AccountID != "other" {
+		t.Fatal("missing membership changed bindings or imported unauthorized facts")
+	}
+}
+
+func TestSyncTransactionFailuresPreserveProgressAndSafeErrors(t *testing.T) {
+	for _, scenario := range []string{"rate limit", "reconnect", "private response"} {
+		t.Run(scenario, func(t *testing.T) {
+			a, s, b := backfillApp(t)
+			s = seed(t, a, s)
+			old := a.ops.LastSync
+			cursors := map[string]string{}
+			for id, cursor := range a.ops.AccountSync {
+				cursors[id] = cursor
+			}
+			switch scenario {
+			case "rate limit":
+				b.fetchErr = bankRateError(t)
+			case "reconnect":
+				b.fetchErr = fmt.Errorf("private provider response: %w", banking.ErrReconnect)
+			case "private response":
+				b.fetchErr = errors.New("private provider response")
+			}
+			failed, err := a.Sync(context.Background())
+			if err != nil {
+				t.Fatal(err)
+			}
+			meta := a.ops.Consents["current"]
+			if failed.Status.SyncError == "" || meta.Error == "" || strings.Contains(failed.Status.SyncError+meta.Error, "private provider response") {
+				t.Fatal("transaction failure was lost or exposed provider data")
+			}
+			if scenario == "rate limit" && (!strings.Contains(failed.Status.SyncError, "429") || !strings.Contains(meta.Error, "429") || !strings.Contains(meta.Error, "retry")) {
+				t.Fatal("transaction rate error was obscured")
+			}
+			if meta.NeedsReconnect != (scenario == "reconnect") {
+				t.Fatal("transaction failure classified consent incorrectly")
+			}
+			if !reflect.DeepEqual(s.Data, failed.Data) || !reflect.DeepEqual(cursors, a.ops.AccountSync) || a.ops.LastSync != old {
+				t.Fatal("failed retrieval imported partial data or advanced synchronization")
+			}
+			b.fetchErr = nil
+			recovered, err := a.Sync(context.Background())
+			if err != nil || recovered.Status.SyncError != "" || a.ops.Consents["current"].Error != "" || a.ops.Consents["current"].NeedsReconnect {
+				t.Fatal("successful retrieval did not clear the failure")
+			}
+		})
+	}
+}