Pace provider traffic and identify genuine foreground bank requests

2026-09-10 17:47:44 +02:00
parent ba3ea6ae5a
commit 4324660888
13 changed files with 817 additions and 32 deletions
@@ -17,8 +17,10 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
 	"time"

@@ -62,6 +64,8 @@ type EnableBanking struct {
 	key         *rsa.PrivateKey
 	redirectURL string
 	requests    ratelimit.Controller
+	// Accessed only while requests is acquired. Keys exclude pagination/query data.
+	backgroundQuotas map[string]time.Time
 }

 var _ Provider = (*EnableBanking)(nil)
@@ -116,7 +120,7 @@ func NewEnableBanking(appID string, keyPEM []byte, redirectURL string) (*EnableB
 	if err = key.Validate(); err != nil {
 		return nil, fmt.Errorf("invalid Enable Banking RSA private key")
 	}
-	return &EnableBanking{HTTPClient: &http.Client{Timeout: 30 * time.Second}, BaseURL: "https://api.enablebanking.com", appID: appID, key: key, redirectURL: redirectURL}, nil
+	return &EnableBanking{HTTPClient: &http.Client{Timeout: 30 * time.Second}, BaseURL: "https://api.enablebanking.com", appID: appID, key: key, redirectURL: redirectURL, requests: ratelimit.Controller{MinimumInterval: time.Second, InitialBackoff: 30 * time.Second}}, nil
 }
 func (p *EnableBanking) jwt() (string, error) {
 	if p.key == nil || p.appID == "" {
@@ -133,11 +137,114 @@ func (p *EnableBanking) jwt() (string, error) {
 	}
 	return unsigned + "." + base64.RawURLEncoding.EncodeToString(signature), nil
 }
+
+// PSU contains only the documented, nonsecret browser metadata for a person
+// actively requesting account data. It must never be stored on a shared client.
+type PSU struct {
+	IPAddress      string
+	UserAgent      string
+	Accept         string
+	AcceptCharset  string
+	AcceptEncoding string
+	AcceptLanguage string
+}
+
+type psuContextKey struct{}
+
+// WithPSU marks this request context as user initiated. Call only at a manual
+// HTTP boundary, never for scheduled retrieval. Unknown metadata stays empty.
+func WithPSU(ctx context.Context, psu PSU) context.Context {
+	return context.WithValue(ctx, psuContextKey{}, psu)
+}
+
+func (psu PSU) setHeaders(header http.Header) {
+	if ip := net.ParseIP(psu.IPAddress); ip != nil {
+		header.Set("Psu-Ip-Address", ip.String())
+	}
+	for _, field := range []struct{ name, value string }{
+		{"Psu-User-Agent", psu.UserAgent},
+		{"Psu-Accept", psu.Accept},
+		{"Psu-Accept-Charset", psu.AcceptCharset},
+		{"Psu-Accept-Encoding", psu.AcceptEncoding},
+		{"Psu-Accept-Language", psu.AcceptLanguage},
+	} {
+		if field.value != "" {
+			header.Set(field.name, field.value)
+		}
+	}
+}
+
+// accountDataPath admits only the internally generated account GET endpoints.
+// The escaped account segment is retained; dates and continuation keys are not.
+func accountDataPath(method, path string) string {
+	if method != http.MethodGet {
+		return ""
+	}
+	path, _, _ = strings.Cut(path, "?")
+	parts := strings.Split(path, "/")
+	if len(parts) != 4 || parts[0] != "" || parts[1] != "accounts" || (parts[3] != "balances" && parts[3] != "transactions") {
+		return ""
+	}
+	account, err := url.PathUnescape(parts[2])
+	if err != nil || account == "" || account == "." || account == ".." || strings.ContainsAny(account, "/\\") || url.PathEscape(account) != parts[2] {
+		return ""
+	}
+	return path
+}
+
+// BackgroundQuotaError identifies the bank's background-only account quota.
+// It preserves RateLimitError identity and exposes no provider response text.
+type BackgroundQuotaError struct {
+	*ratelimit.RateLimitError
+}
+
+func (e *BackgroundQuotaError) Error() string {
+	return "background bank retrieval quota (normally six hours): " + e.RateLimitError.Error()
+}
+
+func (e *BackgroundQuotaError) Unwrap() error {
+	return e.RateLimitError
+}
+
+// backgroundRetryAt follows the bank's six-hour guidance, never shortening a
+// longer provider deadline. Zero retains the limiter's unbounded-delay meaning.
+func backgroundRetryAt(header string, now time.Time) time.Time {
+	next := now.Add(6 * time.Hour)
+	header = strings.TrimSpace(header)
+	if header == "" {
+		return next
+	}
+	if strings.Trim(header, "0123456789") == "" {
+		seconds, err := strconv.ParseUint(header, 10, 64)
+		if err != nil || seconds > uint64((1<<63-1)/int64(time.Second)) {
+			return time.Time{}
+		}
+		if delay := time.Duration(seconds) * time.Second; delay > 6*time.Hour {
+			return now.Add(delay)
+		}
+	} else if date, err := http.ParseTime(header); err == nil && date.After(next) {
+		return date
+	}
+	return next
+}
 func (p *EnableBanking) request(ctx context.Context, method, path string, input, output any) error {
 	if err := p.requests.Acquire(ctx); err != nil {
 		return fmt.Errorf("Enable Banking: %w", err)
 	}
 	defer p.requests.Release()
+	scope := accountDataPath(method, path)
+	psu, foreground := ctx.Value(psuContextKey{}).(PSU)
+	now := time.Now()
+	for key, next := range p.backgroundQuotas {
+		if !next.IsZero() && !next.After(now) {
+			delete(p.backgroundQuotas, key)
+		}
+	}
+	if scope != "" && !foreground {
+		if next, limited := p.backgroundQuotas[scope]; limited {
+			return fmt.Errorf("Enable Banking: %w", &BackgroundQuotaError{ratelimit.NewError(next)})
+		}
+	}
 	token, err := p.jwt()
 	if err != nil {
 		return err
@@ -160,6 +267,9 @@ func (p *EnableBanking) request(ctx context.Context, method, path string, input,
 	}
 	req.Header.Set("Authorization", "Bearer "+token)
 	req.Header.Set("Accept", "application/json")
+	if scope != "" && foreground {
+		psu.setHeaders(req.Header)
+	}
 	if input != nil {
 		req.Header.Set("Content-Type", "application/json")
 	}
@@ -184,6 +294,25 @@ func (p *EnableBanking) request(ctx context.Context, method, path string, input,
 			}
 			return nil, errors.New("connection failed")
 		}
+		if response.StatusCode == http.StatusTooManyRequests && scope != "" && !foreground {
+			// Inspect only a small structured error envelope, never exposing its
+			// detail/message or retaining response bytes. Other 429s remain the
+			// common controller's responsibility (including body closure).
+			const errorLimit = 16 << 10
+			body, readErr := io.ReadAll(io.LimitReader(response.Body, errorLimit+1))
+			var envelope struct {
+				Error string `json:"error"`
+			}
+			if readErr == nil && len(body) <= errorLimit && json.Unmarshal(body, &envelope) == nil && envelope.Error == "ASPSP_RATE_LIMIT_EXCEEDED" {
+				next := backgroundRetryAt(response.Header.Get("Retry-After"), time.Now())
+				if p.backgroundQuotas == nil {
+					p.backgroundQuotas = make(map[string]time.Time)
+				}
+				p.backgroundQuotas[scope] = next
+				response.Body.Close()
+				return nil, &BackgroundQuotaError{ratelimit.NewError(next)}
+			}
+		}
 		return response, nil
 	}, method == http.MethodGet)
 	if err != nil {
@@ -39,6 +39,7 @@ func testProvider(t *testing.T, handler http.HandlerFunc) (*EnableBanking, *rsa.
 	t.Cleanup(server.Close)
 	p.BaseURL = server.URL
 	p.HTTPClient = server.Client()
+	p.requests = ratelimit.Controller{}
 	return p, key
 }
 func assertJWT(t *testing.T, r *http.Request, key *rsa.PrivateKey) {
@@ -462,6 +463,185 @@ func TestEnableBankingCanceledRetryRetainsCooldown(t *testing.T) {
 	}
 }

+func TestEnableBankingPSUSurvivesRetryAndPaginationWithoutLeakingToBackground(t *testing.T) {
+	var manualCalls, backgroundCalls atomic.Int32
+	psu := PSU{IPAddress: "203.0.113.42", UserAgent: "test-browser", Accept: "application/json", AcceptCharset: "utf-8", AcceptEncoding: "gzip", AcceptLanguage: "de"}
+	p, _ := testProvider(t, func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/accounts/uid/transactions" {
+			for name, want := range map[string]string{
+				"Psu-Ip-Address": psu.IPAddress, "Psu-User-Agent": psu.UserAgent,
+				"Psu-Accept": psu.Accept, "Psu-Accept-Charset": psu.AcceptCharset,
+				"Psu-Accept-Encoding": psu.AcceptEncoding, "Psu-Accept-Language": psu.AcceptLanguage,
+			} {
+				if got := r.Header.Get(name); got != want {
+					t.Errorf("%s = %q, want %q", name, got, want)
+				}
+			}
+			switch manualCalls.Add(1) {
+			case 1:
+				w.WriteHeader(http.StatusTooManyRequests)
+				fmt.Fprint(w, `{"error":"ASPSP_RATE_LIMIT_EXCEEDED","detail":"private"}`)
+			case 2:
+				fmt.Fprint(w, `{"transactions":[],"continuation_key":"next-private-page"}`)
+			case 3:
+				if r.URL.Query().Get("continuation_key") != "next-private-page" || r.URL.Query().Get("date_from") != "2026-01-01" {
+					t.Error("pagination lost original filters or continuation")
+				}
+				fmt.Fprint(w, `{"transactions":[{"entry_reference":"entry","transaction_amount":{"amount":"1.00","currency":"EUR"},"credit_debit_indicator":"CRDT","status":"BOOK","booking_date":"2026-09-01"}]}`)
+			default:
+				t.Error("unexpected manual replay")
+			}
+			return
+		}
+		backgroundCalls.Add(1)
+		for name := range r.Header {
+			if strings.HasPrefix(strings.ToLower(name), "psu-") {
+				t.Errorf("PSU metadata escaped manual account request: %s", name)
+			}
+		}
+		if r.URL.Path == "/sessions/session" {
+			fmt.Fprint(w, `{"status":"AUTHORIZED","accounts":["uid"],"access":{"valid_until":"2099-01-01T00:00:00Z"}}`)
+		} else {
+			fmt.Fprint(w, `{"balances":[]}`)
+		}
+	})
+	p.requests = ratelimit.Controller{InitialBackoff: time.Millisecond}
+	ctx := WithPSU(context.Background(), psu)
+	account := fixtureDataset().Accounts[0]
+	account.ExternalAccountID = "uid"
+	rows, err := p.Transactions(ctx, account, "2026-01-01", "")
+	if err != nil || len(rows) != 1 || rows[0].ExternalID != "entry" {
+		t.Fatalf("manual history failed: %+v %v", rows, err)
+	}
+	if _, err := p.Balances(context.Background(), "uid"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := p.Status(ctx, "session"); err != nil {
+		t.Fatal(err)
+	}
+	if manualCalls.Load() != 3 || backgroundCalls.Load() != 2 {
+		t.Fatalf("unexpected request counts: manual=%d other=%d", manualCalls.Load(), backgroundCalls.Load())
+	}
+}
+
+func TestEnableBankingBackgroundQuotaIsScopedAndExpires(t *testing.T) {
+	var calls atomic.Int32
+	p, _ := testProvider(t, func(w http.ResponseWriter, r *http.Request) {
+		if calls.Add(1) == 1 {
+			w.WriteHeader(http.StatusTooManyRequests)
+			fmt.Fprint(w, `{"error":"ASPSP_RATE_LIMIT_EXCEEDED","message":"private-bank-message"}`)
+			return
+		}
+		if strings.HasSuffix(r.URL.Path, "/transactions") {
+			fmt.Fprint(w, `{"transactions":[]}`)
+		} else {
+			fmt.Fprint(w, `{"balances":[]}`)
+		}
+	})
+	before := time.Now()
+	_, err := p.Balances(context.Background(), "uid")
+	var first *ratelimit.RateLimitError
+	if !errors.As(err, &first) || first.RetryAt().Before(before.Add(6*time.Hour)) || first.RetryAt().After(time.Now().Add(6*time.Hour)) || strings.Contains(err.Error(), "private") || errors.Is(err, ErrReconnect) {
+		t.Fatalf("missing safe six-hour bank quota: %v", err)
+	}
+	var quota *BackgroundQuotaError
+	if !errors.As(err, &quota) || !quota.RetryAt().Equal(first.RetryAt()) {
+		t.Fatalf("background quota identity was lost: %v", err)
+	}
+	_, err = p.Balances(context.Background(), "uid")
+	var retained *ratelimit.RateLimitError
+	if !errors.As(err, &retained) || !retained.RetryAt().Equal(first.RetryAt()) || calls.Load() != 1 {
+		t.Fatalf("background quota was replayed: calls=%d err=%v", calls.Load(), err)
+	}
+	if _, err := p.Balances(WithPSU(context.Background(), PSU{IPAddress: "203.0.113.42", UserAgent: "test-browser"}), "uid"); err != nil {
+		t.Fatalf("background quota blocked real user: %v", err)
+	}
+	_, err = p.Balances(context.Background(), "uid")
+	if !errors.As(err, &quota) || !quota.RetryAt().Equal(first.RetryAt()) || calls.Load() != 2 {
+		t.Fatalf("foreground success erased background quota: calls=%d err=%v", calls.Load(), err)
+	}
+	if _, err := p.Balances(context.Background(), "other-uid"); err != nil {
+		t.Fatalf("background quota blocked another account: %v", err)
+	}
+	account := fixtureDataset().Accounts[0]
+	account.ExternalAccountID = "uid"
+	if _, err := p.Transactions(context.Background(), account, "", ""); err != nil {
+		t.Fatalf("balance quota blocked transaction endpoint: %v", err)
+	}
+	// Simulate the stored deadline passing without a six-hour wall-clock wait.
+	p.backgroundQuotas["/accounts/uid/balances"] = time.Now().Add(-time.Second)
+	if _, err := p.Balances(context.Background(), "uid"); err != nil {
+		t.Fatalf("expired quota blocked retrieval: %v", err)
+	}
+	if calls.Load() != 5 {
+		t.Fatalf("unexpected bank replays: %d", calls.Load())
+	}
+}
+
+func TestEnableBankingBackgroundQuotaPreservesLongerHints(t *testing.T) {
+	for _, hint := range []string{"43200", time.Now().Add(24 * time.Hour).UTC().Format(http.TimeFormat), "999999999999999999999999999999999"} {
+		t.Run(hint, func(t *testing.T) {
+			var calls atomic.Int32
+			p, _ := testProvider(t, func(w http.ResponseWriter, r *http.Request) {
+				calls.Add(1)
+				w.Header().Set("Retry-After", hint)
+				w.WriteHeader(http.StatusTooManyRequests)
+				fmt.Fprint(w, `{"error":"ASPSP_RATE_LIMIT_EXCEEDED"}`)
+			})
+			before := time.Now()
+			_, err := p.Balances(context.Background(), "uid")
+			var limit *ratelimit.RateLimitError
+			if !errors.As(err, &limit) || calls.Load() != 1 {
+				t.Fatalf("quota replayed or lost: calls=%d err=%v", calls.Load(), err)
+			}
+			switch hint {
+			case "43200":
+				if limit.RetryAt().Before(before.Add(12 * time.Hour)) {
+					t.Fatal("shortened numeric provider hint")
+				}
+			case "999999999999999999999999999999999":
+				if !limit.RetryAt().IsZero() {
+					t.Fatal("overflow became a finite short retry")
+				}
+			default:
+				date, _ := http.ParseTime(hint)
+				if !limit.RetryAt().Equal(date) {
+					t.Fatal("shortened absolute provider hint")
+				}
+			}
+		})
+	}
+}
+
+func TestEnableBankingOnlyExactBoundedBankErrorUsesBackgroundQuota(t *testing.T) {
+	for name, payload := range map[string]string{
+		"message only": `{"error":"OTHER","message":"ASPSP_RATE_LIMIT_EXCEEDED"}`,
+		"prefix":       `{"error":"ASPSP_RATE_LIMIT_EXCEEDED_OTHER"}`,
+		"nested":       `{"error":{"code":"ASPSP_RATE_LIMIT_EXCEEDED"}}`,
+		"oversized":    `{"error":"ASPSP_RATE_LIMIT_EXCEEDED","detail":"` + strings.Repeat("x", 16<<10) + `"}`,
+	} {
+		t.Run(name, func(t *testing.T) {
+			var calls atomic.Int32
+			p, _ := testProvider(t, func(w http.ResponseWriter, r *http.Request) {
+				calls.Add(1)
+				w.Header().Set("Retry-After", "300")
+				w.WriteHeader(http.StatusTooManyRequests)
+				fmt.Fprint(w, payload)
+			})
+			_, err := p.Balances(context.Background(), "uid")
+			var first *ratelimit.RateLimitError
+			if !errors.As(err, &first) || first.RetryAt().After(time.Now().Add(6*time.Minute)) {
+				t.Fatalf("unrecognized error used bank quota: %v", err)
+			}
+			_, err = p.Balances(WithPSU(context.Background(), PSU{IPAddress: "203.0.113.42"}), "other")
+			var common *ratelimit.RateLimitError
+			if !errors.As(err, &common) || !common.RetryAt().Equal(first.RetryAt()) || calls.Load() != 1 {
+				t.Fatalf("generic platform cooldown not shared: calls=%d err=%v", calls.Load(), err)
+			}
+		})
+	}
+}
+
 func TestEnableBankingValidatesUploadedCredentials(t *testing.T) {
 	_, key := testProvider(t, func(w http.ResponseWriter, r *http.Request) {
 		t.Error("credential validation must not call provider")