Pace provider traffic and identify genuine foreground bank requests

internal/ratelimit/controller.go

@@ -13,18 +13,30 @@ import (
 )
 
 const (
-	maxRateAttempts = 4
-	maxRateWait     = 2 * time.Minute
+	maxRateAttempts    = 4
+	maxRateWait        = 2 * time.Minute
+	maxLearnedInterval = 30 * time.Second
+	maxBackoff         = 15 * time.Minute
 )
 
 // Controller serializes provider calls and retains their HTTP 429 cooldown.
 // Its zero value is ready for use. A Controller must not be copied after use.
 // The active caller owns retries; other callers fail fast during known cooldowns.
 type Controller struct {
+	// Configure before first use; these fields must remain immutable afterward.
+	// Zero values preserve unpaced requests and a one-second initial backoff.
+	MinimumInterval time.Duration
+	InitialBackoff  time.Duration
+
 	once   sync.Once
 	active chan struct{}
 	mu     sync.Mutex
 	limit  *RateLimitError
+
+	// Only the acquired caller accesses pacing and consecutive-failure state.
+	lastAttempt     time.Time
+	learnedInterval time.Duration
+	backoff         time.Duration
 }
 
 // RateLimitError is a safe provider HTTP 429 error. Its message contains only
@@ -35,6 +47,12 @@ type RateLimitError struct {
 	unbounded bool
 }
 
+// NewError records a trusted provider-specific retry deadline without exposing
+// provider response text. A zero deadline disables automatic retries.
+func NewError(retryAt time.Time) *RateLimitError {
+	return &RateLimitError{next: retryAt, unbounded: retryAt.IsZero()}
+}
+
 func (r *RateLimitError) Error() string {
 	if r.unbounded {
 		return "provider rate limit (HTTP 429): retry time exceeds the supported range; automatic retry disabled"
@@ -128,7 +146,7 @@ func retryLimit(header string, now time.Time, fallback time.Duration) *RateLimit
 // Its per-attempt timeout must not include this controller's retry waiting.
 func (g *Controller) Do(ctx context.Context, attempt func(context.Context) (*http.Response, error), retry bool) (*http.Response, error) {
 	remainingWait := maxRateWait
-	var lastLimit error
+	var lastLimit *RateLimitError
 	canceled := func(err error) error {
 		if lastLimit != nil {
 			return fmt.Errorf("%w: %w", lastLimit, err)
@@ -139,6 +157,31 @@ func (g *Controller) Do(ctx context.Context, attempt func(context.Context) (*htt
 		if err := ctx.Err(); err != nil {
 			return nil, canceled(err)
 		}
+		// Pace all attempts, not just retries, including after successful calls.
+		next := g.lastAttempt.Add(max(g.MinimumInterval, g.learnedInterval))
+		if lastLimit != nil && lastLimit.next.After(next) {
+			next = lastLimit.next
+		}
+		delay := time.Until(next)
+		if lastLimit != nil && (lastLimit.unbounded || delay > remainingWait) {
+			return nil, lastLimit
+		}
+		if delay > 0 {
+			if lastLimit != nil {
+				remainingWait -= delay
+			}
+			timer := time.NewTimer(delay)
+			select {
+			case <-ctx.Done():
+				timer.Stop()
+				return nil, canceled(ctx.Err())
+			case <-timer.C:
+			}
+		}
+		if err := ctx.Err(); err != nil {
+			return nil, canceled(err)
+		}
+		g.lastAttempt = time.Now()
 		resp, err := attempt(ctx)
 		if err != nil {
 			if ctx.Err() != nil {
@@ -150,12 +193,36 @@ func (g *Controller) Do(ctx context.Context, attempt func(context.Context) (*htt
 			return nil, err
 		}
 		if resp.StatusCode != http.StatusTooManyRequests {
+			if resp.StatusCode < http.StatusBadRequest {
+				// Recovery resets consecutive-failure escalation, but never
+				// erases learned spacing: a single success is not a quota reset.
+				g.backoff = 0
+			}
 			return resp, nil
 		}
-		limit := retryLimit(resp.Header.Get("Retry-After"), time.Now(), time.Second<<number)
+		if g.backoff <= 0 {
+			g.backoff = g.InitialBackoff
+			if g.backoff <= 0 {
+				g.backoff = time.Second
+			}
+		} else if g.backoff >= maxBackoff/2 {
+			g.backoff = max(g.backoff, maxBackoff)
+		} else {
+			g.backoff *= 2
+		}
+		fallback := max(g.backoff, g.MinimumInterval, g.learnedInterval)
+		limit := retryLimit(resp.Header.Get("Retry-After"), time.Now(), fallback)
 		g.mu.Lock()
 		g.limit = limit
 		g.mu.Unlock()
+		// Keep the most conservative learned cadence for this controller's
+		// lifetime, capped at 30 seconds. The actual provider deadline is never
+		// capped; persistent failures separately escalate up to 15 minutes.
+		learned := maxLearnedInterval
+		if !limit.unbounded {
+			learned = min(learned, time.Until(limit.next))
+		}
+		g.learnedInterval = max(g.learnedInterval, learned)
 		// Never read or expose provider errors, and release each response before
 		// any sleep or retry. Other responses are processed by the caller.
 		resp.Body.Close()
@@ -163,21 +230,9 @@ func (g *Controller) Do(ctx context.Context, attempt func(context.Context) (*htt
 		if err := ctx.Err(); err != nil {
 			return nil, canceled(err)
 		}
-		delay := time.Until(limit.next)
-		if !retry || number == maxRateAttempts-1 || limit.unbounded || delay > remainingWait {
+		if !retry || number == maxRateAttempts-1 {
 			return nil, limit
 		}
-		if delay <= 0 {
-			continue
-		}
-		remainingWait -= delay
-		timer := time.NewTimer(delay)
-		select {
-		case <-ctx.Done():
-			timer.Stop()
-			return nil, canceled(ctx.Err())
-		case <-timer.C:
-		}
 	}
 	return nil, lastLimit
 }