Add per-account historical bank imports without resetting sync cursors

This commit is contained in:
Lars Nolden
2026-09-10 17:02:17 +02:00
parent f4c7d54575
commit 2259db3e85
7 changed files with 561 additions and 8 deletions
+13
View File
@@ -219,6 +219,19 @@ Sync now can retry sooner. Balances are fetched
on demand, with exact amount/currency/type values, rather than inferred from an
incomplete historical journal.
Historical imports for connected accounts
-----------------------------------------
Accounts -> account card -> Import older history opens a per-account dialog.
Choose Months back (12 by default, whole numbers 1-120) and confirm. The request
covers the selected past calendar months through today, subject to bank limits.
Only the chosen account is fetched. Existing transactions are deduplicated, and
new records follow normal transfer matching and classification. The dialog shows
the imported count, including zero for a repeated range with no new records.
Normal sync cursors, last-sync time and saved initial-history choices are not
changed. Inactive connected accounts may import history manually. Expired or
revoked bank authorization must be reconnected first. On failure, the UI refreshes
the journal to reveal any records already committed; it does not retry the import.
CSV and identity
----------------
The initial real CSV adapter is N26, not generic ING/Kontist CSV autodetection.
+6
View File
@@ -142,6 +142,12 @@ Finance Duck verifies the callback state, exchanges the returned code for a `ses
Initial synchronization requests the selected number of **calendar months of booked transactions per account**, defaulting to **12 months**. The bank may provide less history. The choice is saved with the bank connection and reused on reconnection. Subsequent daily synchronization overlaps each account's last successful sync by **14 days**. **Sync now** starts a manual synchronization. Existing accounts keep their successful-sync cursors: changing the history choice or reconnecting does **not** backfill them. Older history can be imported with CSV.
### Import older history for a connected account
Open **Accounts**, find the account, and click **Import older history**. Choose **Months back** (default **12**, whole numbers from **1 to 120**) and confirm. This requests that account's booked transactions from the selected number of calendar months ago through today; the bank may provide less history.
The dialog reports how many new transactions were imported, including zero when nothing new was found. Repeated or overlapping ranges skip existing transactions. New records use the normal import and classification process. The account's regular sync cursor, last-sync time, and saved initial-history choice stay unchanged; other accounts are not fetched. Inactive connected accounts can also use this explicit action. If authorization has expired, reconnect first.
### Reauthorize expired consent
When consent expires or is revoked, the dashboard shows a warning such as **“ING needs reconnection.”** Open **Accounts** and click **Reconnect ING**. The bank and country are already selected.
+249
View File
@@ -0,0 +1,249 @@
package app
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"time"
"finance-duck/internal/banking"
"finance-duck/internal/domain"
)
type backfillBank struct {
historyBank
statusIDs []string
accounts []domain.Account
toDates []string
statusErr error
fetchErr error
}
func (b *backfillBank) Status(ctx context.Context, id string) (banking.Session, error) {
b.statusIDs = append(b.statusIDs, id)
if b.statusErr != nil {
return banking.Session{}, b.statusErr
}
return b.historyBank.Status(ctx, id)
}
func (b *backfillBank) Transactions(ctx context.Context, account domain.Account, from, to string) ([]domain.Facts, error) {
b.accounts = append(b.accounts, account)
b.toDates = append(b.toDates, to)
rows, err := b.historyBank.Transactions(ctx, account, from, to)
if b.fetchErr != nil {
// A provider can fail after accumulating a page: none of it is importable.
return rows, b.fetchErr
}
return rows, err
}
func backfillApp(t *testing.T) (*App, State, *backfillBank) {
t.Helper()
a, s := testApp(t)
s, err := a.Mutate(context.Background(), s.Revision, func(d *domain.Dataset) error {
d.Accounts[0].ExternalAccountID = "selected_uid"
d.Accounts = append(d.Accounts, domain.Account{ID: "other", DisplayName: "Other", Currency: "EUR", Active: true, ExternalAccountID: "other_uid"})
return nil
})
if err != nil {
t.Fatal(err)
}
session := banking.Session{ID: "current", ValidUntil: time.Now().Add(24 * time.Hour).Format(time.RFC3339), Accounts: s.Data.Accounts}
b := &backfillBank{historyBank: historyBank{bankScenario: bankScenario{session: session}}}
a.bank = b
a.ops.Sessions = []banking.Session{session}
a.ops.Consents[session.ID] = Consent{Institution: "Bank", Country: "DE", HistoryMonths: 3}
a.ops.LastSync = time.Now().UTC().Add(-time.Hour).Format(time.RFC3339)
for _, account := range s.Data.Accounts {
a.ops.AccountSync[account.ID] = a.ops.LastSync
}
if err := a.saveOps(); err != nil {
t.Fatal(err)
}
return a, s, b
}
func TestBackfillImportsOlderFactsOnceWithoutChangingSyncState(t *testing.T) {
a, _, b := backfillApp(t)
ctx := context.Background()
s, err := a.Sync(ctx)
if err != nil {
t.Fatal(err)
}
if len(s.Data.Transactions) != 2 {
t.Fatal("recent sync did not seed both accounts")
}
s, err = a.Mutate(ctx, s.Revision, func(d *domain.Dataset) error {
d.Accounts[0].Active = false // Inactive disables scheduling, not explicit backfill.
return nil
})
if err != nil {
t.Fatal(err)
}
old := a.ops.Sessions[0]
old.ID = "superseded"
a.ops.Sessions = append([]banking.Session{old}, a.ops.Sessions...)
a.ops.Consents["superseded"] = Consent{HistoryMonths: 120}
meta := a.ops.Consents["current"]
meta.Error = "Previous temporary failure"
a.ops.Consents["current"] = meta
a.ops.SyncError = "Previous temporary failure"
if err := a.saveOps(); err != nil {
t.Fatal(err)
}
beforeOps, err := json.Marshal(a.ops)
if err != nil {
t.Fatal(err)
}
b.statusIDs, b.accounts, b.fromDates, b.toDates = nil, nil, nil, nil
// Provider-local IDs need not match our canonical account ID.
providerAccount := s.Data.Accounts[0]
providerAccount.ID = "provider_generated_id"
b.session.Accounts = []domain.Account{providerAccount}
fromBefore := time.Now().UTC().AddDate(0, -24, 0).Format("2006-01-02")
toBefore := time.Now().UTC().Format("2006-01-02")
result, err := a.Backfill(ctx, s.Revision, s.Data.Accounts[0].ID, 24)
if err != nil {
t.Fatal(err)
}
fromAfter := time.Now().UTC().AddDate(0, -24, 0).Format("2006-01-02")
toAfter := time.Now().UTC().Format("2006-01-02")
if result.Imported != 2 || len(result.State.Data.Transactions) != 4 {
t.Fatalf("older history import count is wrong: %+v", result)
}
if !reflect.DeepEqual(b.statusIDs, []string{"current"}) || !reflect.DeepEqual(b.accounts, []domain.Account{s.Data.Accounts[0]}) {
t.Fatal("backfill did not use only the newest matching consent and canonical account")
}
if len(b.fromDates) != 1 || (b.fromDates[0] != fromBefore && b.fromDates[0] != fromAfter) || (b.toDates[0] != toBefore && b.toDates[0] != toAfter) {
t.Fatalf("backfill did not request the selected calendar-month range: %v to %v", b.fromDates, b.toDates)
}
for _, existing := range s.Data.Transactions {
found := false
for _, tx := range result.State.Data.Transactions {
if tx.Facts.ID == existing.Facts.ID {
found = reflect.DeepEqual(tx, existing)
}
}
if !found {
t.Fatal("backfill changed an existing transaction")
}
}
counts := map[string]int{}
for _, tx := range result.State.Data.Transactions {
counts[tx.Facts.AccountID]++
}
if counts[s.Data.Accounts[0].ID] != 3 || counts["other"] != 1 || !reflect.DeepEqual(result.State.Data.Accounts, s.Data.Accounts) {
t.Fatal("backfill changed unrelated accounts or imported their history")
}
again, err := a.Backfill(ctx, result.State.Revision, s.Data.Accounts[0].ID, 24)
if err != nil {
t.Fatal(err)
}
if again.Imported != 0 || !reflect.DeepEqual(again.State.Data, result.State.Data) {
t.Fatal("repeated historical range was not idempotent")
}
afterOps, err := json.Marshal(a.ops)
if err != nil || string(afterOps) != string(beforeOps) {
t.Fatal("backfill changed operational state")
}
a = reopenBankingApp(t, a)
afterOps, err = json.Marshal(a.ops)
if err != nil || string(afterOps) != string(beforeOps) {
t.Fatal("backfill changed persisted cursors, consent history or bindings")
}
persisted, err := a.Snapshot(ctx)
if err != nil || !reflect.DeepEqual(persisted.Data, result.State.Data) {
t.Fatal("historical facts did not survive restart")
}
}
func TestBackfillRejectsUnsafePrerequisitesBeforeProviderContact(t *testing.T) {
for _, scenario := range []string{"stale revision", "zero months", "too many months", "missing account", "local account", "unbound account", "mismatched provider ID", "mismatched canonical ID", "expired consent", "reconnect required"} {
t.Run(scenario, func(t *testing.T) {
a, s, b := backfillApp(t)
rev, id, months := s.Revision, s.Data.Accounts[0].ID, 12
switch scenario {
case "stale revision":
rev = "stale"
case "zero months":
months = 0
case "too many months":
months = 121
case "missing account":
id = "missing"
case "local account":
var err error
s, err = a.Mutate(context.Background(), rev, func(d *domain.Dataset) error { d.Accounts[0].ExternalAccountID = ""; return nil })
if err != nil {
t.Fatal(err)
}
rev = s.Revision
case "unbound account":
a.ops.Sessions = nil
case "mismatched provider ID":
a.ops.Sessions[0].Accounts[0].ExternalAccountID = "wrong_uid"
case "mismatched canonical ID":
a.ops.Sessions[0].Accounts[0].ID = "wrong_id"
case "expired consent":
a.ops.Sessions[0].ValidUntil = time.Now().Add(-time.Hour).Format(time.RFC3339)
case "reconnect required":
a.ops.Consents["current"] = Consent{NeedsReconnect: true}
}
if _, err := a.Backfill(context.Background(), rev, id, months); err == nil {
t.Fatal("unsafe backfill was accepted")
}
if len(b.statusIDs) != 0 || len(b.accounts) != 0 {
t.Fatal("unsafe prerequisites contacted the provider")
}
})
}
}
func TestBackfillProviderFailuresDoNotImportPartialData(t *testing.T) {
for _, scenario := range []string{"not configured", "unavailable", "revoked", "provider expired", "account absent", "partial retrieval"} {
t.Run(scenario, func(t *testing.T) {
a, s, b := backfillApp(t)
s = seed(t, a, s)
switch scenario {
case "not configured":
a.bank = nil
case "unavailable":
b.statusErr = errors.New("private provider response")
case "revoked":
b.statusErr = banking.ErrReconnect
case "provider expired":
b.session.ValidUntil = time.Now().Add(-time.Hour).Format(time.RFC3339)
case "account absent":
b.session.Accounts = []domain.Account{{ID: s.Data.Accounts[0].ID, ExternalAccountID: "wrong_uid"}}
case "partial retrieval":
b.fetchErr = errors.New("private provider response")
}
beforeOps, err := json.Marshal(a.ops)
if err != nil {
t.Fatal(err)
}
result, err := a.Backfill(context.Background(), s.Revision, s.Data.Accounts[0].ID, 24)
if err == nil || result.Imported != 0 {
t.Fatal("failed provider request reported a successful import")
}
if strings.Contains(err.Error(), "private provider response") {
t.Fatal("provider error exposed private response data")
}
if scenario != "partial retrieval" && len(b.accounts) != 0 {
t.Fatal("unavailable consent reached transaction retrieval")
}
current, err := a.Snapshot(context.Background())
if err != nil || current.Revision != s.Revision || !reflect.DeepEqual(current.Data, s.Data) {
t.Fatal("provider failure changed canonical data")
}
afterOps, err := json.Marshal(a.ops)
if err != nil || string(afterOps) != string(beforeOps) {
t.Fatal("provider failure changed operational state")
}
})
}
}
+70
View File
@@ -92,6 +92,76 @@ func (a *App) ImportCSV(ctx context.Context, rev, accountID string, r io.Reader)
}
return ImportResult{}, errors.New("unknown account")
}
func (a *App) Backfill(ctx context.Context, rev, accountID string, historyMonths int) (ImportResult, error) {
a.mu.Lock()
defer a.mu.Unlock()
if historyMonths < 1 || historyMonths > 120 {
return ImportResult{}, errors.New("history_months must be an integer between 1 and 120")
}
s, err := a.snapshot(ctx)
if err != nil {
return ImportResult{}, err
}
if rev != s.Revision {
return ImportResult{}, errors.New("revision conflict: reload before importing")
}
if a.bank == nil {
return ImportResult{}, errors.New("Enable Banking is not configured")
}
index := slices.IndexFunc(s.Data.Accounts, func(account domain.Account) bool { return account.ID == accountID })
if index < 0 {
return ImportResult{}, errors.New("unknown account")
}
account := s.Data.Accounts[index]
if account.ExternalAccountID == "" {
return ImportResult{}, errors.New("account is not connected")
}
var session *banking.Session
for i := len(a.ops.Sessions) - 1; i >= 0; i-- {
saved := &a.ops.Sessions[i]
if slices.ContainsFunc(saved.Accounts, func(linked domain.Account) bool {
return linked.ID == account.ID && linked.ExternalAccountID == account.ExternalAccountID
}) {
session = saved
break
}
}
if session == nil || session.ID == "" {
return ImportResult{}, errors.New("account is not connected")
}
expiry, err := time.Parse(time.RFC3339, session.ValidUntil)
if a.ops.Consents[session.ID].NeedsReconnect || err != nil || !expiry.After(time.Now()) {
return ImportResult{}, banking.ErrReconnect
}
current, err := a.bank.Status(ctx, session.ID)
if err != nil {
if errors.Is(err, banking.ErrReconnect) {
return ImportResult{}, banking.ErrReconnect
}
return ImportResult{}, errors.New("bank connection unavailable; retry importing history")
}
expiry, err = time.Parse(time.RFC3339, current.ValidUntil)
if err != nil || !expiry.After(time.Now()) {
return ImportResult{}, banking.ErrReconnect
}
if !slices.ContainsFunc(current.Accounts, func(linked domain.Account) bool {
return linked.ExternalAccountID == account.ExternalAccountID
}) {
return ImportResult{}, banking.ErrReconnect
}
now := time.Now().UTC()
facts, err := a.bank.Transactions(ctx, account, now.AddDate(0, -historyMonths, 0).Format("2006-01-02"), now.Format("2006-01-02"))
if err != nil {
if errors.Is(err, banking.ErrReconnect) {
return ImportResult{}, banking.ErrReconnect
}
return ImportResult{}, errors.New("transaction retrieval failed; retry importing history")
}
// Use normal import processing without changing sync cursors or saved consent
// settings, including when the requested range adds no transactions.
return a.importFacts(ctx, s, facts)
}
func (a *App) Authorize(ctx context.Context, institution, country string, historyMonths int) (string, error) {
a.mu.Lock()
defer a.mu.Unlock()
+13
View File
@@ -41,6 +41,7 @@ func New(a *app.App, assets fs.FS, publicURL string) (http.Handler, error) {
s.mux.HandleFunc("POST /api/transactions/{id}", s.transaction)
s.mux.HandleFunc("POST /api/manage", s.manage)
s.mux.HandleFunc("POST /api/import", s.importCSV)
s.mux.HandleFunc("POST /api/backfill", s.backfill)
s.mux.HandleFunc("POST /api/rebuild", func(w http.ResponseWriter, r *http.Request) { v, e := a.Rebuild(r.Context()); respond(w, v, e) })
s.mux.HandleFunc("POST /api/sync", func(w http.ResponseWriter, r *http.Request) { v, e := a.Sync(r.Context()); respond(w, v, e) })
s.mux.HandleFunc("POST /api/settings", s.settings)
@@ -281,6 +282,18 @@ func (s *Server) importCSV(w http.ResponseWriter, r *http.Request) {
v, e := s.app.ImportCSV(r.Context(), r.FormValue("revision"), r.FormValue("account_id"), f)
respond(w, v, e)
}
func (s *Server) backfill(w http.ResponseWriter, r *http.Request) {
var b struct {
Revision string `json:"revision"`
AccountID string `json:"account_id"`
HistoryMonths int `json:"history_months"`
}
if !decode(w, r, &b) {
return
}
v, e := s.app.Backfill(r.Context(), b.Revision, b.AccountID, b.HistoryMonths)
respond(w, v, e)
}
func (s *Server) settings(w http.ResponseWriter, r *http.Request) {
var b app.Settings
if !decode(w, r, &b) {
+209 -6
View File
@@ -17,15 +17,41 @@ interface Balance {
currency: string;
type: string;
}
export function Accounts({
state,
mutate,
acceptState,
}: {
interface AccountsProps {
state: State;
mutate: Mutate;
acceptState: (state: State, message?: string) => void;
}) {
}
export function Accounts(props: AccountsProps) {
const [backfilling, setBackfilling] = useState<Account | null>(null);
return (
<>
<AccountsContent
key={props.state.revision}
{...props}
backfill={setBackfilling}
/>
{backfilling && (
<BackfillHistory
account={backfilling}
state={props.state}
acceptState={props.acceptState}
close={() => setBackfilling(null)}
/>
)}
</>
);
}
// Ordinary account forms reset on a new revision; the backfill dialog must
// survive its own imports and error recovery to retain its range and result.
function AccountsContent({
state,
mutate,
acceptState,
backfill,
}: AccountsProps & { backfill: (account: Account) => void }) {
const [editing, setEditing] = useState<Account | null>(null);
const [deleting, setDeleting] = useState<Account | null>(null);
const [error, setError] = useState("");
@@ -62,6 +88,7 @@ export function Accounts({
state={state}
edit={() => setEditing(account)}
remove={() => setDeleting(account)}
backfill={() => backfill(account)}
onError={setError}
/>
))}
@@ -172,17 +199,47 @@ async function authorize(
throw new Error("Bank authorization returned an unsafe redirect URL.");
window.location.assign(url.href);
}
function backfillUnavailable(account: Account, state: State): string {
if (!state.status.banking_configured)
return "Configure Enable Banking in Settings before importing older history.";
const connection = state.connections.find((c) => c.account_id === account.id);
if (connection?.status === "reconnect_required")
return "Reconnect this account before importing older history.";
if (
!account.id ||
!account.external_account_id ||
!connection ||
(connection.status !== "connected" && connection.status !== "error")
)
return "Connect this account to your bank before importing older history.";
for (let i = state.sessions.length - 1; i >= 0; i--) {
const session = state.sessions[i];
if (
session.accounts.some(
(linked) =>
linked.id === account.id &&
linked.external_account_id === account.external_account_id,
)
)
return session.session_id && Date.parse(session.valid_until) > Date.now()
? ""
: "Reconnect this account to renew its saved bank authorization.";
}
return "Reconnect this account to save a matching bank connection.";
}
function AccountCard({
account,
state,
edit,
remove,
backfill,
onError,
}: {
account: Account;
state: State;
edit: () => void;
remove: () => void;
backfill: () => void;
onError: (error: string) => void;
}) {
const [balances, setBalances] = useState<Balance[] | null>(null);
@@ -191,6 +248,7 @@ function AccountCard({
const connection = state.connections.find((c) => c.account_id === account.id);
const institution = connection?.institution || account.institution;
const needsReconnect = connection?.status === "reconnect_required";
const backfillReason = backfillUnavailable(account, state);
return (
<section className="panel account-card">
<div className="account-card-heading">
@@ -255,6 +313,16 @@ function AccountCard({
{connecting ? "Opening bank…" : `Reconnect ${institution}`}
</button>
)}
<button
className="button secondary"
disabled={connecting || !!backfillReason}
title={backfillReason || undefined}
onClick={backfill}
>
<Upload size={14} />
Import older history
</button>
{backfillReason && <small>{backfillReason}</small>}
</div>
<div className="balance-area">
{balances ? (
@@ -329,6 +397,141 @@ function AccountCard({
</section>
);
}
function BackfillHistory({
account,
state,
acceptState,
close,
}: {
account: Account;
state: State;
acceptState: (state: State, message?: string) => void;
close: () => void;
}) {
const [historyMonths, setHistoryMonths] = useState("12");
const [busy, setBusy] = useState(false);
const [error, setError] = useState("");
const [success, setSuccess] = useState("");
const submitting = useRef(false);
const currentAccount = state.data.accounts.find((a) => a.id === account.id);
const unavailable = currentAccount
? backfillUnavailable(currentAccount, state)
: "This account is no longer saved. Close this dialog and choose an account.";
const closeWhenIdle = () => {
if (!submitting.current) close();
};
return (
<Modal title="Import older history" close={closeWhenIdle}>
<form
aria-busy={busy}
onSubmit={async (e) => {
e.preventDefault();
if (submitting.current || !e.currentTarget.reportValidity()) return;
const months = Number(historyMonths);
if (!Number.isInteger(months) || months < 1 || months > 120) {
setError("Choose a whole number of months from 1 to 120.");
return;
}
if (unavailable) {
setError(unavailable);
return;
}
submitting.current = true;
setBusy(true);
setError("");
setSuccess("");
try {
const response = await request<{ imported: number; state: State }>(
"/api/backfill",
{
revision: state.revision,
account_id: account.id,
history_months: months,
},
);
const message =
response.imported === 0
? `No new transactions imported for ${account.display_name}. Existing transactions were not duplicated.`
: `Imported ${response.imported} new transactions for ${account.display_name}. Existing transactions were not duplicated.`;
acceptState(response.state, message);
setSuccess(message);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
setError(message);
try {
acceptState(await request<State>("/api/state"));
setError(
`${message} The journal has been refreshed in case any records were saved. Review it before trying again; no import was automatically retried.`,
);
} catch (refreshError) {
setError(
`${message} Could not refresh the journal: ${refreshError instanceof Error ? refreshError.message : String(refreshError)} Reload the page to check for saved records before trying again. No import was automatically retried.`,
);
}
} finally {
submitting.current = false;
setBusy(false);
}
}}
>
<div className="form-body">
<p>
Import bank transactions for <strong>{account.display_name}</strong>{" "}
({account.institution} · {account.currency}).
</p>
<ErrorMessage error={error} />
{success && <p role="status">{success}</p>}
<Field
label="Months back"
hint="Request history from this many calendar months ago through today. Your bank may provide less history."
>
<input
autoFocus
type="number"
required
min={1}
max={120}
step={1}
disabled={busy}
value={historyMonths}
onChange={(e) => setHistoryMonths(e.target.value)}
/>
</Field>
<p className="muted small">
Repeating or overlapping a date range skips transactions already
imported. Normal sync and its cursor stay unchanged, as does the
initial-history setting. Inactive connected accounts can also import
older history.
</p>
{unavailable && <p className="muted small">{unavailable}</p>}
{busy && (
<p className="muted small" role="status">
Importing older history Keep this dialog open while the bank
request and journal update finish.
</p>
)}
</div>
<div className="form-actions">
<button
type="button"
className="button secondary"
onClick={closeWhenIdle}
disabled={busy}
>
{success ? "Close" : "Cancel"}
</button>
<button
type="submit"
className="button primary"
disabled={busy || !!unavailable}
>
{busy ? "Importing history…" : "Import older history"}
</button>
</div>
</form>
</Modal>
);
}
function ImportForm({
state,
acceptState,
+1 -2
View File
@@ -162,7 +162,7 @@ function App() {
</span>
<span>
finance<span className="brand-light">duck</span>
<small>YOUR MONEY, CLEARLY</small>
<small>YOUR MONEY</small>
</span>
</a>
<span className="nav-label">WORKSPACE</span>
@@ -355,7 +355,6 @@ function App() {
)}
{page === "accounts" && (
<Accounts
key={state.revision}
state={state}
mutate={mutate}
acceptState={acceptState}