Lars Nolden
922ae507bd
An account now has a kind, and an investment account holds positions as well as
cash. A broker row is not a new entity: it is a bank fact with an optional
position leg, so deduplication, the journal, fact immutability, the DuckDB
projection and the transactions view carry it unchanged. Facts.Amount stays the
cash leg and is zero on the rows that move only a position.
Scalable Capital exports are recognized locally as a fourth format, read by
their own parser because a column mapping cannot describe them: the amount
column is settled cash on a cash row, a gross to be netted on a trade, and a
position valuation that must never touch cash on a corporate action or a depot
transfer. A cash amount is already net of the tax the broker withheld or
refunded, so that tax is recorded on the fact and never subtracted a second
time; treating a corporate action's valuation as money conjures cash, and a
depot switch would do it once per instrument. The share column is signed only
for those two types, so buys and sells take their direction from the type. Every
security row is checked against shares times price at 128-bit width, because a
lost decimal separator survives every other check. An unknown status, type or
assetType, a foreign currency, a missing ISIN, or one failed check rejects the
whole file with the record number.
Instruments live in instruments.finance, keyed by ISIN with an ID derived from
it, so re-importing never registers a security twice. One ISIN appears under
several broker descriptions over the years and sometimes under the ISIN itself:
the most recent real description names it, and an import never renames one that
already exists. A broker also reuses a single reference across every leg of one
event, so transaction identity includes the event and its instrument.
domain.Fallback returns kind "investment" for any fact carrying a position leg,
so no broker row reaches the sign-based branch. That single rule is what stops
an unmatched deposit from counting as income and a broker fee from counting as
household spending; the monthly PRIME fee and its matching credit now cancel in
clearing:investments with no configuration at all. Investment rows are excluded
from spending analytics, from bulk reclassification and from the model, exactly
as transfers are.
Equal competing transfers are paired instead of skipped. Every connected
component of the candidate graph is a complete bipartite graph between two fixed
accounts at one amount and currency, so every pairing produces the same
accounts, kinds and postings and only the displayed counterpart differs.
Refusing to choose was the expensive option: both legs fell through to the
sign-based fallback and appeared as spending and income that never happened.
Pairing follows the nearest booking date, then the transaction ID, so iteration
order decides nothing. POST /api/transactions/{id}/transfer rewrites the old and
the new pair in one commit, because reciprocity is validated and a half-applied
link is an invalid dataset, and the matcher now skips any record classified
manually so a hand-made link or unlink outlives the next import.
Wealth reports each account's cash and positions from the journal rather than
the index, with named checks - row arithmetic, cash never negative, holdings
never negative - because it exists to be compared against the figures a broker
shows on its own screen. A negative holding means the imported history is
partial. Share counts are exact to eight places; a reinvested distribution
quoted to six is rounded to money's four and the residue is reported rather than
hidden. Market prices, market value, net worth over time, FIFO lot accounting,
realised gains and currency conversion are deliberately absent.
396 lines
14 KiB
Go
396 lines
14 KiB
Go
package banking
|
|
|
|
import (
|
|
"reflect"
|
|
"testing"
|
|
|
|
"finance-duck/internal/domain"
|
|
)
|
|
|
|
func fixtureDataset() domain.Dataset {
|
|
d := domain.NewDataset()
|
|
d.Accounts = []domain.Account{{ID: "account_a", DisplayName: "N26", Currency: "EUR", IBAN: "DE02120300000000202051", Active: true}, {ID: "account_b", DisplayName: "Savings", Currency: "EUR", IBAN: "DE89370400440532013000", Active: true}}
|
|
return d
|
|
}
|
|
func fixtureFacts() domain.Facts {
|
|
return domain.Facts{Source: "n26_csv", AccountID: "account_a", BookingDate: "2026-09-01", Amount: "-12.30", Currency: "EUR", RawDescription: "Lunch", Counterparty: "Cafe", Fingerprint: "fixture"}
|
|
}
|
|
|
|
func TestFallbackOccurrenceMultiplicityAndRepeatImport(t *testing.T) {
|
|
d := fixtureDataset()
|
|
f := fixtureFacts()
|
|
rows, err := NormalizeAndDedupe(d, []domain.Facts{f, f})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(rows) != 2 || rows[0].Facts.ID == rows[1].Facts.ID {
|
|
t.Fatalf("legitimate duplicate rows lost: %+v", rows)
|
|
}
|
|
d.Transactions = append(d.Transactions, rows...)
|
|
again, err := NormalizeAndDedupe(d, []domain.Facts{f, f})
|
|
if err != nil || len(again) != 0 {
|
|
t.Fatalf("repeat not idempotent: %v %+v", err, again)
|
|
}
|
|
added, err := NormalizeAndDedupe(d, []domain.Facts{f, f, f})
|
|
if err != nil || len(added) != 1 {
|
|
t.Fatalf("new occurrence lost: %v %+v", err, added)
|
|
}
|
|
d.Transactions = append(d.Transactions, added...)
|
|
again, err = NormalizeAndDedupe(d, []domain.Facts{f, f, f})
|
|
if err != nil || len(again) != 0 {
|
|
t.Fatalf("expanded repeat not idempotent: %v %+v", err, again)
|
|
}
|
|
if err := domain.Validate(d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
func TestUpstreamIdentityPreferredAndAccountScoped(t *testing.T) {
|
|
d := fixtureDataset()
|
|
a := fixtureFacts()
|
|
a.Source = "enablebanking"
|
|
a.ExternalID = "bank-entry-1"
|
|
b := a
|
|
b.AccountID = "account_b"
|
|
rows, err := NormalizeAndDedupe(d, []domain.Facts{a, a, b})
|
|
if err != nil || len(rows) != 2 {
|
|
t.Fatalf("account identity lost: %v %+v", err, rows)
|
|
}
|
|
d.Transactions = rows
|
|
a.RawDescription = "Updated upstream display"
|
|
a.ValueDate = "2026-09-02"
|
|
again, err := NormalizeAndDedupe(d, []domain.Facts{a})
|
|
if err != nil || len(again) != 0 {
|
|
t.Fatalf("upstream identity not preferred: %v %+v", err, again)
|
|
}
|
|
a.Amount = "-99.00"
|
|
again, err = NormalizeAndDedupe(d, []domain.Facts{a})
|
|
if err == nil || again != nil {
|
|
t.Fatal("changed immutable upstream money accepted")
|
|
}
|
|
}
|
|
func TestUpstreamReferenceSeparatesDebitAndCredit(t *testing.T) {
|
|
d := fixtureDataset()
|
|
debit := fixtureFacts()
|
|
debit.Source = "enablebanking"
|
|
debit.ExternalID = "shared-bank-reference"
|
|
credit := debit
|
|
credit.Amount = "12.30"
|
|
rows, err := NormalizeAndDedupe(d, []domain.Facts{debit, credit, debit, credit})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
amounts := map[domain.Money]int{}
|
|
for _, row := range rows {
|
|
amounts[row.Facts.Amount]++
|
|
}
|
|
if !reflect.DeepEqual(amounts, map[domain.Money]int{"-12.30": 1, "12.30": 1}) {
|
|
t.Fatalf("debit/credit postings lost or duplicated: %+v", rows)
|
|
}
|
|
d.Transactions = rows
|
|
if err := domain.Validate(d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again, err := NormalizeAndDedupe(d, []domain.Facts{credit, debit})
|
|
if err != nil || len(again) != 0 {
|
|
t.Fatalf("repeated debit/credit pair was not idempotent: %+v %v", again, err)
|
|
}
|
|
}
|
|
|
|
func TestUpstreamDirectionPreservesPreviouslyStoredIdentity(t *testing.T) {
|
|
for _, amount := range []domain.Money{"-12.30", "12.30"} {
|
|
t.Run(string(amount), func(t *testing.T) {
|
|
d := fixtureDataset()
|
|
stored := fixtureFacts()
|
|
stored.Source = "enablebanking"
|
|
stored.ExternalID = "shared-bank-reference"
|
|
stored.Amount = amount
|
|
// Reproduce the journal identity written before direction scoping.
|
|
stored.ID = "tx_" + digest(stored.AccountID, stored.Source, stored.ExternalID)
|
|
stored.Fingerprint = fingerprint(stored)
|
|
d.Transactions = []domain.Transaction{{Facts: stored, Enrichment: domain.Fallback(stored)}}
|
|
opposite := stored
|
|
opposite.Amount = "12.30"
|
|
if amount == "12.30" {
|
|
opposite.Amount = "-12.30"
|
|
}
|
|
incoming := stored
|
|
incoming.RawDescription = "Updated upstream display"
|
|
added, err := NormalizeAndDedupe(d, []domain.Facts{opposite, incoming})
|
|
if err != nil || len(added) != 1 || added[0].Facts.Amount != opposite.Amount {
|
|
t.Fatalf("stored posting duplicated or opposite posting lost: %+v %v", added, err)
|
|
}
|
|
if !reflect.DeepEqual(d.Transactions[0].Facts, stored) {
|
|
t.Fatal("previously stored bank facts were rewritten")
|
|
}
|
|
d.Transactions = append(d.Transactions, added...)
|
|
if err := domain.Validate(d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again, err := NormalizeAndDedupe(d, []domain.Facts{incoming, opposite})
|
|
if err != nil || len(again) != 0 {
|
|
t.Fatalf("upgraded journal was not idempotent: %+v %v", again, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestUpstreamReferenceRejectsSameDirectionConflicts(t *testing.T) {
|
|
original := fixtureFacts()
|
|
original.Source = "enablebanking"
|
|
original.ExternalID = "shared-bank-reference"
|
|
original.Amount = "12.30"
|
|
for _, tc := range []struct {
|
|
name string
|
|
change func(*domain.Facts)
|
|
}{
|
|
{"amount", func(f *domain.Facts) { f.Amount = "99.00" }},
|
|
{"booking date", func(f *domain.Facts) { f.BookingDate = "2026-09-02" }},
|
|
{"currency", func(f *domain.Facts) { f.Currency = "USD" }},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
d := fixtureDataset()
|
|
changed := original
|
|
tc.change(&changed)
|
|
if added, err := NormalizeAndDedupe(d, []domain.Facts{original, changed}); err == nil || added != nil {
|
|
t.Fatal("conflicting incoming postings were accepted")
|
|
}
|
|
var err error
|
|
d.Transactions, err = NormalizeAndDedupe(d, []domain.Facts{original})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if added, err := NormalizeAndDedupe(d, []domain.Facts{changed}); err == nil || added != nil {
|
|
t.Fatal("changed stored booking facts were accepted")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDistinctUpstreamIDsPreserveEqualTransactions(t *testing.T) {
|
|
d := fixtureDataset()
|
|
a := fixtureFacts()
|
|
a.Source = "enablebanking"
|
|
a.ExternalID = "one"
|
|
b := a
|
|
b.ExternalID = "two"
|
|
rows, err := NormalizeAndDedupe(d, []domain.Facts{a, b})
|
|
if err != nil || len(rows) != 2 {
|
|
t.Fatalf("distinct IDs collapsed: %v %+v", err, rows)
|
|
}
|
|
reverse, err := NormalizeAndDedupe(d, []domain.Facts{b, a})
|
|
if err != nil || !reflect.DeepEqual(rows, reverse) {
|
|
t.Fatalf("order changed IDs: %v", err)
|
|
}
|
|
d.Transactions = rows[:1]
|
|
added, err := NormalizeAndDedupe(d, []domain.Facts{a, b})
|
|
if err != nil || len(added) != 1 {
|
|
t.Fatalf("new equal upstream record suppressed: %v %+v", err, added)
|
|
}
|
|
}
|
|
func TestCrossSourceExactMatchAndUncertainty(t *testing.T) {
|
|
d := fixtureDataset()
|
|
csv := fixtureFacts()
|
|
rows, err := NormalizeAndDedupe(d, []domain.Facts{csv})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d.Transactions = rows
|
|
api := csv
|
|
api.Source = "enablebanking"
|
|
api.ExternalID = "upstream"
|
|
matched, err := NormalizeAndDedupe(d, []domain.Facts{api})
|
|
if err != nil || len(matched) != 0 {
|
|
t.Fatalf("double counted matching cross-source transaction: %v %+v", err, matched)
|
|
}
|
|
api.RawDescription = "Different bank text"
|
|
matched, err = NormalizeAndDedupe(d, []domain.Facts{api})
|
|
const wantConflict = "bank-synced and CSV transactions overlap on 2026-09-01 for -12.30 EUR but differ in description (bank-synced \"Different bank text\"; CSV \"Lunch\"). Finance Duck cannot tell whether they are one transaction or two; to prevent double counting, nothing was imported. Compare both records, then correct or remove the duplicate before retrying"
|
|
if err == nil || err.Error() != wantConflict || matched != nil {
|
|
t.Fatalf("unexpected cross-source conflict: %v %+v", err, matched)
|
|
}
|
|
api.RawDescription = csv.RawDescription
|
|
b := api
|
|
b.ExternalID = "second"
|
|
matched, err = NormalizeAndDedupe(d, []domain.Facts{api, b})
|
|
if err == nil || matched != nil {
|
|
t.Fatal("unequal cross-source multiplicity was guessed")
|
|
}
|
|
d.Transactions[0].Facts.RawDescription = ""
|
|
d.Transactions[0].Facts.Counterparty = ""
|
|
api.RawDescription = ""
|
|
api.Counterparty = ""
|
|
matched, err = NormalizeAndDedupe(d, []domain.Facts{api})
|
|
if err == nil || matched != nil {
|
|
t.Fatal("matched cross-source money without descriptive evidence")
|
|
}
|
|
}
|
|
func transferDataset() domain.Dataset {
|
|
d := fixtureDataset()
|
|
a := fixtureFacts()
|
|
a.ID = "tx_a"
|
|
a.Amount = "-10.00"
|
|
a.CounterpartyIBAN = d.Accounts[1].IBAN
|
|
b := a
|
|
b.ID = "tx_b"
|
|
b.AccountID = "account_b"
|
|
b.Amount = "10.00"
|
|
b.BookingDate = "2026-09-03"
|
|
b.CounterpartyIBAN = d.Accounts[0].IBAN
|
|
d.Transactions = []domain.Transaction{{Facts: a, Enrichment: domain.Fallback(a)}, {Facts: b, Enrichment: domain.Fallback(b)}}
|
|
return d
|
|
}
|
|
func TestTransfersRequireUniqueReciprocalOwnBankEvidence(t *testing.T) {
|
|
d := transferDataset()
|
|
MatchTransfers(&d)
|
|
if d.Transactions[0].Enrichment.TransferPeerID != "tx_b" || d.Transactions[1].Enrichment.TransferPeerID != "tx_a" {
|
|
t.Fatal("unique own-account transfer not linked")
|
|
}
|
|
if err := domain.Validate(d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, change := range []func(*domain.Dataset){
|
|
func(d *domain.Dataset) { d.Transactions[1].Facts.CounterpartyIBAN = "" },
|
|
func(d *domain.Dataset) { d.Transactions[1].Facts.Currency = "USD" },
|
|
func(d *domain.Dataset) { d.Transactions[1].Facts.Amount = "9.99" },
|
|
func(d *domain.Dataset) { d.Transactions[1].Facts.BookingDate = "2026-09-05" },
|
|
func(d *domain.Dataset) {
|
|
d.Accounts = append(d.Accounts, domain.Account{ID: "ambiguous_account", IBAN: d.Accounts[1].IBAN})
|
|
},
|
|
} {
|
|
d := transferDataset()
|
|
change(&d)
|
|
before := domain.Clone(d)
|
|
MatchTransfers(&d)
|
|
if !reflect.DeepEqual(d, before) {
|
|
t.Fatalf("unsupported transfer evidence linked: %+v", d.Transactions)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Two equal top-ups in one week give every leg two candidates. Refusing to
|
|
// pair them is what turned both legs into spending and income that never
|
|
// happened, so the pairing must happen, must follow the nearest booking date,
|
|
// and must not depend on the order the records arrive in.
|
|
func TestEqualCompetingTransfersPairByNearestDate(t *testing.T) {
|
|
build := func(reverse bool) domain.Dataset {
|
|
d := fixtureDataset()
|
|
leg := func(id, account, amount, date, peerIBAN string) domain.Transaction {
|
|
f := fixtureFacts()
|
|
f.ID, f.AccountID, f.Amount, f.BookingDate, f.CounterpartyIBAN = id, account, domain.Money(amount), date, peerIBAN
|
|
return domain.Transaction{Facts: f, Enrichment: domain.Fallback(f)}
|
|
}
|
|
a, b := d.Accounts[0].IBAN, d.Accounts[1].IBAN
|
|
d.Transactions = []domain.Transaction{
|
|
leg("tx_out_mon", "account_a", "-800.00", "2026-09-05", b),
|
|
leg("tx_out_wed", "account_a", "-800.00", "2026-09-07", b),
|
|
leg("tx_in_tue", "account_b", "800.00", "2026-09-06", a),
|
|
leg("tx_in_thu", "account_b", "800.00", "2026-09-08", a),
|
|
}
|
|
if reverse {
|
|
for i, j := 0, len(d.Transactions)-1; i < j; i, j = i+1, j-1 {
|
|
d.Transactions[i], d.Transactions[j] = d.Transactions[j], d.Transactions[i]
|
|
}
|
|
}
|
|
return d
|
|
}
|
|
want := map[string]string{"tx_out_mon": "tx_in_tue", "tx_in_tue": "tx_out_mon", "tx_out_wed": "tx_in_thu", "tx_in_thu": "tx_out_wed"}
|
|
for _, reverse := range []bool{false, true} {
|
|
d := build(reverse)
|
|
MatchTransfers(&d)
|
|
for _, tx := range d.Transactions {
|
|
if tx.Enrichment.Kind != "transfer" || tx.Enrichment.TransferPeerID != want[tx.Facts.ID] {
|
|
t.Fatalf("reverse=%v: %s linked to %q as %q, want %q as transfer", reverse, tx.Facts.ID, tx.Enrichment.TransferPeerID, tx.Enrichment.Kind, want[tx.Facts.ID])
|
|
}
|
|
}
|
|
if err := domain.Validate(d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A hand-made decision must outlive the next import, or unlinking a pair that
|
|
// is not really a transfer is undone the moment anything is imported again.
|
|
func TestManualClassificationSurvivesMatching(t *testing.T) {
|
|
d := transferDataset()
|
|
d.Transactions[0].Enrichment.Classification.Source = "manual"
|
|
before := domain.Clone(d)
|
|
MatchTransfers(&d)
|
|
if !reflect.DeepEqual(d, before) {
|
|
t.Fatalf("matcher overrode a manual decision: %+v", d.Transactions)
|
|
}
|
|
}
|
|
|
|
func TestMixedReferencedAndAnonymousMultiplicity(t *testing.T) {
|
|
d := fixtureDataset()
|
|
anonymous := fixtureFacts()
|
|
anonymous.Source = "enablebanking"
|
|
referenced := anonymous
|
|
referenced.ExternalID = "known-reference"
|
|
original, err := NormalizeAndDedupe(d, []domain.Facts{referenced})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d.Transactions = original
|
|
for _, window := range [][]domain.Facts{{referenced, anonymous}, {anonymous, referenced}} {
|
|
added, err := NormalizeAndDedupe(d, window)
|
|
if err != nil || len(added) != 1 || added[0].Facts.ExternalID != "" {
|
|
t.Fatalf("lost additional anonymous booking beside matched reference: %+v %v", added, err)
|
|
}
|
|
if added[0].Facts.ID == original[0].Facts.ID {
|
|
t.Fatal("anonymous booking reused referenced identity")
|
|
}
|
|
}
|
|
added, err := NormalizeAndDedupe(d, []domain.Facts{referenced, anonymous})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d.Transactions = append(d.Transactions, added...)
|
|
repeated, err := NormalizeAndDedupe(d, []domain.Facts{anonymous, referenced})
|
|
if err != nil || len(repeated) != 0 {
|
|
t.Fatalf("mixed repeat is not idempotent: %+v %v", repeated, err)
|
|
}
|
|
second, err := NormalizeAndDedupe(d, []domain.Facts{anonymous, referenced, anonymous})
|
|
if err != nil || len(second) != 1 || second[0].Facts.ID == added[0].Facts.ID {
|
|
t.Fatalf("second anonymous occurrence lost or ID reused: %+v %v", second, err)
|
|
}
|
|
d.Transactions = append(d.Transactions, second...)
|
|
repeated, err = NormalizeAndDedupe(d, []domain.Facts{referenced, anonymous, anonymous})
|
|
if err != nil || len(repeated) != 0 {
|
|
t.Fatalf("expanded mixed repeat is not idempotent: %+v %v", repeated, err)
|
|
}
|
|
if err := domain.Validate(d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestChangingReferenceAvailabilityFailsClosed(t *testing.T) {
|
|
anonymous := fixtureFacts()
|
|
anonymous.Source = "enablebanking"
|
|
referenced := anonymous
|
|
referenced.ExternalID = "new-reference"
|
|
for _, pair := range [][2]domain.Facts{{anonymous, referenced}, {referenced, anonymous}} {
|
|
d := fixtureDataset()
|
|
original, err := NormalizeAndDedupe(d, []domain.Facts{pair[0]})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d.Transactions = original
|
|
added, err := NormalizeAndDedupe(d, []domain.Facts{pair[1]})
|
|
if err == nil || added != nil {
|
|
t.Fatalf("identity availability change silently added/dropped money: %+v %v", added, err)
|
|
}
|
|
}
|
|
// A complete window containing the known anonymous booking separately proves
|
|
// that an additional referenced booking increases multiplicity.
|
|
d := fixtureDataset()
|
|
original, err := NormalizeAndDedupe(d, []domain.Facts{anonymous})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d.Transactions = original
|
|
added, err := NormalizeAndDedupe(d, []domain.Facts{anonymous, referenced})
|
|
if err != nil || len(added) != 1 || added[0].Facts.ExternalID != "new-reference" {
|
|
t.Fatalf("proven additional referenced booking was lost: %+v %v", added, err)
|
|
}
|
|
}
|