From cc43a2f9a7231960a7c0eec1d9831a79a0f0cbf2 Mon Sep 17 00:00:00 2001 From: Lars Nolden Date: Fri, 11 Sep 2026 22:29:47 +0200 Subject: [PATCH] Read broker amounts at the precision the export actually uses A real Scalable export reinvests a distribution as -29,579989728, and the import refused the whole file: "broker record 14 has an invalid amount, require signed 64-bit value with at most eight fractional digits". An amount is the row's share count times its price, so it carries as many decimal places as the two columns together need - nine here, from six places of shares and three of price - and scalableMoney was computing its discarded remainder by parsing the cell through the share-count parser, whose ceiling is eight. The leaked wording "invalid quantity" for an amount cell was the tell. Money cells are now read at arbitrary precision, rounded to money's four places half away from zero, and the residue is accumulated exactly and reported as a trimmed decimal. ScalableImport.Rounding stops being a domain.Quantity, which carried the same eight-place ceiling, and becomes the exact decimal string it always claimed to be; the JSON shape and the review copy are unchanged. Share counts and prices are still refused beyond their own precision instead of rounded. Rounding a share count misstates a holding, and rounding a price would break the shares-times-price identity that every security row's amount is checked against. The reported row is now covered end to end: both legs of that distribution, the single reference the broker reuses across them surviving deduplication, the exact 0.000010272 residue, and the dividend paid in cancelling to the cent against the units bought with it. --- OPERATIONS.txt | 14 ++-- internal/banking/scalable.go | 109 ++++++++++++++++++++++-------- internal/banking/scalable_test.go | 50 ++++++++++++++ 3 files changed, 141 insertions(+), 32 deletions(-) diff --git a/OPERATIONS.txt b/OPERATIONS.txt index 567ca22..cf7bf2a 100644 --- a/OPERATIONS.txt +++ b/OPERATIONS.txt @@ -444,11 +444,15 @@ action or depot transfer carrying a fee or tax, and any failed arithmetic check. A zero amount is accepted; it corrupts nothing, and a free share allocation is legitimately priced at zero. -Money holds four decimal places and share counts hold eight. A reinvested -distribution is quoted to six, so its amount is rounded half away from zero and -the exact residue is reported in the import review and never hidden. A share -count beyond eight places is refused rather than truncated, because a holding is -verified against the broker's own figure. +Money holds four decimal places and share counts hold eight. An amount is the +row's share count times its price, so it carries as many decimal places as the +two together need: a reinvested distribution in a real export reaches nine, +past both. Amounts are therefore read at arbitrary precision, rounded to four +places half away from zero, and the exact discarded residue is summed and +reported in the import review rather than hidden. A share count or a price +beyond its own precision is refused instead of truncated: rounding a share +count misstates a holding, and rounding a price would break the shares-times- +price check that the amount is verified against. Instruments are registered from the export, keyed by ISIN, with an ID derived from the ISIN so re-importing never creates a second entry for one security. One diff --git a/internal/banking/scalable.go b/internal/banking/scalable.go index 6ad02da..1ca6d59 100644 --- a/internal/banking/scalable.go +++ b/internal/banking/scalable.go @@ -3,6 +3,7 @@ package banking import ( "errors" "fmt" + "math/big" "strings" "finance-duck/internal/domain" @@ -62,9 +63,10 @@ type ScalableImport struct { // otherwise import as phantom trades. Cancelled int `json:"cancelled"` // Rounded counts rows whose money carried more than four decimal places, - // and Rounding is the exact total adjustment that rounding applied. - Rounded int `json:"rounded"` - Rounding domain.Quantity `json:"rounding"` + // and Rounding is the exact total adjustment that rounding applied, at + // whatever precision the export used. + Rounded int `json:"rounded"` + Rounding string `json:"rounding"` // Unapplied lists cash rows carrying a fee or tax. A broker cash amount is // already net of them, so subtracting them again would double-count; they // are recorded on the fact and reported here. @@ -152,7 +154,7 @@ func ParseScalableCSV(f CSVFile, account domain.Account, registry []domain.Instr } created := map[string]int{} named := map[string]string{} - drift := int64(0) + drift := new(big.Int) for offset, row := range f.rows[header:] { record := header + offset + 1 if blankCSVRow(row) { @@ -226,9 +228,9 @@ func ParseScalableCSV(f CSVFile, account domain.Account, registry []domain.Instr if err != nil { return result, fmt.Errorf("broker record %d has an invalid tax %q: %w", record, cell(row, "tax"), err) } - if amountDrift|feeDrift|taxDrift != 0 { + if amountDrift.Sign() != 0 || feeDrift.Sign() != 0 || taxDrift.Sign() != 0 { result.Rounded++ - drift += amountDrift + feeDrift + taxDrift + drift.Add(drift, amountDrift).Add(drift, feeDrift).Add(drift, taxDrift) } cash := amount if investment.CashOnly() { @@ -248,7 +250,7 @@ func ParseScalableCSV(f CSVFile, account domain.Account, registry []domain.Instr if err != nil { return result, fmt.Errorf("broker record %d has an invalid price %q: %w", record, cell(row, "price"), err) } - if priceDrift != 0 { + if priceDrift.Sign() != 0 { return result, fmt.Errorf("broker record %d has a price %q beyond four decimal places", record, cell(row, "price")) } signed, err := scalableSignedShares(event, shares) @@ -288,7 +290,7 @@ func ParseScalableCSV(f CSVFile, account domain.Account, registry []domain.Instr if len(result.Facts) == 0 { return result, errors.New("broker export contains no executed records") } - result.Rounding = domain.FormatQuantity(drift) + result.Rounding = decimalString(drift, residueScale) return result, nil } @@ -301,32 +303,85 @@ func nonzeroMoney(m domain.Money) bool { return err == nil && minor != 0 } -// scalableMoney reads one German-formatted money cell and returns the exact -// remainder that rounding discarded, in hundred-millionths. The export quotes a -// reinvested distribution to six decimal places, which money's four cannot -// hold; the residue is reported rather than hidden. An empty cell is empty -// money, not zero: blank marks a column that does not apply to the row. -func scalableMoney(value string) (domain.Money, int64, error) { +// residueScale is the precision the discarded remainder is accumulated at. +// A broker amount is its share count times its price, so it carries as many +// decimal places as the two together need: a real export reinvests to nine. +// Eighteen is far past anything a settlement can produce and still exact. +const residueScale = 18 + +// scalableMoney reads one German-formatted money cell, rounds it to money's +// four decimal places half away from zero, and returns the exact remainder +// that rounding discarded, in units of 1e-18. The remainder is reported rather +// than hidden, and never guessed at: it is the only honest account of why a +// computed balance can differ from the broker's by a fraction of a cent. +// +// An empty cell is empty money, not zero: blank marks a column that does not +// apply to the row. +func scalableMoney(value string) (domain.Money, *big.Int, error) { plain, ok, err := scalablePlain(value) if !ok || err != nil { - return "", 0, err + return "", new(big.Int), err } - exact, err := domain.ParseQuantity(plain) + magnitude, negative, err := scalableDigits(plain) if err != nil { - return "", 0, err + return "", new(big.Int), err } - units, err := exact.Units() - if err != nil { - return "", 0, err + // One money place is 1e14 residue units. Rounding compares twice the + // remainder against that, so a tie rounds away from zero. + place := new(big.Int).Exp(big.NewInt(10), big.NewInt(residueScale-4), nil) + rounded, remainder := new(big.Int).QuoRem(magnitude, place, new(big.Int)) + if new(big.Int).Lsh(remainder, 1).Cmp(place) >= 0 { + rounded.Add(rounded, big.NewInt(1)) } - rounded := units / 10000 - switch remainder := units % 10000; { - case remainder >= 5000: - rounded++ - case remainder <= -5000: - rounded-- + if !rounded.IsInt64() { + return "", new(big.Int), fmt.Errorf("value is out of range for money") } - return domain.FormatMoney(rounded), units - rounded*10000, nil + residue := new(big.Int).Sub(magnitude, new(big.Int).Mul(rounded, place)) + minor := rounded.Int64() + if negative { + minor, residue = -minor, residue.Neg(residue) + } + return domain.FormatMoney(minor), residue, nil +} + +// scalableDigits splits a plain decimal string into its exact magnitude in +// residue units and its sign. +func scalableDigits(plain string) (magnitude *big.Int, negative bool, err error) { + digits := plain + if rest, cut := strings.CutPrefix(digits, "-"); cut { + negative, digits = true, rest + } + whole, decimals, _ := strings.Cut(digits, ".") + if whole == "" { + return nil, false, fmt.Errorf("decimal needs a digit before the separator") + } + if len(decimals) > residueScale { + return nil, false, fmt.Errorf("more than %d fractional digits", residueScale) + } + scaled, ok := new(big.Int).SetString(whole+decimals+strings.Repeat("0", residueScale-len(decimals)), 10) + if !ok { + return nil, false, fmt.Errorf("not a decimal number") + } + return scaled, negative, nil +} + +// decimalString renders exact units at a scale without trailing zeros, so an +// adjustment of 1e-9 is reported as such rather than padded to eighteen places. +func decimalString(units *big.Int, scale int) string { + sign := "" + magnitude := new(big.Int).Abs(units) + if units.Sign() < 0 { + sign = "-" + } + digits := magnitude.String() + if len(digits) <= scale { + digits = strings.Repeat("0", scale+1-len(digits)) + digits + } + whole, fraction := digits[:len(digits)-scale], strings.TrimRight(digits[len(digits)-scale:], "0") + if fraction == "" { + return sign + whole + } + return sign + whole + "." + fraction } // scalableQuantity reads one German-formatted share count. Nothing is rounded: diff --git a/internal/banking/scalable_test.go b/internal/banking/scalable_test.go index fcb7452..bbfca6e 100644 --- a/internal/banking/scalable_test.go +++ b/internal/banking/scalable_test.go @@ -228,6 +228,56 @@ func TestSingleShareRowCatchesOnlyInconsistentArithmetic(t *testing.T) { } } +// A reinvested distribution settles shares times price, so it carries as many +// decimal places as the two together need. A real export reinvests to nine, +// which is past what money holds and past what a share count holds, so reading +// the cell at either precision rejects the row outright. It is rounded to +// money's four and the discarded remainder is reported exactly. +func TestReinvestmentKeepsNineDecimalPlacesOutOfTheBalance(t *testing.T) { + const reference = "617007_rrCjP4EcbpefpNiVQeD495" + result := readBroker(t, + `2026-05-28;02:00:00;Executed;`+reference+`;iShares Global Clean Energy Transition (Dist);Cash;Distribution;IE00B1XNHC34;;;29,58;0,00;8,56;EUR`, + `2026-05-28;02:00:00;Executed;`+reference+`;iShares Global Clean Energy Transition (Dist);Security;Reinvestment_Distribution;IE00B1XNHC34;3,144131;9,408;-29,579984448;0,00;0,00;EUR`, + ) + if len(result.Facts) != 2 { + t.Fatalf("read %d of 2 legs", len(result.Facts)) + } + cash, reinvest := result.Facts[0], result.Facts[1] + if cash.Amount != "29.58" || cash.Investment.Tax != "8.56" { + t.Errorf("distribution settled %s with tax %s, want 29.58 and 8.56 recorded", cash.Amount, cash.Investment.Tax) + } + // 29.579984448 rounds up at the fifth place, and the residue is exact. + if reinvest.Amount != "-29.58" || reinvest.Investment.Gross != "-29.58" { + t.Errorf("reinvestment settled %s against gross %s, want -29.58 for both", reinvest.Amount, reinvest.Investment.Gross) + } + if reinvest.Investment.Quantity != "3.144131" { + t.Errorf("reinvested %s shares, want 3.144131", reinvest.Investment.Quantity) + } + if result.Rounded != 1 || result.Rounding != "0.000015552" { + t.Errorf("rounding reported as %d row(s) and %s, want 1 and 0.000015552", result.Rounded, result.Rounding) + } + // The dividend paid in and the units bought with it cancel to the cent. + total := int64(0) + for _, f := range result.Facts { + minor, err := f.Amount.Minor() + if err != nil { + t.Fatal(err) + } + total += minor + } + if total != 0 { + t.Errorf("the pair moved %s of net cash, want none", domain.FormatMoney(total)) + } + // Both legs carry one reference byte for byte and must both survive. + data := domain.NewDataset() + data.Accounts = []domain.Account{brokerAccount()} + data.Instruments = result.Instruments + added, err := NormalizeAndDedupe(data, result.Facts) + if err != nil || len(added) != 2 { + t.Fatalf("dedupe kept %d of 2 legs sharing a reference: %v", len(added), err) + } +} + // A thousands dot and a decimal dot are both present in one share column. func TestBrokerShareColumnDistinguishesGroupingFromDecimals(t *testing.T) { result := readBroker(t,