Read broker amounts at the precision the export actually uses
A real Scalable export reinvests a distribution as -29,579989728, and the import refused the whole file: "broker record 14 has an invalid amount, require signed 64-bit value with at most eight fractional digits". An amount is the row's share count times its price, so it carries as many decimal places as the two columns together need - nine here, from six places of shares and three of price - and scalableMoney was computing its discarded remainder by parsing the cell through the share-count parser, whose ceiling is eight. The leaked wording "invalid quantity" for an amount cell was the tell. Money cells are now read at arbitrary precision, rounded to money's four places half away from zero, and the residue is accumulated exactly and reported as a trimmed decimal. ScalableImport.Rounding stops being a domain.Quantity, which carried the same eight-place ceiling, and becomes the exact decimal string it always claimed to be; the JSON shape and the review copy are unchanged. Share counts and prices are still refused beyond their own precision instead of rounded. Rounding a share count misstates a holding, and rounding a price would break the shares-times-price identity that every security row's amount is checked against. The reported row is now covered end to end: both legs of that distribution, the single reference the broker reuses across them surviving deduplication, the exact 0.000010272 residue, and the dividend paid in cancelling to the cent against the units bought with it.
This commit is contained in:
+9
-5
@@ -444,11 +444,15 @@ action or depot transfer carrying a fee or tax, and any failed arithmetic check.
|
|||||||
A zero amount is accepted; it corrupts nothing, and a free share allocation is
|
A zero amount is accepted; it corrupts nothing, and a free share allocation is
|
||||||
legitimately priced at zero.
|
legitimately priced at zero.
|
||||||
|
|
||||||
Money holds four decimal places and share counts hold eight. A reinvested
|
Money holds four decimal places and share counts hold eight. An amount is the
|
||||||
distribution is quoted to six, so its amount is rounded half away from zero and
|
row's share count times its price, so it carries as many decimal places as the
|
||||||
the exact residue is reported in the import review and never hidden. A share
|
two together need: a reinvested distribution in a real export reaches nine,
|
||||||
count beyond eight places is refused rather than truncated, because a holding is
|
past both. Amounts are therefore read at arbitrary precision, rounded to four
|
||||||
verified against the broker's own figure.
|
places half away from zero, and the exact discarded residue is summed and
|
||||||
|
reported in the import review rather than hidden. A share count or a price
|
||||||
|
beyond its own precision is refused instead of truncated: rounding a share
|
||||||
|
count misstates a holding, and rounding a price would break the shares-times-
|
||||||
|
price check that the amount is verified against.
|
||||||
|
|
||||||
Instruments are registered from the export, keyed by ISIN, with an ID derived
|
Instruments are registered from the export, keyed by ISIN, with an ID derived
|
||||||
from the ISIN so re-importing never creates a second entry for one security. One
|
from the ISIN so re-importing never creates a second entry for one security. One
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package banking
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"math/big"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"finance-duck/internal/domain"
|
"finance-duck/internal/domain"
|
||||||
@@ -62,9 +63,10 @@ type ScalableImport struct {
|
|||||||
// otherwise import as phantom trades.
|
// otherwise import as phantom trades.
|
||||||
Cancelled int `json:"cancelled"`
|
Cancelled int `json:"cancelled"`
|
||||||
// Rounded counts rows whose money carried more than four decimal places,
|
// Rounded counts rows whose money carried more than four decimal places,
|
||||||
// and Rounding is the exact total adjustment that rounding applied.
|
// and Rounding is the exact total adjustment that rounding applied, at
|
||||||
Rounded int `json:"rounded"`
|
// whatever precision the export used.
|
||||||
Rounding domain.Quantity `json:"rounding"`
|
Rounded int `json:"rounded"`
|
||||||
|
Rounding string `json:"rounding"`
|
||||||
// Unapplied lists cash rows carrying a fee or tax. A broker cash amount is
|
// Unapplied lists cash rows carrying a fee or tax. A broker cash amount is
|
||||||
// already net of them, so subtracting them again would double-count; they
|
// already net of them, so subtracting them again would double-count; they
|
||||||
// are recorded on the fact and reported here.
|
// are recorded on the fact and reported here.
|
||||||
@@ -152,7 +154,7 @@ func ParseScalableCSV(f CSVFile, account domain.Account, registry []domain.Instr
|
|||||||
}
|
}
|
||||||
created := map[string]int{}
|
created := map[string]int{}
|
||||||
named := map[string]string{}
|
named := map[string]string{}
|
||||||
drift := int64(0)
|
drift := new(big.Int)
|
||||||
for offset, row := range f.rows[header:] {
|
for offset, row := range f.rows[header:] {
|
||||||
record := header + offset + 1
|
record := header + offset + 1
|
||||||
if blankCSVRow(row) {
|
if blankCSVRow(row) {
|
||||||
@@ -226,9 +228,9 @@ func ParseScalableCSV(f CSVFile, account domain.Account, registry []domain.Instr
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return result, fmt.Errorf("broker record %d has an invalid tax %q: %w", record, cell(row, "tax"), err)
|
return result, fmt.Errorf("broker record %d has an invalid tax %q: %w", record, cell(row, "tax"), err)
|
||||||
}
|
}
|
||||||
if amountDrift|feeDrift|taxDrift != 0 {
|
if amountDrift.Sign() != 0 || feeDrift.Sign() != 0 || taxDrift.Sign() != 0 {
|
||||||
result.Rounded++
|
result.Rounded++
|
||||||
drift += amountDrift + feeDrift + taxDrift
|
drift.Add(drift, amountDrift).Add(drift, feeDrift).Add(drift, taxDrift)
|
||||||
}
|
}
|
||||||
cash := amount
|
cash := amount
|
||||||
if investment.CashOnly() {
|
if investment.CashOnly() {
|
||||||
@@ -248,7 +250,7 @@ func ParseScalableCSV(f CSVFile, account domain.Account, registry []domain.Instr
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return result, fmt.Errorf("broker record %d has an invalid price %q: %w", record, cell(row, "price"), err)
|
return result, fmt.Errorf("broker record %d has an invalid price %q: %w", record, cell(row, "price"), err)
|
||||||
}
|
}
|
||||||
if priceDrift != 0 {
|
if priceDrift.Sign() != 0 {
|
||||||
return result, fmt.Errorf("broker record %d has a price %q beyond four decimal places", record, cell(row, "price"))
|
return result, fmt.Errorf("broker record %d has a price %q beyond four decimal places", record, cell(row, "price"))
|
||||||
}
|
}
|
||||||
signed, err := scalableSignedShares(event, shares)
|
signed, err := scalableSignedShares(event, shares)
|
||||||
@@ -288,7 +290,7 @@ func ParseScalableCSV(f CSVFile, account domain.Account, registry []domain.Instr
|
|||||||
if len(result.Facts) == 0 {
|
if len(result.Facts) == 0 {
|
||||||
return result, errors.New("broker export contains no executed records")
|
return result, errors.New("broker export contains no executed records")
|
||||||
}
|
}
|
||||||
result.Rounding = domain.FormatQuantity(drift)
|
result.Rounding = decimalString(drift, residueScale)
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -301,32 +303,85 @@ func nonzeroMoney(m domain.Money) bool {
|
|||||||
return err == nil && minor != 0
|
return err == nil && minor != 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// scalableMoney reads one German-formatted money cell and returns the exact
|
// residueScale is the precision the discarded remainder is accumulated at.
|
||||||
// remainder that rounding discarded, in hundred-millionths. The export quotes a
|
// A broker amount is its share count times its price, so it carries as many
|
||||||
// reinvested distribution to six decimal places, which money's four cannot
|
// decimal places as the two together need: a real export reinvests to nine.
|
||||||
// hold; the residue is reported rather than hidden. An empty cell is empty
|
// Eighteen is far past anything a settlement can produce and still exact.
|
||||||
// money, not zero: blank marks a column that does not apply to the row.
|
const residueScale = 18
|
||||||
func scalableMoney(value string) (domain.Money, int64, error) {
|
|
||||||
|
// scalableMoney reads one German-formatted money cell, rounds it to money's
|
||||||
|
// four decimal places half away from zero, and returns the exact remainder
|
||||||
|
// that rounding discarded, in units of 1e-18. The remainder is reported rather
|
||||||
|
// than hidden, and never guessed at: it is the only honest account of why a
|
||||||
|
// computed balance can differ from the broker's by a fraction of a cent.
|
||||||
|
//
|
||||||
|
// An empty cell is empty money, not zero: blank marks a column that does not
|
||||||
|
// apply to the row.
|
||||||
|
func scalableMoney(value string) (domain.Money, *big.Int, error) {
|
||||||
plain, ok, err := scalablePlain(value)
|
plain, ok, err := scalablePlain(value)
|
||||||
if !ok || err != nil {
|
if !ok || err != nil {
|
||||||
return "", 0, err
|
return "", new(big.Int), err
|
||||||
}
|
}
|
||||||
exact, err := domain.ParseQuantity(plain)
|
magnitude, negative, err := scalableDigits(plain)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", 0, err
|
return "", new(big.Int), err
|
||||||
}
|
}
|
||||||
units, err := exact.Units()
|
// One money place is 1e14 residue units. Rounding compares twice the
|
||||||
if err != nil {
|
// remainder against that, so a tie rounds away from zero.
|
||||||
return "", 0, err
|
place := new(big.Int).Exp(big.NewInt(10), big.NewInt(residueScale-4), nil)
|
||||||
|
rounded, remainder := new(big.Int).QuoRem(magnitude, place, new(big.Int))
|
||||||
|
if new(big.Int).Lsh(remainder, 1).Cmp(place) >= 0 {
|
||||||
|
rounded.Add(rounded, big.NewInt(1))
|
||||||
}
|
}
|
||||||
rounded := units / 10000
|
if !rounded.IsInt64() {
|
||||||
switch remainder := units % 10000; {
|
return "", new(big.Int), fmt.Errorf("value is out of range for money")
|
||||||
case remainder >= 5000:
|
|
||||||
rounded++
|
|
||||||
case remainder <= -5000:
|
|
||||||
rounded--
|
|
||||||
}
|
}
|
||||||
return domain.FormatMoney(rounded), units - rounded*10000, nil
|
residue := new(big.Int).Sub(magnitude, new(big.Int).Mul(rounded, place))
|
||||||
|
minor := rounded.Int64()
|
||||||
|
if negative {
|
||||||
|
minor, residue = -minor, residue.Neg(residue)
|
||||||
|
}
|
||||||
|
return domain.FormatMoney(minor), residue, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// scalableDigits splits a plain decimal string into its exact magnitude in
|
||||||
|
// residue units and its sign.
|
||||||
|
func scalableDigits(plain string) (magnitude *big.Int, negative bool, err error) {
|
||||||
|
digits := plain
|
||||||
|
if rest, cut := strings.CutPrefix(digits, "-"); cut {
|
||||||
|
negative, digits = true, rest
|
||||||
|
}
|
||||||
|
whole, decimals, _ := strings.Cut(digits, ".")
|
||||||
|
if whole == "" {
|
||||||
|
return nil, false, fmt.Errorf("decimal needs a digit before the separator")
|
||||||
|
}
|
||||||
|
if len(decimals) > residueScale {
|
||||||
|
return nil, false, fmt.Errorf("more than %d fractional digits", residueScale)
|
||||||
|
}
|
||||||
|
scaled, ok := new(big.Int).SetString(whole+decimals+strings.Repeat("0", residueScale-len(decimals)), 10)
|
||||||
|
if !ok {
|
||||||
|
return nil, false, fmt.Errorf("not a decimal number")
|
||||||
|
}
|
||||||
|
return scaled, negative, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// decimalString renders exact units at a scale without trailing zeros, so an
|
||||||
|
// adjustment of 1e-9 is reported as such rather than padded to eighteen places.
|
||||||
|
func decimalString(units *big.Int, scale int) string {
|
||||||
|
sign := ""
|
||||||
|
magnitude := new(big.Int).Abs(units)
|
||||||
|
if units.Sign() < 0 {
|
||||||
|
sign = "-"
|
||||||
|
}
|
||||||
|
digits := magnitude.String()
|
||||||
|
if len(digits) <= scale {
|
||||||
|
digits = strings.Repeat("0", scale+1-len(digits)) + digits
|
||||||
|
}
|
||||||
|
whole, fraction := digits[:len(digits)-scale], strings.TrimRight(digits[len(digits)-scale:], "0")
|
||||||
|
if fraction == "" {
|
||||||
|
return sign + whole
|
||||||
|
}
|
||||||
|
return sign + whole + "." + fraction
|
||||||
}
|
}
|
||||||
|
|
||||||
// scalableQuantity reads one German-formatted share count. Nothing is rounded:
|
// scalableQuantity reads one German-formatted share count. Nothing is rounded:
|
||||||
|
|||||||
@@ -228,6 +228,56 @@ func TestSingleShareRowCatchesOnlyInconsistentArithmetic(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A reinvested distribution settles shares times price, so it carries as many
|
||||||
|
// decimal places as the two together need. A real export reinvests to nine,
|
||||||
|
// which is past what money holds and past what a share count holds, so reading
|
||||||
|
// the cell at either precision rejects the row outright. It is rounded to
|
||||||
|
// money's four and the discarded remainder is reported exactly.
|
||||||
|
func TestReinvestmentKeepsNineDecimalPlacesOutOfTheBalance(t *testing.T) {
|
||||||
|
const reference = "617007_rrCjP4EcbpefpNiVQeD495"
|
||||||
|
result := readBroker(t,
|
||||||
|
`2026-05-28;02:00:00;Executed;`+reference+`;iShares Global Clean Energy Transition (Dist);Cash;Distribution;IE00B1XNHC34;;;29,58;0,00;8,56;EUR`,
|
||||||
|
`2026-05-28;02:00:00;Executed;`+reference+`;iShares Global Clean Energy Transition (Dist);Security;Reinvestment_Distribution;IE00B1XNHC34;3,144131;9,408;-29,579984448;0,00;0,00;EUR`,
|
||||||
|
)
|
||||||
|
if len(result.Facts) != 2 {
|
||||||
|
t.Fatalf("read %d of 2 legs", len(result.Facts))
|
||||||
|
}
|
||||||
|
cash, reinvest := result.Facts[0], result.Facts[1]
|
||||||
|
if cash.Amount != "29.58" || cash.Investment.Tax != "8.56" {
|
||||||
|
t.Errorf("distribution settled %s with tax %s, want 29.58 and 8.56 recorded", cash.Amount, cash.Investment.Tax)
|
||||||
|
}
|
||||||
|
// 29.579984448 rounds up at the fifth place, and the residue is exact.
|
||||||
|
if reinvest.Amount != "-29.58" || reinvest.Investment.Gross != "-29.58" {
|
||||||
|
t.Errorf("reinvestment settled %s against gross %s, want -29.58 for both", reinvest.Amount, reinvest.Investment.Gross)
|
||||||
|
}
|
||||||
|
if reinvest.Investment.Quantity != "3.144131" {
|
||||||
|
t.Errorf("reinvested %s shares, want 3.144131", reinvest.Investment.Quantity)
|
||||||
|
}
|
||||||
|
if result.Rounded != 1 || result.Rounding != "0.000015552" {
|
||||||
|
t.Errorf("rounding reported as %d row(s) and %s, want 1 and 0.000015552", result.Rounded, result.Rounding)
|
||||||
|
}
|
||||||
|
// The dividend paid in and the units bought with it cancel to the cent.
|
||||||
|
total := int64(0)
|
||||||
|
for _, f := range result.Facts {
|
||||||
|
minor, err := f.Amount.Minor()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
total += minor
|
||||||
|
}
|
||||||
|
if total != 0 {
|
||||||
|
t.Errorf("the pair moved %s of net cash, want none", domain.FormatMoney(total))
|
||||||
|
}
|
||||||
|
// Both legs carry one reference byte for byte and must both survive.
|
||||||
|
data := domain.NewDataset()
|
||||||
|
data.Accounts = []domain.Account{brokerAccount()}
|
||||||
|
data.Instruments = result.Instruments
|
||||||
|
added, err := NormalizeAndDedupe(data, result.Facts)
|
||||||
|
if err != nil || len(added) != 2 {
|
||||||
|
t.Fatalf("dedupe kept %d of 2 legs sharing a reference: %v", len(added), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// A thousands dot and a decimal dot are both present in one share column.
|
// A thousands dot and a decimal dot are both present in one share column.
|
||||||
func TestBrokerShareColumnDistinguishesGroupingFromDecimals(t *testing.T) {
|
func TestBrokerShareColumnDistinguishesGroupingFromDecimals(t *testing.T) {
|
||||||
result := readBroker(t,
|
result := readBroker(t,
|
||||||
|
|||||||
Reference in New Issue
Block a user