package banking import ( "errors" "fmt" "strings" "finance-duck/internal/domain" ) // SourceScalable identifies facts imported from a Scalable Capital broker // export. const SourceScalable = "scalable_csv" // scalableColumns are the exact normalized headers of a Scalable Capital // transaction export. The layout is matched in full rather than column by // column: a row's meaning depends on the combination of status, assetType and // type, so a partial match would be a different file wearing the same names. var scalableColumns = []string{ "date", "time", "status", "reference", "description", "assettype", "type", "isin", "shares", "price", "amount", "fee", "tax", "currency", } // scalableEvents maps the export's complete type vocabulary to journal events. // The set is closed on purpose: two of the ten types move a position without // moving money, so an unrecognized type cannot be defaulted either way without // risking a silent balance error. Keys are lowercased with collapsed spaces. var scalableEvents = map[string]string{ "deposit": domain.EventDeposit, "withdrawal": domain.EventWithdrawal, "fee": domain.EventFee, "interest": domain.EventInterest, "distribution": domain.EventDistribution, "buy": domain.EventBuy, "sell": domain.EventSell, "reinvestment_distribution": domain.EventReinvest, "corporate action": domain.EventCorporateAction, "security transfer": domain.EventPositionTransfer, } // ScalableNote records a figure the export carried that the import deliberately // did not apply, so it can be reviewed before confirming and recognized later // if a balance disagrees. type ScalableNote struct { Record int `json:"record"` Date string `json:"date"` Description string `json:"description"` Fee domain.Money `json:"fee,omitempty"` Tax domain.Money `json:"tax,omitempty"` } // ScalableImport is a read broker export awaiting review. type ScalableImport struct { Facts []domain.Facts `json:"-"` // Instruments are securities the export named that the registry does not // hold yet. An import never renames an existing instrument: the name is // editable display text, and the export's own description for one ISIN // changes over time. Instruments []domain.Instrument `json:"instruments"` // Cancelled counts rows the broker did not execute. Their money and share // columns are all zeros, so they satisfy every arithmetic check and would // otherwise import as phantom trades. Cancelled int `json:"cancelled"` // Rounded counts rows whose money carried more than four decimal places, // and Rounding is the exact total adjustment that rounding applied. Rounded int `json:"rounded"` Rounding domain.Quantity `json:"rounding"` // Unapplied lists cash rows carrying a fee or tax. A broker cash amount is // already net of them, so subtracting them again would double-count; they // are recorded on the fact and reported here. Unapplied []ScalableNote `json:"unapplied"` } // DetectScalableCSV reports whether a document is a Scalable Capital export and // which 1-based record holds its header. func DetectScalableCSV(f CSVFile) (header int, ok bool) { for i, row := range f.rows { if i >= maxCSVPreambleRows { break } columns, usable := csvColumnIndex(row) if !usable || len(columns) != len(scalableColumns) { continue } matched := true for _, name := range scalableColumns { if _, exists := columns[name]; !exists { matched = false break } } if matched { return i + 1, true } } return 0, false } // ParseScalableCSV converts a broker export into bank facts carrying position // legs. // // The amount column means a different thing per row class, and reading it // wrongly moves money that never moved: // // - a cash row's amount is the money that actually settled, already net of // the tax the broker withheld or refunded, so its tax is recorded and not // applied; // - a buy, sell or reinvestment quotes gross shares times price and settles // gross minus fee minus tax; // - a corporate action or depot transfer quotes a position valuation and // settles no cash at all. // // The share column is signed only for those last two types; buys and sells are // unsigned and take their direction from the type. Both conventions are // resolved here, once. // // The booking date is the date column exactly as printed. Batch rows are // stamped midnight UTC rendered in local time, so the time column crosses // midnight for part of the year and reading date and time together would move // those rows to the previous day. // // A single unrecognized status, type or assetType, or one failed arithmetic // check, rejects the whole file. Every one of those cases can move money, and a // partially imported broker history cannot be told from a truncated export // afterwards. func ParseScalableCSV(f CSVFile, account domain.Account, registry []domain.Instrument) (ScalableImport, error) { // Empty rather than nil: these are arrays in the reviewed JSON, and a null // where a caller expects a list is a bug waiting on a different machine. result := ScalableImport{Instruments: []domain.Instrument{}, Unapplied: []ScalableNote{}} if account.ID == "" { return result, errors.New("broker import requires a selected account") } if !account.Investing() { return result, fmt.Errorf("account %q must be an investment account to hold a broker export", account.DisplayName) } header, ok := DetectScalableCSV(f) if !ok { return result, errors.New("not a Scalable Capital export") } headers := f.rows[header-1] index := map[string]int{} for i, raw := range headers { index[headerName(raw)] = i } cell := func(row []string, name string) string { return strings.TrimSpace(row[index[name]]) } instruments := map[string]domain.Instrument{} byISIN := map[string]domain.Instrument{} for _, v := range registry { instruments[v.ID] = v byISIN[v.ISIN] = v } created := map[string]int{} named := map[string]string{} drift := int64(0) for offset, row := range f.rows[header:] { record := header + offset + 1 if blankCSVRow(row) { continue } if len(row) != len(headers) { return result, fmt.Errorf("broker record %d has %d columns, expected %d", record, len(row), len(headers)) } switch status := cell(row, "status"); { case strings.EqualFold(status, "executed"): case strings.EqualFold(status, "cancelled"), strings.EqualFold(status, "canceled"): result.Cancelled++ continue default: return result, fmt.Errorf("broker record %d has unknown status %q: only executed and cancelled rows are understood", record, status) } rawType := cell(row, "type") event, known := scalableEvents[strings.ToLower(strings.Join(strings.Fields(rawType), " "))] if !known { return result, fmt.Errorf("broker record %d has unknown type %q: it may or may not move cash, so nothing was imported", record, rawType) } investment := domain.Investment{Event: event} asset, wanted := cell(row, "assettype"), "Security" if investment.CashOnly() { wanted = "Cash" } if !strings.EqualFold(asset, wanted) { return result, fmt.Errorf("broker record %d pairs type %q with assetType %q, expected %q", record, rawType, asset, wanted) } currency := strings.ToUpper(cell(row, "currency")) if currency != strings.ToUpper(account.Currency) { return result, fmt.Errorf("broker record %d settles in %q but account %q holds %s: currency conversion is not supported", record, currency, account.DisplayName, account.Currency) } booking, err := parseMappedCSVDate(cell(row, "date"), "yyyy-mm-dd") if err != nil { return result, fmt.Errorf("broker record %d has an invalid date %q", record, cell(row, "date")) } description := cell(row, "description") isin := strings.ToUpper(strings.Join(strings.Fields(cell(row, "isin")), "")) if isin != "" && !domain.ValidISIN(isin) { return result, fmt.Errorf("broker record %d has an invalid ISIN %q", record, isin) } if isin != "" { held, exists := byISIN[isin] if !exists { held = domain.Instrument{ID: domain.InstrumentID(isin), ISIN: isin, Name: isin, Currency: currency} byISIN[isin] = held instruments[held.ID] = held created[isin] = len(result.Instruments) result.Instruments = append(result.Instruments, held) } investment.InstrumentID = held.ID // One ISIN appears under several descriptions over the years, and // once under the ISIN itself. The most recent real description // names it, and only when this import is the one creating it. slot, mine := created[isin] if mine && description != "" && description != isin && booking >= named[isin] { named[isin] = booking result.Instruments[slot].Name = description } } amount, amountDrift, err := scalableMoney(cell(row, "amount")) if err != nil { return result, fmt.Errorf("broker record %d has an invalid amount %q: %w", record, cell(row, "amount"), err) } fee, feeDrift, err := scalableMoney(cell(row, "fee")) if err != nil { return result, fmt.Errorf("broker record %d has an invalid fee %q: %w", record, cell(row, "fee"), err) } tax, taxDrift, err := scalableMoney(cell(row, "tax")) if err != nil { return result, fmt.Errorf("broker record %d has an invalid tax %q: %w", record, cell(row, "tax"), err) } if amountDrift|feeDrift|taxDrift != 0 { result.Rounded++ drift += amountDrift + feeDrift + taxDrift } cash := amount if investment.CashOnly() { if nonzeroMoney(fee) || nonzeroMoney(tax) { result.Unapplied = append(result.Unapplied, ScalableNote{Record: record, Date: booking, Description: description, Fee: fee, Tax: tax}) } investment.Fee, investment.Tax = fee, tax } else { if isin == "" { return result, fmt.Errorf("broker record %d moves a position without an ISIN", record) } shares, err := scalableQuantity(cell(row, "shares")) if err != nil { return result, fmt.Errorf("broker record %d has an invalid share count %q: %w", record, cell(row, "shares"), err) } price, priceDrift, err := scalableMoney(cell(row, "price")) if err != nil { return result, fmt.Errorf("broker record %d has an invalid price %q: %w", record, cell(row, "price"), err) } if priceDrift != 0 { return result, fmt.Errorf("broker record %d has a price %q beyond four decimal places", record, cell(row, "price")) } signed, err := scalableSignedShares(event, shares) if err != nil { return result, fmt.Errorf("broker record %d: %w", record, err) } investment.Quantity, investment.Price, investment.Gross = signed, price, amount if investment.PositionOnly() { if fee != "" || tax != "" { return result, fmt.Errorf("broker record %d is a %s carrying fee %q and tax %q, which have no settled cash to apply to", record, rawType, fee, tax) } cash = "0.00" } else { investment.Fee, investment.Tax = fee, tax if cash, err = scalableSettlement(amount, fee, tax); err != nil { return result, fmt.Errorf("broker record %d: %w", record, err) } } } facts := domain.Facts{ Source: SourceScalable, AccountID: account.ID, BookingDate: booking, Amount: cash, Currency: currency, RawDescription: description, ExternalID: cell(row, "reference"), Investment: &investment, } // A broker export has no counterparty column, so a deposit or // withdrawal takes the account's configured settlement IBAN. That is // what lets the ordinary transfer matcher pair it with the funding // account instead of leaving it to look like income. if investment.Event == domain.EventDeposit || investment.Event == domain.EventWithdrawal { facts.CounterpartyIBAN = normalizeIBAN(account.ReferenceIBAN) } if err := domain.ValidateInvestment(facts, account, instruments); err != nil { return result, fmt.Errorf("broker record %d: %w", record, err) } result.Facts = append(result.Facts, facts) } if len(result.Facts) == 0 { return result, errors.New("broker export contains no executed records") } result.Rounding = domain.FormatQuantity(drift) return result, nil } // nonzeroMoney reports a figure that could change a balance. The export leaves // a column blank where it does not apply and writes an explicit zero where it // applies but is nil; only the second kind is worth putting in front of // someone before they confirm an import. func nonzeroMoney(m domain.Money) bool { minor, err := m.Minor() return err == nil && minor != 0 } // scalableMoney reads one German-formatted money cell and returns the exact // remainder that rounding discarded, in hundred-millionths. The export quotes a // reinvested distribution to six decimal places, which money's four cannot // hold; the residue is reported rather than hidden. An empty cell is empty // money, not zero: blank marks a column that does not apply to the row. func scalableMoney(value string) (domain.Money, int64, error) { plain, ok, err := scalablePlain(value) if !ok || err != nil { return "", 0, err } exact, err := domain.ParseQuantity(plain) if err != nil { return "", 0, err } units, err := exact.Units() if err != nil { return "", 0, err } rounded := units / 10000 switch remainder := units % 10000; { case remainder >= 5000: rounded++ case remainder <= -5000: rounded-- } return domain.FormatMoney(rounded), units - rounded*10000, nil } // scalableQuantity reads one German-formatted share count. Nothing is rounded: // a holding is verified against the broker's own figure, so a count beyond // eight decimal places is refused instead of silently truncated. func scalableQuantity(value string) (domain.Quantity, error) { plain, ok, err := scalablePlain(value) if !ok || err != nil { return "", err } return domain.ParseQuantity(plain) } // scalablePlain normalizes one numeric cell to a plain decimal string, or // reports that the cell was blank. func scalablePlain(value string) (string, bool, error) { value = strings.NewReplacer("\u00a0", "", "\u202f", "", "'", "").Replace(strings.TrimSpace(value)) if value == "" { return "", false, nil } plain, err := germanDecimal(value) return plain, err == nil, err } // scalableSignedShares resolves the export's two sign conventions. A buy, sell // or reinvestment carries an unsigned count and takes its direction from the // type; a corporate action or depot transfer is already signed. func scalableSignedShares(event string, shares domain.Quantity) (domain.Quantity, error) { units, err := shares.Units() if err != nil { return "", err } if units == 0 { return "", fmt.Errorf("%s requires a nonzero share count", event) } switch event { case domain.EventBuy, domain.EventReinvest, domain.EventSell: if units < 0 { return "", fmt.Errorf("%s carries a signed share count %s; only corporate actions and depot transfers are signed", event, shares) } if event == domain.EventSell { units = -units } } return domain.FormatQuantity(units), nil } // scalableSettlement is gross minus fee minus tax: the cash a trade moved. The // broker states fee and tax as positive deductions whichever way the trade // went, so both are subtracted from a signed gross. func scalableSettlement(gross, fee, tax domain.Money) (domain.Money, error) { total := int64(0) for _, deduction := range []struct { sign int64 money domain.Money }{{1, gross}, {-1, fee}, {-1, tax}} { if deduction.money == "" { continue } minor, err := deduction.money.Minor() if err != nil { return "", err } total += deduction.sign * minor } return domain.FormatMoney(total), nil }