finance-duck/internal/banking/scalable.go

package banking

import (
	"errors"
	"fmt"
	"math/big"
	"strings"

	"finance-duck/internal/domain"
)

// SourceScalable identifies facts imported from a Scalable Capital broker
// export.
const SourceScalable = "scalable_csv"

// scalableColumns are the exact normalized headers of a Scalable Capital
// transaction export. The layout is matched in full rather than column by
// column: a row's meaning depends on the combination of status, assetType and
// type, so a partial match would be a different file wearing the same names.
var scalableColumns = []string{
	"date", "time", "status", "reference", "description",
	"assettype", "type", "isin", "shares", "price", "amount", "fee", "tax", "currency",
}

// scalableEvents maps the export's complete type vocabulary to journal events.
// The set is closed on purpose: two of the ten types move a position without
// moving money, so an unrecognized type cannot be defaulted either way without
// risking a silent balance error. Keys are lowercased with collapsed spaces.
var scalableEvents = map[string]string{
	"deposit":                   domain.EventDeposit,
	"withdrawal":                domain.EventWithdrawal,
	"fee":                       domain.EventFee,
	"interest":                  domain.EventInterest,
	"distribution":              domain.EventDistribution,
	"buy":                       domain.EventBuy,
	"sell":                      domain.EventSell,
	"reinvestment_distribution": domain.EventReinvest,
	"corporate action":          domain.EventCorporateAction,
	"security transfer":         domain.EventPositionTransfer,
}

// ScalableNote records a figure the export carried that the import deliberately
// did not apply, so it can be reviewed before confirming and recognized later
// if a balance disagrees.
type ScalableNote struct {
	Record      int          `json:"record"`
	Date        string       `json:"date"`
	Description string       `json:"description"`
	Fee         domain.Money `json:"fee,omitempty"`
	Tax         domain.Money `json:"tax,omitempty"`
}

// ScalableImport is a read broker export awaiting review.
type ScalableImport struct {
	Facts []domain.Facts `json:"-"`
	// Instruments are securities the export named that the registry does not
	// hold yet. An import never renames an existing instrument: the name is
	// editable display text, and the export's own description for one ISIN
	// changes over time.
	Instruments []domain.Instrument `json:"instruments"`
	// Cancelled counts rows the broker did not execute. Their money and share
	// columns are all zeros, so they satisfy every arithmetic check and would
	// otherwise import as phantom trades.
	Cancelled int `json:"cancelled"`
	// Rounded counts rows whose money carried more than four decimal places,
	// and Rounding is the exact total adjustment that rounding applied, at
	// whatever precision the export used.
	Rounded  int    `json:"rounded"`
	Rounding string `json:"rounding"`
	// Unapplied lists cash rows carrying a fee or tax. A broker cash amount is
	// already net of them, so subtracting them again would double-count; they
	// are recorded on the fact and reported here.
	Unapplied []ScalableNote `json:"unapplied"`
}

// DetectScalableCSV reports whether a document is a Scalable Capital export and
// which 1-based record holds its header.
func DetectScalableCSV(f CSVFile) (header int, ok bool) {
	for i, row := range f.rows {
		if i >= maxCSVPreambleRows {
			break
		}
		columns, usable := csvColumnIndex(row)
		if !usable || len(columns) != len(scalableColumns) {
			continue
		}
		matched := true
		for _, name := range scalableColumns {
			if _, exists := columns[name]; !exists {
				matched = false
				break
			}
		}
		if matched {
			return i + 1, true
		}
	}
	return 0, false
}

// ParseScalableCSV converts a broker export into bank facts carrying position
// legs.
//
// The amount column means a different thing per row class, and reading it
// wrongly moves money that never moved:
//
//   - a cash row's amount is the money that actually settled, already net of
//     the tax the broker withheld or refunded, so its tax is recorded and not
//     applied;
//   - a buy, sell or reinvestment quotes gross shares times price and settles
//     gross minus fee minus tax;
//   - a corporate action or depot transfer quotes a position valuation and
//     settles no cash at all.
//
// The share column is signed only for those last two types; buys and sells are
// unsigned and take their direction from the type. Both conventions are
// resolved here, once.
//
// The booking date is the date column exactly as printed. Batch rows are
// stamped midnight UTC rendered in local time, so the time column crosses
// midnight for part of the year and reading date and time together would move
// those rows to the previous day.
//
// A single unrecognized status, type or assetType, or one failed arithmetic
// check, rejects the whole file. Every one of those cases can move money, and a
// partially imported broker history cannot be told from a truncated export
// afterwards.
func ParseScalableCSV(f CSVFile, account domain.Account, registry []domain.Instrument) (ScalableImport, error) {
	// Empty rather than nil: these are arrays in the reviewed JSON, and a null
	// where a caller expects a list is a bug waiting on a different machine.
	result := ScalableImport{Instruments: []domain.Instrument{}, Unapplied: []ScalableNote{}}
	if account.ID == "" {
		return result, errors.New("broker import requires a selected account")
	}
	if !account.Investing() {
		return result, fmt.Errorf("account %q must be an investment account to hold a broker export", account.DisplayName)
	}
	header, ok := DetectScalableCSV(f)
	if !ok {
		return result, errors.New("not a Scalable Capital export")
	}
	headers := f.rows[header-1]
	index := map[string]int{}
	for i, raw := range headers {
		index[headerName(raw)] = i
	}
	cell := func(row []string, name string) string { return strings.TrimSpace(row[index[name]]) }

	instruments := map[string]domain.Instrument{}
	byISIN := map[string]domain.Instrument{}
	for _, v := range registry {
		instruments[v.ID] = v
		byISIN[v.ISIN] = v
	}
	created := map[string]int{}
	named := map[string]string{}
	drift := new(big.Int)
	for offset, row := range f.rows[header:] {
		record := header + offset + 1
		if blankCSVRow(row) {
			continue
		}
		if len(row) != len(headers) {
			return result, fmt.Errorf("broker record %d has %d columns, expected %d", record, len(row), len(headers))
		}
		switch status := cell(row, "status"); {
		case strings.EqualFold(status, "executed"):
		case strings.EqualFold(status, "cancelled"), strings.EqualFold(status, "canceled"):
			result.Cancelled++
			continue
		default:
			return result, fmt.Errorf("broker record %d has unknown status %q: only executed and cancelled rows are understood", record, status)
		}
		rawType := cell(row, "type")
		event, known := scalableEvents[strings.ToLower(strings.Join(strings.Fields(rawType), " "))]
		if !known {
			return result, fmt.Errorf("broker record %d has unknown type %q: it may or may not move cash, so nothing was imported", record, rawType)
		}
		investment := domain.Investment{Event: event}
		asset, wanted := cell(row, "assettype"), "Security"
		if investment.CashOnly() {
			wanted = "Cash"
		}
		if !strings.EqualFold(asset, wanted) {
			return result, fmt.Errorf("broker record %d pairs type %q with assetType %q, expected %q", record, rawType, asset, wanted)
		}
		currency := strings.ToUpper(cell(row, "currency"))
		if currency != strings.ToUpper(account.Currency) {
			return result, fmt.Errorf("broker record %d settles in %q but account %q holds %s: currency conversion is not supported", record, currency, account.DisplayName, account.Currency)
		}
		booking, err := parseMappedCSVDate(cell(row, "date"), "yyyy-mm-dd")
		if err != nil {
			return result, fmt.Errorf("broker record %d has an invalid date %q", record, cell(row, "date"))
		}
		description := cell(row, "description")
		isin := strings.ToUpper(strings.Join(strings.Fields(cell(row, "isin")), ""))
		if isin != "" && !domain.ValidISIN(isin) {
			return result, fmt.Errorf("broker record %d has an invalid ISIN %q", record, isin)
		}
		if isin != "" {
			held, exists := byISIN[isin]
			if !exists {
				held = domain.Instrument{ID: domain.InstrumentID(isin), ISIN: isin, Name: isin, Currency: currency}
				byISIN[isin] = held
				instruments[held.ID] = held
				created[isin] = len(result.Instruments)
				result.Instruments = append(result.Instruments, held)
			}
			investment.InstrumentID = held.ID
			// One ISIN appears under several descriptions over the years, and
			// once under the ISIN itself. The most recent real description
			// names it, and only when this import is the one creating it.
			slot, mine := created[isin]
			if mine && description != "" && description != isin && booking >= named[isin] {
				named[isin] = booking
				result.Instruments[slot].Name = description
			}
		}
		amount, amountDrift, err := scalableMoney(cell(row, "amount"))
		if err != nil {
			return result, fmt.Errorf("broker record %d has an invalid amount %q: %w", record, cell(row, "amount"), err)
		}
		fee, feeDrift, err := scalableMoney(cell(row, "fee"))
		if err != nil {
			return result, fmt.Errorf("broker record %d has an invalid fee %q: %w", record, cell(row, "fee"), err)
		}
		tax, taxDrift, err := scalableMoney(cell(row, "tax"))
		if err != nil {
			return result, fmt.Errorf("broker record %d has an invalid tax %q: %w", record, cell(row, "tax"), err)
		}
		if amountDrift.Sign() != 0 || feeDrift.Sign() != 0 || taxDrift.Sign() != 0 {
			result.Rounded++
			drift.Add(drift, amountDrift).Add(drift, feeDrift).Add(drift, taxDrift)
		}
		cash := amount
		if investment.CashOnly() {
			if nonzeroMoney(fee) || nonzeroMoney(tax) {
				result.Unapplied = append(result.Unapplied, ScalableNote{Record: record, Date: booking, Description: description, Fee: fee, Tax: tax})
			}
			investment.Fee, investment.Tax = fee, tax
		} else {
			if isin == "" {
				return result, fmt.Errorf("broker record %d moves a position without an ISIN", record)
			}
			shares, err := scalableQuantity(cell(row, "shares"))
			if err != nil {
				return result, fmt.Errorf("broker record %d has an invalid share count %q: %w", record, cell(row, "shares"), err)
			}
			price, priceDrift, err := scalableMoney(cell(row, "price"))
			if err != nil {
				return result, fmt.Errorf("broker record %d has an invalid price %q: %w", record, cell(row, "price"), err)
			}
			if priceDrift.Sign() != 0 {
				return result, fmt.Errorf("broker record %d has a price %q beyond four decimal places", record, cell(row, "price"))
			}
			signed, err := scalableSignedShares(event, shares)
			if err != nil {
				return result, fmt.Errorf("broker record %d: %w", record, err)
			}
			investment.Quantity, investment.Price, investment.Gross = signed, price, amount
			if investment.PositionOnly() {
				if fee != "" || tax != "" {
					return result, fmt.Errorf("broker record %d is a %s carrying fee %q and tax %q, which have no settled cash to apply to", record, rawType, fee, tax)
				}
				cash = "0.00"
			} else {
				investment.Fee, investment.Tax = fee, tax
				if cash, err = scalableSettlement(amount, fee, tax); err != nil {
					return result, fmt.Errorf("broker record %d: %w", record, err)
				}
			}
		}
		facts := domain.Facts{
			Source: SourceScalable, AccountID: account.ID, BookingDate: booking,
			Amount: cash, Currency: currency, RawDescription: description,
			ExternalID: cell(row, "reference"), Investment: &investment,
		}
		// A broker export has no counterparty column, so a deposit or
		// withdrawal takes the account's configured settlement IBAN. That is
		// what lets the ordinary transfer matcher pair it with the funding
		// account instead of leaving it to look like income.
		if investment.Event == domain.EventDeposit || investment.Event == domain.EventWithdrawal {
			facts.CounterpartyIBAN = normalizeIBAN(account.ReferenceIBAN)
		}
		if err := domain.ValidateInvestment(facts, account, instruments); err != nil {
			return result, fmt.Errorf("broker record %d: %w", record, err)
		}
		result.Facts = append(result.Facts, facts)
	}
	if len(result.Facts) == 0 {
		return result, errors.New("broker export contains no executed records")
	}
	result.Rounding = decimalString(drift, residueScale)
	return result, nil
}

// nonzeroMoney reports a figure that could change a balance. The export leaves
// a column blank where it does not apply and writes an explicit zero where it
// applies but is nil; only the second kind is worth putting in front of
// someone before they confirm an import.
func nonzeroMoney(m domain.Money) bool {
	minor, err := m.Minor()
	return err == nil && minor != 0
}

// residueScale is the precision the discarded remainder is accumulated at.
// A broker amount is its share count times its price, so it carries as many
// decimal places as the two together need: a real export reinvests to nine.
// Eighteen is far past anything a settlement can produce and still exact.
const residueScale = 18

// scalableMoney reads one German-formatted money cell, rounds it to money's
// four decimal places half away from zero, and returns the exact remainder
// that rounding discarded, in units of 1e-18. The remainder is reported rather
// than hidden, and never guessed at: it is the only honest account of why a
// computed balance can differ from the broker's by a fraction of a cent.
//
// An empty cell is empty money, not zero: blank marks a column that does not
// apply to the row.
func scalableMoney(value string) (domain.Money, *big.Int, error) {
	plain, ok, err := scalablePlain(value)
	if !ok || err != nil {
		return "", new(big.Int), err
	}
	magnitude, negative, err := scalableDigits(plain)
	if err != nil {
		return "", new(big.Int), err
	}
	// One money place is 1e14 residue units. Rounding compares twice the
	// remainder against that, so a tie rounds away from zero.
	place := new(big.Int).Exp(big.NewInt(10), big.NewInt(residueScale-4), nil)
	rounded, remainder := new(big.Int).QuoRem(magnitude, place, new(big.Int))
	if new(big.Int).Lsh(remainder, 1).Cmp(place) >= 0 {
		rounded.Add(rounded, big.NewInt(1))
	}
	if !rounded.IsInt64() {
		return "", new(big.Int), fmt.Errorf("value is out of range for money")
	}
	residue := new(big.Int).Sub(magnitude, new(big.Int).Mul(rounded, place))
	minor := rounded.Int64()
	if negative {
		minor, residue = -minor, residue.Neg(residue)
	}
	return domain.FormatMoney(minor), residue, nil
}

// scalableDigits splits a plain decimal string into its exact magnitude in
// residue units and its sign.
func scalableDigits(plain string) (magnitude *big.Int, negative bool, err error) {
	digits := plain
	if rest, cut := strings.CutPrefix(digits, "-"); cut {
		negative, digits = true, rest
	}
	whole, decimals, _ := strings.Cut(digits, ".")
	if whole == "" {
		return nil, false, fmt.Errorf("decimal needs a digit before the separator")
	}
	if len(decimals) > residueScale {
		return nil, false, fmt.Errorf("more than %d fractional digits", residueScale)
	}
	scaled, ok := new(big.Int).SetString(whole+decimals+strings.Repeat("0", residueScale-len(decimals)), 10)
	if !ok {
		return nil, false, fmt.Errorf("not a decimal number")
	}
	return scaled, negative, nil
}

// decimalString renders exact units at a scale without trailing zeros, so an
// adjustment of 1e-9 is reported as such rather than padded to eighteen places.
func decimalString(units *big.Int, scale int) string {
	sign := ""
	magnitude := new(big.Int).Abs(units)
	if units.Sign() < 0 {
		sign = "-"
	}
	digits := magnitude.String()
	if len(digits) <= scale {
		digits = strings.Repeat("0", scale+1-len(digits)) + digits
	}
	whole, fraction := digits[:len(digits)-scale], strings.TrimRight(digits[len(digits)-scale:], "0")
	if fraction == "" {
		return sign + whole
	}
	return sign + whole + "." + fraction
}

// scalableQuantity reads one German-formatted share count. Nothing is rounded:
// a holding is verified against the broker's own figure, so a count beyond
// eight decimal places is refused instead of silently truncated.
func scalableQuantity(value string) (domain.Quantity, error) {
	plain, ok, err := scalablePlain(value)
	if !ok || err != nil {
		return "", err
	}
	return domain.ParseQuantity(plain)
}

// scalablePlain normalizes one numeric cell to a plain decimal string, or
// reports that the cell was blank.
func scalablePlain(value string) (string, bool, error) {
	value = strings.NewReplacer("\u00a0", "", "\u202f", "", "'", "").Replace(strings.TrimSpace(value))
	if value == "" {
		return "", false, nil
	}
	plain, err := germanDecimal(value)
	return plain, err == nil, err
}

// scalableSignedShares resolves the export's two sign conventions. A buy, sell
// or reinvestment carries an unsigned count and takes its direction from the
// type; a corporate action or depot transfer is already signed.
func scalableSignedShares(event string, shares domain.Quantity) (domain.Quantity, error) {
	units, err := shares.Units()
	if err != nil {
		return "", err
	}
	if units == 0 {
		return "", fmt.Errorf("%s requires a nonzero share count", event)
	}
	switch event {
	case domain.EventBuy, domain.EventReinvest, domain.EventSell:
		if units < 0 {
			return "", fmt.Errorf("%s carries a signed share count %s; only corporate actions and depot transfers are signed", event, shares)
		}
		if event == domain.EventSell {
			units = -units
		}
	}
	return domain.FormatQuantity(units), nil
}

// scalableSettlement is gross minus fee minus tax: the cash a trade moved. The
// broker states fee and tax as positive deductions whichever way the trade
// went, so both are subtracted from a signed gross.
func scalableSettlement(gross, fee, tax domain.Money) (domain.Money, error) {
	total := int64(0)
	for _, deduction := range []struct {
		sign  int64
		money domain.Money
	}{{1, gross}, {-1, fee}, {-1, tax}} {
		if deduction.money == "" {
			continue
		}
		minor, err := deduction.money.Minor()
		if err != nil {
			return "", err
		}
		total += deduction.sign * minor
	}
	return domain.FormatMoney(total), nil
}