A position was a share count. An instrument now carries a market symbol and the last close fetched for it, so Wealth and the dashboard report cash plus market value instead of cash alone. The symbol is chosen by hand and never derived: one ISIN lists on several exchanges in different currencies, and a price from the wrong listing misstates wealth without failing any check. The refresh refuses a quote whose currency differs from the instrument's, keeps the previous quote when a symbol cannot be priced, and counts an instrument with no symbol as unpriced - naming it in a check and leaving it out of every total, because cost is not value. The quote belongs to the job: saving an instrument can neither set nor erase it, and changing the symbol discards it. Two things the provider forced. It answers HTTP 429 to every request whose User-Agent names a programming language, so the client identifies as a browser; without that header the first call of the day fails. Its closes are 32-bit floats widened to 64 - 165.26 arrives as 165.25999450683594 - so a figure is rounded to seven significant digits, which is what 24 mantissa bits carry; eight would have stored 165.25999 as a price. Accepted quotes are written in one commit against a revision re-read after the fetches, and nothing is committed when no quote changed. The automatic run starts shortly after launch and repeats daily on its own timer, so a sync backoff cannot delay it and prices arrive with no bank connected. Verified against live quotes end to end: 80 shares at 125.45 and 40 at 165.26 on 6000.00 cash report 22646.40 with one holding named as unpriced; giving that holding a symbol through the UI moves the figure to 23530.50, and a second refresh leaves the revision untouched.
742 lines
26 KiB
Go
742 lines
26 KiB
Go
package domain
|
|
|
|
import (
|
|
"crypto/rand"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"math"
|
|
"math/big"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
var idPattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,127}$`)
|
|
var currencyPattern = regexp.MustCompile(`^[A-Z]{3}$`)
|
|
var isinPattern = regexp.MustCompile(`^[A-Z]{2}[A-Z0-9]{9}[0-9]$`)
|
|
|
|
const moneyScale = 4
|
|
const quantityScale = 8
|
|
|
|
// ParseMoney accepts exact decimal values representable as signed 64-bit ten-thousandths.
|
|
// This intentionally bounds the otherwise larger DECIMAL(24,4) database domain.
|
|
func ParseMoney(s string) (Money, error) {
|
|
n, err := parseScaled(s, moneyScale, "money", "four")
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return Money(formatScaled(n, moneyScale, 2)), nil
|
|
}
|
|
|
|
// ParseQuantity accepts exact share counts representable as signed 64-bit
|
|
// hundred-millionths. Money's four places cannot hold a reinvested fraction of
|
|
// a share, and a truncated share count silently misstates a holding.
|
|
func ParseQuantity(s string) (Quantity, error) {
|
|
n, err := parseScaled(s, quantityScale, "quantity", "eight")
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return Quantity(formatScaled(n, quantityScale, 0)), nil
|
|
}
|
|
func parseScaled(s string, scale int, noun, places string) (int64, error) {
|
|
invalid := func() (int64, error) {
|
|
return 0, fmt.Errorf("invalid or out-of-range %s %q: require signed 64-bit value with at most %s fractional digits", noun, s, places)
|
|
}
|
|
if s == "" {
|
|
return invalid()
|
|
}
|
|
start := 0
|
|
negative := s[0] == '-'
|
|
if negative {
|
|
start = 1
|
|
}
|
|
if start == len(s) {
|
|
return invalid()
|
|
}
|
|
if s[start] < '0' || s[start] > '9' {
|
|
return invalid()
|
|
}
|
|
if s[start] == '0' && start+1 < len(s) && s[start+1] != '.' {
|
|
return invalid()
|
|
}
|
|
limit := uint64(math.MaxInt64)
|
|
if negative {
|
|
limit++
|
|
}
|
|
magnitude := uint64(0)
|
|
fraction := -1
|
|
for i := start; i < len(s); i++ {
|
|
c := s[i]
|
|
if c == '.' {
|
|
if fraction >= 0 || i == len(s)-1 {
|
|
return invalid()
|
|
}
|
|
fraction = 0
|
|
continue
|
|
}
|
|
if c < '0' || c > '9' {
|
|
return invalid()
|
|
}
|
|
if fraction >= 0 {
|
|
fraction++
|
|
if fraction > scale {
|
|
return invalid()
|
|
}
|
|
}
|
|
digit := uint64(c - '0')
|
|
if magnitude > (limit-digit)/10 {
|
|
return invalid()
|
|
}
|
|
magnitude = magnitude*10 + digit
|
|
}
|
|
if fraction < 0 {
|
|
fraction = 0
|
|
}
|
|
for range scale - fraction {
|
|
if magnitude > limit/10 {
|
|
return invalid()
|
|
}
|
|
magnitude *= 10
|
|
}
|
|
if negative {
|
|
if magnitude == uint64(math.MaxInt64)+1 {
|
|
return math.MinInt64, nil
|
|
}
|
|
return -int64(magnitude), nil
|
|
}
|
|
return int64(magnitude), nil
|
|
}
|
|
|
|
// formatScaled renders exact units. minFraction keeps money at two places for
|
|
// display while letting a whole share count render without eight zeros.
|
|
func formatScaled(n int64, scale, minFraction int) string {
|
|
s := strconv.FormatInt(n, 10)
|
|
sign := ""
|
|
if strings.HasPrefix(s, "-") {
|
|
sign, s = "-", s[1:]
|
|
}
|
|
if len(s) < scale+1 {
|
|
s = strings.Repeat("0", scale+1-len(s)) + s
|
|
}
|
|
whole, fraction := s[:len(s)-scale], strings.TrimRight(s[len(s)-scale:], "0")
|
|
if len(fraction) < minFraction {
|
|
fraction += strings.Repeat("0", minFraction-len(fraction))
|
|
}
|
|
if fraction == "" {
|
|
return sign + whole
|
|
}
|
|
return sign + whole + "." + fraction
|
|
}
|
|
func (m Money) Minor() (int64, error) { return parseScaled(string(m), moneyScale, "money", "four") }
|
|
func (m Money) String() string {
|
|
parsed, err := ParseMoney(string(m))
|
|
if err != nil {
|
|
return string(m)
|
|
}
|
|
return string(parsed)
|
|
}
|
|
func (q Quantity) Units() (int64, error) {
|
|
return parseScaled(string(q), quantityScale, "quantity", "eight")
|
|
}
|
|
func (q Quantity) String() string {
|
|
parsed, err := ParseQuantity(string(q))
|
|
if err != nil {
|
|
return string(q)
|
|
}
|
|
return string(parsed)
|
|
}
|
|
|
|
// FormatMoney renders exact ten-thousandths as money, and FormatQuantity
|
|
// renders exact hundred-millionths as a share count. Exact units are the only
|
|
// safe currency for arithmetic, and these are how a computed total re-enters
|
|
// the journal without a float ever being involved.
|
|
func FormatMoney(minor int64) Money { return Money(formatScaled(minor, moneyScale, 2)) }
|
|
func FormatQuantity(units int64) Quantity {
|
|
return Quantity(formatScaled(units, quantityScale, 0))
|
|
}
|
|
func NewID(prefix string) string {
|
|
if !idPattern.MatchString(prefix) || len(prefix) > 94 {
|
|
panic("invalid ID prefix")
|
|
}
|
|
var b [16]byte
|
|
if _, err := rand.Read(b[:]); err != nil {
|
|
panic(fmt.Errorf("secure identifier generation: %w", err))
|
|
}
|
|
return prefix + "_" + hex.EncodeToString(b[:])
|
|
}
|
|
func NewDataset() Dataset {
|
|
return Dataset{Accounts: []Account{}, Categories: []Category{
|
|
{ID: "cat_expenses", Name: "Expenses", Kind: "expense"}, {ID: ExpenseFallback, Name: "Unclassified", ParentID: "cat_expenses", Kind: "expense"},
|
|
{ID: "cat_income", Name: "Income", Kind: "income"}, {ID: IncomeFallback, Name: "Unclassified", ParentID: "cat_income", Kind: "income"},
|
|
}, Tags: []Tag{}, Merchants: []Merchant{}, Instruments: []Instrument{}, Transactions: []Transaction{}}
|
|
}
|
|
|
|
// InstrumentID derives a stable registry ID from an ISIN so re-importing the
|
|
// same export never creates a second instrument for one security.
|
|
func InstrumentID(isin string) string {
|
|
sum := sha256.Sum256([]byte("instrument\x00" + strings.ToUpper(strings.TrimSpace(isin))))
|
|
return "ins_" + hex.EncodeToString(sum[:16])
|
|
}
|
|
func Clone(d Dataset) Dataset {
|
|
c := Dataset{Accounts: append([]Account{}, d.Accounts...), Categories: append([]Category{}, d.Categories...), Tags: append([]Tag{}, d.Tags...), Merchants: append([]Merchant{}, d.Merchants...), Instruments: append([]Instrument{}, d.Instruments...), Transactions: append([]Transaction{}, d.Transactions...)}
|
|
for i := range c.Merchants {
|
|
c.Merchants[i].Aliases = append([]string{}, d.Merchants[i].Aliases...)
|
|
c.Merchants[i].DefaultTagIDs = append([]string{}, d.Merchants[i].DefaultTagIDs...)
|
|
}
|
|
for i := range c.Transactions {
|
|
c.Transactions[i].Enrichment.TagIDs = append([]string{}, d.Transactions[i].Enrichment.TagIDs...)
|
|
// Facts are immutable, but a shared pointer would let one dataset's
|
|
// edit reach another's copy.
|
|
if inv := d.Transactions[i].Facts.Investment; inv != nil {
|
|
copied := *inv
|
|
c.Transactions[i].Facts.Investment = &copied
|
|
}
|
|
}
|
|
return c
|
|
}
|
|
|
|
// Fallback classifies a fact that no rule or model claimed. Broker facts never
|
|
// take the sign-based branch: an unmatched deposit is not income and a broker
|
|
// fee paid out of an investment account is not household spending.
|
|
func Fallback(f Facts) Enrichment {
|
|
if f.Investment != nil {
|
|
return Enrichment{Kind: KindInvestment, TagIDs: []string{}, Classification: Provenance{Source: "fallback"}}
|
|
}
|
|
kind, category := "expense", ExpenseFallback
|
|
n, err := f.Amount.Minor()
|
|
if err == nil && n > 0 {
|
|
kind, category = "income", IncomeFallback
|
|
}
|
|
return Enrichment{Kind: kind, CategoryID: category, TagIDs: []string{}, Classification: Provenance{Source: "fallback"}}
|
|
}
|
|
func CategoryPath(d Dataset, id string) string {
|
|
byID := map[string]Category{}
|
|
for _, c := range d.Categories {
|
|
byID[c.ID] = c
|
|
}
|
|
parts := []string{}
|
|
seen := map[string]bool{}
|
|
for id != "" {
|
|
c, ok := byID[id]
|
|
if !ok || seen[id] {
|
|
return ""
|
|
}
|
|
seen[id] = true
|
|
parts = append(parts, c.Name)
|
|
id = c.ParentID
|
|
}
|
|
for i, j := 0, len(parts)-1; i < j; i, j = i+1, j-1 {
|
|
parts[i], parts[j] = parts[j], parts[i]
|
|
}
|
|
return strings.Join(parts, " / ")
|
|
}
|
|
func validDate(s string) bool {
|
|
t, err := time.Parse("2006-01-02", s)
|
|
return err == nil && t.Year() > 0 && t.Format("2006-01-02") == s
|
|
}
|
|
func nonblank(s string) bool { return utf8.ValidString(s) && strings.TrimSpace(s) != "" }
|
|
func validText(values ...string) bool {
|
|
for _, s := range values {
|
|
if !utf8.ValidString(s) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
func validHint(s string) bool {
|
|
return utf8.ValidString(s) && utf8.RuneCountInString(s) <= 200
|
|
}
|
|
|
|
// ValidISIN reports a syntactically valid ISIN: two country letters, nine
|
|
// alphanumerics and a check digit.
|
|
func ValidISIN(s string) bool { return isinPattern.MatchString(s) }
|
|
|
|
// ValidateInvestment checks one broker fact against the investment model. An
|
|
// importer calls it per record so a malformed export is refused with the
|
|
// record that caused it, rather than at commit with only an ID.
|
|
func ValidateInvestment(f Facts, account Account, instruments map[string]Instrument) error {
|
|
return validateInvestment(f, account, instruments)
|
|
}
|
|
func Validate(d Dataset) error {
|
|
ids := map[string]string{}
|
|
register := func(id, kind string) error {
|
|
if !idPattern.MatchString(id) {
|
|
return fmt.Errorf("%s %q: invalid ID", kind, id)
|
|
}
|
|
if old, ok := ids[id]; ok {
|
|
return fmt.Errorf("%s %q: duplicate ID (already %s)", kind, id, old)
|
|
}
|
|
ids[id] = kind
|
|
return nil
|
|
}
|
|
accounts := map[string]Account{}
|
|
categories := map[string]Category{}
|
|
children := map[string]bool{}
|
|
tags := map[string]bool{}
|
|
for _, a := range d.Accounts {
|
|
if err := register(a.ID, "account"); err != nil {
|
|
return err
|
|
}
|
|
if !nonblank(a.DisplayName) || !currencyPattern.MatchString(a.Currency) || !validText(a.Institution, a.ExternalAccountID, a.IBAN, a.ReferenceIBAN) {
|
|
return fmt.Errorf("account %q: valid UTF-8 name and three-letter uppercase currency required", a.ID)
|
|
}
|
|
if a.Kind != "" && a.Kind != AccountCash && a.Kind != AccountInvestment {
|
|
return fmt.Errorf("account %q: kind must be %q or %q", a.ID, AccountCash, AccountInvestment)
|
|
}
|
|
accounts[a.ID] = a
|
|
}
|
|
for _, c := range d.Categories {
|
|
if err := register(c.ID, "category"); err != nil {
|
|
return err
|
|
}
|
|
if !nonblank(c.Name) || !validHint(c.Hint) || (c.Kind != "expense" && c.Kind != "income") {
|
|
return fmt.Errorf("category %q: invalid name, hint or kind", c.ID)
|
|
}
|
|
categories[c.ID] = c
|
|
if c.ParentID != "" {
|
|
children[c.ParentID] = true
|
|
}
|
|
}
|
|
for _, c := range d.Categories {
|
|
seen := map[string]bool{c.ID: true}
|
|
for p := c.ParentID; p != ""; {
|
|
parent, ok := categories[p]
|
|
if !ok {
|
|
return fmt.Errorf("category %q: missing parent %q", c.ID, p)
|
|
}
|
|
if seen[p] {
|
|
return fmt.Errorf("category %q: taxonomy cycle", c.ID)
|
|
}
|
|
if parent.Kind != c.Kind {
|
|
return fmt.Errorf("category %q: parent kind differs", c.ID)
|
|
}
|
|
seen[p] = true
|
|
p = parent.ParentID
|
|
}
|
|
}
|
|
for _, spec := range []struct{ id, parent, kind string }{{"cat_expenses", "", "expense"}, {ExpenseFallback, "cat_expenses", "expense"}, {"cat_income", "", "income"}, {IncomeFallback, "cat_income", "income"}} {
|
|
c, ok := categories[spec.id]
|
|
if !ok || c.ParentID != spec.parent || c.Kind != spec.kind {
|
|
return fmt.Errorf("category %q: required fallback hierarchy cannot be removed or moved", spec.id)
|
|
}
|
|
}
|
|
if children[ExpenseFallback] || children[IncomeFallback] {
|
|
return fmt.Errorf("fallback categories must remain leaves")
|
|
}
|
|
for _, t := range d.Tags {
|
|
if err := register(t.ID, "tag"); err != nil {
|
|
return err
|
|
}
|
|
if !nonblank(t.Name) || !validHint(t.Hint) {
|
|
return fmt.Errorf("tag %q: name or hint invalid", t.ID)
|
|
}
|
|
tags[t.ID] = true
|
|
}
|
|
for _, m := range d.Merchants {
|
|
if err := register(m.ID, "merchant"); err != nil {
|
|
return err
|
|
}
|
|
if !nonblank(m.Name) {
|
|
return fmt.Errorf("merchant %q: name required", m.ID)
|
|
}
|
|
if m.DefaultCategoryID != "" {
|
|
if _, ok := categories[m.DefaultCategoryID]; !ok || children[m.DefaultCategoryID] {
|
|
return fmt.Errorf("merchant %q: default category must be existing leaf", m.ID)
|
|
}
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, id := range m.DefaultTagIDs {
|
|
if !tags[id] || seen[id] {
|
|
return fmt.Errorf("merchant %q: invalid or duplicate default tag %q", m.ID, id)
|
|
}
|
|
seen[id] = true
|
|
}
|
|
aliases := map[string]bool{}
|
|
for _, alias := range m.Aliases {
|
|
key := strings.ToLower(strings.TrimSpace(alias))
|
|
if !nonblank(alias) || aliases[key] {
|
|
return fmt.Errorf("merchant %q: invalid or duplicate alias", m.ID)
|
|
}
|
|
aliases[key] = true
|
|
}
|
|
}
|
|
instruments := map[string]Instrument{}
|
|
isins := map[string]string{}
|
|
for _, v := range d.Instruments {
|
|
if err := register(v.ID, "instrument"); err != nil {
|
|
return err
|
|
}
|
|
if !isinPattern.MatchString(v.ISIN) {
|
|
return fmt.Errorf("instrument %q: ISIN must be two letters, nine alphanumerics and a check digit", v.ID)
|
|
}
|
|
if other, ok := isins[v.ISIN]; ok {
|
|
return fmt.Errorf("instrument %q: ISIN %s already held by %q", v.ID, v.ISIN, other)
|
|
}
|
|
if !nonblank(v.Name) || !currencyPattern.MatchString(v.Currency) || !validText(v.Symbol) {
|
|
return fmt.Errorf("instrument %q: valid UTF-8 name and symbol and three-letter uppercase currency required", v.ID)
|
|
}
|
|
// A quote without its day cannot be judged stale, and a day without a
|
|
// quote values nothing, so neither stands alone.
|
|
if (v.Quote == "") != (v.QuotedAt == "") {
|
|
return fmt.Errorf("instrument %q: a quote and the day it is from are recorded together", v.ID)
|
|
}
|
|
if v.Quote != "" {
|
|
units, err := v.Quote.Units()
|
|
if err != nil {
|
|
return fmt.Errorf("instrument %q: %w", v.ID, err)
|
|
}
|
|
if units < 0 {
|
|
return fmt.Errorf("instrument %q: a quote cannot be negative", v.ID)
|
|
}
|
|
if !validDate(v.QuotedAt) {
|
|
return fmt.Errorf("instrument %q: invalid quote date %q", v.ID, v.QuotedAt)
|
|
}
|
|
}
|
|
isins[v.ISIN] = v.ID
|
|
instruments[v.ID] = v
|
|
}
|
|
for _, t := range d.Transactions {
|
|
f := t.Facts
|
|
if err := register(f.ID, "transaction"); err != nil {
|
|
return err
|
|
}
|
|
a, ok := accounts[f.AccountID]
|
|
if !ok {
|
|
return fmt.Errorf("transaction %q: unknown account %q", f.ID, f.AccountID)
|
|
}
|
|
if !currencyPattern.MatchString(f.Currency) || a.Currency != f.Currency {
|
|
return fmt.Errorf("transaction %q: currency differs from account", f.ID)
|
|
}
|
|
if _, err := f.Amount.Minor(); err != nil {
|
|
return fmt.Errorf("transaction %q: %w", f.ID, err)
|
|
}
|
|
if !validDate(f.BookingDate) || (f.ValueDate != "" && !validDate(f.ValueDate)) {
|
|
return fmt.Errorf("transaction %q: invalid booking/value date", f.ID)
|
|
}
|
|
if !nonblank(f.Source) || !nonblank(f.Fingerprint) {
|
|
return fmt.Errorf("transaction %q: source and fingerprint required", f.ID)
|
|
}
|
|
if !validText(f.RawDescription, f.ExternalID, f.Counterparty, f.CounterpartyIBAN) {
|
|
return fmt.Errorf("transaction %q: bank facts must be valid UTF-8", f.ID)
|
|
}
|
|
if err := validateInvestment(f, a, instruments); err != nil {
|
|
return fmt.Errorf("transaction %q: %w", f.ID, err)
|
|
}
|
|
}
|
|
index := enrichmentIndex{categories: categories, children: children, tags: tags, merchants: map[string]bool{}, transactions: map[string]Transaction{}}
|
|
for _, m := range d.Merchants {
|
|
index.merchants[m.ID] = true
|
|
}
|
|
for _, t := range d.Transactions {
|
|
index.transactions[t.Facts.ID] = t
|
|
}
|
|
for _, t := range d.Transactions {
|
|
if err := index.validate(t.Facts, t.Enrichment); err != nil {
|
|
return fmt.Errorf("transaction %q: %w", t.Facts.ID, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type enrichmentIndex struct {
|
|
categories map[string]Category
|
|
children, tags, merchants map[string]bool
|
|
transactions map[string]Transaction
|
|
}
|
|
|
|
func ValidateEnrichment(d Dataset, f Facts, e Enrichment) error {
|
|
index := enrichmentIndex{categories: map[string]Category{}, children: map[string]bool{}, tags: map[string]bool{}, merchants: map[string]bool{}, transactions: map[string]Transaction{}}
|
|
for _, c := range d.Categories {
|
|
index.categories[c.ID] = c
|
|
if c.ParentID != "" {
|
|
index.children[c.ParentID] = true
|
|
}
|
|
}
|
|
for _, t := range d.Tags {
|
|
index.tags[t.ID] = true
|
|
}
|
|
for _, m := range d.Merchants {
|
|
index.merchants[m.ID] = true
|
|
}
|
|
for _, t := range d.Transactions {
|
|
index.transactions[t.Facts.ID] = t
|
|
}
|
|
return index.validate(f, e)
|
|
}
|
|
func (index enrichmentIndex) validate(f Facts, e Enrichment) error {
|
|
if e.Kind != "expense" && e.Kind != "income" && e.Kind != "transfer" && e.Kind != KindInvestment {
|
|
return fmt.Errorf("invalid enrichment kind %q", e.Kind)
|
|
}
|
|
if (e.Kind == KindInvestment) != (f.Investment != nil && e.Kind != "transfer") {
|
|
return fmt.Errorf("only broker facts carry kind %q, and every unlinked broker fact must", KindInvestment)
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, id := range e.TagIDs {
|
|
if !index.tags[id] || seen[id] {
|
|
return fmt.Errorf("invalid or duplicate tag %q", id)
|
|
}
|
|
seen[id] = true
|
|
}
|
|
if e.MerchantID != "" && !index.merchants[e.MerchantID] {
|
|
return fmt.Errorf("unknown merchant %q", e.MerchantID)
|
|
}
|
|
if !validText(e.Classification.Source, e.Classification.Model, e.Classification.Confidence, e.Classification.Error) {
|
|
return fmt.Errorf("classification metadata must be valid UTF-8")
|
|
}
|
|
if e.Classification.Confidence != "" && e.Classification.Confidence != "high" && e.Classification.Confidence != "medium" && e.Classification.Confidence != "low" {
|
|
return fmt.Errorf("invalid classification confidence")
|
|
}
|
|
if e.Classification.Timestamp != "" {
|
|
if _, err := time.Parse(time.RFC3339Nano, e.Classification.Timestamp); err != nil {
|
|
return fmt.Errorf("invalid classification timestamp")
|
|
}
|
|
}
|
|
if e.Kind == KindInvestment {
|
|
if e.CategoryID != "" || e.MerchantID != "" || e.TransferPeerID != "" {
|
|
return fmt.Errorf("investment must not have category, merchant or transfer peer")
|
|
}
|
|
if e.Classification.Source == "ai" || e.Classification.Source == "openrouter" {
|
|
return fmt.Errorf("AI cannot classify investments")
|
|
}
|
|
return nil
|
|
}
|
|
if e.Kind == "transfer" {
|
|
if e.CategoryID != "" || e.MerchantID != "" {
|
|
return fmt.Errorf("transfer must not have category or merchant")
|
|
}
|
|
if e.Classification.Source == "ai" || e.Classification.Source == "openrouter" {
|
|
return fmt.Errorf("AI cannot classify transfers")
|
|
}
|
|
if f.Investment != nil && !f.Investment.CashOnly() {
|
|
return fmt.Errorf("only a broker cash movement can be linked as a transfer, not %q", f.Investment.Event)
|
|
}
|
|
amount, err := f.Amount.Minor()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if amount == 0 || amount == math.MinInt64 {
|
|
return fmt.Errorf("transfer requires nonzero negatable amount")
|
|
}
|
|
if peer, ok := index.transactions[e.TransferPeerID]; ok && peer.Facts.ID != f.ID {
|
|
other, err := peer.Facts.Amount.Minor()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if peer.Facts.AccountID == f.AccountID || peer.Facts.Currency != f.Currency || other != -amount || peer.Enrichment.Kind != "transfer" || peer.Enrichment.TransferPeerID != f.ID {
|
|
return fmt.Errorf("transfer peer must be reciprocal, opposite, same-currency and different-account")
|
|
}
|
|
return nil
|
|
}
|
|
return fmt.Errorf("missing transfer peer %q", e.TransferPeerID)
|
|
}
|
|
if e.TransferPeerID != "" {
|
|
return fmt.Errorf("non-transfer cannot have transfer peer")
|
|
}
|
|
category, found := index.categories[e.CategoryID]
|
|
if !found {
|
|
return fmt.Errorf("unknown category %q", e.CategoryID)
|
|
}
|
|
if category.Kind != e.Kind {
|
|
return fmt.Errorf("category kind differs from enrichment")
|
|
}
|
|
if index.children[e.CategoryID] {
|
|
return fmt.Errorf("category %q is not a leaf", e.CategoryID)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func optionalMoney(m Money) (int64, error) {
|
|
if m == "" {
|
|
return 0, nil
|
|
}
|
|
return m.Minor()
|
|
}
|
|
func optionalQuantity(q Quantity) (int64, error) {
|
|
if q == "" {
|
|
return 0, nil
|
|
}
|
|
return q.Units()
|
|
}
|
|
|
|
// RoundedProduct multiplies an exact share count by an exact unit price and
|
|
// rounds to money's four places, half away from zero. Both operands are 1e-8
|
|
// units, so the product is 1e-16 and needs 128-bit width.
|
|
func RoundedProduct(quantity, price int64) (int64, bool) {
|
|
product := new(big.Int).Mul(big.NewInt(quantity), big.NewInt(price))
|
|
half := big.NewInt(500_000_000_000)
|
|
if product.Sign() < 0 {
|
|
product.Sub(product, half)
|
|
} else {
|
|
product.Add(product, half)
|
|
}
|
|
rounded := product.Quo(product, big.NewInt(1_000_000_000_000))
|
|
if !rounded.IsInt64() {
|
|
return 0, false
|
|
}
|
|
return rounded.Int64(), true
|
|
}
|
|
|
|
// validateInvestment enforces the broker row model.
|
|
//
|
|
// A cash row's amount is the money that actually moved and is already net of
|
|
// the tax the broker withheld or refunded, so its tax column is recorded but
|
|
// never applied. A buy, sell or reinvestment quotes a gross of shares times
|
|
// price and settles gross minus fee minus tax. A corporate action or depot
|
|
// transfer moves a position at a valuation and must never touch cash: treating
|
|
// its amount as money conjures or destroys it.
|
|
//
|
|
// Every security row is checked against shares times price. That is the only
|
|
// check that catches a lost decimal separator, and it is worthless without it:
|
|
// a one-share row satisfies every other invariant at any scale.
|
|
func validateInvestment(f Facts, a Account, instruments map[string]Instrument) error {
|
|
inv := f.Investment
|
|
if inv == nil {
|
|
return nil
|
|
}
|
|
if !a.Investing() {
|
|
return fmt.Errorf("investment leg requires an account of kind %q", AccountInvestment)
|
|
}
|
|
if !inv.CashOnly() && !inv.Settling() && !inv.PositionOnly() {
|
|
return fmt.Errorf("unknown investment event %q", inv.Event)
|
|
}
|
|
if inv.InstrumentID != "" {
|
|
v, ok := instruments[inv.InstrumentID]
|
|
if !ok {
|
|
return fmt.Errorf("unknown instrument %q", inv.InstrumentID)
|
|
}
|
|
if v.Currency != f.Currency {
|
|
return fmt.Errorf("instrument %s trades in %s but this fact settles in %s", v.ISIN, v.Currency, f.Currency)
|
|
}
|
|
}
|
|
quantity, err := optionalQuantity(inv.Quantity)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
price, err := optionalQuantity(inv.Price)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
gross, err := optionalMoney(inv.Gross)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fee, err := optionalMoney(inv.Fee)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tax, err := optionalMoney(inv.Tax)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
amount, err := f.Amount.Minor()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if inv.CashOnly() {
|
|
if quantity != 0 || inv.Price != "" {
|
|
return fmt.Errorf("%s moves cash only: it carries no quantity or price", inv.Event)
|
|
}
|
|
// The gross is optional here. One broker states a cash row already net
|
|
// of the tax it withheld, and then only the net is knowable, so the
|
|
// tax is recorded and never applied. Another states the gross and the
|
|
// deductions separately, and then the settlement is checkable like any
|
|
// trade's. Which one is a fact about the source, decided at import.
|
|
if inv.Gross == "" {
|
|
return nil
|
|
}
|
|
return settles(inv, gross, fee, tax, amount)
|
|
}
|
|
if inv.InstrumentID == "" {
|
|
return fmt.Errorf("%s requires an instrument", inv.Event)
|
|
}
|
|
if quantity == 0 {
|
|
return fmt.Errorf("%s requires a nonzero quantity", inv.Event)
|
|
}
|
|
// A position-only valuation carries the sign of the position change; a
|
|
// settled trade carries the sign of the cash, which is the opposite. The
|
|
// product is kept exact at 1e-16 so the comparison never rounds first.
|
|
product := new(big.Int).Mul(big.NewInt(quantity), big.NewInt(price))
|
|
if inv.Settling() {
|
|
product.Neg(product)
|
|
}
|
|
difference := new(big.Int).Sub(product, new(big.Int).Mul(big.NewInt(gross), productPerMoney))
|
|
if difference.Abs(difference).Cmp(grossSlack(gross, inv.Gross)) > 0 {
|
|
expected, ok := RoundedProduct(quantity, price)
|
|
if !ok {
|
|
return fmt.Errorf("%s quantity times price is out of range", inv.Event)
|
|
}
|
|
if inv.Settling() {
|
|
expected = -expected
|
|
}
|
|
return fmt.Errorf("%s gross %s does not equal quantity %s times price %s, which is %s", inv.Event, inv.Gross.String(), inv.Quantity.String(), inv.Price.String(), Money(formatScaled(expected, moneyScale, 2)))
|
|
}
|
|
if inv.PositionOnly() {
|
|
if amount != 0 {
|
|
return fmt.Errorf("%s moves position only, but this fact carries cash %s", inv.Event, f.Amount.String())
|
|
}
|
|
if fee != 0 || tax != 0 {
|
|
return fmt.Errorf("%s cannot carry a fee or tax", inv.Event)
|
|
}
|
|
return nil
|
|
}
|
|
if (inv.Event == EventSell) != (quantity < 0) {
|
|
return fmt.Errorf("%s must %s the position", inv.Event, map[bool]string{true: "reduce", false: "increase"}[inv.Event == EventSell])
|
|
}
|
|
return settles(inv, gross, fee, tax, amount)
|
|
}
|
|
|
|
// settles enforces that the cash a fact moved is its gross less the fee and
|
|
// the tax deducted from it. Fee and tax are stored as deductions whichever sign
|
|
// the source printed, so a refunded tax is a negative deduction and a broker
|
|
// that writes its fee as a negative adjustment is normalized at import.
|
|
func settles(inv *Investment, gross, fee, tax, amount int64) error {
|
|
settled := new(big.Int).Sub(big.NewInt(gross), big.NewInt(fee))
|
|
settled.Sub(settled, big.NewInt(tax))
|
|
if !settled.IsInt64() || settled.Int64() != amount {
|
|
return fmt.Errorf("%s cash %s does not equal gross %s minus fee %s minus tax %s", inv.Event, Money(formatScaled(amount, moneyScale, 2)), inv.Gross.String(), inv.Fee.String(), inv.Tax.String())
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// productPerMoney converts money's ten-thousandths to the 1e-16 units a
|
|
// quantity times a price lands in.
|
|
var productPerMoney = new(big.Int).Exp(big.NewInt(10), big.NewInt(productScale-moneyScale), nil)
|
|
|
|
const productScale = quantityScale * 2
|
|
|
|
// grossSlack is how far a printed gross may sit from the product of the printed
|
|
// quantity and price before the row is refused. Both ends are rounded, and
|
|
// neither states by how much.
|
|
//
|
|
// The gross is rounded to its own last decimal place: one broker prints the
|
|
// notional to the cent, so 0.426581 shares at 63.06 settle as 26.90 where the
|
|
// product is 26.90019786, and demanding exactness there rejects half a
|
|
// portfolio. The price is rounded to a precision the file does not state: the
|
|
// same export settles six NVIDIA shares at 808.5599 while printing the price
|
|
// as 134.76, whose product is 808.56, because the real fill was 134.759983.
|
|
// So the slack is half a unit of the gross's stated precision, plus one part
|
|
// in a hundred thousand of the gross itself.
|
|
//
|
|
// Measured over a complete real export of 88 security rows, exactly one
|
|
// deviates at all, by one part in eight million - eighty times inside this
|
|
// bound. What it refuses: any deviation above one part in a hundred thousand,
|
|
// which covers a price taken from the wrong share class and the lost decimal
|
|
// separator this check exists for, four orders of magnitude out. What it
|
|
// accepts: the broker's own rounding. On a gross stated to the cent the slack
|
|
// reaches a whole cent at around five hundred euro, above which a genuine
|
|
// one-cent error is indistinguishable from that rounding and is allowed.
|
|
func grossSlack(gross int64, printed Money) *big.Int {
|
|
_, fraction, _ := strings.Cut(string(printed), ".")
|
|
places := len(fraction)
|
|
if places > moneyScale {
|
|
places = moneyScale
|
|
}
|
|
half := new(big.Int).Exp(big.NewInt(10), big.NewInt(int64(productScale-places)), nil)
|
|
half.Quo(half, big.NewInt(2))
|
|
relative := new(big.Int).Abs(new(big.Int).Mul(big.NewInt(gross), productPerMoney))
|
|
return half.Add(half, relative.Quo(relative, big.NewInt(100_000)))
|
|
}
|