init
This commit is contained in:
Lars Nolden
@@ -0,0 +1,335 @@
|
||||
package banking
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"finance-duck/internal/domain"
|
||||
)
|
||||
|
||||
func digest(parts ...string) string {
|
||||
b, _ := json.Marshal(parts)
|
||||
h := sha256.Sum256(b)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
func identity(f domain.Facts) string { return digest(f.AccountID, f.Source, f.ExternalID) }
|
||||
func fingerprint(f domain.Facts) string {
|
||||
return digest(f.AccountID, f.BookingDate, f.ValueDate, f.Amount.String(), f.Currency, strings.Join(strings.Fields(f.RawDescription), " "), strings.ToLower(strings.Join(strings.Fields(f.Counterparty), " ")), f.CounterpartyIBAN)
|
||||
}
|
||||
func looseFingerprint(f domain.Facts) string {
|
||||
return digest(f.AccountID, f.BookingDate, f.Amount.String(), f.Currency)
|
||||
}
|
||||
func sameBookedMoney(a, b domain.Facts) bool {
|
||||
return a.AccountID == b.AccountID && a.BookingDate == b.BookingDate && a.Amount == b.Amount && a.Currency == b.Currency
|
||||
}
|
||||
|
||||
// NormalizeAndDedupe returns new records without mutating the input. Stable bank
|
||||
// entry references take precedence over text. CSV rows without references use
|
||||
// occurrence counts, not a set: two identical rows remain two transactions and
|
||||
// importing the same export again creates none. For overlapping partial exports,
|
||||
// indistinguishable rows cannot prove an additional occurrence; import complete
|
||||
// overlapping date windows to establish multiplicity.
|
||||
//
|
||||
// Cross-source reconciliation only suppresses equal full-fingerprint groups with
|
||||
// equal multiplicity. Same-day/same-money cross-source discrepancies fail closed
|
||||
// for user review rather than guessing or silently inflating balances. No alias
|
||||
// or bank fact is rewritten, so later upstream metadata drift remains visible.
|
||||
func NormalizeAndDedupe(data domain.Dataset, incoming []domain.Facts) ([]domain.Transaction, error) {
|
||||
accounts := make(map[string]bool, len(data.Accounts))
|
||||
for _, a := range data.Accounts {
|
||||
accounts[a.ID] = true
|
||||
}
|
||||
type group struct {
|
||||
source, fp string
|
||||
facts []domain.Facts
|
||||
}
|
||||
groups := map[string]*group{}
|
||||
existing := map[string]map[string]int{}
|
||||
existingAnonymous := map[string]int{}
|
||||
existingIDs := map[string]domain.Facts{}
|
||||
loose := map[string]map[string]map[string]bool{}
|
||||
addLoose := func(f domain.Facts, fp string) {
|
||||
k := looseFingerprint(f)
|
||||
if loose[k] == nil {
|
||||
loose[k] = map[string]map[string]bool{}
|
||||
}
|
||||
if loose[k][f.Source] == nil {
|
||||
loose[k][f.Source] = map[string]bool{}
|
||||
}
|
||||
loose[k][f.Source][fp] = true
|
||||
}
|
||||
for _, t := range data.Transactions {
|
||||
f, err := normalizeFacts(t.Facts, accounts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("existing transaction %s: %w", t.Facts.ID, err)
|
||||
}
|
||||
fp := fingerprint(f)
|
||||
if existing[fp] == nil {
|
||||
existing[fp] = map[string]int{}
|
||||
}
|
||||
existing[fp][f.Source]++
|
||||
if f.ExternalID == "" {
|
||||
existingAnonymous[digest(f.Source, fp)]++
|
||||
}
|
||||
if f.ExternalID != "" {
|
||||
existingIDs[identity(f)] = f
|
||||
}
|
||||
addLoose(f, fp)
|
||||
}
|
||||
seenIDs := map[string]domain.Facts{}
|
||||
for index, original := range incoming {
|
||||
f, err := normalizeFacts(original, accounts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("incoming record %d: %w", index+1, err)
|
||||
}
|
||||
if f.ExternalID != "" {
|
||||
key := identity(f)
|
||||
if old, ok := seenIDs[key]; ok {
|
||||
if !sameBookedMoney(old, f) {
|
||||
return nil, fmt.Errorf("conflicting upstream transaction identity in incoming records")
|
||||
}
|
||||
continue
|
||||
}
|
||||
seenIDs[key] = f
|
||||
if old, ok := existingIDs[key]; ok {
|
||||
if !sameBookedMoney(old, f) {
|
||||
return nil, fmt.Errorf("upstream transaction changed immutable booking facts")
|
||||
}
|
||||
// Use stored metadata to keep this matched occurrence in its original group.
|
||||
f = old
|
||||
}
|
||||
}
|
||||
fp := fingerprint(f)
|
||||
key := digest(f.Source, fp)
|
||||
if groups[key] == nil {
|
||||
groups[key] = &group{source: f.Source, fp: fp}
|
||||
}
|
||||
groups[key].facts = append(groups[key].facts, f)
|
||||
addLoose(f, fp)
|
||||
}
|
||||
// Reject ambiguous collisions even when one exact match also exists.
|
||||
for _, g := range groups {
|
||||
for _, f := range g.facts {
|
||||
for source, fps := range loose[looseFingerprint(f)] {
|
||||
if source != g.source {
|
||||
for fp := range fps {
|
||||
if fp != g.fp {
|
||||
return nil, fmt.Errorf("uncertain cross-source match on account %s at %s; reconcile differing bank/CSV records before importing", f.AccountID, f.BookingDate)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
keys := make([]string, 0, len(groups))
|
||||
for k := range groups {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
result := make([]domain.Transaction, 0)
|
||||
accepted := map[string]map[string]int{}
|
||||
for _, key := range keys {
|
||||
g := groups[key]
|
||||
crossCount := -1
|
||||
for source, n := range existing[g.fp] {
|
||||
if source != g.source {
|
||||
if crossCount >= 0 && crossCount != n {
|
||||
return nil, fmt.Errorf("uncertain cross-source occurrence counts")
|
||||
}
|
||||
crossCount = n
|
||||
}
|
||||
}
|
||||
for source, n := range accepted[g.fp] {
|
||||
if source != g.source {
|
||||
if crossCount >= 0 && crossCount != n {
|
||||
return nil, fmt.Errorf("uncertain cross-source occurrence counts")
|
||||
}
|
||||
crossCount = n
|
||||
}
|
||||
}
|
||||
if crossCount >= 0 {
|
||||
f := g.facts[0]
|
||||
if strings.TrimSpace(f.RawDescription) == "" && strings.TrimSpace(f.Counterparty) == "" && f.CounterpartyIBAN == "" {
|
||||
return nil, fmt.Errorf("uncertain cross-source match lacks descriptive bank evidence")
|
||||
}
|
||||
if crossCount != len(g.facts) {
|
||||
return nil, fmt.Errorf("uncertain cross-source occurrence counts on account %s at %s", g.facts[0].AccountID, g.facts[0].BookingDate)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Sorting IDs makes equal-fingerprint upstream records input-order independent.
|
||||
sort.SliceStable(g.facts, func(i, j int) bool { return g.facts[i].ExternalID < g.facts[j].ExternalID })
|
||||
// Referenced and anonymous records consume separate occurrence pools. When a
|
||||
// reference appears/disappears, a spare record in the other pool is ambiguous:
|
||||
// it may be an existing booking with changed identity metadata, not new money.
|
||||
baseline := existingAnonymous[key]
|
||||
anonymousCount, matchedReferences, newReferences := 0, 0, 0
|
||||
for _, f := range g.facts {
|
||||
if f.ExternalID == "" {
|
||||
anonymousCount++
|
||||
} else if _, ok := existingIDs[identity(f)]; ok {
|
||||
matchedReferences++
|
||||
} else {
|
||||
newReferences++
|
||||
}
|
||||
}
|
||||
unmatchedReferences := existing[g.fp][g.source] - baseline - matchedReferences
|
||||
if (newReferences > 0 && baseline > anonymousCount) || (anonymousCount > baseline && unmatchedReferences > 0) {
|
||||
return nil, fmt.Errorf("uncertain transaction identity changed between referenced and anonymous records on account %s at %s", g.facts[0].AccountID, g.facts[0].BookingDate)
|
||||
}
|
||||
occurrence := 0
|
||||
for _, f := range g.facts {
|
||||
if f.ExternalID != "" {
|
||||
if _, ok := existingIDs[identity(f)]; ok {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
occurrence++
|
||||
if occurrence <= baseline {
|
||||
continue
|
||||
}
|
||||
}
|
||||
f.Fingerprint = g.fp
|
||||
if f.ExternalID != "" {
|
||||
f.ID = "tx_" + identity(f)
|
||||
} else {
|
||||
f.ID = "tx_" + digest(f.Source, g.fp, strconv.Itoa(occurrence))
|
||||
}
|
||||
result = append(result, domain.Transaction{Facts: f, Enrichment: domain.Fallback(f)})
|
||||
}
|
||||
if accepted[g.fp] == nil {
|
||||
accepted[g.fp] = map[string]int{}
|
||||
}
|
||||
accepted[g.fp][g.source] = len(g.facts)
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
a, b := result[i].Facts, result[j].Facts
|
||||
if a.BookingDate != b.BookingDate {
|
||||
return a.BookingDate < b.BookingDate
|
||||
}
|
||||
return a.ID < b.ID
|
||||
})
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func normalizeFacts(f domain.Facts, accounts map[string]bool) (domain.Facts, error) {
|
||||
if !accounts[f.AccountID] {
|
||||
return f, fmt.Errorf("unknown account")
|
||||
}
|
||||
if f.Source == "" {
|
||||
return f, fmt.Errorf("missing import source")
|
||||
}
|
||||
date, err := parseDate(f.BookingDate)
|
||||
if err != nil {
|
||||
return f, fmt.Errorf("invalid booking date")
|
||||
}
|
||||
f.BookingDate = date
|
||||
if f.ValueDate != "" {
|
||||
f.ValueDate, err = parseDate(f.ValueDate)
|
||||
if err != nil {
|
||||
return f, fmt.Errorf("invalid value date")
|
||||
}
|
||||
}
|
||||
f.Amount, err = domain.ParseMoney(string(f.Amount))
|
||||
if err != nil {
|
||||
return f, fmt.Errorf("invalid amount")
|
||||
}
|
||||
f.Currency = strings.ToUpper(strings.TrimSpace(f.Currency))
|
||||
if !validCurrency(f.Currency) {
|
||||
return f, fmt.Errorf("invalid currency")
|
||||
}
|
||||
f.CounterpartyIBAN = normalizeIBAN(f.CounterpartyIBAN)
|
||||
f.ExternalID = strings.TrimSpace(f.ExternalID)
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// MatchTransfers links only mutually unique candidates, with reciprocal own
|
||||
// IBANs, inverse exact money in one currency, and booking dates within 3 calendar
|
||||
// days. Existing manual links are retained. Ambiguous equal payments stay ordinary
|
||||
// transactions: iteration order must never decide which transfer gets linked.
|
||||
func MatchTransfers(data *domain.Dataset) {
|
||||
if data == nil {
|
||||
return
|
||||
}
|
||||
own := map[string]string{}
|
||||
duplicates := map[string]bool{}
|
||||
for _, a := range data.Accounts {
|
||||
iban := normalizeIBAN(a.IBAN)
|
||||
if iban == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := own[iban]; ok {
|
||||
duplicates[iban] = true
|
||||
}
|
||||
own[iban] = a.ID
|
||||
}
|
||||
byAccount := map[string]string{}
|
||||
for iban, id := range own {
|
||||
if !duplicates[iban] {
|
||||
byAccount[id] = iban
|
||||
}
|
||||
}
|
||||
candidates := make([][]int, len(data.Transactions))
|
||||
for i := range data.Transactions {
|
||||
a := data.Transactions[i]
|
||||
if a.Enrichment.Kind == "transfer" || a.Enrichment.TransferPeerID != "" {
|
||||
continue
|
||||
}
|
||||
ai := byAccount[a.Facts.AccountID]
|
||||
target := normalizeIBAN(a.Facts.CounterpartyIBAN)
|
||||
if ai == "" || target == "" || duplicates[target] || own[target] == "" || own[target] == a.Facts.AccountID {
|
||||
continue
|
||||
}
|
||||
am, err := a.Facts.Amount.Minor()
|
||||
if err != nil || am == 0 {
|
||||
continue
|
||||
}
|
||||
ad, err := time.Parse("2006-01-02", a.Facts.BookingDate)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for j := i + 1; j < len(data.Transactions); j++ {
|
||||
b := data.Transactions[j]
|
||||
if b.Enrichment.Kind == "transfer" || b.Enrichment.TransferPeerID != "" || b.Facts.AccountID != own[target] || normalizeIBAN(b.Facts.CounterpartyIBAN) != ai || a.Facts.Currency != b.Facts.Currency {
|
||||
continue
|
||||
}
|
||||
bm, err := b.Facts.Amount.Minor()
|
||||
if err != nil || (am > 0) == (bm > 0) || am+bm != 0 {
|
||||
continue
|
||||
}
|
||||
bd, err := time.Parse("2006-01-02", b.Facts.BookingDate)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
delta := ad.Sub(bd)
|
||||
if delta < -72*time.Hour || delta > 72*time.Hour {
|
||||
continue
|
||||
}
|
||||
candidates[i] = append(candidates[i], j)
|
||||
candidates[j] = append(candidates[j], i)
|
||||
}
|
||||
}
|
||||
for i, matches := range candidates {
|
||||
if len(matches) != 1 {
|
||||
continue
|
||||
}
|
||||
j := matches[0]
|
||||
if j <= i || len(candidates[j]) != 1 {
|
||||
continue
|
||||
}
|
||||
for _, pair := range [][2]int{{i, j}, {j, i}} {
|
||||
t := &data.Transactions[pair[0]]
|
||||
tags := t.Enrichment.TagIDs
|
||||
if tags == nil {
|
||||
tags = []string{}
|
||||
}
|
||||
t.Enrichment = domain.Enrichment{Kind: "transfer", TagIDs: tags, TransferPeerID: data.Transactions[pair[1]].Facts.ID, Classification: domain.Provenance{Source: "transfer_match"}}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user