391 lines
13 KiB
Go
391 lines
13 KiB
Go
package banking
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"finance-duck/internal/domain"
|
|
)
|
|
|
|
func digest(parts ...string) string {
|
|
b, _ := json.Marshal(parts)
|
|
h := sha256.Sum256(b)
|
|
return hex.EncodeToString(h[:])
|
|
}
|
|
func identity(f domain.Facts) string {
|
|
// Banks can reuse a reference for a debit and its credit counterpart.
|
|
// Facts are normalized before lookup; never rewrite stored journal IDs.
|
|
direction := "credit"
|
|
if strings.HasPrefix(string(f.Amount), "-") {
|
|
direction = "debit"
|
|
}
|
|
return digest(f.AccountID, f.Source, f.ExternalID, direction)
|
|
}
|
|
func fingerprint(f domain.Facts) string {
|
|
return digest(f.AccountID, f.BookingDate, f.ValueDate, f.Amount.String(), f.Currency, strings.Join(strings.Fields(f.RawDescription), " "), strings.ToLower(strings.Join(strings.Fields(f.Counterparty), " ")), f.CounterpartyIBAN)
|
|
}
|
|
func looseFingerprint(f domain.Facts) string {
|
|
return digest(f.AccountID, f.BookingDate, f.Amount.String(), f.Currency)
|
|
}
|
|
func sameBookedMoney(a, b domain.Facts) bool {
|
|
return a.AccountID == b.AccountID && a.BookingDate == b.BookingDate && a.Amount == b.Amount && a.Currency == b.Currency
|
|
}
|
|
func sourceLabel(source string) string {
|
|
switch source {
|
|
case "enablebanking":
|
|
return "bank-synced"
|
|
case "n26_csv":
|
|
return "CSV"
|
|
default:
|
|
return "source " + strconv.Quote(source)
|
|
}
|
|
}
|
|
func normalizedDetail(value string) string {
|
|
return strings.Join(strings.Fields(value), " ")
|
|
}
|
|
func conflictValue(value string) string {
|
|
if value == "" {
|
|
return "none"
|
|
}
|
|
const maxRunes = 120
|
|
runes := []rune(value)
|
|
if len(runes) > maxRunes {
|
|
value = string(runes[:maxRunes]) + "…"
|
|
}
|
|
return strconv.Quote(value)
|
|
}
|
|
func crossSourceDetails(left, right domain.Facts) string {
|
|
fields := []struct {
|
|
name, left, right string
|
|
}{
|
|
{"value date", left.ValueDate, right.ValueDate},
|
|
{"description", normalizedDetail(left.RawDescription), normalizedDetail(right.RawDescription)},
|
|
{"counterparty", strings.ToLower(normalizedDetail(left.Counterparty)), strings.ToLower(normalizedDetail(right.Counterparty))},
|
|
{"counterparty IBAN", left.CounterpartyIBAN, right.CounterpartyIBAN},
|
|
}
|
|
differences := make([]string, 0, len(fields))
|
|
for _, field := range fields {
|
|
if field.left != field.right {
|
|
differences = append(differences, fmt.Sprintf("%s (%s %s; %s %s)", field.name, sourceLabel(left.Source), conflictValue(field.left), sourceLabel(right.Source), conflictValue(field.right)))
|
|
}
|
|
}
|
|
if len(differences) == 0 {
|
|
return "unrecorded transaction details"
|
|
}
|
|
return strings.Join(differences, ", ")
|
|
}
|
|
|
|
// NormalizeAndDedupe returns new records without mutating the input. Stable bank
|
|
// entry references, scoped by account, source and debit/credit direction, take
|
|
// precedence over text. Rows without references use occurrence counts, not a set:
|
|
// two identical rows remain two transactions; reimporting creates none.
|
|
// For overlapping partial exports,
|
|
// indistinguishable rows cannot prove an additional occurrence; import complete
|
|
// overlapping date windows to establish multiplicity.
|
|
//
|
|
// Cross-source reconciliation only suppresses equal full-fingerprint groups with
|
|
// equal multiplicity. Same-day/same-money cross-source discrepancies fail closed
|
|
// for user review rather than guessing or silently inflating balances. No alias
|
|
// or bank fact is rewritten, so later upstream metadata drift remains visible.
|
|
func NormalizeAndDedupe(data domain.Dataset, incoming []domain.Facts) ([]domain.Transaction, error) {
|
|
accounts := make(map[string]bool, len(data.Accounts))
|
|
for _, a := range data.Accounts {
|
|
accounts[a.ID] = true
|
|
}
|
|
type group struct {
|
|
source, fp string
|
|
facts []domain.Facts
|
|
}
|
|
groups := map[string]*group{}
|
|
existing := map[string]map[string]int{}
|
|
existingAnonymous := map[string]int{}
|
|
existingIDs := map[string]domain.Facts{}
|
|
loose := map[string]map[string]map[string]domain.Facts{}
|
|
addLoose := func(f domain.Facts, fp string) {
|
|
k := looseFingerprint(f)
|
|
if loose[k] == nil {
|
|
loose[k] = map[string]map[string]domain.Facts{}
|
|
}
|
|
if loose[k][f.Source] == nil {
|
|
loose[k][f.Source] = map[string]domain.Facts{}
|
|
}
|
|
if _, exists := loose[k][f.Source][fp]; !exists {
|
|
loose[k][f.Source][fp] = f
|
|
}
|
|
}
|
|
for _, t := range data.Transactions {
|
|
f, err := normalizeFacts(t.Facts, accounts)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("existing transaction %s: %w", t.Facts.ID, err)
|
|
}
|
|
fp := fingerprint(f)
|
|
if existing[fp] == nil {
|
|
existing[fp] = map[string]int{}
|
|
}
|
|
existing[fp][f.Source]++
|
|
if f.ExternalID == "" {
|
|
existingAnonymous[digest(f.Source, fp)]++
|
|
}
|
|
if f.ExternalID != "" {
|
|
existingIDs[identity(f)] = f
|
|
}
|
|
addLoose(f, fp)
|
|
}
|
|
seenIDs := map[string]domain.Facts{}
|
|
for index, original := range incoming {
|
|
f, err := normalizeFacts(original, accounts)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("incoming record %d: %w", index+1, err)
|
|
}
|
|
if f.ExternalID != "" {
|
|
key := identity(f)
|
|
if old, ok := seenIDs[key]; ok {
|
|
if !sameBookedMoney(old, f) {
|
|
return nil, fmt.Errorf("conflicting upstream transaction identity in incoming records")
|
|
}
|
|
continue
|
|
}
|
|
seenIDs[key] = f
|
|
if old, ok := existingIDs[key]; ok {
|
|
if !sameBookedMoney(old, f) {
|
|
return nil, fmt.Errorf("upstream transaction changed immutable booking facts")
|
|
}
|
|
// Use stored metadata to keep this matched occurrence in its original group.
|
|
f = old
|
|
}
|
|
}
|
|
fp := fingerprint(f)
|
|
key := digest(f.Source, fp)
|
|
if groups[key] == nil {
|
|
groups[key] = &group{source: f.Source, fp: fp}
|
|
}
|
|
groups[key].facts = append(groups[key].facts, f)
|
|
addLoose(f, fp)
|
|
}
|
|
// Reject ambiguous collisions even when one exact match also exists.
|
|
for _, g := range groups {
|
|
for _, f := range g.facts {
|
|
for source, fps := range loose[looseFingerprint(f)] {
|
|
if source != g.source {
|
|
for fp, other := range fps {
|
|
if fp != g.fp {
|
|
return nil, fmt.Errorf("%s and %s transactions overlap on %s for %s %s but differ in %s. Finance Duck cannot tell whether they are one transaction or two; to prevent double counting, nothing was imported. Compare both records, then correct or remove the duplicate before retrying", sourceLabel(f.Source), sourceLabel(other.Source), f.BookingDate, f.Amount, f.Currency, crossSourceDetails(f, other))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
keys := make([]string, 0, len(groups))
|
|
for k := range groups {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
result := make([]domain.Transaction, 0)
|
|
accepted := map[string]map[string]int{}
|
|
for _, key := range keys {
|
|
g := groups[key]
|
|
crossCount := -1
|
|
for source, n := range existing[g.fp] {
|
|
if source != g.source {
|
|
if crossCount >= 0 && crossCount != n {
|
|
return nil, fmt.Errorf("uncertain cross-source occurrence counts")
|
|
}
|
|
crossCount = n
|
|
}
|
|
}
|
|
for source, n := range accepted[g.fp] {
|
|
if source != g.source {
|
|
if crossCount >= 0 && crossCount != n {
|
|
return nil, fmt.Errorf("uncertain cross-source occurrence counts")
|
|
}
|
|
crossCount = n
|
|
}
|
|
}
|
|
if crossCount >= 0 {
|
|
f := g.facts[0]
|
|
if strings.TrimSpace(f.RawDescription) == "" && strings.TrimSpace(f.Counterparty) == "" && f.CounterpartyIBAN == "" {
|
|
return nil, fmt.Errorf("uncertain cross-source match lacks descriptive bank evidence")
|
|
}
|
|
if crossCount != len(g.facts) {
|
|
return nil, fmt.Errorf("uncertain cross-source occurrence counts on account %s at %s", g.facts[0].AccountID, g.facts[0].BookingDate)
|
|
}
|
|
continue
|
|
}
|
|
// Sorting IDs makes equal-fingerprint upstream records input-order independent.
|
|
sort.SliceStable(g.facts, func(i, j int) bool { return g.facts[i].ExternalID < g.facts[j].ExternalID })
|
|
// Referenced and anonymous records consume separate occurrence pools. When a
|
|
// reference appears/disappears, a spare record in the other pool is ambiguous:
|
|
// it may be an existing booking with changed identity metadata, not new money.
|
|
baseline := existingAnonymous[key]
|
|
anonymousCount, matchedReferences, newReferences := 0, 0, 0
|
|
for _, f := range g.facts {
|
|
if f.ExternalID == "" {
|
|
anonymousCount++
|
|
} else if _, ok := existingIDs[identity(f)]; ok {
|
|
matchedReferences++
|
|
} else {
|
|
newReferences++
|
|
}
|
|
}
|
|
unmatchedReferences := existing[g.fp][g.source] - baseline - matchedReferences
|
|
if (newReferences > 0 && baseline > anonymousCount) || (anonymousCount > baseline && unmatchedReferences > 0) {
|
|
return nil, fmt.Errorf("uncertain transaction identity changed between referenced and anonymous records on account %s at %s", g.facts[0].AccountID, g.facts[0].BookingDate)
|
|
}
|
|
occurrence := 0
|
|
for _, f := range g.facts {
|
|
if f.ExternalID != "" {
|
|
if _, ok := existingIDs[identity(f)]; ok {
|
|
continue
|
|
}
|
|
} else {
|
|
occurrence++
|
|
if occurrence <= baseline {
|
|
continue
|
|
}
|
|
}
|
|
f.Fingerprint = g.fp
|
|
if f.ExternalID != "" {
|
|
f.ID = "tx_" + identity(f)
|
|
} else {
|
|
f.ID = "tx_" + digest(f.Source, g.fp, strconv.Itoa(occurrence))
|
|
}
|
|
result = append(result, domain.Transaction{Facts: f, Enrichment: domain.Fallback(f)})
|
|
}
|
|
if accepted[g.fp] == nil {
|
|
accepted[g.fp] = map[string]int{}
|
|
}
|
|
accepted[g.fp][g.source] = len(g.facts)
|
|
}
|
|
sort.Slice(result, func(i, j int) bool {
|
|
a, b := result[i].Facts, result[j].Facts
|
|
if a.BookingDate != b.BookingDate {
|
|
return a.BookingDate < b.BookingDate
|
|
}
|
|
return a.ID < b.ID
|
|
})
|
|
return result, nil
|
|
}
|
|
|
|
func normalizeFacts(f domain.Facts, accounts map[string]bool) (domain.Facts, error) {
|
|
if !accounts[f.AccountID] {
|
|
return f, fmt.Errorf("unknown account")
|
|
}
|
|
if f.Source == "" {
|
|
return f, fmt.Errorf("missing import source")
|
|
}
|
|
date, err := parseDate(f.BookingDate)
|
|
if err != nil {
|
|
return f, fmt.Errorf("invalid booking date")
|
|
}
|
|
f.BookingDate = date
|
|
if f.ValueDate != "" {
|
|
f.ValueDate, err = parseDate(f.ValueDate)
|
|
if err != nil {
|
|
return f, fmt.Errorf("invalid value date")
|
|
}
|
|
}
|
|
f.Amount, err = domain.ParseMoney(string(f.Amount))
|
|
if err != nil {
|
|
return f, fmt.Errorf("invalid amount")
|
|
}
|
|
f.Currency = strings.ToUpper(strings.TrimSpace(f.Currency))
|
|
if !validCurrency(f.Currency) {
|
|
return f, fmt.Errorf("invalid currency")
|
|
}
|
|
f.CounterpartyIBAN = normalizeIBAN(f.CounterpartyIBAN)
|
|
f.ExternalID = strings.TrimSpace(f.ExternalID)
|
|
return f, nil
|
|
}
|
|
|
|
// MatchTransfers links only mutually unique candidates, with reciprocal own
|
|
// IBANs, inverse exact money in one currency, and booking dates within 3 calendar
|
|
// days. Existing manual links are retained. Ambiguous equal payments stay ordinary
|
|
// transactions: iteration order must never decide which transfer gets linked.
|
|
func MatchTransfers(data *domain.Dataset) {
|
|
if data == nil {
|
|
return
|
|
}
|
|
own := map[string]string{}
|
|
duplicates := map[string]bool{}
|
|
for _, a := range data.Accounts {
|
|
iban := normalizeIBAN(a.IBAN)
|
|
if iban == "" {
|
|
continue
|
|
}
|
|
if _, ok := own[iban]; ok {
|
|
duplicates[iban] = true
|
|
}
|
|
own[iban] = a.ID
|
|
}
|
|
byAccount := map[string]string{}
|
|
for iban, id := range own {
|
|
if !duplicates[iban] {
|
|
byAccount[id] = iban
|
|
}
|
|
}
|
|
candidates := make([][]int, len(data.Transactions))
|
|
for i := range data.Transactions {
|
|
a := data.Transactions[i]
|
|
if a.Enrichment.Kind == "transfer" || a.Enrichment.TransferPeerID != "" {
|
|
continue
|
|
}
|
|
ai := byAccount[a.Facts.AccountID]
|
|
target := normalizeIBAN(a.Facts.CounterpartyIBAN)
|
|
if ai == "" || target == "" || duplicates[target] || own[target] == "" || own[target] == a.Facts.AccountID {
|
|
continue
|
|
}
|
|
am, err := a.Facts.Amount.Minor()
|
|
if err != nil || am == 0 {
|
|
continue
|
|
}
|
|
ad, err := time.Parse("2006-01-02", a.Facts.BookingDate)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
for j := i + 1; j < len(data.Transactions); j++ {
|
|
b := data.Transactions[j]
|
|
if b.Enrichment.Kind == "transfer" || b.Enrichment.TransferPeerID != "" || b.Facts.AccountID != own[target] || normalizeIBAN(b.Facts.CounterpartyIBAN) != ai || a.Facts.Currency != b.Facts.Currency {
|
|
continue
|
|
}
|
|
bm, err := b.Facts.Amount.Minor()
|
|
if err != nil || (am > 0) == (bm > 0) || am+bm != 0 {
|
|
continue
|
|
}
|
|
bd, err := time.Parse("2006-01-02", b.Facts.BookingDate)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
delta := ad.Sub(bd)
|
|
if delta < -72*time.Hour || delta > 72*time.Hour {
|
|
continue
|
|
}
|
|
candidates[i] = append(candidates[i], j)
|
|
candidates[j] = append(candidates[j], i)
|
|
}
|
|
}
|
|
for i, matches := range candidates {
|
|
if len(matches) != 1 {
|
|
continue
|
|
}
|
|
j := matches[0]
|
|
if j <= i || len(candidates[j]) != 1 {
|
|
continue
|
|
}
|
|
for _, pair := range [][2]int{{i, j}, {j, i}} {
|
|
t := &data.Transactions[pair[0]]
|
|
tags := t.Enrichment.TagIDs
|
|
if tags == nil {
|
|
tags = []string{}
|
|
}
|
|
t.Enrichment = domain.Enrichment{Kind: "transfer", TagIDs: tags, TransferPeerID: data.Transactions[pair[1]].Facts.ID, Classification: domain.Provenance{Source: "transfer_match"}}
|
|
}
|
|
}
|
|
}
|