package banking import ( "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "sort" "strconv" "strings" "time" "finance-duck/internal/domain" ) func digest(parts ...string) string { b, _ := json.Marshal(parts) h := sha256.Sum256(b) return hex.EncodeToString(h[:]) } func identity(f domain.Facts) string { // Banks can reuse a reference for a debit and its credit counterpart. // Facts are normalized before lookup; never rewrite stored journal IDs. direction := "credit" if strings.HasPrefix(string(f.Amount), "-") { direction = "debit" } return digest(f.AccountID, f.Source, f.ExternalID, direction) } func fingerprint(f domain.Facts) string { return digest(f.AccountID, f.BookingDate, f.ValueDate, f.Amount.String(), f.Currency, strings.Join(strings.Fields(f.RawDescription), " "), strings.ToLower(strings.Join(strings.Fields(f.Counterparty), " ")), f.CounterpartyIBAN) } func looseFingerprint(f domain.Facts) string { return digest(f.AccountID, f.BookingDate, f.Amount.String(), f.Currency) } func sameBookedMoney(a, b domain.Facts) bool { return a.AccountID == b.AccountID && a.BookingDate == b.BookingDate && a.Amount == b.Amount && a.Currency == b.Currency } func sourceLabel(source string) string { switch source { case "enablebanking": return "bank-synced" case "n26_csv": return "CSV" case "ing_csv": return "ING CSV" case "kontist_csv": return "Kontist CSV" case "csv": return "mapped CSV" default: return "source " + strconv.Quote(source) } } func normalizedDetail(value string) string { return strings.Join(strings.Fields(value), " ") } func conflictValue(value string) string { if value == "" { return "none" } const maxRunes = 120 runes := []rune(value) if len(runes) > maxRunes { value = string(runes[:maxRunes]) + "…" } return strconv.Quote(value) } func crossSourceDetails(left, right domain.Facts) string { fields := []struct { name, left, right string }{ {"value date", left.ValueDate, right.ValueDate}, {"description", normalizedDetail(left.RawDescription), normalizedDetail(right.RawDescription)}, {"counterparty", strings.ToLower(normalizedDetail(left.Counterparty)), strings.ToLower(normalizedDetail(right.Counterparty))}, {"counterparty IBAN", left.CounterpartyIBAN, right.CounterpartyIBAN}, } differences := make([]string, 0, len(fields)) for _, field := range fields { if field.left != field.right { differences = append(differences, fmt.Sprintf("%s (%s %s; %s %s)", field.name, sourceLabel(left.Source), conflictValue(field.left), sourceLabel(right.Source), conflictValue(field.right))) } } if len(differences) == 0 { return "unrecorded transaction details" } return strings.Join(differences, ", ") } // NormalizeAndDedupe returns new records without mutating the input. Stable bank // entry references, scoped by account, source and debit/credit direction, take // precedence over text. Rows without references use occurrence counts, not a set: // two identical rows remain two transactions; reimporting creates none. // For overlapping partial exports, // indistinguishable rows cannot prove an additional occurrence; import complete // overlapping date windows to establish multiplicity. // // Cross-source reconciliation only suppresses equal full-fingerprint groups with // equal multiplicity. Same-day/same-money cross-source discrepancies fail closed // for user review rather than guessing or silently inflating balances. No alias // or bank fact is rewritten, so later upstream metadata drift remains visible. func NormalizeAndDedupe(data domain.Dataset, incoming []domain.Facts) ([]domain.Transaction, error) { accounts := make(map[string]bool, len(data.Accounts)) for _, a := range data.Accounts { accounts[a.ID] = true } type group struct { source, fp string facts []domain.Facts } groups := map[string]*group{} existing := map[string]map[string]int{} existingAnonymous := map[string]int{} existingIDs := map[string]domain.Facts{} loose := map[string]map[string]map[string]domain.Facts{} addLoose := func(f domain.Facts, fp string) { k := looseFingerprint(f) if loose[k] == nil { loose[k] = map[string]map[string]domain.Facts{} } if loose[k][f.Source] == nil { loose[k][f.Source] = map[string]domain.Facts{} } if _, exists := loose[k][f.Source][fp]; !exists { loose[k][f.Source][fp] = f } } for _, t := range data.Transactions { f, err := normalizeFacts(t.Facts, accounts) if err != nil { return nil, fmt.Errorf("existing transaction %s: %w", t.Facts.ID, err) } fp := fingerprint(f) if existing[fp] == nil { existing[fp] = map[string]int{} } existing[fp][f.Source]++ if f.ExternalID == "" { existingAnonymous[digest(f.Source, fp)]++ } if f.ExternalID != "" { existingIDs[identity(f)] = f } addLoose(f, fp) } seenIDs := map[string]domain.Facts{} for index, original := range incoming { f, err := normalizeFacts(original, accounts) if err != nil { return nil, fmt.Errorf("incoming record %d: %w", index+1, err) } if f.ExternalID != "" { key := identity(f) if old, ok := seenIDs[key]; ok { if !sameBookedMoney(old, f) { return nil, fmt.Errorf("conflicting upstream transaction identity in incoming records") } continue } seenIDs[key] = f if old, ok := existingIDs[key]; ok { if !sameBookedMoney(old, f) { return nil, fmt.Errorf("upstream transaction changed immutable booking facts") } // Use stored metadata to keep this matched occurrence in its original group. f = old } } fp := fingerprint(f) key := digest(f.Source, fp) if groups[key] == nil { groups[key] = &group{source: f.Source, fp: fp} } groups[key].facts = append(groups[key].facts, f) addLoose(f, fp) } // Reject ambiguous collisions even when one exact match also exists. for _, g := range groups { for _, f := range g.facts { for source, fps := range loose[looseFingerprint(f)] { if source != g.source { for fp, other := range fps { if fp != g.fp { return nil, fmt.Errorf("%s and %s transactions overlap on %s for %s %s but differ in %s. Finance Duck cannot tell whether they are one transaction or two; to prevent double counting, nothing was imported. Compare both records, then correct or remove the duplicate before retrying", sourceLabel(f.Source), sourceLabel(other.Source), f.BookingDate, f.Amount, f.Currency, crossSourceDetails(f, other)) } } } } } } keys := make([]string, 0, len(groups)) for k := range groups { keys = append(keys, k) } sort.Strings(keys) result := make([]domain.Transaction, 0) accepted := map[string]map[string]int{} for _, key := range keys { g := groups[key] crossCount := -1 for source, n := range existing[g.fp] { if source != g.source { if crossCount >= 0 && crossCount != n { return nil, fmt.Errorf("uncertain cross-source occurrence counts") } crossCount = n } } for source, n := range accepted[g.fp] { if source != g.source { if crossCount >= 0 && crossCount != n { return nil, fmt.Errorf("uncertain cross-source occurrence counts") } crossCount = n } } if crossCount >= 0 { f := g.facts[0] if strings.TrimSpace(f.RawDescription) == "" && strings.TrimSpace(f.Counterparty) == "" && f.CounterpartyIBAN == "" { return nil, fmt.Errorf("uncertain cross-source match lacks descriptive bank evidence") } if crossCount != len(g.facts) { return nil, fmt.Errorf("uncertain cross-source occurrence counts on account %s at %s", g.facts[0].AccountID, g.facts[0].BookingDate) } continue } // Sorting IDs makes equal-fingerprint upstream records input-order independent. sort.SliceStable(g.facts, func(i, j int) bool { return g.facts[i].ExternalID < g.facts[j].ExternalID }) // Referenced and anonymous records consume separate occurrence pools. When a // reference appears/disappears, a spare record in the other pool is ambiguous: // it may be an existing booking with changed identity metadata, not new money. baseline := existingAnonymous[key] anonymousCount, matchedReferences, newReferences := 0, 0, 0 for _, f := range g.facts { if f.ExternalID == "" { anonymousCount++ } else if _, ok := existingIDs[identity(f)]; ok { matchedReferences++ } else { newReferences++ } } unmatchedReferences := existing[g.fp][g.source] - baseline - matchedReferences if (newReferences > 0 && baseline > anonymousCount) || (anonymousCount > baseline && unmatchedReferences > 0) { return nil, fmt.Errorf("uncertain transaction identity changed between referenced and anonymous records on account %s at %s", g.facts[0].AccountID, g.facts[0].BookingDate) } occurrence := 0 for _, f := range g.facts { if f.ExternalID != "" { if _, ok := existingIDs[identity(f)]; ok { continue } } else { occurrence++ if occurrence <= baseline { continue } } f.Fingerprint = g.fp if f.ExternalID != "" { f.ID = "tx_" + identity(f) } else { f.ID = "tx_" + digest(f.Source, g.fp, strconv.Itoa(occurrence)) } result = append(result, domain.Transaction{Facts: f, Enrichment: domain.Fallback(f)}) } if accepted[g.fp] == nil { accepted[g.fp] = map[string]int{} } accepted[g.fp][g.source] = len(g.facts) } sort.Slice(result, func(i, j int) bool { a, b := result[i].Facts, result[j].Facts if a.BookingDate != b.BookingDate { return a.BookingDate < b.BookingDate } return a.ID < b.ID }) return result, nil } func normalizeFacts(f domain.Facts, accounts map[string]bool) (domain.Facts, error) { if !accounts[f.AccountID] { return f, fmt.Errorf("unknown account") } if f.Source == "" { return f, fmt.Errorf("missing import source") } date, err := parseDate(f.BookingDate) if err != nil { return f, fmt.Errorf("invalid booking date") } f.BookingDate = date if f.ValueDate != "" { f.ValueDate, err = parseDate(f.ValueDate) if err != nil { return f, fmt.Errorf("invalid value date") } } f.Amount, err = domain.ParseMoney(string(f.Amount)) if err != nil { return f, fmt.Errorf("invalid amount") } f.Currency = strings.ToUpper(strings.TrimSpace(f.Currency)) if !validCurrency(f.Currency) { return f, fmt.Errorf("invalid currency") } f.CounterpartyIBAN = normalizeIBAN(f.CounterpartyIBAN) f.ExternalID = strings.TrimSpace(f.ExternalID) return f, nil } // MatchTransfers links only mutually unique candidates, with reciprocal own // IBANs, inverse exact money in one currency, and booking dates within 3 calendar // days. Existing manual links are retained. Ambiguous equal payments stay ordinary // transactions: iteration order must never decide which transfer gets linked. func MatchTransfers(data *domain.Dataset) { if data == nil { return } own := map[string]string{} duplicates := map[string]bool{} for _, a := range data.Accounts { iban := normalizeIBAN(a.IBAN) if iban == "" { continue } if _, ok := own[iban]; ok { duplicates[iban] = true } own[iban] = a.ID } byAccount := map[string]string{} for iban, id := range own { if !duplicates[iban] { byAccount[id] = iban } } candidates := make([][]int, len(data.Transactions)) for i := range data.Transactions { a := data.Transactions[i] if a.Enrichment.Kind == "transfer" || a.Enrichment.TransferPeerID != "" { continue } ai := byAccount[a.Facts.AccountID] target := normalizeIBAN(a.Facts.CounterpartyIBAN) if ai == "" || target == "" || duplicates[target] || own[target] == "" || own[target] == a.Facts.AccountID { continue } am, err := a.Facts.Amount.Minor() if err != nil || am == 0 { continue } ad, err := time.Parse("2006-01-02", a.Facts.BookingDate) if err != nil { continue } for j := i + 1; j < len(data.Transactions); j++ { b := data.Transactions[j] if b.Enrichment.Kind == "transfer" || b.Enrichment.TransferPeerID != "" || b.Facts.AccountID != own[target] || normalizeIBAN(b.Facts.CounterpartyIBAN) != ai || a.Facts.Currency != b.Facts.Currency { continue } bm, err := b.Facts.Amount.Minor() if err != nil || (am > 0) == (bm > 0) || am+bm != 0 { continue } bd, err := time.Parse("2006-01-02", b.Facts.BookingDate) if err != nil { continue } delta := ad.Sub(bd) if delta < -72*time.Hour || delta > 72*time.Hour { continue } candidates[i] = append(candidates[i], j) candidates[j] = append(candidates[j], i) } } for i, matches := range candidates { if len(matches) != 1 { continue } j := matches[0] if j <= i || len(candidates[j]) != 1 { continue } for _, pair := range [][2]int{{i, j}, {j, i}} { t := &data.Transactions[pair[0]] tags := t.Enrichment.TagIDs if tags == nil { tags = []string{} } t.Enrichment = domain.Enrichment{Kind: "transfer", TagIDs: tags, TransferPeerID: data.Transactions[pair[1]].Facts.ID, Classification: domain.Provenance{Source: "transfer_match"}} } } }