finance-duck/internal/banking/import.go

package banking

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"sort"
	"strconv"
	"strings"
	"time"

	"finance-duck/internal/domain"
)

func digest(parts ...string) string {
	b, _ := json.Marshal(parts)
	h := sha256.Sum256(b)
	return hex.EncodeToString(h[:])
}
func identity(f domain.Facts) string {
	// Banks can reuse a reference for a debit and its credit counterpart.
	// Facts are normalized before lookup; never rewrite stored journal IDs.
	direction := "credit"
	if strings.HasPrefix(string(f.Amount), "-") {
		direction = "debit"
	}
	return digest(f.AccountID, f.Source, f.ExternalID, direction, leg(f))
}

// leg distinguishes the records of one broker event. A broker reuses a single
// reference across every leg: the cash side of a corporate action and its
// position side arrive with the same reference byte for byte, and a position
// leg's zero amount does not even differ in direction. The event and its
// instrument separate them without making money part of an identity, so a
// corrected upstream figure is still reported rather than imported twice.
func leg(f domain.Facts) string {
	if f.Investment == nil {
		return ""
	}
	return f.Investment.Event + "\x00" + f.Investment.InstrumentID
}
func fingerprint(f domain.Facts) string {
	inv := domain.Investment{}
	if f.Investment != nil {
		inv = *f.Investment
	}
	return digest(f.AccountID, f.BookingDate, f.ValueDate, f.Amount.String(), f.Currency, strings.Join(strings.Fields(f.RawDescription), " "), strings.ToLower(strings.Join(strings.Fields(f.Counterparty), " ")), f.CounterpartyIBAN,
		inv.Event, inv.InstrumentID, string(inv.Quantity), string(inv.Price), string(inv.Gross), string(inv.Fee), string(inv.Tax))
}
func looseFingerprint(f domain.Facts) string {
	return digest(f.AccountID, f.BookingDate, f.Amount.String(), f.Currency, leg(f))
}
func sameBookedMoney(a, b domain.Facts) bool {
	return a.AccountID == b.AccountID && a.BookingDate == b.BookingDate && a.Amount == b.Amount && a.Currency == b.Currency
}
func sourceLabel(source string) string {
	switch source {
	case "enablebanking":
		return "bank-synced"
	case "n26_csv":
		return "CSV"
	case "ing_csv":
		return "ING CSV"
	case "kontist_csv":
		return "Kontist CSV"
	case SourceScalable:
		return "Scalable CSV"
	case "csv":
		return "mapped CSV"
	default:
		return "source " + strconv.Quote(source)
	}
}
func normalizedDetail(value string) string {
	return strings.Join(strings.Fields(value), " ")
}
func conflictValue(value string) string {
	if value == "" {
		return "none"
	}
	const maxRunes = 120
	runes := []rune(value)
	if len(runes) > maxRunes {
		value = string(runes[:maxRunes]) + "…"
	}
	return strconv.Quote(value)
}
func crossSourceDetails(left, right domain.Facts) string {
	fields := []struct {
		name, left, right string
	}{
		{"value date", left.ValueDate, right.ValueDate},
		{"description", normalizedDetail(left.RawDescription), normalizedDetail(right.RawDescription)},
		{"counterparty", strings.ToLower(normalizedDetail(left.Counterparty)), strings.ToLower(normalizedDetail(right.Counterparty))},
		{"counterparty IBAN", left.CounterpartyIBAN, right.CounterpartyIBAN},
	}
	differences := make([]string, 0, len(fields))
	for _, field := range fields {
		if field.left != field.right {
			differences = append(differences, fmt.Sprintf("%s (%s %s; %s %s)", field.name, sourceLabel(left.Source), conflictValue(field.left), sourceLabel(right.Source), conflictValue(field.right)))
		}
	}
	if len(differences) == 0 {
		return "unrecorded transaction details"
	}
	return strings.Join(differences, ", ")
}

// NormalizeAndDedupe returns new records without mutating the input. Stable bank
// entry references, scoped by account, source and debit/credit direction, take
// precedence over text. Rows without references use occurrence counts, not a set:
// two identical rows remain two transactions; reimporting creates none.
// For overlapping partial exports,
// indistinguishable rows cannot prove an additional occurrence; import complete
// overlapping date windows to establish multiplicity.
//
// Cross-source reconciliation only suppresses equal full-fingerprint groups with
// equal multiplicity. Same-day/same-money cross-source discrepancies fail closed
// for user review rather than guessing or silently inflating balances. No alias
// or bank fact is rewritten, so later upstream metadata drift remains visible.
func NormalizeAndDedupe(data domain.Dataset, incoming []domain.Facts) ([]domain.Transaction, error) {
	accounts := make(map[string]bool, len(data.Accounts))
	for _, a := range data.Accounts {
		accounts[a.ID] = true
	}
	type group struct {
		source, fp string
		facts      []domain.Facts
	}
	groups := map[string]*group{}
	existing := map[string]map[string]int{}
	existingAnonymous := map[string]int{}
	existingIDs := map[string]domain.Facts{}
	loose := map[string]map[string]map[string]domain.Facts{}
	addLoose := func(f domain.Facts, fp string) {
		k := looseFingerprint(f)
		if loose[k] == nil {
			loose[k] = map[string]map[string]domain.Facts{}
		}
		if loose[k][f.Source] == nil {
			loose[k][f.Source] = map[string]domain.Facts{}
		}
		if _, exists := loose[k][f.Source][fp]; !exists {
			loose[k][f.Source][fp] = f
		}
	}
	for _, t := range data.Transactions {
		f, err := normalizeFacts(t.Facts, accounts)
		if err != nil {
			return nil, fmt.Errorf("existing transaction %s: %w", t.Facts.ID, err)
		}
		fp := fingerprint(f)
		if existing[fp] == nil {
			existing[fp] = map[string]int{}
		}
		existing[fp][f.Source]++
		if f.ExternalID == "" {
			existingAnonymous[digest(f.Source, fp)]++
		}
		if f.ExternalID != "" {
			existingIDs[identity(f)] = f
		}
		addLoose(f, fp)
	}
	seenIDs := map[string]domain.Facts{}
	for index, original := range incoming {
		f, err := normalizeFacts(original, accounts)
		if err != nil {
			return nil, fmt.Errorf("incoming record %d: %w", index+1, err)
		}
		if f.ExternalID != "" {
			key := identity(f)
			if old, ok := seenIDs[key]; ok {
				if !sameBookedMoney(old, f) {
					return nil, fmt.Errorf("conflicting upstream transaction identity in incoming records")
				}
				continue
			}
			seenIDs[key] = f
			if old, ok := existingIDs[key]; ok {
				if !sameBookedMoney(old, f) {
					return nil, fmt.Errorf("upstream transaction changed immutable booking facts")
				}
				// Use stored metadata to keep this matched occurrence in its original group.
				f = old
			}
		}
		fp := fingerprint(f)
		key := digest(f.Source, fp)
		if groups[key] == nil {
			groups[key] = &group{source: f.Source, fp: fp}
		}
		groups[key].facts = append(groups[key].facts, f)
		addLoose(f, fp)
	}
	// Reject ambiguous collisions even when one exact match also exists.
	for _, g := range groups {
		for _, f := range g.facts {
			for source, fps := range loose[looseFingerprint(f)] {
				if source != g.source {
					for fp, other := range fps {
						if fp != g.fp {
							return nil, fmt.Errorf("%s and %s transactions overlap on %s for %s %s but differ in %s. Finance Duck cannot tell whether they are one transaction or two; to prevent double counting, nothing was imported. Compare both records, then correct or remove the duplicate before retrying", sourceLabel(f.Source), sourceLabel(other.Source), f.BookingDate, f.Amount, f.Currency, crossSourceDetails(f, other))
						}
					}
				}
			}
		}
	}
	keys := make([]string, 0, len(groups))
	for k := range groups {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	result := make([]domain.Transaction, 0)
	accepted := map[string]map[string]int{}
	for _, key := range keys {
		g := groups[key]
		crossCount := -1
		for source, n := range existing[g.fp] {
			if source != g.source {
				if crossCount >= 0 && crossCount != n {
					return nil, fmt.Errorf("uncertain cross-source occurrence counts")
				}
				crossCount = n
			}
		}
		for source, n := range accepted[g.fp] {
			if source != g.source {
				if crossCount >= 0 && crossCount != n {
					return nil, fmt.Errorf("uncertain cross-source occurrence counts")
				}
				crossCount = n
			}
		}
		if crossCount >= 0 {
			f := g.facts[0]
			if strings.TrimSpace(f.RawDescription) == "" && strings.TrimSpace(f.Counterparty) == "" && f.CounterpartyIBAN == "" {
				return nil, fmt.Errorf("uncertain cross-source match lacks descriptive bank evidence")
			}
			if crossCount != len(g.facts) {
				return nil, fmt.Errorf("uncertain cross-source occurrence counts on account %s at %s", g.facts[0].AccountID, g.facts[0].BookingDate)
			}
			continue
		}
		// Sorting IDs makes equal-fingerprint upstream records input-order independent.
		sort.SliceStable(g.facts, func(i, j int) bool { return g.facts[i].ExternalID < g.facts[j].ExternalID })
		// Referenced and anonymous records consume separate occurrence pools. When a
		// reference appears/disappears, a spare record in the other pool is ambiguous:
		// it may be an existing booking with changed identity metadata, not new money.
		baseline := existingAnonymous[key]
		anonymousCount, matchedReferences, newReferences := 0, 0, 0
		for _, f := range g.facts {
			if f.ExternalID == "" {
				anonymousCount++
			} else if _, ok := existingIDs[identity(f)]; ok {
				matchedReferences++
			} else {
				newReferences++
			}
		}
		unmatchedReferences := existing[g.fp][g.source] - baseline - matchedReferences
		if (newReferences > 0 && baseline > anonymousCount) || (anonymousCount > baseline && unmatchedReferences > 0) {
			return nil, fmt.Errorf("uncertain transaction identity changed between referenced and anonymous records on account %s at %s", g.facts[0].AccountID, g.facts[0].BookingDate)
		}
		occurrence := 0
		for _, f := range g.facts {
			if f.ExternalID != "" {
				if _, ok := existingIDs[identity(f)]; ok {
					continue
				}
			} else {
				occurrence++
				if occurrence <= baseline {
					continue
				}
			}
			f.Fingerprint = g.fp
			if f.ExternalID != "" {
				f.ID = "tx_" + identity(f)
			} else {
				f.ID = "tx_" + digest(f.Source, g.fp, strconv.Itoa(occurrence))
			}
			result = append(result, domain.Transaction{Facts: f, Enrichment: domain.Fallback(f)})
		}
		if accepted[g.fp] == nil {
			accepted[g.fp] = map[string]int{}
		}
		accepted[g.fp][g.source] = len(g.facts)
	}
	sort.Slice(result, func(i, j int) bool {
		a, b := result[i].Facts, result[j].Facts
		if a.BookingDate != b.BookingDate {
			return a.BookingDate < b.BookingDate
		}
		return a.ID < b.ID
	})
	return result, nil
}

func normalizeFacts(f domain.Facts, accounts map[string]bool) (domain.Facts, error) {
	if !accounts[f.AccountID] {
		return f, fmt.Errorf("unknown account")
	}
	if f.Source == "" {
		return f, fmt.Errorf("missing import source")
	}
	date, err := parseDate(f.BookingDate)
	if err != nil {
		return f, fmt.Errorf("invalid booking date")
	}
	f.BookingDate = date
	if f.ValueDate != "" {
		f.ValueDate, err = parseDate(f.ValueDate)
		if err != nil {
			return f, fmt.Errorf("invalid value date")
		}
	}
	f.Amount, err = domain.ParseMoney(string(f.Amount))
	if err != nil {
		return f, fmt.Errorf("invalid amount")
	}
	f.Currency = strings.ToUpper(strings.TrimSpace(f.Currency))
	if !validCurrency(f.Currency) {
		return f, fmt.Errorf("invalid currency")
	}
	f.CounterpartyIBAN = normalizeIBAN(f.CounterpartyIBAN)
	f.ExternalID = strings.TrimSpace(f.ExternalID)
	return f, nil
}

// MatchTransfers links own-account pairs with reciprocal own IBANs, inverse
// exact money in one currency, and booking dates within 3 calendar days.
//
// Equal competing payments are paired by nearest booking date rather than left
// alone. Every connected component of the candidate graph is a complete
// bipartite graph between two fixed accounts at one amount and one currency:
// an edge needs exactly inverse money, and a record's own counterparty IBAN
// names exactly one other account. So every perfect matching produces the same
// accounts, amounts, kinds and postings, and the only thing a choice decides is
// which row displays as which one's counterpart. Refusing to choose is the
// expensive option: both legs then fall through to the sign-based fallback and
// show up as spending and income that never happened.
//
// Ordering is by date gap, then by transaction ID, so iteration order cannot
// decide anything. Existing links and hand-made decisions are never revisited.
func MatchTransfers(data *domain.Dataset) {
	if data == nil {
		return
	}
	own := map[string]string{}
	duplicates := map[string]bool{}
	for _, a := range data.Accounts {
		iban := normalizeIBAN(a.IBAN)
		if iban == "" {
			continue
		}
		if _, ok := own[iban]; ok {
			duplicates[iban] = true
		}
		own[iban] = a.ID
	}
	byAccount := map[string]string{}
	for iban, id := range own {
		if !duplicates[iban] {
			byAccount[id] = iban
		}
	}
	matchable := func(t domain.Transaction) bool {
		if t.Enrichment.Kind == "transfer" || t.Enrichment.TransferPeerID != "" {
			return false
		}
		// A hand-made decision outlives the next import. Without this, an
		// operator who unlinks a pair that is not really a transfer watches the
		// matcher relink it on the following import, forever.
		if t.Enrichment.Classification.Source == "manual" {
			return false
		}
		// Only a broker cash movement can be a transfer leg; a trade's cash
		// side settles against a position, not against another account.
		return t.Facts.Investment == nil || t.Facts.Investment.CashOnly()
	}
	type candidate struct {
		i, j int
		gap  time.Duration
	}
	candidates := []candidate{}
	for i := range data.Transactions {
		a := data.Transactions[i]
		if !matchable(a) {
			continue
		}
		ai := byAccount[a.Facts.AccountID]
		target := normalizeIBAN(a.Facts.CounterpartyIBAN)
		if ai == "" || target == "" || duplicates[target] || own[target] == "" || own[target] == a.Facts.AccountID {
			continue
		}
		am, err := a.Facts.Amount.Minor()
		if err != nil || am == 0 {
			continue
		}
		ad, err := time.Parse("2006-01-02", a.Facts.BookingDate)
		if err != nil {
			continue
		}
		for j := i + 1; j < len(data.Transactions); j++ {
			b := data.Transactions[j]
			if !matchable(b) || b.Facts.AccountID != own[target] || normalizeIBAN(b.Facts.CounterpartyIBAN) != ai || a.Facts.Currency != b.Facts.Currency {
				continue
			}
			bm, err := b.Facts.Amount.Minor()
			if err != nil || (am > 0) == (bm > 0) || am+bm != 0 {
				continue
			}
			bd, err := time.Parse("2006-01-02", b.Facts.BookingDate)
			if err != nil {
				continue
			}
			gap := ad.Sub(bd)
			if gap < 0 {
				gap = -gap
			}
			if gap > 72*time.Hour {
				continue
			}
			candidates = append(candidates, candidate{i: i, j: j, gap: gap})
		}
	}
	sort.Slice(candidates, func(x, y int) bool {
		if candidates[x].gap != candidates[y].gap {
			return candidates[x].gap < candidates[y].gap
		}
		left, right := data.Transactions[candidates[x].i].Facts.ID, data.Transactions[candidates[y].i].Facts.ID
		if left != right {
			return left < right
		}
		return data.Transactions[candidates[x].j].Facts.ID < data.Transactions[candidates[y].j].Facts.ID
	})
	linked := make([]bool, len(data.Transactions))
	for _, c := range candidates {
		if linked[c.i] || linked[c.j] {
			continue
		}
		linked[c.i], linked[c.j] = true, true
		for _, ends := range [][2]int{{c.i, c.j}, {c.j, c.i}} {
			t := &data.Transactions[ends[0]]
			tags := t.Enrichment.TagIDs
			if tags == nil {
				tags = []string{}
			}
			t.Enrichment = domain.Enrichment{Kind: "transfer", TagIDs: tags, TransferPeerID: data.Transactions[ends[1]].Facts.ID, Classification: domain.Provenance{Source: "transfer_match"}}
		}
	}
}