better import csv error
This commit is contained in:
@@ -36,6 +36,50 @@ func looseFingerprint(f domain.Facts) string {
|
|||||||
func sameBookedMoney(a, b domain.Facts) bool {
|
func sameBookedMoney(a, b domain.Facts) bool {
|
||||||
return a.AccountID == b.AccountID && a.BookingDate == b.BookingDate && a.Amount == b.Amount && a.Currency == b.Currency
|
return a.AccountID == b.AccountID && a.BookingDate == b.BookingDate && a.Amount == b.Amount && a.Currency == b.Currency
|
||||||
}
|
}
|
||||||
|
func sourceLabel(source string) string {
|
||||||
|
switch source {
|
||||||
|
case "enablebanking":
|
||||||
|
return "bank-synced"
|
||||||
|
case "n26_csv":
|
||||||
|
return "CSV"
|
||||||
|
default:
|
||||||
|
return "source " + strconv.Quote(source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func normalizedDetail(value string) string {
|
||||||
|
return strings.Join(strings.Fields(value), " ")
|
||||||
|
}
|
||||||
|
func conflictValue(value string) string {
|
||||||
|
if value == "" {
|
||||||
|
return "none"
|
||||||
|
}
|
||||||
|
const maxRunes = 120
|
||||||
|
runes := []rune(value)
|
||||||
|
if len(runes) > maxRunes {
|
||||||
|
value = string(runes[:maxRunes]) + "…"
|
||||||
|
}
|
||||||
|
return strconv.Quote(value)
|
||||||
|
}
|
||||||
|
func crossSourceDetails(left, right domain.Facts) string {
|
||||||
|
fields := []struct {
|
||||||
|
name, left, right string
|
||||||
|
}{
|
||||||
|
{"value date", left.ValueDate, right.ValueDate},
|
||||||
|
{"description", normalizedDetail(left.RawDescription), normalizedDetail(right.RawDescription)},
|
||||||
|
{"counterparty", strings.ToLower(normalizedDetail(left.Counterparty)), strings.ToLower(normalizedDetail(right.Counterparty))},
|
||||||
|
{"counterparty IBAN", left.CounterpartyIBAN, right.CounterpartyIBAN},
|
||||||
|
}
|
||||||
|
differences := make([]string, 0, len(fields))
|
||||||
|
for _, field := range fields {
|
||||||
|
if field.left != field.right {
|
||||||
|
differences = append(differences, fmt.Sprintf("%s (%s %s; %s %s)", field.name, sourceLabel(left.Source), conflictValue(field.left), sourceLabel(right.Source), conflictValue(field.right)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(differences) == 0 {
|
||||||
|
return "unrecorded transaction details"
|
||||||
|
}
|
||||||
|
return strings.Join(differences, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
// NormalizeAndDedupe returns new records without mutating the input. Stable bank
|
// NormalizeAndDedupe returns new records without mutating the input. Stable bank
|
||||||
// entry references, scoped by account, source and debit/credit direction, take
|
// entry references, scoped by account, source and debit/credit direction, take
|
||||||
@@ -62,16 +106,18 @@ func NormalizeAndDedupe(data domain.Dataset, incoming []domain.Facts) ([]domain.
|
|||||||
existing := map[string]map[string]int{}
|
existing := map[string]map[string]int{}
|
||||||
existingAnonymous := map[string]int{}
|
existingAnonymous := map[string]int{}
|
||||||
existingIDs := map[string]domain.Facts{}
|
existingIDs := map[string]domain.Facts{}
|
||||||
loose := map[string]map[string]map[string]bool{}
|
loose := map[string]map[string]map[string]domain.Facts{}
|
||||||
addLoose := func(f domain.Facts, fp string) {
|
addLoose := func(f domain.Facts, fp string) {
|
||||||
k := looseFingerprint(f)
|
k := looseFingerprint(f)
|
||||||
if loose[k] == nil {
|
if loose[k] == nil {
|
||||||
loose[k] = map[string]map[string]bool{}
|
loose[k] = map[string]map[string]domain.Facts{}
|
||||||
}
|
}
|
||||||
if loose[k][f.Source] == nil {
|
if loose[k][f.Source] == nil {
|
||||||
loose[k][f.Source] = map[string]bool{}
|
loose[k][f.Source] = map[string]domain.Facts{}
|
||||||
|
}
|
||||||
|
if _, exists := loose[k][f.Source][fp]; !exists {
|
||||||
|
loose[k][f.Source][fp] = f
|
||||||
}
|
}
|
||||||
loose[k][f.Source][fp] = true
|
|
||||||
}
|
}
|
||||||
for _, t := range data.Transactions {
|
for _, t := range data.Transactions {
|
||||||
f, err := normalizeFacts(t.Facts, accounts)
|
f, err := normalizeFacts(t.Facts, accounts)
|
||||||
@@ -127,9 +173,9 @@ func NormalizeAndDedupe(data domain.Dataset, incoming []domain.Facts) ([]domain.
|
|||||||
for _, f := range g.facts {
|
for _, f := range g.facts {
|
||||||
for source, fps := range loose[looseFingerprint(f)] {
|
for source, fps := range loose[looseFingerprint(f)] {
|
||||||
if source != g.source {
|
if source != g.source {
|
||||||
for fp := range fps {
|
for fp, other := range fps {
|
||||||
if fp != g.fp {
|
if fp != g.fp {
|
||||||
return nil, fmt.Errorf("uncertain cross-source match on account %s at %s; reconcile differing bank/CSV records before importing", f.AccountID, f.BookingDate)
|
return nil, fmt.Errorf("%s and %s transactions overlap on %s for %s %s but differ in %s. Finance Duck cannot tell whether they are one transaction or two; to prevent double counting, nothing was imported. Compare both records, then correct or remove the duplicate before retrying", sourceLabel(f.Source), sourceLabel(other.Source), f.BookingDate, f.Amount, f.Currency, crossSourceDetails(f, other))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -241,8 +241,9 @@ func TestCrossSourceExactMatchAndUncertainty(t *testing.T) {
|
|||||||
}
|
}
|
||||||
api.RawDescription = "Different bank text"
|
api.RawDescription = "Different bank text"
|
||||||
matched, err = NormalizeAndDedupe(d, []domain.Facts{api})
|
matched, err = NormalizeAndDedupe(d, []domain.Facts{api})
|
||||||
if err == nil || matched != nil {
|
const wantConflict = "bank-synced and CSV transactions overlap on 2026-09-01 for -12.30 EUR but differ in description (bank-synced \"Different bank text\"; CSV \"Lunch\"). Finance Duck cannot tell whether they are one transaction or two; to prevent double counting, nothing was imported. Compare both records, then correct or remove the duplicate before retrying"
|
||||||
t.Fatal("uncertain overlap was silently counted")
|
if err == nil || err.Error() != wantConflict || matched != nil {
|
||||||
|
t.Fatalf("unexpected cross-source conflict: %v %+v", err, matched)
|
||||||
}
|
}
|
||||||
api.RawDescription = csv.RawDescription
|
api.RawDescription = csv.RawDescription
|
||||||
b := api
|
b := api
|
||||||
|
|||||||
Reference in New Issue
Block a user