Import ING and Kontist statements behind a reviewed column mapping
CSV import is now mapping-driven: N26, ING (metadata preamble, Windows-1252, German decimals) and Kontist exports are recognized locally, and any other layout can have its columns proposed by the configured model from a sample in which letters are replaced by x and digits by 0. Proposals are untrusted: every column must name a supplied header, money must come from one signed column or one debit/credit pair, and formats must be from a closed list. Uploading no longer imports. /api/import is replaced by prepare/confirm/cancel: prepare parses, deduplicates and previews the exact facts, and only confirming at the reviewed revision writes them. ING and AI-mapped facts carry no transaction reference, because repeating SEPA mandate references must never become a transaction identity.
This commit is contained in:
Lars Nolden
+677
-126
@@ -1,159 +1,715 @@
|
||||
package banking
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/csv"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"finance-duck/internal/domain"
|
||||
)
|
||||
|
||||
// ParseCSV accepts N26 English and German account-activity exports, including
|
||||
// their older Date/Datum and newer Booking Date/Buchungsdatum schemas. Supported
|
||||
// columns: Date/Datum/Booking Date/Buchungsdatum, Value Date/Wertstellung/
|
||||
// Wertstellungsdatum, Payee/Partner Name/Zahlungsempfänger/Empfänger/Auftraggeber,
|
||||
// Account number/Kontonummer/IBAN, Payment reference/Verwendungszweck,
|
||||
// Payment type/Transaktionstyp, Amount (EUR)/Betrag (EUR), and optional
|
||||
// Currency/Währung and Transaction ID/Transaktions-ID. Foreign-original-amount,
|
||||
// exchange-rate and category columns are deliberately not used for account money.
|
||||
// Comma and semicolon delimiters, UTF-8 BOM, CRLF, RFC4180 quoted multiline
|
||||
// descriptions, ISO and German dates, decimal comma and decimal point are accepted.
|
||||
// Missing required booking-date or account-amount columns fail the entire import.
|
||||
func ParseCSV(input io.Reader, account domain.Account) ([]domain.Facts, error) {
|
||||
if account.ID == "" {
|
||||
return nil, fmt.Errorf("CSV requires a selected account")
|
||||
}
|
||||
reader := bufio.NewReader(input)
|
||||
first, err := reader.ReadString('\n')
|
||||
if err != nil && err != io.EOF {
|
||||
return nil, fmt.Errorf("read CSV header: %w", err)
|
||||
}
|
||||
first = strings.TrimPrefix(first, "\ufeff")
|
||||
delimiter := ','
|
||||
// Count separators outside quotes; descriptions may contain either delimiter.
|
||||
quoted := false
|
||||
commas, semicolons := 0, 0
|
||||
for _, r := range first {
|
||||
if r == '"' {
|
||||
quoted = !quoted
|
||||
}
|
||||
if !quoted {
|
||||
if r == ',' {
|
||||
commas++
|
||||
}
|
||||
if r == ';' {
|
||||
semicolons++
|
||||
}
|
||||
}
|
||||
}
|
||||
if semicolons > commas {
|
||||
delimiter = ';'
|
||||
}
|
||||
parser := csv.NewReader(io.MultiReader(strings.NewReader(first), reader))
|
||||
parser.Comma = delimiter
|
||||
headers, err := parser.Read()
|
||||
// maxCSVBytes bounds an uploaded statement. Statements are held in memory so a
|
||||
// mapping can be proposed, previewed and confirmed without re-uploading.
|
||||
const maxCSVBytes = 2 << 20
|
||||
|
||||
const maxCSVColumns = 128
|
||||
const maxCSVPreambleRows = 50
|
||||
const maxCSVSampleRows = 4
|
||||
|
||||
// CSVFile is a parsed CSV document: its detected delimiter and every record,
|
||||
// including the bank preamble records that precede the column header. Blank
|
||||
// lines are not records, so indexes count parsed records, not file lines.
|
||||
type CSVFile struct {
|
||||
delimiter rune
|
||||
rows [][]string
|
||||
}
|
||||
|
||||
// CSVMapping assigns source columns, by their exact header text, to bank facts.
|
||||
// Money comes either from one signed AmountColumn or from separate
|
||||
// DebitColumn/CreditColumn pairs, never from both. FixedCurrency records a
|
||||
// currency carried by a header such as "Amount (EUR)" rather than a column.
|
||||
type CSVMapping struct {
|
||||
// HeaderRow is the 1-based parsed record holding the column names.
|
||||
HeaderRow int `json:"header_row"`
|
||||
BookingDateColumn string `json:"booking_date_column"`
|
||||
ValueDateColumn string `json:"value_date_column,omitempty"`
|
||||
AmountColumn string `json:"amount_column,omitempty"`
|
||||
DebitColumn string `json:"debit_column,omitempty"`
|
||||
CreditColumn string `json:"credit_column,omitempty"`
|
||||
CurrencyColumn string `json:"currency_column,omitempty"`
|
||||
DescriptionColumn string `json:"description_column"`
|
||||
FallbackDescriptionColumn string `json:"fallback_description_column,omitempty"`
|
||||
CounterpartyColumn string `json:"counterparty_column,omitempty"`
|
||||
CounterpartyIBANColumn string `json:"counterparty_iban_column,omitempty"`
|
||||
ExternalIDColumn string `json:"external_id_column,omitempty"`
|
||||
DateFormat string `json:"date_format"`
|
||||
DecimalFormat string `json:"decimal_format"`
|
||||
FixedCurrency string `json:"fixed_currency,omitempty"`
|
||||
}
|
||||
|
||||
// CSVSample describes a statement's shape for column mapping. Cell values are
|
||||
// replaced by their character shape: no account text, name, reference or amount
|
||||
// digit is retained.
|
||||
type CSVSample struct {
|
||||
Delimiter string `json:"delimiter"`
|
||||
HeaderRow int `json:"header_row"`
|
||||
Headers []string `json:"headers"`
|
||||
ShapedRows [][]string `json:"shaped_rows"`
|
||||
RecordCount int `json:"record_count"`
|
||||
}
|
||||
|
||||
// ReadCSV decodes an uploaded statement. UTF-8 and Windows-1252 (still emitted
|
||||
// by some ING exports) are accepted, along with a BOM, CRLF, comma/semicolon/tab
|
||||
// delimiters and RFC4180 quoted multiline fields.
|
||||
func ReadCSV(input io.Reader) (CSVFile, error) {
|
||||
raw, err := io.ReadAll(io.LimitReader(input, maxCSVBytes+1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid N26 CSV header")
|
||||
return CSVFile{}, fmt.Errorf("read CSV: %w", err)
|
||||
}
|
||||
columns := make(map[string]int)
|
||||
amountCurrency := ""
|
||||
for i, h := range headers {
|
||||
name := headerName(h)
|
||||
key := ""
|
||||
switch name {
|
||||
case "date", "datum", "booking date", "buchungsdatum":
|
||||
key = "date"
|
||||
case "value date", "wertstellung", "wertstellungsdatum", "valutadatum":
|
||||
key = "value"
|
||||
case "payee", "partner name", "zahlungsempfänger", "zahlungsempfänger name", "empfänger", "empfänger/auftraggeber", "partnername", "name zahlungspartner":
|
||||
key = "party"
|
||||
case "account number", "partner iban", "kontonummer", "iban", "konto":
|
||||
key = "iban"
|
||||
case "payment reference", "verwendungszweck", "reference", "beschreibung":
|
||||
key = "description"
|
||||
case "payment type", "transaktionstyp", "zahlungstyp", "type", "typ":
|
||||
key = "type"
|
||||
case "currency", "währung":
|
||||
key = "currency"
|
||||
case "transaction id", "transaktions-id", "transaktions id":
|
||||
key = "external"
|
||||
case "amount", "betrag":
|
||||
key = "amount"
|
||||
if len(raw) > maxCSVBytes {
|
||||
return CSVFile{}, fmt.Errorf("CSV statement exceeds %d MiB", maxCSVBytes>>20)
|
||||
}
|
||||
text, err := decodeCSVText(raw)
|
||||
if err != nil {
|
||||
return CSVFile{}, err
|
||||
}
|
||||
if strings.TrimSpace(text) == "" {
|
||||
return CSVFile{}, errors.New("CSV statement is empty")
|
||||
}
|
||||
// Choose the delimiter that yields the widest consistently parsed records:
|
||||
// descriptions routinely contain the delimiters used by other banks.
|
||||
delimiter, best := ' ', -1
|
||||
for _, candidate := range []rune{',', ';', '\t'} {
|
||||
if _, score := parseCSVRecords(text, candidate); score > best {
|
||||
delimiter, best = candidate, score
|
||||
}
|
||||
}
|
||||
if best <= 0 {
|
||||
return CSVFile{}, errors.New("unreadable CSV statement: no delimiter produced multi-column records")
|
||||
}
|
||||
rows, _ := parseCSVRecords(text, delimiter)
|
||||
for _, row := range rows {
|
||||
if len(row) > maxCSVColumns {
|
||||
return CSVFile{}, fmt.Errorf("CSV statement has more than %d columns", maxCSVColumns)
|
||||
}
|
||||
}
|
||||
return CSVFile{delimiter: delimiter, rows: rows}, nil
|
||||
}
|
||||
|
||||
func decodeCSVText(raw []byte) (string, error) {
|
||||
text := strings.TrimPrefix(string(raw), "\ufeff")
|
||||
if strings.IndexByte(text, 0) >= 0 {
|
||||
return "", errors.New("CSV statement contains a NUL byte")
|
||||
}
|
||||
if utf8.ValidString(text) {
|
||||
return text, nil
|
||||
}
|
||||
// Windows-1252 is decoded locally: undecodable account text must never be
|
||||
// forwarded to a model or stored as invalid UTF-8 bank facts.
|
||||
windows1252 := [...]rune{'€', 0, '‚', 'ƒ', '„', '…', '†', '‡', 'ˆ', '‰', 'Š', '‹', 'Œ', 0, 'Ž', 0, 0, '‘', '’', '“', '”', '•', '–', '—', '˜', '™', 'š', '›', 'œ', 0, 'ž', 'Ÿ'}
|
||||
var out strings.Builder
|
||||
out.Grow(len(raw))
|
||||
for _, b := range []byte(text) {
|
||||
switch {
|
||||
case b < 0x80 || b >= 0xa0:
|
||||
out.WriteRune(rune(b))
|
||||
default:
|
||||
for _, prefix := range []string{"amount (", "betrag ("} {
|
||||
if strings.HasPrefix(name, prefix) && strings.HasSuffix(name, ")") {
|
||||
candidate := strings.ToUpper(strings.TrimSuffix(strings.TrimPrefix(name, prefix), ")"))
|
||||
if validCurrency(candidate) {
|
||||
key = "amount"
|
||||
amountCurrency = candidate
|
||||
}
|
||||
}
|
||||
r := windows1252[int(b)-0x80]
|
||||
if r == 0 {
|
||||
return "", errors.New("CSV statement is neither valid UTF-8 nor Windows-1252")
|
||||
}
|
||||
}
|
||||
if key != "" {
|
||||
if _, exists := columns[key]; exists {
|
||||
return nil, fmt.Errorf("duplicate N26 CSV column %s", key)
|
||||
}
|
||||
columns[key] = i
|
||||
out.WriteRune(r)
|
||||
}
|
||||
}
|
||||
if _, ok := columns["date"]; !ok {
|
||||
return nil, fmt.Errorf("N26 CSV requires Date/Datum or Booking Date/Buchungsdatum")
|
||||
}
|
||||
if _, ok := columns["amount"]; !ok {
|
||||
return nil, fmt.Errorf("N26 CSV requires Amount (currency)/Betrag (currency)")
|
||||
}
|
||||
get := func(row []string, key string) string {
|
||||
if i, ok := columns[key]; ok {
|
||||
return strings.TrimSpace(row[i])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
result := make([]domain.Facts, 0)
|
||||
for rowNumber := 2; ; rowNumber++ {
|
||||
return out.String(), nil
|
||||
}
|
||||
|
||||
// parseCSVRecords returns every record plus a preference score. Malformed
|
||||
// quoting scores below zero so a delimiter is never silently accepted for a
|
||||
// document it cannot represent.
|
||||
func parseCSVRecords(text string, delimiter rune) ([][]string, int) {
|
||||
parser := csv.NewReader(strings.NewReader(text))
|
||||
parser.Comma = delimiter
|
||||
parser.FieldsPerRecord = -1
|
||||
parser.ReuseRecord = false
|
||||
rows := make([][]string, 0, 64)
|
||||
widest, populated := 0, 0
|
||||
for {
|
||||
row, err := parser.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid N26 CSV record %d", rowNumber)
|
||||
return nil, -1
|
||||
}
|
||||
date, err := parseDate(get(row, "date"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid booking date in CSV record %d", rowNumber)
|
||||
}
|
||||
value := get(row, "value")
|
||||
if value != "" {
|
||||
value, err = parseDate(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid value date in CSV record %d", rowNumber)
|
||||
rows = append(rows, row)
|
||||
if len(row) > 1 && !blankCSVRow(row) {
|
||||
populated++
|
||||
if len(row) > widest {
|
||||
widest = len(row)
|
||||
}
|
||||
}
|
||||
amount, err := parseCSVAmount(get(row, "amount"))
|
||||
}
|
||||
if widest == 0 {
|
||||
return rows, 0
|
||||
}
|
||||
return rows, widest*1_000_000 + populated
|
||||
}
|
||||
|
||||
func blankCSVRow(row []string) bool {
|
||||
for _, value := range row {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Sample locates the column header and describes the statement's shape. The
|
||||
// header is the first widest record, which skips the metadata preamble that
|
||||
// banks such as ING place above their columns.
|
||||
func (f CSVFile) Sample() (CSVSample, error) {
|
||||
header, width := 0, 0
|
||||
for i, row := range f.rows {
|
||||
if i >= maxCSVPreambleRows {
|
||||
break
|
||||
}
|
||||
if len(row) > width && !blankCSVRow(row) {
|
||||
header, width = i+1, len(row)
|
||||
}
|
||||
}
|
||||
if width < 2 {
|
||||
return CSVSample{}, errors.New("CSV statement has no column header row")
|
||||
}
|
||||
headers, err := csvHeaders(f, header)
|
||||
if err != nil {
|
||||
return CSVSample{}, err
|
||||
}
|
||||
sample := CSVSample{Delimiter: string(f.delimiter), HeaderRow: header, Headers: headers, ShapedRows: [][]string{}}
|
||||
for _, row := range f.rows[header:] {
|
||||
if blankCSVRow(row) {
|
||||
continue
|
||||
}
|
||||
if len(row) != width {
|
||||
return CSVSample{}, fmt.Errorf("CSV statement has ragged records: expected %d columns", width)
|
||||
}
|
||||
sample.RecordCount++
|
||||
if len(sample.ShapedRows) < maxCSVSampleRows {
|
||||
shaped := make([]string, len(row))
|
||||
for i, value := range row {
|
||||
shaped[i] = shapeCSVValue(value)
|
||||
}
|
||||
sample.ShapedRows = append(sample.ShapedRows, shaped)
|
||||
}
|
||||
}
|
||||
if sample.RecordCount == 0 {
|
||||
return CSVSample{}, errors.New("CSV statement contains no transaction records")
|
||||
}
|
||||
return sample, nil
|
||||
}
|
||||
|
||||
func csvHeaders(f CSVFile, header int) ([]string, error) {
|
||||
if header < 1 || header > len(f.rows) {
|
||||
return nil, errors.New("CSV mapping has an invalid header row")
|
||||
}
|
||||
headers := make([]string, 0, len(f.rows[header-1]))
|
||||
seen := map[string]bool{}
|
||||
for _, raw := range f.rows[header-1] {
|
||||
name := strings.TrimSpace(raw)
|
||||
key := headerName(name)
|
||||
if key == "" {
|
||||
return nil, errors.New("CSV column headers must all be named")
|
||||
}
|
||||
if seen[key] {
|
||||
return nil, fmt.Errorf("duplicate CSV column %q", name)
|
||||
}
|
||||
seen[key] = true
|
||||
headers = append(headers, name)
|
||||
}
|
||||
return headers, nil
|
||||
}
|
||||
|
||||
// shapeCSVValue keeps only structure: letters become x, digits 0. This is what a
|
||||
// column mapping needs, and it keeps descriptions, names, references, IBANs and
|
||||
// amounts out of any request that leaves this machine.
|
||||
func shapeCSVValue(value string) string {
|
||||
var out strings.Builder
|
||||
for i, r := range strings.TrimSpace(value) {
|
||||
if i >= 64 {
|
||||
out.WriteRune('…')
|
||||
break
|
||||
}
|
||||
switch {
|
||||
case unicode.IsLetter(r):
|
||||
out.WriteRune('x')
|
||||
case unicode.IsDigit(r):
|
||||
out.WriteRune('0')
|
||||
case unicode.IsSpace(r):
|
||||
out.WriteRune(' ')
|
||||
default:
|
||||
out.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// DetectCSVMapping recognizes known bank exports without any model. The returned
|
||||
// source identifies imported facts; label names the export for the operator.
|
||||
func DetectCSVMapping(f CSVFile) (mapping CSVMapping, source, label string, ok bool) {
|
||||
for i, row := range f.rows {
|
||||
if i >= maxCSVPreambleRows {
|
||||
break
|
||||
}
|
||||
columns, usable := csvColumnIndex(row)
|
||||
if !usable {
|
||||
continue
|
||||
}
|
||||
if mapping, ok := n26Mapping(i+1, columns); ok {
|
||||
return mapping, "n26_csv", "N26", true
|
||||
}
|
||||
if mapping, ok := ingMapping(i+1, columns); ok {
|
||||
return mapping, "ing_csv", "ING", true
|
||||
}
|
||||
if mapping, ok := kontistMapping(f, i+1, columns); ok {
|
||||
return mapping, "kontist_csv", "Kontist", true
|
||||
}
|
||||
}
|
||||
return CSVMapping{}, "", "", false
|
||||
}
|
||||
|
||||
// csvColumnIndex maps normalized column names to their exact header text.
|
||||
func csvColumnIndex(row []string) (map[string]string, bool) {
|
||||
columns := make(map[string]string, len(row))
|
||||
for _, raw := range row {
|
||||
actual := strings.TrimSpace(raw)
|
||||
name := headerName(actual)
|
||||
if name == "" {
|
||||
return nil, false
|
||||
}
|
||||
if _, exists := columns[name]; exists {
|
||||
return nil, false
|
||||
}
|
||||
columns[name] = actual
|
||||
}
|
||||
return columns, len(columns) > 1
|
||||
}
|
||||
|
||||
func csvColumn(columns map[string]string, names ...string) string {
|
||||
for _, name := range names {
|
||||
if actual, ok := columns[name]; ok {
|
||||
return actual
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// n26Mapping accepts N26's English and German account-activity exports,
|
||||
// including their older Date/Datum and newer Booking Date/Buchungsdatum
|
||||
// schemas. Foreign original amounts, exchange rates and categories are
|
||||
// deliberately never used as account money.
|
||||
func n26Mapping(header int, columns map[string]string) (CSVMapping, bool) {
|
||||
amount, currency := "", ""
|
||||
for name, actual := range columns {
|
||||
if name == "amount" || name == "betrag" {
|
||||
amount = actual
|
||||
continue
|
||||
}
|
||||
for _, prefix := range []string{"amount (", "betrag ("} {
|
||||
if strings.HasPrefix(name, prefix) && strings.HasSuffix(name, ")") {
|
||||
code := strings.ToUpper(strings.TrimSuffix(strings.TrimPrefix(name, prefix), ")"))
|
||||
if validCurrency(code) {
|
||||
amount, currency = actual, code
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
mapping := CSVMapping{
|
||||
HeaderRow: header,
|
||||
BookingDateColumn: csvColumn(columns, "date", "datum", "booking date", "buchungsdatum"),
|
||||
ValueDateColumn: csvColumn(columns, "value date", "wertstellung", "wertstellungsdatum", "valutadatum"),
|
||||
AmountColumn: amount,
|
||||
CurrencyColumn: csvColumn(columns, "currency", "währung"),
|
||||
DescriptionColumn: csvColumn(columns, "payment reference", "verwendungszweck", "reference", "beschreibung"),
|
||||
FallbackDescriptionColumn: csvColumn(columns, "payment type", "transaktionstyp", "zahlungstyp", "type", "typ"),
|
||||
CounterpartyColumn: csvColumn(columns, "payee", "partner name", "zahlungsempfänger", "zahlungsempfänger name", "empfänger", "empfänger/auftraggeber", "partnername", "name zahlungspartner"),
|
||||
CounterpartyIBANColumn: csvColumn(columns, "account number", "partner iban", "kontonummer", "iban", "konto"),
|
||||
ExternalIDColumn: csvColumn(columns, "transaction id", "transaktions-id", "transaktions id"),
|
||||
DateFormat: "iso-or-german",
|
||||
DecimalFormat: "dot-or-comma",
|
||||
FixedCurrency: currency,
|
||||
}
|
||||
// An N26 export always carries a payment reference and a typed transaction.
|
||||
marker := mapping.FallbackDescriptionColumn != "" || csvColumn(columns, "original amount", "betrag (fremdwährung)", "account name", "partner name", "payee") != ""
|
||||
if mapping.BookingDateColumn == "" || mapping.AmountColumn == "" || mapping.DescriptionColumn == "" || !marker {
|
||||
return CSVMapping{}, false
|
||||
}
|
||||
return mapping, true
|
||||
}
|
||||
|
||||
// ingMapping accepts ING's Umsatzanzeige export, whose columns sit below a
|
||||
// metadata preamble. Gläubiger-ID, Mandatsreferenz and Kundenreferenz are SEPA
|
||||
// mandate references that repeat across bookings, so they are never used as a
|
||||
// transaction identity.
|
||||
func ingMapping(header int, columns map[string]string) (CSVMapping, bool) {
|
||||
mapping := CSVMapping{
|
||||
HeaderRow: header,
|
||||
BookingDateColumn: csvColumn(columns, "buchung"),
|
||||
ValueDateColumn: csvColumn(columns, "wertstellungsdatum", "valuta"),
|
||||
AmountColumn: csvColumn(columns, "betrag"),
|
||||
CurrencyColumn: csvColumn(columns, "währung", "waehrung"),
|
||||
DescriptionColumn: csvColumn(columns, "verwendungszweck"),
|
||||
FallbackDescriptionColumn: csvColumn(columns, "buchungstext"),
|
||||
CounterpartyColumn: csvColumn(columns, "auftraggeber/empfänger", "auftraggeber/empfaenger"),
|
||||
DateFormat: "dd.mm.yyyy",
|
||||
DecimalFormat: "comma",
|
||||
}
|
||||
if mapping.BookingDateColumn == "" || mapping.AmountColumn == "" || mapping.DescriptionColumn == "" || mapping.CounterpartyColumn == "" {
|
||||
return CSVMapping{}, false
|
||||
}
|
||||
return mapping, true
|
||||
}
|
||||
|
||||
// kontistMapping accepts Kontist's documented transaction vocabulary: a payment
|
||||
// date, an amount, a purpose and a counterparty name. Date and decimal
|
||||
// conventions are inferred from the file's own first populated values, and the
|
||||
// mapping is always reviewed before anything is imported.
|
||||
func kontistMapping(f CSVFile, header int, columns map[string]string) (CSVMapping, bool) {
|
||||
mapping := CSVMapping{
|
||||
HeaderRow: header,
|
||||
BookingDateColumn: csvColumn(columns, "payment date", "booking date", "buchungsdatum", "zahlungsdatum"),
|
||||
ValueDateColumn: csvColumn(columns, "value date", "wertstellungsdatum", "valuta"),
|
||||
AmountColumn: csvColumn(columns, "amount", "betrag"),
|
||||
CurrencyColumn: csvColumn(columns, "currency", "währung", "waehrung"),
|
||||
DescriptionColumn: csvColumn(columns, "purpose", "verwendungszweck", "payment reference"),
|
||||
CounterpartyColumn: csvColumn(columns, "name", "counterparty", "zahlungspartner"),
|
||||
CounterpartyIBANColumn: csvColumn(columns, "iban", "counterparty iban"),
|
||||
ExternalIDColumn: csvColumn(columns, "transaction id", "transaction_id", "transaktions-id"),
|
||||
}
|
||||
if mapping.BookingDateColumn == "" || mapping.AmountColumn == "" || mapping.DescriptionColumn == "" || mapping.CounterpartyColumn == "" {
|
||||
return CSVMapping{}, false
|
||||
}
|
||||
mapping.DateFormat = inferCSVDateFormat(csvFirstValue(f, header, mapping.BookingDateColumn))
|
||||
mapping.DecimalFormat = inferCSVDecimalFormat(csvFirstValue(f, header, mapping.AmountColumn))
|
||||
return mapping, true
|
||||
}
|
||||
|
||||
func csvFirstValue(f CSVFile, header int, column string) string {
|
||||
headers := f.rows[header-1]
|
||||
index := -1
|
||||
for i, name := range headers {
|
||||
if strings.TrimSpace(name) == column {
|
||||
index = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if index < 0 {
|
||||
return ""
|
||||
}
|
||||
for _, row := range f.rows[header:] {
|
||||
if len(row) == len(headers) && strings.TrimSpace(row[index]) != "" {
|
||||
return strings.TrimSpace(row[index])
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func inferCSVDateFormat(value string) string {
|
||||
switch {
|
||||
case len(value) >= 11 && value[4] == '-' && (value[10] == 'T' || value[10] == ' '):
|
||||
return "iso-date-time"
|
||||
case len(value) == 10 && value[4] == '-' && value[7] == '-':
|
||||
return "yyyy-mm-dd"
|
||||
case strings.Count(value, ".") == 2:
|
||||
return "dd.mm.yyyy"
|
||||
case strings.Count(value, "/") == 2:
|
||||
// Kontist documents month/day/year for interchange. A first component
|
||||
// above twelve can only be a day; the preview shows the parsed result.
|
||||
if first, err := strconv.Atoi(strings.SplitN(value, "/", 2)[0]); err == nil && first > 12 {
|
||||
return "dd/mm/yyyy"
|
||||
}
|
||||
return "mm/dd/yyyy"
|
||||
default:
|
||||
return "yyyy-mm-dd"
|
||||
}
|
||||
}
|
||||
|
||||
func inferCSVDecimalFormat(value string) string {
|
||||
if strings.Contains(value, ",") {
|
||||
return "comma"
|
||||
}
|
||||
return "dot"
|
||||
}
|
||||
|
||||
// ParseMappedCSV converts every record into bank facts. A single malformed
|
||||
// record fails the whole statement: a partially imported statement cannot be
|
||||
// distinguished from a truncated export later.
|
||||
func ParseMappedCSV(f CSVFile, account domain.Account, mapping CSVMapping, source string) ([]domain.Facts, error) {
|
||||
if account.ID == "" {
|
||||
return nil, errors.New("CSV requires a selected account")
|
||||
}
|
||||
if source == "" {
|
||||
return nil, errors.New("CSV import source is required")
|
||||
}
|
||||
columns, err := validateCSVMapping(f, mapping)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
headers := f.rows[mapping.HeaderRow-1]
|
||||
get := func(row []string, column string) string {
|
||||
if column == "" {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(row[columns[column]])
|
||||
}
|
||||
facts := make([]domain.Facts, 0, len(f.rows)-mapping.HeaderRow)
|
||||
for offset, row := range f.rows[mapping.HeaderRow:] {
|
||||
record := mapping.HeaderRow + offset + 1
|
||||
if blankCSVRow(row) {
|
||||
continue
|
||||
}
|
||||
if len(row) != len(headers) {
|
||||
return nil, fmt.Errorf("CSV record %d has %d columns, expected %d", record, len(row), len(headers))
|
||||
}
|
||||
booking, err := parseMappedCSVDate(get(row, mapping.BookingDateColumn), mapping.DateFormat)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid account amount in CSV record %d", rowNumber)
|
||||
return nil, fmt.Errorf("invalid booking date in CSV record %d", record)
|
||||
}
|
||||
value := get(row, mapping.ValueDateColumn)
|
||||
if value != "" {
|
||||
if value, err = parseMappedCSVDate(value, mapping.DateFormat); err != nil {
|
||||
return nil, fmt.Errorf("invalid value date in CSV record %d", record)
|
||||
}
|
||||
}
|
||||
amount, err := mappedCSVAmount(row, get, mapping)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid account amount in CSV record %d", record)
|
||||
}
|
||||
currency := strings.ToUpper(get(row, mapping.CurrencyColumn))
|
||||
if currency == "€" {
|
||||
currency = "EUR"
|
||||
}
|
||||
currency := strings.ToUpper(get(row, "currency"))
|
||||
if currency == "" {
|
||||
currency = amountCurrency
|
||||
currency = strings.ToUpper(mapping.FixedCurrency)
|
||||
}
|
||||
if currency == "" {
|
||||
currency = strings.ToUpper(account.Currency)
|
||||
}
|
||||
if !validCurrency(currency) || (amountCurrency != "" && currency != amountCurrency) || (account.Currency != "" && currency != strings.ToUpper(account.Currency)) {
|
||||
return nil, fmt.Errorf("invalid or conflicting account currency in CSV record %d", rowNumber)
|
||||
fixed := strings.ToUpper(mapping.FixedCurrency)
|
||||
if !validCurrency(currency) || (fixed != "" && currency != fixed) || (account.Currency != "" && currency != strings.ToUpper(account.Currency)) {
|
||||
return nil, fmt.Errorf("invalid or conflicting account currency in CSV record %d", record)
|
||||
}
|
||||
description := get(row, "description")
|
||||
description := get(row, mapping.DescriptionColumn)
|
||||
if description == "" {
|
||||
description = get(row, "type")
|
||||
description = get(row, mapping.FallbackDescriptionColumn)
|
||||
}
|
||||
result = append(result, domain.Facts{Source: "n26_csv", AccountID: account.ID, BookingDate: date, ValueDate: value, Amount: amount, Currency: currency, RawDescription: description, ExternalID: get(row, "external"), Counterparty: get(row, "party"), CounterpartyIBAN: normalizeIBAN(get(row, "iban"))})
|
||||
facts = append(facts, domain.Facts{
|
||||
Source: source, AccountID: account.ID, BookingDate: booking, ValueDate: value,
|
||||
Amount: amount, Currency: currency, RawDescription: description,
|
||||
ExternalID: get(row, mapping.ExternalIDColumn), Counterparty: get(row, mapping.CounterpartyColumn),
|
||||
CounterpartyIBAN: normalizeIBAN(get(row, mapping.CounterpartyIBANColumn)),
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
if len(facts) == 0 {
|
||||
return nil, errors.New("CSV statement contains no transaction records")
|
||||
}
|
||||
return facts, nil
|
||||
}
|
||||
|
||||
// validateCSVMapping resolves a mapping against the document and returns each
|
||||
// mapped column's index. Every referenced column must exist exactly, no column
|
||||
// may serve two fields, and money must come from exactly one strategy.
|
||||
func validateCSVMapping(f CSVFile, mapping CSVMapping) (map[string]int, error) {
|
||||
if mapping.HeaderRow < 1 || mapping.HeaderRow > len(f.rows) || mapping.HeaderRow > maxCSVPreambleRows {
|
||||
return nil, errors.New("CSV mapping has an invalid header row")
|
||||
}
|
||||
if _, err := csvHeaders(f, mapping.HeaderRow); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
indexes := make(map[string]int, len(f.rows[mapping.HeaderRow-1]))
|
||||
for i, raw := range f.rows[mapping.HeaderRow-1] {
|
||||
indexes[strings.TrimSpace(raw)] = i
|
||||
}
|
||||
if mapping.BookingDateColumn == "" {
|
||||
return nil, errors.New("CSV mapping requires a booking date column")
|
||||
}
|
||||
if mapping.DescriptionColumn == "" {
|
||||
return nil, errors.New("CSV mapping requires a description column")
|
||||
}
|
||||
signed, split := mapping.AmountColumn != "", mapping.DebitColumn != "" || mapping.CreditColumn != ""
|
||||
if signed == split {
|
||||
return nil, errors.New("CSV mapping requires either one signed amount column or separate debit and credit columns")
|
||||
}
|
||||
if split && (mapping.DebitColumn == "" || mapping.CreditColumn == "") {
|
||||
return nil, errors.New("CSV mapping requires both a debit and a credit column")
|
||||
}
|
||||
if !validCSVDateFormat(mapping.DateFormat) {
|
||||
return nil, fmt.Errorf("unsupported CSV date format %q", mapping.DateFormat)
|
||||
}
|
||||
if !validCSVDecimalFormat(mapping.DecimalFormat) {
|
||||
return nil, fmt.Errorf("unsupported CSV decimal format %q", mapping.DecimalFormat)
|
||||
}
|
||||
if mapping.FixedCurrency != "" && !validCurrency(strings.ToUpper(mapping.FixedCurrency)) {
|
||||
return nil, errors.New("CSV mapping has an invalid fixed currency")
|
||||
}
|
||||
assigned := map[string]string{}
|
||||
for _, field := range []struct{ name, column string }{
|
||||
{"booking date", mapping.BookingDateColumn}, {"value date", mapping.ValueDateColumn},
|
||||
{"amount", mapping.AmountColumn}, {"debit", mapping.DebitColumn}, {"credit", mapping.CreditColumn},
|
||||
{"currency", mapping.CurrencyColumn}, {"description", mapping.DescriptionColumn},
|
||||
{"secondary description", mapping.FallbackDescriptionColumn}, {"counterparty", mapping.CounterpartyColumn},
|
||||
{"counterparty IBAN", mapping.CounterpartyIBANColumn}, {"transaction reference", mapping.ExternalIDColumn},
|
||||
} {
|
||||
if field.column == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := indexes[field.column]; !ok {
|
||||
return nil, fmt.Errorf("CSV mapping references unknown %s column %q", field.name, field.column)
|
||||
}
|
||||
if previous, ok := assigned[field.column]; ok {
|
||||
return nil, fmt.Errorf("CSV column %q is mapped to both %s and %s", field.column, previous, field.name)
|
||||
}
|
||||
assigned[field.column] = field.name
|
||||
}
|
||||
return indexes, nil
|
||||
}
|
||||
|
||||
// CSVDateFormats and CSVDecimalFormats are the exact accepted conventions. A
|
||||
// proposed mapping outside them is rejected rather than guessed.
|
||||
func CSVDateFormats() []string {
|
||||
return []string{"yyyy-mm-dd", "dd.mm.yyyy", "mm/dd/yyyy", "dd/mm/yyyy", "iso-date-time", "iso-or-german"}
|
||||
}
|
||||
func CSVDecimalFormats() []string { return []string{"dot", "comma", "dot-or-comma"} }
|
||||
|
||||
func validCSVDateFormat(format string) bool {
|
||||
for _, valid := range CSVDateFormats() {
|
||||
if format == valid {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
func validCSVDecimalFormat(format string) bool {
|
||||
for _, valid := range CSVDecimalFormats() {
|
||||
if format == valid {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func parseMappedCSVDate(value, format string) (string, error) {
|
||||
layouts := map[string][]string{
|
||||
"yyyy-mm-dd": {"2006-01-02"},
|
||||
"dd.mm.yyyy": {"02.01.2006", "2.1.2006"},
|
||||
"mm/dd/yyyy": {"01/02/2006", "1/2/2006"},
|
||||
"dd/mm/yyyy": {"02/01/2006", "2/1/2006"},
|
||||
"iso-date-time": {time.RFC3339, "2006-01-02T15:04:05", "2006-01-02 15:04:05", "2006-01-02"},
|
||||
"iso-or-german": {"2006-01-02", "02.01.2006", "2.1.2006"},
|
||||
}
|
||||
for _, layout := range layouts[format] {
|
||||
if parsed, err := time.Parse(layout, value); err == nil {
|
||||
return parsed.Format("2006-01-02"), nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("invalid date %q", value)
|
||||
}
|
||||
|
||||
// mappedCSVAmount returns signed account money. With split columns a debit is
|
||||
// negative however the bank wrote its sign, a credit must not be negative, and
|
||||
// the unused column may be empty or an explicit zero but never carry money.
|
||||
func mappedCSVAmount(row []string, get func([]string, string) string, mapping CSVMapping) (domain.Money, error) {
|
||||
if mapping.AmountColumn != "" {
|
||||
return parseMappedCSVDecimal(get(row, mapping.AmountColumn), mapping.DecimalFormat)
|
||||
}
|
||||
debit, credit := get(row, mapping.DebitColumn), get(row, mapping.CreditColumn)
|
||||
if debit == "" && credit == "" {
|
||||
return "", errors.New("debit and credit are both empty")
|
||||
}
|
||||
parse := func(value string) (domain.Money, int64, error) {
|
||||
if value == "" {
|
||||
return "0.00", 0, nil
|
||||
}
|
||||
money, err := parseMappedCSVDecimal(value, mapping.DecimalFormat)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
minor, err := money.Minor()
|
||||
return money, minor, err
|
||||
}
|
||||
debitMoney, debited, err := parse(debit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
creditMoney, credited, err := parse(credit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if debited != 0 && credited != 0 {
|
||||
return "", errors.New("debit and credit both carry money")
|
||||
}
|
||||
if credited < 0 {
|
||||
return "", errors.New("a credit column must not hold negative money")
|
||||
}
|
||||
if credited != 0 {
|
||||
return creditMoney, nil
|
||||
}
|
||||
if debited != 0 {
|
||||
return domain.ParseMoney("-" + strings.TrimPrefix(debitMoney.String(), "-"))
|
||||
}
|
||||
return domain.ParseMoney("0")
|
||||
}
|
||||
|
||||
func parseMappedCSVDecimal(value, format string) (domain.Money, error) {
|
||||
value = strings.NewReplacer("\u00a0", "", "\u202f", "", "'", "").Replace(strings.TrimSpace(value))
|
||||
switch format {
|
||||
case "dot-or-comma":
|
||||
return parseCSVAmount(value)
|
||||
case "comma":
|
||||
// A dot can only be grouping here, and only in exact thousands groups.
|
||||
if !strings.Contains(value, ",") && strings.Contains(value, ".") {
|
||||
if digits, ok := ungroup(value, "."); ok {
|
||||
value = digits
|
||||
}
|
||||
}
|
||||
return parseCSVAmount(value)
|
||||
case "dot":
|
||||
value = strings.TrimPrefix(value, "+")
|
||||
if strings.Contains(value, ",") {
|
||||
digits, ok := ungroup(value, ",")
|
||||
if !ok {
|
||||
return "", errors.New("invalid grouping")
|
||||
}
|
||||
value = digits
|
||||
}
|
||||
return domain.ParseMoney(value)
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported CSV decimal format %q", format)
|
||||
}
|
||||
}
|
||||
|
||||
// ungroup removes thousands separators, and only when every group is exactly
|
||||
// three digits: "1.234" is 1234, while "1.23" stays a decimal value.
|
||||
func ungroup(value, separator string) (string, bool) {
|
||||
sign := ""
|
||||
if rest, found := strings.CutPrefix(value, "-"); found {
|
||||
sign, value = "-", rest
|
||||
}
|
||||
groups := strings.Split(value, separator)
|
||||
if len(groups) < 2 || len(groups[0]) < 1 || len(groups[0]) > 3 {
|
||||
return "", false
|
||||
}
|
||||
for _, group := range groups[1:] {
|
||||
if len(group) != 3 {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
return sign + strings.Join(groups, ""), true
|
||||
}
|
||||
|
||||
func headerName(s string) string {
|
||||
@@ -195,16 +751,11 @@ func parseCSVAmount(s string) (domain.Money, error) {
|
||||
}
|
||||
pair := strings.SplitN(s, ",", 2)
|
||||
if strings.Contains(pair[0], ".") {
|
||||
groups := strings.Split(strings.TrimLeft(pair[0], "+-"), ".")
|
||||
if len(groups[0]) < 1 || len(groups[0]) > 3 {
|
||||
digits, ok := ungroup(pair[0], ".")
|
||||
if !ok {
|
||||
return "", fmt.Errorf("invalid grouping")
|
||||
}
|
||||
for _, g := range groups[1:] {
|
||||
if len(g) != 3 {
|
||||
return "", fmt.Errorf("invalid grouping")
|
||||
}
|
||||
}
|
||||
pair[0] = strings.ReplaceAll(pair[0], ".", "")
|
||||
pair[0] = digits
|
||||
}
|
||||
s = pair[0] + "." + pair[1]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
package banking
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode"
|
||||
|
||||
"finance-duck/internal/domain"
|
||||
)
|
||||
|
||||
// ingExport reproduces ING's Umsatzanzeige: a metadata preamble above the
|
||||
// columns, semicolons, German dates and decimals, and Windows-1252 umlauts.
|
||||
const ingExport = "Umsatzanzeige;Datei erstellt am: 09.12.2025 15:23\n" +
|
||||
"\nIBAN;DE89370400440532013000\nKontoname;Girokonto\nBank;ING\nKunde;Max Mustermann\n" +
|
||||
"Zeitraum;17.11.2025 - 09.12.2025\nSaldo;66.331,90;EUR\n\nSortierung;Datum absteigend\n\n" +
|
||||
"In der CSV-Datei finden Sie alle bereits gebuchten Ums\xe4tze, nicht die vorgemerkten.\n\n" +
|
||||
"Buchung;Wertstellungsdatum;Auftraggeber/Empf\xe4nger;Buchungstext;Verwendungszweck;Betrag;W\xe4hrung\n" +
|
||||
"09.12.2025;08.12.2025;VISA Firma;Lastschrift;NR XXXX 4025 KAUFUMSATZ;-13,98;EUR\n" +
|
||||
"28.11.2025;28.11.2025;Rente;Gehalt/Rente;RV-RENTE 11.2025;2.647,74;EUR\n"
|
||||
|
||||
func readFixture(t *testing.T, text string) CSVFile {
|
||||
t.Helper()
|
||||
file, err := ReadCSV(strings.NewReader(text))
|
||||
if err != nil {
|
||||
t.Fatalf("read fixture: %v", err)
|
||||
}
|
||||
return file
|
||||
}
|
||||
|
||||
func TestPresetMappingsParseKnownBankExports(t *testing.T) {
|
||||
cases := []struct {
|
||||
name, csv, source, label string
|
||||
records int
|
||||
booking, value, amount string
|
||||
description, party, iban string
|
||||
}{
|
||||
{
|
||||
name: "N26 English legacy quoted multiline", source: "n26_csv", label: "N26",
|
||||
csv: "Date,Payee,Account number,Payment type,Payment reference,Amount (EUR),Amount (Foreign Currency),Type Foreign Currency,Exchange Rate\r\n2026-09-01,\"Cafe, Berlin\",DE02120300000000202051,MasterCard Payment,\"Lunch, first line\nsecond line\",-12.30,-14.50,USD,0.85\r\n",
|
||||
records: 1, booking: "2026-09-01", amount: "-12.30",
|
||||
description: "Lunch, first line\nsecond line", party: "Cafe, Berlin", iban: "DE02120300000000202051",
|
||||
},
|
||||
{
|
||||
name: "N26 German decimal comma semicolon BOM", source: "n26_csv", label: "N26",
|
||||
csv: "\ufeffDatum;Zahlungsempfänger;Kontonummer;Transaktionstyp;Verwendungszweck;Betrag (EUR);Betrag (Fremdwährung);Fremdwährung;Wechselkurs\n01.09.2026;Arbeitgeber;DE89 3704 0044 0532 0130 00;Überweisung;Gehalt;\"1.234,56\";;;\n",
|
||||
records: 1, booking: "2026-09-01", amount: "1234.56",
|
||||
description: "Gehalt", party: "Arbeitgeber", iban: "DE89370400440532013000",
|
||||
},
|
||||
{
|
||||
name: "N26 English booking and value dates", source: "n26_csv", label: "N26",
|
||||
csv: "Booking Date,Value Date,Partner Name,Partner IBAN,Type,Payment Reference,Account Name,Amount (EUR),Original Amount,Original Currency,Exchange Rate\n2026-09-01,2026-08-31,Cafe,DE02120300000000202051,Card,Lunch,Main,-12.30,-14.50,USD,0.85\n",
|
||||
records: 1, booking: "2026-09-01", value: "2026-08-31", amount: "-12.30",
|
||||
description: "Lunch", party: "Cafe", iban: "DE02120300000000202051",
|
||||
},
|
||||
{
|
||||
name: "ING Umsatzanzeige with preamble and Windows-1252", source: "ing_csv", label: "ING",
|
||||
csv: ingExport,
|
||||
records: 2, booking: "2025-12-09", value: "2025-12-08", amount: "-13.98",
|
||||
description: "NR XXXX 4025 KAUFUMSATZ", party: "VISA Firma",
|
||||
},
|
||||
{
|
||||
name: "Kontist English month-first dates", source: "kontist_csv", label: "Kontist",
|
||||
csv: "Payment Date,Name,Amount,Purpose,Currency\n01/15/2026,Client GmbH,1200.00,Invoice 2026-01,EUR\n01/16/2026,Cafe,-4.20,Espresso,EUR\n",
|
||||
records: 2, booking: "2026-01-15", amount: "1200.00",
|
||||
description: "Invoice 2026-01", party: "Client GmbH",
|
||||
},
|
||||
{
|
||||
name: "Kontist German day-first dates and grouping", source: "kontist_csv", label: "Kontist",
|
||||
csv: "Buchungsdatum;Name;Betrag;Verwendungszweck;Währung\n15/01/2026;Client GmbH;1.200,00;Rechnung;EUR\n",
|
||||
records: 1, booking: "2026-01-15", amount: "1200.00",
|
||||
description: "Rechnung", party: "Client GmbH",
|
||||
},
|
||||
}
|
||||
for _, tt := range cases {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
file := readFixture(t, tt.csv)
|
||||
mapping, source, label, ok := DetectCSVMapping(file)
|
||||
if !ok || source != tt.source || label != tt.label {
|
||||
t.Fatalf("detected %q/%q (ok=%t), want %q/%q", source, label, ok, tt.source, tt.label)
|
||||
}
|
||||
facts, err := ParseMappedCSV(file, fixtureDataset().Accounts[0], mapping, source)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(facts) != tt.records {
|
||||
t.Fatalf("records: %d, want %d", len(facts), tt.records)
|
||||
}
|
||||
f := facts[0]
|
||||
if f.BookingDate != tt.booking || f.ValueDate != tt.value || f.Amount.String() != tt.amount ||
|
||||
f.RawDescription != tt.description || f.Counterparty != tt.party || f.CounterpartyIBAN != tt.iban ||
|
||||
f.Currency != "EUR" || f.Source != tt.source || f.AccountID != "account_a" {
|
||||
t.Fatalf("unexpected facts: %+v", f)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ING's second record is income; a preset must not flatten the direction or the
|
||||
// German grouping of larger amounts.
|
||||
func TestINGRetainsDirectionAndGrouping(t *testing.T) {
|
||||
file := readFixture(t, ingExport)
|
||||
mapping, source, _, _ := DetectCSVMapping(file)
|
||||
facts, err := ParseMappedCSV(file, fixtureDataset().Accounts[0], mapping, source)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(facts) != 2 || facts[1].Amount.String() != "2647.74" || facts[1].BookingDate != "2025-11-28" {
|
||||
t.Fatalf("income record lost: %+v", facts)
|
||||
}
|
||||
// Mandate and customer references repeat across bookings, so no preset may
|
||||
// present them as a stable per-transaction identity.
|
||||
for _, f := range facts {
|
||||
if f.ExternalID != "" {
|
||||
t.Fatalf("ING facts must not carry a synthetic reference: %q", f.ExternalID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnreadableAndUnrecognizedStatementsAreRejected(t *testing.T) {
|
||||
for name, input := range map[string]string{
|
||||
"empty": "",
|
||||
"blank": "\n\n",
|
||||
"unterminated quote": "Date,Amount (EUR)\n2026-09-01,\"unterminated\n",
|
||||
"NUL byte": "Date;Betrag\n2026-09-01;-1,00\x00\n",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := ReadCSV(strings.NewReader(input)); err == nil {
|
||||
t.Fatal("accepted an unreadable statement")
|
||||
}
|
||||
})
|
||||
}
|
||||
for name, input := range map[string]string{
|
||||
"no known columns": "Foo;Bar\n1;2\n",
|
||||
"duplicate columns": "Datum;Datum;Betrag\n2026-09-01;2026-09-01;-1,00\n",
|
||||
"foreign amount only": "Date,Payee,Amount (Foreign Currency)\n2026-09-01,Cafe,-1.00\n",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
file := readFixture(t, input)
|
||||
if _, _, _, ok := DetectCSVMapping(file); ok {
|
||||
t.Fatal("guessed a mapping for an unrecognized layout")
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := readFixture(t, "A;B;C\n1;2\n").Sample(); err == nil {
|
||||
t.Fatal("ragged records were offered for mapping")
|
||||
}
|
||||
if _, err := readFixture(t, "A;A\n1;2\n").Sample(); err == nil {
|
||||
t.Fatal("duplicate columns were offered for mapping")
|
||||
}
|
||||
if _, err := readFixture(t, "Buchung;Betrag;Verwendungszweck;Auftraggeber/Empfänger\n").Sample(); err == nil {
|
||||
t.Fatal("a statement without records was offered for mapping")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMappedCSVRejectsInvalidMappingsAndRecords(t *testing.T) {
|
||||
const document = "Datum;Betrag;Text;Waehrung\n2026-09-01;-1,00;Lunch;EUR\n"
|
||||
valid := CSVMapping{HeaderRow: 1, BookingDateColumn: "Datum", AmountColumn: "Betrag", DescriptionColumn: "Text", DateFormat: "iso-or-german", DecimalFormat: "comma"}
|
||||
with := func(change func(*CSVMapping)) CSVMapping {
|
||||
mapping := valid
|
||||
change(&mapping)
|
||||
return mapping
|
||||
}
|
||||
mappings := map[string]CSVMapping{
|
||||
"unknown column": with(func(m *CSVMapping) { m.AmountColumn = "Amount" }),
|
||||
"missing description": with(func(m *CSVMapping) { m.DescriptionColumn = "" }),
|
||||
"missing booking date": with(func(m *CSVMapping) { m.BookingDateColumn = "" }),
|
||||
"both money strategies": with(func(m *CSVMapping) { m.DebitColumn = "Betrag"; m.CreditColumn = "Betrag" }),
|
||||
"debit without credit": with(func(m *CSVMapping) { m.AmountColumn = ""; m.DebitColumn = "Betrag" }),
|
||||
"no money strategy": with(func(m *CSVMapping) { m.AmountColumn = "" }),
|
||||
"column used twice": with(func(m *CSVMapping) { m.CounterpartyColumn = "Text" }),
|
||||
"unsupported date": with(func(m *CSVMapping) { m.DateFormat = "%Y-%m-%d" }),
|
||||
"unsupported decimal": with(func(m *CSVMapping) { m.DecimalFormat = "german" }),
|
||||
"header row before file": with(func(m *CSVMapping) { m.HeaderRow = 0 }),
|
||||
"header row past file": with(func(m *CSVMapping) { m.HeaderRow = 9 }),
|
||||
"invalid fixed currency": with(func(m *CSVMapping) { m.FixedCurrency = "Euro" }),
|
||||
}
|
||||
for name, mapping := range mappings {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if facts, err := ParseMappedCSV(readFixture(t, document), fixtureDataset().Accounts[0], mapping, "csv"); err == nil || facts != nil {
|
||||
t.Fatalf("accepted an invalid mapping: %+v", mapping)
|
||||
}
|
||||
})
|
||||
}
|
||||
records := map[string]string{
|
||||
"unparseable amount": "Datum;Betrag;Text;Waehrung\n2026-09-01;-1,00;Lunch;EUR\n2026-09-02;nope;Lunch;EUR\n",
|
||||
"impossible date": "Datum;Betrag;Text;Waehrung\n2026-02-30;-1,00;Lunch;EUR\n",
|
||||
"missing amount": "Datum;Betrag;Text;Waehrung\n2026-09-01;;Lunch;EUR\n",
|
||||
"short record": "Datum;Betrag;Text;Waehrung\n2026-09-01;-1,00\n",
|
||||
"three comma groups": "Datum;Betrag;Text;Waehrung\n2026-09-01;1,00,00;Lunch;EUR\n",
|
||||
}
|
||||
for name, document := range records {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if facts, err := ParseMappedCSV(readFixture(t, document), fixtureDataset().Accounts[0], valid, "csv"); err == nil || facts != nil {
|
||||
t.Fatalf("accepted a malformed record: %q", document)
|
||||
}
|
||||
})
|
||||
}
|
||||
// A currency the account does not hold must fail the whole statement rather
|
||||
// than silently booking foreign money against it.
|
||||
conflicting := valid
|
||||
conflicting.CurrencyColumn = "Waehrung"
|
||||
if _, err := ParseMappedCSV(readFixture(t, "Datum;Betrag;Text;Waehrung\n2026-09-01;-1,00;Lunch;USD\n"), fixtureDataset().Accounts[0], conflicting, "csv"); err == nil {
|
||||
t.Fatal("imported a foreign currency into a EUR account")
|
||||
}
|
||||
if _, err := ParseMappedCSV(readFixture(t, document), fixtureDataset().Accounts[0], valid, ""); err == nil {
|
||||
t.Fatal("imported facts without an import source")
|
||||
}
|
||||
if _, err := ParseMappedCSV(readFixture(t, document), domain.Account{Currency: "EUR"}, valid, "csv"); err == nil {
|
||||
t.Fatal("imported facts without an account")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitDebitAndCreditColumnsCarryDirection(t *testing.T) {
|
||||
const document = "Date;Debit;Credit;Text\n" +
|
||||
"2026-09-01;12,30;;Lunch\n" +
|
||||
"2026-09-02;;100,00;Salary\n" +
|
||||
"2026-09-03;0,00;5,00;Refund\n" +
|
||||
"2026-09-04;-7,50;;Signed debit\n"
|
||||
mapping := CSVMapping{HeaderRow: 1, BookingDateColumn: "Date", DebitColumn: "Debit", CreditColumn: "Credit", DescriptionColumn: "Text", DateFormat: "yyyy-mm-dd", DecimalFormat: "comma"}
|
||||
facts, err := ParseMappedCSV(readFixture(t, document), fixtureDataset().Accounts[0], mapping, "csv")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"-12.30", "100.00", "5.00", "-7.50"}
|
||||
if len(facts) != len(want) {
|
||||
t.Fatalf("records: %d", len(facts))
|
||||
}
|
||||
for i, amount := range want {
|
||||
if facts[i].Amount.String() != amount {
|
||||
t.Fatalf("record %d: %s, want %s", i+1, facts[i].Amount, amount)
|
||||
}
|
||||
}
|
||||
for name, document := range map[string]string{
|
||||
"both populated": "Date;Debit;Credit;Text\n2026-09-01;12,30;5,00;Ambiguous\n",
|
||||
"both empty": "Date;Debit;Credit;Text\n2026-09-01;;;Empty\n",
|
||||
"negative credit": "Date;Debit;Credit;Text\n2026-09-01;;-5,00;Negative\n",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := ParseMappedCSV(readFixture(t, document), fixtureDataset().Accounts[0], mapping, "csv"); err == nil {
|
||||
t.Fatalf("accepted ambiguous split money: %q", document)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A mapping sample must describe structure only: no account text, counterparty,
|
||||
// reference, IBAN or amount digit may survive redaction.
|
||||
func TestSampleRedactsValuesAndLocatesHeaderBelowPreamble(t *testing.T) {
|
||||
sample, err := readFixture(t, ingExport).Sample()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sample.Delimiter != ";" || len(sample.Headers) != 7 || sample.Headers[0] != "Buchung" {
|
||||
t.Fatalf("header row not located: %+v", sample)
|
||||
}
|
||||
// Blank lines are not records: the header is the tenth parsed record.
|
||||
if sample.HeaderRow != 10 || sample.RecordCount != 2 || len(sample.ShapedRows) != 2 {
|
||||
t.Fatalf("unexpected sample shape: record=%d records=%d rows=%d", sample.HeaderRow, sample.RecordCount, len(sample.ShapedRows))
|
||||
}
|
||||
for _, row := range sample.ShapedRows {
|
||||
for _, value := range row {
|
||||
for _, r := range value {
|
||||
if unicode.IsLetter(r) && r != 'x' || unicode.IsDigit(r) && r != '0' {
|
||||
t.Fatalf("sample leaked statement content: %q", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, secret := range []string{"VISA", "Rente", "KAUFUMSATZ", "13", "98", "2647", "DE89"} {
|
||||
for _, row := range sample.ShapedRows {
|
||||
if strings.Contains(strings.Join(row, "|"), secret) {
|
||||
t.Fatalf("sample leaked %q", secret)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -42,6 +42,12 @@ func sourceLabel(source string) string {
|
||||
return "bank-synced"
|
||||
case "n26_csv":
|
||||
return "CSV"
|
||||
case "ing_csv":
|
||||
return "ING CSV"
|
||||
case "kontist_csv":
|
||||
return "Kontist CSV"
|
||||
case "csv":
|
||||
return "mapped CSV"
|
||||
default:
|
||||
return "source " + strconv.Quote(source)
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@ package banking
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"finance-duck/internal/domain"
|
||||
@@ -17,42 +16,6 @@ func fixtureFacts() domain.Facts {
|
||||
return domain.Facts{Source: "n26_csv", AccountID: "account_a", BookingDate: "2026-09-01", Amount: "-12.30", Currency: "EUR", RawDescription: "Lunch", Counterparty: "Cafe", Fingerprint: "fixture"}
|
||||
}
|
||||
|
||||
func TestN26SupportedExportSchemas(t *testing.T) {
|
||||
cases := []struct{ name, csv, amount, description, party, iban, value string }{
|
||||
{"English legacy quoted multiline", "Date,Payee,Account number,Payment type,Payment reference,Amount (EUR),Amount (Foreign Currency),Type Foreign Currency,Exchange Rate\r\n2026-09-01,\"Cafe, Berlin\",DE02120300000000202051,MasterCard Payment,\"Lunch, first line\nsecond line\",-12.30,-14.50,USD,0.85\r\n", "-12.30", "Lunch, first line\nsecond line", "Cafe, Berlin", "DE02120300000000202051", ""},
|
||||
{"German decimal comma semicolon BOM", "\ufeffDatum;Zahlungsempfänger;Kontonummer;Transaktionstyp;Verwendungszweck;Betrag (EUR);Betrag (Fremdwährung);Fremdwährung;Wechselkurs\n01.09.2026;Arbeitgeber;DE89 3704 0044 0532 0130 00;Überweisung;Gehalt;\"1.234,56\";;;\n", "1234.56", "Gehalt", "Arbeitgeber", "DE89370400440532013000", ""},
|
||||
{"English booking and value dates", "Booking Date,Value Date,Partner Name,Partner IBAN,Type,Payment Reference,Account Name,Amount (EUR),Original Amount,Original Currency,Exchange Rate\n2026-09-01,2026-08-31,Cafe,DE02120300000000202051,Card,Lunch,Main,-12.30,-14.50,USD,0.85\n", "-12.30", "Lunch", "Cafe", "DE02120300000000202051", "2026-08-31"},
|
||||
}
|
||||
for _, tt := range cases {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
rows, err := ParseCSV(strings.NewReader(tt.csv), fixtureDataset().Accounts[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("records: %d", len(rows))
|
||||
}
|
||||
f := rows[0]
|
||||
if f.Amount.String() != tt.amount || f.RawDescription != tt.description || f.Counterparty != tt.party || f.CounterpartyIBAN != tt.iban || f.ValueDate != tt.value || f.BookingDate != "2026-09-01" || f.Currency != "EUR" {
|
||||
t.Fatalf("unexpected parsed facts: %+v", f)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
func TestCSVRejectsPartialAndMalformedImports(t *testing.T) {
|
||||
for _, input := range []string{
|
||||
"Date,Payee,Amount (Foreign Currency)\n2026-09-01,Cafe,-1.00\n",
|
||||
"Date,Amount (EUR)\n2026-09-01,-1.00\n2026-09-02,nope\n",
|
||||
"Date,Amount (EUR)\n2026-02-30,-1.00\n",
|
||||
"Date,Amount (EUR),Currency\n2026-09-01,-1.00,USD\n",
|
||||
"Date,Amount (EUR)\n2026-09-01,\"unterminated\n",
|
||||
} {
|
||||
rows, err := ParseCSV(strings.NewReader(input), fixtureDataset().Accounts[0])
|
||||
if err == nil || rows != nil {
|
||||
t.Fatalf("accepted malformed/partial import %q", input)
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestFallbackOccurrenceMultiplicityAndRepeatImport(t *testing.T) {
|
||||
d := fixtureDataset()
|
||||
f := fixtureFacts()
|
||||
|
||||
Reference in New Issue
Block a user