Import ING and Kontist statements behind a reviewed column mapping
CSV import is now mapping-driven: N26, ING (metadata preamble, Windows-1252, German decimals) and Kontist exports are recognized locally, and any other layout can have its columns proposed by the configured model from a sample in which letters are replaced by x and digits by 0. Proposals are untrusted: every column must name a supplied header, money must come from one signed column or one debit/credit pair, and formats must be from a closed list. Uploading no longer imports. /api/import is replaced by prepare/confirm/cancel: prepare parses, deduplicates and previews the exact facts, and only confirming at the reviewed revision writes them. ING and AI-mapped facts carry no transaction reference, because repeating SEPA mandate references must never become a transaction identity.
This commit is contained in:
Lars Nolden
+677
-126
@@ -1,159 +1,715 @@
|
||||
package banking
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/csv"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"finance-duck/internal/domain"
|
||||
)
|
||||
|
||||
// ParseCSV accepts N26 English and German account-activity exports, including
|
||||
// their older Date/Datum and newer Booking Date/Buchungsdatum schemas. Supported
|
||||
// columns: Date/Datum/Booking Date/Buchungsdatum, Value Date/Wertstellung/
|
||||
// Wertstellungsdatum, Payee/Partner Name/Zahlungsempfänger/Empfänger/Auftraggeber,
|
||||
// Account number/Kontonummer/IBAN, Payment reference/Verwendungszweck,
|
||||
// Payment type/Transaktionstyp, Amount (EUR)/Betrag (EUR), and optional
|
||||
// Currency/Währung and Transaction ID/Transaktions-ID. Foreign-original-amount,
|
||||
// exchange-rate and category columns are deliberately not used for account money.
|
||||
// Comma and semicolon delimiters, UTF-8 BOM, CRLF, RFC4180 quoted multiline
|
||||
// descriptions, ISO and German dates, decimal comma and decimal point are accepted.
|
||||
// Missing required booking-date or account-amount columns fail the entire import.
|
||||
func ParseCSV(input io.Reader, account domain.Account) ([]domain.Facts, error) {
|
||||
if account.ID == "" {
|
||||
return nil, fmt.Errorf("CSV requires a selected account")
|
||||
}
|
||||
reader := bufio.NewReader(input)
|
||||
first, err := reader.ReadString('\n')
|
||||
if err != nil && err != io.EOF {
|
||||
return nil, fmt.Errorf("read CSV header: %w", err)
|
||||
}
|
||||
first = strings.TrimPrefix(first, "\ufeff")
|
||||
delimiter := ','
|
||||
// Count separators outside quotes; descriptions may contain either delimiter.
|
||||
quoted := false
|
||||
commas, semicolons := 0, 0
|
||||
for _, r := range first {
|
||||
if r == '"' {
|
||||
quoted = !quoted
|
||||
}
|
||||
if !quoted {
|
||||
if r == ',' {
|
||||
commas++
|
||||
}
|
||||
if r == ';' {
|
||||
semicolons++
|
||||
}
|
||||
}
|
||||
}
|
||||
if semicolons > commas {
|
||||
delimiter = ';'
|
||||
}
|
||||
parser := csv.NewReader(io.MultiReader(strings.NewReader(first), reader))
|
||||
parser.Comma = delimiter
|
||||
headers, err := parser.Read()
|
||||
// maxCSVBytes bounds an uploaded statement. Statements are held in memory so a
|
||||
// mapping can be proposed, previewed and confirmed without re-uploading.
|
||||
const maxCSVBytes = 2 << 20
|
||||
|
||||
const maxCSVColumns = 128
|
||||
const maxCSVPreambleRows = 50
|
||||
const maxCSVSampleRows = 4
|
||||
|
||||
// CSVFile is a parsed CSV document: its detected delimiter and every record,
|
||||
// including the bank preamble records that precede the column header. Blank
|
||||
// lines are not records, so indexes count parsed records, not file lines.
|
||||
type CSVFile struct {
|
||||
delimiter rune
|
||||
rows [][]string
|
||||
}
|
||||
|
||||
// CSVMapping assigns source columns, by their exact header text, to bank facts.
|
||||
// Money comes either from one signed AmountColumn or from separate
|
||||
// DebitColumn/CreditColumn pairs, never from both. FixedCurrency records a
|
||||
// currency carried by a header such as "Amount (EUR)" rather than a column.
|
||||
type CSVMapping struct {
|
||||
// HeaderRow is the 1-based parsed record holding the column names.
|
||||
HeaderRow int `json:"header_row"`
|
||||
BookingDateColumn string `json:"booking_date_column"`
|
||||
ValueDateColumn string `json:"value_date_column,omitempty"`
|
||||
AmountColumn string `json:"amount_column,omitempty"`
|
||||
DebitColumn string `json:"debit_column,omitempty"`
|
||||
CreditColumn string `json:"credit_column,omitempty"`
|
||||
CurrencyColumn string `json:"currency_column,omitempty"`
|
||||
DescriptionColumn string `json:"description_column"`
|
||||
FallbackDescriptionColumn string `json:"fallback_description_column,omitempty"`
|
||||
CounterpartyColumn string `json:"counterparty_column,omitempty"`
|
||||
CounterpartyIBANColumn string `json:"counterparty_iban_column,omitempty"`
|
||||
ExternalIDColumn string `json:"external_id_column,omitempty"`
|
||||
DateFormat string `json:"date_format"`
|
||||
DecimalFormat string `json:"decimal_format"`
|
||||
FixedCurrency string `json:"fixed_currency,omitempty"`
|
||||
}
|
||||
|
||||
// CSVSample describes a statement's shape for column mapping. Cell values are
|
||||
// replaced by their character shape: no account text, name, reference or amount
|
||||
// digit is retained.
|
||||
type CSVSample struct {
|
||||
Delimiter string `json:"delimiter"`
|
||||
HeaderRow int `json:"header_row"`
|
||||
Headers []string `json:"headers"`
|
||||
ShapedRows [][]string `json:"shaped_rows"`
|
||||
RecordCount int `json:"record_count"`
|
||||
}
|
||||
|
||||
// ReadCSV decodes an uploaded statement. UTF-8 and Windows-1252 (still emitted
|
||||
// by some ING exports) are accepted, along with a BOM, CRLF, comma/semicolon/tab
|
||||
// delimiters and RFC4180 quoted multiline fields.
|
||||
func ReadCSV(input io.Reader) (CSVFile, error) {
|
||||
raw, err := io.ReadAll(io.LimitReader(input, maxCSVBytes+1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid N26 CSV header")
|
||||
return CSVFile{}, fmt.Errorf("read CSV: %w", err)
|
||||
}
|
||||
columns := make(map[string]int)
|
||||
amountCurrency := ""
|
||||
for i, h := range headers {
|
||||
name := headerName(h)
|
||||
key := ""
|
||||
switch name {
|
||||
case "date", "datum", "booking date", "buchungsdatum":
|
||||
key = "date"
|
||||
case "value date", "wertstellung", "wertstellungsdatum", "valutadatum":
|
||||
key = "value"
|
||||
case "payee", "partner name", "zahlungsempfänger", "zahlungsempfänger name", "empfänger", "empfänger/auftraggeber", "partnername", "name zahlungspartner":
|
||||
key = "party"
|
||||
case "account number", "partner iban", "kontonummer", "iban", "konto":
|
||||
key = "iban"
|
||||
case "payment reference", "verwendungszweck", "reference", "beschreibung":
|
||||
key = "description"
|
||||
case "payment type", "transaktionstyp", "zahlungstyp", "type", "typ":
|
||||
key = "type"
|
||||
case "currency", "währung":
|
||||
key = "currency"
|
||||
case "transaction id", "transaktions-id", "transaktions id":
|
||||
key = "external"
|
||||
case "amount", "betrag":
|
||||
key = "amount"
|
||||
if len(raw) > maxCSVBytes {
|
||||
return CSVFile{}, fmt.Errorf("CSV statement exceeds %d MiB", maxCSVBytes>>20)
|
||||
}
|
||||
text, err := decodeCSVText(raw)
|
||||
if err != nil {
|
||||
return CSVFile{}, err
|
||||
}
|
||||
if strings.TrimSpace(text) == "" {
|
||||
return CSVFile{}, errors.New("CSV statement is empty")
|
||||
}
|
||||
// Choose the delimiter that yields the widest consistently parsed records:
|
||||
// descriptions routinely contain the delimiters used by other banks.
|
||||
delimiter, best := ' ', -1
|
||||
for _, candidate := range []rune{',', ';', '\t'} {
|
||||
if _, score := parseCSVRecords(text, candidate); score > best {
|
||||
delimiter, best = candidate, score
|
||||
}
|
||||
}
|
||||
if best <= 0 {
|
||||
return CSVFile{}, errors.New("unreadable CSV statement: no delimiter produced multi-column records")
|
||||
}
|
||||
rows, _ := parseCSVRecords(text, delimiter)
|
||||
for _, row := range rows {
|
||||
if len(row) > maxCSVColumns {
|
||||
return CSVFile{}, fmt.Errorf("CSV statement has more than %d columns", maxCSVColumns)
|
||||
}
|
||||
}
|
||||
return CSVFile{delimiter: delimiter, rows: rows}, nil
|
||||
}
|
||||
|
||||
func decodeCSVText(raw []byte) (string, error) {
|
||||
text := strings.TrimPrefix(string(raw), "\ufeff")
|
||||
if strings.IndexByte(text, 0) >= 0 {
|
||||
return "", errors.New("CSV statement contains a NUL byte")
|
||||
}
|
||||
if utf8.ValidString(text) {
|
||||
return text, nil
|
||||
}
|
||||
// Windows-1252 is decoded locally: undecodable account text must never be
|
||||
// forwarded to a model or stored as invalid UTF-8 bank facts.
|
||||
windows1252 := [...]rune{'€', 0, '‚', 'ƒ', '„', '…', '†', '‡', 'ˆ', '‰', 'Š', '‹', 'Œ', 0, 'Ž', 0, 0, '‘', '’', '“', '”', '•', '–', '—', '˜', '™', 'š', '›', 'œ', 0, 'ž', 'Ÿ'}
|
||||
var out strings.Builder
|
||||
out.Grow(len(raw))
|
||||
for _, b := range []byte(text) {
|
||||
switch {
|
||||
case b < 0x80 || b >= 0xa0:
|
||||
out.WriteRune(rune(b))
|
||||
default:
|
||||
for _, prefix := range []string{"amount (", "betrag ("} {
|
||||
if strings.HasPrefix(name, prefix) && strings.HasSuffix(name, ")") {
|
||||
candidate := strings.ToUpper(strings.TrimSuffix(strings.TrimPrefix(name, prefix), ")"))
|
||||
if validCurrency(candidate) {
|
||||
key = "amount"
|
||||
amountCurrency = candidate
|
||||
}
|
||||
}
|
||||
r := windows1252[int(b)-0x80]
|
||||
if r == 0 {
|
||||
return "", errors.New("CSV statement is neither valid UTF-8 nor Windows-1252")
|
||||
}
|
||||
}
|
||||
if key != "" {
|
||||
if _, exists := columns[key]; exists {
|
||||
return nil, fmt.Errorf("duplicate N26 CSV column %s", key)
|
||||
}
|
||||
columns[key] = i
|
||||
out.WriteRune(r)
|
||||
}
|
||||
}
|
||||
if _, ok := columns["date"]; !ok {
|
||||
return nil, fmt.Errorf("N26 CSV requires Date/Datum or Booking Date/Buchungsdatum")
|
||||
}
|
||||
if _, ok := columns["amount"]; !ok {
|
||||
return nil, fmt.Errorf("N26 CSV requires Amount (currency)/Betrag (currency)")
|
||||
}
|
||||
get := func(row []string, key string) string {
|
||||
if i, ok := columns[key]; ok {
|
||||
return strings.TrimSpace(row[i])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
result := make([]domain.Facts, 0)
|
||||
for rowNumber := 2; ; rowNumber++ {
|
||||
return out.String(), nil
|
||||
}
|
||||
|
||||
// parseCSVRecords returns every record plus a preference score. Malformed
|
||||
// quoting scores below zero so a delimiter is never silently accepted for a
|
||||
// document it cannot represent.
|
||||
func parseCSVRecords(text string, delimiter rune) ([][]string, int) {
|
||||
parser := csv.NewReader(strings.NewReader(text))
|
||||
parser.Comma = delimiter
|
||||
parser.FieldsPerRecord = -1
|
||||
parser.ReuseRecord = false
|
||||
rows := make([][]string, 0, 64)
|
||||
widest, populated := 0, 0
|
||||
for {
|
||||
row, err := parser.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid N26 CSV record %d", rowNumber)
|
||||
return nil, -1
|
||||
}
|
||||
date, err := parseDate(get(row, "date"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid booking date in CSV record %d", rowNumber)
|
||||
}
|
||||
value := get(row, "value")
|
||||
if value != "" {
|
||||
value, err = parseDate(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid value date in CSV record %d", rowNumber)
|
||||
rows = append(rows, row)
|
||||
if len(row) > 1 && !blankCSVRow(row) {
|
||||
populated++
|
||||
if len(row) > widest {
|
||||
widest = len(row)
|
||||
}
|
||||
}
|
||||
amount, err := parseCSVAmount(get(row, "amount"))
|
||||
}
|
||||
if widest == 0 {
|
||||
return rows, 0
|
||||
}
|
||||
return rows, widest*1_000_000 + populated
|
||||
}
|
||||
|
||||
func blankCSVRow(row []string) bool {
|
||||
for _, value := range row {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Sample locates the column header and describes the statement's shape. The
|
||||
// header is the first widest record, which skips the metadata preamble that
|
||||
// banks such as ING place above their columns.
|
||||
func (f CSVFile) Sample() (CSVSample, error) {
|
||||
header, width := 0, 0
|
||||
for i, row := range f.rows {
|
||||
if i >= maxCSVPreambleRows {
|
||||
break
|
||||
}
|
||||
if len(row) > width && !blankCSVRow(row) {
|
||||
header, width = i+1, len(row)
|
||||
}
|
||||
}
|
||||
if width < 2 {
|
||||
return CSVSample{}, errors.New("CSV statement has no column header row")
|
||||
}
|
||||
headers, err := csvHeaders(f, header)
|
||||
if err != nil {
|
||||
return CSVSample{}, err
|
||||
}
|
||||
sample := CSVSample{Delimiter: string(f.delimiter), HeaderRow: header, Headers: headers, ShapedRows: [][]string{}}
|
||||
for _, row := range f.rows[header:] {
|
||||
if blankCSVRow(row) {
|
||||
continue
|
||||
}
|
||||
if len(row) != width {
|
||||
return CSVSample{}, fmt.Errorf("CSV statement has ragged records: expected %d columns", width)
|
||||
}
|
||||
sample.RecordCount++
|
||||
if len(sample.ShapedRows) < maxCSVSampleRows {
|
||||
shaped := make([]string, len(row))
|
||||
for i, value := range row {
|
||||
shaped[i] = shapeCSVValue(value)
|
||||
}
|
||||
sample.ShapedRows = append(sample.ShapedRows, shaped)
|
||||
}
|
||||
}
|
||||
if sample.RecordCount == 0 {
|
||||
return CSVSample{}, errors.New("CSV statement contains no transaction records")
|
||||
}
|
||||
return sample, nil
|
||||
}
|
||||
|
||||
func csvHeaders(f CSVFile, header int) ([]string, error) {
|
||||
if header < 1 || header > len(f.rows) {
|
||||
return nil, errors.New("CSV mapping has an invalid header row")
|
||||
}
|
||||
headers := make([]string, 0, len(f.rows[header-1]))
|
||||
seen := map[string]bool{}
|
||||
for _, raw := range f.rows[header-1] {
|
||||
name := strings.TrimSpace(raw)
|
||||
key := headerName(name)
|
||||
if key == "" {
|
||||
return nil, errors.New("CSV column headers must all be named")
|
||||
}
|
||||
if seen[key] {
|
||||
return nil, fmt.Errorf("duplicate CSV column %q", name)
|
||||
}
|
||||
seen[key] = true
|
||||
headers = append(headers, name)
|
||||
}
|
||||
return headers, nil
|
||||
}
|
||||
|
||||
// shapeCSVValue keeps only structure: letters become x, digits 0. This is what a
|
||||
// column mapping needs, and it keeps descriptions, names, references, IBANs and
|
||||
// amounts out of any request that leaves this machine.
|
||||
func shapeCSVValue(value string) string {
|
||||
var out strings.Builder
|
||||
for i, r := range strings.TrimSpace(value) {
|
||||
if i >= 64 {
|
||||
out.WriteRune('…')
|
||||
break
|
||||
}
|
||||
switch {
|
||||
case unicode.IsLetter(r):
|
||||
out.WriteRune('x')
|
||||
case unicode.IsDigit(r):
|
||||
out.WriteRune('0')
|
||||
case unicode.IsSpace(r):
|
||||
out.WriteRune(' ')
|
||||
default:
|
||||
out.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// DetectCSVMapping recognizes known bank exports without any model. The returned
|
||||
// source identifies imported facts; label names the export for the operator.
|
||||
func DetectCSVMapping(f CSVFile) (mapping CSVMapping, source, label string, ok bool) {
|
||||
for i, row := range f.rows {
|
||||
if i >= maxCSVPreambleRows {
|
||||
break
|
||||
}
|
||||
columns, usable := csvColumnIndex(row)
|
||||
if !usable {
|
||||
continue
|
||||
}
|
||||
if mapping, ok := n26Mapping(i+1, columns); ok {
|
||||
return mapping, "n26_csv", "N26", true
|
||||
}
|
||||
if mapping, ok := ingMapping(i+1, columns); ok {
|
||||
return mapping, "ing_csv", "ING", true
|
||||
}
|
||||
if mapping, ok := kontistMapping(f, i+1, columns); ok {
|
||||
return mapping, "kontist_csv", "Kontist", true
|
||||
}
|
||||
}
|
||||
return CSVMapping{}, "", "", false
|
||||
}
|
||||
|
||||
// csvColumnIndex maps normalized column names to their exact header text.
|
||||
func csvColumnIndex(row []string) (map[string]string, bool) {
|
||||
columns := make(map[string]string, len(row))
|
||||
for _, raw := range row {
|
||||
actual := strings.TrimSpace(raw)
|
||||
name := headerName(actual)
|
||||
if name == "" {
|
||||
return nil, false
|
||||
}
|
||||
if _, exists := columns[name]; exists {
|
||||
return nil, false
|
||||
}
|
||||
columns[name] = actual
|
||||
}
|
||||
return columns, len(columns) > 1
|
||||
}
|
||||
|
||||
func csvColumn(columns map[string]string, names ...string) string {
|
||||
for _, name := range names {
|
||||
if actual, ok := columns[name]; ok {
|
||||
return actual
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// n26Mapping accepts N26's English and German account-activity exports,
|
||||
// including their older Date/Datum and newer Booking Date/Buchungsdatum
|
||||
// schemas. Foreign original amounts, exchange rates and categories are
|
||||
// deliberately never used as account money.
|
||||
func n26Mapping(header int, columns map[string]string) (CSVMapping, bool) {
|
||||
amount, currency := "", ""
|
||||
for name, actual := range columns {
|
||||
if name == "amount" || name == "betrag" {
|
||||
amount = actual
|
||||
continue
|
||||
}
|
||||
for _, prefix := range []string{"amount (", "betrag ("} {
|
||||
if strings.HasPrefix(name, prefix) && strings.HasSuffix(name, ")") {
|
||||
code := strings.ToUpper(strings.TrimSuffix(strings.TrimPrefix(name, prefix), ")"))
|
||||
if validCurrency(code) {
|
||||
amount, currency = actual, code
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
mapping := CSVMapping{
|
||||
HeaderRow: header,
|
||||
BookingDateColumn: csvColumn(columns, "date", "datum", "booking date", "buchungsdatum"),
|
||||
ValueDateColumn: csvColumn(columns, "value date", "wertstellung", "wertstellungsdatum", "valutadatum"),
|
||||
AmountColumn: amount,
|
||||
CurrencyColumn: csvColumn(columns, "currency", "währung"),
|
||||
DescriptionColumn: csvColumn(columns, "payment reference", "verwendungszweck", "reference", "beschreibung"),
|
||||
FallbackDescriptionColumn: csvColumn(columns, "payment type", "transaktionstyp", "zahlungstyp", "type", "typ"),
|
||||
CounterpartyColumn: csvColumn(columns, "payee", "partner name", "zahlungsempfänger", "zahlungsempfänger name", "empfänger", "empfänger/auftraggeber", "partnername", "name zahlungspartner"),
|
||||
CounterpartyIBANColumn: csvColumn(columns, "account number", "partner iban", "kontonummer", "iban", "konto"),
|
||||
ExternalIDColumn: csvColumn(columns, "transaction id", "transaktions-id", "transaktions id"),
|
||||
DateFormat: "iso-or-german",
|
||||
DecimalFormat: "dot-or-comma",
|
||||
FixedCurrency: currency,
|
||||
}
|
||||
// An N26 export always carries a payment reference and a typed transaction.
|
||||
marker := mapping.FallbackDescriptionColumn != "" || csvColumn(columns, "original amount", "betrag (fremdwährung)", "account name", "partner name", "payee") != ""
|
||||
if mapping.BookingDateColumn == "" || mapping.AmountColumn == "" || mapping.DescriptionColumn == "" || !marker {
|
||||
return CSVMapping{}, false
|
||||
}
|
||||
return mapping, true
|
||||
}
|
||||
|
||||
// ingMapping accepts ING's Umsatzanzeige export, whose columns sit below a
|
||||
// metadata preamble. Gläubiger-ID, Mandatsreferenz and Kundenreferenz are SEPA
|
||||
// mandate references that repeat across bookings, so they are never used as a
|
||||
// transaction identity.
|
||||
func ingMapping(header int, columns map[string]string) (CSVMapping, bool) {
|
||||
mapping := CSVMapping{
|
||||
HeaderRow: header,
|
||||
BookingDateColumn: csvColumn(columns, "buchung"),
|
||||
ValueDateColumn: csvColumn(columns, "wertstellungsdatum", "valuta"),
|
||||
AmountColumn: csvColumn(columns, "betrag"),
|
||||
CurrencyColumn: csvColumn(columns, "währung", "waehrung"),
|
||||
DescriptionColumn: csvColumn(columns, "verwendungszweck"),
|
||||
FallbackDescriptionColumn: csvColumn(columns, "buchungstext"),
|
||||
CounterpartyColumn: csvColumn(columns, "auftraggeber/empfänger", "auftraggeber/empfaenger"),
|
||||
DateFormat: "dd.mm.yyyy",
|
||||
DecimalFormat: "comma",
|
||||
}
|
||||
if mapping.BookingDateColumn == "" || mapping.AmountColumn == "" || mapping.DescriptionColumn == "" || mapping.CounterpartyColumn == "" {
|
||||
return CSVMapping{}, false
|
||||
}
|
||||
return mapping, true
|
||||
}
|
||||
|
||||
// kontistMapping accepts Kontist's documented transaction vocabulary: a payment
|
||||
// date, an amount, a purpose and a counterparty name. Date and decimal
|
||||
// conventions are inferred from the file's own first populated values, and the
|
||||
// mapping is always reviewed before anything is imported.
|
||||
func kontistMapping(f CSVFile, header int, columns map[string]string) (CSVMapping, bool) {
|
||||
mapping := CSVMapping{
|
||||
HeaderRow: header,
|
||||
BookingDateColumn: csvColumn(columns, "payment date", "booking date", "buchungsdatum", "zahlungsdatum"),
|
||||
ValueDateColumn: csvColumn(columns, "value date", "wertstellungsdatum", "valuta"),
|
||||
AmountColumn: csvColumn(columns, "amount", "betrag"),
|
||||
CurrencyColumn: csvColumn(columns, "currency", "währung", "waehrung"),
|
||||
DescriptionColumn: csvColumn(columns, "purpose", "verwendungszweck", "payment reference"),
|
||||
CounterpartyColumn: csvColumn(columns, "name", "counterparty", "zahlungspartner"),
|
||||
CounterpartyIBANColumn: csvColumn(columns, "iban", "counterparty iban"),
|
||||
ExternalIDColumn: csvColumn(columns, "transaction id", "transaction_id", "transaktions-id"),
|
||||
}
|
||||
if mapping.BookingDateColumn == "" || mapping.AmountColumn == "" || mapping.DescriptionColumn == "" || mapping.CounterpartyColumn == "" {
|
||||
return CSVMapping{}, false
|
||||
}
|
||||
mapping.DateFormat = inferCSVDateFormat(csvFirstValue(f, header, mapping.BookingDateColumn))
|
||||
mapping.DecimalFormat = inferCSVDecimalFormat(csvFirstValue(f, header, mapping.AmountColumn))
|
||||
return mapping, true
|
||||
}
|
||||
|
||||
func csvFirstValue(f CSVFile, header int, column string) string {
|
||||
headers := f.rows[header-1]
|
||||
index := -1
|
||||
for i, name := range headers {
|
||||
if strings.TrimSpace(name) == column {
|
||||
index = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if index < 0 {
|
||||
return ""
|
||||
}
|
||||
for _, row := range f.rows[header:] {
|
||||
if len(row) == len(headers) && strings.TrimSpace(row[index]) != "" {
|
||||
return strings.TrimSpace(row[index])
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func inferCSVDateFormat(value string) string {
|
||||
switch {
|
||||
case len(value) >= 11 && value[4] == '-' && (value[10] == 'T' || value[10] == ' '):
|
||||
return "iso-date-time"
|
||||
case len(value) == 10 && value[4] == '-' && value[7] == '-':
|
||||
return "yyyy-mm-dd"
|
||||
case strings.Count(value, ".") == 2:
|
||||
return "dd.mm.yyyy"
|
||||
case strings.Count(value, "/") == 2:
|
||||
// Kontist documents month/day/year for interchange. A first component
|
||||
// above twelve can only be a day; the preview shows the parsed result.
|
||||
if first, err := strconv.Atoi(strings.SplitN(value, "/", 2)[0]); err == nil && first > 12 {
|
||||
return "dd/mm/yyyy"
|
||||
}
|
||||
return "mm/dd/yyyy"
|
||||
default:
|
||||
return "yyyy-mm-dd"
|
||||
}
|
||||
}
|
||||
|
||||
func inferCSVDecimalFormat(value string) string {
|
||||
if strings.Contains(value, ",") {
|
||||
return "comma"
|
||||
}
|
||||
return "dot"
|
||||
}
|
||||
|
||||
// ParseMappedCSV converts every record into bank facts. A single malformed
|
||||
// record fails the whole statement: a partially imported statement cannot be
|
||||
// distinguished from a truncated export later.
|
||||
func ParseMappedCSV(f CSVFile, account domain.Account, mapping CSVMapping, source string) ([]domain.Facts, error) {
|
||||
if account.ID == "" {
|
||||
return nil, errors.New("CSV requires a selected account")
|
||||
}
|
||||
if source == "" {
|
||||
return nil, errors.New("CSV import source is required")
|
||||
}
|
||||
columns, err := validateCSVMapping(f, mapping)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
headers := f.rows[mapping.HeaderRow-1]
|
||||
get := func(row []string, column string) string {
|
||||
if column == "" {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(row[columns[column]])
|
||||
}
|
||||
facts := make([]domain.Facts, 0, len(f.rows)-mapping.HeaderRow)
|
||||
for offset, row := range f.rows[mapping.HeaderRow:] {
|
||||
record := mapping.HeaderRow + offset + 1
|
||||
if blankCSVRow(row) {
|
||||
continue
|
||||
}
|
||||
if len(row) != len(headers) {
|
||||
return nil, fmt.Errorf("CSV record %d has %d columns, expected %d", record, len(row), len(headers))
|
||||
}
|
||||
booking, err := parseMappedCSVDate(get(row, mapping.BookingDateColumn), mapping.DateFormat)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid account amount in CSV record %d", rowNumber)
|
||||
return nil, fmt.Errorf("invalid booking date in CSV record %d", record)
|
||||
}
|
||||
value := get(row, mapping.ValueDateColumn)
|
||||
if value != "" {
|
||||
if value, err = parseMappedCSVDate(value, mapping.DateFormat); err != nil {
|
||||
return nil, fmt.Errorf("invalid value date in CSV record %d", record)
|
||||
}
|
||||
}
|
||||
amount, err := mappedCSVAmount(row, get, mapping)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid account amount in CSV record %d", record)
|
||||
}
|
||||
currency := strings.ToUpper(get(row, mapping.CurrencyColumn))
|
||||
if currency == "€" {
|
||||
currency = "EUR"
|
||||
}
|
||||
currency := strings.ToUpper(get(row, "currency"))
|
||||
if currency == "" {
|
||||
currency = amountCurrency
|
||||
currency = strings.ToUpper(mapping.FixedCurrency)
|
||||
}
|
||||
if currency == "" {
|
||||
currency = strings.ToUpper(account.Currency)
|
||||
}
|
||||
if !validCurrency(currency) || (amountCurrency != "" && currency != amountCurrency) || (account.Currency != "" && currency != strings.ToUpper(account.Currency)) {
|
||||
return nil, fmt.Errorf("invalid or conflicting account currency in CSV record %d", rowNumber)
|
||||
fixed := strings.ToUpper(mapping.FixedCurrency)
|
||||
if !validCurrency(currency) || (fixed != "" && currency != fixed) || (account.Currency != "" && currency != strings.ToUpper(account.Currency)) {
|
||||
return nil, fmt.Errorf("invalid or conflicting account currency in CSV record %d", record)
|
||||
}
|
||||
description := get(row, "description")
|
||||
description := get(row, mapping.DescriptionColumn)
|
||||
if description == "" {
|
||||
description = get(row, "type")
|
||||
description = get(row, mapping.FallbackDescriptionColumn)
|
||||
}
|
||||
result = append(result, domain.Facts{Source: "n26_csv", AccountID: account.ID, BookingDate: date, ValueDate: value, Amount: amount, Currency: currency, RawDescription: description, ExternalID: get(row, "external"), Counterparty: get(row, "party"), CounterpartyIBAN: normalizeIBAN(get(row, "iban"))})
|
||||
facts = append(facts, domain.Facts{
|
||||
Source: source, AccountID: account.ID, BookingDate: booking, ValueDate: value,
|
||||
Amount: amount, Currency: currency, RawDescription: description,
|
||||
ExternalID: get(row, mapping.ExternalIDColumn), Counterparty: get(row, mapping.CounterpartyColumn),
|
||||
CounterpartyIBAN: normalizeIBAN(get(row, mapping.CounterpartyIBANColumn)),
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
if len(facts) == 0 {
|
||||
return nil, errors.New("CSV statement contains no transaction records")
|
||||
}
|
||||
return facts, nil
|
||||
}
|
||||
|
||||
// validateCSVMapping resolves a mapping against the document and returns each
|
||||
// mapped column's index. Every referenced column must exist exactly, no column
|
||||
// may serve two fields, and money must come from exactly one strategy.
|
||||
func validateCSVMapping(f CSVFile, mapping CSVMapping) (map[string]int, error) {
|
||||
if mapping.HeaderRow < 1 || mapping.HeaderRow > len(f.rows) || mapping.HeaderRow > maxCSVPreambleRows {
|
||||
return nil, errors.New("CSV mapping has an invalid header row")
|
||||
}
|
||||
if _, err := csvHeaders(f, mapping.HeaderRow); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
indexes := make(map[string]int, len(f.rows[mapping.HeaderRow-1]))
|
||||
for i, raw := range f.rows[mapping.HeaderRow-1] {
|
||||
indexes[strings.TrimSpace(raw)] = i
|
||||
}
|
||||
if mapping.BookingDateColumn == "" {
|
||||
return nil, errors.New("CSV mapping requires a booking date column")
|
||||
}
|
||||
if mapping.DescriptionColumn == "" {
|
||||
return nil, errors.New("CSV mapping requires a description column")
|
||||
}
|
||||
signed, split := mapping.AmountColumn != "", mapping.DebitColumn != "" || mapping.CreditColumn != ""
|
||||
if signed == split {
|
||||
return nil, errors.New("CSV mapping requires either one signed amount column or separate debit and credit columns")
|
||||
}
|
||||
if split && (mapping.DebitColumn == "" || mapping.CreditColumn == "") {
|
||||
return nil, errors.New("CSV mapping requires both a debit and a credit column")
|
||||
}
|
||||
if !validCSVDateFormat(mapping.DateFormat) {
|
||||
return nil, fmt.Errorf("unsupported CSV date format %q", mapping.DateFormat)
|
||||
}
|
||||
if !validCSVDecimalFormat(mapping.DecimalFormat) {
|
||||
return nil, fmt.Errorf("unsupported CSV decimal format %q", mapping.DecimalFormat)
|
||||
}
|
||||
if mapping.FixedCurrency != "" && !validCurrency(strings.ToUpper(mapping.FixedCurrency)) {
|
||||
return nil, errors.New("CSV mapping has an invalid fixed currency")
|
||||
}
|
||||
assigned := map[string]string{}
|
||||
for _, field := range []struct{ name, column string }{
|
||||
{"booking date", mapping.BookingDateColumn}, {"value date", mapping.ValueDateColumn},
|
||||
{"amount", mapping.AmountColumn}, {"debit", mapping.DebitColumn}, {"credit", mapping.CreditColumn},
|
||||
{"currency", mapping.CurrencyColumn}, {"description", mapping.DescriptionColumn},
|
||||
{"secondary description", mapping.FallbackDescriptionColumn}, {"counterparty", mapping.CounterpartyColumn},
|
||||
{"counterparty IBAN", mapping.CounterpartyIBANColumn}, {"transaction reference", mapping.ExternalIDColumn},
|
||||
} {
|
||||
if field.column == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := indexes[field.column]; !ok {
|
||||
return nil, fmt.Errorf("CSV mapping references unknown %s column %q", field.name, field.column)
|
||||
}
|
||||
if previous, ok := assigned[field.column]; ok {
|
||||
return nil, fmt.Errorf("CSV column %q is mapped to both %s and %s", field.column, previous, field.name)
|
||||
}
|
||||
assigned[field.column] = field.name
|
||||
}
|
||||
return indexes, nil
|
||||
}
|
||||
|
||||
// CSVDateFormats and CSVDecimalFormats are the exact accepted conventions. A
|
||||
// proposed mapping outside them is rejected rather than guessed.
|
||||
func CSVDateFormats() []string {
|
||||
return []string{"yyyy-mm-dd", "dd.mm.yyyy", "mm/dd/yyyy", "dd/mm/yyyy", "iso-date-time", "iso-or-german"}
|
||||
}
|
||||
func CSVDecimalFormats() []string { return []string{"dot", "comma", "dot-or-comma"} }
|
||||
|
||||
func validCSVDateFormat(format string) bool {
|
||||
for _, valid := range CSVDateFormats() {
|
||||
if format == valid {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
func validCSVDecimalFormat(format string) bool {
|
||||
for _, valid := range CSVDecimalFormats() {
|
||||
if format == valid {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func parseMappedCSVDate(value, format string) (string, error) {
|
||||
layouts := map[string][]string{
|
||||
"yyyy-mm-dd": {"2006-01-02"},
|
||||
"dd.mm.yyyy": {"02.01.2006", "2.1.2006"},
|
||||
"mm/dd/yyyy": {"01/02/2006", "1/2/2006"},
|
||||
"dd/mm/yyyy": {"02/01/2006", "2/1/2006"},
|
||||
"iso-date-time": {time.RFC3339, "2006-01-02T15:04:05", "2006-01-02 15:04:05", "2006-01-02"},
|
||||
"iso-or-german": {"2006-01-02", "02.01.2006", "2.1.2006"},
|
||||
}
|
||||
for _, layout := range layouts[format] {
|
||||
if parsed, err := time.Parse(layout, value); err == nil {
|
||||
return parsed.Format("2006-01-02"), nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("invalid date %q", value)
|
||||
}
|
||||
|
||||
// mappedCSVAmount returns signed account money. With split columns a debit is
|
||||
// negative however the bank wrote its sign, a credit must not be negative, and
|
||||
// the unused column may be empty or an explicit zero but never carry money.
|
||||
func mappedCSVAmount(row []string, get func([]string, string) string, mapping CSVMapping) (domain.Money, error) {
|
||||
if mapping.AmountColumn != "" {
|
||||
return parseMappedCSVDecimal(get(row, mapping.AmountColumn), mapping.DecimalFormat)
|
||||
}
|
||||
debit, credit := get(row, mapping.DebitColumn), get(row, mapping.CreditColumn)
|
||||
if debit == "" && credit == "" {
|
||||
return "", errors.New("debit and credit are both empty")
|
||||
}
|
||||
parse := func(value string) (domain.Money, int64, error) {
|
||||
if value == "" {
|
||||
return "0.00", 0, nil
|
||||
}
|
||||
money, err := parseMappedCSVDecimal(value, mapping.DecimalFormat)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
minor, err := money.Minor()
|
||||
return money, minor, err
|
||||
}
|
||||
debitMoney, debited, err := parse(debit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
creditMoney, credited, err := parse(credit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if debited != 0 && credited != 0 {
|
||||
return "", errors.New("debit and credit both carry money")
|
||||
}
|
||||
if credited < 0 {
|
||||
return "", errors.New("a credit column must not hold negative money")
|
||||
}
|
||||
if credited != 0 {
|
||||
return creditMoney, nil
|
||||
}
|
||||
if debited != 0 {
|
||||
return domain.ParseMoney("-" + strings.TrimPrefix(debitMoney.String(), "-"))
|
||||
}
|
||||
return domain.ParseMoney("0")
|
||||
}
|
||||
|
||||
func parseMappedCSVDecimal(value, format string) (domain.Money, error) {
|
||||
value = strings.NewReplacer("\u00a0", "", "\u202f", "", "'", "").Replace(strings.TrimSpace(value))
|
||||
switch format {
|
||||
case "dot-or-comma":
|
||||
return parseCSVAmount(value)
|
||||
case "comma":
|
||||
// A dot can only be grouping here, and only in exact thousands groups.
|
||||
if !strings.Contains(value, ",") && strings.Contains(value, ".") {
|
||||
if digits, ok := ungroup(value, "."); ok {
|
||||
value = digits
|
||||
}
|
||||
}
|
||||
return parseCSVAmount(value)
|
||||
case "dot":
|
||||
value = strings.TrimPrefix(value, "+")
|
||||
if strings.Contains(value, ",") {
|
||||
digits, ok := ungroup(value, ",")
|
||||
if !ok {
|
||||
return "", errors.New("invalid grouping")
|
||||
}
|
||||
value = digits
|
||||
}
|
||||
return domain.ParseMoney(value)
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported CSV decimal format %q", format)
|
||||
}
|
||||
}
|
||||
|
||||
// ungroup removes thousands separators, and only when every group is exactly
|
||||
// three digits: "1.234" is 1234, while "1.23" stays a decimal value.
|
||||
func ungroup(value, separator string) (string, bool) {
|
||||
sign := ""
|
||||
if rest, found := strings.CutPrefix(value, "-"); found {
|
||||
sign, value = "-", rest
|
||||
}
|
||||
groups := strings.Split(value, separator)
|
||||
if len(groups) < 2 || len(groups[0]) < 1 || len(groups[0]) > 3 {
|
||||
return "", false
|
||||
}
|
||||
for _, group := range groups[1:] {
|
||||
if len(group) != 3 {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
return sign + strings.Join(groups, ""), true
|
||||
}
|
||||
|
||||
func headerName(s string) string {
|
||||
@@ -195,16 +751,11 @@ func parseCSVAmount(s string) (domain.Money, error) {
|
||||
}
|
||||
pair := strings.SplitN(s, ",", 2)
|
||||
if strings.Contains(pair[0], ".") {
|
||||
groups := strings.Split(strings.TrimLeft(pair[0], "+-"), ".")
|
||||
if len(groups[0]) < 1 || len(groups[0]) > 3 {
|
||||
digits, ok := ungroup(pair[0], ".")
|
||||
if !ok {
|
||||
return "", fmt.Errorf("invalid grouping")
|
||||
}
|
||||
for _, g := range groups[1:] {
|
||||
if len(g) != 3 {
|
||||
return "", fmt.Errorf("invalid grouping")
|
||||
}
|
||||
}
|
||||
pair[0] = strings.ReplaceAll(pair[0], ".", "")
|
||||
pair[0] = digits
|
||||
}
|
||||
s = pair[0] + "." + pair[1]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user