Lars Nolden
dc767799bc
CSV import is now mapping-driven: N26, ING (metadata preamble, Windows-1252, German decimals) and Kontist exports are recognized locally, and any other layout can have its columns proposed by the configured model from a sample in which letters are replaced by x and digits by 0. Proposals are untrusted: every column must name a supplied header, money must come from one signed column or one debit/credit pair, and formats must be from a closed list. Uploading no longer imports. /api/import is replaced by prepare/confirm/cancel: prepare parses, deduplicates and previews the exact facts, and only confirming at the reviewed revision writes them. ING and AI-mapped facts carry no transaction reference, because repeating SEPA mandate references must never become a transaction identity.
764 lines
26 KiB
Go
764 lines
26 KiB
Go
package banking
|
||
|
||
import (
|
||
"encoding/csv"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
"unicode"
|
||
"unicode/utf8"
|
||
|
||
"finance-duck/internal/domain"
|
||
)
|
||
|
||
// maxCSVBytes bounds an uploaded statement. Statements are held in memory so a
|
||
// mapping can be proposed, previewed and confirmed without re-uploading.
|
||
const maxCSVBytes = 2 << 20
|
||
|
||
const maxCSVColumns = 128
|
||
const maxCSVPreambleRows = 50
|
||
const maxCSVSampleRows = 4
|
||
|
||
// CSVFile is a parsed CSV document: its detected delimiter and every record,
|
||
// including the bank preamble records that precede the column header. Blank
|
||
// lines are not records, so indexes count parsed records, not file lines.
|
||
type CSVFile struct {
|
||
delimiter rune
|
||
rows [][]string
|
||
}
|
||
|
||
// CSVMapping assigns source columns, by their exact header text, to bank facts.
|
||
// Money comes either from one signed AmountColumn or from separate
|
||
// DebitColumn/CreditColumn pairs, never from both. FixedCurrency records a
|
||
// currency carried by a header such as "Amount (EUR)" rather than a column.
|
||
type CSVMapping struct {
|
||
// HeaderRow is the 1-based parsed record holding the column names.
|
||
HeaderRow int `json:"header_row"`
|
||
BookingDateColumn string `json:"booking_date_column"`
|
||
ValueDateColumn string `json:"value_date_column,omitempty"`
|
||
AmountColumn string `json:"amount_column,omitempty"`
|
||
DebitColumn string `json:"debit_column,omitempty"`
|
||
CreditColumn string `json:"credit_column,omitempty"`
|
||
CurrencyColumn string `json:"currency_column,omitempty"`
|
||
DescriptionColumn string `json:"description_column"`
|
||
FallbackDescriptionColumn string `json:"fallback_description_column,omitempty"`
|
||
CounterpartyColumn string `json:"counterparty_column,omitempty"`
|
||
CounterpartyIBANColumn string `json:"counterparty_iban_column,omitempty"`
|
||
ExternalIDColumn string `json:"external_id_column,omitempty"`
|
||
DateFormat string `json:"date_format"`
|
||
DecimalFormat string `json:"decimal_format"`
|
||
FixedCurrency string `json:"fixed_currency,omitempty"`
|
||
}
|
||
|
||
// CSVSample describes a statement's shape for column mapping. Cell values are
|
||
// replaced by their character shape: no account text, name, reference or amount
|
||
// digit is retained.
|
||
type CSVSample struct {
|
||
Delimiter string `json:"delimiter"`
|
||
HeaderRow int `json:"header_row"`
|
||
Headers []string `json:"headers"`
|
||
ShapedRows [][]string `json:"shaped_rows"`
|
||
RecordCount int `json:"record_count"`
|
||
}
|
||
|
||
// ReadCSV decodes an uploaded statement. UTF-8 and Windows-1252 (still emitted
|
||
// by some ING exports) are accepted, along with a BOM, CRLF, comma/semicolon/tab
|
||
// delimiters and RFC4180 quoted multiline fields.
|
||
func ReadCSV(input io.Reader) (CSVFile, error) {
|
||
raw, err := io.ReadAll(io.LimitReader(input, maxCSVBytes+1))
|
||
if err != nil {
|
||
return CSVFile{}, fmt.Errorf("read CSV: %w", err)
|
||
}
|
||
if len(raw) > maxCSVBytes {
|
||
return CSVFile{}, fmt.Errorf("CSV statement exceeds %d MiB", maxCSVBytes>>20)
|
||
}
|
||
text, err := decodeCSVText(raw)
|
||
if err != nil {
|
||
return CSVFile{}, err
|
||
}
|
||
if strings.TrimSpace(text) == "" {
|
||
return CSVFile{}, errors.New("CSV statement is empty")
|
||
}
|
||
// Choose the delimiter that yields the widest consistently parsed records:
|
||
// descriptions routinely contain the delimiters used by other banks.
|
||
delimiter, best := ' ', -1
|
||
for _, candidate := range []rune{',', ';', '\t'} {
|
||
if _, score := parseCSVRecords(text, candidate); score > best {
|
||
delimiter, best = candidate, score
|
||
}
|
||
}
|
||
if best <= 0 {
|
||
return CSVFile{}, errors.New("unreadable CSV statement: no delimiter produced multi-column records")
|
||
}
|
||
rows, _ := parseCSVRecords(text, delimiter)
|
||
for _, row := range rows {
|
||
if len(row) > maxCSVColumns {
|
||
return CSVFile{}, fmt.Errorf("CSV statement has more than %d columns", maxCSVColumns)
|
||
}
|
||
}
|
||
return CSVFile{delimiter: delimiter, rows: rows}, nil
|
||
}
|
||
|
||
func decodeCSVText(raw []byte) (string, error) {
|
||
text := strings.TrimPrefix(string(raw), "\ufeff")
|
||
if strings.IndexByte(text, 0) >= 0 {
|
||
return "", errors.New("CSV statement contains a NUL byte")
|
||
}
|
||
if utf8.ValidString(text) {
|
||
return text, nil
|
||
}
|
||
// Windows-1252 is decoded locally: undecodable account text must never be
|
||
// forwarded to a model or stored as invalid UTF-8 bank facts.
|
||
windows1252 := [...]rune{'€', 0, '‚', 'ƒ', '„', '…', '†', '‡', 'ˆ', '‰', 'Š', '‹', 'Œ', 0, 'Ž', 0, 0, '‘', '’', '“', '”', '•', '–', '—', '˜', '™', 'š', '›', 'œ', 0, 'ž', 'Ÿ'}
|
||
var out strings.Builder
|
||
out.Grow(len(raw))
|
||
for _, b := range []byte(text) {
|
||
switch {
|
||
case b < 0x80 || b >= 0xa0:
|
||
out.WriteRune(rune(b))
|
||
default:
|
||
r := windows1252[int(b)-0x80]
|
||
if r == 0 {
|
||
return "", errors.New("CSV statement is neither valid UTF-8 nor Windows-1252")
|
||
}
|
||
out.WriteRune(r)
|
||
}
|
||
}
|
||
return out.String(), nil
|
||
}
|
||
|
||
// parseCSVRecords returns every record plus a preference score. Malformed
|
||
// quoting scores below zero so a delimiter is never silently accepted for a
|
||
// document it cannot represent.
|
||
func parseCSVRecords(text string, delimiter rune) ([][]string, int) {
|
||
parser := csv.NewReader(strings.NewReader(text))
|
||
parser.Comma = delimiter
|
||
parser.FieldsPerRecord = -1
|
||
parser.ReuseRecord = false
|
||
rows := make([][]string, 0, 64)
|
||
widest, populated := 0, 0
|
||
for {
|
||
row, err := parser.Read()
|
||
if err == io.EOF {
|
||
break
|
||
}
|
||
if err != nil {
|
||
return nil, -1
|
||
}
|
||
rows = append(rows, row)
|
||
if len(row) > 1 && !blankCSVRow(row) {
|
||
populated++
|
||
if len(row) > widest {
|
||
widest = len(row)
|
||
}
|
||
}
|
||
}
|
||
if widest == 0 {
|
||
return rows, 0
|
||
}
|
||
return rows, widest*1_000_000 + populated
|
||
}
|
||
|
||
func blankCSVRow(row []string) bool {
|
||
for _, value := range row {
|
||
if strings.TrimSpace(value) != "" {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// Sample locates the column header and describes the statement's shape. The
|
||
// header is the first widest record, which skips the metadata preamble that
|
||
// banks such as ING place above their columns.
|
||
func (f CSVFile) Sample() (CSVSample, error) {
|
||
header, width := 0, 0
|
||
for i, row := range f.rows {
|
||
if i >= maxCSVPreambleRows {
|
||
break
|
||
}
|
||
if len(row) > width && !blankCSVRow(row) {
|
||
header, width = i+1, len(row)
|
||
}
|
||
}
|
||
if width < 2 {
|
||
return CSVSample{}, errors.New("CSV statement has no column header row")
|
||
}
|
||
headers, err := csvHeaders(f, header)
|
||
if err != nil {
|
||
return CSVSample{}, err
|
||
}
|
||
sample := CSVSample{Delimiter: string(f.delimiter), HeaderRow: header, Headers: headers, ShapedRows: [][]string{}}
|
||
for _, row := range f.rows[header:] {
|
||
if blankCSVRow(row) {
|
||
continue
|
||
}
|
||
if len(row) != width {
|
||
return CSVSample{}, fmt.Errorf("CSV statement has ragged records: expected %d columns", width)
|
||
}
|
||
sample.RecordCount++
|
||
if len(sample.ShapedRows) < maxCSVSampleRows {
|
||
shaped := make([]string, len(row))
|
||
for i, value := range row {
|
||
shaped[i] = shapeCSVValue(value)
|
||
}
|
||
sample.ShapedRows = append(sample.ShapedRows, shaped)
|
||
}
|
||
}
|
||
if sample.RecordCount == 0 {
|
||
return CSVSample{}, errors.New("CSV statement contains no transaction records")
|
||
}
|
||
return sample, nil
|
||
}
|
||
|
||
func csvHeaders(f CSVFile, header int) ([]string, error) {
|
||
if header < 1 || header > len(f.rows) {
|
||
return nil, errors.New("CSV mapping has an invalid header row")
|
||
}
|
||
headers := make([]string, 0, len(f.rows[header-1]))
|
||
seen := map[string]bool{}
|
||
for _, raw := range f.rows[header-1] {
|
||
name := strings.TrimSpace(raw)
|
||
key := headerName(name)
|
||
if key == "" {
|
||
return nil, errors.New("CSV column headers must all be named")
|
||
}
|
||
if seen[key] {
|
||
return nil, fmt.Errorf("duplicate CSV column %q", name)
|
||
}
|
||
seen[key] = true
|
||
headers = append(headers, name)
|
||
}
|
||
return headers, nil
|
||
}
|
||
|
||
// shapeCSVValue keeps only structure: letters become x, digits 0. This is what a
|
||
// column mapping needs, and it keeps descriptions, names, references, IBANs and
|
||
// amounts out of any request that leaves this machine.
|
||
func shapeCSVValue(value string) string {
|
||
var out strings.Builder
|
||
for i, r := range strings.TrimSpace(value) {
|
||
if i >= 64 {
|
||
out.WriteRune('…')
|
||
break
|
||
}
|
||
switch {
|
||
case unicode.IsLetter(r):
|
||
out.WriteRune('x')
|
||
case unicode.IsDigit(r):
|
||
out.WriteRune('0')
|
||
case unicode.IsSpace(r):
|
||
out.WriteRune(' ')
|
||
default:
|
||
out.WriteRune(r)
|
||
}
|
||
}
|
||
return out.String()
|
||
}
|
||
|
||
// DetectCSVMapping recognizes known bank exports without any model. The returned
|
||
// source identifies imported facts; label names the export for the operator.
|
||
func DetectCSVMapping(f CSVFile) (mapping CSVMapping, source, label string, ok bool) {
|
||
for i, row := range f.rows {
|
||
if i >= maxCSVPreambleRows {
|
||
break
|
||
}
|
||
columns, usable := csvColumnIndex(row)
|
||
if !usable {
|
||
continue
|
||
}
|
||
if mapping, ok := n26Mapping(i+1, columns); ok {
|
||
return mapping, "n26_csv", "N26", true
|
||
}
|
||
if mapping, ok := ingMapping(i+1, columns); ok {
|
||
return mapping, "ing_csv", "ING", true
|
||
}
|
||
if mapping, ok := kontistMapping(f, i+1, columns); ok {
|
||
return mapping, "kontist_csv", "Kontist", true
|
||
}
|
||
}
|
||
return CSVMapping{}, "", "", false
|
||
}
|
||
|
||
// csvColumnIndex maps normalized column names to their exact header text.
|
||
func csvColumnIndex(row []string) (map[string]string, bool) {
|
||
columns := make(map[string]string, len(row))
|
||
for _, raw := range row {
|
||
actual := strings.TrimSpace(raw)
|
||
name := headerName(actual)
|
||
if name == "" {
|
||
return nil, false
|
||
}
|
||
if _, exists := columns[name]; exists {
|
||
return nil, false
|
||
}
|
||
columns[name] = actual
|
||
}
|
||
return columns, len(columns) > 1
|
||
}
|
||
|
||
func csvColumn(columns map[string]string, names ...string) string {
|
||
for _, name := range names {
|
||
if actual, ok := columns[name]; ok {
|
||
return actual
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// n26Mapping accepts N26's English and German account-activity exports,
|
||
// including their older Date/Datum and newer Booking Date/Buchungsdatum
|
||
// schemas. Foreign original amounts, exchange rates and categories are
|
||
// deliberately never used as account money.
|
||
func n26Mapping(header int, columns map[string]string) (CSVMapping, bool) {
|
||
amount, currency := "", ""
|
||
for name, actual := range columns {
|
||
if name == "amount" || name == "betrag" {
|
||
amount = actual
|
||
continue
|
||
}
|
||
for _, prefix := range []string{"amount (", "betrag ("} {
|
||
if strings.HasPrefix(name, prefix) && strings.HasSuffix(name, ")") {
|
||
code := strings.ToUpper(strings.TrimSuffix(strings.TrimPrefix(name, prefix), ")"))
|
||
if validCurrency(code) {
|
||
amount, currency = actual, code
|
||
}
|
||
}
|
||
}
|
||
}
|
||
mapping := CSVMapping{
|
||
HeaderRow: header,
|
||
BookingDateColumn: csvColumn(columns, "date", "datum", "booking date", "buchungsdatum"),
|
||
ValueDateColumn: csvColumn(columns, "value date", "wertstellung", "wertstellungsdatum", "valutadatum"),
|
||
AmountColumn: amount,
|
||
CurrencyColumn: csvColumn(columns, "currency", "währung"),
|
||
DescriptionColumn: csvColumn(columns, "payment reference", "verwendungszweck", "reference", "beschreibung"),
|
||
FallbackDescriptionColumn: csvColumn(columns, "payment type", "transaktionstyp", "zahlungstyp", "type", "typ"),
|
||
CounterpartyColumn: csvColumn(columns, "payee", "partner name", "zahlungsempfänger", "zahlungsempfänger name", "empfänger", "empfänger/auftraggeber", "partnername", "name zahlungspartner"),
|
||
CounterpartyIBANColumn: csvColumn(columns, "account number", "partner iban", "kontonummer", "iban", "konto"),
|
||
ExternalIDColumn: csvColumn(columns, "transaction id", "transaktions-id", "transaktions id"),
|
||
DateFormat: "iso-or-german",
|
||
DecimalFormat: "dot-or-comma",
|
||
FixedCurrency: currency,
|
||
}
|
||
// An N26 export always carries a payment reference and a typed transaction.
|
||
marker := mapping.FallbackDescriptionColumn != "" || csvColumn(columns, "original amount", "betrag (fremdwährung)", "account name", "partner name", "payee") != ""
|
||
if mapping.BookingDateColumn == "" || mapping.AmountColumn == "" || mapping.DescriptionColumn == "" || !marker {
|
||
return CSVMapping{}, false
|
||
}
|
||
return mapping, true
|
||
}
|
||
|
||
// ingMapping accepts ING's Umsatzanzeige export, whose columns sit below a
|
||
// metadata preamble. Gläubiger-ID, Mandatsreferenz and Kundenreferenz are SEPA
|
||
// mandate references that repeat across bookings, so they are never used as a
|
||
// transaction identity.
|
||
func ingMapping(header int, columns map[string]string) (CSVMapping, bool) {
|
||
mapping := CSVMapping{
|
||
HeaderRow: header,
|
||
BookingDateColumn: csvColumn(columns, "buchung"),
|
||
ValueDateColumn: csvColumn(columns, "wertstellungsdatum", "valuta"),
|
||
AmountColumn: csvColumn(columns, "betrag"),
|
||
CurrencyColumn: csvColumn(columns, "währung", "waehrung"),
|
||
DescriptionColumn: csvColumn(columns, "verwendungszweck"),
|
||
FallbackDescriptionColumn: csvColumn(columns, "buchungstext"),
|
||
CounterpartyColumn: csvColumn(columns, "auftraggeber/empfänger", "auftraggeber/empfaenger"),
|
||
DateFormat: "dd.mm.yyyy",
|
||
DecimalFormat: "comma",
|
||
}
|
||
if mapping.BookingDateColumn == "" || mapping.AmountColumn == "" || mapping.DescriptionColumn == "" || mapping.CounterpartyColumn == "" {
|
||
return CSVMapping{}, false
|
||
}
|
||
return mapping, true
|
||
}
|
||
|
||
// kontistMapping accepts Kontist's documented transaction vocabulary: a payment
|
||
// date, an amount, a purpose and a counterparty name. Date and decimal
|
||
// conventions are inferred from the file's own first populated values, and the
|
||
// mapping is always reviewed before anything is imported.
|
||
func kontistMapping(f CSVFile, header int, columns map[string]string) (CSVMapping, bool) {
|
||
mapping := CSVMapping{
|
||
HeaderRow: header,
|
||
BookingDateColumn: csvColumn(columns, "payment date", "booking date", "buchungsdatum", "zahlungsdatum"),
|
||
ValueDateColumn: csvColumn(columns, "value date", "wertstellungsdatum", "valuta"),
|
||
AmountColumn: csvColumn(columns, "amount", "betrag"),
|
||
CurrencyColumn: csvColumn(columns, "currency", "währung", "waehrung"),
|
||
DescriptionColumn: csvColumn(columns, "purpose", "verwendungszweck", "payment reference"),
|
||
CounterpartyColumn: csvColumn(columns, "name", "counterparty", "zahlungspartner"),
|
||
CounterpartyIBANColumn: csvColumn(columns, "iban", "counterparty iban"),
|
||
ExternalIDColumn: csvColumn(columns, "transaction id", "transaction_id", "transaktions-id"),
|
||
}
|
||
if mapping.BookingDateColumn == "" || mapping.AmountColumn == "" || mapping.DescriptionColumn == "" || mapping.CounterpartyColumn == "" {
|
||
return CSVMapping{}, false
|
||
}
|
||
mapping.DateFormat = inferCSVDateFormat(csvFirstValue(f, header, mapping.BookingDateColumn))
|
||
mapping.DecimalFormat = inferCSVDecimalFormat(csvFirstValue(f, header, mapping.AmountColumn))
|
||
return mapping, true
|
||
}
|
||
|
||
func csvFirstValue(f CSVFile, header int, column string) string {
|
||
headers := f.rows[header-1]
|
||
index := -1
|
||
for i, name := range headers {
|
||
if strings.TrimSpace(name) == column {
|
||
index = i
|
||
break
|
||
}
|
||
}
|
||
if index < 0 {
|
||
return ""
|
||
}
|
||
for _, row := range f.rows[header:] {
|
||
if len(row) == len(headers) && strings.TrimSpace(row[index]) != "" {
|
||
return strings.TrimSpace(row[index])
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func inferCSVDateFormat(value string) string {
|
||
switch {
|
||
case len(value) >= 11 && value[4] == '-' && (value[10] == 'T' || value[10] == ' '):
|
||
return "iso-date-time"
|
||
case len(value) == 10 && value[4] == '-' && value[7] == '-':
|
||
return "yyyy-mm-dd"
|
||
case strings.Count(value, ".") == 2:
|
||
return "dd.mm.yyyy"
|
||
case strings.Count(value, "/") == 2:
|
||
// Kontist documents month/day/year for interchange. A first component
|
||
// above twelve can only be a day; the preview shows the parsed result.
|
||
if first, err := strconv.Atoi(strings.SplitN(value, "/", 2)[0]); err == nil && first > 12 {
|
||
return "dd/mm/yyyy"
|
||
}
|
||
return "mm/dd/yyyy"
|
||
default:
|
||
return "yyyy-mm-dd"
|
||
}
|
||
}
|
||
|
||
func inferCSVDecimalFormat(value string) string {
|
||
if strings.Contains(value, ",") {
|
||
return "comma"
|
||
}
|
||
return "dot"
|
||
}
|
||
|
||
// ParseMappedCSV converts every record into bank facts. A single malformed
|
||
// record fails the whole statement: a partially imported statement cannot be
|
||
// distinguished from a truncated export later.
|
||
func ParseMappedCSV(f CSVFile, account domain.Account, mapping CSVMapping, source string) ([]domain.Facts, error) {
|
||
if account.ID == "" {
|
||
return nil, errors.New("CSV requires a selected account")
|
||
}
|
||
if source == "" {
|
||
return nil, errors.New("CSV import source is required")
|
||
}
|
||
columns, err := validateCSVMapping(f, mapping)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
headers := f.rows[mapping.HeaderRow-1]
|
||
get := func(row []string, column string) string {
|
||
if column == "" {
|
||
return ""
|
||
}
|
||
return strings.TrimSpace(row[columns[column]])
|
||
}
|
||
facts := make([]domain.Facts, 0, len(f.rows)-mapping.HeaderRow)
|
||
for offset, row := range f.rows[mapping.HeaderRow:] {
|
||
record := mapping.HeaderRow + offset + 1
|
||
if blankCSVRow(row) {
|
||
continue
|
||
}
|
||
if len(row) != len(headers) {
|
||
return nil, fmt.Errorf("CSV record %d has %d columns, expected %d", record, len(row), len(headers))
|
||
}
|
||
booking, err := parseMappedCSVDate(get(row, mapping.BookingDateColumn), mapping.DateFormat)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("invalid booking date in CSV record %d", record)
|
||
}
|
||
value := get(row, mapping.ValueDateColumn)
|
||
if value != "" {
|
||
if value, err = parseMappedCSVDate(value, mapping.DateFormat); err != nil {
|
||
return nil, fmt.Errorf("invalid value date in CSV record %d", record)
|
||
}
|
||
}
|
||
amount, err := mappedCSVAmount(row, get, mapping)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("invalid account amount in CSV record %d", record)
|
||
}
|
||
currency := strings.ToUpper(get(row, mapping.CurrencyColumn))
|
||
if currency == "€" {
|
||
currency = "EUR"
|
||
}
|
||
if currency == "" {
|
||
currency = strings.ToUpper(mapping.FixedCurrency)
|
||
}
|
||
if currency == "" {
|
||
currency = strings.ToUpper(account.Currency)
|
||
}
|
||
fixed := strings.ToUpper(mapping.FixedCurrency)
|
||
if !validCurrency(currency) || (fixed != "" && currency != fixed) || (account.Currency != "" && currency != strings.ToUpper(account.Currency)) {
|
||
return nil, fmt.Errorf("invalid or conflicting account currency in CSV record %d", record)
|
||
}
|
||
description := get(row, mapping.DescriptionColumn)
|
||
if description == "" {
|
||
description = get(row, mapping.FallbackDescriptionColumn)
|
||
}
|
||
facts = append(facts, domain.Facts{
|
||
Source: source, AccountID: account.ID, BookingDate: booking, ValueDate: value,
|
||
Amount: amount, Currency: currency, RawDescription: description,
|
||
ExternalID: get(row, mapping.ExternalIDColumn), Counterparty: get(row, mapping.CounterpartyColumn),
|
||
CounterpartyIBAN: normalizeIBAN(get(row, mapping.CounterpartyIBANColumn)),
|
||
})
|
||
}
|
||
if len(facts) == 0 {
|
||
return nil, errors.New("CSV statement contains no transaction records")
|
||
}
|
||
return facts, nil
|
||
}
|
||
|
||
// validateCSVMapping resolves a mapping against the document and returns each
|
||
// mapped column's index. Every referenced column must exist exactly, no column
|
||
// may serve two fields, and money must come from exactly one strategy.
|
||
func validateCSVMapping(f CSVFile, mapping CSVMapping) (map[string]int, error) {
|
||
if mapping.HeaderRow < 1 || mapping.HeaderRow > len(f.rows) || mapping.HeaderRow > maxCSVPreambleRows {
|
||
return nil, errors.New("CSV mapping has an invalid header row")
|
||
}
|
||
if _, err := csvHeaders(f, mapping.HeaderRow); err != nil {
|
||
return nil, err
|
||
}
|
||
indexes := make(map[string]int, len(f.rows[mapping.HeaderRow-1]))
|
||
for i, raw := range f.rows[mapping.HeaderRow-1] {
|
||
indexes[strings.TrimSpace(raw)] = i
|
||
}
|
||
if mapping.BookingDateColumn == "" {
|
||
return nil, errors.New("CSV mapping requires a booking date column")
|
||
}
|
||
if mapping.DescriptionColumn == "" {
|
||
return nil, errors.New("CSV mapping requires a description column")
|
||
}
|
||
signed, split := mapping.AmountColumn != "", mapping.DebitColumn != "" || mapping.CreditColumn != ""
|
||
if signed == split {
|
||
return nil, errors.New("CSV mapping requires either one signed amount column or separate debit and credit columns")
|
||
}
|
||
if split && (mapping.DebitColumn == "" || mapping.CreditColumn == "") {
|
||
return nil, errors.New("CSV mapping requires both a debit and a credit column")
|
||
}
|
||
if !validCSVDateFormat(mapping.DateFormat) {
|
||
return nil, fmt.Errorf("unsupported CSV date format %q", mapping.DateFormat)
|
||
}
|
||
if !validCSVDecimalFormat(mapping.DecimalFormat) {
|
||
return nil, fmt.Errorf("unsupported CSV decimal format %q", mapping.DecimalFormat)
|
||
}
|
||
if mapping.FixedCurrency != "" && !validCurrency(strings.ToUpper(mapping.FixedCurrency)) {
|
||
return nil, errors.New("CSV mapping has an invalid fixed currency")
|
||
}
|
||
assigned := map[string]string{}
|
||
for _, field := range []struct{ name, column string }{
|
||
{"booking date", mapping.BookingDateColumn}, {"value date", mapping.ValueDateColumn},
|
||
{"amount", mapping.AmountColumn}, {"debit", mapping.DebitColumn}, {"credit", mapping.CreditColumn},
|
||
{"currency", mapping.CurrencyColumn}, {"description", mapping.DescriptionColumn},
|
||
{"secondary description", mapping.FallbackDescriptionColumn}, {"counterparty", mapping.CounterpartyColumn},
|
||
{"counterparty IBAN", mapping.CounterpartyIBANColumn}, {"transaction reference", mapping.ExternalIDColumn},
|
||
} {
|
||
if field.column == "" {
|
||
continue
|
||
}
|
||
if _, ok := indexes[field.column]; !ok {
|
||
return nil, fmt.Errorf("CSV mapping references unknown %s column %q", field.name, field.column)
|
||
}
|
||
if previous, ok := assigned[field.column]; ok {
|
||
return nil, fmt.Errorf("CSV column %q is mapped to both %s and %s", field.column, previous, field.name)
|
||
}
|
||
assigned[field.column] = field.name
|
||
}
|
||
return indexes, nil
|
||
}
|
||
|
||
// CSVDateFormats and CSVDecimalFormats are the exact accepted conventions. A
|
||
// proposed mapping outside them is rejected rather than guessed.
|
||
func CSVDateFormats() []string {
|
||
return []string{"yyyy-mm-dd", "dd.mm.yyyy", "mm/dd/yyyy", "dd/mm/yyyy", "iso-date-time", "iso-or-german"}
|
||
}
|
||
func CSVDecimalFormats() []string { return []string{"dot", "comma", "dot-or-comma"} }
|
||
|
||
func validCSVDateFormat(format string) bool {
|
||
for _, valid := range CSVDateFormats() {
|
||
if format == valid {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
func validCSVDecimalFormat(format string) bool {
|
||
for _, valid := range CSVDecimalFormats() {
|
||
if format == valid {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func parseMappedCSVDate(value, format string) (string, error) {
|
||
layouts := map[string][]string{
|
||
"yyyy-mm-dd": {"2006-01-02"},
|
||
"dd.mm.yyyy": {"02.01.2006", "2.1.2006"},
|
||
"mm/dd/yyyy": {"01/02/2006", "1/2/2006"},
|
||
"dd/mm/yyyy": {"02/01/2006", "2/1/2006"},
|
||
"iso-date-time": {time.RFC3339, "2006-01-02T15:04:05", "2006-01-02 15:04:05", "2006-01-02"},
|
||
"iso-or-german": {"2006-01-02", "02.01.2006", "2.1.2006"},
|
||
}
|
||
for _, layout := range layouts[format] {
|
||
if parsed, err := time.Parse(layout, value); err == nil {
|
||
return parsed.Format("2006-01-02"), nil
|
||
}
|
||
}
|
||
return "", fmt.Errorf("invalid date %q", value)
|
||
}
|
||
|
||
// mappedCSVAmount returns signed account money. With split columns a debit is
|
||
// negative however the bank wrote its sign, a credit must not be negative, and
|
||
// the unused column may be empty or an explicit zero but never carry money.
|
||
func mappedCSVAmount(row []string, get func([]string, string) string, mapping CSVMapping) (domain.Money, error) {
|
||
if mapping.AmountColumn != "" {
|
||
return parseMappedCSVDecimal(get(row, mapping.AmountColumn), mapping.DecimalFormat)
|
||
}
|
||
debit, credit := get(row, mapping.DebitColumn), get(row, mapping.CreditColumn)
|
||
if debit == "" && credit == "" {
|
||
return "", errors.New("debit and credit are both empty")
|
||
}
|
||
parse := func(value string) (domain.Money, int64, error) {
|
||
if value == "" {
|
||
return "0.00", 0, nil
|
||
}
|
||
money, err := parseMappedCSVDecimal(value, mapping.DecimalFormat)
|
||
if err != nil {
|
||
return "", 0, err
|
||
}
|
||
minor, err := money.Minor()
|
||
return money, minor, err
|
||
}
|
||
debitMoney, debited, err := parse(debit)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
creditMoney, credited, err := parse(credit)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
if debited != 0 && credited != 0 {
|
||
return "", errors.New("debit and credit both carry money")
|
||
}
|
||
if credited < 0 {
|
||
return "", errors.New("a credit column must not hold negative money")
|
||
}
|
||
if credited != 0 {
|
||
return creditMoney, nil
|
||
}
|
||
if debited != 0 {
|
||
return domain.ParseMoney("-" + strings.TrimPrefix(debitMoney.String(), "-"))
|
||
}
|
||
return domain.ParseMoney("0")
|
||
}
|
||
|
||
func parseMappedCSVDecimal(value, format string) (domain.Money, error) {
|
||
value = strings.NewReplacer("\u00a0", "", "\u202f", "", "'", "").Replace(strings.TrimSpace(value))
|
||
switch format {
|
||
case "dot-or-comma":
|
||
return parseCSVAmount(value)
|
||
case "comma":
|
||
// A dot can only be grouping here, and only in exact thousands groups.
|
||
if !strings.Contains(value, ",") && strings.Contains(value, ".") {
|
||
if digits, ok := ungroup(value, "."); ok {
|
||
value = digits
|
||
}
|
||
}
|
||
return parseCSVAmount(value)
|
||
case "dot":
|
||
value = strings.TrimPrefix(value, "+")
|
||
if strings.Contains(value, ",") {
|
||
digits, ok := ungroup(value, ",")
|
||
if !ok {
|
||
return "", errors.New("invalid grouping")
|
||
}
|
||
value = digits
|
||
}
|
||
return domain.ParseMoney(value)
|
||
default:
|
||
return "", fmt.Errorf("unsupported CSV decimal format %q", format)
|
||
}
|
||
}
|
||
|
||
// ungroup removes thousands separators, and only when every group is exactly
|
||
// three digits: "1.234" is 1234, while "1.23" stays a decimal value.
|
||
func ungroup(value, separator string) (string, bool) {
|
||
sign := ""
|
||
if rest, found := strings.CutPrefix(value, "-"); found {
|
||
sign, value = "-", rest
|
||
}
|
||
groups := strings.Split(value, separator)
|
||
if len(groups) < 2 || len(groups[0]) < 1 || len(groups[0]) > 3 {
|
||
return "", false
|
||
}
|
||
for _, group := range groups[1:] {
|
||
if len(group) != 3 {
|
||
return "", false
|
||
}
|
||
}
|
||
return sign + strings.Join(groups, ""), true
|
||
}
|
||
|
||
func headerName(s string) string {
|
||
return strings.ToLower(strings.Join(strings.Fields(strings.TrimPrefix(s, "\ufeff")), " "))
|
||
}
|
||
func normalizeIBAN(s string) string {
|
||
return strings.ToUpper(strings.Map(func(r rune) rune {
|
||
if unicode.IsSpace(r) {
|
||
return -1
|
||
}
|
||
return r
|
||
}, s))
|
||
}
|
||
func validCurrency(s string) bool {
|
||
if len(s) != 3 {
|
||
return false
|
||
}
|
||
for _, c := range s {
|
||
if c < 'A' || c > 'Z' {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
func parseDate(s string) (string, error) {
|
||
for _, layout := range []string{"2006-01-02", "02.01.2006", "2.1.2006"} {
|
||
if d, e := time.Parse(layout, s); e == nil {
|
||
return d.Format("2006-01-02"), nil
|
||
}
|
||
}
|
||
return "", fmt.Errorf("invalid date")
|
||
}
|
||
func parseCSVAmount(s string) (domain.Money, error) {
|
||
s = strings.TrimPrefix(strings.TrimSpace(s), "+")
|
||
// German grouping is only accepted when every group is exactly three digits.
|
||
if strings.Contains(s, ",") {
|
||
if strings.Count(s, ",") != 1 {
|
||
return "", fmt.Errorf("invalid decimal separator")
|
||
}
|
||
pair := strings.SplitN(s, ",", 2)
|
||
if strings.Contains(pair[0], ".") {
|
||
digits, ok := ungroup(pair[0], ".")
|
||
if !ok {
|
||
return "", fmt.Errorf("invalid grouping")
|
||
}
|
||
pair[0] = digits
|
||
}
|
||
s = pair[0] + "." + pair[1]
|
||
}
|
||
return domain.ParseMoney(s)
|
||
}
|