Import ING and Kontist statements behind a reviewed column mapping
CSV import is now mapping-driven: N26, ING (metadata preamble, Windows-1252, German decimals) and Kontist exports are recognized locally, and any other layout can have its columns proposed by the configured model from a sample in which letters are replaced by x and digits by 0. Proposals are untrusted: every column must name a supplied header, money must come from one signed column or one debit/credit pair, and formats must be from a closed list. Uploading no longer imports. /api/import is replaced by prepare/confirm/cancel: prepare parses, deduplicates and previews the exact facts, and only confirming at the reviewed revision writes them. ING and AI-mapped facts carry no transaction reference, because repeating SEPA mandate references must never become a transaction identity.
This commit is contained in:
Lars Nolden
+258
-11
@@ -77,26 +77,273 @@ func (a *App) importFacts(ctx context.Context, s State, facts []domain.Facts) (I
|
||||
}
|
||||
return ImportResult{Imported: len(added), State: state}, nil
|
||||
}
|
||||
func (a *App) ImportCSV(ctx context.Context, rev, accountID string, r io.Reader) (ImportResult, error) {
|
||||
|
||||
// CSVColumn is one reviewable source-column assignment.
|
||||
type CSVColumn struct {
|
||||
Field string `json:"field"`
|
||||
Column string `json:"column"`
|
||||
}
|
||||
|
||||
// CSVImport is a parsed statement awaiting confirmation. Nothing is written to
|
||||
// the journal until ConfirmCSVImport applies the exact facts previewed here.
|
||||
type CSVImport struct {
|
||||
ID string `json:"id"`
|
||||
Revision string `json:"revision"`
|
||||
AccountID string `json:"account_id"`
|
||||
Source string `json:"source"`
|
||||
SourceLabel string `json:"source_label"`
|
||||
MappedBy string `json:"mapped_by"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Mapping banking.CSVMapping `json:"mapping"`
|
||||
Columns []CSVColumn `json:"columns"`
|
||||
Records int `json:"records"`
|
||||
New int `json:"new"`
|
||||
Duplicates int `json:"duplicates"`
|
||||
Samples []domain.Facts `json:"samples"`
|
||||
|
||||
facts []domain.Facts
|
||||
created time.Time
|
||||
}
|
||||
|
||||
const csvImportLifetime = time.Hour
|
||||
const maxPreparedCSVImports = 5
|
||||
const maxCSVSamples = 10
|
||||
|
||||
// PrepareCSVImport maps and parses an uploaded statement without importing it.
|
||||
// Known N26, ING and Kontist exports are recognized locally; any other layout
|
||||
// needs a configured model to propose a column mapping from the statement's
|
||||
// redacted shape. The result must be reviewed and confirmed.
|
||||
func (a *App) PrepareCSVImport(ctx context.Context, rev, accountID string, r io.Reader) (CSVImport, error) {
|
||||
a.mu.Lock()
|
||||
s, err := a.snapshot(ctx)
|
||||
model := strings.TrimSpace(a.settings.Model)
|
||||
client := a.classifier.WithModel(model)
|
||||
configured := strings.TrimSpace(a.classifier.APIKey) != "" && model != ""
|
||||
a.mu.Unlock()
|
||||
if err != nil {
|
||||
return CSVImport{}, err
|
||||
}
|
||||
if rev != s.Revision {
|
||||
return CSVImport{}, errors.New("revision conflict: reload before importing")
|
||||
}
|
||||
index := slices.IndexFunc(s.Data.Accounts, func(account domain.Account) bool { return account.ID == accountID })
|
||||
if index < 0 {
|
||||
return CSVImport{}, errors.New("unknown account")
|
||||
}
|
||||
account := s.Data.Accounts[index]
|
||||
file, err := banking.ReadCSV(r)
|
||||
if err != nil {
|
||||
return CSVImport{}, err
|
||||
}
|
||||
prepared := CSVImport{ID: domain.NewID("csvimport"), Revision: s.Revision, AccountID: account.ID, MappedBy: "preset", created: time.Now()}
|
||||
mapping, source, label, recognized := banking.DetectCSVMapping(file)
|
||||
if !recognized {
|
||||
sample, e := file.Sample()
|
||||
if e != nil {
|
||||
return CSVImport{}, e
|
||||
}
|
||||
if !configured {
|
||||
return CSVImport{}, errors.New("unrecognized CSV layout: import an N26, ING or Kontist export, or configure an OpenRouter key and model in Settings to map these columns")
|
||||
}
|
||||
proposal, e := client.ProposeCSVMapping(ctx, classification.CSVMappingRequest{
|
||||
Delimiter: sample.Delimiter, Headers: sample.Headers, ShapedRows: sample.ShapedRows,
|
||||
DateFormats: banking.CSVDateFormats(), DecimalFormats: banking.CSVDecimalFormats(),
|
||||
})
|
||||
if e != nil {
|
||||
return CSVImport{}, e
|
||||
}
|
||||
mapping = banking.CSVMapping{
|
||||
HeaderRow: sample.HeaderRow,
|
||||
BookingDateColumn: proposal.BookingDateColumn,
|
||||
ValueDateColumn: proposal.ValueDateColumn,
|
||||
AmountColumn: proposal.AmountColumn,
|
||||
DebitColumn: proposal.DebitColumn,
|
||||
CreditColumn: proposal.CreditColumn,
|
||||
CurrencyColumn: proposal.CurrencyColumn,
|
||||
DescriptionColumn: proposal.DescriptionColumn,
|
||||
CounterpartyColumn: proposal.CounterpartyColumn,
|
||||
CounterpartyIBANColumn: proposal.CounterpartyIBANColumn,
|
||||
DateFormat: proposal.DateFormat,
|
||||
DecimalFormat: proposal.DecimalFormat,
|
||||
}
|
||||
source, label = "csv", "AI-mapped CSV"
|
||||
prepared.MappedBy, prepared.Model = "openrouter", proposal.Model
|
||||
}
|
||||
facts, err := banking.ParseMappedCSV(file, account, mapping, source)
|
||||
if err != nil {
|
||||
return CSVImport{}, err
|
||||
}
|
||||
// Dedupe now so the preview reports what confirming would actually add, and
|
||||
// so cross-source conflicts are reported before anything is written.
|
||||
added, err := banking.NormalizeAndDedupe(s.Data, facts)
|
||||
if err != nil {
|
||||
return CSVImport{}, err
|
||||
}
|
||||
prepared.Source, prepared.SourceLabel, prepared.Mapping = source, label, mapping
|
||||
prepared.Columns = csvColumns(mapping, account)
|
||||
prepared.Records, prepared.New, prepared.Duplicates = len(facts), len(added), len(facts)-len(added)
|
||||
prepared.Samples, prepared.facts = csvSamples(facts), facts
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
for id, old := range a.csvImports {
|
||||
if time.Since(old.created) > csvImportLifetime {
|
||||
delete(a.csvImports, id)
|
||||
}
|
||||
}
|
||||
if len(a.csvImports) >= maxPreparedCSVImports {
|
||||
return CSVImport{}, errors.New("too many statements awaiting confirmation; confirm or cancel one first")
|
||||
}
|
||||
a.csvImports[prepared.ID] = prepared
|
||||
return prepared, nil
|
||||
}
|
||||
|
||||
// ConfirmCSVImport imports exactly the facts that were previewed, provided the
|
||||
// journal has not changed since.
|
||||
func (a *App) ConfirmCSVImport(ctx context.Context, id, rev string) (ImportResult, error) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
prepared, ok := a.csvImports[id]
|
||||
if !ok || time.Since(prepared.created) > csvImportLifetime {
|
||||
return ImportResult{}, errors.New("prepared import expired or unknown; upload the statement again")
|
||||
}
|
||||
if rev != prepared.Revision {
|
||||
return ImportResult{}, errors.New("revision conflict: reload before importing")
|
||||
}
|
||||
s, err := a.snapshot(ctx)
|
||||
if err != nil {
|
||||
return ImportResult{}, err
|
||||
}
|
||||
if rev != s.Revision {
|
||||
return ImportResult{}, errors.New("revision conflict: reload before importing")
|
||||
if s.Revision != prepared.Revision {
|
||||
return ImportResult{}, errors.New("revision conflict: data changed after the preview; upload the statement again")
|
||||
}
|
||||
for _, account := range s.Data.Accounts {
|
||||
if account.ID == accountID {
|
||||
facts, e := banking.ParseCSV(r, account)
|
||||
if e != nil {
|
||||
return ImportResult{}, e
|
||||
}
|
||||
return a.importFacts(ctx, s, facts)
|
||||
result, err := a.importFacts(ctx, s, prepared.facts)
|
||||
if err != nil {
|
||||
return ImportResult{}, err
|
||||
}
|
||||
delete(a.csvImports, id)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// CancelCSVImport discards a prepared statement without importing anything.
|
||||
func (a *App) CancelCSVImport(id string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
delete(a.csvImports, id)
|
||||
}
|
||||
|
||||
// csvColumns lists the mapping as reviewable field/value pairs, including where
|
||||
// the currency comes from and how dates and decimals are read: an inferred
|
||||
// convention is the easiest part of a mapping to get wrong.
|
||||
func csvColumns(mapping banking.CSVMapping, account domain.Account) []CSVColumn {
|
||||
columns := make([]CSVColumn, 0, 13)
|
||||
for _, field := range []CSVColumn{
|
||||
{"Booking date", mapping.BookingDateColumn}, {"Value date", mapping.ValueDateColumn},
|
||||
{"Amount", mapping.AmountColumn}, {"Debit", mapping.DebitColumn}, {"Credit", mapping.CreditColumn},
|
||||
{"Currency", mapping.CurrencyColumn}, {"Description", mapping.DescriptionColumn},
|
||||
{"Secondary description", mapping.FallbackDescriptionColumn}, {"Counterparty", mapping.CounterpartyColumn},
|
||||
{"Counterparty IBAN", mapping.CounterpartyIBANColumn}, {"Transaction reference", mapping.ExternalIDColumn},
|
||||
} {
|
||||
if field.Column != "" {
|
||||
columns = append(columns, field)
|
||||
}
|
||||
}
|
||||
return ImportResult{}, errors.New("unknown account")
|
||||
if mapping.CurrencyColumn == "" {
|
||||
currency, origin := mapping.FixedCurrency, "from the amount column header"
|
||||
if currency == "" {
|
||||
currency, origin = account.Currency, "from the selected account"
|
||||
}
|
||||
columns = append(columns, CSVColumn{"Currency", currency + " (" + origin + ")"})
|
||||
}
|
||||
return append(columns,
|
||||
CSVColumn{"Dates read as", csvDateFormatLabel(mapping.DateFormat)},
|
||||
CSVColumn{"Decimal separator", csvDecimalFormatLabel(mapping.DecimalFormat)},
|
||||
)
|
||||
}
|
||||
|
||||
func csvDateFormatLabel(format string) string {
|
||||
switch format {
|
||||
case "yyyy-mm-dd":
|
||||
return "2026-09-01"
|
||||
case "dd.mm.yyyy":
|
||||
return "01.09.2026 (day first)"
|
||||
case "mm/dd/yyyy":
|
||||
return "09/01/2026 (month first)"
|
||||
case "dd/mm/yyyy":
|
||||
return "01/09/2026 (day first)"
|
||||
case "iso-date-time":
|
||||
return "2026-09-01T14:30:00 (date and time)"
|
||||
case "iso-or-german":
|
||||
return "2026-09-01 or 01.09.2026"
|
||||
default:
|
||||
return format
|
||||
}
|
||||
}
|
||||
|
||||
func csvDecimalFormatLabel(format string) string {
|
||||
switch format {
|
||||
case "dot":
|
||||
return "point (1234.56)"
|
||||
case "comma":
|
||||
return "comma (1.234,56)"
|
||||
case "dot-or-comma":
|
||||
return "point or comma"
|
||||
default:
|
||||
return format
|
||||
}
|
||||
}
|
||||
|
||||
// csvSamples keeps a bounded, ordered excerpt that always shows the extremes and
|
||||
// both directions of money when the statement contains them: an inverted sign or
|
||||
// a misread date convention has to be visible before confirming.
|
||||
func csvSamples(facts []domain.Facts) []domain.Facts {
|
||||
if len(facts) == 0 {
|
||||
return []domain.Facts{}
|
||||
}
|
||||
chosen := map[int]bool{0: true, len(facts) - 1: true}
|
||||
if len(facts) > 1 {
|
||||
chosen[1] = true
|
||||
}
|
||||
if len(facts) > 2 {
|
||||
chosen[len(facts)-2] = true
|
||||
}
|
||||
credit, debit, largest := -1, -1, 0
|
||||
for i, f := range facts {
|
||||
minor, err := f.Amount.Minor()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if minor >= 0 && credit < 0 {
|
||||
credit = i
|
||||
}
|
||||
if minor < 0 && debit < 0 {
|
||||
debit = i
|
||||
}
|
||||
if previous, e := facts[largest].Amount.Minor(); e != nil || abs64(minor) > abs64(previous) {
|
||||
largest = i
|
||||
}
|
||||
}
|
||||
for _, index := range []int{credit, debit, largest} {
|
||||
if index >= 0 && len(chosen) < maxCSVSamples {
|
||||
chosen[index] = true
|
||||
}
|
||||
}
|
||||
indexes := make([]int, 0, len(chosen))
|
||||
for index := range chosen {
|
||||
indexes = append(indexes, index)
|
||||
}
|
||||
slices.Sort(indexes)
|
||||
samples := make([]domain.Facts, 0, len(indexes))
|
||||
for _, index := range indexes {
|
||||
samples = append(samples, facts[index])
|
||||
}
|
||||
return samples
|
||||
}
|
||||
|
||||
func abs64(v int64) int64 {
|
||||
if v < 0 {
|
||||
return -v
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func (a *App) Backfill(ctx context.Context, rev, accountID string, historyMonths int) (ImportResult, error) {
|
||||
|
||||
Reference in New Issue
Block a user