Import ING and Kontist statements behind a reviewed column mapping
CSV import is now mapping-driven: N26, ING (metadata preamble, Windows-1252, German decimals) and Kontist exports are recognized locally, and any other layout can have its columns proposed by the configured model from a sample in which letters are replaced by x and digits by 0. Proposals are untrusted: every column must name a supplied header, money must come from one signed column or one debit/credit pair, and formats must be from a closed list. Uploading no longer imports. /api/import is replaced by prepare/confirm/cancel: prepare parses, deduplicates and previews the exact facts, and only confirming at the reviewed revision writes them. ING and AI-mapped facts carry no transaction reference, because repeating SEPA mandate references must never become a transaction identity.
This commit is contained in:
Lars Nolden
+2
-1
@@ -61,6 +61,7 @@ type App struct {
|
||||
bank banking.Provider
|
||||
classifier classification.Client
|
||||
previews map[string]Preview
|
||||
csvImports map[string]CSVImport
|
||||
authStates map[string]authorization
|
||||
callbackURL string
|
||||
bankingSettings bankingSettings
|
||||
@@ -72,7 +73,7 @@ func Open(dir string) (*App, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a := &App{dir: dir, journal: j, previews: make(map[string]Preview), authStates: make(map[string]authorization), syncRequested: make(chan struct{}, 1)}
|
||||
a := &App{dir: dir, journal: j, previews: make(map[string]Preview), csvImports: make(map[string]CSVImport), authStates: make(map[string]authorization), syncRequested: make(chan struct{}, 1)}
|
||||
fail := func(e error) (*App, error) { j.Close(); return nil, e }
|
||||
if err = os.MkdirAll(filepath.Join(dir, "state"), 0700); err != nil {
|
||||
return fail(err)
|
||||
|
||||
@@ -0,0 +1,302 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"finance-duck/internal/classification"
|
||||
"finance-duck/internal/domain"
|
||||
)
|
||||
|
||||
const n26Statement = "Booking Date,Value Date,Partner Name,Partner IBAN,Type,Payment Reference,Account Name,Amount (EUR)\n" +
|
||||
"2026-09-01,2026-08-31,Cafe,DE02120300000000202051,Card,Lunch,Main,-12.30\n" +
|
||||
"2026-09-02,2026-09-02,Employer,DE89370400440532013000,Transfer,Salary,Main,2400.00\n"
|
||||
|
||||
// unmappedStatement uses columns no preset recognizes, so it needs a proposed
|
||||
// mapping: German booking dates, split Soll/Haben money and a free-text note.
|
||||
const unmappedStatement = "Datum;Empfänger;Soll;Haben;Notiz\n" +
|
||||
"01.09.2026;Cafe Sonne;12,30;;Mittagessen Berlin\n" +
|
||||
"02.09.2026;Arbeitgeber GmbH;;1.200,00;Gehalt September\n"
|
||||
|
||||
func mockCSVMapper(t *testing.T, a *App, proposal map[string]any) *[]string {
|
||||
t.Helper()
|
||||
prompts := &[]string{}
|
||||
mock := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var request struct {
|
||||
Messages []struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
} `json:"messages"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
t.Error(err)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
for _, message := range request.Messages {
|
||||
if message.Role == "user" {
|
||||
*prompts = append(*prompts, message.Content)
|
||||
}
|
||||
}
|
||||
content, err := json.Marshal(proposal)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{
|
||||
"finish_reason": "stop", "message": map[string]any{"content": string(content)},
|
||||
}}})
|
||||
}))
|
||||
t.Cleanup(mock.Close)
|
||||
a.classifier = classification.Client{APIKey: "test-private-key", Model: "test/model", BaseURL: mock.URL}
|
||||
a.settings.Model = "test/model"
|
||||
return prompts
|
||||
}
|
||||
|
||||
func prepare(t *testing.T, a *App, s State, statement string) CSVImport {
|
||||
t.Helper()
|
||||
prepared, err := a.PrepareCSVImport(context.Background(), s.Revision, "n26", strings.NewReader(statement))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return prepared
|
||||
}
|
||||
|
||||
func TestPreparedCSVImportWritesNothingUntilConfirmed(t *testing.T) {
|
||||
a, s := testApp(t)
|
||||
before := domain.Clone(s.Data)
|
||||
prepared := prepare(t, a, s, n26Statement)
|
||||
if prepared.Source != "n26_csv" || prepared.SourceLabel != "N26" || prepared.MappedBy != "preset" || prepared.Model != "" {
|
||||
t.Fatalf("recognized export was not mapped locally: %+v", prepared)
|
||||
}
|
||||
if prepared.Records != 2 || prepared.New != 2 || prepared.Duplicates != 0 || len(prepared.Samples) != 2 {
|
||||
t.Fatalf("unexpected preview counts: %+v", prepared)
|
||||
}
|
||||
// The preview must show what would be written, including direction and dates.
|
||||
sample := prepared.Samples[0]
|
||||
if sample.BookingDate != "2026-09-01" || sample.ValueDate != "2026-08-31" || sample.Amount.String() != "-12.30" ||
|
||||
sample.RawDescription != "Lunch" || sample.Counterparty != "Cafe" || sample.Currency != "EUR" {
|
||||
t.Fatalf("unexpected preview sample: %+v", sample)
|
||||
}
|
||||
if prepared.Samples[1].Amount.String() != "2400.00" {
|
||||
t.Fatalf("preview hid the credit record: %+v", prepared.Samples[1])
|
||||
}
|
||||
mapped := map[string]string{}
|
||||
for _, column := range prepared.Columns {
|
||||
mapped[column.Field] = column.Column
|
||||
}
|
||||
if mapped["Booking date"] != "Booking Date" || mapped["Amount"] != "Amount (EUR)" || mapped["Description"] != "Payment Reference" ||
|
||||
mapped["Currency"] != "EUR (from the amount column header)" || mapped["Dates read as"] == "" || mapped["Decimal separator"] == "" {
|
||||
t.Fatalf("mapping was not reviewable: %+v", prepared.Columns)
|
||||
}
|
||||
current, err := a.Snapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(before, current.Data) {
|
||||
t.Fatal("preparing an import changed the journal")
|
||||
}
|
||||
|
||||
// Cancelling must discard the statement without importing anything.
|
||||
a.CancelCSVImport(prepared.ID)
|
||||
if _, err := a.ConfirmCSVImport(context.Background(), prepared.ID, s.Revision); err == nil {
|
||||
t.Fatal("confirmed a cancelled import")
|
||||
}
|
||||
current, err = a.Snapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(before, current.Data) {
|
||||
t.Fatal("cancelling an import changed the journal")
|
||||
}
|
||||
|
||||
prepared = prepare(t, a, s, n26Statement)
|
||||
result, err := a.ConfirmCSVImport(context.Background(), prepared.ID, prepared.Revision)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Imported != 2 || len(result.State.Data.Transactions) != 2 {
|
||||
t.Fatalf("confirmation did not import the previewed records: %+v", result)
|
||||
}
|
||||
for _, tx := range result.State.Data.Transactions {
|
||||
if tx.Facts.Source != "n26_csv" || tx.Facts.Currency != "EUR" || tx.Facts.AccountID != "n26" {
|
||||
t.Fatalf("imported facts differ from the preview: %+v", tx.Facts)
|
||||
}
|
||||
}
|
||||
// A confirmed statement is consumed: confirming again must not double count.
|
||||
if _, err := a.ConfirmCSVImport(context.Background(), prepared.ID, prepared.Revision); err == nil {
|
||||
t.Fatal("confirmed the same prepared import twice")
|
||||
}
|
||||
repeat := prepare(t, a, result.State, n26Statement)
|
||||
if repeat.Records != 2 || repeat.New != 0 || repeat.Duplicates != 2 {
|
||||
t.Fatalf("reimport preview did not report duplicates: %+v", repeat)
|
||||
}
|
||||
again, err := a.ConfirmCSVImport(context.Background(), repeat.ID, repeat.Revision)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again.Imported != 0 || len(again.State.Data.Transactions) != 2 {
|
||||
t.Fatalf("reimport duplicated transactions: %+v", again)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnrecognizedStatementNeedsAConfiguredModel(t *testing.T) {
|
||||
a, s := testApp(t)
|
||||
_, err := a.PrepareCSVImport(context.Background(), s.Revision, "n26", strings.NewReader(unmappedStatement))
|
||||
if err == nil || !strings.Contains(err.Error(), "unrecognized CSV layout") {
|
||||
t.Fatalf("unmapped statement without AI: %v", err)
|
||||
}
|
||||
current, err := a.Snapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(s.Data, current.Data) {
|
||||
t.Fatal("a failed mapping changed the journal")
|
||||
}
|
||||
// Recognized exports must keep working with no AI configured at all.
|
||||
if prepared := prepare(t, a, s, n26Statement); prepared.MappedBy != "preset" {
|
||||
t.Fatalf("preset import required AI: %+v", prepared)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProposedMappingIsPreviewedFromRedactedSample(t *testing.T) {
|
||||
a, s := testApp(t)
|
||||
prompts := mockCSVMapper(t, a, map[string]any{
|
||||
"booking_date_column": "Datum",
|
||||
"value_date_column": "",
|
||||
"amount_column": "",
|
||||
"debit_column": "Soll",
|
||||
"credit_column": "Haben",
|
||||
"currency_column": "",
|
||||
"description_column": "Notiz",
|
||||
"counterparty_column": "Empfänger",
|
||||
"counterparty_iban_column": "",
|
||||
"date_format": "dd.mm.yyyy",
|
||||
"decimal_format": "comma",
|
||||
})
|
||||
prepared := prepare(t, a, s, unmappedStatement)
|
||||
if prepared.MappedBy != "openrouter" || prepared.Model != "test/model" || prepared.Source != "csv" {
|
||||
t.Fatalf("proposed mapping was not attributed: %+v", prepared)
|
||||
}
|
||||
if prepared.Records != 2 || prepared.New != 2 || len(prepared.Samples) != 2 {
|
||||
t.Fatalf("unexpected preview counts: %+v", prepared)
|
||||
}
|
||||
if prepared.Samples[0].Amount.String() != "-12.30" || prepared.Samples[1].Amount.String() != "1200.00" {
|
||||
t.Fatalf("split debit and credit lost direction: %+v", prepared.Samples)
|
||||
}
|
||||
if prepared.Samples[0].RawDescription != "Mittagessen Berlin" || prepared.Samples[0].Counterparty != "Cafe Sonne" || prepared.Samples[0].BookingDate != "2026-09-01" {
|
||||
t.Fatalf("unexpected preview sample: %+v", prepared.Samples[0])
|
||||
}
|
||||
if len(*prompts) != 1 {
|
||||
t.Fatalf("expected exactly one mapping request, got %d", len(*prompts))
|
||||
}
|
||||
// Only column names and value shapes may leave this machine.
|
||||
for _, secret := range []string{"Cafe Sonne", "Mittagessen", "Arbeitgeber", "12,30", "1.200,00", "01.09.2026"} {
|
||||
if strings.Contains((*prompts)[0], secret) {
|
||||
t.Fatalf("mapping request leaked %q: %s", secret, (*prompts)[0])
|
||||
}
|
||||
}
|
||||
for _, header := range []string{"Datum", "Soll", "Haben", "Notiz", "Empfänger"} {
|
||||
if !strings.Contains((*prompts)[0], header) {
|
||||
t.Fatalf("mapping request omitted column %q: %s", header, (*prompts)[0])
|
||||
}
|
||||
}
|
||||
result, err := a.ConfirmCSVImport(context.Background(), prepared.ID, prepared.Revision)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Imported != 2 {
|
||||
t.Fatalf("confirmed mapping imported %d records", result.Imported)
|
||||
}
|
||||
for _, tx := range result.State.Data.Transactions {
|
||||
if tx.Facts.Source != "csv" || tx.Facts.ExternalID != "" {
|
||||
t.Fatalf("mapped facts carry an unexpected identity: %+v", tx.Facts)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsafeProposedMappingsAreRejected(t *testing.T) {
|
||||
valid := map[string]any{
|
||||
"booking_date_column": "Datum", "value_date_column": "", "amount_column": "",
|
||||
"debit_column": "Soll", "credit_column": "Haben", "currency_column": "",
|
||||
"description_column": "Notiz", "counterparty_column": "Empfänger",
|
||||
"counterparty_iban_column": "", "date_format": "dd.mm.yyyy", "decimal_format": "comma",
|
||||
}
|
||||
with := func(changes map[string]any) map[string]any {
|
||||
proposal := map[string]any{}
|
||||
for key, value := range valid {
|
||||
proposal[key] = value
|
||||
}
|
||||
for key, value := range changes {
|
||||
proposal[key] = value
|
||||
}
|
||||
return proposal
|
||||
}
|
||||
cases := map[string]map[string]any{
|
||||
"invented column": with(map[string]any{"description_column": "Verwendungszweck"}),
|
||||
"missing date": with(map[string]any{"booking_date_column": ""}),
|
||||
"missing description": with(map[string]any{"description_column": ""}),
|
||||
"no money": with(map[string]any{"debit_column": "", "credit_column": ""}),
|
||||
"both strategies": with(map[string]any{"amount_column": "Soll"}),
|
||||
"half split": with(map[string]any{"credit_column": ""}),
|
||||
"unsupported format": with(map[string]any{"date_format": "%d.%m.%Y"}),
|
||||
"column reused": with(map[string]any{"counterparty_column": "Notiz"}),
|
||||
"unknown field": with(map[string]any{"balance_column": "Soll"}),
|
||||
}
|
||||
for name, proposal := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
a, s := testApp(t)
|
||||
mockCSVMapper(t, a, proposal)
|
||||
prepared, err := a.PrepareCSVImport(context.Background(), s.Revision, "n26", strings.NewReader(unmappedStatement))
|
||||
if err == nil {
|
||||
t.Fatalf("accepted an unsafe mapping: %+v", prepared)
|
||||
}
|
||||
current, e := a.Snapshot(context.Background())
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if !reflect.DeepEqual(s.Data, current.Data) {
|
||||
t.Fatal("a rejected mapping changed the journal")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfirmationRequiresTheReviewedJournalRevision(t *testing.T) {
|
||||
a, s := testApp(t)
|
||||
prepared := prepare(t, a, s, n26Statement)
|
||||
// An unrelated edit invalidates the preview: its duplicate analysis and
|
||||
// sample were computed against the revision the operator reviewed.
|
||||
changed, err := a.Mutate(context.Background(), s.Revision, func(d *domain.Dataset) error {
|
||||
d.Tags = append(d.Tags, domain.Tag{ID: "tag_new", Name: "new"})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := a.ConfirmCSVImport(context.Background(), prepared.ID, prepared.Revision); err == nil {
|
||||
t.Fatal("confirmed a preview of a superseded journal")
|
||||
}
|
||||
if _, err := a.ConfirmCSVImport(context.Background(), prepared.ID, changed.Revision); err == nil {
|
||||
t.Fatal("confirmed a preview with a substituted revision")
|
||||
}
|
||||
if _, err := a.PrepareCSVImport(context.Background(), s.Revision, "n26", strings.NewReader(n26Statement)); err == nil {
|
||||
t.Fatal("prepared an import against a superseded revision")
|
||||
}
|
||||
if _, err := a.PrepareCSVImport(context.Background(), changed.Revision, "missing", strings.NewReader(n26Statement)); err == nil {
|
||||
t.Fatal("prepared an import for an unknown account")
|
||||
}
|
||||
current, err := a.Snapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(current.Data.Transactions) != 0 {
|
||||
t.Fatalf("rejected confirmations imported %d transactions", len(current.Data.Transactions))
|
||||
}
|
||||
}
|
||||
+258
-11
@@ -77,26 +77,273 @@ func (a *App) importFacts(ctx context.Context, s State, facts []domain.Facts) (I
|
||||
}
|
||||
return ImportResult{Imported: len(added), State: state}, nil
|
||||
}
|
||||
func (a *App) ImportCSV(ctx context.Context, rev, accountID string, r io.Reader) (ImportResult, error) {
|
||||
|
||||
// CSVColumn is one reviewable source-column assignment.
|
||||
type CSVColumn struct {
|
||||
Field string `json:"field"`
|
||||
Column string `json:"column"`
|
||||
}
|
||||
|
||||
// CSVImport is a parsed statement awaiting confirmation. Nothing is written to
|
||||
// the journal until ConfirmCSVImport applies the exact facts previewed here.
|
||||
type CSVImport struct {
|
||||
ID string `json:"id"`
|
||||
Revision string `json:"revision"`
|
||||
AccountID string `json:"account_id"`
|
||||
Source string `json:"source"`
|
||||
SourceLabel string `json:"source_label"`
|
||||
MappedBy string `json:"mapped_by"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Mapping banking.CSVMapping `json:"mapping"`
|
||||
Columns []CSVColumn `json:"columns"`
|
||||
Records int `json:"records"`
|
||||
New int `json:"new"`
|
||||
Duplicates int `json:"duplicates"`
|
||||
Samples []domain.Facts `json:"samples"`
|
||||
|
||||
facts []domain.Facts
|
||||
created time.Time
|
||||
}
|
||||
|
||||
const csvImportLifetime = time.Hour
|
||||
const maxPreparedCSVImports = 5
|
||||
const maxCSVSamples = 10
|
||||
|
||||
// PrepareCSVImport maps and parses an uploaded statement without importing it.
|
||||
// Known N26, ING and Kontist exports are recognized locally; any other layout
|
||||
// needs a configured model to propose a column mapping from the statement's
|
||||
// redacted shape. The result must be reviewed and confirmed.
|
||||
func (a *App) PrepareCSVImport(ctx context.Context, rev, accountID string, r io.Reader) (CSVImport, error) {
|
||||
a.mu.Lock()
|
||||
s, err := a.snapshot(ctx)
|
||||
model := strings.TrimSpace(a.settings.Model)
|
||||
client := a.classifier.WithModel(model)
|
||||
configured := strings.TrimSpace(a.classifier.APIKey) != "" && model != ""
|
||||
a.mu.Unlock()
|
||||
if err != nil {
|
||||
return CSVImport{}, err
|
||||
}
|
||||
if rev != s.Revision {
|
||||
return CSVImport{}, errors.New("revision conflict: reload before importing")
|
||||
}
|
||||
index := slices.IndexFunc(s.Data.Accounts, func(account domain.Account) bool { return account.ID == accountID })
|
||||
if index < 0 {
|
||||
return CSVImport{}, errors.New("unknown account")
|
||||
}
|
||||
account := s.Data.Accounts[index]
|
||||
file, err := banking.ReadCSV(r)
|
||||
if err != nil {
|
||||
return CSVImport{}, err
|
||||
}
|
||||
prepared := CSVImport{ID: domain.NewID("csvimport"), Revision: s.Revision, AccountID: account.ID, MappedBy: "preset", created: time.Now()}
|
||||
mapping, source, label, recognized := banking.DetectCSVMapping(file)
|
||||
if !recognized {
|
||||
sample, e := file.Sample()
|
||||
if e != nil {
|
||||
return CSVImport{}, e
|
||||
}
|
||||
if !configured {
|
||||
return CSVImport{}, errors.New("unrecognized CSV layout: import an N26, ING or Kontist export, or configure an OpenRouter key and model in Settings to map these columns")
|
||||
}
|
||||
proposal, e := client.ProposeCSVMapping(ctx, classification.CSVMappingRequest{
|
||||
Delimiter: sample.Delimiter, Headers: sample.Headers, ShapedRows: sample.ShapedRows,
|
||||
DateFormats: banking.CSVDateFormats(), DecimalFormats: banking.CSVDecimalFormats(),
|
||||
})
|
||||
if e != nil {
|
||||
return CSVImport{}, e
|
||||
}
|
||||
mapping = banking.CSVMapping{
|
||||
HeaderRow: sample.HeaderRow,
|
||||
BookingDateColumn: proposal.BookingDateColumn,
|
||||
ValueDateColumn: proposal.ValueDateColumn,
|
||||
AmountColumn: proposal.AmountColumn,
|
||||
DebitColumn: proposal.DebitColumn,
|
||||
CreditColumn: proposal.CreditColumn,
|
||||
CurrencyColumn: proposal.CurrencyColumn,
|
||||
DescriptionColumn: proposal.DescriptionColumn,
|
||||
CounterpartyColumn: proposal.CounterpartyColumn,
|
||||
CounterpartyIBANColumn: proposal.CounterpartyIBANColumn,
|
||||
DateFormat: proposal.DateFormat,
|
||||
DecimalFormat: proposal.DecimalFormat,
|
||||
}
|
||||
source, label = "csv", "AI-mapped CSV"
|
||||
prepared.MappedBy, prepared.Model = "openrouter", proposal.Model
|
||||
}
|
||||
facts, err := banking.ParseMappedCSV(file, account, mapping, source)
|
||||
if err != nil {
|
||||
return CSVImport{}, err
|
||||
}
|
||||
// Dedupe now so the preview reports what confirming would actually add, and
|
||||
// so cross-source conflicts are reported before anything is written.
|
||||
added, err := banking.NormalizeAndDedupe(s.Data, facts)
|
||||
if err != nil {
|
||||
return CSVImport{}, err
|
||||
}
|
||||
prepared.Source, prepared.SourceLabel, prepared.Mapping = source, label, mapping
|
||||
prepared.Columns = csvColumns(mapping, account)
|
||||
prepared.Records, prepared.New, prepared.Duplicates = len(facts), len(added), len(facts)-len(added)
|
||||
prepared.Samples, prepared.facts = csvSamples(facts), facts
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
for id, old := range a.csvImports {
|
||||
if time.Since(old.created) > csvImportLifetime {
|
||||
delete(a.csvImports, id)
|
||||
}
|
||||
}
|
||||
if len(a.csvImports) >= maxPreparedCSVImports {
|
||||
return CSVImport{}, errors.New("too many statements awaiting confirmation; confirm or cancel one first")
|
||||
}
|
||||
a.csvImports[prepared.ID] = prepared
|
||||
return prepared, nil
|
||||
}
|
||||
|
||||
// ConfirmCSVImport imports exactly the facts that were previewed, provided the
|
||||
// journal has not changed since.
|
||||
func (a *App) ConfirmCSVImport(ctx context.Context, id, rev string) (ImportResult, error) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
prepared, ok := a.csvImports[id]
|
||||
if !ok || time.Since(prepared.created) > csvImportLifetime {
|
||||
return ImportResult{}, errors.New("prepared import expired or unknown; upload the statement again")
|
||||
}
|
||||
if rev != prepared.Revision {
|
||||
return ImportResult{}, errors.New("revision conflict: reload before importing")
|
||||
}
|
||||
s, err := a.snapshot(ctx)
|
||||
if err != nil {
|
||||
return ImportResult{}, err
|
||||
}
|
||||
if rev != s.Revision {
|
||||
return ImportResult{}, errors.New("revision conflict: reload before importing")
|
||||
if s.Revision != prepared.Revision {
|
||||
return ImportResult{}, errors.New("revision conflict: data changed after the preview; upload the statement again")
|
||||
}
|
||||
for _, account := range s.Data.Accounts {
|
||||
if account.ID == accountID {
|
||||
facts, e := banking.ParseCSV(r, account)
|
||||
if e != nil {
|
||||
return ImportResult{}, e
|
||||
}
|
||||
return a.importFacts(ctx, s, facts)
|
||||
result, err := a.importFacts(ctx, s, prepared.facts)
|
||||
if err != nil {
|
||||
return ImportResult{}, err
|
||||
}
|
||||
delete(a.csvImports, id)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// CancelCSVImport discards a prepared statement without importing anything.
|
||||
func (a *App) CancelCSVImport(id string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
delete(a.csvImports, id)
|
||||
}
|
||||
|
||||
// csvColumns lists the mapping as reviewable field/value pairs, including where
|
||||
// the currency comes from and how dates and decimals are read: an inferred
|
||||
// convention is the easiest part of a mapping to get wrong.
|
||||
func csvColumns(mapping banking.CSVMapping, account domain.Account) []CSVColumn {
|
||||
columns := make([]CSVColumn, 0, 13)
|
||||
for _, field := range []CSVColumn{
|
||||
{"Booking date", mapping.BookingDateColumn}, {"Value date", mapping.ValueDateColumn},
|
||||
{"Amount", mapping.AmountColumn}, {"Debit", mapping.DebitColumn}, {"Credit", mapping.CreditColumn},
|
||||
{"Currency", mapping.CurrencyColumn}, {"Description", mapping.DescriptionColumn},
|
||||
{"Secondary description", mapping.FallbackDescriptionColumn}, {"Counterparty", mapping.CounterpartyColumn},
|
||||
{"Counterparty IBAN", mapping.CounterpartyIBANColumn}, {"Transaction reference", mapping.ExternalIDColumn},
|
||||
} {
|
||||
if field.Column != "" {
|
||||
columns = append(columns, field)
|
||||
}
|
||||
}
|
||||
return ImportResult{}, errors.New("unknown account")
|
||||
if mapping.CurrencyColumn == "" {
|
||||
currency, origin := mapping.FixedCurrency, "from the amount column header"
|
||||
if currency == "" {
|
||||
currency, origin = account.Currency, "from the selected account"
|
||||
}
|
||||
columns = append(columns, CSVColumn{"Currency", currency + " (" + origin + ")"})
|
||||
}
|
||||
return append(columns,
|
||||
CSVColumn{"Dates read as", csvDateFormatLabel(mapping.DateFormat)},
|
||||
CSVColumn{"Decimal separator", csvDecimalFormatLabel(mapping.DecimalFormat)},
|
||||
)
|
||||
}
|
||||
|
||||
func csvDateFormatLabel(format string) string {
|
||||
switch format {
|
||||
case "yyyy-mm-dd":
|
||||
return "2026-09-01"
|
||||
case "dd.mm.yyyy":
|
||||
return "01.09.2026 (day first)"
|
||||
case "mm/dd/yyyy":
|
||||
return "09/01/2026 (month first)"
|
||||
case "dd/mm/yyyy":
|
||||
return "01/09/2026 (day first)"
|
||||
case "iso-date-time":
|
||||
return "2026-09-01T14:30:00 (date and time)"
|
||||
case "iso-or-german":
|
||||
return "2026-09-01 or 01.09.2026"
|
||||
default:
|
||||
return format
|
||||
}
|
||||
}
|
||||
|
||||
func csvDecimalFormatLabel(format string) string {
|
||||
switch format {
|
||||
case "dot":
|
||||
return "point (1234.56)"
|
||||
case "comma":
|
||||
return "comma (1.234,56)"
|
||||
case "dot-or-comma":
|
||||
return "point or comma"
|
||||
default:
|
||||
return format
|
||||
}
|
||||
}
|
||||
|
||||
// csvSamples keeps a bounded, ordered excerpt that always shows the extremes and
|
||||
// both directions of money when the statement contains them: an inverted sign or
|
||||
// a misread date convention has to be visible before confirming.
|
||||
func csvSamples(facts []domain.Facts) []domain.Facts {
|
||||
if len(facts) == 0 {
|
||||
return []domain.Facts{}
|
||||
}
|
||||
chosen := map[int]bool{0: true, len(facts) - 1: true}
|
||||
if len(facts) > 1 {
|
||||
chosen[1] = true
|
||||
}
|
||||
if len(facts) > 2 {
|
||||
chosen[len(facts)-2] = true
|
||||
}
|
||||
credit, debit, largest := -1, -1, 0
|
||||
for i, f := range facts {
|
||||
minor, err := f.Amount.Minor()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if minor >= 0 && credit < 0 {
|
||||
credit = i
|
||||
}
|
||||
if minor < 0 && debit < 0 {
|
||||
debit = i
|
||||
}
|
||||
if previous, e := facts[largest].Amount.Minor(); e != nil || abs64(minor) > abs64(previous) {
|
||||
largest = i
|
||||
}
|
||||
}
|
||||
for _, index := range []int{credit, debit, largest} {
|
||||
if index >= 0 && len(chosen) < maxCSVSamples {
|
||||
chosen[index] = true
|
||||
}
|
||||
}
|
||||
indexes := make([]int, 0, len(chosen))
|
||||
for index := range chosen {
|
||||
indexes = append(indexes, index)
|
||||
}
|
||||
slices.Sort(indexes)
|
||||
samples := make([]domain.Facts, 0, len(indexes))
|
||||
for _, index := range indexes {
|
||||
samples = append(samples, facts[index])
|
||||
}
|
||||
return samples
|
||||
}
|
||||
|
||||
func abs64(v int64) int64 {
|
||||
if v < 0 {
|
||||
return -v
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func (a *App) Backfill(ctx context.Context, rev, accountID string, historyMonths int) (ImportResult, error) {
|
||||
|
||||
Reference in New Issue
Block a user