finance-duck/internal/app/csv_import_test.go

package app

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"reflect"
	"strings"
	"testing"

	"finance-duck/internal/classification"
	"finance-duck/internal/domain"
)

const n26Statement = "Booking Date,Value Date,Partner Name,Partner IBAN,Type,Payment Reference,Account Name,Amount (EUR)\n" +
	"2026-09-01,2026-08-31,Cafe,DE02120300000000202051,Card,Lunch,Main,-12.30\n" +
	"2026-09-02,2026-09-02,Employer,DE89370400440532013000,Transfer,Salary,Main,2400.00\n"

// unmappedStatement uses columns no preset recognizes, so it needs a proposed
// mapping: German booking dates, split Soll/Haben money and a free-text note.
const unmappedStatement = "Datum;Empfänger;Soll;Haben;Notiz\n" +
	"01.09.2026;Cafe Sonne;12,30;;Mittagessen Berlin\n" +
	"02.09.2026;Arbeitgeber GmbH;;1.200,00;Gehalt September\n"

func mockCSVMapper(t *testing.T, a *App, proposal map[string]any) *[]string {
	t.Helper()
	prompts := &[]string{}
	mock := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		var request struct {
			Messages []struct {
				Role    string `json:"role"`
				Content string `json:"content"`
			} `json:"messages"`
		}
		if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
			t.Error(err)
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		for _, message := range request.Messages {
			if message.Role == "user" {
				*prompts = append(*prompts, message.Content)
			}
		}
		content, err := json.Marshal(proposal)
		if err != nil {
			t.Error(err)
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{
			"finish_reason": "stop", "message": map[string]any{"content": string(content)},
		}}})
	}))
	t.Cleanup(mock.Close)
	a.classifier = classification.Client{APIKey: "test-private-key", Model: "test/model", BaseURL: mock.URL}
	a.settings.Model = "test/model"
	return prompts
}

func prepare(t *testing.T, a *App, s State, statement string) CSVImport {
	t.Helper()
	prepared, err := a.PrepareCSVImport(context.Background(), s.Revision, "n26", strings.NewReader(statement))
	if err != nil {
		t.Fatal(err)
	}
	return prepared
}

func TestPreparedCSVImportWritesNothingUntilConfirmed(t *testing.T) {
	a, s := testApp(t)
	before := domain.Clone(s.Data)
	prepared := prepare(t, a, s, n26Statement)
	if prepared.Source != "n26_csv" || prepared.SourceLabel != "N26" || prepared.MappedBy != "preset" || prepared.Model != "" {
		t.Fatalf("recognized export was not mapped locally: %+v", prepared)
	}
	if prepared.Records != 2 || prepared.New != 2 || prepared.Duplicates != 0 || len(prepared.Samples) != 2 {
		t.Fatalf("unexpected preview counts: %+v", prepared)
	}
	// The preview must show what would be written, including direction and dates.
	sample := prepared.Samples[0]
	if sample.BookingDate != "2026-09-01" || sample.ValueDate != "2026-08-31" || sample.Amount.String() != "-12.30" ||
		sample.RawDescription != "Lunch" || sample.Counterparty != "Cafe" || sample.Currency != "EUR" {
		t.Fatalf("unexpected preview sample: %+v", sample)
	}
	if prepared.Samples[1].Amount.String() != "2400.00" {
		t.Fatalf("preview hid the credit record: %+v", prepared.Samples[1])
	}
	mapped := map[string]string{}
	for _, column := range prepared.Columns {
		mapped[column.Field] = column.Column
	}
	if mapped["Booking date"] != "Booking Date" || mapped["Amount"] != "Amount (EUR)" || mapped["Description"] != "Payment Reference" ||
		mapped["Currency"] != "EUR (from the amount column header)" || mapped["Dates read as"] == "" || mapped["Decimal separator"] == "" {
		t.Fatalf("mapping was not reviewable: %+v", prepared.Columns)
	}
	current, err := a.Snapshot(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(before, current.Data) {
		t.Fatal("preparing an import changed the journal")
	}

	// Cancelling must discard the statement without importing anything.
	a.CancelCSVImport(prepared.ID)
	if _, err := a.ConfirmCSVImport(context.Background(), prepared.ID, s.Revision); err == nil {
		t.Fatal("confirmed a cancelled import")
	}
	current, err = a.Snapshot(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(before, current.Data) {
		t.Fatal("cancelling an import changed the journal")
	}

	prepared = prepare(t, a, s, n26Statement)
	result, err := a.ConfirmCSVImport(context.Background(), prepared.ID, prepared.Revision)
	if err != nil {
		t.Fatal(err)
	}
	if result.Imported != 2 || len(result.State.Data.Transactions) != 2 {
		t.Fatalf("confirmation did not import the previewed records: %+v", result)
	}
	for _, tx := range result.State.Data.Transactions {
		if tx.Facts.Source != "n26_csv" || tx.Facts.Currency != "EUR" || tx.Facts.AccountID != "n26" {
			t.Fatalf("imported facts differ from the preview: %+v", tx.Facts)
		}
	}
	// A confirmed statement is consumed: confirming again must not double count.
	if _, err := a.ConfirmCSVImport(context.Background(), prepared.ID, prepared.Revision); err == nil {
		t.Fatal("confirmed the same prepared import twice")
	}
	repeat := prepare(t, a, result.State, n26Statement)
	if repeat.Records != 2 || repeat.New != 0 || repeat.Duplicates != 2 {
		t.Fatalf("reimport preview did not report duplicates: %+v", repeat)
	}
	again, err := a.ConfirmCSVImport(context.Background(), repeat.ID, repeat.Revision)
	if err != nil {
		t.Fatal(err)
	}
	if again.Imported != 0 || len(again.State.Data.Transactions) != 2 {
		t.Fatalf("reimport duplicated transactions: %+v", again)
	}
}

func TestUnrecognizedStatementNeedsAConfiguredModel(t *testing.T) {
	a, s := testApp(t)
	_, err := a.PrepareCSVImport(context.Background(), s.Revision, "n26", strings.NewReader(unmappedStatement))
	if err == nil || !strings.Contains(err.Error(), "unrecognized CSV layout") {
		t.Fatalf("unmapped statement without AI: %v", err)
	}
	current, err := a.Snapshot(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(s.Data, current.Data) {
		t.Fatal("a failed mapping changed the journal")
	}
	// Recognized exports must keep working with no AI configured at all.
	if prepared := prepare(t, a, s, n26Statement); prepared.MappedBy != "preset" {
		t.Fatalf("preset import required AI: %+v", prepared)
	}
}

func TestProposedMappingIsPreviewedFromRedactedSample(t *testing.T) {
	a, s := testApp(t)
	prompts := mockCSVMapper(t, a, map[string]any{
		"booking_date_column":      "Datum",
		"value_date_column":        "",
		"amount_column":            "",
		"debit_column":             "Soll",
		"credit_column":            "Haben",
		"currency_column":          "",
		"description_column":       "Notiz",
		"counterparty_column":      "Empfänger",
		"counterparty_iban_column": "",
		"date_format":              "dd.mm.yyyy",
		"decimal_format":           "comma",
	})
	prepared := prepare(t, a, s, unmappedStatement)
	if prepared.MappedBy != "openrouter" || prepared.Model != "test/model" || prepared.Source != "csv" {
		t.Fatalf("proposed mapping was not attributed: %+v", prepared)
	}
	if prepared.Records != 2 || prepared.New != 2 || len(prepared.Samples) != 2 {
		t.Fatalf("unexpected preview counts: %+v", prepared)
	}
	if prepared.Samples[0].Amount.String() != "-12.30" || prepared.Samples[1].Amount.String() != "1200.00" {
		t.Fatalf("split debit and credit lost direction: %+v", prepared.Samples)
	}
	if prepared.Samples[0].RawDescription != "Mittagessen Berlin" || prepared.Samples[0].Counterparty != "Cafe Sonne" || prepared.Samples[0].BookingDate != "2026-09-01" {
		t.Fatalf("unexpected preview sample: %+v", prepared.Samples[0])
	}
	if len(*prompts) != 1 {
		t.Fatalf("expected exactly one mapping request, got %d", len(*prompts))
	}
	// Only column names and value shapes may leave this machine.
	for _, secret := range []string{"Cafe Sonne", "Mittagessen", "Arbeitgeber", "12,30", "1.200,00", "01.09.2026"} {
		if strings.Contains((*prompts)[0], secret) {
			t.Fatalf("mapping request leaked %q: %s", secret, (*prompts)[0])
		}
	}
	for _, header := range []string{"Datum", "Soll", "Haben", "Notiz", "Empfänger"} {
		if !strings.Contains((*prompts)[0], header) {
			t.Fatalf("mapping request omitted column %q: %s", header, (*prompts)[0])
		}
	}
	result, err := a.ConfirmCSVImport(context.Background(), prepared.ID, prepared.Revision)
	if err != nil {
		t.Fatal(err)
	}
	if result.Imported != 2 {
		t.Fatalf("confirmed mapping imported %d records", result.Imported)
	}
	for _, tx := range result.State.Data.Transactions {
		if tx.Facts.Source != "csv" || tx.Facts.ExternalID != "" {
			t.Fatalf("mapped facts carry an unexpected identity: %+v", tx.Facts)
		}
	}
}

func TestUnsafeProposedMappingsAreRejected(t *testing.T) {
	valid := map[string]any{
		"booking_date_column": "Datum", "value_date_column": "", "amount_column": "",
		"debit_column": "Soll", "credit_column": "Haben", "currency_column": "",
		"description_column": "Notiz", "counterparty_column": "Empfänger",
		"counterparty_iban_column": "", "date_format": "dd.mm.yyyy", "decimal_format": "comma",
	}
	with := func(changes map[string]any) map[string]any {
		proposal := map[string]any{}
		for key, value := range valid {
			proposal[key] = value
		}
		for key, value := range changes {
			proposal[key] = value
		}
		return proposal
	}
	cases := map[string]map[string]any{
		"invented column":     with(map[string]any{"description_column": "Verwendungszweck"}),
		"missing date":        with(map[string]any{"booking_date_column": ""}),
		"missing description": with(map[string]any{"description_column": ""}),
		"no money":            with(map[string]any{"debit_column": "", "credit_column": ""}),
		"both strategies":     with(map[string]any{"amount_column": "Soll"}),
		"half split":          with(map[string]any{"credit_column": ""}),
		"unsupported format":  with(map[string]any{"date_format": "%d.%m.%Y"}),
		"column reused":       with(map[string]any{"counterparty_column": "Notiz"}),
		"unknown field":       with(map[string]any{"balance_column": "Soll"}),
	}
	for name, proposal := range cases {
		t.Run(name, func(t *testing.T) {
			a, s := testApp(t)
			mockCSVMapper(t, a, proposal)
			prepared, err := a.PrepareCSVImport(context.Background(), s.Revision, "n26", strings.NewReader(unmappedStatement))
			if err == nil {
				t.Fatalf("accepted an unsafe mapping: %+v", prepared)
			}
			current, e := a.Snapshot(context.Background())
			if e != nil {
				t.Fatal(e)
			}
			if !reflect.DeepEqual(s.Data, current.Data) {
				t.Fatal("a rejected mapping changed the journal")
			}
		})
	}
}

func TestConfirmationRequiresTheReviewedJournalRevision(t *testing.T) {
	a, s := testApp(t)
	prepared := prepare(t, a, s, n26Statement)
	// An unrelated edit invalidates the preview: its duplicate analysis and
	// sample were computed against the revision the operator reviewed.
	changed, err := a.Mutate(context.Background(), s.Revision, func(d *domain.Dataset) error {
		d.Tags = append(d.Tags, domain.Tag{ID: "tag_new", Name: "new"})
		return nil
	})
	if err != nil {
		t.Fatal(err)
	}
	if _, err := a.ConfirmCSVImport(context.Background(), prepared.ID, prepared.Revision); err == nil {
		t.Fatal("confirmed a preview of a superseded journal")
	}
	if _, err := a.ConfirmCSVImport(context.Background(), prepared.ID, changed.Revision); err == nil {
		t.Fatal("confirmed a preview with a substituted revision")
	}
	if _, err := a.PrepareCSVImport(context.Background(), s.Revision, "n26", strings.NewReader(n26Statement)); err == nil {
		t.Fatal("prepared an import against a superseded revision")
	}
	if _, err := a.PrepareCSVImport(context.Background(), changed.Revision, "missing", strings.NewReader(n26Statement)); err == nil {
		t.Fatal("prepared an import for an unknown account")
	}
	current, err := a.Snapshot(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	if len(current.Data.Transactions) != 0 {
		t.Fatalf("rejected confirmations imported %d transactions", len(current.Data.Transactions))
	}
}