finance-duck/internal/app/propose.go

package app

import (
	"context"
	"errors"
	"fmt"
	"math/rand/v2"
	"reflect"
	"sort"
	"strings"
	"time"
	"unicode"

	"finance-duck/internal/classification"
	"finance-duck/internal/domain"
)

const taxonomySampleLimit = 300

type TaxonomyProposalRequest struct {
	Revision string `json:"revision"`
	Model    string `json:"model"`
}

type TaxonomyPreview struct {
	ID       string                          `json:"id"`
	Revision string                          `json:"revision"`
	Sample   []classification.TaxonomySample `json:"sample"`
	Proposal classification.TaxonomyProposal `json:"proposal"`
	created  time.Time                       `json:"-"`
}

func taxonomyTextKey(value string) string {
	return strings.Join(strings.Fields(strings.Map(func(r rune) rune {
		if unicode.IsLetter(r) || unicode.IsDigit(r) {
			return unicode.ToLower(r)
		}
		return ' '
	}, value)), " ")
}

func taxonomySamples(d domain.Dataset, private []string) []classification.TaxonomySample {
	indices := make([]int, 0, len(d.Transactions))
	for i, tx := range d.Transactions {
		if tx.Enrichment.Kind == "transfer" || tx.Enrichment.Kind == domain.KindInvestment || (tx.Enrichment.Kind != "expense" && tx.Enrichment.Kind != "income") {
			continue
		}
		indices = append(indices, i)
	}
	if len(indices) == 0 {
		return nil
	}
	groups := map[string]int{}
	for _, i := range indices {
		tx := d.Transactions[i]
		key := taxonomyTextKey(tx.Facts.Counterparty)
		if key == "" {
			key = taxonomyTextKey(tx.Facts.RawDescription)
		}
		if key == "" {
			key = tx.Facts.ID
		}
		if current, ok := groups[key]; !ok || tx.Facts.BookingDate < d.Transactions[current].Facts.BookingDate || tx.Facts.BookingDate == d.Transactions[current].Facts.BookingDate && tx.Facts.ID < d.Transactions[current].Facts.ID {
			groups[key] = i
		}
	}
	keys := make([]string, 0, len(groups))
	for key := range groups {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	selected := make([]int, 0, taxonomyMin(taxonomySampleLimit, len(indices)))
	seen := map[int]bool{}
	add := func(i int) {
		if len(selected) == taxonomySampleLimit || seen[i] {
			return
		}
		selected = append(selected, i)
		seen[i] = true
	}
	for _, kind := range []string{"expense", "income"} {
		var first, last = -1, -1
		for _, i := range indices {
			if d.Transactions[i].Enrichment.Kind != kind {
				continue
			}
			if first == -1 || d.Transactions[i].Facts.BookingDate < d.Transactions[first].Facts.BookingDate {
				first = i
			}
			if last == -1 || d.Transactions[i].Facts.BookingDate > d.Transactions[last].Facts.BookingDate {
				last = i
			}
		}
		if first >= 0 {
			add(first)
		}
		if last >= 0 {
			add(last)
		}
	}
	for _, key := range keys {
		if len(selected) == taxonomySampleLimit {
			break
		}
		add(groups[key])
	}
	remainder := make([]int, 0, len(indices)-len(selected))
	for _, i := range indices {
		if !seen[i] {
			remainder = append(remainder, i)
		}
	}
	rand.Shuffle(len(remainder), func(i, j int) { remainder[i], remainder[j] = remainder[j], remainder[i] })
	for _, i := range remainder {
		if len(selected) == taxonomySampleLimit {
			break
		}
		add(i)
	}
	out := make([]classification.TaxonomySample, 0, len(selected))
	for _, i := range selected {
		tx := d.Transactions[i]
		out = append(out, classification.TaxonomySample{
			Date: tx.Facts.BookingDate, Amount: string(tx.Facts.Amount), Currency: tx.Facts.Currency,
			Kind:         tx.Enrichment.Kind,
			Description:  classification.Redact(tx.Facts.RawDescription, d, tx.Facts, private),
			Counterparty: classification.Redact(tx.Facts.Counterparty, d, tx.Facts, private),
		})
	}
	return out
}

func filterExistingTaxonomy(d domain.Dataset, p classification.TaxonomyProposal) classification.TaxonomyProposal {
	categoryKeys := map[string]bool{}
	for _, c := range d.Categories {
		categoryKeys[taxonomyTextKey(c.Name)+"\x00"+c.Kind] = true
	}
	filtered := classification.TaxonomyProposal{}
	for _, c := range p.Categories {
		if !categoryKeys[taxonomyTextKey(c.Name)+"\x00"+c.Kind] {
			filtered.Categories = append(filtered.Categories, c)
		}
	}
	tagKeys := map[string]bool{}
	for _, t := range d.Tags {
		tagKeys[taxonomyTextKey(t.Name)] = true
	}
	for _, t := range p.Tags {
		if !tagKeys[taxonomyTextKey(t.Name)] {
			filtered.Tags = append(filtered.Tags, t)
		}
	}
	merchantOwners := map[string]bool{}
	for _, m := range d.Merchants {
		merchantOwners[taxonomyTextKey(m.Name)] = true
		for _, alias := range m.Aliases {
			merchantOwners[taxonomyTextKey(alias)] = true
		}
	}
	for _, m := range p.Merchants {
		if merchantOwners[taxonomyTextKey(m.Name)] {
			continue
		}
		aliases := make([]string, 0, len(m.Aliases))
		for _, alias := range m.Aliases {
			key := taxonomyTextKey(alias)
			if key != "" && !merchantOwners[key] && !slicesContains(aliases, alias) {
				aliases = append(aliases, alias)
			}
		}
		m.Aliases = aliases
		filtered.Merchants = append(filtered.Merchants, m)
	}
	return filtered
}

func (a *App) ProposeTaxonomy(ctx context.Context, r TaxonomyProposalRequest) (TaxonomyPreview, error) {
	if strings.TrimSpace(r.Model) == "" {
		return TaxonomyPreview{}, errors.New("model is required")
	}
	a.mu.Lock()
	s, err := a.snapshot(ctx)
	client := a.classifier.WithModel(r.Model)
	private := append([]string{}, a.settings.PrivateNames...)
	a.mu.Unlock()
	if err != nil {
		return TaxonomyPreview{}, err
	}
	if r.Revision != s.Revision {
		return TaxonomyPreview{}, errors.New("revision conflict: reload before proposing a taxonomy")
	}
	sample := taxonomySamples(s.Data, private)
	if len(sample) == 0 {
		return TaxonomyPreview{}, errors.New("import transactions before proposing a taxonomy")
	}
	proposal, err := client.ProposeTaxonomy(ctx, sample)
	if err != nil {
		return TaxonomyPreview{}, err
	}
	proposal = filterExistingTaxonomy(s.Data, proposal)
	p := TaxonomyPreview{ID: domain.NewID("taxonomy"), Revision: s.Revision, Sample: sample, Proposal: proposal, created: time.Now()}
	a.mu.Lock()
	defer a.mu.Unlock()
	for id, old := range a.taxonomies {
		if time.Since(old.created) > time.Hour {
			delete(a.taxonomies, id)
		}
	}
	if len(a.taxonomies) >= 20 {
		return TaxonomyPreview{}, errors.New("too many active taxonomy proposals; apply or discard one first")
	}
	a.taxonomies[p.ID] = p
	return p, nil
}

func proposalContainsCategory(values []classification.ProposedCategory, value classification.ProposedCategory) bool {
	return slicesContains(values, value)
}
func slicesContains[T any](values []T, value T) bool {
	for _, candidate := range values {
		if reflect.DeepEqual(candidate, value) {
			return true
		}
	}
	return false
}

func closeApprovedTaxonomy(full, approved classification.TaxonomyProposal) classification.TaxonomyProposal {
	out := approved
	for changed := true; changed; {
		changed = false
		for _, c := range append([]classification.ProposedCategory{}, out.Categories...) {
			if c.Parent == "" {
				continue
			}
			for _, parent := range full.Categories {
				if parent.Kind == c.Kind && strings.EqualFold(parent.Name, c.Parent) && !proposalContainsCategory(out.Categories, parent) {
					out.Categories = append(out.Categories, parent)
					changed = true
				}
			}
		}
	}
	return out
}

func applyTaxonomyCategories(d *domain.Dataset, approved []classification.ProposedCategory) error {
	key := func(name, kind string) string { return taxonomyTextKey(name) + "\x00" + kind }
	idByName := map[string]string{}
	for _, c := range d.Categories {
		idByName[key(c.Name, c.Kind)] = c.ID
	}
	assigned := map[string]int{}
	for _, tx := range d.Transactions {
		assigned[tx.Enrichment.CategoryID]++
	}
	for _, p := range approved {
		if _, exists := idByName[key(p.Name, p.Kind)]; exists {
			return fmt.Errorf("category %q already exists", p.Name)
		}
	}
	for pass := range 2 {
		for _, p := range approved {
			if _, exists := idByName[key(p.Name, p.Kind)]; exists {
				continue
			}
			parent := "cat_expenses"
			if p.Kind == "income" {
				parent = "cat_income"
			}
			if p.Parent != "" {
				if id, ok := idByName[key(p.Parent, p.Kind)]; ok {
					if assigned[id] > 0 {
						return fmt.Errorf("category %q holds %d transactions and cannot gain a subcategory; reclassify them first", p.Parent, assigned[id])
					}
					parent = id
				} else if pass == 0 {
					continue
				}
			}
			category := domain.Category{ID: domain.NewID("cat"), Name: p.Name, ParentID: parent, Kind: p.Kind, Hint: p.Hint}
			if err := SaveCategory(d, category); err != nil {
				return err
			}
			idByName[key(p.Name, p.Kind)] = category.ID
		}
	}
	return nil
}

func applyTaxonomyTags(d *domain.Dataset, approved []classification.ProposedTag) error {
	seen := map[string]bool{}
	for _, tag := range d.Tags {
		seen[taxonomyTextKey(tag.Name)] = true
	}
	for _, p := range approved {
		if seen[taxonomyTextKey(p.Name)] {
			return fmt.Errorf("tag %q already exists", p.Name)
		}
		if err := SaveTag(d, domain.Tag{ID: domain.NewID("tag"), Name: p.Name, Hint: p.Hint}); err != nil {
			return err
		}
		seen[taxonomyTextKey(p.Name)] = true
	}
	return nil
}

func applyTaxonomyMerchants(d *domain.Dataset, approved []classification.ProposedMerchant) error {
	owners := map[string]string{}
	for _, merchant := range d.Merchants {
		owners[taxonomyTextKey(merchant.Name)] = merchant.ID
		for _, alias := range merchant.Aliases {
			owners[taxonomyTextKey(alias)] = merchant.ID
		}
	}
	for _, p := range approved {
		if owner := owners[taxonomyTextKey(p.Name)]; owner != "" {
			return fmt.Errorf("merchant %q already exists or is an alias", p.Name)
		}
		aliases := make([]string, 0, len(p.Aliases))
		for _, alias := range p.Aliases {
			if owner := owners[taxonomyTextKey(alias)]; owner != "" {
				return fmt.Errorf("merchant alias %q collides with another merchant", alias)
			}
			if !slicesContains(aliases, alias) {
				aliases = append(aliases, alias)
			}
		}
		merchant := domain.Merchant{ID: domain.NewID("mer"), Name: p.Name, Aliases: aliases, DefaultTagIDs: []string{}, UseDefaults: false}
		if err := SaveMerchant(d, merchant); err != nil {
			return err
		}
		owners[taxonomyTextKey(merchant.Name)] = merchant.ID
		for _, alias := range aliases {
			owners[taxonomyTextKey(alias)] = merchant.ID
		}
	}
	return nil
}

func (a *App) ApplyTaxonomy(ctx context.Context, id, rev string, approved classification.TaxonomyProposal) (State, error) {
	a.mu.Lock()
	defer a.mu.Unlock()
	cached, ok := a.taxonomies[id]
	if !ok || time.Since(cached.created) > time.Hour {
		return State{}, errors.New("taxonomy proposal expired or unknown; propose again")
	}
	if rev != cached.Revision {
		return State{}, errors.New("revision conflict: taxonomy was generated from different records")
	}
	if err := classification.ValidateTaxonomyProposal(approved); err != nil {
		return State{}, err
	}
	contains := func() bool {
		for _, c := range approved.Categories {
			if !slicesContains(cached.Proposal.Categories, c) {
				return false
			}
		}
		for _, t := range approved.Tags {
			if !slicesContains(cached.Proposal.Tags, t) {
				return false
			}
		}
		for _, m := range approved.Merchants {
			if !slicesContains(cached.Proposal.Merchants, m) {
				return false
			}
		}
		return true
	}()
	if !contains {
		return State{}, errors.New("approved taxonomy item was not in the proposal")
	}
	approved = closeApprovedTaxonomy(cached.Proposal, approved)
	if len(approved.Categories)+len(approved.Tags)+len(approved.Merchants) == 0 {
		return State{}, errors.New("approve at least one taxonomy item")
	}
	s, err := a.snapshot(ctx)
	if err != nil {
		return State{}, err
	}
	if s.Revision != rev {
		return State{}, errors.New("revision conflict: data changed after proposal; propose again")
	}
	if err := applyTaxonomyCategories(&s.Data, approved.Categories); err != nil {
		return State{}, err
	}
	if err := applyTaxonomyTags(&s.Data, approved.Tags); err != nil {
		return State{}, err
	}
	if err := applyTaxonomyMerchants(&s.Data, approved.Merchants); err != nil {
		return State{}, err
	}
	state, err := a.commit(ctx, rev, s.Data)
	if err != nil {
		return State{}, err
	}
	delete(a.taxonomies, id)
	return state, nil
}

func taxonomyMin(a, b int) int {
	if a < b {
		return a
	}
	return b
}