Implement classification redesign

2026-09-11 22:46:17 +02:00
parent cc43a2f9a7
commit 87f052a3ea
23 changed files with 1602 additions and 296 deletions
@@ -0,0 +1,408 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand/v2"
+	"reflect"
+	"sort"
+	"strings"
+	"time"
+	"unicode"
+
+	"finance-duck/internal/classification"
+	"finance-duck/internal/domain"
+)
+
+const taxonomySampleLimit = 300
+
+type TaxonomyProposalRequest struct {
+	Revision string `json:"revision"`
+	Model    string `json:"model"`
+}
+
+type TaxonomyPreview struct {
+	ID       string                          `json:"id"`
+	Revision string                          `json:"revision"`
+	Sample   []classification.TaxonomySample `json:"sample"`
+	Proposal classification.TaxonomyProposal `json:"proposal"`
+	created  time.Time                       `json:"-"`
+}
+
+func taxonomyTextKey(value string) string {
+	return strings.Join(strings.Fields(strings.Map(func(r rune) rune {
+		if unicode.IsLetter(r) || unicode.IsDigit(r) {
+			return unicode.ToLower(r)
+		}
+		return ' '
+	}, value)), " ")
+}
+
+func taxonomySamples(d domain.Dataset, private []string) []classification.TaxonomySample {
+	indices := make([]int, 0, len(d.Transactions))
+	for i, tx := range d.Transactions {
+		if tx.Enrichment.Kind == "transfer" || tx.Enrichment.Kind == domain.KindInvestment || (tx.Enrichment.Kind != "expense" && tx.Enrichment.Kind != "income") {
+			continue
+		}
+		indices = append(indices, i)
+	}
+	if len(indices) == 0 {
+		return nil
+	}
+	groups := map[string]int{}
+	for _, i := range indices {
+		tx := d.Transactions[i]
+		key := taxonomyTextKey(tx.Facts.Counterparty)
+		if key == "" {
+			key = taxonomyTextKey(tx.Facts.RawDescription)
+		}
+		if key == "" {
+			key = tx.Facts.ID
+		}
+		if current, ok := groups[key]; !ok || tx.Facts.BookingDate < d.Transactions[current].Facts.BookingDate || tx.Facts.BookingDate == d.Transactions[current].Facts.BookingDate && tx.Facts.ID < d.Transactions[current].Facts.ID {
+			groups[key] = i
+		}
+	}
+	keys := make([]string, 0, len(groups))
+	for key := range groups {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	selected := make([]int, 0, taxonomyMin(taxonomySampleLimit, len(indices)))
+	seen := map[int]bool{}
+	add := func(i int) {
+		if len(selected) == taxonomySampleLimit || seen[i] {
+			return
+		}
+		selected = append(selected, i)
+		seen[i] = true
+	}
+	for _, kind := range []string{"expense", "income"} {
+		var first, last = -1, -1
+		for _, i := range indices {
+			if d.Transactions[i].Enrichment.Kind != kind {
+				continue
+			}
+			if first == -1 || d.Transactions[i].Facts.BookingDate < d.Transactions[first].Facts.BookingDate {
+				first = i
+			}
+			if last == -1 || d.Transactions[i].Facts.BookingDate > d.Transactions[last].Facts.BookingDate {
+				last = i
+			}
+		}
+		if first >= 0 {
+			add(first)
+		}
+		if last >= 0 {
+			add(last)
+		}
+	}
+	for _, key := range keys {
+		if len(selected) == taxonomySampleLimit {
+			break
+		}
+		add(groups[key])
+	}
+	remainder := make([]int, 0, len(indices)-len(selected))
+	for _, i := range indices {
+		if !seen[i] {
+			remainder = append(remainder, i)
+		}
+	}
+	rand.Shuffle(len(remainder), func(i, j int) { remainder[i], remainder[j] = remainder[j], remainder[i] })
+	for _, i := range remainder {
+		if len(selected) == taxonomySampleLimit {
+			break
+		}
+		add(i)
+	}
+	out := make([]classification.TaxonomySample, 0, len(selected))
+	for _, i := range selected {
+		tx := d.Transactions[i]
+		out = append(out, classification.TaxonomySample{
+			Date: tx.Facts.BookingDate, Amount: string(tx.Facts.Amount), Currency: tx.Facts.Currency,
+			Kind:         tx.Enrichment.Kind,
+			Description:  classification.Redact(tx.Facts.RawDescription, d, tx.Facts, private),
+			Counterparty: classification.Redact(tx.Facts.Counterparty, d, tx.Facts, private),
+		})
+	}
+	return out
+}
+
+func filterExistingTaxonomy(d domain.Dataset, p classification.TaxonomyProposal) classification.TaxonomyProposal {
+	categoryKeys := map[string]bool{}
+	for _, c := range d.Categories {
+		categoryKeys[taxonomyTextKey(c.Name)+"\x00"+c.Kind] = true
+	}
+	filtered := classification.TaxonomyProposal{}
+	for _, c := range p.Categories {
+		if !categoryKeys[taxonomyTextKey(c.Name)+"\x00"+c.Kind] {
+			filtered.Categories = append(filtered.Categories, c)
+		}
+	}
+	tagKeys := map[string]bool{}
+	for _, t := range d.Tags {
+		tagKeys[taxonomyTextKey(t.Name)] = true
+	}
+	for _, t := range p.Tags {
+		if !tagKeys[taxonomyTextKey(t.Name)] {
+			filtered.Tags = append(filtered.Tags, t)
+		}
+	}
+	merchantOwners := map[string]bool{}
+	for _, m := range d.Merchants {
+		merchantOwners[taxonomyTextKey(m.Name)] = true
+		for _, alias := range m.Aliases {
+			merchantOwners[taxonomyTextKey(alias)] = true
+		}
+	}
+	for _, m := range p.Merchants {
+		if merchantOwners[taxonomyTextKey(m.Name)] {
+			continue
+		}
+		aliases := make([]string, 0, len(m.Aliases))
+		for _, alias := range m.Aliases {
+			key := taxonomyTextKey(alias)
+			if key != "" && !merchantOwners[key] && !slicesContains(aliases, alias) {
+				aliases = append(aliases, alias)
+			}
+		}
+		m.Aliases = aliases
+		filtered.Merchants = append(filtered.Merchants, m)
+	}
+	return filtered
+}
+
+func (a *App) ProposeTaxonomy(ctx context.Context, r TaxonomyProposalRequest) (TaxonomyPreview, error) {
+	if strings.TrimSpace(r.Model) == "" {
+		return TaxonomyPreview{}, errors.New("model is required")
+	}
+	a.mu.Lock()
+	s, err := a.snapshot(ctx)
+	client := a.classifier.WithModel(r.Model)
+	private := append([]string{}, a.settings.PrivateNames...)
+	a.mu.Unlock()
+	if err != nil {
+		return TaxonomyPreview{}, err
+	}
+	if r.Revision != s.Revision {
+		return TaxonomyPreview{}, errors.New("revision conflict: reload before proposing a taxonomy")
+	}
+	sample := taxonomySamples(s.Data, private)
+	if len(sample) == 0 {
+		return TaxonomyPreview{}, errors.New("import transactions before proposing a taxonomy")
+	}
+	proposal, err := client.ProposeTaxonomy(ctx, sample)
+	if err != nil {
+		return TaxonomyPreview{}, err
+	}
+	proposal = filterExistingTaxonomy(s.Data, proposal)
+	p := TaxonomyPreview{ID: domain.NewID("taxonomy"), Revision: s.Revision, Sample: sample, Proposal: proposal, created: time.Now()}
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	for id, old := range a.taxonomies {
+		if time.Since(old.created) > time.Hour {
+			delete(a.taxonomies, id)
+		}
+	}
+	if len(a.taxonomies) >= 20 {
+		return TaxonomyPreview{}, errors.New("too many active taxonomy proposals; apply or discard one first")
+	}
+	a.taxonomies[p.ID] = p
+	return p, nil
+}
+
+func proposalContainsCategory(values []classification.ProposedCategory, value classification.ProposedCategory) bool {
+	return slicesContains(values, value)
+}
+func slicesContains[T any](values []T, value T) bool {
+	for _, candidate := range values {
+		if reflect.DeepEqual(candidate, value) {
+			return true
+		}
+	}
+	return false
+}
+
+func closeApprovedTaxonomy(full, approved classification.TaxonomyProposal) classification.TaxonomyProposal {
+	out := approved
+	for changed := true; changed; {
+		changed = false
+		for _, c := range append([]classification.ProposedCategory{}, out.Categories...) {
+			if c.Parent == "" {
+				continue
+			}
+			for _, parent := range full.Categories {
+				if parent.Kind == c.Kind && strings.EqualFold(parent.Name, c.Parent) && !proposalContainsCategory(out.Categories, parent) {
+					out.Categories = append(out.Categories, parent)
+					changed = true
+				}
+			}
+		}
+	}
+	return out
+}
+
+func applyTaxonomyCategories(d *domain.Dataset, approved []classification.ProposedCategory) error {
+	key := func(name, kind string) string { return taxonomyTextKey(name) + "\x00" + kind }
+	idByName := map[string]string{}
+	for _, c := range d.Categories {
+		idByName[key(c.Name, c.Kind)] = c.ID
+	}
+	assigned := map[string]int{}
+	for _, tx := range d.Transactions {
+		assigned[tx.Enrichment.CategoryID]++
+	}
+	for _, p := range approved {
+		if _, exists := idByName[key(p.Name, p.Kind)]; exists {
+			return fmt.Errorf("category %q already exists", p.Name)
+		}
+	}
+	for pass := range 2 {
+		for _, p := range approved {
+			if _, exists := idByName[key(p.Name, p.Kind)]; exists {
+				continue
+			}
+			parent := "cat_expenses"
+			if p.Kind == "income" {
+				parent = "cat_income"
+			}
+			if p.Parent != "" {
+				if id, ok := idByName[key(p.Parent, p.Kind)]; ok {
+					if assigned[id] > 0 {
+						return fmt.Errorf("category %q holds %d transactions and cannot gain a subcategory; reclassify them first", p.Parent, assigned[id])
+					}
+					parent = id
+				} else if pass == 0 {
+					continue
+				}
+			}
+			category := domain.Category{ID: domain.NewID("cat"), Name: p.Name, ParentID: parent, Kind: p.Kind, Hint: p.Hint}
+			if err := SaveCategory(d, category); err != nil {
+				return err
+			}
+			idByName[key(p.Name, p.Kind)] = category.ID
+		}
+	}
+	return nil
+}
+
+func applyTaxonomyTags(d *domain.Dataset, approved []classification.ProposedTag) error {
+	seen := map[string]bool{}
+	for _, tag := range d.Tags {
+		seen[taxonomyTextKey(tag.Name)] = true
+	}
+	for _, p := range approved {
+		if seen[taxonomyTextKey(p.Name)] {
+			return fmt.Errorf("tag %q already exists", p.Name)
+		}
+		if err := SaveTag(d, domain.Tag{ID: domain.NewID("tag"), Name: p.Name, Hint: p.Hint}); err != nil {
+			return err
+		}
+		seen[taxonomyTextKey(p.Name)] = true
+	}
+	return nil
+}
+
+func applyTaxonomyMerchants(d *domain.Dataset, approved []classification.ProposedMerchant) error {
+	owners := map[string]string{}
+	for _, merchant := range d.Merchants {
+		owners[taxonomyTextKey(merchant.Name)] = merchant.ID
+		for _, alias := range merchant.Aliases {
+			owners[taxonomyTextKey(alias)] = merchant.ID
+		}
+	}
+	for _, p := range approved {
+		if owner := owners[taxonomyTextKey(p.Name)]; owner != "" {
+			return fmt.Errorf("merchant %q already exists or is an alias", p.Name)
+		}
+		aliases := make([]string, 0, len(p.Aliases))
+		for _, alias := range p.Aliases {
+			if owner := owners[taxonomyTextKey(alias)]; owner != "" {
+				return fmt.Errorf("merchant alias %q collides with another merchant", alias)
+			}
+			if !slicesContains(aliases, alias) {
+				aliases = append(aliases, alias)
+			}
+		}
+		merchant := domain.Merchant{ID: domain.NewID("mer"), Name: p.Name, Aliases: aliases, DefaultTagIDs: []string{}, UseDefaults: false}
+		if err := SaveMerchant(d, merchant); err != nil {
+			return err
+		}
+		owners[taxonomyTextKey(merchant.Name)] = merchant.ID
+		for _, alias := range aliases {
+			owners[taxonomyTextKey(alias)] = merchant.ID
+		}
+	}
+	return nil
+}
+
+func (a *App) ApplyTaxonomy(ctx context.Context, id, rev string, approved classification.TaxonomyProposal) (State, error) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	cached, ok := a.taxonomies[id]
+	if !ok || time.Since(cached.created) > time.Hour {
+		return State{}, errors.New("taxonomy proposal expired or unknown; propose again")
+	}
+	if rev != cached.Revision {
+		return State{}, errors.New("revision conflict: taxonomy was generated from different records")
+	}
+	if err := classification.ValidateTaxonomyProposal(approved); err != nil {
+		return State{}, err
+	}
+	contains := func() bool {
+		for _, c := range approved.Categories {
+			if !slicesContains(cached.Proposal.Categories, c) {
+				return false
+			}
+		}
+		for _, t := range approved.Tags {
+			if !slicesContains(cached.Proposal.Tags, t) {
+				return false
+			}
+		}
+		for _, m := range approved.Merchants {
+			if !slicesContains(cached.Proposal.Merchants, m) {
+				return false
+			}
+		}
+		return true
+	}()
+	if !contains {
+		return State{}, errors.New("approved taxonomy item was not in the proposal")
+	}
+	approved = closeApprovedTaxonomy(cached.Proposal, approved)
+	if len(approved.Categories)+len(approved.Tags)+len(approved.Merchants) == 0 {
+		return State{}, errors.New("approve at least one taxonomy item")
+	}
+	s, err := a.snapshot(ctx)
+	if err != nil {
+		return State{}, err
+	}
+	if s.Revision != rev {
+		return State{}, errors.New("revision conflict: data changed after proposal; propose again")
+	}
+	if err := applyTaxonomyCategories(&s.Data, approved.Categories); err != nil {
+		return State{}, err
+	}
+	if err := applyTaxonomyTags(&s.Data, approved.Tags); err != nil {
+		return State{}, err
+	}
+	if err := applyTaxonomyMerchants(&s.Data, approved.Merchants); err != nil {
+		return State{}, err
+	}
+	state, err := a.commit(ctx, rev, s.Data)
+	if err != nil {
+		return State{}, err
+	}
+	delete(a.taxonomies, id)
+	return state, nil
+}
+
+func taxonomyMin(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}