package app import ( "context" "errors" "fmt" "math/rand/v2" "reflect" "sort" "strings" "time" "unicode" "finance-duck/internal/classification" "finance-duck/internal/domain" ) const taxonomySampleLimit = 300 type TaxonomyProposalRequest struct { Revision string `json:"revision"` Model string `json:"model"` } type TaxonomyPreview struct { ID string `json:"id"` Revision string `json:"revision"` Sample []classification.TaxonomySample `json:"sample"` Proposal classification.TaxonomyProposal `json:"proposal"` created time.Time `json:"-"` } func taxonomyTextKey(value string) string { return strings.Join(strings.Fields(strings.Map(func(r rune) rune { if unicode.IsLetter(r) || unicode.IsDigit(r) { return unicode.ToLower(r) } return ' ' }, value)), " ") } func taxonomySamples(d domain.Dataset, private []string) []classification.TaxonomySample { indices := make([]int, 0, len(d.Transactions)) for i, tx := range d.Transactions { if tx.Enrichment.Kind == "transfer" || tx.Enrichment.Kind == domain.KindInvestment || (tx.Enrichment.Kind != "expense" && tx.Enrichment.Kind != "income") { continue } indices = append(indices, i) } if len(indices) == 0 { return nil } groups := map[string]int{} for _, i := range indices { tx := d.Transactions[i] key := taxonomyTextKey(tx.Facts.Counterparty) if key == "" { key = taxonomyTextKey(tx.Facts.RawDescription) } if key == "" { key = tx.Facts.ID } if current, ok := groups[key]; !ok || tx.Facts.BookingDate < d.Transactions[current].Facts.BookingDate || tx.Facts.BookingDate == d.Transactions[current].Facts.BookingDate && tx.Facts.ID < d.Transactions[current].Facts.ID { groups[key] = i } } keys := make([]string, 0, len(groups)) for key := range groups { keys = append(keys, key) } sort.Strings(keys) selected := make([]int, 0, taxonomyMin(taxonomySampleLimit, len(indices))) seen := map[int]bool{} add := func(i int) { if len(selected) == taxonomySampleLimit || seen[i] { return } selected = append(selected, i) seen[i] = true } for _, kind := range []string{"expense", "income"} { var first, last = -1, -1 for _, i := range indices { if d.Transactions[i].Enrichment.Kind != kind { continue } if first == -1 || d.Transactions[i].Facts.BookingDate < d.Transactions[first].Facts.BookingDate { first = i } if last == -1 || d.Transactions[i].Facts.BookingDate > d.Transactions[last].Facts.BookingDate { last = i } } if first >= 0 { add(first) } if last >= 0 { add(last) } } for _, key := range keys { if len(selected) == taxonomySampleLimit { break } add(groups[key]) } remainder := make([]int, 0, len(indices)-len(selected)) for _, i := range indices { if !seen[i] { remainder = append(remainder, i) } } rand.Shuffle(len(remainder), func(i, j int) { remainder[i], remainder[j] = remainder[j], remainder[i] }) for _, i := range remainder { if len(selected) == taxonomySampleLimit { break } add(i) } out := make([]classification.TaxonomySample, 0, len(selected)) for _, i := range selected { tx := d.Transactions[i] out = append(out, classification.TaxonomySample{ Date: tx.Facts.BookingDate, Amount: string(tx.Facts.Amount), Currency: tx.Facts.Currency, Kind: tx.Enrichment.Kind, Description: classification.Redact(tx.Facts.RawDescription, d, tx.Facts, private), Counterparty: classification.Redact(tx.Facts.Counterparty, d, tx.Facts, private), }) } return out } func filterExistingTaxonomy(d domain.Dataset, p classification.TaxonomyProposal) classification.TaxonomyProposal { categoryKeys := map[string]bool{} for _, c := range d.Categories { categoryKeys[taxonomyTextKey(c.Name)+"\x00"+c.Kind] = true } filtered := classification.TaxonomyProposal{} for _, c := range p.Categories { if !categoryKeys[taxonomyTextKey(c.Name)+"\x00"+c.Kind] { filtered.Categories = append(filtered.Categories, c) } } tagKeys := map[string]bool{} for _, t := range d.Tags { tagKeys[taxonomyTextKey(t.Name)] = true } for _, t := range p.Tags { if !tagKeys[taxonomyTextKey(t.Name)] { filtered.Tags = append(filtered.Tags, t) } } merchantOwners := map[string]bool{} for _, m := range d.Merchants { merchantOwners[taxonomyTextKey(m.Name)] = true for _, alias := range m.Aliases { merchantOwners[taxonomyTextKey(alias)] = true } } for _, m := range p.Merchants { if merchantOwners[taxonomyTextKey(m.Name)] { continue } aliases := make([]string, 0, len(m.Aliases)) for _, alias := range m.Aliases { key := taxonomyTextKey(alias) if key != "" && !merchantOwners[key] && !slicesContains(aliases, alias) { aliases = append(aliases, alias) } } m.Aliases = aliases filtered.Merchants = append(filtered.Merchants, m) } return filtered } func (a *App) ProposeTaxonomy(ctx context.Context, r TaxonomyProposalRequest) (TaxonomyPreview, error) { if strings.TrimSpace(r.Model) == "" { return TaxonomyPreview{}, errors.New("model is required") } a.mu.Lock() s, err := a.snapshot(ctx) client := a.classifier.WithModel(r.Model) private := append([]string{}, a.settings.PrivateNames...) a.mu.Unlock() if err != nil { return TaxonomyPreview{}, err } if r.Revision != s.Revision { return TaxonomyPreview{}, errors.New("revision conflict: reload before proposing a taxonomy") } sample := taxonomySamples(s.Data, private) if len(sample) == 0 { return TaxonomyPreview{}, errors.New("import transactions before proposing a taxonomy") } proposal, err := client.ProposeTaxonomy(ctx, sample) if err != nil { return TaxonomyPreview{}, err } proposal = filterExistingTaxonomy(s.Data, proposal) p := TaxonomyPreview{ID: domain.NewID("taxonomy"), Revision: s.Revision, Sample: sample, Proposal: proposal, created: time.Now()} a.mu.Lock() defer a.mu.Unlock() for id, old := range a.taxonomies { if time.Since(old.created) > time.Hour { delete(a.taxonomies, id) } } if len(a.taxonomies) >= 20 { return TaxonomyPreview{}, errors.New("too many active taxonomy proposals; apply or discard one first") } a.taxonomies[p.ID] = p return p, nil } func proposalContainsCategory(values []classification.ProposedCategory, value classification.ProposedCategory) bool { return slicesContains(values, value) } func slicesContains[T any](values []T, value T) bool { for _, candidate := range values { if reflect.DeepEqual(candidate, value) { return true } } return false } func closeApprovedTaxonomy(full, approved classification.TaxonomyProposal) classification.TaxonomyProposal { out := approved for changed := true; changed; { changed = false for _, c := range append([]classification.ProposedCategory{}, out.Categories...) { if c.Parent == "" { continue } for _, parent := range full.Categories { if parent.Kind == c.Kind && strings.EqualFold(parent.Name, c.Parent) && !proposalContainsCategory(out.Categories, parent) { out.Categories = append(out.Categories, parent) changed = true } } } } return out } func applyTaxonomyCategories(d *domain.Dataset, approved []classification.ProposedCategory) error { key := func(name, kind string) string { return taxonomyTextKey(name) + "\x00" + kind } idByName := map[string]string{} for _, c := range d.Categories { idByName[key(c.Name, c.Kind)] = c.ID } assigned := map[string]int{} for _, tx := range d.Transactions { assigned[tx.Enrichment.CategoryID]++ } for _, p := range approved { if _, exists := idByName[key(p.Name, p.Kind)]; exists { return fmt.Errorf("category %q already exists", p.Name) } } for pass := range 2 { for _, p := range approved { if _, exists := idByName[key(p.Name, p.Kind)]; exists { continue } parent := "cat_expenses" if p.Kind == "income" { parent = "cat_income" } if p.Parent != "" { if id, ok := idByName[key(p.Parent, p.Kind)]; ok { if assigned[id] > 0 { return fmt.Errorf("category %q holds %d transactions and cannot gain a subcategory; reclassify them first", p.Parent, assigned[id]) } parent = id } else if pass == 0 { continue } } category := domain.Category{ID: domain.NewID("cat"), Name: p.Name, ParentID: parent, Kind: p.Kind, Hint: p.Hint} if err := SaveCategory(d, category); err != nil { return err } idByName[key(p.Name, p.Kind)] = category.ID } } return nil } func applyTaxonomyTags(d *domain.Dataset, approved []classification.ProposedTag) error { seen := map[string]bool{} for _, tag := range d.Tags { seen[taxonomyTextKey(tag.Name)] = true } for _, p := range approved { if seen[taxonomyTextKey(p.Name)] { return fmt.Errorf("tag %q already exists", p.Name) } if err := SaveTag(d, domain.Tag{ID: domain.NewID("tag"), Name: p.Name, Hint: p.Hint}); err != nil { return err } seen[taxonomyTextKey(p.Name)] = true } return nil } func applyTaxonomyMerchants(d *domain.Dataset, approved []classification.ProposedMerchant) error { owners := map[string]string{} for _, merchant := range d.Merchants { owners[taxonomyTextKey(merchant.Name)] = merchant.ID for _, alias := range merchant.Aliases { owners[taxonomyTextKey(alias)] = merchant.ID } } for _, p := range approved { if owner := owners[taxonomyTextKey(p.Name)]; owner != "" { return fmt.Errorf("merchant %q already exists or is an alias", p.Name) } aliases := make([]string, 0, len(p.Aliases)) for _, alias := range p.Aliases { if owner := owners[taxonomyTextKey(alias)]; owner != "" { return fmt.Errorf("merchant alias %q collides with another merchant", alias) } if !slicesContains(aliases, alias) { aliases = append(aliases, alias) } } merchant := domain.Merchant{ID: domain.NewID("mer"), Name: p.Name, Aliases: aliases, DefaultTagIDs: []string{}, UseDefaults: false} if err := SaveMerchant(d, merchant); err != nil { return err } owners[taxonomyTextKey(merchant.Name)] = merchant.ID for _, alias := range aliases { owners[taxonomyTextKey(alias)] = merchant.ID } } return nil } func (a *App) ApplyTaxonomy(ctx context.Context, id, rev string, approved classification.TaxonomyProposal) (State, error) { a.mu.Lock() defer a.mu.Unlock() cached, ok := a.taxonomies[id] if !ok || time.Since(cached.created) > time.Hour { return State{}, errors.New("taxonomy proposal expired or unknown; propose again") } if rev != cached.Revision { return State{}, errors.New("revision conflict: taxonomy was generated from different records") } if err := classification.ValidateTaxonomyProposal(approved); err != nil { return State{}, err } contains := func() bool { for _, c := range approved.Categories { if !slicesContains(cached.Proposal.Categories, c) { return false } } for _, t := range approved.Tags { if !slicesContains(cached.Proposal.Tags, t) { return false } } for _, m := range approved.Merchants { if !slicesContains(cached.Proposal.Merchants, m) { return false } } return true }() if !contains { return State{}, errors.New("approved taxonomy item was not in the proposal") } approved = closeApprovedTaxonomy(cached.Proposal, approved) if len(approved.Categories)+len(approved.Tags)+len(approved.Merchants) == 0 { return State{}, errors.New("approve at least one taxonomy item") } s, err := a.snapshot(ctx) if err != nil { return State{}, err } if s.Revision != rev { return State{}, errors.New("revision conflict: data changed after proposal; propose again") } if err := applyTaxonomyCategories(&s.Data, approved.Categories); err != nil { return State{}, err } if err := applyTaxonomyTags(&s.Data, approved.Tags); err != nil { return State{}, err } if err := applyTaxonomyMerchants(&s.Data, approved.Merchants); err != nil { return State{}, err } state, err := a.commit(ctx, rev, s.Data) if err != nil { return State{}, err } delete(a.taxonomies, id) return state, nil } func taxonomyMin(a, b int) int { if a < b { return a } return b }