Implement classification redesign
This commit is contained in:
+44
-10
@@ -12,6 +12,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"unicode/utf8"
|
||||
|
||||
"finance-duck/internal/analytics"
|
||||
"finance-duck/internal/banking"
|
||||
@@ -22,11 +23,12 @@ import (
|
||||
|
||||
// Settings holds preferences only, never credentials. ClassifyOnImport controls
|
||||
// whether newly imported transactions are sent to the model at all; merchant
|
||||
// rules always apply.
|
||||
// rules always apply. PrivateNames is a semicolon-separated household redaction
|
||||
// list when persisted in config.toml.
|
||||
type Settings struct {
|
||||
Model string `json:"model"`
|
||||
IncludeAmount bool `json:"include_amount"`
|
||||
ClassifyOnImport bool `json:"classify_on_import"`
|
||||
Model string `json:"model"`
|
||||
ClassifyOnImport bool `json:"classify_on_import"`
|
||||
PrivateNames []string `json:"private_names"`
|
||||
}
|
||||
type Status struct {
|
||||
SyncError string `json:"sync_error"`
|
||||
@@ -69,6 +71,7 @@ type App struct {
|
||||
bank banking.Provider
|
||||
classifier classification.Client
|
||||
previews map[string]Preview
|
||||
taxonomies map[string]TaxonomyPreview
|
||||
csvImports map[string]CSVImport
|
||||
authStates map[string]authorization
|
||||
callbackURL string
|
||||
@@ -81,7 +84,7 @@ func Open(dir string) (*App, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a := &App{dir: dir, journal: j, previews: make(map[string]Preview), csvImports: make(map[string]CSVImport), authStates: make(map[string]authorization), syncRequested: make(chan struct{}, 1)}
|
||||
a := &App{dir: dir, journal: j, previews: make(map[string]Preview), taxonomies: make(map[string]TaxonomyPreview), csvImports: make(map[string]CSVImport), authStates: make(map[string]authorization), syncRequested: make(chan struct{}, 1)}
|
||||
// Configurations written before this preference existed keep classifying
|
||||
// imports; only an explicit key switches it off.
|
||||
a.settings.ClassifyOnImport = true
|
||||
@@ -107,8 +110,8 @@ func Open(dir string) (*App, error) {
|
||||
switch k {
|
||||
case "classification_model":
|
||||
a.settings.Model, err = strconv.Unquote(v)
|
||||
case "include_amount":
|
||||
a.settings.IncludeAmount, err = strconv.ParseBool(v)
|
||||
case "private_names":
|
||||
a.settings.PrivateNames, err = parseNames(v)
|
||||
case "classify_on_import":
|
||||
a.settings.ClassifyOnImport, err = strconv.ParseBool(v)
|
||||
default:
|
||||
@@ -138,7 +141,7 @@ func Open(dir string) (*App, error) {
|
||||
if err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
a.classifier = classification.Client{APIKey: apiKey, Model: a.settings.Model, IncludeAmount: a.settings.IncludeAmount}
|
||||
a.classifier = classification.Client{APIKey: apiKey, Model: a.settings.Model, PrivateNames: append([]string{}, a.settings.PrivateNames...)}
|
||||
if err = a.loadBankingSettings(); err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
@@ -268,6 +271,32 @@ func normalizeOpenRouterKey(key string) (string, error) {
|
||||
return key, nil
|
||||
}
|
||||
|
||||
func normalizePrivateNames(values []string) ([]string, error) {
|
||||
out := make([]string, 0, len(values))
|
||||
for _, raw := range values {
|
||||
if !utf8.ValidString(raw) {
|
||||
return nil, errors.New("private names must be valid UTF-8")
|
||||
}
|
||||
name := strings.Join(strings.Fields(raw), " ")
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
if utf8.RuneCountInString(name) > 200 || strings.ContainsRune(name, ';') {
|
||||
return nil, errors.New("private names must be at most 200 characters and cannot contain semicolons")
|
||||
}
|
||||
out = append(out, name)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func parseNames(v string) ([]string, error) {
|
||||
raw, err := strconv.Unquote(v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return normalizePrivateNames(strings.Split(raw, ";"))
|
||||
}
|
||||
|
||||
func loadOpenRouterKey(path string) (string, error) {
|
||||
f, err := os.Open(path)
|
||||
if os.IsNotExist(err) {
|
||||
@@ -340,15 +369,20 @@ func (a *App) SaveSettings(ctx context.Context, s Settings) (State, error) {
|
||||
if len(s.Model) > 200 {
|
||||
return State{}, errors.New("model name is too long")
|
||||
}
|
||||
names, err := normalizePrivateNames(s.PrivateNames)
|
||||
if err != nil {
|
||||
return State{}, err
|
||||
}
|
||||
s.PrivateNames = names
|
||||
b := []byte("# Preferences only. Manage secrets in Settings or environment variables, never this file.\n" +
|
||||
"classification_model = " + strconv.Quote(s.Model) + "\n" +
|
||||
"include_amount = " + strconv.FormatBool(s.IncludeAmount) + "\n" +
|
||||
"private_names = " + strconv.Quote(strings.Join(s.PrivateNames, "; ")) + "\n" +
|
||||
"classify_on_import = " + strconv.FormatBool(s.ClassifyOnImport) + "\n")
|
||||
if err := atomicFile(filepath.Join(a.dir, "config.toml"), b); err != nil {
|
||||
return State{}, err
|
||||
}
|
||||
a.settings = s
|
||||
a.classifier.Model = s.Model
|
||||
a.classifier.IncludeAmount = s.IncludeAmount
|
||||
a.classifier.PrivateNames = append([]string{}, s.PrivateNames...)
|
||||
return a.snapshot(ctx)
|
||||
}
|
||||
|
||||
@@ -190,7 +190,7 @@ func mockClassifier(t *testing.T, a *App, inspect ...func(*http.Request)) {
|
||||
return
|
||||
}
|
||||
var prompt struct {
|
||||
Categories []struct{ ID, Name string } `json:"categories"`
|
||||
Categories []struct{ ID, Path string } `json:"categories"`
|
||||
}
|
||||
if len(req.Messages) != 2 || json.Unmarshal([]byte(req.Messages[1].Content), &prompt) != nil {
|
||||
w.WriteHeader(400)
|
||||
@@ -198,11 +198,11 @@ func mockClassifier(t *testing.T, a *App, inspect ...func(*http.Request)) {
|
||||
}
|
||||
category := ""
|
||||
for _, c := range prompt.Categories {
|
||||
if strings.Contains(strings.ToLower(c.Name), "groceries") {
|
||||
if strings.Contains(strings.ToLower(c.Path), "groceries") {
|
||||
category = c.ID
|
||||
}
|
||||
}
|
||||
content, _ := json.Marshal(map[string]any{"merchant_id": nil, "new_merchant": "REWE", "category_id": category, "tag_ids": []string{}})
|
||||
content, _ := json.Marshal(map[string]any{"merchant_id": nil, "new_merchant": "REWE", "category_id": category, "tag_ids": []string{}, "confidence": "high"})
|
||||
json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"finish_reason": "stop", "message": map[string]any{"content": string(content)}}}})
|
||||
}))
|
||||
t.Cleanup(mock.Close)
|
||||
@@ -289,3 +289,46 @@ func TestStalePreviewCannotOverwriteManualCorrection(t *testing.T) {
|
||||
t.Fatal("stale apply partially changed records")
|
||||
}
|
||||
}
|
||||
func TestTaxonomyProposalApprovalMintsOnlyApprovedEntries(t *testing.T) {
|
||||
a, s := testApp(t)
|
||||
s = seed(t, a, s)
|
||||
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
content := `{"categories":[{"name":"Food","parent":"","kind":"expense","hint":"Food purchases","because":["REWE"]},{"name":"Dining","parent":"Food","kind":"expense","hint":"Restaurants","because":["EDEKA"]}],"tags":[{"name":"Recurring","hint":"Repeats regularly"}],"merchants":[{"name":"REWE","aliases":["REWE"]}]}`
|
||||
json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{
|
||||
"finish_reason": "stop",
|
||||
"message": map[string]any{"content": content},
|
||||
}}})
|
||||
}))
|
||||
defer provider.Close()
|
||||
a.classifier = classification.Client{APIKey: "test", Model: "test/model", BaseURL: provider.URL}
|
||||
preview, err := a.ProposeTaxonomy(context.Background(), TaxonomyProposalRequest{
|
||||
Revision: s.Revision,
|
||||
Model: "test/model",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(preview.Sample) != 2 || len(preview.Proposal.Categories) != 2 {
|
||||
t.Fatalf("unexpected taxonomy preview: %+v", preview)
|
||||
}
|
||||
approved := classification.TaxonomyProposal{
|
||||
Categories: []classification.ProposedCategory{
|
||||
preview.Proposal.Categories[1],
|
||||
},
|
||||
}
|
||||
applied, err := a.ApplyTaxonomy(context.Background(), preview.ID, preview.Revision, approved)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
foundFood, foundDining := false, false
|
||||
for _, category := range applied.Data.Categories {
|
||||
foundFood = foundFood || category.Name == "Food"
|
||||
foundDining = foundDining || category.Name == "Dining"
|
||||
}
|
||||
if !foundFood || !foundDining {
|
||||
t.Fatalf("approved child did not bring its parent: %+v", applied.Data.Categories)
|
||||
}
|
||||
if len(applied.Data.Tags) != len(s.Data.Tags) || len(applied.Data.Merchants) != len(s.Data.Merchants) {
|
||||
t.Fatal("unapproved taxonomy entries were written")
|
||||
}
|
||||
}
|
||||
|
||||
+10
-2
@@ -25,9 +25,14 @@ type ImportResult struct {
|
||||
State State `json:"state"`
|
||||
}
|
||||
|
||||
func addProposal(d *domain.Dataset, p classification.Proposal) error {
|
||||
func addProposal(d *domain.Dataset, p classification.Proposal, facts ...domain.Facts) error {
|
||||
if p.NewMerchant != nil {
|
||||
m := *p.NewMerchant
|
||||
if len(facts) > 0 {
|
||||
if alias := strings.Join(strings.Fields(facts[0].Counterparty), " "); alias != "" && !slices.Contains(m.Aliases, alias) {
|
||||
m.Aliases = append(m.Aliases, alias)
|
||||
}
|
||||
}
|
||||
if slices.ContainsFunc(d.Merchants, func(v domain.Merchant) bool { return v.ID == m.ID }) {
|
||||
return errors.New("proposed merchant ID already exists")
|
||||
}
|
||||
@@ -79,7 +84,10 @@ func (a *App) importFacts(ctx context.Context, s State, facts []domain.Facts, in
|
||||
p, e = a.classifier.Classify(ctx, t.Facts, s.Data, false)
|
||||
}
|
||||
if e == nil {
|
||||
e = addProposal(&s.Data, p)
|
||||
e = addProposal(&s.Data, p, t.Facts)
|
||||
if e == nil && p.Enrichment.MerchantID != "" {
|
||||
classification.LearnAlias(&s.Data, t.Facts, p.Enrichment.MerchantID)
|
||||
}
|
||||
}
|
||||
if e == nil {
|
||||
e = domain.ValidateEnrichment(s.Data, t.Facts, p.Enrichment)
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"finance-duck/internal/banking"
|
||||
"finance-duck/internal/classification"
|
||||
"finance-duck/internal/domain"
|
||||
)
|
||||
|
||||
@@ -108,6 +109,12 @@ func SaveCategory(d *domain.Dataset, v domain.Category) error {
|
||||
d.Categories = append(d.Categories, v)
|
||||
return nil
|
||||
}
|
||||
|
||||
// LearnAlias records the chosen counterparty as a merchant alias when the
|
||||
// classification matcher remains unambiguous.
|
||||
func LearnAlias(d *domain.Dataset, f domain.Facts, merchantID string) bool {
|
||||
return classification.LearnAlias(d, f, merchantID)
|
||||
}
|
||||
func SaveTag(d *domain.Dataset, v domain.Tag) error {
|
||||
v.Name = strings.TrimSpace(v.Name)
|
||||
if v.ID == "" {
|
||||
|
||||
@@ -0,0 +1,408 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"finance-duck/internal/classification"
|
||||
"finance-duck/internal/domain"
|
||||
)
|
||||
|
||||
const taxonomySampleLimit = 300
|
||||
|
||||
type TaxonomyProposalRequest struct {
|
||||
Revision string `json:"revision"`
|
||||
Model string `json:"model"`
|
||||
}
|
||||
|
||||
type TaxonomyPreview struct {
|
||||
ID string `json:"id"`
|
||||
Revision string `json:"revision"`
|
||||
Sample []classification.TaxonomySample `json:"sample"`
|
||||
Proposal classification.TaxonomyProposal `json:"proposal"`
|
||||
created time.Time `json:"-"`
|
||||
}
|
||||
|
||||
func taxonomyTextKey(value string) string {
|
||||
return strings.Join(strings.Fields(strings.Map(func(r rune) rune {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
||||
return unicode.ToLower(r)
|
||||
}
|
||||
return ' '
|
||||
}, value)), " ")
|
||||
}
|
||||
|
||||
func taxonomySamples(d domain.Dataset, private []string) []classification.TaxonomySample {
|
||||
indices := make([]int, 0, len(d.Transactions))
|
||||
for i, tx := range d.Transactions {
|
||||
if tx.Enrichment.Kind == "transfer" || tx.Enrichment.Kind == domain.KindInvestment || (tx.Enrichment.Kind != "expense" && tx.Enrichment.Kind != "income") {
|
||||
continue
|
||||
}
|
||||
indices = append(indices, i)
|
||||
}
|
||||
if len(indices) == 0 {
|
||||
return nil
|
||||
}
|
||||
groups := map[string]int{}
|
||||
for _, i := range indices {
|
||||
tx := d.Transactions[i]
|
||||
key := taxonomyTextKey(tx.Facts.Counterparty)
|
||||
if key == "" {
|
||||
key = taxonomyTextKey(tx.Facts.RawDescription)
|
||||
}
|
||||
if key == "" {
|
||||
key = tx.Facts.ID
|
||||
}
|
||||
if current, ok := groups[key]; !ok || tx.Facts.BookingDate < d.Transactions[current].Facts.BookingDate || tx.Facts.BookingDate == d.Transactions[current].Facts.BookingDate && tx.Facts.ID < d.Transactions[current].Facts.ID {
|
||||
groups[key] = i
|
||||
}
|
||||
}
|
||||
keys := make([]string, 0, len(groups))
|
||||
for key := range groups {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
selected := make([]int, 0, taxonomyMin(taxonomySampleLimit, len(indices)))
|
||||
seen := map[int]bool{}
|
||||
add := func(i int) {
|
||||
if len(selected) == taxonomySampleLimit || seen[i] {
|
||||
return
|
||||
}
|
||||
selected = append(selected, i)
|
||||
seen[i] = true
|
||||
}
|
||||
for _, kind := range []string{"expense", "income"} {
|
||||
var first, last = -1, -1
|
||||
for _, i := range indices {
|
||||
if d.Transactions[i].Enrichment.Kind != kind {
|
||||
continue
|
||||
}
|
||||
if first == -1 || d.Transactions[i].Facts.BookingDate < d.Transactions[first].Facts.BookingDate {
|
||||
first = i
|
||||
}
|
||||
if last == -1 || d.Transactions[i].Facts.BookingDate > d.Transactions[last].Facts.BookingDate {
|
||||
last = i
|
||||
}
|
||||
}
|
||||
if first >= 0 {
|
||||
add(first)
|
||||
}
|
||||
if last >= 0 {
|
||||
add(last)
|
||||
}
|
||||
}
|
||||
for _, key := range keys {
|
||||
if len(selected) == taxonomySampleLimit {
|
||||
break
|
||||
}
|
||||
add(groups[key])
|
||||
}
|
||||
remainder := make([]int, 0, len(indices)-len(selected))
|
||||
for _, i := range indices {
|
||||
if !seen[i] {
|
||||
remainder = append(remainder, i)
|
||||
}
|
||||
}
|
||||
rand.Shuffle(len(remainder), func(i, j int) { remainder[i], remainder[j] = remainder[j], remainder[i] })
|
||||
for _, i := range remainder {
|
||||
if len(selected) == taxonomySampleLimit {
|
||||
break
|
||||
}
|
||||
add(i)
|
||||
}
|
||||
out := make([]classification.TaxonomySample, 0, len(selected))
|
||||
for _, i := range selected {
|
||||
tx := d.Transactions[i]
|
||||
out = append(out, classification.TaxonomySample{
|
||||
Date: tx.Facts.BookingDate, Amount: string(tx.Facts.Amount), Currency: tx.Facts.Currency,
|
||||
Kind: tx.Enrichment.Kind,
|
||||
Description: classification.Redact(tx.Facts.RawDescription, d, tx.Facts, private),
|
||||
Counterparty: classification.Redact(tx.Facts.Counterparty, d, tx.Facts, private),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func filterExistingTaxonomy(d domain.Dataset, p classification.TaxonomyProposal) classification.TaxonomyProposal {
|
||||
categoryKeys := map[string]bool{}
|
||||
for _, c := range d.Categories {
|
||||
categoryKeys[taxonomyTextKey(c.Name)+"\x00"+c.Kind] = true
|
||||
}
|
||||
filtered := classification.TaxonomyProposal{}
|
||||
for _, c := range p.Categories {
|
||||
if !categoryKeys[taxonomyTextKey(c.Name)+"\x00"+c.Kind] {
|
||||
filtered.Categories = append(filtered.Categories, c)
|
||||
}
|
||||
}
|
||||
tagKeys := map[string]bool{}
|
||||
for _, t := range d.Tags {
|
||||
tagKeys[taxonomyTextKey(t.Name)] = true
|
||||
}
|
||||
for _, t := range p.Tags {
|
||||
if !tagKeys[taxonomyTextKey(t.Name)] {
|
||||
filtered.Tags = append(filtered.Tags, t)
|
||||
}
|
||||
}
|
||||
merchantOwners := map[string]bool{}
|
||||
for _, m := range d.Merchants {
|
||||
merchantOwners[taxonomyTextKey(m.Name)] = true
|
||||
for _, alias := range m.Aliases {
|
||||
merchantOwners[taxonomyTextKey(alias)] = true
|
||||
}
|
||||
}
|
||||
for _, m := range p.Merchants {
|
||||
if merchantOwners[taxonomyTextKey(m.Name)] {
|
||||
continue
|
||||
}
|
||||
aliases := make([]string, 0, len(m.Aliases))
|
||||
for _, alias := range m.Aliases {
|
||||
key := taxonomyTextKey(alias)
|
||||
if key != "" && !merchantOwners[key] && !slicesContains(aliases, alias) {
|
||||
aliases = append(aliases, alias)
|
||||
}
|
||||
}
|
||||
m.Aliases = aliases
|
||||
filtered.Merchants = append(filtered.Merchants, m)
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
func (a *App) ProposeTaxonomy(ctx context.Context, r TaxonomyProposalRequest) (TaxonomyPreview, error) {
|
||||
if strings.TrimSpace(r.Model) == "" {
|
||||
return TaxonomyPreview{}, errors.New("model is required")
|
||||
}
|
||||
a.mu.Lock()
|
||||
s, err := a.snapshot(ctx)
|
||||
client := a.classifier.WithModel(r.Model)
|
||||
private := append([]string{}, a.settings.PrivateNames...)
|
||||
a.mu.Unlock()
|
||||
if err != nil {
|
||||
return TaxonomyPreview{}, err
|
||||
}
|
||||
if r.Revision != s.Revision {
|
||||
return TaxonomyPreview{}, errors.New("revision conflict: reload before proposing a taxonomy")
|
||||
}
|
||||
sample := taxonomySamples(s.Data, private)
|
||||
if len(sample) == 0 {
|
||||
return TaxonomyPreview{}, errors.New("import transactions before proposing a taxonomy")
|
||||
}
|
||||
proposal, err := client.ProposeTaxonomy(ctx, sample)
|
||||
if err != nil {
|
||||
return TaxonomyPreview{}, err
|
||||
}
|
||||
proposal = filterExistingTaxonomy(s.Data, proposal)
|
||||
p := TaxonomyPreview{ID: domain.NewID("taxonomy"), Revision: s.Revision, Sample: sample, Proposal: proposal, created: time.Now()}
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
for id, old := range a.taxonomies {
|
||||
if time.Since(old.created) > time.Hour {
|
||||
delete(a.taxonomies, id)
|
||||
}
|
||||
}
|
||||
if len(a.taxonomies) >= 20 {
|
||||
return TaxonomyPreview{}, errors.New("too many active taxonomy proposals; apply or discard one first")
|
||||
}
|
||||
a.taxonomies[p.ID] = p
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func proposalContainsCategory(values []classification.ProposedCategory, value classification.ProposedCategory) bool {
|
||||
return slicesContains(values, value)
|
||||
}
|
||||
func slicesContains[T any](values []T, value T) bool {
|
||||
for _, candidate := range values {
|
||||
if reflect.DeepEqual(candidate, value) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func closeApprovedTaxonomy(full, approved classification.TaxonomyProposal) classification.TaxonomyProposal {
|
||||
out := approved
|
||||
for changed := true; changed; {
|
||||
changed = false
|
||||
for _, c := range append([]classification.ProposedCategory{}, out.Categories...) {
|
||||
if c.Parent == "" {
|
||||
continue
|
||||
}
|
||||
for _, parent := range full.Categories {
|
||||
if parent.Kind == c.Kind && strings.EqualFold(parent.Name, c.Parent) && !proposalContainsCategory(out.Categories, parent) {
|
||||
out.Categories = append(out.Categories, parent)
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func applyTaxonomyCategories(d *domain.Dataset, approved []classification.ProposedCategory) error {
|
||||
key := func(name, kind string) string { return taxonomyTextKey(name) + "\x00" + kind }
|
||||
idByName := map[string]string{}
|
||||
for _, c := range d.Categories {
|
||||
idByName[key(c.Name, c.Kind)] = c.ID
|
||||
}
|
||||
assigned := map[string]int{}
|
||||
for _, tx := range d.Transactions {
|
||||
assigned[tx.Enrichment.CategoryID]++
|
||||
}
|
||||
for _, p := range approved {
|
||||
if _, exists := idByName[key(p.Name, p.Kind)]; exists {
|
||||
return fmt.Errorf("category %q already exists", p.Name)
|
||||
}
|
||||
}
|
||||
for pass := range 2 {
|
||||
for _, p := range approved {
|
||||
if _, exists := idByName[key(p.Name, p.Kind)]; exists {
|
||||
continue
|
||||
}
|
||||
parent := "cat_expenses"
|
||||
if p.Kind == "income" {
|
||||
parent = "cat_income"
|
||||
}
|
||||
if p.Parent != "" {
|
||||
if id, ok := idByName[key(p.Parent, p.Kind)]; ok {
|
||||
if assigned[id] > 0 {
|
||||
return fmt.Errorf("category %q holds %d transactions and cannot gain a subcategory; reclassify them first", p.Parent, assigned[id])
|
||||
}
|
||||
parent = id
|
||||
} else if pass == 0 {
|
||||
continue
|
||||
}
|
||||
}
|
||||
category := domain.Category{ID: domain.NewID("cat"), Name: p.Name, ParentID: parent, Kind: p.Kind, Hint: p.Hint}
|
||||
if err := SaveCategory(d, category); err != nil {
|
||||
return err
|
||||
}
|
||||
idByName[key(p.Name, p.Kind)] = category.ID
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func applyTaxonomyTags(d *domain.Dataset, approved []classification.ProposedTag) error {
|
||||
seen := map[string]bool{}
|
||||
for _, tag := range d.Tags {
|
||||
seen[taxonomyTextKey(tag.Name)] = true
|
||||
}
|
||||
for _, p := range approved {
|
||||
if seen[taxonomyTextKey(p.Name)] {
|
||||
return fmt.Errorf("tag %q already exists", p.Name)
|
||||
}
|
||||
if err := SaveTag(d, domain.Tag{ID: domain.NewID("tag"), Name: p.Name, Hint: p.Hint}); err != nil {
|
||||
return err
|
||||
}
|
||||
seen[taxonomyTextKey(p.Name)] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func applyTaxonomyMerchants(d *domain.Dataset, approved []classification.ProposedMerchant) error {
|
||||
owners := map[string]string{}
|
||||
for _, merchant := range d.Merchants {
|
||||
owners[taxonomyTextKey(merchant.Name)] = merchant.ID
|
||||
for _, alias := range merchant.Aliases {
|
||||
owners[taxonomyTextKey(alias)] = merchant.ID
|
||||
}
|
||||
}
|
||||
for _, p := range approved {
|
||||
if owner := owners[taxonomyTextKey(p.Name)]; owner != "" {
|
||||
return fmt.Errorf("merchant %q already exists or is an alias", p.Name)
|
||||
}
|
||||
aliases := make([]string, 0, len(p.Aliases))
|
||||
for _, alias := range p.Aliases {
|
||||
if owner := owners[taxonomyTextKey(alias)]; owner != "" {
|
||||
return fmt.Errorf("merchant alias %q collides with another merchant", alias)
|
||||
}
|
||||
if !slicesContains(aliases, alias) {
|
||||
aliases = append(aliases, alias)
|
||||
}
|
||||
}
|
||||
merchant := domain.Merchant{ID: domain.NewID("mer"), Name: p.Name, Aliases: aliases, DefaultTagIDs: []string{}, UseDefaults: false}
|
||||
if err := SaveMerchant(d, merchant); err != nil {
|
||||
return err
|
||||
}
|
||||
owners[taxonomyTextKey(merchant.Name)] = merchant.ID
|
||||
for _, alias := range aliases {
|
||||
owners[taxonomyTextKey(alias)] = merchant.ID
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) ApplyTaxonomy(ctx context.Context, id, rev string, approved classification.TaxonomyProposal) (State, error) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
cached, ok := a.taxonomies[id]
|
||||
if !ok || time.Since(cached.created) > time.Hour {
|
||||
return State{}, errors.New("taxonomy proposal expired or unknown; propose again")
|
||||
}
|
||||
if rev != cached.Revision {
|
||||
return State{}, errors.New("revision conflict: taxonomy was generated from different records")
|
||||
}
|
||||
if err := classification.ValidateTaxonomyProposal(approved); err != nil {
|
||||
return State{}, err
|
||||
}
|
||||
contains := func() bool {
|
||||
for _, c := range approved.Categories {
|
||||
if !slicesContains(cached.Proposal.Categories, c) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for _, t := range approved.Tags {
|
||||
if !slicesContains(cached.Proposal.Tags, t) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for _, m := range approved.Merchants {
|
||||
if !slicesContains(cached.Proposal.Merchants, m) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}()
|
||||
if !contains {
|
||||
return State{}, errors.New("approved taxonomy item was not in the proposal")
|
||||
}
|
||||
approved = closeApprovedTaxonomy(cached.Proposal, approved)
|
||||
if len(approved.Categories)+len(approved.Tags)+len(approved.Merchants) == 0 {
|
||||
return State{}, errors.New("approve at least one taxonomy item")
|
||||
}
|
||||
s, err := a.snapshot(ctx)
|
||||
if err != nil {
|
||||
return State{}, err
|
||||
}
|
||||
if s.Revision != rev {
|
||||
return State{}, errors.New("revision conflict: data changed after proposal; propose again")
|
||||
}
|
||||
if err := applyTaxonomyCategories(&s.Data, approved.Categories); err != nil {
|
||||
return State{}, err
|
||||
}
|
||||
if err := applyTaxonomyTags(&s.Data, approved.Tags); err != nil {
|
||||
return State{}, err
|
||||
}
|
||||
if err := applyTaxonomyMerchants(&s.Data, approved.Merchants); err != nil {
|
||||
return State{}, err
|
||||
}
|
||||
state, err := a.commit(ctx, rev, s.Data)
|
||||
if err != nil {
|
||||
return State{}, err
|
||||
}
|
||||
delete(a.taxonomies, id)
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func taxonomyMin(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
@@ -99,7 +99,7 @@ func (a *App) Preview(ctx context.Context, r PreviewRequest) (Preview, error) {
|
||||
after := t.Enrichment
|
||||
if r.Fields.Merchant {
|
||||
after.MerchantID = proposal.Enrichment.MerchantID
|
||||
if e = addProposal(&s.Data, proposal); e != nil {
|
||||
if e = addProposal(&s.Data, proposal, t.Facts); e != nil {
|
||||
p.Errors = append(p.Errors, ClassificationError{t.Facts.ID, e.Error()})
|
||||
continue
|
||||
}
|
||||
@@ -175,16 +175,24 @@ func (a *App) ApplyPreview(ctx context.Context, id, rev string, ids []string) (S
|
||||
}
|
||||
needed := map[string]bool{}
|
||||
for i, t := range s.Data.Transactions {
|
||||
if selected[t.Facts.ID] {
|
||||
s.Data.Transactions[i].Enrichment = changes[t.Facts.ID]
|
||||
needed[changes[t.Facts.ID].MerchantID] = true
|
||||
if !selected[t.Facts.ID] {
|
||||
continue
|
||||
}
|
||||
s.Data.Transactions[i].Enrichment = changes[t.Facts.ID]
|
||||
needed[changes[t.Facts.ID].MerchantID] = true
|
||||
}
|
||||
for _, m := range p.NewMerchants {
|
||||
if needed[m.ID] {
|
||||
s.Data.Merchants = append(s.Data.Merchants, m)
|
||||
}
|
||||
}
|
||||
for _, t := range s.Data.Transactions {
|
||||
if selected[t.Facts.ID] && t.Enrichment.MerchantID != "" {
|
||||
// New merchants already carry their first alias; existing merchants
|
||||
// learn only when the real matcher stays unambiguous.
|
||||
LearnAlias(&s.Data, t.Facts, t.Enrichment.MerchantID)
|
||||
}
|
||||
}
|
||||
state, err := a.commit(ctx, rev, s.Data)
|
||||
if err != nil {
|
||||
return State{}, err
|
||||
|
||||
Reference in New Issue
Block a user