init
This commit is contained in:
Lars Nolden
@@ -0,0 +1,240 @@
|
||||
package classification
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"finance-duck/internal/domain"
|
||||
)
|
||||
|
||||
func normalize(text string) string {
|
||||
return strings.Join(strings.Fields(strings.Map(func(r rune) rune {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
||||
return unicode.ToLower(r)
|
||||
}
|
||||
return ' '
|
||||
}, text)), " ")
|
||||
}
|
||||
|
||||
// Only whole normalized phrases match, so e.g. Shell does not match Seashell.
|
||||
// Equal-length aliases shared by different merchants are ambiguous, not rules.
|
||||
func aliasMatch(description string, merchants []domain.Merchant) *domain.Merchant {
|
||||
text := " " + normalize(description) + " "
|
||||
var best *domain.Merchant
|
||||
score := 0
|
||||
ambiguous := false
|
||||
for i := range merchants {
|
||||
m := &merchants[i]
|
||||
names := append([]string{m.Name}, m.Aliases...)
|
||||
for _, name := range names {
|
||||
alias := normalize(name)
|
||||
if alias == "" || !strings.Contains(text, " "+alias+" ") {
|
||||
continue
|
||||
}
|
||||
if len(alias) > score {
|
||||
best = m
|
||||
score = len(alias)
|
||||
ambiguous = false
|
||||
} else if len(alias) == score && best != nil && best.ID != m.ID {
|
||||
ambiguous = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if ambiguous {
|
||||
return nil
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
func duplicateMerchant(name string, merchants []domain.Merchant) *domain.Merchant {
|
||||
key := normalize(name)
|
||||
var best *domain.Merchant
|
||||
for i := range merchants {
|
||||
m := &merchants[i]
|
||||
match := normalize(m.Name) == key
|
||||
for _, alias := range m.Aliases {
|
||||
match = match || normalize(alias) == key
|
||||
}
|
||||
if match && (best == nil || m.ID < best.ID) {
|
||||
best = m
|
||||
}
|
||||
}
|
||||
if best != nil {
|
||||
return best
|
||||
}
|
||||
// A near spelling can reuse an existing merchant only when exactly one
|
||||
// registry entry is similar. Token counts protect e.g. REWE vs REWE To Go.
|
||||
for i := range merchants {
|
||||
m := &merchants[i]
|
||||
match := nearMerchant(key, normalize(m.Name))
|
||||
for _, alias := range m.Aliases {
|
||||
match = match || nearMerchant(key, normalize(alias))
|
||||
}
|
||||
if !match {
|
||||
continue
|
||||
}
|
||||
if best != nil && best.ID != m.ID {
|
||||
return nil
|
||||
}
|
||||
best = m
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
func nearMerchant(a, b string) bool {
|
||||
if a == b {
|
||||
return true
|
||||
}
|
||||
left, right := []rune(a), []rune(b)
|
||||
if len(left) < 8 || len(right) < 8 || len(strings.Fields(a)) != len(strings.Fields(b)) {
|
||||
return false
|
||||
}
|
||||
if len(left)*100 < len(right)*85 || len(right)*100 < len(left)*85 {
|
||||
return false
|
||||
}
|
||||
trigrams := func(runes []rune) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for i := range len(runes) - 2 {
|
||||
out[string(runes[i:i+3])] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
x, y := trigrams(left), trigrams(right)
|
||||
shared := 0
|
||||
for gram := range x {
|
||||
if y[gram] {
|
||||
shared++
|
||||
}
|
||||
}
|
||||
return shared*200 >= (len(x)+len(y))*92
|
||||
}
|
||||
|
||||
type candidate struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
type candidateSet struct {
|
||||
categories, tags, merchants []candidate
|
||||
categoryIDs, tagIDs, merchantIDs map[string]string
|
||||
}
|
||||
type ranked struct {
|
||||
id, name string
|
||||
score int
|
||||
}
|
||||
|
||||
func similarity(description, name string) int {
|
||||
a, b := normalize(description), normalize(name)
|
||||
if b == "" {
|
||||
return 0
|
||||
}
|
||||
if strings.Contains(" "+a+" ", " "+b+" ") {
|
||||
return 10000 + len(b)
|
||||
}
|
||||
words := strings.Fields(a)
|
||||
score := 0
|
||||
for _, word := range strings.Fields(b) {
|
||||
for _, input := range words {
|
||||
if input == word {
|
||||
score += len(word)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return score
|
||||
}
|
||||
|
||||
func bounded(rows []ranked, prefix string, limit int, clean func(string) string) ([]candidate, map[string]string) {
|
||||
sort.Slice(rows, func(i, j int) bool {
|
||||
if rows[i].score != rows[j].score {
|
||||
return rows[i].score > rows[j].score
|
||||
}
|
||||
return rows[i].id < rows[j].id
|
||||
})
|
||||
if limit > 0 && len(rows) > limit {
|
||||
rows = rows[:limit]
|
||||
}
|
||||
out := make([]candidate, 0, len(rows))
|
||||
ids := make(map[string]string, len(rows))
|
||||
for i, row := range rows {
|
||||
id := fmt.Sprintf("%s%d", prefix, i+1)
|
||||
name := clean(row.name)
|
||||
if name == "" {
|
||||
name = "unnamed"
|
||||
}
|
||||
out = append(out, candidate{ID: id, Name: name})
|
||||
ids[id] = row.id
|
||||
}
|
||||
return out, ids
|
||||
}
|
||||
|
||||
func retrieve(description, kind string, data domain.Dataset, clean, merchantClean func(string) string) candidateSet {
|
||||
var categories, tags, merchants []ranked
|
||||
fallback := domain.ExpenseFallback
|
||||
if kind == "income" {
|
||||
fallback = domain.IncomeFallback
|
||||
}
|
||||
parents := map[string]bool{}
|
||||
for _, cat := range data.Categories {
|
||||
parents[cat.ParentID] = true
|
||||
}
|
||||
for _, cat := range data.Categories {
|
||||
if cat.Kind != kind || parents[cat.ID] {
|
||||
continue
|
||||
}
|
||||
name := domain.CategoryPath(data, cat.ID)
|
||||
score := similarity(description, name)
|
||||
if cat.ID == fallback {
|
||||
score = int(^uint(0) >> 1)
|
||||
}
|
||||
categories = append(categories, ranked{id: cat.ID, name: name, score: score})
|
||||
}
|
||||
for _, tag := range data.Tags {
|
||||
tags = append(tags, ranked{id: tag.ID, name: tag.Name, score: similarity(description, tag.Name)})
|
||||
}
|
||||
for _, m := range data.Merchants {
|
||||
score := similarity(description, m.Name)
|
||||
for _, alias := range m.Aliases {
|
||||
if s := similarity(description, alias); s > score {
|
||||
score = s
|
||||
}
|
||||
}
|
||||
merchants = append(merchants, ranked{id: m.ID, name: m.Name, score: score})
|
||||
}
|
||||
var set candidateSet
|
||||
set.categories, set.categoryIDs = bounded(categories, "c", 0, clean)
|
||||
set.tags, set.tagIDs = bounded(tags, "t", 0, clean)
|
||||
set.merchants, set.merchantIDs = bounded(merchants, "m", 20, merchantClean)
|
||||
return set
|
||||
}
|
||||
|
||||
func candidateEnums(candidates []candidate) []string {
|
||||
ids := make([]string, 0, len(candidates))
|
||||
for _, c := range candidates {
|
||||
ids = append(ids, c.ID)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func (c candidateSet) schema() map[string]any {
|
||||
merchantEnums := []any{nil}
|
||||
for _, m := range c.merchants {
|
||||
merchantEnums = append(merchantEnums, m.ID)
|
||||
}
|
||||
tagItems := map[string]any{"type": "string"}
|
||||
if len(c.tags) > 0 {
|
||||
tagItems["enum"] = candidateEnums(c.tags)
|
||||
}
|
||||
tags := map[string]any{"type": "array", "items": tagItems, "maxItems": len(c.tags), "uniqueItems": true}
|
||||
return map[string]any{
|
||||
"type": "object", "additionalProperties": false,
|
||||
"required": []string{"merchant_id", "new_merchant", "category_id", "tag_ids"},
|
||||
"properties": map[string]any{
|
||||
"merchant_id": map[string]any{"type": []string{"string", "null"}, "enum": merchantEnums, "description": "Existing merchant candidate ID, or null."},
|
||||
"new_merchant": map[string]any{"type": []string{"string", "null"}, "maxLength": 100, "description": "Public business name only when no existing merchant matches, otherwise null."},
|
||||
"category_id": map[string]any{"type": "string", "enum": candidateEnums(c.categories)},
|
||||
"tag_ids": tags,
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user