105 lines
3.2 KiB
Go
105 lines
3.2 KiB
Go
package classification
|
|
|
|
import (
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"unicode"
|
|
|
|
"finance-duck/internal/domain"
|
|
)
|
|
|
|
var bankingPatterns = []*regexp.Regexp{
|
|
// Apply before tokenization to capture formatted identifiers as a unit.
|
|
regexp.MustCompile(`(?i)\b[a-z]{2}\s*\d{2}(?:[ -]?[a-z0-9]){11,30}\b`),
|
|
regexp.MustCompile(`(?i)\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b`),
|
|
regexp.MustCompile(`(?i)\b(?:iban|bic|swift|account(?:\s*(?:number|no))?|konto(?:nummer)?|reference|ref|payment\s*(?:id|reference)|end\s*to\s*end(?:\s*id)?|e2e|eref|mref|kref|cred|mandate|mandat(?:sreferenz)?|kunden(?:nummer|referenz)|kreditornummer|glaeubiger\s*id|gläubiger\s*id)\b[^;\n|]*`),
|
|
regexp.MustCompile(`(?i)\b[A-Z]{6}[A-Z0-9]{2}(?:[A-Z0-9]{3})?\b`),
|
|
regexp.MustCompile(`(?i)\b(?:https?://|www\.)\S+|\b[^\s@]+@[^\s@]+\b`),
|
|
}
|
|
|
|
// No raw bank object is serialized. Known private values are removed from every
|
|
// allowlisted text field; all digit-bearing tokens are additionally discarded.
|
|
// This deliberately sacrifices numeric/BIC-shaped merchant names and reference-heavy text.
|
|
// It is data minimization, not a guarantee of anonymization of arbitrary prose.
|
|
func newSanitizer(facts domain.Facts, data domain.Dataset, publicMerchantLabels bool) func(string) string {
|
|
secrets := map[string]bool{}
|
|
publicNames := map[string]bool{}
|
|
if publicMerchantLabels {
|
|
for _, merchant := range data.Merchants {
|
|
publicNames[normalize(merchant.Name)] = true
|
|
}
|
|
}
|
|
add := func(value string) {
|
|
normalized := normalize(value)
|
|
if normalized != "" {
|
|
secrets[normalized] = true
|
|
}
|
|
for _, part := range strings.Fields(normalized) {
|
|
if len([]rune(part)) >= 2 {
|
|
secrets[part] = true
|
|
}
|
|
}
|
|
}
|
|
addFacts := func(f domain.Facts) {
|
|
add(f.ID)
|
|
add(f.Source)
|
|
add(f.AccountID)
|
|
add(f.ExternalID)
|
|
add(f.Fingerprint)
|
|
add(f.CounterpartyIBAN)
|
|
// This exception applies only to registered public merchant labels, never
|
|
// transaction prose or raw payee fields. Banking identifiers remain private.
|
|
if !publicNames[normalize(f.Counterparty)] {
|
|
add(f.Counterparty)
|
|
}
|
|
}
|
|
addFacts(facts)
|
|
for _, tx := range data.Transactions {
|
|
addFacts(tx.Facts)
|
|
}
|
|
for _, account := range data.Accounts {
|
|
add(account.ID)
|
|
add(account.ExternalAccountID)
|
|
add(account.IBAN)
|
|
add(account.DisplayName)
|
|
add(account.Institution)
|
|
}
|
|
values := make([]string, 0, len(secrets))
|
|
for value := range secrets {
|
|
values = append(values, value)
|
|
}
|
|
sort.Slice(values, func(i, j int) bool {
|
|
if len(values[i]) != len(values[j]) {
|
|
return len(values[i]) > len(values[j])
|
|
}
|
|
return values[i] < values[j]
|
|
})
|
|
return func(text string) string {
|
|
for _, pattern := range bankingPatterns {
|
|
text = pattern.ReplaceAllString(text, " ")
|
|
}
|
|
text = " " + normalize(text) + " "
|
|
for _, value := range values {
|
|
needle := " " + value + " "
|
|
for strings.Contains(text, needle) {
|
|
text = strings.ReplaceAll(text, needle, " ")
|
|
}
|
|
}
|
|
tokens := strings.Fields(text)
|
|
kept := make([]string, 0, len(tokens))
|
|
length := 0
|
|
for _, token := range tokens {
|
|
if strings.IndexFunc(token, unicode.IsDigit) >= 0 || len([]rune(token)) > 40 {
|
|
continue
|
|
}
|
|
if length+len(token) > 500 {
|
|
break
|
|
}
|
|
kept = append(kept, token)
|
|
length += len(token) + 1
|
|
}
|
|
return strings.Join(kept, " ")
|
|
}
|
|
}
|