Implement classification redesign

internal/classification/privacy.go

@@ -5,6 +5,7 @@ import (
 	"sort"
 	"strings"
 	"unicode"
+	"unicode/utf8"
 
 	"finance-duck/internal/domain"
 )
@@ -18,52 +19,47 @@ var bankingPatterns = []*regexp.Regexp{
 	regexp.MustCompile(`(?i)\b(?:https?://|www\.)\S+|\b[^\s@]+@[^\s@]+\b`),
 }
 
-// No raw bank object is serialized. Known private values are removed from every
-// allowlisted text field; all digit-bearing tokens are additionally discarded.
-// This deliberately sacrifices numeric/BIC-shaped merchant names and reference-heavy text.
-// It is data minimization, not a guarantee of anonymization of arbitrary prose.
-func newSanitizer(facts domain.Facts, data domain.Dataset, publicMerchantLabels bool) func(string) string {
+var identifierPatterns = append(append([]*regexp.Regexp{}, bankingPatterns...),
+	regexp.MustCompile(`\b\d{4,6}[\*x]{4,}\d{2,4}\b`),
+	regexp.MustCompile(`\b\d{4}-\d{2}-\d{2}T[\d:]+\b`),
+)
+
+// countDigits counts decimal digits in a token. The redaction rule drops a
+// token with four or more, or with three among letters, so the count has to be
+// over runes rather than bytes.
+func countDigits(text string) int {
+	digits := 0
+	for _, r := range text {
+		if unicode.IsDigit(r) {
+			digits++
+		}
+	}
+	return digits
+}
+
+func addSecret(secrets map[string]bool, value string) {
+	normalized := normalize(value)
+	if normalized == "" {
+		return
+	}
+	secrets[normalized] = true
+}
+
+// redactor builds one text filter per request from the account registry, the
+// facts being classified, and configured private names. Counterparties and
+// stored transaction facts are deliberately not secrets.
+func redactor(d domain.Dataset, f domain.Facts, private []string) func(string) string {
 	secrets := map[string]bool{}
-	publicNames := map[string]bool{}
-	if publicMerchantLabels {
-		for _, merchant := range data.Merchants {
-			publicNames[normalize(merchant.Name)] = true
-		}
+	for _, a := range d.Accounts {
+		addSecret(secrets, a.ID)
+		addSecret(secrets, a.IBAN)
+		addSecret(secrets, a.ExternalAccountID)
 	}
-	add := func(value string) {
-		normalized := normalize(value)
-		if normalized != "" {
-			secrets[normalized] = true
-		}
-		for _, part := range strings.Fields(normalized) {
-			if len([]rune(part)) >= 2 {
-				secrets[part] = true
-			}
-		}
+	for _, value := range []string{f.ID, f.ExternalID, f.Fingerprint, f.CounterpartyIBAN} {
+		addSecret(secrets, value)
 	}
-	addFacts := func(f domain.Facts) {
-		add(f.ID)
-		add(f.Source)
-		add(f.AccountID)
-		add(f.ExternalID)
-		add(f.Fingerprint)
-		add(f.CounterpartyIBAN)
-		// This exception applies only to registered public merchant labels, never
-		// transaction prose or raw payee fields. Banking identifiers remain private.
-		if !publicNames[normalize(f.Counterparty)] {
-			add(f.Counterparty)
-		}
-	}
-	addFacts(facts)
-	for _, tx := range data.Transactions {
-		addFacts(tx.Facts)
-	}
-	for _, account := range data.Accounts {
-		add(account.ID)
-		add(account.ExternalAccountID)
-		add(account.IBAN)
-		add(account.DisplayName)
-		add(account.Institution)
+	for _, name := range private {
+		addSecret(secrets, name)
 	}
 	values := make([]string, 0, len(secrets))
 	for value := range secrets {
@@ -76,7 +72,10 @@ func newSanitizer(facts domain.Facts, data domain.Dataset, publicMerchantLabels
 		return values[i] < values[j]
 	})
 	return func(text string) string {
-		for _, pattern := range bankingPatterns {
+		if !utf8.ValidString(text) {
+			return ""
+		}
+		for _, pattern := range identifierPatterns {
 			text = pattern.ReplaceAllString(text, " ")
 		}
 		text = " " + normalize(text) + " "
@@ -86,11 +85,10 @@ func newSanitizer(facts domain.Facts, data domain.Dataset, publicMerchantLabels
 				text = strings.ReplaceAll(text, needle, " ")
 			}
 		}
-		tokens := strings.Fields(text)
-		kept := make([]string, 0, len(tokens))
-		length := 0
-		for _, token := range tokens {
-			if strings.IndexFunc(token, unicode.IsDigit) >= 0 || len([]rune(token)) > 40 {
+		kept, length := make([]string, 0, 16), 0
+		for _, token := range strings.Fields(text) {
+			digits := countDigits(token)
+			if digits >= 4 || (digits >= 3 && digits < utf8.RuneCountInString(token)) || utf8.RuneCountInString(token) > 40 {
 				continue
 			}
 			if length+len(token) > 500 {
@@ -102,3 +100,15 @@ func newSanitizer(facts domain.Facts, data domain.Dataset, publicMerchantLabels
 		return strings.Join(kept, " ")
 	}
 }
+
+// redact is the stateless dataset-only form used when no current Facts object
+// is available. Classification uses redactor so the current row's own ids are
+// also removed.
+func redact(text string, d domain.Dataset, private []string) string {
+	return redactor(d, domain.Facts{}, private)(text)
+}
+
+// Redact applies the identifier-only policy to one text field.
+func Redact(text string, data domain.Dataset, facts domain.Facts, private []string) string {
+	return redactor(data, facts, private)(text)
+}