115 lines
3.6 KiB
Go
115 lines
3.6 KiB
Go
package classification
|
|
|
|
import (
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
|
|
"finance-duck/internal/domain"
|
|
)
|
|
|
|
var bankingPatterns = []*regexp.Regexp{
|
|
// Apply before tokenization to capture formatted identifiers as a unit.
|
|
regexp.MustCompile(`(?i)\b[a-z]{2}\s*\d{2}(?:[ -]?[a-z0-9]){11,30}\b`),
|
|
regexp.MustCompile(`(?i)\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b`),
|
|
regexp.MustCompile(`(?i)\b(?:iban|bic|swift|account(?:\s*(?:number|no))?|konto(?:nummer)?|reference|ref|payment\s*(?:id|reference)|end\s*to\s*end(?:\s*id)?|e2e|eref|mref|kref|cred|mandate|mandat(?:sreferenz)?|kunden(?:nummer|referenz)|kreditornummer|glaeubiger\s*id|gläubiger\s*id)\b[^;\n|]*`),
|
|
regexp.MustCompile(`(?i)\b[A-Z]{6}[A-Z0-9]{2}(?:[A-Z0-9]{3})?\b`),
|
|
regexp.MustCompile(`(?i)\b(?:https?://|www\.)\S+|\b[^\s@]+@[^\s@]+\b`),
|
|
}
|
|
|
|
var identifierPatterns = append(append([]*regexp.Regexp{}, bankingPatterns...),
|
|
regexp.MustCompile(`\b\d{4,6}[\*x]{4,}\d{2,4}\b`),
|
|
regexp.MustCompile(`\b\d{4}-\d{2}-\d{2}T[\d:]+\b`),
|
|
)
|
|
|
|
// countDigits counts decimal digits in a token. The redaction rule drops a
|
|
// token with four or more, or with three among letters, so the count has to be
|
|
// over runes rather than bytes.
|
|
func countDigits(text string) int {
|
|
digits := 0
|
|
for _, r := range text {
|
|
if unicode.IsDigit(r) {
|
|
digits++
|
|
}
|
|
}
|
|
return digits
|
|
}
|
|
|
|
func addSecret(secrets map[string]bool, value string) {
|
|
normalized := normalize(value)
|
|
if normalized == "" {
|
|
return
|
|
}
|
|
secrets[normalized] = true
|
|
}
|
|
|
|
// redactor builds one text filter per request from the account registry, the
|
|
// facts being classified, and configured private names. Counterparties and
|
|
// stored transaction facts are deliberately not secrets.
|
|
func redactor(d domain.Dataset, f domain.Facts, private []string) func(string) string {
|
|
secrets := map[string]bool{}
|
|
for _, a := range d.Accounts {
|
|
addSecret(secrets, a.ID)
|
|
addSecret(secrets, a.IBAN)
|
|
addSecret(secrets, a.ExternalAccountID)
|
|
}
|
|
for _, value := range []string{f.ID, f.ExternalID, f.Fingerprint, f.CounterpartyIBAN} {
|
|
addSecret(secrets, value)
|
|
}
|
|
for _, name := range private {
|
|
addSecret(secrets, name)
|
|
}
|
|
values := make([]string, 0, len(secrets))
|
|
for value := range secrets {
|
|
values = append(values, value)
|
|
}
|
|
sort.Slice(values, func(i, j int) bool {
|
|
if len(values[i]) != len(values[j]) {
|
|
return len(values[i]) > len(values[j])
|
|
}
|
|
return values[i] < values[j]
|
|
})
|
|
return func(text string) string {
|
|
if !utf8.ValidString(text) {
|
|
return ""
|
|
}
|
|
for _, pattern := range identifierPatterns {
|
|
text = pattern.ReplaceAllString(text, " ")
|
|
}
|
|
text = " " + normalize(text) + " "
|
|
for _, value := range values {
|
|
needle := " " + value + " "
|
|
for strings.Contains(text, needle) {
|
|
text = strings.ReplaceAll(text, needle, " ")
|
|
}
|
|
}
|
|
kept, length := make([]string, 0, 16), 0
|
|
for _, token := range strings.Fields(text) {
|
|
digits := countDigits(token)
|
|
if digits >= 4 || (digits >= 3 && digits < utf8.RuneCountInString(token)) || utf8.RuneCountInString(token) > 40 {
|
|
continue
|
|
}
|
|
if length+len(token) > 500 {
|
|
break
|
|
}
|
|
kept = append(kept, token)
|
|
length += len(token) + 1
|
|
}
|
|
return strings.Join(kept, " ")
|
|
}
|
|
}
|
|
|
|
// redact is the stateless dataset-only form used when no current Facts object
|
|
// is available. Classification uses redactor so the current row's own ids are
|
|
// also removed.
|
|
func redact(text string, d domain.Dataset, private []string) string {
|
|
return redactor(d, domain.Facts{}, private)(text)
|
|
}
|
|
|
|
// Redact applies the identifier-only policy to one text field.
|
|
func Redact(text string, data domain.Dataset, facts domain.Facts, private []string) string {
|
|
return redactor(data, facts, private)(text)
|
|
}
|