Batch Analyse requests and survive opaque provider schema budgets
Analyse now classifies up to ten same-kind transactions per provider request: the registry and history travel once per batch, so a thousand-row backfill costs about a hundred paced requests instead of a thousand. The answer schema appears once — an array item carrying an enum-bound ref — because providers meter strict schemas by token cost: duplicating registry enums per row, or bounding arrays with minItems/maxItems that Gemini expands per element, rejects real registries with a bare HTTP 400. Row count, duplicate refs, duplicate tags and taxonomy bounds are all enforced server-side instead, and a request still rejected outright halves until accepted, remembering the working size for the run. Batch requests scale the HTTP budget by row count, chunk failures cannot abort a run whose later rows succeeded, and rows resolved against one snapshot share one minted merchant. Measured on a real 165-row month over a zero-data-retention route: 165 analysed, 152 proposals, 0 errors, 17 requests, under 8 minutes. Fresh installs default to google/gemini-3.8-flash, the model that demonstrably honors strict structured outputs over a ZDR route. Preview changes now carry counterparty, amount and currency, and the review list shows the amount with a counterparty fallback for banks that leave descriptions empty.
This commit is contained in:
Lars Nolden
+90
-52
@@ -26,10 +26,13 @@ type PreviewRequest struct {
|
||||
Fields Fields `json:"fields"`
|
||||
}
|
||||
type Change struct {
|
||||
ID string `json:"id"`
|
||||
Description string `json:"description"`
|
||||
Before domain.Enrichment `json:"before"`
|
||||
After domain.Enrichment `json:"after"`
|
||||
ID string `json:"id"`
|
||||
Description string `json:"description"`
|
||||
Counterparty string `json:"counterparty"`
|
||||
Amount domain.Money `json:"amount"`
|
||||
Currency string `json:"currency"`
|
||||
Before domain.Enrichment `json:"before"`
|
||||
After domain.Enrichment `json:"after"`
|
||||
}
|
||||
type ClassificationError struct {
|
||||
ID string `json:"id"`
|
||||
@@ -198,12 +201,13 @@ func (a *App) PreviewProgress(id string) (PreviewProgress, error) {
|
||||
func classifyRange(ctx context.Context, client *classification.Client, s State, r PreviewRequest, id string, report func(PreviewProgress)) (Preview, error) {
|
||||
p := Preview{ID: id, Revision: s.Revision, Changes: []Change{}, Errors: []ClassificationError{}, created: time.Now()}
|
||||
baseMerchants := len(s.Data.Merchants)
|
||||
total := 0
|
||||
eligible := []domain.Transaction{}
|
||||
for _, t := range s.Data.Transactions {
|
||||
if previewEligible(t, r) {
|
||||
total++
|
||||
eligible = append(eligible, t)
|
||||
}
|
||||
}
|
||||
total := len(eligible)
|
||||
progress := func() {
|
||||
if report != nil {
|
||||
report(PreviewProgress{ID: id, Total: total, Analysed: p.Analysed, Changes: len(p.Changes), Unchanged: p.Unchanged, Errors: append([]ClassificationError{}, p.Errors...)})
|
||||
@@ -211,71 +215,105 @@ func classifyRange(ctx context.Context, client *classification.Client, s State,
|
||||
}
|
||||
succeeded := false
|
||||
repeated := 0
|
||||
for _, t := range s.Data.Transactions {
|
||||
if !previewEligible(t, r) {
|
||||
continue
|
||||
// One provider request classifies a whole chunk. Rows are partitioned by
|
||||
// transaction kind because expense and income use different category
|
||||
// enums; within a kind they keep journal order. New merchants proposed by
|
||||
// one chunk are registered before the next chunk runs, so later
|
||||
// duplicates link instead of minting again.
|
||||
chunks := [][]domain.Transaction{}
|
||||
for _, kind := range []string{"expense", "income"} {
|
||||
group := []domain.Transaction{}
|
||||
for _, t := range eligible {
|
||||
if domain.Fallback(t.Facts).Kind == kind {
|
||||
group = append(group, t)
|
||||
}
|
||||
}
|
||||
for start := 0; start < len(group); start += classification.MaxBatch {
|
||||
chunks = append(chunks, group[start:min(start+classification.MaxBatch, len(group))])
|
||||
}
|
||||
}
|
||||
for _, chunk := range chunks {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Preview{}, err
|
||||
}
|
||||
p.Analysed++
|
||||
proposal, e := client.Classify(ctx, t.Facts, s.Data, true)
|
||||
facts := make([]domain.Facts, len(chunk))
|
||||
for i, t := range chunk {
|
||||
facts[i] = t.Facts
|
||||
}
|
||||
results := client.ClassifyBatch(ctx, facts, s.Data)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Preview{}, err
|
||||
}
|
||||
if e != nil {
|
||||
if n := len(p.Errors); n > 0 && p.Errors[n-1].Error == e.Error() {
|
||||
repeated++
|
||||
} else {
|
||||
repeated = 1
|
||||
// A chunk can mix one slow request's failures with later successes;
|
||||
// count the successes first so a working run is never aborted by the
|
||||
// repeated-identical-failure heuristic.
|
||||
for _, result := range results {
|
||||
if result.Err == nil {
|
||||
succeeded = true
|
||||
}
|
||||
p.Errors = append(p.Errors, ClassificationError{t.Facts.ID, e.Error()})
|
||||
if !succeeded && repeated >= 3 {
|
||||
return Preview{}, fmt.Errorf("stopped after %d identical failures — %s — with %d of %d transactions not analysed", repeated, e.Error(), total-p.Analysed, total)
|
||||
}
|
||||
progress()
|
||||
continue
|
||||
}
|
||||
succeeded = true
|
||||
after := t.Enrichment
|
||||
if r.Fields.Merchant {
|
||||
after.MerchantID = proposal.Enrichment.MerchantID
|
||||
if e = addProposal(&s.Data, proposal, t.Facts); e != nil {
|
||||
for i, t := range chunk {
|
||||
p.Analysed++
|
||||
proposal, e := results[i].Proposal, results[i].Err
|
||||
if e != nil {
|
||||
if n := len(p.Errors); n > 0 && p.Errors[n-1].Error == e.Error() {
|
||||
repeated++
|
||||
} else {
|
||||
repeated = 1
|
||||
}
|
||||
p.Errors = append(p.Errors, ClassificationError{t.Facts.ID, e.Error()})
|
||||
if !succeeded && repeated >= 3 {
|
||||
return Preview{}, fmt.Errorf("stopped after %d identical failures — %s — with %d of %d transactions not analysed", repeated, e.Error(), total-p.Analysed, total)
|
||||
}
|
||||
progress()
|
||||
continue
|
||||
}
|
||||
succeeded = true
|
||||
after := t.Enrichment
|
||||
if r.Fields.Merchant {
|
||||
after.MerchantID = proposal.Enrichment.MerchantID
|
||||
if e = addProposal(&s.Data, proposal, t.Facts); e != nil {
|
||||
p.Errors = append(p.Errors, ClassificationError{t.Facts.ID, e.Error()})
|
||||
progress()
|
||||
continue
|
||||
}
|
||||
}
|
||||
if r.Fields.Category {
|
||||
after.CategoryID = proposal.Enrichment.CategoryID
|
||||
}
|
||||
if r.Fields.Tags {
|
||||
after.TagIDs = slices.Clone(proposal.Enrichment.TagIDs)
|
||||
}
|
||||
if e = domain.ValidateEnrichment(s.Data, t.Facts, after); e != nil {
|
||||
p.Errors = append(p.Errors, ClassificationError{t.Facts.ID, e.Error()})
|
||||
progress()
|
||||
continue
|
||||
}
|
||||
}
|
||||
if r.Fields.Category {
|
||||
after.CategoryID = proposal.Enrichment.CategoryID
|
||||
}
|
||||
if r.Fields.Tags {
|
||||
after.TagIDs = slices.Clone(proposal.Enrichment.TagIDs)
|
||||
}
|
||||
if e = domain.ValidateEnrichment(s.Data, t.Facts, after); e != nil {
|
||||
p.Errors = append(p.Errors, ClassificationError{t.Facts.ID, e.Error()})
|
||||
beforeComparable, afterComparable := t.Enrichment, after
|
||||
beforeComparable.Classification = domain.Provenance{}
|
||||
afterComparable.Classification = domain.Provenance{}
|
||||
beforeComparable.TagIDs = slices.Clone(beforeComparable.TagIDs)
|
||||
afterComparable.TagIDs = slices.Clone(afterComparable.TagIDs)
|
||||
slices.Sort(beforeComparable.TagIDs)
|
||||
slices.Sort(afterComparable.TagIDs)
|
||||
if reflect.DeepEqual(beforeComparable, afterComparable) {
|
||||
p.Unchanged++
|
||||
progress()
|
||||
continue
|
||||
}
|
||||
after.Classification = proposal.Enrichment.Classification
|
||||
p.Changes = append(p.Changes, Change{
|
||||
ID: t.Facts.ID, Description: t.Facts.RawDescription, Counterparty: t.Facts.Counterparty,
|
||||
Amount: t.Facts.Amount, Currency: t.Facts.Currency,
|
||||
Before: t.Enrichment, After: after,
|
||||
})
|
||||
progress()
|
||||
continue
|
||||
}
|
||||
beforeComparable, afterComparable := t.Enrichment, after
|
||||
beforeComparable.Classification = domain.Provenance{}
|
||||
afterComparable.Classification = domain.Provenance{}
|
||||
beforeComparable.TagIDs = slices.Clone(beforeComparable.TagIDs)
|
||||
afterComparable.TagIDs = slices.Clone(afterComparable.TagIDs)
|
||||
slices.Sort(beforeComparable.TagIDs)
|
||||
slices.Sort(afterComparable.TagIDs)
|
||||
if reflect.DeepEqual(beforeComparable, afterComparable) {
|
||||
p.Unchanged++
|
||||
progress()
|
||||
continue
|
||||
}
|
||||
after.Classification = proposal.Enrichment.Classification
|
||||
p.Changes = append(p.Changes, Change{t.Facts.ID, t.Facts.RawDescription, t.Enrichment, after})
|
||||
progress()
|
||||
}
|
||||
p.NewMerchants = append([]domain.Merchant{}, s.Data.Merchants[baseMerchants:]...)
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// enrichmentEqual compares enrichment semantically: tag order is not a change.
|
||||
func enrichmentEqual(a, b domain.Enrichment) bool {
|
||||
a.TagIDs = slices.Clone(a.TagIDs)
|
||||
|
||||
Reference in New Issue
Block a user