Value positions from a daily price feed

A position was a share count. An instrument now carries a market symbol and the last close fetched for it, so Wealth and the dashboard report cash plus market value instead of cash alone. The symbol is chosen by hand and never derived: one ISIN lists on several exchanges in different currencies, and a price from the wrong listing misstates wealth without failing any check. The refresh refuses a quote whose currency differs from the instrument's, keeps the previous quote when a symbol cannot be priced, and counts an instrument with no symbol as unpriced - naming it in a check and leaving it out of every total, because cost is not value. The quote belongs to the job: saving an instrument can neither set nor erase it, and changing the symbol discards it. Two things the provider forced. It answers HTTP 429 to every request whose User-Agent names a programming language, so the client identifies as a browser; without that header the first call of the day fails. Its closes are 32-bit floats widened to 64 - 165.26 arrives as 165.25999450683594 - so a figure is rounded to seven significant digits, which is what 24 mantissa bits carry; eight would have stored 165.25999 as a price. Accepted quotes are written in one commit against a revision re-read after the fetches, and nothing is committed when no quote changed. The automatic run starts shortly after launch and repeats daily on its own timer, so a sync backoff cannot delay it and prices arrive with no bank connected. Verified against live quotes end to end: 80 shares at 125.45 and 40 at 165.26 on 6000.00 cash report 22646.40 with one holding named as unpriced; giving that holding a symbol through the UI moves the figure to 23530.50, and a second refresh leaves the revision untouched.
2026-09-12 18:42:07 +02:00
parent 2373790be3
commit 588c16ad19
19 changed files with 1580 additions and 55 deletions
@@ -0,0 +1,302 @@
+// Package quotes retrieves daily closing prices for listed instruments so a
+// holding can be valued without anyone typing a price by hand. Prices enter the
+// journal as exact decimals: a float would make two runs of the same valuation
+// disagree in the last cents.
+package quotes
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"finance-duck/internal/domain"
+)
+
+// Client fetches the latest close for a market symbol. It holds no mutable
+// state, so a zero Client is usable and a copy is as good as the original.
+type Client struct {
+	HTTPClient *http.Client
+	BaseURL    string // defaults to https://query1.finance.yahoo.com
+}
+
+// Quote is one instrument's latest close. Symbol is the caller's own symbol
+// rather than the one echoed by the provider, so nothing derived from response
+// text can end up keyed against an instrument.
+type Quote struct {
+	Symbol   string
+	Price    domain.Quantity
+	Currency string
+	Day      string // YYYY-MM-DD
+}
+
+// Error reports a price lookup that failed for a reason Finance Duck
+// determined itself: the provider could not be reached, or its response could
+// not be used. Reason is written here and never taken from provider response
+// text, so callers may show the whole message to the user. Returning it for
+// every provider failure lets a caller tell provider trouble apart from a
+// programming error such as an unusable base URL.
+type Error struct {
+	Symbol string
+	Reason string
+}
+
+func (e Error) Error() string {
+	if e.Symbol == "" {
+		return "price lookup failed: " + e.Reason
+	}
+	return "price lookup for " + e.Symbol + " failed: " + e.Reason
+}
+
+// symbolPattern admits the listing symbols the chart endpoint uses, including
+// exchange suffixes ("VWCE.DE"), share classes ("BRK-B"), indices ("^GSPC")
+// and currency pairs ("EURUSD=X"). Anything else is rejected before a request
+// is built, so no caller-supplied text can reshape the request path.
+var symbolPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9.=^-]{0,31}$`)
+
+var currencyPattern = regexp.MustCompile(`^[A-Z]{3}$`)
+
+// defaultTimeout caps a lookup including the response read. A scheduled
+// refresh walks many instruments, so one unresponsive symbol must not hold the
+// whole run.
+const defaultTimeout = 15 * time.Second
+
+// A version-pinned desktop agent, not a bare "Mozilla/5.0": a real-looking
+// string is what the endpoint serves, and it carries no identifying data.
+const userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+
+// maxResponse bounds the chart response. Five daily candles are a few kilobytes
+// even with the metadata Yahoo attaches; a megabyte is a decoding accident.
+const maxResponse = 1 << 20
+
+// Latest returns the most recent usable close for symbol. A day whose close is
+// still null (today before the exchange settles, or a holiday) is skipped, so
+// the five-day window is what makes a Monday morning refresh return Friday's
+// price instead of nothing.
+func (c Client) Latest(ctx context.Context, symbol string) (Quote, error) {
+	if !symbolPattern.MatchString(symbol) || strings.Contains(symbol, "..") {
+		return Quote{}, Error{Symbol: symbol, Reason: "the symbol is not a valid market listing"}
+	}
+	base := strings.TrimRight(c.BaseURL, "/")
+	if base == "" {
+		base = "https://query1.finance.yahoo.com"
+	}
+	endpoint, err := url.Parse(base)
+	if err != nil || endpoint.Host == "" || endpoint.User != nil || endpoint.RawQuery != "" || endpoint.Fragment != "" {
+		return Quote{}, Error{Symbol: symbol, Reason: "the configured price provider address is invalid"}
+	}
+	// Plain HTTP is allowed only for a loopback stub; a real lookup must not
+	// take prices from an unauthenticated connection.
+	if endpoint.Scheme != "https" && !(endpoint.Scheme == "http" && (endpoint.Hostname() == "localhost" || endpoint.Hostname() == "127.0.0.1" || endpoint.Hostname() == "::1")) {
+		return Quote{}, Error{Symbol: symbol, Reason: "the price provider address must use HTTPS"}
+	}
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/v8/finance/chart/"+url.PathEscape(symbol)+"?range=5d&interval=1d", nil)
+	if err != nil {
+		return Quote{}, Error{Symbol: symbol, Reason: "the price request could not be created"}
+	}
+	request.Header.Set("Accept", "application/json")
+	// The endpoint answers 429 to every request whose User-Agent names a
+	// programming language, whatever the rate: an empty or Go-default agent is
+	// refused on the first call of the day, a browser agent is served. This is
+	// the price of an unkeyed provider and the only reason a real symbol
+	// resolves at all.
+	request.Header.Set("User-Agent", userAgent)
+	client := http.Client{Timeout: defaultTimeout}
+	if c.HTTPClient != nil {
+		client = *c.HTTPClient
+		if client.Timeout <= 0 {
+			client.Timeout = defaultTimeout
+		}
+	}
+	// A redirect to a consent or login page would answer with HTML that only
+	// fails later and less clearly than the redirect status itself.
+	client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
+	response, err := client.Do(request)
+	if err != nil {
+		// Cancellation and deadlines keep their identity: a caller shutting the
+		// scheduler down must not read that as the provider being broken.
+		if cause := ctx.Err(); cause != nil {
+			return Quote{}, cause
+		}
+		return Quote{}, Error{Symbol: symbol, Reason: "the price provider could not be reached"}
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return Quote{}, Error{Symbol: symbol, Reason: fmt.Sprintf("the price provider returned HTTP %d", response.StatusCode)}
+	}
+	var envelope struct {
+		Chart struct {
+			Result []struct {
+				Meta struct {
+					Currency string `json:"currency"`
+				} `json:"meta"`
+				Timestamp  []int64 `json:"timestamp"`
+				Indicators struct {
+					Quote []struct {
+						// json.Number keeps the provider's own decimal text: the
+						// price must never pass through a float. A null close
+						// decodes as the empty string and means "no trading".
+						Close []json.Number `json:"close"`
+					} `json:"quote"`
+				} `json:"indicators"`
+			} `json:"result"`
+			Error json.RawMessage `json:"error"`
+		} `json:"chart"`
+	}
+	// Unknown keys are tolerated because Yahoo adds metadata freely, but the
+	// fields read below are decoded strictly. The limit bounds the decode
+	// itself, so an oversized response fails as a truncated document.
+	decoder := json.NewDecoder(io.LimitReader(response.Body, maxResponse))
+	if err := decoder.Decode(&envelope); err != nil {
+		if cause := ctx.Err(); cause != nil {
+			return Quote{}, cause
+		}
+		return Quote{}, Error{Symbol: symbol, Reason: "the price provider sent a response that could not be read"}
+	}
+	if len(envelope.Chart.Error) > 0 && string(envelope.Chart.Error) != "null" {
+		return Quote{}, Error{Symbol: symbol, Reason: "the price provider reported an error for this symbol"}
+	}
+	if len(envelope.Chart.Result) == 0 {
+		return Quote{}, Error{Symbol: symbol, Reason: "the price provider knows no data for this symbol"}
+	}
+	result := envelope.Chart.Result[0]
+	if !currencyPattern.MatchString(result.Meta.Currency) {
+		return Quote{}, Error{Symbol: symbol, Reason: "the price provider did not report a currency"}
+	}
+	if len(result.Indicators.Quote) == 0 {
+		return Quote{}, Error{Symbol: symbol, Reason: "the price provider returned no closing prices"}
+	}
+	closes := result.Indicators.Quote[0].Close
+	// Walk backwards for the newest close that actually traded, and keep the
+	// timestamp of that same candle: the day shown must be the day priced.
+	for i := len(closes) - 1; i >= 0; i-- {
+		if closes[i] == "" {
+			continue
+		}
+		if i >= len(result.Timestamp) || result.Timestamp[i] <= 0 {
+			return Quote{}, Error{Symbol: symbol, Reason: "the price provider returned a closing price without a date"}
+		}
+		price, err := decimalQuantity(string(closes[i]))
+		if err != nil {
+			return Quote{}, Error{Symbol: symbol, Reason: "the price provider returned an unusable closing price"}
+		}
+		if units, err := price.Units(); err != nil || units <= 0 {
+			return Quote{}, Error{Symbol: symbol, Reason: "the price provider returned a closing price that is not positive"}
+		}
+		return Quote{
+			Symbol:   symbol,
+			Price:    price,
+			Currency: result.Meta.Currency,
+			Day:      time.Unix(result.Timestamp[i], 0).UTC().Format("2006-01-02"),
+		}, nil
+	}
+	return Quote{}, Error{Symbol: symbol, Reason: "the price provider returned no closing price for the last five days"}
+}
+
+// quantityScale is the journal's eight fractional places, and maxUnitDigits
+// bounds the scaled result: a price needing more than eight digits before the
+// point is not a security price, and the bound keeps the value inside the
+// signed 64-bit units the journal stores.
+const quantityScale = 8
+const maxUnitDigits = 8 + quantityScale
+
+// significantDigits is where a provider price stops being price and starts
+// being float noise. Yahoo's closes are 32-bit floats widened to 64: a real
+// response carries 165.26 as "165.25999450683594" and 9.408 as
+// "9.4079999923706". A 32-bit float holds 24 bits of mantissa, which is 7.22
+// decimal digits, so the eighth digit onwards is an artefact of the encoding
+// and never a figure that traded - rounding at eight would keep the visible
+// nonsense "165.25999". Seven recovers the decimal the exchange published for
+// every price quoted to cents, which is every equity and fund price, and is
+// still four orders of magnitude finer than a price needs to value a holding.
+const significantDigits = 7
+
+// decimalQuantity converts a provider's decimal literal to the journal's
+// eight-place scale, working on the digit text so the value never passes
+// through binary floating point. It rounds to significantDigits and then to
+// eight fractional places, half rounding away from zero both times. Exponent
+// notation is rejected rather than guessed at: the endpoint does not use it,
+// and a price misread by a factor of ten is worse than a failed refresh.
+func decimalQuantity(text string) (domain.Quantity, error) {
+	invalid := fmt.Errorf("invalid decimal price")
+	negative := strings.HasPrefix(text, "-")
+	literal := strings.TrimPrefix(text, "-")
+	whole, fraction, point := strings.Cut(literal, ".")
+	// A trailing or repeated point, or digits absent on either side, is not a
+	// number this endpoint produces; so is exponent notation, caught by the
+	// digit scan below.
+	if whole == "" || (point && fraction == "") || strings.Contains(fraction, ".") {
+		return "", invalid
+	}
+	digits := whole + fraction
+	for i := range len(digits) {
+		if digits[i] < '0' || digits[i] > '9' {
+			return "", invalid
+		}
+	}
+	// value holds the significant digits and exponent counts how many of them
+	// stand before the decimal point, so the point can move under rounding
+	// without the digits being re-parsed.
+	value := []byte(strings.TrimLeft(digits, "0"))
+	exponent := len(whole) - (len(digits) - len(value))
+	if len(value) == 0 {
+		return domain.FormatQuantity(0), nil
+	}
+	if len(value) > significantDigits {
+		roundUp := value[significantDigits] >= '5'
+		value = value[:significantDigits]
+		if roundUp {
+			// A carry off the front ("99999999" to "100000000") moves the point.
+			if value = increment(value); len(value) > significantDigits {
+				exponent++
+			}
+		}
+	}
+	// Scale to hundred-millionths: appending zeros multiplies, and dropping
+	// digits divides with the same half-away-from-zero rounding.
+	if shift := exponent - len(value) + quantityScale; shift >= 0 {
+		value = append(value, strings.Repeat("0", shift)...)
+	} else if keep := len(value) + shift; keep < 0 {
+		value = []byte("0")
+	} else {
+		roundUp := value[keep] >= '5'
+		value = value[:keep]
+		if len(value) == 0 {
+			value = []byte("0")
+		}
+		if roundUp {
+			value = increment(value)
+		}
+	}
+	if len(value) > maxUnitDigits {
+		return "", invalid
+	}
+	units, err := strconv.ParseInt(string(value), 10, 64)
+	if err != nil {
+		return "", invalid
+	}
+	if negative {
+		units = -units
+	}
+	return domain.FormatQuantity(units), nil
+}
+
+// increment adds one to a decimal digit string, growing it when the carry runs
+// off the front ("999" becomes "1000"). Rounding up the last kept place of
+// 0.99999999|9 has to carry into the whole part, not wrap it.
+func increment(digits []byte) []byte {
+	for i := len(digits) - 1; i >= 0; i-- {
+		if digits[i] != '9' {
+			digits[i]++
+			return digits
+		}
+		digits[i] = '0'
+	}
+	return append([]byte{'1'}, digits...)
+}
@@ -0,0 +1,196 @@
+package quotes
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+// secret stands in for anything a provider might put in a response body: no
+// part of it may reach a message shown to the user.
+const secret = "SUPER-SECRET-PROVIDER-TEXT"
+
+func stub(t *testing.T, handler http.HandlerFunc) Client {
+	t.Helper()
+	server := httptest.NewServer(handler)
+	t.Cleanup(server.Close)
+	return Client{BaseURL: server.URL, HTTPClient: server.Client()}
+}
+
+func body(payload string) http.HandlerFunc {
+	return func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(payload))
+	}
+}
+
+const chartVWCE = `{"chart":{"result":[{"meta":{"currency":"EUR","symbol":"VWCE.DE","exchangeName":"GER"},
+  "timestamp":[1757376000,1757462400],
+  "indicators":{"quote":[{"close":[127.11,128.42],"volume":[1,2]}]}}],"error":null}}`
+
+func TestLatestReadsLastClose(t *testing.T) {
+	var path, query string
+	client := stub(t, func(w http.ResponseWriter, r *http.Request) {
+		path, query = r.URL.Path, r.URL.RawQuery
+		body(chartVWCE)(w, r)
+	})
+	quote, err := client.Latest(context.Background(), "VWCE.DE")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if quote.Symbol != "VWCE.DE" || quote.Price != "128.42" || quote.Currency != "EUR" || quote.Day != "2025-09-10" {
+		t.Fatalf("quote: %+v", quote)
+	}
+	if path != "/v8/finance/chart/VWCE.DE" || query != "range=5d&interval=1d" {
+		t.Fatalf("request: %q %q", path, query)
+	}
+}
+
+func TestLatestSkipsTrailingNullCloses(t *testing.T) {
+	client := stub(t, body(`{"chart":{"result":[{"meta":{"currency":"EUR"},
+	  "timestamp":[1757376000,1757462400,1757548800],
+	  "indicators":{"quote":[{"close":[127.11,128.42,null]}]}}],"error":null}}`))
+	quote, err := client.Latest(context.Background(), "VWCE.DE")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// The day must come from the candle that priced, not from the newest one.
+	if quote.Price != "128.42" || quote.Day != "2025-09-10" {
+		t.Fatalf("quote: %+v", quote)
+	}
+}
+
+func TestLatestReportsForeignCurrency(t *testing.T) {
+	client := stub(t, body(`{"chart":{"result":[{"meta":{"currency":"USD"},
+	  "timestamp":[1757376000],"indicators":{"quote":[{"close":[9.4079999923706]}]}}],"error":null}}`))
+	quote, err := client.Latest(context.Background(), "VUSA")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// A foreign currency is the caller's decision to reject, not a fetch failure.
+	if quote.Currency != "USD" || quote.Price != "9.408" {
+		t.Fatalf("quote: %+v", quote)
+	}
+}
+
+func TestLatestRejectsUnusableResponses(t *testing.T) {
+	cases := []struct {
+		name    string
+		handler http.HandlerFunc
+	}{
+		{"every close null", body(`{"chart":{"result":[{"meta":{"currency":"EUR"},
+		  "timestamp":[1757376000,1757462400],"indicators":{"quote":[{"close":[null,null]}]}}],"error":null}}`)},
+		{"server failure", func(w http.ResponseWriter, _ *http.Request) {
+			w.WriteHeader(http.StatusInternalServerError)
+			_, _ = w.Write([]byte(`{"chart":{"result":null,"error":{"description":"` + secret + `"}}}`))
+		}},
+		{"chart error", body(`{"chart":{"result":null,"error":{"code":"Not Found","description":"` + secret + `"}}}`)},
+		{"empty result", body(`{"chart":{"result":[],"error":null}}`)},
+		{"no currency", body(`{"chart":{"result":[{"meta":{"currency":"eur"},
+		  "timestamp":[1757376000],"indicators":{"quote":[{"close":[128.42]}]}}],"error":null}}`)},
+		{"close not positive", body(`{"chart":{"result":[{"meta":{"currency":"EUR"},
+		  "timestamp":[1757376000],"indicators":{"quote":[{"close":[0]}]}}],"error":null}}`)},
+		{"close without timestamp", body(`{"chart":{"result":[{"meta":{"currency":"EUR"},
+		  "timestamp":[],"indicators":{"quote":[{"close":[128.42]}]}}],"error":null}}`)},
+		{"not json", func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("<html>" + secret + "</html>")) }},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			quote, err := stub(t, c.handler).Latest(context.Background(), "VWCE.DE")
+			if err == nil {
+				t.Fatalf("expected failure, got %+v", quote)
+			}
+			var provider Error
+			if !errors.As(err, &provider) || provider.Symbol != "VWCE.DE" || provider.Reason == "" {
+				t.Fatalf("want typed provider error, got %#v", err)
+			}
+			if strings.Contains(err.Error(), secret) {
+				t.Fatalf("response text leaked into %q", err)
+			}
+			if !strings.Contains(err.Error(), "VWCE.DE") {
+				t.Fatalf("error must name the symbol: %q", err)
+			}
+		})
+	}
+}
+
+func TestLatestRejectsUnusableSymbolAndAddress(t *testing.T) {
+	client := stub(t, func(http.ResponseWriter, *http.Request) {
+		t.Fatal("no request may be made for a rejected symbol or address")
+	})
+	if _, err := client.Latest(context.Background(), "../secrets"); err == nil {
+		t.Fatal("expected a path-shaping symbol to be rejected")
+	}
+	plain := Client{BaseURL: "http://prices.example.com"}
+	if _, err := plain.Latest(context.Background(), "VWCE.DE"); err == nil {
+		t.Fatal("expected non-loopback plain HTTP to be rejected")
+	}
+}
+
+func TestLatestKeepsCancellationIdentity(t *testing.T) {
+	client := stub(t, body(chartVWCE))
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if _, err := client.Latest(ctx, "VWCE.DE"); !errors.Is(err, context.Canceled) {
+		t.Fatalf("want context.Canceled, got %#v", err)
+	}
+}
+
+func TestDecimalQuantityRoundsHalfAwayFromZero(t *testing.T) {
+	cases := []struct {
+		text string
+		want string
+	}{
+		// Real closes, copied from a live response: every one is a 32-bit float
+		// widened to 64, and the decimal the exchange published has to come
+		// back out of it.
+		{"165.25999450683594", "165.26"},
+		{"125.44999694824219", "125.45"},
+		{"127.1449966430664", "127.145"},
+		{"167.77999877929688", "167.78"},
+		{"0.41578700000001", "0.415787"},
+		{"9.4079999923706", "9.408"},
+		{"-9.4079999923706", "-9.408"},
+		{"128.42", "128.42"},
+		{"0.000000005", "0.00000001"},
+		{"0.000000004", "0"},
+		{"0.999999995", "1"},
+		{"42", "42"},
+		{"0007.5", "7.5"},
+		// Past the seventh digit the provider is describing its own encoding,
+		// so the eighth place moves rather than being preserved.
+		{"12345.678912345", "12345.68"},
+		{"12345678.94999999", "12345680"},
+	}
+	for _, c := range cases {
+		got, err := decimalQuantity(c.text)
+		if err != nil || string(got) != c.want {
+			t.Fatalf("decimalQuantity(%q) = %q, %v; want %q", c.text, got, err, c.want)
+		}
+	}
+	for _, text := range []string{"", "-", ".5", "5.", "1.2.3", "1e5", "12e-3", "abc", "1 2", "999999999", "99999999.999999995"} {
+		if got, err := decimalQuantity(text); err == nil {
+			t.Fatalf("decimalQuantity(%q) = %q, want an error", text, got)
+		}
+	}
+}
+
+// The provider answers 429 to every request whose agent names a programming
+// language, so a missing or Go-default User-Agent breaks every quote on the
+// first call rather than under load. The header is load-bearing, not decor.
+func TestLatestIdentifiesAsABrowser(t *testing.T) {
+	agent := "unset"
+	client := stub(t, func(w http.ResponseWriter, r *http.Request) {
+		agent = r.Header.Get("User-Agent")
+		body(chartVWCE)(w, r)
+	})
+	if _, err := client.Latest(context.Background(), "VWCE.DE"); err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasPrefix(agent, "Mozilla/") || strings.Contains(agent, "Go-http-client") {
+		t.Fatalf("User-Agent %q is refused by the provider", agent)
+	}
+}