From d5aa9bcc8e8c6728ff1160fc3dc357aba4220fff Mon Sep 17 00:00:00 2001 From: Lars Nolden Date: Tue, 1 Sep 2026 17:52:18 +0200 Subject: [PATCH] beacon mode + meter-anchored verification; discovery: intermittent post-chip TX RF death; withdraw single-burst confirmations --- docs/FIELD_TEST_2026-09-01.md | 33 ++++++++ docs/FINDINGS.md | 24 ++++++ firmware/main/console.c | 6 ++ firmware/main/repeater.c | 44 ++++++++++ firmware/main/repeater.h | 1 + hil/meter_anchored.py | 151 ++++++++++++++++++++++++++++++++++ 6 files changed, 259 insertions(+) create mode 100755 hil/meter_anchored.py diff --git a/docs/FIELD_TEST_2026-09-01.md b/docs/FIELD_TEST_2026-09-01.md index c618510..a7cf299 100644 --- a/docs/FIELD_TEST_2026-09-01.md +++ b/docs/FIELD_TEST_2026-09-01.md @@ -85,3 +85,36 @@ because the sequence of wrong theories is itself a finding. identical geometry, cable in vs out (LiPo), antenna close. - Proper antenna A/B (e.g. quarter-wave with radials vs stock whip) -- directly relevant to the gateway RF architecture. + +## Location 3 (evening session) -- the beacon, and the real fault + +Coordinating physical actions with an async agent kept failing (missed +timestamps, tuner deaths mid-capture), so we changed instruments: a +firmware **beacon mode** (`set beacon_period_ms N`) transmits a synthetic +OMS frame (manufacturer SLN, ID 00000001, access_nr = sequence) every N ms. +No meter traffic, no serial coordination, dense uniform sampling. + +The beacon caught what two days of repeat-chasing could not: + +- **TX radiated power is intermittent at minute scale.** One 90 s window: + 0/25 beacons produced ANY RF (max +0.5 dB over floor) while 27 ambient + meter frames in the same IQ showed median +26.8 dB (observer healthy) and + the chip reported TX_DONE, mode 6, device_errors 0 for every beacon. + Thirty minutes later, same boot, same config: repeats, test_tx and + beacons all radiated at a constant ~+10 dB. +- RX stayed healthy through dead TX windows. Chip-side diagnostics are + identical in dead and live states. The fault therefore sits between the + SX1262 PA output and the antenna: KCT8103L PA path (CPS/DIO2, CTX, PA + supply) or a mechanical/thermal TX-chain fault. Firmware cannot observe + it -- exactly the RADIO_VDD_SENSE / conducted-test-port gap in + docs/devboard_signals.csv. +- **Methodology bug found and fixed:** verify_repeats_iq's single-burst + clock fit can lock at -repeat_delay and "confirm" every repeat with its + own original's burst. The "7/7 at +37 dB" laptop confirmations were this + artifact. Trustworthy methods: structural same-duration pairs at the + configured delay, content decodes (duplicate payloads), and + hil/meter_anchored.py (clock fitted on strong meter receptions; ambient + meter levels double as per-capture observer health). +- Positive control: with a healthy observer, 30 structural pairs at + 137 ms and 35 duplicate decoded payloads in 90 s -- the repeater + function itself is solid whenever the TX chain is in its live state. diff --git a/docs/FINDINGS.md b/docs/FINDINGS.md index 7b78abb..4232d65 100644 --- a/docs/FINDINGS.md +++ b/docs/FINDINGS.md @@ -147,6 +147,30 @@ - Long-run stability at >=100 and >=500 verified repeat events. - Early-stop RX (poll GetRxBufferStatus) to cut the ~20 ms deaf tail. +## Location 3 follow-up (2026-09-01 evening): intermittent TX RF death + +- CONFIRMED: the DUT's radiated TX output is intermittent on minute scales + -- genuinely zero (not weak) for >=10 min windows, then a stable ~+10 dB + at the SDR. Verified with beacon mode against in-capture ambient-meter + controls (observer healthy: meters +26.8 dB median while 25/25 beacons + produced <= +0.5 dB). RX unaffected; chip reports TX_DONE/mode 6/no + errors in both states => fault is post-chip (KCT8103L PA path or TX + chain), invisible to firmware. Hardware follow-up required; strengthens + the RADIO_VDD_SENSE and conducted-RF-test-port devboard requirements. +- WITHDRAWN: every single-burst "TX aligned to IQ" confirmation (e.g. + "7/7 at +37 dB", "33/41 at +19.3 dB"). verify_repeats_iq's clock fit on + TX events can lock at -repeat_delay and match each repeat to its own + original's burst. Borrow-immune evidence only: structural same-duration + pairs at the configured delay, duplicate content decodes, or + hil/meter_anchored.py (meter-anchored clock, exact TX windows). +- CONFIRMED: observer health must be judged per capture from ambient meter + levels inside the same IQ (meter median >= +15 dB over floor). The + Nooelec went deaf mid-session again; runs without this control are void. +- CORRECTION to yesterday's "placement dominates": placement governs + whether PAIRS are detectable (originals must be SDR-visible), but the + standalone invisibility episodes are better explained by the intermittent + TX death now that it has been directly observed under controls. + ## Field test 2026-09-01 (onsite standalone deployment) Full narrative with the evidence table: docs/FIELD_TEST_2026-09-01.md. diff --git a/firmware/main/console.c b/firmware/main/console.c index 8c59a67..bfae1c0 100644 --- a/firmware/main/console.c +++ b/firmware/main/console.c @@ -3,6 +3,7 @@ // status | stats | clear_stats | radio_status | radio_reset | test_tx // set repeat_delay_ms N | set tx_power N | set repeat_enabled 0|1 // set verbose 0|1 | set whitelist ID[,ID...] | set whitelist clear +// set beacon_period_ms N (0=off, uses test_lfield) | set duty_budget_ppm N // reboot #include #include @@ -49,6 +50,11 @@ static void handle_set(char *args) if (v < 0 || v > 100000) { ack("set", false, "range 0..100000 ppm"); return; } g_cfg.duty_budget_ppm = (uint32_t)v; ack("set duty_budget_ppm", true, v == 0 ? "guard disabled" : NULL); + } else if (strcmp(key, "beacon_period_ms") == 0) { + long v = strtol(val, NULL, 10); + if (v != 0 && (v < 250 || v > 600000)) { ack("set", false, "0 or 250..600000 ms"); return; } + g_cfg.beacon_period_ms = (uint32_t)v; + ack("set beacon_period_ms", true, v == 0 ? "beacon off" : NULL); } else if (strcmp(key, "preamble_bits") == 0) { long v = strtol(val, NULL, 10); uint8_t code; diff --git a/firmware/main/repeater.c b/firmware/main/repeater.c index 94e69ad..c9d06ec 100644 --- a/firmware/main/repeater.c +++ b/firmware/main/repeater.c @@ -27,6 +27,7 @@ repeater_cfg_t g_cfg = { // environment, so the experiment enforces the budget itself rather than // relying on traffic being sparse. See docs/STANDARDS_GAPS.md. .duty_budget_ppm = 1000, // 1000 ppm = 0.1 % + .beacon_period_ms = 0, .tx_power_dbm = -9, .repeat_enabled = true, #ifdef CONFIG_WMBUS_VERBOSE_DEFAULT @@ -49,6 +50,7 @@ static volatile int s_inject; // 1 = standby, 2 = sleep (config loss) static uint8_t s_test_access_nr; static uint8_t s_test_lfield = 19; static uint32_t s_seq; +static int64_t s_last_beacon; static struct { uint32_t hash; int64_t ts; } s_dup[DUP_RING]; static int s_dup_idx; @@ -347,6 +349,39 @@ static void run_test_tx(void) sx126x_restore_rx_payload_len(); if (!sx126x_set_rx()) recover_ladder("test_tx_rx_restore"); } +// Periodic synthetic OMS beacon: standalone RF measurements without meter +// traffic or serial coordination. access_nr doubles as sequence number and +// is recoverable from every SDR-side decode (SLN manufacturer, ID 00000001). +// Respects the duty guard; suppressed beacons skip to the next period. +static void run_beacon(void) +{ + uint8_t frame[256]; + uint8_t acc = s_test_access_nr++; + int n = wmbus_build_test_frame(frame, sizeof(frame), acc, s_test_lfield); + if (n < 0) return; + uint64_t duty_used = 0, duty_budget = 0; + if (duty_would_exceed(esp_timer_get_time(), n + 2, &duty_used, &duty_budget)) { + g_stats.tx_suppressed_duty++; + LOGJ("\"event\":\"beacon_suppressed\",\"reason\":\"duty_cycle\"," + "\"used_us\":%llu,\"budget_us\":%llu", + (unsigned long long)duty_used, (unsigned long long)duty_budget); + return; + } + static uint8_t payload[258]; + payload[0] = WMBUS_MODE_BYTE_A_HI; + payload[1] = WMBUS_MODE_BYTE_A_LO; + memcpy(payload + 2, frame, n); + + int64_t t0, t1; + const char *res = do_tx(payload, n + 2, &t0, &t1); + if (t1 > t0) duty_record(t0, (uint32_t)(t1 - t0)); + LOGJ("\"event\":\"beacon\",\"acc\":%u,\"lfield\":%u,\"result\":\"%s\"," + "\"timestamp_us\":%lld,\"tx_done_us\":%lld", + acc, s_test_lfield, res, (long long)t0, (long long)t1); + + sx126x_restore_rx_payload_len(); + if (!sx126x_set_rx()) recover_ladder("beacon_rx_restore"); +} // Transmit the SAME frame twice, separated by repeat_delay_ms. This // reproduces the original+repeat timing pattern without needing a meter, @@ -614,6 +649,13 @@ static void repeater_task(void *arg) LOGJ("\"event\":\"irq_other\",\"mask\":%u", irq); } } + if (g_cfg.beacon_period_ms) { + int64_t bnow = esp_timer_get_time(); + if (bnow - s_last_beacon >= (int64_t)g_cfg.beacon_period_ms * 1000) { + s_last_beacon = bnow; + run_beacon(); + } + } int64_t now = esp_timer_get_time(); if (now - last_heartbeat > 10000000LL) { @@ -658,6 +700,7 @@ void repeater_print_status(void) "\"repeat_delay_ms\":%" PRIu32 ",\"configured_tx_dbm\":%d," "\"repeat_enabled\":%s,\"verbose\":%s,\"whitelist_len\":%d," "\"preamble_detect_code\":%u,\"duty_budget_ppm\":%" PRIu32 "," + "\"beacon_period_ms\":%" PRIu32 "," "\"duty_used_us\":%llu,\"reset_reason\":%d", (long long)esp_timer_get_time(), g_cfg.repeat_delay_ms, (int)g_cfg.tx_power_dbm, @@ -666,6 +709,7 @@ void repeater_print_status(void) g_cfg.whitelist_len, sx126x_get_preamble_detect(), g_cfg.duty_budget_ppm, + g_cfg.beacon_period_ms, (unsigned long long)duty_used_us(esp_timer_get_time()), (int)esp_reset_reason()); } diff --git a/firmware/main/repeater.h b/firmware/main/repeater.h index 0017dbf..9c52c75 100644 --- a/firmware/main/repeater.h +++ b/firmware/main/repeater.h @@ -26,6 +26,7 @@ typedef struct { typedef struct { uint32_t repeat_delay_ms; // 20..200 uint32_t duty_budget_ppm; // rolling-hour TX airtime cap, 0 = disabled + uint32_t beacon_period_ms; // periodic synthetic test frame, 0 = off int8_t tx_power_dbm; // configured SX1262 level, not radiated power bool repeat_enabled; bool verbose; diff --git a/hil/meter_anchored.py b/hil/meter_anchored.py new file mode 100755 index 0000000..d2bdc1c --- /dev/null +++ b/hil/meter_anchored.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Meter-anchored TX verification. + +The only trustworthy way we have found to verify DUT transmissions in IQ: + +1. Fit the serial<->IQ clock offset using STRONG METER RECEPTIONS as anchors + (independent events, visible to both DUT and SDR, abundant). +2. Judge each firmware TX event by the envelope inside its EXACT window. +3. Report ambient meter levels alongside -- they are the in-band health + control for the observer (deaf dongle => meters compressed => run void). + +Why not fit the clock on TX events (hil/verify_repeats_iq.py, single-burst +mode): repeats fire RX + delay by construction, so an offset shifted by +-delay "matches" every repeat to its own original's burst. That artifact +produced fake "7/7 at +37 dB" confirmations. See docs/FINDINGS.md. + +Usage: meter_anchored.py --iq FILE --serial FILE [--json] +""" +import argparse, json, sys +import numpy as np + +FS = 1600000 +WIN = FS // 1000 # 1 ms envelope bins + + +def envelope(path): + raw = np.fromfile(path, dtype=np.uint8).astype(np.float32) - 127.5 + iq = raw[0::2] + 1j * raw[1::2] + p = np.abs(iq) ** 2 + env = p[:(len(p) // WIN) * WIN].reshape(-1, WIN).mean(axis=1) + db = 10 * np.log10(env + 1e-9) + return db, float(np.median(db)) + + +def bursts_of(db, floor, thresh=5): + idx = np.where(db > floor + thresh)[0] + if not len(idx): + return [] + out, s, p0 = [], idx[0], idx[0] + for i in idx[1:]: + if i > p0 + 3: + out.append((s, p0)) + s = i + p0 = i + out.append((s, p0)) + return [(a, b - a + 1, float(db[a:b + 1].max() - floor)) + for a, b in out if (b - a + 1) >= 3] + + +def parse_serial(path): + rx, tx = [], [] + for line in open(path, errors="replace"): + for tag, dst in (("wmbus_rx", rx), ("wmbus_tx", tx), ("beacon", tx)): + if f'"event":"{tag}"' in line: + try: + dst.append(json.loads(line[line.index("{"):])) + except (ValueError, KeyError): + pass + break + return rx, tx + + +def fit_clock(bursts, anchors_ms, span_ms): + bst = np.array([b[0] for b in bursts], dtype=float) + best = (None, -1, 1e9) + for off in np.arange(anchors_ms[0] - span_ms - 3000, anchors_ms[0] + 3000, 2): + d = [np.min(np.abs(bst - (t - off))) for t in anchors_ms] + m = sum(1 for x in d if x < 15) + e = float(np.median(d)) + if m > best[1] or (m == best[1] and e < best[2]): + best = (off, m, e) + return best # offset, matched, median_err + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--iq", required=True) + ap.add_argument("--serial", required=True) + ap.add_argument("--rssi-min", type=float, default=-65, + help="anchor threshold: DUT-side RSSI of meter frames") + ap.add_argument("--json", action="store_true") + args = ap.parse_args() + + db, floor = envelope(args.iq) + bursts = bursts_of(db, floor) + rx, tx = parse_serial(args.serial) + strong = [r for r in rx + if r.get("rssi_dbm", -99) > args.rssi_min and r.get("timestamp_us")] + if len(strong) < 5: + print(f"only {len(strong)} strong meter anchors -- cannot fit clock", + file=sys.stderr) + sys.exit(2) + + anchors = np.array([r["timestamp_us"] / 1000.0 for r in strong]) + span = len(db) + off, matched, err = fit_clock(bursts, anchors, span) + + bst = np.array([b[0] for b in bursts], dtype=float) + meter_lv = [] + for t in anchors: + k = int(np.argmin(np.abs(bst - (t - off)))) + if abs(bst[k] - (t - off)) < 15: + meter_lv.append(bursts[k][2]) + + def winmax(ts_ms): + a0, a1 = int(ts_ms) - 2, int(ts_ms) + 12 + if not (0 <= a0 < a1 < len(db)): + return None + return float((db[a0:a1] - floor).max()) + + events = [] + for t in tx: + w = winmax(t["timestamp_us"] / 1000.0 - off) + if w is None: + continue + events.append({"event": t.get("event", "wmbus_tx"), + "result": t.get("result"), + "window_db": round(w, 1), + "rf": bool(w >= 6)}) + + meter_med = float(np.median(meter_lv)) if meter_lv else 0.0 + observer = "ok" if meter_med >= 15 else ("degraded" if meter_med >= 8 else "deaf") + n_rf = sum(1 for e in events if e["rf"]) + report = { + "clock_matched": f"{matched}/{len(strong)}", + "clock_err_ms": round(err, 1), + "floor_db": round(floor, 1), + "meter_median_db": round(meter_med, 1), + "observer": observer, + "tx_events": len(events), + "tx_with_rf": n_rf, + "tx_levels_db": [e["window_db"] for e in events], + } + if args.json: + print(json.dumps(report, indent=2)) + else: + print(f"clock: {report['clock_matched']} anchors, err {report['clock_err_ms']} ms") + print(f"observer: {observer} (meter median +{report['meter_median_db']} dB, " + f"floor {report['floor_db']})") + print(f"TX with RF: {n_rf}/{len(events)}") + for e in events: + print(f" {e['event']:>9} {e['result'] or '':>10} " + f"+{e['window_db']:>5.1f} dB {'RF' if e['rf'] else '--'}") + # exit 3 = observer unusable, 1 = TX dead while observer ok + if observer == "deaf": + sys.exit(3) + sys.exit(0 if (n_rf or not events) else 1) + + +if __name__ == "__main__": + main()