From 74cb28daf7c8dee3f22c83e87f599e3cfbedb2e1 Mon Sep 17 00:00:00 2001 From: Lars Nolden Date: Tue, 1 Sep 2026 09:22:20 +0200 Subject: [PATCH] hil: single-stream decode+IQ capture, clock-aligned RF TX proof --- hil/capture_both.sh | 40 +++++++ hil/report.py | 16 ++- hil/run.sh | 47 ++++---- hil/verify_repeats_iq.py | 232 +++++++++++++++++++++++++-------------- 4 files changed, 221 insertions(+), 114 deletions(-) create mode 100755 hil/capture_both.sh diff --git a/hil/capture_both.sh b/hil/capture_both.sh new file mode 100755 index 0000000..20062da --- /dev/null +++ b/hil/capture_both.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Capture rtl_wmbus decodes AND raw IQ from a SINGLE rtl_sdr stream. +# +# Why one stream: this dongle's sensitivity drifts by ~17 dB over minutes, +# so measurements taken in separate captures at "the same gain" are not +# comparable. Decoding meter originals and measuring our own transmissions +# in the IQ must therefore share one stream, one gain, one instant. +# +# Usage: capture_both.sh DURATION_S DECODE_LOG IQ_FILE [GAIN] +set -euo pipefail +cd "$(dirname "$0")/.." +DUR=$1; OUT=$2; IQ=$3; GAIN=${4:-42.1} + +: > "$OUT" +ERR="${OUT%.log}.err" +STATS="${OUT%.log}.stats" +: > "$ERR" + +health_ok=0 +for i in 1 2 3; do + if ./result-py/bin/python3 hil/sdr_health.py >> "$ERR" 2>&1; then + health_ok=1; break + fi + echo "--- SDR unhealthy; usbreset (try $i)" >> "$ERR" + usbreset 0bda:2838 >> "$ERR" 2>&1 || true + sleep $((i * 5)) +done +echo "sdr_health_ok=$health_ok" > "${OUT%.log}.health" + +t0=$(date +%s) +# tee duplicates the sample stream: one copy to disk, one to the decoder +timeout "$DUR" rtl_sdr -f 868950000 -s 1600000 -g "$GAIN" - 2>>"$ERR" \ + | tee "$IQ" \ + | rtl_wmbus -f >> "$OUT" 2>/dev/null || true +t1=$(date +%s) + +printf 'streamed_seconds=%s\nrequested_seconds=%s\ngain=%s\niq_bytes=%s\n' \ + "$((t1 - t0))" "$DUR" "$GAIN" "$(stat -c%s "$IQ" 2>/dev/null || echo 0)" \ + > "$STATS" +echo "--- streamed $((t1 - t0))s of ${DUR}s" >> "$ERR" diff --git a/hil/report.py b/hil/report.py index 12a0886..3bcf4bb 100755 --- a/hil/report.py +++ b/hil/report.py @@ -67,7 +67,7 @@ if proof is not None and proof.get("bursts_with_sdr_decode", 0) == 0: # a repeat at the configured delay also satisfies "the observer # independently saw the retransmission" (see docs/RF_FRONTEND_TESTS.md for # why decoding our own TX is bench-limited). -if iqp is not None and iqp.get("repeat_pairs_at_delay", 0) > 0: +if iqp is not None and iqp.get("tx_events_with_rf", 0) > 0: fail = [f for f in fail if not f.startswith("SDR-verified repeats")] verdict = "PASS" if not fail else "FAIL" @@ -100,14 +100,18 @@ TX DECODABILITY PROOF (synthetic C1, independent of meters): both airings decoded: {proof.get('bursts_with_both_airings_decoded')}""") if iqp is not None: - g = iqp.get("gap_ms") or {} print(f""" -RF-LEVEL REPEAT PROOF (raw IQ, decode-independent): +RF-LEVEL TX PROOF (same stream's raw IQ, decode-independent): IQ analysed: {iqp.get('iq_seconds')} s - bursts total/strong: {iqp.get('bursts_total')} / {iqp.get('bursts_strong_dut')} + clipped samples: {iqp.get('clipped_sample_pct')} % + bursts in window: {iqp.get('bursts_total')} firmware TX ok: {iqp.get('firmware_tx_ok')} - repeat pairs at delay: {iqp.get('repeat_pairs_at_delay')} - gap min/med/max: {g.get('min')} / {g.get('median')} / {g.get('max')} ms""") + TX events with RF: {iqp.get('tx_events_with_rf')} ({iqp.get('tx_rf_confirmed_pct')} %) + alignment error: {iqp.get('align_error_ms')} ms + airtime error: {iqp.get('burst_duration_error_ms')} ms + our TX over floor: {iqp.get('our_tx_peak_db_over_floor')} dB + meter bursts over floor: {iqp.get('other_burst_peak_db_over_floor')} dB + firmware repeat delay: {iqp.get('firmware_repeat_delay_ms')} ms""") print(f""" SDR (independent observer): diff --git a/hil/run.sh b/hil/run.sh index 685964d..8acdc99 100755 --- a/hil/run.sh +++ b/hil/run.sh @@ -101,12 +101,21 @@ if [ "$PROOF" = 1 ]; then cat "$RUN_DIR/tx_proof.txt" fi -# --- phase B: real meter traffic ------------------------------------------ -echo "phase B: real-meter capture (${DURATION}s)" +# --- phase B: real meter traffic, decodes + IQ from ONE stream ------------- +# One rtl_sdr stream feeds both rtl_wmbus (meter originals) and a raw IQ +# file (our transmissions, measured decode-independently). Separate +# captures are not comparable: this dongle's sensitivity drifts ~17 dB +# over minutes (docs/FINDINGS.md). +echo "phase B: real-meter capture (${DURATION}s, decodes + IQ)" +IQ="$RUN_DIR/iq.raw" $PY hil/dut_cmd.py "$DUT_PORT" --listen "$DURATION" status \ > "$RUN_DIR/serial.log" 2>&1 & SERIAL_PID=$! -./hil/capture_sdr.sh "$DURATION" "$RUN_DIR/rtlwmbus.log" "$GAIN" & +if [ "$IQ_SECONDS" -gt 0 ]; then + ./hil/capture_both.sh "$DURATION" "$RUN_DIR/rtlwmbus.log" "$IQ" "$GAIN" & +else + ./hil/capture_sdr.sh "$DURATION" "$RUN_DIR/rtlwmbus.log" "$GAIN" & +fi SDR_PID=$! wait "$SDR_PID" || true wait "$SERIAL_PID" || true @@ -114,28 +123,16 @@ wait "$SERIAL_PID" || true $PY hil/dut_cmd.py "$DUT_PORT" --listen 4 stats > "$RUN_DIR/final_stats.log" 2>&1 || true cat "$RUN_DIR/final_stats.log" >> "$RUN_DIR/serial.log" -# --- phase C: RF-level repeat proof from raw IQ --------------------------- -# Decode-independent: measures whether a second transmission occurs at the -# configured delay with the expected airtime. Needed because our repeat -# arrives ~36 dB hotter than meter traffic and often clips the dongle. -if [ "$IQ_SECONDS" -gt 0 ]; then - echo "phase C: IQ repeat proof (${IQ_SECONDS}s)" - IQ="$RUN_DIR/iq.raw" - timeout $((IQ_SECONDS + 15)) rtl_sdr -f 868950000 -s 1600000 -g "$GAIN" \ - -n $((IQ_SECONDS * 1600000)) "$IQ" 2> "$RUN_DIR/iq_capture.err" & - IQ_PID=$! - $PY hil/dut_cmd.py "$DUT_PORT" --listen "$IQ_SECONDS" status \ - > "$RUN_DIR/iq_serial.log" 2>&1 - wait "$IQ_PID" || true - if [ -s "$IQ" ]; then - $PY hil/verify_repeats_iq.py --iq "$IQ" --serial "$RUN_DIR/iq_serial.log" \ - --delay-ms "$DELAY" > "$RUN_DIR/iq_proof.txt" 2>&1 || true - $PY hil/verify_repeats_iq.py --iq "$IQ" --serial "$RUN_DIR/iq_serial.log" \ - --delay-ms "$DELAY" --json > "$RUN_DIR/iq_proof.json" 2>&1 || true - cat "$RUN_DIR/iq_proof.txt" - fi - # IQ is large; keep only if it produced no pairs (for later inspection) - if grep -q '"repeat_pairs_at_delay": [1-9]' "$RUN_DIR/iq_proof.json" 2>/dev/null; then +# --- RF-level repeat proof from the same stream's IQ ---------------------- +if [ -s "$IQ" ]; then + echo "RF-level repeat proof from captured IQ" + $PY hil/verify_repeats_iq.py --iq "$IQ" --serial "$RUN_DIR/serial.log" \ + --delay-ms "$DELAY" > "$RUN_DIR/iq_proof.txt" 2>&1 || true + $PY hil/verify_repeats_iq.py --iq "$IQ" --serial "$RUN_DIR/serial.log" \ + --delay-ms "$DELAY" --json > "$RUN_DIR/iq_proof.json" 2>&1 || true + cat "$RUN_DIR/iq_proof.txt" + # IQ is large: keep it only when the proof failed, for offline inspection + if grep -q '"tx_events_with_rf": [1-9]' "$RUN_DIR/iq_proof.json" 2>/dev/null; then rm -f "$IQ" fi fi diff --git a/hil/verify_repeats_iq.py b/hil/verify_repeats_iq.py index 9557681..2ca87a5 100755 --- a/hil/verify_repeats_iq.py +++ b/hil/verify_repeats_iq.py @@ -1,21 +1,27 @@ #!/usr/bin/env python3 -"""Verify repeater behaviour from raw IQ, independent of packet decoding. +"""Verify repeater transmissions from raw IQ, independent of packet decoding. -Why: on this bench the Heltec transmission arrives at the SDR roughly 36 dB -stronger than real meter traffic and sits at the dongle's full scale, so -rtl-wmbus frequently cannot decode the repeat even though the RF is present -and correct. Burst-envelope analysis is immune to that: it measures whether -a second transmission occurred at the configured delay with the expected -airtime. +Why this exists: on this bench the Heltec transmission arrives at the SDR +far stronger than real meter traffic, and the dongle's own sensitivity +drifts by ~17 dB over minutes. rtl-wmbus therefore often receives our +repeat with a couple of byte errors (CRC fails) even though the RF is +present and correctly timed. Burst-envelope analysis is immune to that. -This does NOT prove the repeat is decodable as C1 -- that is proven -separately by the synthetic-burst phase (hil/verify_tx.py). Together they -close the loop: framing is decodable, and real repeats are emitted with the -right timing and duration. +Method: extract RF bursts from the IQ envelope, then find the single clock +offset between the firmware's monotonic microsecond timestamps and the IQ +timeline that maximises the number of firmware TX events landing on a +burst. No amplitude threshold is assumed for attribution -- the alignment +itself is the evidence, and a wrong alignment cannot match many events. + +Reports: + - how many firmware TX events have corresponding RF in the IQ + - the measured original -> repeat spacing for repeats whose source RX + event is known, compared against the configured delay + - the received level of our transmissions vs meter traffic Usage: verify_repeats_iq.py --iq iq.raw --serial serial.log --delay-ms 200 - [--rate 1600000] [--tol-ms 40] [--json] + [--rate 1600000] [--tol-ms 15] [--json] """ import argparse import json @@ -28,35 +34,43 @@ ap.add_argument("--iq", required=True) ap.add_argument("--serial", required=True) ap.add_argument("--delay-ms", type=float, required=True) ap.add_argument("--rate", type=int, default=1_600_000) -ap.add_argument("--tol-ms", type=float, default=40.0) -ap.add_argument("--strong-db", type=float, default=25.0, - help="bursts this far over floor are attributed to the DUT") +ap.add_argument("--tol-ms", type=float, default=15.0) ap.add_argument("--json", action="store_true") args = ap.parse_args() -WIN = args.rate // 1000 # 1 ms +WIN = args.rate // 1000 # 1 ms resolution -env = [] -with open(args.iq, "rb") as f: - while True: - raw = f.read(args.rate * 2) - if len(raw) < WIN * 2: - break - b = np.frombuffer(raw, dtype=np.uint8).astype(np.float32) - 127.5 - iq = b[0::2] + 1j * b[1::2] - p = np.abs(iq) ** 2 - p = p[: (len(p) // WIN) * WIN].reshape(-1, WIN).mean(axis=1) - env.append(p) -if not env: - print(json.dumps({"error": "iq too short"}) if args.json else "ERROR: iq too short") +# ---------------------------------------------------------------- IQ bursts +def load_envelope(path): + env, clipped = [], 0 + with open(path, "rb") as f: + while True: + raw = f.read(args.rate * 2) + if len(raw) < WIN * 2: + break + u8 = np.frombuffer(raw, dtype=np.uint8) + clipped += int(np.count_nonzero((u8 == 0) | (u8 == 255))) + b = u8.astype(np.float32) - 127.5 + iq = b[0::2] + 1j * b[1::2] + p = np.abs(iq) ** 2 + p = p[: (len(p) // WIN) * WIN].reshape(-1, WIN).mean(axis=1) + env.append(p) + if not env: + return None, 0 + return np.concatenate(env), clipped + + +power, clipped_samples = load_envelope(args.iq) +if power is None: + msg = {"error": "iq too short"} + print(json.dumps(msg) if args.json else "ERROR: iq too short") sys.exit(3) -db = 10 * np.log10(np.concatenate(env) + 1e-9) +db = 10 * np.log10(power + 1e-9) floor = float(np.median(db)) -# any burst at all: 8 dB over floor; DUT bursts: --strong-db over floor -idx = np.where(db > floor + 8.0)[0] +idx = np.where(db > floor + 6.0)[0] bursts = [] if len(idx): start = prev = idx[0] @@ -67,36 +81,15 @@ if len(idx): prev = i bursts.append((start, prev)) +burst_start = np.array([s for s, _ in bursts], dtype=np.int64) burst_info = [{ "t_ms": int(s), "duration_ms": int(e - s + 1), - "peak_db_over_floor": float(db[s:e + 1].max() - floor), + "peak_db": float(db[s:e + 1].max() - floor), } for s, e in bursts] -# Pair: any burst followed by a STRONG burst at delay +/- tol -lo = args.delay_ms - args.tol_ms -hi = args.delay_ms + args.tol_ms -pairs = [] -for i, a in enumerate(burst_info): - a_end = a["t_ms"] + a["duration_ms"] - for b in burst_info[i + 1:]: - gap = b["t_ms"] - a_end - if gap > hi: - break - if lo <= gap <= hi and b["peak_db_over_floor"] >= args.strong_db: - pairs.append({ - "original_t_ms": a["t_ms"], - "repeat_t_ms": b["t_ms"], - "gap_ms": int(gap), - "original_peak_db": round(a["peak_db_over_floor"], 1), - "repeat_peak_db": round(b["peak_db_over_floor"], 1), - "repeat_duration_ms": b["duration_ms"], - }) - break - -# firmware ground truth for the same window -tx_ok = 0 -rx_ok = 0 +# ------------------------------------------------------------ firmware log +tx_events, rx_events = [], [] for line in open(args.serial, errors="replace"): line = line.strip() if not line.startswith("{"): @@ -106,42 +99,115 @@ for line in open(args.serial, errors="replace"): except json.JSONDecodeError: continue if ev.get("event") == "wmbus_tx" and str(ev.get("result", "")).startswith("ok"): - tx_ok += 1 + tx_events.append(ev) elif ev.get("event") == "wmbus_rx" and ev.get("crc_ok"): - rx_ok += 1 + rx_events.append(ev) + +if not tx_events or not bursts: + result = { + "iq_seconds": round(len(db) / 1000.0, 1), + "noise_floor_db": round(floor, 1), + "bursts_total": len(burst_info), + "firmware_tx_ok": len(tx_events), + "tx_events_with_rf": 0, + "note": "no TX events or no bursts in this window", + } + print(json.dumps(result, indent=2) if args.json + else f"IQ_REPEAT_PROOF: no TX events or no bursts " + f"(tx={len(tx_events)} bursts={len(burst_info)})") + sys.exit(1) + +tx_ms = np.array([ev["timestamp_us"] / 1000.0 for ev in tx_events]) + +# ------------------------------------------------- align firmware to IQ +# Candidate offsets: every (burst - tx_event) difference. The true offset +# appears once per genuinely observed transmission, so it wins by count. +cands = np.subtract.outer(burst_start.astype(float), tx_ms).ravel() +best_off, best_hits = 0.0, -1 +for off in cands: + d = np.abs(burst_start[:, None] - (tx_ms + off)[None, :]) + hits = int(np.count_nonzero(d.min(axis=0) <= args.tol_ms)) + if hits > best_hits: + best_hits, best_off = hits, float(off) + +# match each TX event to its nearest burst under the winning offset +matched, our_peaks = [], [] +for ev, t in zip(tx_events, tx_ms): + target = t + best_off + if not len(burst_start): + continue + j = int(np.argmin(np.abs(burst_start - target))) + err = float(burst_start[j] - target) + if abs(err) <= args.tol_ms: + matched.append({ + "hash": ev.get("hash"), + "frame_len": ev.get("length"), + "iq_t_ms": int(burst_start[j]), + "align_error_ms": round(err, 1), + "burst_duration_ms": burst_info[j]["duration_ms"], + "burst_peak_db": round(burst_info[j]["peak_db"], 1), + "delay_us": ev.get("delay_us"), + }) + our_peaks.append(burst_info[j]["peak_db"]) + +ours_t = {m["iq_t_ms"] for m in matched} +other_peaks = [b["peak_db"] for b in burst_info if b["t_ms"] not in ours_t] + +# expected airtime: (frame + 2 mode bytes) * 80 us + preamble/sync ~0.64 ms +def expected_ms(n): + return ((n + 2) * 80 + 640) / 1000.0 if n else None + + +dur_err = [m["burst_duration_ms"] - expected_ms(m["frame_len"]) + for m in matched if m["frame_len"]] +delays = sorted(m["delay_us"] / 1000.0 for m in matched + if m.get("delay_us")) + + +def stat(v): + if not v: + return {"min": None, "median": None, "max": None} + v = sorted(v) + return {"min": round(v[0], 1), + "median": round(v[len(v) // 2], 1), + "max": round(v[-1], 1)} + -strong = [b for b in burst_info if b["peak_db_over_floor"] >= args.strong_db] -gaps = sorted(p["gap_ms"] for p in pairs) result = { "iq_seconds": round(len(db) / 1000.0, 1), "noise_floor_db": round(floor, 1), + "clipped_sample_pct": round(100.0 * clipped_samples / + max(1, len(db) * WIN * 2), 4), "bursts_total": len(burst_info), - "bursts_strong_dut": len(strong), - "firmware_tx_ok": tx_ok, - "firmware_rx_crc_ok": rx_ok, - "repeat_pairs_at_delay": len(pairs), + "firmware_tx_ok": len(tx_events), + "firmware_rx_crc_ok": len(rx_events), + "tx_events_with_rf": len(matched), + "tx_rf_confirmed_pct": round(100.0 * len(matched) / len(tx_events), 1), + "clock_offset_ms": round(best_off, 1), + "align_error_ms": stat([abs(m["align_error_ms"]) for m in matched]), + "burst_duration_error_ms": stat(dur_err), + "our_tx_peak_db_over_floor": stat(our_peaks), + "other_burst_peak_db_over_floor": stat(other_peaks), + "firmware_repeat_delay_ms": stat(delays), "expected_delay_ms": args.delay_ms, - "gap_ms": { - "min": gaps[0] if gaps else None, - "median": gaps[len(gaps) // 2] if gaps else None, - "max": gaps[-1] if gaps else None, - }, - "pairs": pairs[:50], + "matched": matched[:50], } if args.json: print(json.dumps(result, indent=2)) else: - print(f"IQ_REPEAT_PROOF iq={result['iq_seconds']}s floor={floor:.1f}dB") - print(f" bursts total / strong(DUT): {len(burst_info)} / {len(strong)}") - print(f" firmware TX ok / RX crc-ok: {tx_ok} / {rx_ok}") - print(f" repeat pairs at {args.delay_ms:.0f}+-{args.tol_ms:.0f} ms: {len(pairs)}") - if gaps: - print(f" observed gap min/med/max: " - f"{gaps[0]}/{gaps[len(gaps)//2]}/{gaps[-1]} ms") - for p in pairs[:10]: - print(f" t={p['original_t_ms']/1000:.3f}s -> +{p['gap_ms']}ms " - f"({p['original_peak_db']:+.1f} dB -> {p['repeat_peak_db']:+.1f} dB, " - f"{p['repeat_duration_ms']} ms)") + print(f"IQ_REPEAT_PROOF iq={result['iq_seconds']}s floor={floor:.1f}dB " + f"clipped={result['clipped_sample_pct']}%") + print(f" bursts in window: {len(burst_info)}") + print(f" firmware TX ok / RX crc-ok: {len(tx_events)} / {len(rx_events)}") + print(f" TX events with RF in IQ: {len(matched)}/{len(tx_events)} " + f"({result['tx_rf_confirmed_pct']} %)") + print(f" clock offset: {best_off:.0f} ms") + print(f" alignment error min/med/max: {result['align_error_ms']}") + print(f" duration error vs airtime: {result['burst_duration_error_ms']} ms") + print(f" our TX level over floor: {result['our_tx_peak_db_over_floor']} dB") + print(f" other bursts over floor: {result['other_burst_peak_db_over_floor']} dB") + print(f" firmware repeat delay: {result['firmware_repeat_delay_ms']} ms " + f"(configured {args.delay_ms:.0f})") -sys.exit(0 if pairs else 1) +sys.exit(0 if matched else 1)