diff --git a/hil/capture_sdr.sh b/hil/capture_sdr.sh index 5a213f4..792e06e 100755 --- a/hil/capture_sdr.sh +++ b/hil/capture_sdr.sh @@ -20,6 +20,18 @@ ERR="${OUT%.log}.err" STATS="${OUT%.log}.stats" : > "$ERR" +# Verify the observer is actually receiving before trusting any capture. +health_ok=0 +for i in 1 2 3; do + if ./result-py/bin/python3 hil/sdr_health.py >> "$ERR" 2>&1; then + health_ok=1; break + fi + echo "--- SDR unhealthy; usbreset (try $i)" >> "$ERR" + usbreset 0bda:2838 >> "$ERR" 2>&1 || true + sleep $((i * 5)) +done +echo "sdr_health_ok=$health_ok" > "${OUT%.log}.health" + deadline=$(( $(date +%s) + DUR )) attempt=0 streamed=0 diff --git a/hil/report.py b/hil/report.py index f148caf..2971150 100755 --- a/hil/report.py +++ b/hil/report.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 """Render a human-readable HIL report from correlate.py JSON. -Usage: report.py report.json [--min-repeats N] [--min-rx-rate PCT] +Usage: report.py report.json [options] Exit code 0 = PASS, 1 = FAIL. """ import argparse import json +import os import sys ap = argparse.ArgumentParser() @@ -17,70 +18,99 @@ ap.add_argument("--min-rx-rate", type=float, default=0, ap.add_argument("--duration", type=float, default=0) ap.add_argument("--tx-power", default="?") ap.add_argument("--repeat-delay", default="?") +ap.add_argument("--sdr-streamed", default="?") +ap.add_argument("--tx-proof", default=None) args = ap.parse_args() r = json.load(open(args.report)) - d = r.get("repeat_delay_ms") or {} fw = r.get("firmware_events") or {} -fail_reasons = [] +proof = None +if args.tx_proof and os.path.exists(args.tx_proof): + try: + proof = json.load(open(args.tx_proof)) + except (json.JSONDecodeError, OSError): + proof = None + + +def fmt(v, unit=""): + if v is None: + return "n/a" + if isinstance(v, float): + return f"{v:.1f}{unit}" + return f"{v}{unit}" + + +fail = [] if r.get("sdr_originals", 0) == 0: - fail_reasons.append("SDR saw no original meter frames") + fail.append("SDR observed no original meter frames") if r.get("heltec_rx", 0) == 0: - fail_reasons.append("Heltec received nothing") + fail.append("Heltec received nothing") if r.get("sdr_repeats", 0) < args.min_repeats: - fail_reasons.append( - f"SDR-verified repeats {r.get('sdr_repeats', 0)} < {args.min_repeats}") + fail.append(f"SDR-verified repeats {r.get('sdr_repeats', 0)} < {args.min_repeats}") if args.min_rx_rate and (r.get("rx_match_rate_pct") or 0) < args.min_rx_rate: - fail_reasons.append("RX match rate below threshold") + fail.append("RX match rate below threshold") if fw.get("boots", 0) > 1: - fail_reasons.append(f"unexpected MCU resets: {fw.get('boots', 0) - 1}") + fail.append(f"unexpected MCU resets: {fw['boots'] - 1}") +if proof is not None and proof.get("bursts_with_sdr_decode", 0) == 0: + fail.append("SDR could not decode any Heltec C1 transmission") -verdict = "PASS" if not fail_reasons else "FAIL" +verdict = "PASS" if not fail else "FAIL" -W = 50 -print("=" * W) +print("=" * 50) print("SOLENOS WMBUS HIL TEST") -print("=" * W) +print("=" * 50) print(f""" DUT: Heltec WiFi LoRa 32 V4.3 (HTIT-WB32LAF) + ESP32-S3 + SX1262 + KCT8103L FEM RF: frequency: 868.950 MHz - mode: C1 - configured TX power: {args.tx_power} dBm + mode: C1 (2-FSK, 100 kbps, +/-45 kHz) + configured TX power: {args.tx_power} dBm (SX1262; external PA adds gain) repeat delay: {args.repeat_delay} ms TEST: duration: {args.duration:.0f} s + SDR streaming time: {args.sdr_streamed} s""") -SDR: - frames total: {r.get('sdr_frames_total')} - C1 CRC-ok: {r.get('sdr_c1_crc_ok')} - originals: {r.get('sdr_originals')} +if proof is not None: + print(f""" +TX DECODABILITY PROOF (synthetic C1, independent of meters): + bursts transmitted: {proof.get('bursts_requested')} + firmware TX ok: {proof.get('tx_reported_ok')} + SDR decoded as C1: {proof.get('bursts_with_sdr_decode')} + both airings decoded: {proof.get('bursts_with_both_airings_decoded')}""") + +print(f""" +SDR (independent observer): + decode lines: {r.get('sdr_frames_total')} + C1 CRC-ok bursts: {r.get('sdr_c1_crc_ok')} + classified originals: {r.get('sdr_originals')} + classified repeats: {r.get('sdr_repeats')} unique meters: {len(r.get('per_meter') or {})} HELTEC RX: frames: {r.get('heltec_rx')} - crc ok: {r.get('heltec_rx_crc_ok')} - matched originals: {r.get('rx_matched')} - match rate: {r.get('rx_match_rate_pct')} % + CRC ok: {r.get('heltec_rx_crc_ok')} + matched SDR originals: {r.get('rx_matched')} + match rate: {fmt(r.get('rx_match_rate_pct'), ' %')} HELTEC TX: attempts: {r.get('heltec_tx_attempts')} reported success: {r.get('heltec_tx_ok')} -SDR VERIFIED REPEATS: - matched: {r.get('sdr_repeats')} +SDR-VERIFIED REPEATS: + byte-identical pairs: {r.get('sdr_repeats')} fully correlated: {r.get('fully_correlated_repeats')} - success rate: {r.get('repeat_success_rate_pct')} % + vs TX attempts: {fmt(r.get('repeat_success_rate_pct'), ' %')} -TIMING: - median repeat delay: {d.get('median')} ms - minimum: {d.get('min')} ms - maximum: {d.get('max')} ms +TIMING (SDR-observed original -> repeat): + median: {fmt(d.get('median'), ' ms')} + minimum: {fmt(d.get('min'), ' ms')} + maximum: {fmt(d.get('max'), ' ms')} ERRORS: radio recoveries: {fw.get('recoveries')} @@ -89,14 +119,15 @@ ERRORS: if r.get("per_meter"): print(f"{'METER':>10} {'SDR':>6} {'HELTEC':>7} {'MATCHED':>8}") - for m, v in sorted(r["per_meter"].items()): + for m, v in sorted(r["per_meter"].items(), + key=lambda kv: -kv[1]["sdr"]): print(f"{m:>10} {v['sdr']:>6} {v['heltec']:>7} {v['matched']:>8}") print() -if fail_reasons: +if fail: print("FAIL REASONS:") - for fr in fail_reasons: - print(f" - {fr}") + for f in fail: + print(f" - {f}") print() print(f"RESULT:\n {verdict}") diff --git a/hil/run.sh b/hil/run.sh index 692938b..1e90109 100755 --- a/hil/run.sh +++ b/hil/run.sh @@ -1,22 +1,28 @@ #!/usr/bin/env bash -# Full HIL loop: discover -> (build+flash) -> capture serial+SDR -> correlate -> report. +# Full HIL loop for the Heltec V4.3 wM-Bus C1 repeater. +# +# discover devices -> verify SDR actually receives -> build+flash +# -> phase A: TX decodability proof (synthetic C1 bursts) +# -> phase B: real-meter RX/repeat correlation +# -> report (text + JSON), nonzero exit on FAIL # # Usage: ./hil/run.sh [--duration N] [--no-flash] [--tx-power N] [--delay MS] -# [--gain G] [--min-repeats N] -# Exit: 0 = PASS, nonzero = FAIL (report thresholds or infrastructure error). +# [--gain G] [--min-repeats N] [--no-proof] set -euo pipefail cd "$(dirname "$0")/.." DURATION=300 FLASH=1 TXPOW=-9 -DELAY=50 +DELAY=200 GAIN=42.1 MIN_REPEATS=1 +PROOF=1 while [ $# -gt 0 ]; do case "$1" in --duration) DURATION=$2; shift 2 ;; --no-flash) FLASH=0; shift ;; + --no-proof) PROOF=0; shift ;; --tx-power) TXPOW=$2; shift 2 ;; --delay) DELAY=$2; shift 2 ;; --gain) GAIN=$2; shift 2 ;; @@ -28,69 +34,99 @@ done PY=./result-py/bin/python3 [ -x "$PY" ] || { echo "run 'nix build .#hil-python -o result-py' first" >&2; exit 2; } -# 1-2. discover devices (abort on ambiguity) +# --- 1/2. discover devices (abort on ambiguity) ----------------------------- eval "$(./tools/discover_devices.sh)" echo "DUT: $DUT_PORT" echo "SDR: Nooelec NESDR SMArt v5, index $SDR_INDEX, serial $SDR_SERIAL" -# 3. SDR sanity: reset USB (rtl PLL can wedge; see docs/FINDINGS.md) and -# verify the device opens. -usbreset 0bda:2838 >/dev/null 2>&1 || true -sleep 1 - -# 4-6. build + flash + reset -if [ "$FLASH" = 1 ]; then - (cd firmware && idf.py build) >/dev/null - (cd firmware && idf.py -p "$DUT_PORT" flash) >/dev/null 2>&1 || { - # port re-enumerates when entering download mode; rediscover once - sleep 2 - eval "$(./tools/discover_devices.sh)" - (cd firmware && idf.py -p "$DUT_PORT" flash) >/dev/null - } - sleep 2 - eval "$(./tools/discover_devices.sh)" -fi - RUN_DIR="captures/$(date +%Y-%m-%dT%H-%M-%S)-hil" mkdir -p "$RUN_DIR" echo "run dir: $RUN_DIR" -# 7. configure DUT + start serial capture -$PY hil/dut_cmd.py "$DUT_PORT" --listen 2 \ +# --- 3. SDR must demonstrably receive before any capture is trusted -------- +for i in 1 2 3; do + if $PY hil/sdr_health.py > "$RUN_DIR/sdr_health.txt" 2>&1; then break; fi + echo "SDR unhealthy (try $i) - resetting" >&2 + usbreset 0bda:2838 >/dev/null 2>&1 || true + sleep $((i * 5)) +done +cat "$RUN_DIR/sdr_health.txt" +grep -q "SDR_HEALTH=ok" "$RUN_DIR/sdr_health.txt" || { + echo "ABORT: SDR observer is deaf; captures would be unattributable" >&2 + exit 3 +} + +# --- 4-6. build + flash + reset -------------------------------------------- +if [ "$FLASH" = 1 ]; then + (cd firmware && idf.py build) > "$RUN_DIR/build.log" 2>&1 + if ! (cd firmware && idf.py -p "$DUT_PORT" flash) >> "$RUN_DIR/build.log" 2>&1; then + # the port re-enumerates when entering download mode; rediscover once + sleep 2 + eval "$(./tools/discover_devices.sh)" + (cd firmware && idf.py -p "$DUT_PORT" flash) >> "$RUN_DIR/build.log" 2>&1 + fi + sleep 2 + eval "$(./tools/discover_devices.sh)" +fi + +# --- configure DUT -------------------------------------------------------- +$PY hil/dut_cmd.py "$DUT_PORT" --listen 3 \ "set repeat_delay_ms $DELAY" "set tx_power $TXPOW" "clear_stats" \ > "$RUN_DIR/setup.log" 2>&1 + +# --- phase A: TX decodability proof --------------------------------------- +# Synthetic C1 frames, each aired twice at the repeat delay. Independent of +# meter traffic, and (unlike real repeats) not preceded by a weak frame, +# so it isolates "can the SDR decode our C1 TX" from bench dynamic range. +if [ "$PROOF" = 1 ]; then + echo "phase A: TX decodability proof" + ./hil/capture_sdr.sh 70 "$RUN_DIR/proof_sdr.log" "$GAIN" & + SDR_PID=$! + sleep 10 + for L in 19 60 98; do + $PY hil/dut_cmd.py "$DUT_PORT" --listen 8 \ + "set test_lfield $L" test_burst >> "$RUN_DIR/proof_serial.log" 2>&1 + done + wait "$SDR_PID" || true + $PY hil/verify_tx.py --serial "$RUN_DIR/proof_serial.log" \ + --sdr "$RUN_DIR/proof_sdr.log" > "$RUN_DIR/tx_proof.txt" 2>&1 || true + $PY hil/verify_tx.py --serial "$RUN_DIR/proof_serial.log" \ + --sdr "$RUN_DIR/proof_sdr.log" --json > "$RUN_DIR/tx_proof.json" 2>&1 || true + cat "$RUN_DIR/tx_proof.txt" +fi + +# --- phase B: real meter traffic ------------------------------------------ +echo "phase B: real-meter capture (${DURATION}s)" $PY hil/dut_cmd.py "$DUT_PORT" --listen "$DURATION" status \ > "$RUN_DIR/serial.log" 2>&1 & SERIAL_PID=$! - -# 8. SDR capture (timeout applied to rtl_sdr itself; children die with it) ./hil/capture_sdr.sh "$DURATION" "$RUN_DIR/rtlwmbus.log" "$GAIN" & SDR_PID=$! - -# 9-12. wait for both collectors wait "$SDR_PID" || true wait "$SERIAL_PID" || true -# final stats snapshot -$PY hil/dut_cmd.py "$DUT_PORT" --listen 3 stats > "$RUN_DIR/final_stats.log" 2>&1 || true +$PY hil/dut_cmd.py "$DUT_PORT" --listen 4 stats > "$RUN_DIR/final_stats.log" 2>&1 || true cat "$RUN_DIR/final_stats.log" >> "$RUN_DIR/serial.log" -# 13-14. correlate +# --- 13-14. correlate ------------------------------------------------------ $PY hil/correlate.py --serial "$RUN_DIR/serial.log" --sdr "$RUN_DIR/rtlwmbus.log" \ - > "$RUN_DIR/report.json" + --min-delay 0.010 --max-delay 0.400 > "$RUN_DIR/report.json" -# 15. lockup detection: heartbeats should keep arriving +# --- 15. lockup / reset detection ----------------------------------------- HB=$(grep -c '"event":"heartbeat"' "$RUN_DIR/serial.log" || true) EXPECTED_HB=$((DURATION / 10 - 2)) if [ "$HB" -lt "$EXPECTED_HB" ]; then - echo "WARNING: only $HB heartbeats seen (expected >= $EXPECTED_HB) - possible lockup" \ + echo "WARNING: only $HB heartbeats (expected >= $EXPECTED_HB) - possible lockup" \ | tee "$RUN_DIR/lockup_warning.txt" fi -# 16-18. report +# --- 16-18. report --------------------------------------------------------- +STREAMED=$(sed -n 's/^streamed_seconds=//p' "$RUN_DIR/rtlwmbus.stats" 2>/dev/null || echo "?") $PY hil/report.py "$RUN_DIR/report.json" \ --min-repeats "$MIN_REPEATS" --duration "$DURATION" \ --tx-power "$TXPOW" --repeat-delay "$DELAY" \ + --sdr-streamed "$STREAMED" \ + --tx-proof "$RUN_DIR/tx_proof.json" \ | tee "$RUN_DIR/report.txt" RC=${PIPESTATUS[0]} echo "artifacts: $RUN_DIR" diff --git a/hil/sdr_health.py b/hil/sdr_health.py new file mode 100755 index 0000000..effbce2 --- /dev/null +++ b/hil/sdr_health.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Functional health check for the RTL-SDR observer. + +Why this exists: this dongle intermittently enters a state where the R820T +tuner is not actually receiving (I2C init failure, PLL wedge, or +direct-sampling bypass). Captures then contain gain-independent noise -- +indistinguishable from "no meter traffic" unless the receiver is tested. +Every empty capture must be attributable. + +Validated signatures (this hardware, 1.6 Msps at 868.95 MHz): + healthy : mean power ~ -3 dB @20.7 dB gain, ~ +5 dB @42.1 dB gain, + averaged spectrum span 18-26 dB (filter shape + signals) + wedged : mean power ~ +40 dB (railed), spectrum span < 10 dB, and + power does not track tuner gain + +Usage: sdr_health.py [--freq HZ] [--json] +Exit 0 = healthy, 2 = deaf/wedged, 3 = could not capture. +""" +import argparse +import json +import os +import subprocess +import sys +import tempfile + +import numpy as np + +ap = argparse.ArgumentParser() +ap.add_argument("--freq", type=int, default=868_950_000) +ap.add_argument("--json", action="store_true") +args = ap.parse_args() + +FS = 1_600_000 +NS = 480_000 # 0.3 s +NFFT = 4096 + +RAILED_DB = 25.0 # mean power above this => railed/bypassed +MIN_SPAN_DB = 12.0 # averaged-spectrum structure +MIN_GAIN_DELTA_DB = 3.0 # noise must track tuner gain + + +def measure(gain): + fd, path = tempfile.mkstemp(suffix=".raw") + os.close(fd) + try: + p = subprocess.run( + ["rtl_sdr", "-f", str(args.freq), "-s", str(FS), + "-g", str(gain), "-n", str(NS), path], + capture_output=True, timeout=30) + raw = np.fromfile(path, dtype=np.uint8) + if raw.size < NS: + return None, p.stderr.decode(errors="replace") + b = raw.astype(np.float32) - 127.5 + iq = b[0::2] + 1j * b[1::2] + n = (len(iq) // NFFT) * NFFT + blocks = iq[:n].reshape(-1, NFFT) * np.hanning(NFFT) + sp = np.abs(np.fft.fftshift(np.fft.fft(blocks, axis=1))) ** 2 + db = 10 * np.log10(sp.mean(axis=0) + 1e-9) + return { + "gain": gain, + "mean_power_db": float(10 * np.log10(np.mean(np.abs(iq) ** 2) + 1e-9)), + "spectrum_span_db": float(db.max() - db.min()), + }, p.stderr.decode(errors="replace") + finally: + os.unlink(path) + + +lo, err_lo = measure(20.7) +hi, err_hi = measure(42.1) + +if lo is None or hi is None: + out = {"health": "capture_failed", + "stderr": ((err_lo or "") + (err_hi or ""))[-500:]} + print(json.dumps(out) if args.json else f"SDR_HEALTH=capture_failed\n{out['stderr']}") + sys.exit(3) + +delta = hi["mean_power_db"] - lo["mean_power_db"] +reasons = [] +if hi["mean_power_db"] > RAILED_DB: + reasons.append(f"railed: mean power {hi['mean_power_db']:.1f} dB > {RAILED_DB}") +if hi["spectrum_span_db"] < MIN_SPAN_DB: + reasons.append(f"flat spectrum: span {hi['spectrum_span_db']:.1f} dB < {MIN_SPAN_DB}") +if delta < MIN_GAIN_DELTA_DB: + reasons.append(f"noise does not track gain: delta {delta:.1f} dB < {MIN_GAIN_DELTA_DB}") + +healthy = not reasons +result = { + "health": "ok" if healthy else "deaf", + "low": lo, "high": hi, + "gain_delta_db": round(delta, 2), + "reasons": reasons, +} + +if args.json: + print(json.dumps(result)) +else: + print(f"SDR_HEALTH={result['health']}") + print(f"SDR_MEAN_POWER_20DB={lo['mean_power_db']:.2f}") + print(f"SDR_MEAN_POWER_42DB={hi['mean_power_db']:.2f}") + print(f"SDR_SPECTRUM_SPAN_DB={hi['spectrum_span_db']:.2f}") + print(f"SDR_GAIN_DELTA_DB={delta:.2f}") + for r in reasons: + print(f"SDR_REASON={r}") + +sys.exit(0 if healthy else 2) diff --git a/hil/verify_tx.py b/hil/verify_tx.py new file mode 100755 index 0000000..f554746 --- /dev/null +++ b/hil/verify_tx.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Evaluate the TX-decodability proof phase. + +The DUT transmits synthetic C1 frames twice, separated by the configured +repeat delay (`test_burst`). The SDR must independently decode them as +Wireless M-Bus C1. This proves, without depending on real meter traffic: + - the Heltec emits standards-shaped C1 frames + - a second frame at the repeat delay is independently observable + - the exact bytes match what the firmware said it sent + +Usage: verify_tx.py --serial burst_serial.log --sdr burst_sdr.log [--json] +Exit 0 if at least one transmitted frame was decoded by the SDR. +""" +import argparse +import json +import sys + +sys.path.insert(0, __file__.rsplit("/", 1)[0]) +from correlate import cook_a, SDR_RE # noqa: E402 + +ap = argparse.ArgumentParser() +ap.add_argument("--serial", required=True) +ap.add_argument("--sdr", required=True) +ap.add_argument("--json", action="store_true") +args = ap.parse_args() + +frames = [] # synthetic frames the DUT reported building +tx_results = [] +for line in open(args.serial, errors="replace"): + line = line.strip() + if not line.startswith("{"): + continue + try: + ev = json.loads(line) + except json.JSONDecodeError: + continue + if ev.get("event") == "test_burst_frame": + frames.append(ev) + elif ev.get("event") == "wmbus_tx": + tx_results.append(ev) + +sdr_counts = {} +for line in open(args.sdr, errors="replace"): + m = SDR_RE.match(line.strip()) + if not m or m["mode"] != "C1" or m["crc"] != "1": + continue + sdr_counts[m["hex"].lower()] = sdr_counts.get(m["hex"].lower(), 0) + 1 + +per_frame = [] +for f in frames: + raw = bytes.fromhex(f["raw"]) + cooked = cook_a(raw) + n = sdr_counts.get(cooked.hex(), 0) if cooked else 0 + # each airing yields 1-2 lines (rtl_wmbus runs two demodulators) + airings = 0 if n == 0 else (1 if n <= 2 else 2) + per_frame.append({ + "lfield": f.get("lfield"), + "delay_ms": f.get("delay_ms"), + "sdr_lines": n, + "airings_decoded": airings, + }) + +decoded = sum(1 for p in per_frame if p["airings_decoded"] >= 1) +both = sum(1 for p in per_frame if p["airings_decoded"] >= 2) +tx_ok = sum(1 for t in tx_results if t.get("result", "").startswith("ok")) + +result = { + "bursts_requested": len(frames), + "tx_events": len(tx_results), + "tx_reported_ok": tx_ok, + "bursts_with_sdr_decode": decoded, + "bursts_with_both_airings_decoded": both, + "per_burst": per_frame, +} + +if args.json: + print(json.dumps(result, indent=2)) +else: + print(f"TX_PROOF bursts={len(frames)} tx_ok={tx_ok} " + f"sdr_decoded={decoded} both_airings={both}") + for p in per_frame: + print(f" L={p['lfield']} delay={p['delay_ms']}ms " + f"sdr_lines={p['sdr_lines']} airings={p['airings_decoded']}") + +sys.exit(0 if decoded >= 1 else 1)