HIL harness, docs, SDR PLL-wedge workaround; firmware flashed and in RX

This commit is contained in:
Lars Nolden
2026-08-31 22:23:26 +02:00
parent 5b46f647f6
commit 3b8547fb8a
13 changed files with 727 additions and 1 deletions
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
# Capture wM-Bus frames with the RTL-SDR for a fixed duration.
# Usage: capture_sdr.sh DURATION_S OUTFILE [GAIN]
# timeout is applied to rtl_sdr itself so the pipeline dies cleanly
# (killing a wrapping shell orphans the children -- learned the hard way).
set -euo pipefail
cd "$(dirname "$0")/.."
DUR=$1; OUT=$2; GAIN=${3:-42.1}
timeout "$DUR" rtl_sdr -f 868950000 -s 1600000 -g "$GAIN" - 2>"${OUT%.log}.err" \
| rtl_wmbus -f > "$OUT" 2>/dev/null || true
+7
View File
@@ -0,0 +1,7 @@
#!/usr/bin/env bash
# Capture structured serial output from the DUT for a fixed duration.
# Usage: capture_serial.sh PORT DURATION_S OUTFILE
set -euo pipefail
cd "$(dirname "$0")/.."
PORT=$1; DUR=$2; OUT=$3
exec ./result-py/bin/python3 hil/dut_cmd.py "$PORT" --listen "$DUR" status > "$OUT" 2>&1
+222
View File
@@ -0,0 +1,222 @@
#!/usr/bin/env python3
"""Correlate Heltec serial events with SDR (rtl_wmbus) captures.
Inputs:
--serial serial.log JSON-lines from the Heltec ({"event": ...})
--sdr rtlwmbus.log rtl_wmbus CSV-ish lines:
[algo]C1;crc_ok;3of6ok;timestamp;pkt_rssi;rssi;serial;0xHEX
Output: JSON report on stdout.
Correlation model (developer repeat delay ~50 ms):
SDR original <-> Heltec RX : same cooked bytes, |dt| < rx_window
Heltec RX <-> Heltec TX : same hash (firmware log)
Heltec TX <-> SDR repeat : same cooked bytes, dt in [min_delay, max_delay]
after the SDR original
The Heltec logs raw over-the-air frames (with CRCs); rtl_wmbus prints
"cooked" frames (CRCs stripped). cook() below mirrors rtl-wmbus.
"""
import argparse
import json
import re
import sys
from datetime import datetime
def cook_a(frame: bytes):
"""Strip block CRCs from a format A over-the-air frame."""
if len(frame) < 12:
return None
out = bytearray(frame[:10])
pos = 12
while pos < len(frame):
n = min(16, len(frame) - pos - 2)
if n <= 0:
return None
out += frame[pos:pos + n]
pos += n + 2
return bytes(out)
def cook_b(frame: bytes):
"""Strip CRCs from a format B frame and rewrite L (mirrors rtl-wmbus)."""
if len(frame) < 12:
return None
out = bytearray()
rem = frame
while rem:
if len(rem) >= 128:
out += rem[:126]
rem = rem[128:]
else:
out += rem[:-2]
rem = b""
# rewrite L: original L includes CRC bytes; cooked excludes them
out[0] = len(out) - 1
return bytes(out)
def parse_serial(path):
"""Heltec JSON-lines -> (rx_events, tx_events, other)."""
rx, tx, other = [], [], []
with open(path, "r", errors="replace") as f:
for line in f:
line = line.strip()
if not line.startswith("{"):
continue
try:
ev = json.loads(line)
except json.JSONDecodeError:
continue
e = ev.get("event")
if e == "wmbus_rx":
raw = bytes.fromhex(ev.get("raw", ""))
fmt = ev.get("format", "A")
cooked = cook_a(raw) if fmt == "A" else cook_b(raw)
ev["_raw"] = raw
ev["_cooked"] = cooked
rx.append(ev)
elif e == "wmbus_tx":
tx.append(ev)
else:
other.append(ev)
return rx, tx, other
SDR_RE = re.compile(
r"^(?:[a-z0-9_]+;)?(?P<mode>[TCS]1);(?P<crc>\d);(?P<x>\d);"
r"(?P<ts>[\d\- :.]+);(?P<prssi>\d+);(?P<rssi>\d+);"
r"(?P<serial>[0-9A-Fa-f]{8});0x(?P<hex>[0-9a-fA-F]+)"
)
def parse_sdr(path):
frames = []
with open(path, "r", errors="replace") as f:
for line in f:
m = SDR_RE.match(line.strip())
if not m:
continue
ts = datetime.strptime(m["ts"].strip(), "%Y-%m-%d %H:%M:%S.%f")
frames.append({
"mode": m["mode"],
"crc_ok": m["crc"] == "1",
"t": ts.timestamp(),
"serial": m["serial"].lower(),
"cooked": bytes.fromhex(m["hex"]),
})
return frames
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--serial", required=True)
ap.add_argument("--sdr", required=True)
ap.add_argument("--min-delay", type=float, default=0.010)
ap.add_argument("--max-delay", type=float, default=0.300)
ap.add_argument("--rx-window", type=float, default=2.0,
help="wall/mono clock skew tolerance for RX matching")
args = ap.parse_args()
heltec_rx, heltec_tx, other = parse_serial(args.serial)
sdr = parse_sdr(args.sdr)
sdr_c1 = [s for s in sdr if s["mode"] == "C1" and s["crc_ok"]]
# --- classify SDR frames into originals vs repeats ---------------------
# A repeat is an SDR frame with identical cooked bytes to an earlier SDR
# frame within [min_delay, max_delay]. Everything else is an original.
originals, repeats = [], []
for i, s in enumerate(sdr_c1):
matched = None
for j in range(i - 1, -1, -1):
prev = sdr_c1[j]
dt = s["t"] - prev["t"]
if dt > args.max_delay:
break
if prev["cooked"] == s["cooked"] and args.min_delay <= dt <= args.max_delay:
matched = (j, dt)
break
if matched is not None:
repeats.append({"idx": i, "orig_idx": matched[0], "dt": matched[1], **s})
else:
originals.append({"idx": i, **s})
# --- Heltec RX <-> SDR originals (exact cooked-bytes match) ------------
# Serial timestamps are monotonic-us; SDR timestamps are wall clock.
# Estimate offset from the best byte-match pairs.
matched_rx = 0
per_meter = {}
used_rx = set()
for o in originals:
ser = o["serial"]
pm = per_meter.setdefault(ser, {"sdr": 0, "heltec": 0, "matched": 0})
pm["sdr"] += 1
for k, r in enumerate(heltec_rx):
if k in used_rx or r["_cooked"] != o["cooked"]:
continue
used_rx.add(k)
matched_rx += 1
pm["matched"] += 1
o["heltec_rx"] = r
break
for r in heltec_rx:
if r["_cooked"] is None:
continue
ser = r["_cooked"][4:8][::-1].hex() if len(r["_cooked"]) >= 8 else "?"
per_meter.setdefault(ser, {"sdr": 0, "heltec": 0, "matched": 0})
per_meter[ser]["heltec"] += 1
# --- Heltec TX <-> SDR repeats -----------------------------------------
tx_by_hash = {}
for t in heltec_tx:
tx_by_hash.setdefault(t.get("hash"), []).append(t)
verified_repeats = 0
for rep in repeats:
orig = sdr_c1[rep["orig_idx"]]
o_entry = next((o for o in originals if o["idx"] == rep["orig_idx"]), None)
hr = o_entry.get("heltec_rx") if o_entry else None
if hr and hr.get("hash") in tx_by_hash:
verified_repeats += 1
rep["verified"] = True
tx_ok = [t for t in heltec_tx if t.get("result") == "ok"]
delays = [r["dt"] * 1000 for r in repeats]
delays.sort()
def pct(a, b):
return round(100.0 * a / b, 1) if b else None
report = {
"sdr_frames_total": len(sdr),
"sdr_c1_crc_ok": len(sdr_c1),
"sdr_originals": len(originals),
"sdr_repeats": len(repeats),
"heltec_rx": len(heltec_rx),
"heltec_rx_crc_ok": sum(1 for r in heltec_rx if r.get("crc_ok")),
"heltec_tx_attempts": len(heltec_tx),
"heltec_tx_ok": len(tx_ok),
"rx_matched": matched_rx,
"rx_match_rate_pct": pct(matched_rx, len(originals)),
"sdr_verified_repeats": len(repeats),
"fully_correlated_repeats": verified_repeats,
"repeat_success_rate_pct": pct(len(repeats), len(tx_ok)),
"repeat_delay_ms": {
"median": delays[len(delays) // 2] if delays else None,
"min": delays[0] if delays else None,
"max": delays[-1] if delays else None,
},
"per_meter": per_meter,
"firmware_events": {
"recoveries": sum(1 for e in other if e.get("event") == "radio_recovery"),
"boots": sum(1 for e in other if e.get("event") == "boot"),
},
}
json.dump(report, sys.stdout, indent=2)
print()
return 0
if __name__ == "__main__":
sys.exit(main())
Executable
+31
View File
@@ -0,0 +1,31 @@
#!/usr/bin/env python3
"""Send commands to the DUT and print JSON responses.
Usage: dut_cmd.py PORT [--listen SECONDS] CMD [CMD...]
Each CMD is sent as a line; output is echoed for --listen seconds (default 3).
"""
import sys
import time
import serial
port = sys.argv[1]
args = sys.argv[2:]
listen = 3.0
if args and args[0] == "--listen":
listen = float(args[1])
args = args[2:]
s = serial.Serial(port, 115200, timeout=0.2)
time.sleep(0.2)
s.reset_input_buffer()
for cmd in args:
s.write((cmd + "\n").encode())
s.flush()
time.sleep(0.1)
end = time.time() + listen
while time.time() < end:
line = s.readline()
if line:
sys.stdout.write(line.decode(errors="replace"))
sys.stdout.flush()
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env python3
"""Detect burst energy in a raw rtl_sdr u8 IQ capture.
Usage: iq_energy.py iq.raw [sample_rate]
Prints noise floor, and burst groups >9 dB above floor with 1 ms resolution.
Exit code 0 if at least one burst found, 2 otherwise.
"""
import sys
import numpy as np
path = sys.argv[1]
fs = int(sys.argv[2]) if len(sys.argv) > 2 else 1_600_000
win = fs // 1000 # 1 ms
pows = []
with open(path, "rb") as f:
while True:
raw = f.read(fs * 2) # 1 s
if len(raw) < win * 2:
break
buf = np.frombuffer(raw, dtype=np.uint8).astype(np.float32) - 127.5
iq = buf[0::2] + 1j * buf[1::2]
w = np.abs(iq) ** 2
w = w[: (len(w) // win) * win].reshape(-1, win).mean(axis=1)
pows.append(w)
db = 10 * np.log10(np.concatenate(pows) + 1e-9)
floor = np.median(db)
print(f"windows: {len(db)} x 1ms, floor={floor:.1f} dB, "
f"p99={np.percentile(db, 99):.1f}, max={db.max():.1f}")
idx = np.where(db > floor + 9)[0]
groups = []
if len(idx):
start = prev = idx[0]
for i in idx[1:]:
if i > prev + 2:
groups.append((start, prev))
start = i
prev = i
groups.append((start, prev))
print(f"burst groups (>9 dB over floor): {len(groups)}")
for g in groups[:40]:
print(f" t={g[0]/1000:.3f}s dur={g[1]-g[0]+1}ms "
f"peak=+{db[g[0]:g[1]+1].max()-floor:.1f}dB")
sys.exit(0 if groups else 2)
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env python3
"""Averaged power spectrum of a raw rtl_sdr u8 IQ capture.
Usage: iq_spectrum.py iq.raw [sample_rate] [seconds]
Prints a coarse ASCII spectrum and the top peaks.
"""
import sys
import numpy as np
path = sys.argv[1]
fs = int(sys.argv[2]) if len(sys.argv) > 2 else 1_600_000
secs = float(sys.argv[3]) if len(sys.argv) > 3 else 5.0
NFFT = 4096
acc = np.zeros(NFFT)
count = 0
with open(path, "rb") as f:
raw = f.read(int(fs * 2 * secs))
buf = np.frombuffer(raw, dtype=np.uint8).astype(np.float32) - 127.5
iq = buf[0::2] + 1j * buf[1::2]
n = (len(iq) // NFFT) * NFFT
iq = iq[:n].reshape(-1, NFFT)
win = np.hanning(NFFT)
for row in iq[:2000]:
sp = np.fft.fftshift(np.abs(np.fft.fft(row * win)) ** 2)
acc += sp
count += 1
db = 10 * np.log10(acc / count + 1e-9)
freqs = np.fft.fftshift(np.fft.fftfreq(NFFT, 1 / fs))
# collapse to 64 bins for display
B = 64
cb = db[: (NFFT // B) * B].reshape(B, -1).max(axis=1)
lo, hi = cb.min(), cb.max()
print(f"spectrum dB range: {lo:.1f} .. {hi:.1f} (span {hi-lo:.1f} dB)")
for i, v in enumerate(cb):
bar = "#" * int(max(0, (v - lo) / max(1e-9, hi - lo) * 60))
fk = freqs[i * (NFFT // B)] / 1e3
print(f"{fk:+8.0f} kHz |{bar}")
Executable
+103
View File
@@ -0,0 +1,103 @@
#!/usr/bin/env python3
"""Render a human-readable HIL report from correlate.py JSON.
Usage: report.py report.json [--min-repeats N] [--min-rx-rate PCT]
Exit code 0 = PASS, 1 = FAIL.
"""
import argparse
import json
import sys
ap = argparse.ArgumentParser()
ap.add_argument("report")
ap.add_argument("--min-repeats", type=int, default=1,
help="minimum SDR-verified repeats for PASS")
ap.add_argument("--min-rx-rate", type=float, default=0,
help="minimum Heltec RX match rate in %% for PASS")
ap.add_argument("--duration", type=float, default=0)
ap.add_argument("--tx-power", default="?")
ap.add_argument("--repeat-delay", default="?")
args = ap.parse_args()
r = json.load(open(args.report))
d = r.get("repeat_delay_ms") or {}
fw = r.get("firmware_events") or {}
fail_reasons = []
if r.get("sdr_originals", 0) == 0:
fail_reasons.append("SDR saw no original meter frames")
if r.get("heltec_rx", 0) == 0:
fail_reasons.append("Heltec received nothing")
if r.get("sdr_repeats", 0) < args.min_repeats:
fail_reasons.append(
f"SDR-verified repeats {r.get('sdr_repeats', 0)} < {args.min_repeats}")
if args.min_rx_rate and (r.get("rx_match_rate_pct") or 0) < args.min_rx_rate:
fail_reasons.append("RX match rate below threshold")
if fw.get("boots", 0) > 1:
fail_reasons.append(f"unexpected MCU resets: {fw.get('boots', 0) - 1}")
verdict = "PASS" if not fail_reasons else "FAIL"
W = 50
print("=" * W)
print("SOLENOS WMBUS HIL TEST")
print("=" * W)
print(f"""
DUT:
Heltec WiFi LoRa 32 V4.3 (HTIT-WB32LAF)
RF:
frequency: 868.950 MHz
mode: C1
configured TX power: {args.tx_power} dBm
repeat delay: {args.repeat_delay} ms
TEST:
duration: {args.duration:.0f} s
SDR:
frames total: {r.get('sdr_frames_total')}
C1 CRC-ok: {r.get('sdr_c1_crc_ok')}
originals: {r.get('sdr_originals')}
unique meters: {len(r.get('per_meter') or {})}
HELTEC RX:
frames: {r.get('heltec_rx')}
crc ok: {r.get('heltec_rx_crc_ok')}
matched originals: {r.get('rx_matched')}
match rate: {r.get('rx_match_rate_pct')} %
HELTEC TX:
attempts: {r.get('heltec_tx_attempts')}
reported success: {r.get('heltec_tx_ok')}
SDR VERIFIED REPEATS:
matched: {r.get('sdr_repeats')}
fully correlated: {r.get('fully_correlated_repeats')}
success rate: {r.get('repeat_success_rate_pct')} %
TIMING:
median repeat delay: {d.get('median')} ms
minimum: {d.get('min')} ms
maximum: {d.get('max')} ms
ERRORS:
radio recoveries: {fw.get('recoveries')}
MCU boots seen: {fw.get('boots')}
""")
if r.get("per_meter"):
print(f"{'METER':>10} {'SDR':>6} {'HELTEC':>7} {'MATCHED':>8}")
for m, v in sorted(r["per_meter"].items()):
print(f"{m:>10} {v['sdr']:>6} {v['heltec']:>7} {v['matched']:>8}")
print()
if fail_reasons:
print("FAIL REASONS:")
for fr in fail_reasons:
print(f" - {fr}")
print()
print(f"RESULT:\n {verdict}")
sys.exit(0 if verdict == "PASS" else 1)
Executable
+97
View File
@@ -0,0 +1,97 @@
#!/usr/bin/env bash
# Full HIL loop: discover -> (build+flash) -> capture serial+SDR -> correlate -> report.
#
# Usage: ./hil/run.sh [--duration N] [--no-flash] [--tx-power N] [--delay MS]
# [--gain G] [--min-repeats N]
# Exit: 0 = PASS, nonzero = FAIL (report thresholds or infrastructure error).
set -euo pipefail
cd "$(dirname "$0")/.."
DURATION=300
FLASH=1
TXPOW=-9
DELAY=50
GAIN=42.1
MIN_REPEATS=1
while [ $# -gt 0 ]; do
case "$1" in
--duration) DURATION=$2; shift 2 ;;
--no-flash) FLASH=0; shift ;;
--tx-power) TXPOW=$2; shift 2 ;;
--delay) DELAY=$2; shift 2 ;;
--gain) GAIN=$2; shift 2 ;;
--min-repeats) MIN_REPEATS=$2; shift 2 ;;
*) echo "unknown arg: $1" >&2; exit 2 ;;
esac
done
PY=./result-py/bin/python3
[ -x "$PY" ] || { echo "run 'nix build .#hil-python -o result-py' first" >&2; exit 2; }
# 1-2. discover devices (abort on ambiguity)
eval "$(./tools/discover_devices.sh)"
echo "DUT: $DUT_PORT"
echo "SDR: Nooelec NESDR SMArt v5, index $SDR_INDEX, serial $SDR_SERIAL"
# 3. SDR sanity: reset USB (rtl PLL can wedge; see docs/FINDINGS.md) and
# verify the device opens.
usbreset 0bda:2838 >/dev/null 2>&1 || true
sleep 1
# 4-6. build + flash + reset
if [ "$FLASH" = 1 ]; then
(cd firmware && idf.py build) >/dev/null
(cd firmware && idf.py -p "$DUT_PORT" flash) >/dev/null 2>&1 || {
# port re-enumerates when entering download mode; rediscover once
sleep 2
eval "$(./tools/discover_devices.sh)"
(cd firmware && idf.py -p "$DUT_PORT" flash) >/dev/null
}
sleep 2
eval "$(./tools/discover_devices.sh)"
fi
RUN_DIR="captures/$(date +%Y-%m-%dT%H-%M-%S)-hil"
mkdir -p "$RUN_DIR"
echo "run dir: $RUN_DIR"
# 7. configure DUT + start serial capture
$PY hil/dut_cmd.py "$DUT_PORT" --listen 2 \
"set repeat_delay_ms $DELAY" "set tx_power $TXPOW" "clear_stats" \
> "$RUN_DIR/setup.log" 2>&1
$PY hil/dut_cmd.py "$DUT_PORT" --listen "$DURATION" status \
> "$RUN_DIR/serial.log" 2>&1 &
SERIAL_PID=$!
# 8. SDR capture (timeout applied to rtl_sdr itself; children die with it)
./hil/capture_sdr.sh "$DURATION" "$RUN_DIR/rtlwmbus.log" "$GAIN" &
SDR_PID=$!
# 9-12. wait for both collectors
wait "$SDR_PID" || true
wait "$SERIAL_PID" || true
# final stats snapshot
$PY hil/dut_cmd.py "$DUT_PORT" --listen 3 stats > "$RUN_DIR/final_stats.log" 2>&1 || true
cat "$RUN_DIR/final_stats.log" >> "$RUN_DIR/serial.log"
# 13-14. correlate
$PY hil/correlate.py --serial "$RUN_DIR/serial.log" --sdr "$RUN_DIR/rtlwmbus.log" \
> "$RUN_DIR/report.json"
# 15. lockup detection: heartbeats should keep arriving
HB=$(grep -c '"event":"heartbeat"' "$RUN_DIR/serial.log" || true)
EXPECTED_HB=$((DURATION / 10 - 2))
if [ "$HB" -lt "$EXPECTED_HB" ]; then
echo "WARNING: only $HB heartbeats seen (expected >= $EXPECTED_HB) - possible lockup" \
| tee "$RUN_DIR/lockup_warning.txt"
fi
# 16-18. report
$PY hil/report.py "$RUN_DIR/report.json" \
--min-repeats "$MIN_REPEATS" --duration "$DURATION" \
--tx-power "$TXPOW" --repeat-delay "$DELAY" \
| tee "$RUN_DIR/report.txt"
RC=${PIPESTATUS[0]}
echo "artifacts: $RUN_DIR"
exit "$RC"