hil: single-stream decode+IQ capture, clock-aligned RF TX proof

This commit is contained in:
Lars Nolden
2026-09-01 09:22:20 +02:00
parent 3fea191008
commit 74cb28daf7
4 changed files with 221 additions and 114 deletions
+149 -83
View File
@@ -1,21 +1,27 @@
#!/usr/bin/env python3
"""Verify repeater behaviour from raw IQ, independent of packet decoding.
"""Verify repeater transmissions from raw IQ, independent of packet decoding.
Why: on this bench the Heltec transmission arrives at the SDR roughly 36 dB
stronger than real meter traffic and sits at the dongle's full scale, so
rtl-wmbus frequently cannot decode the repeat even though the RF is present
and correct. Burst-envelope analysis is immune to that: it measures whether
a second transmission occurred at the configured delay with the expected
airtime.
Why this exists: on this bench the Heltec transmission arrives at the SDR
far stronger than real meter traffic, and the dongle's own sensitivity
drifts by ~17 dB over minutes. rtl-wmbus therefore often receives our
repeat with a couple of byte errors (CRC fails) even though the RF is
present and correctly timed. Burst-envelope analysis is immune to that.
This does NOT prove the repeat is decodable as C1 -- that is proven
separately by the synthetic-burst phase (hil/verify_tx.py). Together they
close the loop: framing is decodable, and real repeats are emitted with the
right timing and duration.
Method: extract RF bursts from the IQ envelope, then find the single clock
offset between the firmware's monotonic microsecond timestamps and the IQ
timeline that maximises the number of firmware TX events landing on a
burst. No amplitude threshold is assumed for attribution -- the alignment
itself is the evidence, and a wrong alignment cannot match many events.
Reports:
- how many firmware TX events have corresponding RF in the IQ
- the measured original -> repeat spacing for repeats whose source RX
event is known, compared against the configured delay
- the received level of our transmissions vs meter traffic
Usage:
verify_repeats_iq.py --iq iq.raw --serial serial.log --delay-ms 200
[--rate 1600000] [--tol-ms 40] [--json]
[--rate 1600000] [--tol-ms 15] [--json]
"""
import argparse
import json
@@ -28,35 +34,43 @@ ap.add_argument("--iq", required=True)
ap.add_argument("--serial", required=True)
ap.add_argument("--delay-ms", type=float, required=True)
ap.add_argument("--rate", type=int, default=1_600_000)
ap.add_argument("--tol-ms", type=float, default=40.0)
ap.add_argument("--strong-db", type=float, default=25.0,
help="bursts this far over floor are attributed to the DUT")
ap.add_argument("--tol-ms", type=float, default=15.0)
ap.add_argument("--json", action="store_true")
args = ap.parse_args()
WIN = args.rate // 1000 # 1 ms
WIN = args.rate // 1000 # 1 ms resolution
env = []
with open(args.iq, "rb") as f:
while True:
raw = f.read(args.rate * 2)
if len(raw) < WIN * 2:
break
b = np.frombuffer(raw, dtype=np.uint8).astype(np.float32) - 127.5
iq = b[0::2] + 1j * b[1::2]
p = np.abs(iq) ** 2
p = p[: (len(p) // WIN) * WIN].reshape(-1, WIN).mean(axis=1)
env.append(p)
if not env:
print(json.dumps({"error": "iq too short"}) if args.json else "ERROR: iq too short")
# ---------------------------------------------------------------- IQ bursts
def load_envelope(path):
env, clipped = [], 0
with open(path, "rb") as f:
while True:
raw = f.read(args.rate * 2)
if len(raw) < WIN * 2:
break
u8 = np.frombuffer(raw, dtype=np.uint8)
clipped += int(np.count_nonzero((u8 == 0) | (u8 == 255)))
b = u8.astype(np.float32) - 127.5
iq = b[0::2] + 1j * b[1::2]
p = np.abs(iq) ** 2
p = p[: (len(p) // WIN) * WIN].reshape(-1, WIN).mean(axis=1)
env.append(p)
if not env:
return None, 0
return np.concatenate(env), clipped
power, clipped_samples = load_envelope(args.iq)
if power is None:
msg = {"error": "iq too short"}
print(json.dumps(msg) if args.json else "ERROR: iq too short")
sys.exit(3)
db = 10 * np.log10(np.concatenate(env) + 1e-9)
db = 10 * np.log10(power + 1e-9)
floor = float(np.median(db))
# any burst at all: 8 dB over floor; DUT bursts: --strong-db over floor
idx = np.where(db > floor + 8.0)[0]
idx = np.where(db > floor + 6.0)[0]
bursts = []
if len(idx):
start = prev = idx[0]
@@ -67,36 +81,15 @@ if len(idx):
prev = i
bursts.append((start, prev))
burst_start = np.array([s for s, _ in bursts], dtype=np.int64)
burst_info = [{
"t_ms": int(s),
"duration_ms": int(e - s + 1),
"peak_db_over_floor": float(db[s:e + 1].max() - floor),
"peak_db": float(db[s:e + 1].max() - floor),
} for s, e in bursts]
# Pair: any burst followed by a STRONG burst at delay +/- tol
lo = args.delay_ms - args.tol_ms
hi = args.delay_ms + args.tol_ms
pairs = []
for i, a in enumerate(burst_info):
a_end = a["t_ms"] + a["duration_ms"]
for b in burst_info[i + 1:]:
gap = b["t_ms"] - a_end
if gap > hi:
break
if lo <= gap <= hi and b["peak_db_over_floor"] >= args.strong_db:
pairs.append({
"original_t_ms": a["t_ms"],
"repeat_t_ms": b["t_ms"],
"gap_ms": int(gap),
"original_peak_db": round(a["peak_db_over_floor"], 1),
"repeat_peak_db": round(b["peak_db_over_floor"], 1),
"repeat_duration_ms": b["duration_ms"],
})
break
# firmware ground truth for the same window
tx_ok = 0
rx_ok = 0
# ------------------------------------------------------------ firmware log
tx_events, rx_events = [], []
for line in open(args.serial, errors="replace"):
line = line.strip()
if not line.startswith("{"):
@@ -106,42 +99,115 @@ for line in open(args.serial, errors="replace"):
except json.JSONDecodeError:
continue
if ev.get("event") == "wmbus_tx" and str(ev.get("result", "")).startswith("ok"):
tx_ok += 1
tx_events.append(ev)
elif ev.get("event") == "wmbus_rx" and ev.get("crc_ok"):
rx_ok += 1
rx_events.append(ev)
if not tx_events or not bursts:
result = {
"iq_seconds": round(len(db) / 1000.0, 1),
"noise_floor_db": round(floor, 1),
"bursts_total": len(burst_info),
"firmware_tx_ok": len(tx_events),
"tx_events_with_rf": 0,
"note": "no TX events or no bursts in this window",
}
print(json.dumps(result, indent=2) if args.json
else f"IQ_REPEAT_PROOF: no TX events or no bursts "
f"(tx={len(tx_events)} bursts={len(burst_info)})")
sys.exit(1)
tx_ms = np.array([ev["timestamp_us"] / 1000.0 for ev in tx_events])
# ------------------------------------------------- align firmware to IQ
# Candidate offsets: every (burst - tx_event) difference. The true offset
# appears once per genuinely observed transmission, so it wins by count.
cands = np.subtract.outer(burst_start.astype(float), tx_ms).ravel()
best_off, best_hits = 0.0, -1
for off in cands:
d = np.abs(burst_start[:, None] - (tx_ms + off)[None, :])
hits = int(np.count_nonzero(d.min(axis=0) <= args.tol_ms))
if hits > best_hits:
best_hits, best_off = hits, float(off)
# match each TX event to its nearest burst under the winning offset
matched, our_peaks = [], []
for ev, t in zip(tx_events, tx_ms):
target = t + best_off
if not len(burst_start):
continue
j = int(np.argmin(np.abs(burst_start - target)))
err = float(burst_start[j] - target)
if abs(err) <= args.tol_ms:
matched.append({
"hash": ev.get("hash"),
"frame_len": ev.get("length"),
"iq_t_ms": int(burst_start[j]),
"align_error_ms": round(err, 1),
"burst_duration_ms": burst_info[j]["duration_ms"],
"burst_peak_db": round(burst_info[j]["peak_db"], 1),
"delay_us": ev.get("delay_us"),
})
our_peaks.append(burst_info[j]["peak_db"])
ours_t = {m["iq_t_ms"] for m in matched}
other_peaks = [b["peak_db"] for b in burst_info if b["t_ms"] not in ours_t]
# expected airtime: (frame + 2 mode bytes) * 80 us + preamble/sync ~0.64 ms
def expected_ms(n):
return ((n + 2) * 80 + 640) / 1000.0 if n else None
dur_err = [m["burst_duration_ms"] - expected_ms(m["frame_len"])
for m in matched if m["frame_len"]]
delays = sorted(m["delay_us"] / 1000.0 for m in matched
if m.get("delay_us"))
def stat(v):
if not v:
return {"min": None, "median": None, "max": None}
v = sorted(v)
return {"min": round(v[0], 1),
"median": round(v[len(v) // 2], 1),
"max": round(v[-1], 1)}
strong = [b for b in burst_info if b["peak_db_over_floor"] >= args.strong_db]
gaps = sorted(p["gap_ms"] for p in pairs)
result = {
"iq_seconds": round(len(db) / 1000.0, 1),
"noise_floor_db": round(floor, 1),
"clipped_sample_pct": round(100.0 * clipped_samples /
max(1, len(db) * WIN * 2), 4),
"bursts_total": len(burst_info),
"bursts_strong_dut": len(strong),
"firmware_tx_ok": tx_ok,
"firmware_rx_crc_ok": rx_ok,
"repeat_pairs_at_delay": len(pairs),
"firmware_tx_ok": len(tx_events),
"firmware_rx_crc_ok": len(rx_events),
"tx_events_with_rf": len(matched),
"tx_rf_confirmed_pct": round(100.0 * len(matched) / len(tx_events), 1),
"clock_offset_ms": round(best_off, 1),
"align_error_ms": stat([abs(m["align_error_ms"]) for m in matched]),
"burst_duration_error_ms": stat(dur_err),
"our_tx_peak_db_over_floor": stat(our_peaks),
"other_burst_peak_db_over_floor": stat(other_peaks),
"firmware_repeat_delay_ms": stat(delays),
"expected_delay_ms": args.delay_ms,
"gap_ms": {
"min": gaps[0] if gaps else None,
"median": gaps[len(gaps) // 2] if gaps else None,
"max": gaps[-1] if gaps else None,
},
"pairs": pairs[:50],
"matched": matched[:50],
}
if args.json:
print(json.dumps(result, indent=2))
else:
print(f"IQ_REPEAT_PROOF iq={result['iq_seconds']}s floor={floor:.1f}dB")
print(f" bursts total / strong(DUT): {len(burst_info)} / {len(strong)}")
print(f" firmware TX ok / RX crc-ok: {tx_ok} / {rx_ok}")
print(f" repeat pairs at {args.delay_ms:.0f}+-{args.tol_ms:.0f} ms: {len(pairs)}")
if gaps:
print(f" observed gap min/med/max: "
f"{gaps[0]}/{gaps[len(gaps)//2]}/{gaps[-1]} ms")
for p in pairs[:10]:
print(f" t={p['original_t_ms']/1000:.3f}s -> +{p['gap_ms']}ms "
f"({p['original_peak_db']:+.1f} dB -> {p['repeat_peak_db']:+.1f} dB, "
f"{p['repeat_duration_ms']} ms)")
print(f"IQ_REPEAT_PROOF iq={result['iq_seconds']}s floor={floor:.1f}dB "
f"clipped={result['clipped_sample_pct']}%")
print(f" bursts in window: {len(burst_info)}")
print(f" firmware TX ok / RX crc-ok: {len(tx_events)} / {len(rx_events)}")
print(f" TX events with RF in IQ: {len(matched)}/{len(tx_events)} "
f"({result['tx_rf_confirmed_pct']} %)")
print(f" clock offset: {best_off:.0f} ms")
print(f" alignment error min/med/max: {result['align_error_ms']}")
print(f" duration error vs airtime: {result['burst_duration_error_ms']} ms")
print(f" our TX level over floor: {result['our_tx_peak_db_over_floor']} dB")
print(f" other bursts over floor: {result['other_burst_peak_db_over_floor']} dB")
print(f" firmware repeat delay: {result['firmware_repeat_delay_ms']} ms "
f"(configured {args.delay_ms:.0f})")
sys.exit(0 if pairs else 1)
sys.exit(0 if matched else 1)