hil: single-stream decode+IQ capture, clock-aligned RF TX proof
This commit is contained in:
+149
-83
@@ -1,21 +1,27 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify repeater behaviour from raw IQ, independent of packet decoding.
|
||||
"""Verify repeater transmissions from raw IQ, independent of packet decoding.
|
||||
|
||||
Why: on this bench the Heltec transmission arrives at the SDR roughly 36 dB
|
||||
stronger than real meter traffic and sits at the dongle's full scale, so
|
||||
rtl-wmbus frequently cannot decode the repeat even though the RF is present
|
||||
and correct. Burst-envelope analysis is immune to that: it measures whether
|
||||
a second transmission occurred at the configured delay with the expected
|
||||
airtime.
|
||||
Why this exists: on this bench the Heltec transmission arrives at the SDR
|
||||
far stronger than real meter traffic, and the dongle's own sensitivity
|
||||
drifts by ~17 dB over minutes. rtl-wmbus therefore often receives our
|
||||
repeat with a couple of byte errors (CRC fails) even though the RF is
|
||||
present and correctly timed. Burst-envelope analysis is immune to that.
|
||||
|
||||
This does NOT prove the repeat is decodable as C1 -- that is proven
|
||||
separately by the synthetic-burst phase (hil/verify_tx.py). Together they
|
||||
close the loop: framing is decodable, and real repeats are emitted with the
|
||||
right timing and duration.
|
||||
Method: extract RF bursts from the IQ envelope, then find the single clock
|
||||
offset between the firmware's monotonic microsecond timestamps and the IQ
|
||||
timeline that maximises the number of firmware TX events landing on a
|
||||
burst. No amplitude threshold is assumed for attribution -- the alignment
|
||||
itself is the evidence, and a wrong alignment cannot match many events.
|
||||
|
||||
Reports:
|
||||
- how many firmware TX events have corresponding RF in the IQ
|
||||
- the measured original -> repeat spacing for repeats whose source RX
|
||||
event is known, compared against the configured delay
|
||||
- the received level of our transmissions vs meter traffic
|
||||
|
||||
Usage:
|
||||
verify_repeats_iq.py --iq iq.raw --serial serial.log --delay-ms 200
|
||||
[--rate 1600000] [--tol-ms 40] [--json]
|
||||
[--rate 1600000] [--tol-ms 15] [--json]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
@@ -28,35 +34,43 @@ ap.add_argument("--iq", required=True)
|
||||
ap.add_argument("--serial", required=True)
|
||||
ap.add_argument("--delay-ms", type=float, required=True)
|
||||
ap.add_argument("--rate", type=int, default=1_600_000)
|
||||
ap.add_argument("--tol-ms", type=float, default=40.0)
|
||||
ap.add_argument("--strong-db", type=float, default=25.0,
|
||||
help="bursts this far over floor are attributed to the DUT")
|
||||
ap.add_argument("--tol-ms", type=float, default=15.0)
|
||||
ap.add_argument("--json", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
WIN = args.rate // 1000 # 1 ms
|
||||
WIN = args.rate // 1000 # 1 ms resolution
|
||||
|
||||
env = []
|
||||
with open(args.iq, "rb") as f:
|
||||
while True:
|
||||
raw = f.read(args.rate * 2)
|
||||
if len(raw) < WIN * 2:
|
||||
break
|
||||
b = np.frombuffer(raw, dtype=np.uint8).astype(np.float32) - 127.5
|
||||
iq = b[0::2] + 1j * b[1::2]
|
||||
p = np.abs(iq) ** 2
|
||||
p = p[: (len(p) // WIN) * WIN].reshape(-1, WIN).mean(axis=1)
|
||||
env.append(p)
|
||||
|
||||
if not env:
|
||||
print(json.dumps({"error": "iq too short"}) if args.json else "ERROR: iq too short")
|
||||
# ---------------------------------------------------------------- IQ bursts
|
||||
def load_envelope(path):
|
||||
env, clipped = [], 0
|
||||
with open(path, "rb") as f:
|
||||
while True:
|
||||
raw = f.read(args.rate * 2)
|
||||
if len(raw) < WIN * 2:
|
||||
break
|
||||
u8 = np.frombuffer(raw, dtype=np.uint8)
|
||||
clipped += int(np.count_nonzero((u8 == 0) | (u8 == 255)))
|
||||
b = u8.astype(np.float32) - 127.5
|
||||
iq = b[0::2] + 1j * b[1::2]
|
||||
p = np.abs(iq) ** 2
|
||||
p = p[: (len(p) // WIN) * WIN].reshape(-1, WIN).mean(axis=1)
|
||||
env.append(p)
|
||||
if not env:
|
||||
return None, 0
|
||||
return np.concatenate(env), clipped
|
||||
|
||||
|
||||
power, clipped_samples = load_envelope(args.iq)
|
||||
if power is None:
|
||||
msg = {"error": "iq too short"}
|
||||
print(json.dumps(msg) if args.json else "ERROR: iq too short")
|
||||
sys.exit(3)
|
||||
|
||||
db = 10 * np.log10(np.concatenate(env) + 1e-9)
|
||||
db = 10 * np.log10(power + 1e-9)
|
||||
floor = float(np.median(db))
|
||||
|
||||
# any burst at all: 8 dB over floor; DUT bursts: --strong-db over floor
|
||||
idx = np.where(db > floor + 8.0)[0]
|
||||
idx = np.where(db > floor + 6.0)[0]
|
||||
bursts = []
|
||||
if len(idx):
|
||||
start = prev = idx[0]
|
||||
@@ -67,36 +81,15 @@ if len(idx):
|
||||
prev = i
|
||||
bursts.append((start, prev))
|
||||
|
||||
burst_start = np.array([s for s, _ in bursts], dtype=np.int64)
|
||||
burst_info = [{
|
||||
"t_ms": int(s),
|
||||
"duration_ms": int(e - s + 1),
|
||||
"peak_db_over_floor": float(db[s:e + 1].max() - floor),
|
||||
"peak_db": float(db[s:e + 1].max() - floor),
|
||||
} for s, e in bursts]
|
||||
|
||||
# Pair: any burst followed by a STRONG burst at delay +/- tol
|
||||
lo = args.delay_ms - args.tol_ms
|
||||
hi = args.delay_ms + args.tol_ms
|
||||
pairs = []
|
||||
for i, a in enumerate(burst_info):
|
||||
a_end = a["t_ms"] + a["duration_ms"]
|
||||
for b in burst_info[i + 1:]:
|
||||
gap = b["t_ms"] - a_end
|
||||
if gap > hi:
|
||||
break
|
||||
if lo <= gap <= hi and b["peak_db_over_floor"] >= args.strong_db:
|
||||
pairs.append({
|
||||
"original_t_ms": a["t_ms"],
|
||||
"repeat_t_ms": b["t_ms"],
|
||||
"gap_ms": int(gap),
|
||||
"original_peak_db": round(a["peak_db_over_floor"], 1),
|
||||
"repeat_peak_db": round(b["peak_db_over_floor"], 1),
|
||||
"repeat_duration_ms": b["duration_ms"],
|
||||
})
|
||||
break
|
||||
|
||||
# firmware ground truth for the same window
|
||||
tx_ok = 0
|
||||
rx_ok = 0
|
||||
# ------------------------------------------------------------ firmware log
|
||||
tx_events, rx_events = [], []
|
||||
for line in open(args.serial, errors="replace"):
|
||||
line = line.strip()
|
||||
if not line.startswith("{"):
|
||||
@@ -106,42 +99,115 @@ for line in open(args.serial, errors="replace"):
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if ev.get("event") == "wmbus_tx" and str(ev.get("result", "")).startswith("ok"):
|
||||
tx_ok += 1
|
||||
tx_events.append(ev)
|
||||
elif ev.get("event") == "wmbus_rx" and ev.get("crc_ok"):
|
||||
rx_ok += 1
|
||||
rx_events.append(ev)
|
||||
|
||||
if not tx_events or not bursts:
|
||||
result = {
|
||||
"iq_seconds": round(len(db) / 1000.0, 1),
|
||||
"noise_floor_db": round(floor, 1),
|
||||
"bursts_total": len(burst_info),
|
||||
"firmware_tx_ok": len(tx_events),
|
||||
"tx_events_with_rf": 0,
|
||||
"note": "no TX events or no bursts in this window",
|
||||
}
|
||||
print(json.dumps(result, indent=2) if args.json
|
||||
else f"IQ_REPEAT_PROOF: no TX events or no bursts "
|
||||
f"(tx={len(tx_events)} bursts={len(burst_info)})")
|
||||
sys.exit(1)
|
||||
|
||||
tx_ms = np.array([ev["timestamp_us"] / 1000.0 for ev in tx_events])
|
||||
|
||||
# ------------------------------------------------- align firmware to IQ
|
||||
# Candidate offsets: every (burst - tx_event) difference. The true offset
|
||||
# appears once per genuinely observed transmission, so it wins by count.
|
||||
cands = np.subtract.outer(burst_start.astype(float), tx_ms).ravel()
|
||||
best_off, best_hits = 0.0, -1
|
||||
for off in cands:
|
||||
d = np.abs(burst_start[:, None] - (tx_ms + off)[None, :])
|
||||
hits = int(np.count_nonzero(d.min(axis=0) <= args.tol_ms))
|
||||
if hits > best_hits:
|
||||
best_hits, best_off = hits, float(off)
|
||||
|
||||
# match each TX event to its nearest burst under the winning offset
|
||||
matched, our_peaks = [], []
|
||||
for ev, t in zip(tx_events, tx_ms):
|
||||
target = t + best_off
|
||||
if not len(burst_start):
|
||||
continue
|
||||
j = int(np.argmin(np.abs(burst_start - target)))
|
||||
err = float(burst_start[j] - target)
|
||||
if abs(err) <= args.tol_ms:
|
||||
matched.append({
|
||||
"hash": ev.get("hash"),
|
||||
"frame_len": ev.get("length"),
|
||||
"iq_t_ms": int(burst_start[j]),
|
||||
"align_error_ms": round(err, 1),
|
||||
"burst_duration_ms": burst_info[j]["duration_ms"],
|
||||
"burst_peak_db": round(burst_info[j]["peak_db"], 1),
|
||||
"delay_us": ev.get("delay_us"),
|
||||
})
|
||||
our_peaks.append(burst_info[j]["peak_db"])
|
||||
|
||||
ours_t = {m["iq_t_ms"] for m in matched}
|
||||
other_peaks = [b["peak_db"] for b in burst_info if b["t_ms"] not in ours_t]
|
||||
|
||||
# expected airtime: (frame + 2 mode bytes) * 80 us + preamble/sync ~0.64 ms
|
||||
def expected_ms(n):
|
||||
return ((n + 2) * 80 + 640) / 1000.0 if n else None
|
||||
|
||||
|
||||
dur_err = [m["burst_duration_ms"] - expected_ms(m["frame_len"])
|
||||
for m in matched if m["frame_len"]]
|
||||
delays = sorted(m["delay_us"] / 1000.0 for m in matched
|
||||
if m.get("delay_us"))
|
||||
|
||||
|
||||
def stat(v):
|
||||
if not v:
|
||||
return {"min": None, "median": None, "max": None}
|
||||
v = sorted(v)
|
||||
return {"min": round(v[0], 1),
|
||||
"median": round(v[len(v) // 2], 1),
|
||||
"max": round(v[-1], 1)}
|
||||
|
||||
|
||||
strong = [b for b in burst_info if b["peak_db_over_floor"] >= args.strong_db]
|
||||
gaps = sorted(p["gap_ms"] for p in pairs)
|
||||
result = {
|
||||
"iq_seconds": round(len(db) / 1000.0, 1),
|
||||
"noise_floor_db": round(floor, 1),
|
||||
"clipped_sample_pct": round(100.0 * clipped_samples /
|
||||
max(1, len(db) * WIN * 2), 4),
|
||||
"bursts_total": len(burst_info),
|
||||
"bursts_strong_dut": len(strong),
|
||||
"firmware_tx_ok": tx_ok,
|
||||
"firmware_rx_crc_ok": rx_ok,
|
||||
"repeat_pairs_at_delay": len(pairs),
|
||||
"firmware_tx_ok": len(tx_events),
|
||||
"firmware_rx_crc_ok": len(rx_events),
|
||||
"tx_events_with_rf": len(matched),
|
||||
"tx_rf_confirmed_pct": round(100.0 * len(matched) / len(tx_events), 1),
|
||||
"clock_offset_ms": round(best_off, 1),
|
||||
"align_error_ms": stat([abs(m["align_error_ms"]) for m in matched]),
|
||||
"burst_duration_error_ms": stat(dur_err),
|
||||
"our_tx_peak_db_over_floor": stat(our_peaks),
|
||||
"other_burst_peak_db_over_floor": stat(other_peaks),
|
||||
"firmware_repeat_delay_ms": stat(delays),
|
||||
"expected_delay_ms": args.delay_ms,
|
||||
"gap_ms": {
|
||||
"min": gaps[0] if gaps else None,
|
||||
"median": gaps[len(gaps) // 2] if gaps else None,
|
||||
"max": gaps[-1] if gaps else None,
|
||||
},
|
||||
"pairs": pairs[:50],
|
||||
"matched": matched[:50],
|
||||
}
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(result, indent=2))
|
||||
else:
|
||||
print(f"IQ_REPEAT_PROOF iq={result['iq_seconds']}s floor={floor:.1f}dB")
|
||||
print(f" bursts total / strong(DUT): {len(burst_info)} / {len(strong)}")
|
||||
print(f" firmware TX ok / RX crc-ok: {tx_ok} / {rx_ok}")
|
||||
print(f" repeat pairs at {args.delay_ms:.0f}+-{args.tol_ms:.0f} ms: {len(pairs)}")
|
||||
if gaps:
|
||||
print(f" observed gap min/med/max: "
|
||||
f"{gaps[0]}/{gaps[len(gaps)//2]}/{gaps[-1]} ms")
|
||||
for p in pairs[:10]:
|
||||
print(f" t={p['original_t_ms']/1000:.3f}s -> +{p['gap_ms']}ms "
|
||||
f"({p['original_peak_db']:+.1f} dB -> {p['repeat_peak_db']:+.1f} dB, "
|
||||
f"{p['repeat_duration_ms']} ms)")
|
||||
print(f"IQ_REPEAT_PROOF iq={result['iq_seconds']}s floor={floor:.1f}dB "
|
||||
f"clipped={result['clipped_sample_pct']}%")
|
||||
print(f" bursts in window: {len(burst_info)}")
|
||||
print(f" firmware TX ok / RX crc-ok: {len(tx_events)} / {len(rx_events)}")
|
||||
print(f" TX events with RF in IQ: {len(matched)}/{len(tx_events)} "
|
||||
f"({result['tx_rf_confirmed_pct']} %)")
|
||||
print(f" clock offset: {best_off:.0f} ms")
|
||||
print(f" alignment error min/med/max: {result['align_error_ms']}")
|
||||
print(f" duration error vs airtime: {result['burst_duration_error_ms']} ms")
|
||||
print(f" our TX level over floor: {result['our_tx_peak_db_over_floor']} dB")
|
||||
print(f" other bursts over floor: {result['other_burst_peak_db_over_floor']} dB")
|
||||
print(f" firmware repeat delay: {result['firmware_repeat_delay_ms']} ms "
|
||||
f"(configured {args.delay_ms:.0f})")
|
||||
|
||||
sys.exit(0 if pairs else 1)
|
||||
sys.exit(0 if matched else 1)
|
||||
|
||||
Reference in New Issue
Block a user