Files
heltec-repeater/hil/verify_repeats_iq.py
T

224 lines
8.4 KiB
Python
Executable File

#!/usr/bin/env python3
"""Verify repeater transmissions from raw IQ, independent of packet decoding.
Why this exists: on this bench the Heltec transmission arrives at the SDR
far stronger than real meter traffic, and the dongle's own sensitivity
drifts by ~17 dB over minutes. rtl-wmbus therefore often receives our
repeat with a couple of byte errors (CRC fails) even though the RF is
present and correctly timed. Burst-envelope analysis is immune to that.
Method: extract RF bursts from the IQ envelope, then find the single clock
offset between the firmware's monotonic microsecond timestamps and the IQ
timeline that maximises the number of firmware TX events landing on a
burst. No amplitude threshold is assumed for attribution -- the alignment
itself is the evidence, and a wrong alignment cannot match many events.
Reports:
- how many firmware TX events have corresponding RF in the IQ
- the measured original -> repeat spacing for repeats whose source RX
event is known, compared against the configured delay
- the received level of our transmissions vs meter traffic
Usage:
verify_repeats_iq.py --iq iq.raw --serial serial.log --delay-ms 200
[--rate 1600000] [--tol-ms 15] [--json]
"""
import argparse
import json
import sys
import numpy as np
ap = argparse.ArgumentParser()
ap.add_argument("--iq", required=True)
ap.add_argument("--serial", required=True)
ap.add_argument("--delay-ms", type=float, required=True)
ap.add_argument("--rate", type=int, default=1_600_000)
ap.add_argument("--tol-ms", type=float, default=15.0)
ap.add_argument("--json", action="store_true")
args = ap.parse_args()
WIN = args.rate // 1000 # 1 ms resolution
# ---------------------------------------------------------------- IQ bursts
def load_envelope(path):
env, clipped = [], 0
with open(path, "rb") as f:
while True:
raw = f.read(args.rate * 2)
if len(raw) < WIN * 2:
break
u8 = np.frombuffer(raw, dtype=np.uint8)
clipped += int(np.count_nonzero((u8 == 0) | (u8 == 255)))
b = u8.astype(np.float32) - 127.5
iq = b[0::2] + 1j * b[1::2]
p = np.abs(iq) ** 2
p = p[: (len(p) // WIN) * WIN].reshape(-1, WIN).mean(axis=1)
env.append(p)
if not env:
return None, 0
return np.concatenate(env), clipped
power, clipped_samples = load_envelope(args.iq)
if power is None:
msg = {"error": "iq too short"}
print(json.dumps(msg) if args.json else "ERROR: iq too short")
sys.exit(3)
db = 10 * np.log10(power + 1e-9)
floor = float(np.median(db))
idx = np.where(db > floor + 6.0)[0]
bursts = []
if len(idx):
start = prev = idx[0]
for i in idx[1:]:
if i > prev + 3:
bursts.append((start, prev))
start = i
prev = i
bursts.append((start, prev))
burst_start = np.array([s for s, _ in bursts], dtype=np.int64)
burst_info = [{
"t_ms": int(s),
"duration_ms": int(e - s + 1),
"peak_db": float(db[s:e + 1].max() - floor),
} for s, e in bursts]
# ------------------------------------------------------------ firmware log
tx_events, rx_events = [], []
for line in open(args.serial, errors="replace"):
line = line.strip()
if not line.startswith("{"):
continue
try:
ev = json.loads(line)
except json.JSONDecodeError:
continue
if ev.get("event") == "wmbus_tx" and str(ev.get("result", "")).startswith("ok"):
tx_events.append(ev)
elif ev.get("event") == "wmbus_rx" and ev.get("crc_ok"):
rx_events.append(ev)
if not tx_events or not bursts:
result = {
"iq_seconds": round(len(db) / 1000.0, 1),
"noise_floor_db": round(floor, 1),
"bursts_total": len(burst_info),
"firmware_tx_ok": len(tx_events),
"tx_events_with_rf": 0,
"note": "no TX events or no bursts in this window",
}
print(json.dumps(result, indent=2) if args.json
else f"IQ_REPEAT_PROOF: no TX events or no bursts "
f"(tx={len(tx_events)} bursts={len(burst_info)})")
sys.exit(1)
tx_ms = np.array([ev["timestamp_us"] / 1000.0 for ev in tx_events])
# ------------------------------------------------- align firmware to IQ
# Candidate offsets: every (burst - tx_event) difference. The true offset
# appears once per genuinely observed transmission, so it wins by count.
cands = np.subtract.outer(burst_start.astype(float), tx_ms).ravel()
best_off, best_hits = 0.0, -1
for off in cands:
d = np.abs(burst_start[:, None] - (tx_ms + off)[None, :])
hits = int(np.count_nonzero(d.min(axis=0) <= args.tol_ms))
if hits > best_hits:
best_hits, best_off = hits, float(off)
# Match each TX event to its nearest burst under the winning offset.
# The IQ may cover only part of the serial log (bounded IQ segments, or a
# tuner fault mid-run), so TX events whose aligned time falls outside the
# IQ timeline are "not observable" rather than "not observed" -- counting
# them as failures would understate the result.
iq_span_ms = len(db)
matched, our_peaks, in_window = [], [], 0
for ev, t in zip(tx_events, tx_ms):
target = t + best_off
if target < 0 or target > iq_span_ms:
continue
in_window += 1
if not len(burst_start):
continue
j = int(np.argmin(np.abs(burst_start - target)))
err = float(burst_start[j] - target)
if abs(err) <= args.tol_ms:
matched.append({
"hash": ev.get("hash"),
"frame_len": ev.get("length"),
"iq_t_ms": int(burst_start[j]),
"align_error_ms": round(err, 1),
"burst_duration_ms": burst_info[j]["duration_ms"],
"burst_peak_db": round(burst_info[j]["peak_db"], 1),
"delay_us": ev.get("delay_us"),
})
our_peaks.append(burst_info[j]["peak_db"])
ours_t = {m["iq_t_ms"] for m in matched}
other_peaks = [b["peak_db"] for b in burst_info if b["t_ms"] not in ours_t]
# expected airtime: (frame + 2 mode bytes) * 80 us + preamble/sync ~0.64 ms
def expected_ms(n):
return ((n + 2) * 80 + 640) / 1000.0 if n else None
dur_err = [m["burst_duration_ms"] - expected_ms(m["frame_len"])
for m in matched if m["frame_len"]]
delays = sorted(m["delay_us"] / 1000.0 for m in matched
if m.get("delay_us"))
def stat(v):
if not v:
return {"min": None, "median": None, "max": None}
v = sorted(v)
return {"min": round(v[0], 1),
"median": round(v[len(v) // 2], 1),
"max": round(v[-1], 1)}
result = {
"iq_seconds": round(len(db) / 1000.0, 1),
"noise_floor_db": round(floor, 1),
"clipped_sample_pct": round(100.0 * clipped_samples /
max(1, len(db) * WIN * 2), 4),
"bursts_total": len(burst_info),
"firmware_tx_ok": len(tx_events),
"firmware_rx_crc_ok": len(rx_events),
"tx_events_in_iq_window": in_window,
"tx_events_with_rf": len(matched),
"tx_rf_confirmed_pct": round(100.0 * len(matched) / in_window, 1) if in_window else None,
"clock_offset_ms": round(best_off, 1),
"align_error_ms": stat([abs(m["align_error_ms"]) for m in matched]),
"burst_duration_error_ms": stat(dur_err),
"our_tx_peak_db_over_floor": stat(our_peaks),
"other_burst_peak_db_over_floor": stat(other_peaks),
"firmware_repeat_delay_ms": stat(delays),
"expected_delay_ms": args.delay_ms,
"matched": matched[:50],
}
if args.json:
print(json.dumps(result, indent=2))
else:
print(f"IQ_REPEAT_PROOF iq={result['iq_seconds']}s floor={floor:.1f}dB "
f"clipped={result['clipped_sample_pct']}%")
print(f" bursts in window: {len(burst_info)}")
print(f" firmware TX ok / RX crc-ok: {len(tx_events)} / {len(rx_events)}")
print(f" TX events inside IQ window: {in_window} of {len(tx_events)}")
print(f" TX events with RF in IQ: {len(matched)}/{in_window} "
f"({result['tx_rf_confirmed_pct']} %)")
print(f" clock offset: {best_off:.0f} ms")
print(f" alignment error min/med/max: {result['align_error_ms']}")
print(f" duration error vs airtime: {result['burst_duration_error_ms']} ms")
print(f" our TX level over floor: {result['our_tx_peak_db_over_floor']} dB")
print(f" other bursts over floor: {result['other_burst_peak_db_over_floor']} dB")
print(f" firmware repeat delay: {result['firmware_repeat_delay_ms']} ms "
f"(configured {args.delay_ms:.0f})")
sys.exit(0 if matched else 1)