224 lines
8.4 KiB
Python
Executable File
224 lines
8.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Verify repeater transmissions from raw IQ, independent of packet decoding.
|
|
|
|
Why this exists: on this bench the Heltec transmission arrives at the SDR
|
|
far stronger than real meter traffic, and the dongle's own sensitivity
|
|
drifts by ~17 dB over minutes. rtl-wmbus therefore often receives our
|
|
repeat with a couple of byte errors (CRC fails) even though the RF is
|
|
present and correctly timed. Burst-envelope analysis is immune to that.
|
|
|
|
Method: extract RF bursts from the IQ envelope, then find the single clock
|
|
offset between the firmware's monotonic microsecond timestamps and the IQ
|
|
timeline that maximises the number of firmware TX events landing on a
|
|
burst. No amplitude threshold is assumed for attribution -- the alignment
|
|
itself is the evidence, and a wrong alignment cannot match many events.
|
|
|
|
Reports:
|
|
- how many firmware TX events have corresponding RF in the IQ
|
|
- the measured original -> repeat spacing for repeats whose source RX
|
|
event is known, compared against the configured delay
|
|
- the received level of our transmissions vs meter traffic
|
|
|
|
Usage:
|
|
verify_repeats_iq.py --iq iq.raw --serial serial.log --delay-ms 200
|
|
[--rate 1600000] [--tol-ms 15] [--json]
|
|
"""
|
|
import argparse
|
|
import json
|
|
import sys
|
|
|
|
import numpy as np
|
|
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--iq", required=True)
|
|
ap.add_argument("--serial", required=True)
|
|
ap.add_argument("--delay-ms", type=float, required=True)
|
|
ap.add_argument("--rate", type=int, default=1_600_000)
|
|
ap.add_argument("--tol-ms", type=float, default=15.0)
|
|
ap.add_argument("--json", action="store_true")
|
|
args = ap.parse_args()
|
|
|
|
WIN = args.rate // 1000 # 1 ms resolution
|
|
|
|
|
|
# ---------------------------------------------------------------- IQ bursts
|
|
def load_envelope(path):
|
|
env, clipped = [], 0
|
|
with open(path, "rb") as f:
|
|
while True:
|
|
raw = f.read(args.rate * 2)
|
|
if len(raw) < WIN * 2:
|
|
break
|
|
u8 = np.frombuffer(raw, dtype=np.uint8)
|
|
clipped += int(np.count_nonzero((u8 == 0) | (u8 == 255)))
|
|
b = u8.astype(np.float32) - 127.5
|
|
iq = b[0::2] + 1j * b[1::2]
|
|
p = np.abs(iq) ** 2
|
|
p = p[: (len(p) // WIN) * WIN].reshape(-1, WIN).mean(axis=1)
|
|
env.append(p)
|
|
if not env:
|
|
return None, 0
|
|
return np.concatenate(env), clipped
|
|
|
|
|
|
power, clipped_samples = load_envelope(args.iq)
|
|
if power is None:
|
|
msg = {"error": "iq too short"}
|
|
print(json.dumps(msg) if args.json else "ERROR: iq too short")
|
|
sys.exit(3)
|
|
|
|
db = 10 * np.log10(power + 1e-9)
|
|
floor = float(np.median(db))
|
|
|
|
idx = np.where(db > floor + 6.0)[0]
|
|
bursts = []
|
|
if len(idx):
|
|
start = prev = idx[0]
|
|
for i in idx[1:]:
|
|
if i > prev + 3:
|
|
bursts.append((start, prev))
|
|
start = i
|
|
prev = i
|
|
bursts.append((start, prev))
|
|
|
|
burst_start = np.array([s for s, _ in bursts], dtype=np.int64)
|
|
burst_info = [{
|
|
"t_ms": int(s),
|
|
"duration_ms": int(e - s + 1),
|
|
"peak_db": float(db[s:e + 1].max() - floor),
|
|
} for s, e in bursts]
|
|
|
|
# ------------------------------------------------------------ firmware log
|
|
tx_events, rx_events = [], []
|
|
for line in open(args.serial, errors="replace"):
|
|
line = line.strip()
|
|
if not line.startswith("{"):
|
|
continue
|
|
try:
|
|
ev = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if ev.get("event") == "wmbus_tx" and str(ev.get("result", "")).startswith("ok"):
|
|
tx_events.append(ev)
|
|
elif ev.get("event") == "wmbus_rx" and ev.get("crc_ok"):
|
|
rx_events.append(ev)
|
|
|
|
if not tx_events or not bursts:
|
|
result = {
|
|
"iq_seconds": round(len(db) / 1000.0, 1),
|
|
"noise_floor_db": round(floor, 1),
|
|
"bursts_total": len(burst_info),
|
|
"firmware_tx_ok": len(tx_events),
|
|
"tx_events_with_rf": 0,
|
|
"note": "no TX events or no bursts in this window",
|
|
}
|
|
print(json.dumps(result, indent=2) if args.json
|
|
else f"IQ_REPEAT_PROOF: no TX events or no bursts "
|
|
f"(tx={len(tx_events)} bursts={len(burst_info)})")
|
|
sys.exit(1)
|
|
|
|
tx_ms = np.array([ev["timestamp_us"] / 1000.0 for ev in tx_events])
|
|
|
|
# ------------------------------------------------- align firmware to IQ
|
|
# Candidate offsets: every (burst - tx_event) difference. The true offset
|
|
# appears once per genuinely observed transmission, so it wins by count.
|
|
cands = np.subtract.outer(burst_start.astype(float), tx_ms).ravel()
|
|
best_off, best_hits = 0.0, -1
|
|
for off in cands:
|
|
d = np.abs(burst_start[:, None] - (tx_ms + off)[None, :])
|
|
hits = int(np.count_nonzero(d.min(axis=0) <= args.tol_ms))
|
|
if hits > best_hits:
|
|
best_hits, best_off = hits, float(off)
|
|
|
|
# Match each TX event to its nearest burst under the winning offset.
|
|
# The IQ may cover only part of the serial log (bounded IQ segments, or a
|
|
# tuner fault mid-run), so TX events whose aligned time falls outside the
|
|
# IQ timeline are "not observable" rather than "not observed" -- counting
|
|
# them as failures would understate the result.
|
|
iq_span_ms = len(db)
|
|
matched, our_peaks, in_window = [], [], 0
|
|
for ev, t in zip(tx_events, tx_ms):
|
|
target = t + best_off
|
|
if target < 0 or target > iq_span_ms:
|
|
continue
|
|
in_window += 1
|
|
if not len(burst_start):
|
|
continue
|
|
j = int(np.argmin(np.abs(burst_start - target)))
|
|
err = float(burst_start[j] - target)
|
|
if abs(err) <= args.tol_ms:
|
|
matched.append({
|
|
"hash": ev.get("hash"),
|
|
"frame_len": ev.get("length"),
|
|
"iq_t_ms": int(burst_start[j]),
|
|
"align_error_ms": round(err, 1),
|
|
"burst_duration_ms": burst_info[j]["duration_ms"],
|
|
"burst_peak_db": round(burst_info[j]["peak_db"], 1),
|
|
"delay_us": ev.get("delay_us"),
|
|
})
|
|
our_peaks.append(burst_info[j]["peak_db"])
|
|
|
|
ours_t = {m["iq_t_ms"] for m in matched}
|
|
other_peaks = [b["peak_db"] for b in burst_info if b["t_ms"] not in ours_t]
|
|
|
|
# expected airtime: (frame + 2 mode bytes) * 80 us + preamble/sync ~0.64 ms
|
|
def expected_ms(n):
|
|
return ((n + 2) * 80 + 640) / 1000.0 if n else None
|
|
|
|
|
|
dur_err = [m["burst_duration_ms"] - expected_ms(m["frame_len"])
|
|
for m in matched if m["frame_len"]]
|
|
delays = sorted(m["delay_us"] / 1000.0 for m in matched
|
|
if m.get("delay_us"))
|
|
|
|
|
|
def stat(v):
|
|
if not v:
|
|
return {"min": None, "median": None, "max": None}
|
|
v = sorted(v)
|
|
return {"min": round(v[0], 1),
|
|
"median": round(v[len(v) // 2], 1),
|
|
"max": round(v[-1], 1)}
|
|
|
|
|
|
result = {
|
|
"iq_seconds": round(len(db) / 1000.0, 1),
|
|
"noise_floor_db": round(floor, 1),
|
|
"clipped_sample_pct": round(100.0 * clipped_samples /
|
|
max(1, len(db) * WIN * 2), 4),
|
|
"bursts_total": len(burst_info),
|
|
"firmware_tx_ok": len(tx_events),
|
|
"firmware_rx_crc_ok": len(rx_events),
|
|
"tx_events_in_iq_window": in_window,
|
|
"tx_events_with_rf": len(matched),
|
|
"tx_rf_confirmed_pct": round(100.0 * len(matched) / in_window, 1) if in_window else None,
|
|
"clock_offset_ms": round(best_off, 1),
|
|
"align_error_ms": stat([abs(m["align_error_ms"]) for m in matched]),
|
|
"burst_duration_error_ms": stat(dur_err),
|
|
"our_tx_peak_db_over_floor": stat(our_peaks),
|
|
"other_burst_peak_db_over_floor": stat(other_peaks),
|
|
"firmware_repeat_delay_ms": stat(delays),
|
|
"expected_delay_ms": args.delay_ms,
|
|
"matched": matched[:50],
|
|
}
|
|
|
|
if args.json:
|
|
print(json.dumps(result, indent=2))
|
|
else:
|
|
print(f"IQ_REPEAT_PROOF iq={result['iq_seconds']}s floor={floor:.1f}dB "
|
|
f"clipped={result['clipped_sample_pct']}%")
|
|
print(f" bursts in window: {len(burst_info)}")
|
|
print(f" firmware TX ok / RX crc-ok: {len(tx_events)} / {len(rx_events)}")
|
|
print(f" TX events inside IQ window: {in_window} of {len(tx_events)}")
|
|
print(f" TX events with RF in IQ: {len(matched)}/{in_window} "
|
|
f"({result['tx_rf_confirmed_pct']} %)")
|
|
print(f" clock offset: {best_off:.0f} ms")
|
|
print(f" alignment error min/med/max: {result['align_error_ms']}")
|
|
print(f" duration error vs airtime: {result['burst_duration_error_ms']} ms")
|
|
print(f" our TX level over floor: {result['our_tx_peak_db_over_floor']} dB")
|
|
print(f" other bursts over floor: {result['other_burst_peak_db_over_floor']} dB")
|
|
print(f" firmware repeat delay: {result['firmware_repeat_delay_ms']} ms "
|
|
f"(configured {args.delay_ms:.0f})")
|
|
|
|
sys.exit(0 if matched else 1)
|