Files
heltec-repeater/hil/correlate.py
T

233 lines
7.8 KiB
Python
Executable File

#!/usr/bin/env python3
"""Correlate Heltec serial events with SDR (rtl_wmbus) captures.
Inputs:
--serial serial.log JSON-lines from the Heltec ({"event": ...})
--sdr rtlwmbus.log rtl_wmbus CSV-ish lines:
[algo]C1;crc_ok;3of6ok;timestamp;pkt_rssi;rssi;serial;0xHEX
Output: JSON report on stdout.
Correlation model (developer repeat delay ~50 ms):
SDR original <-> Heltec RX : same cooked bytes, |dt| < rx_window
Heltec RX <-> Heltec TX : same hash (firmware log)
Heltec TX <-> SDR repeat : same cooked bytes, dt in [min_delay, max_delay]
after the SDR original
The Heltec logs raw over-the-air frames (with CRCs); rtl_wmbus prints
"cooked" frames (CRCs stripped). cook() below mirrors rtl-wmbus.
"""
import argparse
import json
import re
import sys
from datetime import datetime
def cook_a(frame: bytes):
"""Strip block CRCs from a format A over-the-air frame."""
if len(frame) < 12:
return None
out = bytearray(frame[:10])
pos = 12
while pos < len(frame):
n = min(16, len(frame) - pos - 2)
if n <= 0:
return None
out += frame[pos:pos + n]
pos += n + 2
return bytes(out)
def cook_b(frame: bytes):
"""Strip CRCs from a format B frame and rewrite L (mirrors rtl-wmbus)."""
if len(frame) < 12:
return None
out = bytearray()
rem = frame
while rem:
if len(rem) >= 128:
out += rem[:126]
rem = rem[128:]
else:
out += rem[:-2]
rem = b""
# rewrite L: original L includes CRC bytes; cooked excludes them
out[0] = len(out) - 1
return bytes(out)
def parse_serial(path):
"""Heltec JSON-lines -> (rx_events, tx_events, other)."""
rx, tx, other = [], [], []
with open(path, "r", errors="replace") as f:
for line in f:
line = line.strip()
if not line.startswith("{"):
continue
try:
ev = json.loads(line)
except json.JSONDecodeError:
continue
e = ev.get("event")
if e == "wmbus_rx":
raw = bytes.fromhex(ev.get("raw", ""))
fmt = ev.get("format", "A")
cooked = cook_a(raw) if fmt == "A" else cook_b(raw)
ev["_raw"] = raw
ev["_cooked"] = cooked
rx.append(ev)
elif e == "wmbus_tx":
tx.append(ev)
else:
other.append(ev)
return rx, tx, other
SDR_RE = re.compile(
r"^(?:[a-z0-9_]+;)?(?P<mode>[TCS]1);(?P<crc>\d);(?P<x>\d);"
r"(?P<ts>[\d\- :.]+);(?P<prssi>\d+);(?P<rssi>\d+);"
r"(?P<serial>[0-9A-Fa-f]{8});0x(?P<hex>[0-9a-fA-F]+)"
)
def parse_sdr(path):
frames = []
with open(path, "r", errors="replace") as f:
for line in f:
m = SDR_RE.match(line.strip())
if not m:
continue
ts = datetime.strptime(m["ts"].strip(), "%Y-%m-%d %H:%M:%S.%f")
frames.append({
"mode": m["mode"],
"crc_ok": m["crc"] == "1",
"t": ts.timestamp(),
"serial": m["serial"].lower(),
"cooked": bytes.fromhex(m["hex"]),
})
return frames
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--serial", required=True)
ap.add_argument("--sdr", required=True)
ap.add_argument("--min-delay", type=float, default=0.010)
ap.add_argument("--max-delay", type=float, default=0.300)
ap.add_argument("--rx-window", type=float, default=2.0,
help="wall/mono clock skew tolerance for RX matching")
args = ap.parse_args()
heltec_rx, heltec_tx, other = parse_serial(args.serial)
sdr = parse_sdr(args.sdr)
sdr_c1 = [s for s in sdr if s["mode"] == "C1" and s["crc_ok"]]
# rtl_wmbus runs two demodulator algorithms concurrently; a single burst
# can be reported twice within ~1 ms. Collapse identical bytes < 5 ms apart.
deduped = []
for s in sorted(sdr_c1, key=lambda x: x["t"]):
if deduped and deduped[-1]["cooked"] == s["cooked"] \
and s["t"] - deduped[-1]["t"] < 0.005:
continue
deduped.append(s)
sdr_c1 = deduped
# --- classify SDR frames into originals vs repeats ---------------------
# A repeat is an SDR frame with identical cooked bytes to an earlier SDR
# frame within [min_delay, max_delay]. Everything else is an original.
originals, repeats = [], []
for i, s in enumerate(sdr_c1):
matched = None
for j in range(i - 1, -1, -1):
prev = sdr_c1[j]
dt = s["t"] - prev["t"]
if dt > args.max_delay:
break
if prev["cooked"] == s["cooked"] and args.min_delay <= dt <= args.max_delay:
matched = (j, dt)
break
if matched is not None:
repeats.append({"idx": i, "orig_idx": matched[0], "dt": matched[1], **s})
else:
originals.append({"idx": i, **s})
# --- Heltec RX <-> SDR originals (exact cooked-bytes match) ------------
# Serial timestamps are monotonic-us; SDR timestamps are wall clock.
# Estimate offset from the best byte-match pairs.
matched_rx = 0
per_meter = {}
used_rx = set()
for o in originals:
ser = o["serial"]
pm = per_meter.setdefault(ser, {"sdr": 0, "heltec": 0, "matched": 0})
pm["sdr"] += 1
for k, r in enumerate(heltec_rx):
if k in used_rx or r["_cooked"] != o["cooked"]:
continue
used_rx.add(k)
matched_rx += 1
pm["matched"] += 1
o["heltec_rx"] = r
break
for r in heltec_rx:
if r["_cooked"] is None:
continue
ser = r["_cooked"][4:8][::-1].hex() if len(r["_cooked"]) >= 8 else "?"
per_meter.setdefault(ser, {"sdr": 0, "heltec": 0, "matched": 0})
per_meter[ser]["heltec"] += 1
# --- Heltec TX <-> SDR repeats -----------------------------------------
tx_by_hash = {}
for t in heltec_tx:
tx_by_hash.setdefault(t.get("hash"), []).append(t)
verified_repeats = 0
for rep in repeats:
orig = sdr_c1[rep["orig_idx"]]
o_entry = next((o for o in originals if o["idx"] == rep["orig_idx"]), None)
hr = o_entry.get("heltec_rx") if o_entry else None
if hr and hr.get("hash") in tx_by_hash:
verified_repeats += 1
rep["verified"] = True
tx_ok = [t for t in heltec_tx if t.get("result") == "ok"]
delays = [r["dt"] * 1000 for r in repeats]
delays.sort()
def pct(a, b):
return round(100.0 * a / b, 1) if b else None
report = {
"sdr_frames_total": len(sdr),
"sdr_c1_crc_ok": len(sdr_c1),
"sdr_originals": len(originals),
"sdr_repeats": len(repeats),
"heltec_rx": len(heltec_rx),
"heltec_rx_crc_ok": sum(1 for r in heltec_rx if r.get("crc_ok")),
"heltec_tx_attempts": len(heltec_tx),
"heltec_tx_ok": len(tx_ok),
"rx_matched": matched_rx,
"rx_match_rate_pct": pct(matched_rx, len(originals)),
"sdr_verified_repeats": len(repeats),
"fully_correlated_repeats": verified_repeats,
"repeat_success_rate_pct": pct(len(repeats), len(tx_ok)),
"repeat_delay_ms": {
"median": delays[len(delays) // 2] if delays else None,
"min": delays[0] if delays else None,
"max": delays[-1] if delays else None,
},
"per_meter": per_meter,
"firmware_events": {
"recoveries": sum(1 for e in other if e.get("event") == "radio_recovery"),
"boots": sum(1 for e in other if e.get("event") == "boot"),
},
}
json.dump(report, sys.stdout, indent=2)
print()
return 0
if __name__ == "__main__":
sys.exit(main())