#!/usr/bin/env python3 """Correlate Heltec serial events with SDR (rtl_wmbus) captures. Inputs: --serial serial.log JSON-lines from the Heltec ({"event": ...}) --sdr rtlwmbus.log rtl_wmbus CSV-ish lines: [algo]C1;crc_ok;3of6ok;timestamp;pkt_rssi;rssi;serial;0xHEX Output: JSON report on stdout. Correlation model (developer repeat delay ~50 ms): SDR original <-> Heltec RX : same cooked bytes, |dt| < rx_window Heltec RX <-> Heltec TX : same hash (firmware log) Heltec TX <-> SDR repeat : same cooked bytes, dt in [min_delay, max_delay] after the SDR original The Heltec logs raw over-the-air frames (with CRCs); rtl_wmbus prints "cooked" frames (CRCs stripped). cook() below mirrors rtl-wmbus. """ import argparse import json import re import sys from datetime import datetime def cook_a(frame: bytes): """Strip block CRCs from a format A over-the-air frame.""" if len(frame) < 12: return None out = bytearray(frame[:10]) pos = 12 while pos < len(frame): n = min(16, len(frame) - pos - 2) if n <= 0: return None out += frame[pos:pos + n] pos += n + 2 return bytes(out) def cook_b(frame: bytes): """Strip CRCs from a format B frame and rewrite L (mirrors rtl-wmbus).""" if len(frame) < 12: return None out = bytearray() rem = frame while rem: if len(rem) >= 128: out += rem[:126] rem = rem[128:] else: out += rem[:-2] rem = b"" # rewrite L: original L includes CRC bytes; cooked excludes them out[0] = len(out) - 1 return bytes(out) def parse_serial(path): """Heltec JSON-lines -> (rx_events, tx_events, other).""" rx, tx, other = [], [], [] with open(path, "r", errors="replace") as f: for line in f: line = line.strip() if not line.startswith("{"): continue try: ev = json.loads(line) except json.JSONDecodeError: continue e = ev.get("event") if e == "wmbus_rx": raw = bytes.fromhex(ev.get("raw", "")) fmt = ev.get("format", "A") cooked = cook_a(raw) if fmt == "A" else cook_b(raw) ev["_raw"] = raw ev["_cooked"] = cooked rx.append(ev) elif e == "wmbus_tx": tx.append(ev) else: other.append(ev) return rx, tx, other SDR_RE = re.compile( r"^(?:[a-z0-9_]+;)?(?P[TCS]1);(?P\d);(?P\d);" r"(?P[\d\- :.]+);(?P\d+);(?P\d+);" r"(?P[0-9A-Fa-f]{8});0x(?P[0-9a-fA-F]+)" ) def parse_sdr(path): frames = [] with open(path, "r", errors="replace") as f: for line in f: m = SDR_RE.match(line.strip()) if not m: continue ts = datetime.strptime(m["ts"].strip(), "%Y-%m-%d %H:%M:%S.%f") frames.append({ "mode": m["mode"], "crc_ok": m["crc"] == "1", "t": ts.timestamp(), "serial": m["serial"].lower(), "cooked": bytes.fromhex(m["hex"]), }) return frames def main(): ap = argparse.ArgumentParser() ap.add_argument("--serial", required=True) ap.add_argument("--sdr", required=True) ap.add_argument("--min-delay", type=float, default=0.010) ap.add_argument("--max-delay", type=float, default=0.300) ap.add_argument("--rx-window", type=float, default=2.0, help="wall/mono clock skew tolerance for RX matching") args = ap.parse_args() heltec_rx, heltec_tx, other = parse_serial(args.serial) sdr = parse_sdr(args.sdr) sdr_c1 = [s for s in sdr if s["mode"] == "C1" and s["crc_ok"]] # rtl_wmbus runs two demodulator algorithms concurrently; a single burst # can be reported twice within ~1 ms. Collapse identical bytes < 5 ms apart. deduped = [] for s in sorted(sdr_c1, key=lambda x: x["t"]): if deduped and deduped[-1]["cooked"] == s["cooked"] \ and s["t"] - deduped[-1]["t"] < 0.005: continue deduped.append(s) sdr_c1 = deduped # --- classify SDR frames into originals vs repeats --------------------- # A repeat is an SDR frame with identical cooked bytes to an earlier SDR # frame within [min_delay, max_delay]. Everything else is an original. originals, repeats = [], [] for i, s in enumerate(sdr_c1): matched = None for j in range(i - 1, -1, -1): prev = sdr_c1[j] dt = s["t"] - prev["t"] if dt > args.max_delay: break if prev["cooked"] == s["cooked"] and args.min_delay <= dt <= args.max_delay: matched = (j, dt) break if matched is not None: repeats.append({"idx": i, "orig_idx": matched[0], "dt": matched[1], **s}) else: originals.append({"idx": i, **s}) # --- Heltec RX <-> SDR originals (exact cooked-bytes match) ------------ # Serial timestamps are monotonic-us; SDR timestamps are wall clock. # Estimate offset from the best byte-match pairs. matched_rx = 0 per_meter = {} used_rx = set() for o in originals: ser = o["serial"] pm = per_meter.setdefault(ser, {"sdr": 0, "heltec": 0, "matched": 0}) pm["sdr"] += 1 for k, r in enumerate(heltec_rx): if k in used_rx or r["_cooked"] != o["cooked"]: continue used_rx.add(k) matched_rx += 1 pm["matched"] += 1 o["heltec_rx"] = r break for r in heltec_rx: if r["_cooked"] is None: continue ser = r["_cooked"][4:8][::-1].hex() if len(r["_cooked"]) >= 8 else "?" per_meter.setdefault(ser, {"sdr": 0, "heltec": 0, "matched": 0}) per_meter[ser]["heltec"] += 1 # --- Heltec TX <-> SDR repeats ----------------------------------------- tx_by_hash = {} for t in heltec_tx: tx_by_hash.setdefault(t.get("hash"), []).append(t) verified_repeats = 0 for rep in repeats: orig = sdr_c1[rep["orig_idx"]] o_entry = next((o for o in originals if o["idx"] == rep["orig_idx"]), None) hr = o_entry.get("heltec_rx") if o_entry else None if hr and hr.get("hash") in tx_by_hash: verified_repeats += 1 rep["verified"] = True tx_ok = [t for t in heltec_tx if t.get("result") == "ok"] delays = [r["dt"] * 1000 for r in repeats] delays.sort() def pct(a, b): return round(100.0 * a / b, 1) if b else None report = { "sdr_frames_total": len(sdr), "sdr_c1_crc_ok": len(sdr_c1), "sdr_originals": len(originals), "sdr_repeats": len(repeats), "heltec_rx": len(heltec_rx), "heltec_rx_crc_ok": sum(1 for r in heltec_rx if r.get("crc_ok")), "heltec_tx_attempts": len(heltec_tx), "heltec_tx_ok": len(tx_ok), "rx_matched": matched_rx, "rx_match_rate_pct": pct(matched_rx, len(originals)), "sdr_verified_repeats": len(repeats), "fully_correlated_repeats": verified_repeats, "repeat_success_rate_pct": pct(len(repeats), len(tx_ok)), "repeat_delay_ms": { "median": delays[len(delays) // 2] if delays else None, "min": delays[0] if delays else None, "max": delays[-1] if delays else None, }, "per_meter": per_meter, "firmware_events": { "recoveries": sum(1 for e in other if e.get("event") == "radio_recovery"), "boots": sum(1 for e in other if e.get("event") == "boot"), }, } json.dump(report, sys.stdout, indent=2) print() return 0 if __name__ == "__main__": sys.exit(main())