#!/usr/bin/env python3 """Correlate Heltec serial events with SDR (rtl_wmbus) captures. Inputs: --serial serial.log JSON-lines from the Heltec ({"event": ...}) --sdr rtlwmbus.log rtl_wmbus CSV-ish lines: [algo]C1;crc_ok;3of6ok;timestamp;pkt_rssi;rssi;serial;0xHEX Output: JSON report on stdout. Correlation model (developer repeat delay ~50 ms): SDR original <-> Heltec RX : same cooked bytes, |dt| < rx_window Heltec RX <-> Heltec TX : same hash (firmware log) SDR original <-> SDR repeat : only H changes 0 -> 1 on a supported header, dt in [min_delay, max_delay] The Heltec logs raw over-the-air frames (with CRCs); rtl_wmbus prints "cooked" frames (CRCs stripped). cook() below mirrors rtl-wmbus. """ import argparse import json import re import sys from datetime import datetime def total_len_a(lfield: int) -> int: """Over-the-air length of a format A frame, incl. L and all CRC bytes. Block 1 = L + C,M,A (10 bytes) + CRC; the remaining (L - 9) L-counted bytes go into blocks of at most 16, each followed by 2 CRC bytes. This matches firmware/main/wmbus.c; see the note there about rtl-wmbus differing for L = 10 (mod 16). """ if lfield < 9: return -1 return 1 + lfield + 2 * (1 + (lfield - 9 + 15) // 16) def cook_a(frame: bytes): """Strip block CRCs from a format A over-the-air frame.""" if len(frame) < 12: return None out = bytearray(frame[:10]) pos = 12 while pos < len(frame): n = min(16, len(frame) - pos - 2) if n <= 0: return None out += frame[pos:pos + n] pos += n + 2 return bytes(out) def cook_b(frame: bytes): """Strip CRCs from a format B frame and rewrite L (mirrors rtl-wmbus).""" if len(frame) < 12: return None out = bytearray() rem = frame while rem: if len(rem) >= 128: out += rem[:126] rem = rem[128:] else: out += rem[:-2] rem = b"" # rewrite L: original L includes CRC bytes; cooked excludes them out[0] = len(out) - 1 return bytes(out) def expected_repeat(cooked: bytes): """Return the H-marked counterpart of a supported CRC-stripped original. Both format A and rtl-wmbus-normalized format B use L = cooked length - 1. ELL hop control takes precedence over any encapsulated transport header. Unsupported layouts, H=1 and R=1 cannot originate a received repeat. """ if len(cooked) < 11 or cooked[0] != len(cooked) - 1: return None if cooked[1] not in (0x44, 0x46): return None ci = cooked[10] if ci in (0x8c, 0x8e): # CI + CC + ACC, optionally eight address bytes, then another CI. if len(cooked) < (14 if ci == 0x8c else 22): return None pos, h_mask = 11, 0x10 else: if ci in (0x67, 0x6e, 0x74, 0x7a, 0x7d, 0x7f, 0x88, 0x8a, 0x9e, 0xc1, 0xc4): pos = 13 elif ci in (0x68, 0x6f, 0x72, 0x75, 0x7c, 0x7e, 0x87, 0x8b, 0x9f, 0xc2, 0xc5): pos = 21 else: return None if len(cooked) <= pos + 1 or cooked[pos + 1] & 0x1f not in (0, 5): return None h_mask = 0x01 if cooked[pos] & (h_mask | 0x02): return None marked = bytearray(cooked) marked[pos] |= h_mask return bytes(marked) def parse_serial(path): """Heltec JSON-lines -> (rx_events, tx_events, other).""" rx, tx, other = [], [], [] with open(path, "r", errors="replace") as f: for line in f: line = line.strip() if not line.startswith("{"): continue try: ev = json.loads(line) except json.JSONDecodeError: continue e = ev.get("event") if e == "wmbus_rx": raw = bytes.fromhex(ev.get("raw", "")) fmt = ev.get("format", "A") cooked = cook_a(raw) if fmt == "A" else cook_b(raw) ev["_raw"] = raw ev["_cooked"] = cooked rx.append(ev) elif e == "wmbus_tx": tx.append(ev) else: other.append(ev) return rx, tx, other SDR_RE = re.compile( r"^(?:[a-z0-9_]+;)?(?P[TCS]1);(?P\d);(?P\d);" r"(?P[\d\- :.]+);(?P\d+);(?P\d+);" r"(?P[0-9A-Fa-f]{8});0x(?P[0-9a-fA-F]+)" ) def parse_sdr(path): frames = [] with open(path, "r", errors="replace") as f: for line in f: m = SDR_RE.match(line.strip()) if not m: continue ts = datetime.strptime(m["ts"].strip(), "%Y-%m-%d %H:%M:%S.%f") frames.append({ "mode": m["mode"], "crc_ok": m["crc"] == "1", "t": ts.timestamp(), "serial": m["serial"].lower(), "cooked": bytes.fromhex(m["hex"]), }) return frames def main(): ap = argparse.ArgumentParser() ap.add_argument("--serial", required=True) ap.add_argument("--sdr", required=True) ap.add_argument("--min-delay", type=float, default=0.010) ap.add_argument("--max-delay", type=float, default=0.300) ap.add_argument("--rx-window", type=float, default=2.0, help="wall/mono clock skew tolerance for RX matching") args = ap.parse_args() heltec_rx, heltec_tx, other = parse_serial(args.serial) sdr = parse_sdr(args.sdr) sdr_frames = [s for s in sdr if s["mode"] in ("C1", "T1") and s["crc_ok"]] # rtl_wmbus runs two demodulator algorithms concurrently; a single burst # can be reported twice within ~1 ms. Collapse identical bytes < 5 ms apart. deduped = [] for s in sorted(sdr_frames, key=lambda x: x["t"]): if deduped and deduped[-1]["mode"] == s["mode"] \ and deduped[-1]["cooked"] == s["cooked"] \ and s["t"] - deduped[-1]["t"] < 0.005: continue deduped.append(s) sdr_frames = deduped expected = [expected_repeat(s["cooked"]) for s in sdr_frames] # --- classify SDR frames into originals vs repeats --------------------- # A repeat changes only H from 0 to 1 on a supported original header # within [min_delay, max_delay]. Same-H duplicates are not repeats. originals, repeats = [], [] used_originals = set() for i, s in enumerate(sdr_frames): matched = None for j in range(i - 1, -1, -1): prev = sdr_frames[j] dt = s["t"] - prev["t"] if dt > args.max_delay: break if (j not in used_originals and prev["mode"] == s["mode"] and expected[j] == s["cooked"] and args.min_delay <= dt <= args.max_delay): matched = (j, dt) used_originals.add(j) break if matched is not None: repeats.append({"idx": i, "orig_idx": matched[0], "dt": matched[1], **s}) else: originals.append({"idx": i, **s}) # --- Heltec RX <-> SDR originals (exact cooked-bytes match) ------------ # Serial timestamps are monotonic-us; SDR timestamps are wall clock. # Estimate offset from the best byte-match pairs. matched_rx = 0 per_meter = {} used_rx = set() for o in originals: ser = o["serial"] pm = per_meter.setdefault(ser, {"sdr": 0, "heltec": 0, "matched": 0}) pm["sdr"] += 1 for k, r in enumerate(heltec_rx): if k in used_rx or r["_cooked"] != o["cooked"]: continue used_rx.add(k) matched_rx += 1 pm["matched"] += 1 o["heltec_rx"] = r break for r in heltec_rx: if r["_cooked"] is None or not r.get("crc_ok"): continue ser = r["_cooked"][4:8][::-1].hex() if len(r["_cooked"]) >= 8 else "?" per_meter.setdefault(ser, {"sdr": 0, "heltec": 0, "matched": 0}) per_meter[ser]["heltec"] += 1 # --- Heltec TX <-> SDR repeats ----------------------------------------- tx_by_hash = {} for t in heltec_tx: tx_by_hash.setdefault(t.get("hash"), []).append(t) verified_repeats = 0 for rep in repeats: o_entry = next((o for o in originals if o["idx"] == rep["orig_idx"]), None) hr = o_entry.get("heltec_rx") if o_entry else None if hr and hr.get("hash") in tx_by_hash: verified_repeats += 1 rep["verified"] = True tx_ok = [t for t in heltec_tx if str(t.get("result", "")).startswith("ok")] delays = [r["dt"] * 1000 for r in repeats] delays.sort() def pct(a, b): return round(100.0 * a / b, 1) if b else None report = { "sdr_frames_total": len(sdr), "sdr_c1_crc_ok": sum(s["mode"] == "C1" for s in sdr_frames), "sdr_t1_crc_ok": sum(s["mode"] == "T1" for s in sdr_frames), "sdr_originals": len(originals), "sdr_repeats": len(repeats), "heltec_rx": len(heltec_rx), "heltec_rx_crc_ok": sum(1 for r in heltec_rx if r.get("crc_ok")), "heltec_tx_attempts": len(heltec_tx), "heltec_tx_ok": len(tx_ok), "rx_matched": matched_rx, "rx_match_rate_pct": pct(matched_rx, len(originals)), "sdr_verified_repeats": len(repeats), "fully_correlated_repeats": verified_repeats, "repeat_success_rate_pct": pct(len(repeats), len(tx_ok)), "repeat_delay_ms": { "median": delays[len(delays) // 2] if delays else None, "min": delays[0] if delays else None, "max": delays[-1] if delays else None, }, "per_meter": per_meter, "firmware_events": { "recoveries": sum(1 for e in other if e.get("event") == "radio_recovery"), "boots": sum(1 for e in other if e.get("event") == "boot"), }, } json.dump(report, sys.stdout, indent=2) print() return 0 if __name__ == "__main__": sys.exit(main())