246 lines
8.3 KiB
Python
Executable File
246 lines
8.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Correlate Heltec serial events with SDR (rtl_wmbus) captures.
|
|
|
|
Inputs:
|
|
--serial serial.log JSON-lines from the Heltec ({"event": ...})
|
|
--sdr rtlwmbus.log rtl_wmbus CSV-ish lines:
|
|
[algo]C1;crc_ok;3of6ok;timestamp;pkt_rssi;rssi;serial;0xHEX
|
|
Output: JSON report on stdout.
|
|
|
|
Correlation model (developer repeat delay ~50 ms):
|
|
SDR original <-> Heltec RX : same cooked bytes, |dt| < rx_window
|
|
Heltec RX <-> Heltec TX : same hash (firmware log)
|
|
Heltec TX <-> SDR repeat : same cooked bytes, dt in [min_delay, max_delay]
|
|
after the SDR original
|
|
|
|
The Heltec logs raw over-the-air frames (with CRCs); rtl_wmbus prints
|
|
"cooked" frames (CRCs stripped). cook() below mirrors rtl-wmbus.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import re
|
|
import sys
|
|
from datetime import datetime
|
|
|
|
|
|
def total_len_a(lfield: int) -> int:
|
|
"""Over-the-air length of a format A frame, incl. L and all CRC bytes.
|
|
|
|
Block 1 = L + C,M,A (10 bytes) + CRC; the remaining (L - 9) L-counted
|
|
bytes go into blocks of at most 16, each followed by 2 CRC bytes. This
|
|
matches firmware/main/wmbus.c; see the note there about rtl-wmbus
|
|
differing for L = 10 (mod 16).
|
|
"""
|
|
if lfield < 9:
|
|
return -1
|
|
return 1 + lfield + 2 * (1 + (lfield - 9 + 15) // 16)
|
|
|
|
|
|
def cook_a(frame: bytes):
|
|
"""Strip block CRCs from a format A over-the-air frame."""
|
|
if len(frame) < 12:
|
|
return None
|
|
out = bytearray(frame[:10])
|
|
pos = 12
|
|
while pos < len(frame):
|
|
n = min(16, len(frame) - pos - 2)
|
|
if n <= 0:
|
|
return None
|
|
out += frame[pos:pos + n]
|
|
pos += n + 2
|
|
return bytes(out)
|
|
|
|
|
|
def cook_b(frame: bytes):
|
|
"""Strip CRCs from a format B frame and rewrite L (mirrors rtl-wmbus)."""
|
|
if len(frame) < 12:
|
|
return None
|
|
out = bytearray()
|
|
rem = frame
|
|
while rem:
|
|
if len(rem) >= 128:
|
|
out += rem[:126]
|
|
rem = rem[128:]
|
|
else:
|
|
out += rem[:-2]
|
|
rem = b""
|
|
# rewrite L: original L includes CRC bytes; cooked excludes them
|
|
out[0] = len(out) - 1
|
|
return bytes(out)
|
|
|
|
|
|
def parse_serial(path):
|
|
"""Heltec JSON-lines -> (rx_events, tx_events, other)."""
|
|
rx, tx, other = [], [], []
|
|
with open(path, "r", errors="replace") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line.startswith("{"):
|
|
continue
|
|
try:
|
|
ev = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
e = ev.get("event")
|
|
if e == "wmbus_rx":
|
|
raw = bytes.fromhex(ev.get("raw", ""))
|
|
fmt = ev.get("format", "A")
|
|
cooked = cook_a(raw) if fmt == "A" else cook_b(raw)
|
|
ev["_raw"] = raw
|
|
ev["_cooked"] = cooked
|
|
rx.append(ev)
|
|
elif e == "wmbus_tx":
|
|
tx.append(ev)
|
|
else:
|
|
other.append(ev)
|
|
return rx, tx, other
|
|
|
|
|
|
SDR_RE = re.compile(
|
|
r"^(?:[a-z0-9_]+;)?(?P<mode>[TCS]1);(?P<crc>\d);(?P<x>\d);"
|
|
r"(?P<ts>[\d\- :.]+);(?P<prssi>\d+);(?P<rssi>\d+);"
|
|
r"(?P<serial>[0-9A-Fa-f]{8});0x(?P<hex>[0-9a-fA-F]+)"
|
|
)
|
|
|
|
|
|
def parse_sdr(path):
|
|
frames = []
|
|
with open(path, "r", errors="replace") as f:
|
|
for line in f:
|
|
m = SDR_RE.match(line.strip())
|
|
if not m:
|
|
continue
|
|
ts = datetime.strptime(m["ts"].strip(), "%Y-%m-%d %H:%M:%S.%f")
|
|
frames.append({
|
|
"mode": m["mode"],
|
|
"crc_ok": m["crc"] == "1",
|
|
"t": ts.timestamp(),
|
|
"serial": m["serial"].lower(),
|
|
"cooked": bytes.fromhex(m["hex"]),
|
|
})
|
|
return frames
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--serial", required=True)
|
|
ap.add_argument("--sdr", required=True)
|
|
ap.add_argument("--min-delay", type=float, default=0.010)
|
|
ap.add_argument("--max-delay", type=float, default=0.300)
|
|
ap.add_argument("--rx-window", type=float, default=2.0,
|
|
help="wall/mono clock skew tolerance for RX matching")
|
|
args = ap.parse_args()
|
|
|
|
heltec_rx, heltec_tx, other = parse_serial(args.serial)
|
|
sdr = parse_sdr(args.sdr)
|
|
sdr_c1 = [s for s in sdr if s["mode"] == "C1" and s["crc_ok"]]
|
|
|
|
# rtl_wmbus runs two demodulator algorithms concurrently; a single burst
|
|
# can be reported twice within ~1 ms. Collapse identical bytes < 5 ms apart.
|
|
deduped = []
|
|
for s in sorted(sdr_c1, key=lambda x: x["t"]):
|
|
if deduped and deduped[-1]["cooked"] == s["cooked"] \
|
|
and s["t"] - deduped[-1]["t"] < 0.005:
|
|
continue
|
|
deduped.append(s)
|
|
sdr_c1 = deduped
|
|
|
|
# --- classify SDR frames into originals vs repeats ---------------------
|
|
# A repeat is an SDR frame with identical cooked bytes to an earlier SDR
|
|
# frame within [min_delay, max_delay]. Everything else is an original.
|
|
originals, repeats = [], []
|
|
for i, s in enumerate(sdr_c1):
|
|
matched = None
|
|
for j in range(i - 1, -1, -1):
|
|
prev = sdr_c1[j]
|
|
dt = s["t"] - prev["t"]
|
|
if dt > args.max_delay:
|
|
break
|
|
if prev["cooked"] == s["cooked"] and args.min_delay <= dt <= args.max_delay:
|
|
matched = (j, dt)
|
|
break
|
|
if matched is not None:
|
|
repeats.append({"idx": i, "orig_idx": matched[0], "dt": matched[1], **s})
|
|
else:
|
|
originals.append({"idx": i, **s})
|
|
|
|
# --- Heltec RX <-> SDR originals (exact cooked-bytes match) ------------
|
|
# Serial timestamps are monotonic-us; SDR timestamps are wall clock.
|
|
# Estimate offset from the best byte-match pairs.
|
|
matched_rx = 0
|
|
per_meter = {}
|
|
used_rx = set()
|
|
for o in originals:
|
|
ser = o["serial"]
|
|
pm = per_meter.setdefault(ser, {"sdr": 0, "heltec": 0, "matched": 0})
|
|
pm["sdr"] += 1
|
|
for k, r in enumerate(heltec_rx):
|
|
if k in used_rx or r["_cooked"] != o["cooked"]:
|
|
continue
|
|
used_rx.add(k)
|
|
matched_rx += 1
|
|
pm["matched"] += 1
|
|
o["heltec_rx"] = r
|
|
break
|
|
|
|
for r in heltec_rx:
|
|
if r["_cooked"] is None or not r.get("crc_ok"):
|
|
continue
|
|
ser = r["_cooked"][4:8][::-1].hex() if len(r["_cooked"]) >= 8 else "?"
|
|
per_meter.setdefault(ser, {"sdr": 0, "heltec": 0, "matched": 0})
|
|
per_meter[ser]["heltec"] += 1
|
|
|
|
# --- Heltec TX <-> SDR repeats -----------------------------------------
|
|
tx_by_hash = {}
|
|
for t in heltec_tx:
|
|
tx_by_hash.setdefault(t.get("hash"), []).append(t)
|
|
|
|
verified_repeats = 0
|
|
for rep in repeats:
|
|
orig = sdr_c1[rep["orig_idx"]]
|
|
o_entry = next((o for o in originals if o["idx"] == rep["orig_idx"]), None)
|
|
hr = o_entry.get("heltec_rx") if o_entry else None
|
|
if hr and hr.get("hash") in tx_by_hash:
|
|
verified_repeats += 1
|
|
rep["verified"] = True
|
|
|
|
tx_ok = [t for t in heltec_tx if str(t.get("result", "")).startswith("ok")]
|
|
delays = [r["dt"] * 1000 for r in repeats]
|
|
delays.sort()
|
|
|
|
def pct(a, b):
|
|
return round(100.0 * a / b, 1) if b else None
|
|
|
|
report = {
|
|
"sdr_frames_total": len(sdr),
|
|
"sdr_c1_crc_ok": len(sdr_c1),
|
|
"sdr_originals": len(originals),
|
|
"sdr_repeats": len(repeats),
|
|
"heltec_rx": len(heltec_rx),
|
|
"heltec_rx_crc_ok": sum(1 for r in heltec_rx if r.get("crc_ok")),
|
|
"heltec_tx_attempts": len(heltec_tx),
|
|
"heltec_tx_ok": len(tx_ok),
|
|
"rx_matched": matched_rx,
|
|
"rx_match_rate_pct": pct(matched_rx, len(originals)),
|
|
"sdr_verified_repeats": len(repeats),
|
|
"fully_correlated_repeats": verified_repeats,
|
|
"repeat_success_rate_pct": pct(len(repeats), len(tx_ok)),
|
|
"repeat_delay_ms": {
|
|
"median": delays[len(delays) // 2] if delays else None,
|
|
"min": delays[0] if delays else None,
|
|
"max": delays[-1] if delays else None,
|
|
},
|
|
"per_meter": per_meter,
|
|
"firmware_events": {
|
|
"recoveries": sum(1 for e in other if e.get("event") == "radio_recovery"),
|
|
"boots": sum(1 for e in other if e.get("event") == "boot"),
|
|
},
|
|
}
|
|
json.dump(report, sys.stdout, indent=2)
|
|
print()
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|