Update HIL verification for hop-marked repeats

This commit is contained in:
Lars Nolden
2026-09-21 12:31:19 +02:00
parent a11a4b3b89
commit 8c44c54131
13 changed files with 525 additions and 309 deletions
+10 -2
View File
@@ -33,15 +33,23 @@ for i in $(seq 1 "$SEGS"); do
V=$(./result-py/bin/python3 hil/meter_anchored.py \ V=$(./result-py/bin/python3 hil/meter_anchored.py \
--iq "$D/iq.raw" --serial "$D/serial.log" --json 2>"$D/verdict.err") --iq "$D/iq.raw" --serial "$D/serial.log" --json 2>"$D/verdict.err")
RC=$? RC=$?
STATE=""
if [ -n "$V" ]; then if [ -n "$V" ]; then
echo "$V" | ./result-py/bin/python3 -c \ STATE=$(printf '%s\n' "$V" | ./result-py/bin/python3 -c \
"import json,sys; print(json.load(sys.stdin).get('rf_state','unknown'))")
printf '%s\n' "$V" | ./result-py/bin/python3 -c \
"import json,sys,time; d=json.load(sys.stdin); d['seg']=$i; d['rc']=$RC; d['t']=int(time.time()); print(json.dumps(d))" \ "import json,sys,time; d=json.load(sys.stdin); d['seg']=$i; d['rc']=$RC; d['t']=int(time.time()); print(json.dumps(d))" \
>> "$OUT/soak.jsonl" >> "$OUT/soak.jsonl"
else else
echo "{\"seg\":$i,\"rc\":$RC,\"error\":\"no_verdict\",\"t\":$(date +%s)}" >> "$OUT/soak.jsonl" echo "{\"seg\":$i,\"rc\":$RC,\"error\":\"no_verdict\",\"t\":$(date +%s)}" >> "$OUT/soak.jsonl"
fi fi
# keep disk bounded: drop IQ after verdict, keep serial # Strong segments are summarized and discarded. Retain every weak,
# absent, or unclassified segment so a hardware conclusion is auditable.
if [ "$STATE" = "strong" ] && [ "$RC" = "0" ]; then
rm -f "$D/iq.raw" rm -f "$D/iq.raw"
else
echo "{\"seg\":$i,\"note\":\"iq_retained\",\"rf_state\":\"$STATE\",\"t\":$(date +%s)}" >> "$OUT/soak.jsonl"
fi
if grep -q "lost at least" "$D/rtl_sdr.err" 2>/dev/null; then if grep -q "lost at least" "$D/rtl_sdr.err" 2>/dev/null; then
echo "{\"seg\":$i,\"note\":\"sample_drops\",\"t\":$(date +%s)}" >> "$OUT/soak.jsonl" echo "{\"seg\":$i,\"note\":\"sample_drops\",\"t\":$(date +%s)}" >> "$OUT/soak.jsonl"
fi fi
+55 -13
View File
@@ -10,8 +10,8 @@ Output: JSON report on stdout.
Correlation model (developer repeat delay ~50 ms): Correlation model (developer repeat delay ~50 ms):
SDR original <-> Heltec RX : same cooked bytes, |dt| < rx_window SDR original <-> Heltec RX : same cooked bytes, |dt| < rx_window
Heltec RX <-> Heltec TX : same hash (firmware log) Heltec RX <-> Heltec TX : same hash (firmware log)
Heltec TX <-> SDR repeat : same cooked bytes, dt in [min_delay, max_delay] SDR original <-> SDR repeat : only H changes 0 -> 1 on a supported header,
after the SDR original dt in [min_delay, max_delay]
The Heltec logs raw over-the-air frames (with CRCs); rtl_wmbus prints The Heltec logs raw over-the-air frames (with CRCs); rtl_wmbus prints
"cooked" frames (CRCs stripped). cook() below mirrors rtl-wmbus. "cooked" frames (CRCs stripped). cook() below mirrors rtl-wmbus.
@@ -69,6 +69,42 @@ def cook_b(frame: bytes):
return bytes(out) return bytes(out)
def expected_repeat(cooked: bytes):
"""Return the H-marked counterpart of a supported CRC-stripped original.
Both format A and rtl-wmbus-normalized format B use L = cooked length - 1.
ELL hop control takes precedence over any encapsulated transport header.
Unsupported layouts, H=1 and R=1 cannot originate a received repeat.
"""
if len(cooked) < 11 or cooked[0] != len(cooked) - 1:
return None
if cooked[1] not in (0x44, 0x46):
return None
ci = cooked[10]
if ci in (0x8c, 0x8e):
# CI + CC + ACC, optionally eight address bytes, then another CI.
if len(cooked) < (14 if ci == 0x8c else 22):
return None
pos, h_mask = 11, 0x10
else:
if ci in (0x67, 0x6e, 0x74, 0x7a, 0x7d, 0x7f, 0x88, 0x8a,
0x9e, 0xc1, 0xc4):
pos = 13
elif ci in (0x68, 0x6f, 0x72, 0x75, 0x7c, 0x7e, 0x87, 0x8b,
0x9f, 0xc2, 0xc5):
pos = 21
else:
return None
if len(cooked) <= pos + 1 or cooked[pos + 1] & 0x1f not in (0, 5):
return None
h_mask = 0x01
if cooked[pos] & (h_mask | 0x02):
return None
marked = bytearray(cooked)
marked[pos] |= h_mask
return bytes(marked)
def parse_serial(path): def parse_serial(path):
"""Heltec JSON-lines -> (rx_events, tx_events, other).""" """Heltec JSON-lines -> (rx_events, tx_events, other)."""
rx, tx, other = [], [], [] rx, tx, other = [], [], []
@@ -133,31 +169,37 @@ def main():
heltec_rx, heltec_tx, other = parse_serial(args.serial) heltec_rx, heltec_tx, other = parse_serial(args.serial)
sdr = parse_sdr(args.sdr) sdr = parse_sdr(args.sdr)
sdr_c1 = [s for s in sdr if s["mode"] == "C1" and s["crc_ok"]] sdr_frames = [s for s in sdr if s["mode"] in ("C1", "T1") and s["crc_ok"]]
# rtl_wmbus runs two demodulator algorithms concurrently; a single burst # rtl_wmbus runs two demodulator algorithms concurrently; a single burst
# can be reported twice within ~1 ms. Collapse identical bytes < 5 ms apart. # can be reported twice within ~1 ms. Collapse identical bytes < 5 ms apart.
deduped = [] deduped = []
for s in sorted(sdr_c1, key=lambda x: x["t"]): for s in sorted(sdr_frames, key=lambda x: x["t"]):
if deduped and deduped[-1]["cooked"] == s["cooked"] \ if deduped and deduped[-1]["mode"] == s["mode"] \
and deduped[-1]["cooked"] == s["cooked"] \
and s["t"] - deduped[-1]["t"] < 0.005: and s["t"] - deduped[-1]["t"] < 0.005:
continue continue
deduped.append(s) deduped.append(s)
sdr_c1 = deduped sdr_frames = deduped
expected = [expected_repeat(s["cooked"]) for s in sdr_frames]
# --- classify SDR frames into originals vs repeats --------------------- # --- classify SDR frames into originals vs repeats ---------------------
# A repeat is an SDR frame with identical cooked bytes to an earlier SDR # A repeat changes only H from 0 to 1 on a supported original header
# frame within [min_delay, max_delay]. Everything else is an original. # within [min_delay, max_delay]. Same-H duplicates are not repeats.
originals, repeats = [], [] originals, repeats = [], []
for i, s in enumerate(sdr_c1): used_originals = set()
for i, s in enumerate(sdr_frames):
matched = None matched = None
for j in range(i - 1, -1, -1): for j in range(i - 1, -1, -1):
prev = sdr_c1[j] prev = sdr_frames[j]
dt = s["t"] - prev["t"] dt = s["t"] - prev["t"]
if dt > args.max_delay: if dt > args.max_delay:
break break
if prev["cooked"] == s["cooked"] and args.min_delay <= dt <= args.max_delay: if (j not in used_originals and prev["mode"] == s["mode"]
and expected[j] == s["cooked"]
and args.min_delay <= dt <= args.max_delay):
matched = (j, dt) matched = (j, dt)
used_originals.add(j)
break break
if matched is not None: if matched is not None:
repeats.append({"idx": i, "orig_idx": matched[0], "dt": matched[1], **s}) repeats.append({"idx": i, "orig_idx": matched[0], "dt": matched[1], **s})
@@ -197,7 +239,6 @@ def main():
verified_repeats = 0 verified_repeats = 0
for rep in repeats: for rep in repeats:
orig = sdr_c1[rep["orig_idx"]]
o_entry = next((o for o in originals if o["idx"] == rep["orig_idx"]), None) o_entry = next((o for o in originals if o["idx"] == rep["orig_idx"]), None)
hr = o_entry.get("heltec_rx") if o_entry else None hr = o_entry.get("heltec_rx") if o_entry else None
if hr and hr.get("hash") in tx_by_hash: if hr and hr.get("hash") in tx_by_hash:
@@ -213,7 +254,8 @@ def main():
report = { report = {
"sdr_frames_total": len(sdr), "sdr_frames_total": len(sdr),
"sdr_c1_crc_ok": len(sdr_c1), "sdr_c1_crc_ok": sum(s["mode"] == "C1" for s in sdr_frames),
"sdr_t1_crc_ok": sum(s["mode"] == "T1" for s in sdr_frames),
"sdr_originals": len(originals), "sdr_originals": len(originals),
"sdr_repeats": len(repeats), "sdr_repeats": len(repeats),
"heltec_rx": len(heltec_rx), "heltec_rx": len(heltec_rx),
+94 -14
View File
@@ -9,10 +9,10 @@ The only trustworthy way we have found to verify DUT transmissions in IQ:
3. Report ambient meter levels alongside -- they are the in-band health 3. Report ambient meter levels alongside -- they are the in-band health
control for the observer (deaf dongle => meters compressed => run void). control for the observer (deaf dongle => meters compressed => run void).
Why not fit the clock on TX events (hil/verify_repeats_iq.py, single-burst Why not fit the clock on TX events (the removed verify_repeats_iq.py did):
mode): repeats fire RX + delay by construction, so an offset shifted by repeats fire RX + delay by construction, so an offset shifted by -delay
-delay "matches" every repeat to its own original's burst. That artifact "matches" every repeat to its own original's burst. That artifact produced
produced fake "7/7 at +37 dB" confirmations. See docs/FINDINGS.md. fake "7/7 at +37 dB" confirmations. See docs/FINDINGS.md.
Usage: meter_anchored.py --iq FILE --serial FILE [--json] Usage: meter_anchored.py --iq FILE --serial FILE [--json]
""" """
@@ -71,6 +71,62 @@ def fit_clock(bursts, anchors_ms, span_ms):
best = (off, m, e) best = (off, m, e)
return best # offset, matched, median_err return best # offset, matched, median_err
def fsk_spectral_excess(path, db, floor, event_times_ms):
"""Return event-correlated 45 kHz FSK excess over nearby control windows."""
nfft = 2048
fft_window = np.hanning(nfft)
raw = np.memmap(path, dtype=np.uint8, mode="r")
sample_count = raw.size // 2
event_times = np.asarray(event_times_ms, dtype=float)
event_psd, control_psd = [], []
def spectrum(center):
lo = center - nfft // 2
hi = lo + nfft
if lo < 0 or hi > sample_count:
return None
values = raw[2 * lo:2 * hi].astype(np.float32) - 127.5
iq = values[0::2] + 1j * values[1::2]
return np.abs(np.fft.fftshift(np.fft.fft(iq * fft_window))) ** 2
for ts_ms in event_times:
a0, a1 = int(ts_ms) - 2, int(ts_ms) + 12
if not (0 <= a0 < a1 < len(db)):
continue
peak_bin = a0 + int(np.argmax(db[a0:a1] - floor))
event = spectrum(peak_bin * WIN + WIN // 2)
if event is None:
continue
control = None
for delta_ms in (100, 200, -100, -200):
control_bin = peak_bin + delta_ms
if not (0 <= control_bin < len(db)):
continue
if np.any(np.abs(event_times - control_bin) < 25):
continue
control = spectrum(control_bin * WIN + WIN // 2)
if control is not None:
break
if control is None:
continue
event_psd.append(event)
control_psd.append(control)
if len(event_psd) < 5:
return None, len(event_psd)
event_med = np.median(np.stack(event_psd), axis=0)
control_med = np.median(np.stack(control_psd), axis=0)
freq = np.fft.fftshift(np.fft.fftfreq(nfft, 1 / FS))
signal = (np.abs(freq) >= 20000) & (np.abs(freq) < 80000)
wide = (np.abs(freq) >= 250000) & (np.abs(freq) < 700000)
signal_db = 10 * np.log10(event_med[signal].sum()
/ control_med[signal].sum())
wide_db = 10 * np.log10(event_med[wide].sum()
/ control_med[wide].sum())
return float(signal_db - wide_db), len(event_psd)
def main(): def main():
ap = argparse.ArgumentParser() ap = argparse.ArgumentParser()
@@ -124,19 +180,35 @@ def main():
return None return None
return float((db[a0:a1] - floor).max()) return float((db[a0:a1] - floor).max())
events = [] events, mapped_tx = [], []
for t in tx: for t in tx:
w = winmax(t["timestamp_us"] / 1000.0 - off) mapped = t["timestamp_us"] / 1000.0 - off
w = winmax(mapped)
if w is None: if w is None:
continue continue
mapped_tx.append(mapped)
events.append({"event": t.get("event", "wmbus_tx"), events.append({"event": t.get("event", "wmbus_tx"),
"result": t.get("result"), "result": t.get("result"),
"window_db": round(w, 1), "window_db": round(w, 1),
"rf": bool(w >= 6)}) "above_6db": bool(w >= 6)})
meter_med = float(np.median(meter_lv)) if meter_lv else 0.0 meter_med = float(np.median(meter_lv)) if meter_lv else 0.0
observer = "ok" if meter_med >= 15 else ("degraded" if meter_med >= 8 else "deaf") observer = "ok" if meter_med >= 15 else ("degraded" if meter_med >= 8 else "deaf")
n_rf = sum(1 for e in events if e["rf"]) n_above = sum(1 for e in events if e["above_6db"])
levels = [e["window_db"] for e in events]
tx_med = float(np.median(levels)) if levels else None
controls = [winmax(t + 100) for t in mapped_tx]
controls = [v for v in controls if v is not None]
control_med = float(np.median(controls)) if controls else None
fsk_db, fsk_windows = fsk_spectral_excess(
args.iq, db, floor, mapped_tx)
if not events:
rf_state = "no_events"
elif fsk_db is not None and fsk_db >= 3.0:
rf_state = "strong" if tx_med >= 6.0 else "weak"
else:
rf_state = "not_observed"
report = { report = {
"clock_matched": f"{matched}/{len(strong)}", "clock_matched": f"{matched}/{len(strong)}",
"clock_err_ms": round(err, 1), "clock_err_ms": round(err, 1),
@@ -144,8 +216,14 @@ def main():
"meter_median_db": round(meter_med, 1), "meter_median_db": round(meter_med, 1),
"observer": observer, "observer": observer,
"tx_events": len(events), "tx_events": len(events),
"tx_with_rf": n_rf, "tx_above_6db": n_above,
"tx_levels_db": [e["window_db"] for e in events], "tx_median_db": round(tx_med, 1) if tx_med is not None else None,
"control_median_db": (round(control_med, 1)
if control_med is not None else None),
"fsk_excess_db": round(fsk_db, 1) if fsk_db is not None else None,
"fsk_windows": fsk_windows,
"rf_state": rf_state,
"tx_levels_db": levels,
} }
if args.json: if args.json:
print(json.dumps(report)) print(json.dumps(report))
@@ -153,14 +231,16 @@ def main():
print(f"clock: {report['clock_matched']} anchors, err {report['clock_err_ms']} ms") print(f"clock: {report['clock_matched']} anchors, err {report['clock_err_ms']} ms")
print(f"observer: {observer} (meter median +{report['meter_median_db']} dB, " print(f"observer: {observer} (meter median +{report['meter_median_db']} dB, "
f"floor {report['floor_db']})") f"floor {report['floor_db']})")
print(f"TX with RF: {n_rf}/{len(events)}") print(f"TX state: {rf_state}; median +{report['tx_median_db']} dB, "
f"FSK excess {report['fsk_excess_db']} dB")
for e in events: for e in events:
print(f" {e['event']:>9} {e['result'] or '':>10} " print(f" {e['event']:>9} {e['result'] or '':>10} "
f"+{e['window_db']:>5.1f} dB {'RF' if e['rf'] else '--'}") f"+{e['window_db']:>5.1f} dB "
# exit 3 = observer unusable, 1 = TX dead while observer ok f"{'>=6dB' if e['above_6db'] else 'weak'}")
# exit 3 = observer unusable, 1 = no event-correlated FSK while observer ok
if observer == "deaf": if observer == "deaf":
sys.exit(3) sys.exit(3)
sys.exit(0 if (n_rf or not events) else 1) sys.exit(0 if rf_state in ("strong", "weak", "no_events") else 1)
if __name__ == "__main__": if __name__ == "__main__":
+23 -21
View File
@@ -20,7 +20,7 @@ ap.add_argument("--tx-power", default="?")
ap.add_argument("--repeat-delay", default="?") ap.add_argument("--repeat-delay", default="?")
ap.add_argument("--sdr-streamed", default="?") ap.add_argument("--sdr-streamed", default="?")
ap.add_argument("--tx-proof", default=None) ap.add_argument("--tx-proof", default=None)
ap.add_argument("--iq-proof", default=None) ap.add_argument("--rf-proof", default=None)
ap.add_argument("--sdr-sensitivity", default="?") ap.add_argument("--sdr-sensitivity", default="?")
args = ap.parse_args() args = ap.parse_args()
@@ -38,7 +38,7 @@ def load_json(path):
proof = load_json(args.tx_proof) proof = load_json(args.tx_proof)
iqp = load_json(args.iq_proof) rfp = load_json(args.rf_proof)
def fmt(v, unit=""): def fmt(v, unit=""):
@@ -63,11 +63,13 @@ if fw.get("boots", 0) > 1:
if proof is not None and proof.get("bursts_with_sdr_decode", 0) == 0: if proof is not None and proof.get("bursts_with_sdr_decode", 0) == 0:
fail.append("SDR could not decode any Heltec C1 transmission") fail.append("SDR could not decode any Heltec C1 transmission")
# A decoded repeat is preferred, but a decode-independent RF observation of # Decoded H=0 -> H=1 transitions are preferred. Meter-anchored, spectrally
# a repeat at the configured delay also satisfies "the observer # classified FSK at firmware repeat times is an independent RF proof when
# independently saw the retransmission" (see docs/RF_FRONTEND_TESTS.md for # the nearby transmission overloads the decoder.
# why decoding our own TX is bench-limited). rf_confirmed = (rfp is not None
if iqp is not None and iqp.get("tx_events_with_rf", 0) > 0: and rfp.get("rf_state") in ("strong", "weak")
and rfp.get("tx_events", 0) >= args.min_repeats)
if rf_confirmed:
fail = [f for f in fail if not f.startswith("SDR-verified repeats")] fail = [f for f in fail if not f.startswith("SDR-verified repeats")]
verdict = "PASS" if not fail else "FAIL" verdict = "PASS" if not fail else "FAIL"
@@ -99,25 +101,25 @@ TX DECODABILITY PROOF (synthetic C1, independent of meters):
SDR decoded as C1: {proof.get('bursts_with_sdr_decode')} SDR decoded as C1: {proof.get('bursts_with_sdr_decode')}
both airings decoded: {proof.get('bursts_with_both_airings_decoded')}""") both airings decoded: {proof.get('bursts_with_both_airings_decoded')}""")
if iqp is not None: if rfp is not None:
print(f""" print(f"""
RF-LEVEL TX PROOF (same stream's raw IQ, decode-independent): RF-LEVEL TX PROOF (meter-anchored IQ + non-TX controls):
IQ analysed: {iqp.get('iq_seconds')} s clock anchors: {rfp.get('clock_matched')}
clipped samples: {iqp.get('clipped_sample_pct')} % clock median error: {rfp.get('clock_err_ms')} ms
bursts in window: {iqp.get('bursts_total')} observer state: {rfp.get('observer')}
firmware TX ok: {iqp.get('firmware_tx_ok')} meter level over floor: {rfp.get('meter_median_db')} dB
TX events in IQ window: {iqp.get('tx_events_in_iq_window')} firmware TX events: {rfp.get('tx_events')}
TX events with RF: {iqp.get('tx_events_with_rf')} ({iqp.get('tx_rf_confirmed_pct')} %) RF state: {rfp.get('rf_state')}
alignment error: {iqp.get('align_error_ms')} ms TX median over floor: {rfp.get('tx_median_db')} dB
airtime error: {iqp.get('burst_duration_error_ms')} ms control median: {rfp.get('control_median_db')} dB
our TX over floor: {iqp.get('our_tx_peak_db_over_floor')} dB FSK-selective excess: {rfp.get('fsk_excess_db')} dB
meter bursts over floor: {iqp.get('other_burst_peak_db_over_floor')} dB spectral windows: {rfp.get('fsk_windows')}""")
firmware repeat delay: {iqp.get('firmware_repeat_delay_ms')} ms""")
print(f""" print(f"""
SDR (independent observer): SDR (independent observer):
decode lines: {r.get('sdr_frames_total')} decode lines: {r.get('sdr_frames_total')}
C1 CRC-ok bursts: {r.get('sdr_c1_crc_ok')} C1 CRC-ok bursts: {r.get('sdr_c1_crc_ok')}
T1 CRC-ok bursts: {r.get('sdr_t1_crc_ok', 0)}
classified originals: {r.get('sdr_originals')} classified originals: {r.get('sdr_originals')}
classified repeats: {r.get('sdr_repeats')} classified repeats: {r.get('sdr_repeats')}
unique meters: {len(r.get('per_meter') or {})} unique meters: {len(r.get('per_meter') or {})}
@@ -133,7 +135,7 @@ HELTEC TX:
reported success: {r.get('heltec_tx_ok')} reported success: {r.get('heltec_tx_ok')}
SDR-VERIFIED REPEATS: SDR-VERIFIED REPEATS:
byte-identical pairs: {r.get('sdr_repeats')} H=0 -> H=1 pairs: {r.get('sdr_repeats')}
fully correlated: {r.get('fully_correlated_repeats')} fully correlated: {r.get('fully_correlated_repeats')}
vs TX attempts: {fmt(r.get('repeat_success_rate_pct'), ' %')} vs TX attempts: {fmt(r.get('repeat_success_rate_pct'), ' %')}
+23 -10
View File
@@ -37,6 +37,11 @@ while [ $# -gt 0 ]; do
esac esac
done done
if ! [[ "$TXPOW" =~ ^-?[0-9]+$ ]] || (( TXPOW < -9 || TXPOW > 2 )); then
echo "--tx-power must be an integer in -9..2" >&2
exit 2
fi
PY=./result-py/bin/python3 PY=./result-py/bin/python3
[ -x "$PY" ] || { echo "run 'nix build .#hil-python -o result-py' first" >&2; exit 2; } [ -x "$PY" ] || { echo "run 'nix build .#hil-python -o result-py' first" >&2; exit 2; }
@@ -154,21 +159,29 @@ cat "$RUN_DIR/final_stats.log" >> "$RUN_DIR/serial.log"
# --- RF-level repeat proof from the same stream's IQ ---------------------- # --- RF-level repeat proof from the same stream's IQ ----------------------
if [ -s "$IQ" ]; then if [ -s "$IQ" ]; then
echo "RF-level repeat proof from captured IQ" echo "RF-level repeat proof from meter-anchored IQ"
$PY hil/verify_repeats_iq.py --iq "$IQ" --serial "$RUN_DIR/serial.log" \ RF_RC=0
--delay-ms "$DELAY" > "$RUN_DIR/iq_proof.txt" 2>&1 || true if ! $PY hil/meter_anchored.py --iq "$IQ" \
$PY hil/verify_repeats_iq.py --iq "$IQ" --serial "$RUN_DIR/serial.log" \ --serial "$RUN_DIR/serial.log" --json \
--delay-ms "$DELAY" --json > "$RUN_DIR/iq_proof.json" 2>&1 || true > "$RUN_DIR/rf_proof.json" 2> "$RUN_DIR/rf_proof.err"; then
cat "$RUN_DIR/iq_proof.txt" RF_RC=$?
# IQ is large: keep it only when the proof failed, for offline inspection fi
if grep -q '"tx_events_with_rf": [1-9]' "$RUN_DIR/iq_proof.json" 2>/dev/null; then if [ -s "$RUN_DIR/rf_proof.json" ]; then
cat "$RUN_DIR/rf_proof.json"
else
cat "$RUN_DIR/rf_proof.err"
fi
# Preserve weak, absent and unclassifiable RF for offline audit.
if [ "$RF_RC" = "0" ] \
&& grep -q '"rf_state": "strong"' "$RUN_DIR/rf_proof.json" 2>/dev/null; then
rm -f "$IQ" rm -f "$IQ"
fi fi
fi fi
# --- 13-14. correlate ------------------------------------------------------ # --- 13-14. correlate ------------------------------------------------------
MAXDELAY=$(awk "BEGIN{printf \"%.3f\", ($DELAY + 200) / 1000}")
$PY hil/correlate.py --serial "$RUN_DIR/serial.log" --sdr "$RUN_DIR/rtlwmbus.log" \ $PY hil/correlate.py --serial "$RUN_DIR/serial.log" --sdr "$RUN_DIR/rtlwmbus.log" \
--min-delay 0.010 --max-delay 0.400 > "$RUN_DIR/report.json" --min-delay 0.010 --max-delay "$MAXDELAY" > "$RUN_DIR/report.json"
# --- 15. lockup / reset detection ----------------------------------------- # --- 15. lockup / reset detection -----------------------------------------
HB=$(grep -c '"event":"heartbeat"' "$RUN_DIR/serial.log" || true) HB=$(grep -c '"event":"heartbeat"' "$RUN_DIR/serial.log" || true)
@@ -189,7 +202,7 @@ $PY hil/report.py "$RUN_DIR/report.json" \
--sdr-streamed "$STREAMED" \ --sdr-streamed "$STREAMED" \
--sdr-sensitivity "${SENS:-unknown}" \ --sdr-sensitivity "${SENS:-unknown}" \
--tx-proof "$RUN_DIR/tx_proof.json" \ --tx-proof "$RUN_DIR/tx_proof.json" \
--iq-proof "$RUN_DIR/iq_proof.json" \ --rf-proof "$RUN_DIR/rf_proof.json" \
| tee "$RUN_DIR/report.txt" | tee "$RUN_DIR/report.txt"
RC=${PIPESTATUS[0]} RC=${PIPESTATUS[0]}
echo "artifacts: $RUN_DIR" echo "artifacts: $RUN_DIR"
+2 -2
View File
@@ -69,8 +69,8 @@ if not tx:
print("TX: no repeat events recorded") print("TX: no repeat events recorded")
print(f" result breakdown: {dict(results)}") print(f" result breakdown: {dict(results)}")
print(f" repeat delay: {s(delays, ' ms')}") print(f" TX-routine delay: {s(delays, ' ms')}")
print(f" TX airtime: {s(air, ' ms')}") print(f" computed RF airtime: {s(air, ' ms')}")
print(f""" print(f"""
HEALTH: HEALTH:
radio recoveries: {len(rec)} radio recoveries: {len(rec)}
+91
View File
@@ -0,0 +1,91 @@
#!/usr/bin/env bash
# Overnight segmented RF soak for a STANDALONE repeater (no serial link).
# Per segment: bounded rtl_sdr capture -> offline rtl_wmbus decode ->
# hil/verify_standalone.py verdict appended to soak.jsonl.
#
# Designed around the known observer faults (docs/HARDWARE.md): bounded
# segments, usbreset recovery on short captures, and a deaf-observer
# heuristic -- the neighbouring meter rails the dongle at a steady rate,
# so consecutive near-empty segments mean a dead receiver, not a quiet
# night. Disk is bounded: confirmed segments drop their IQ; only the two
# newest unconfirmed segments keep raw IQ for offline audit.
#
# Run via the devshell so usbreset is on PATH:
# nix develop --command ./hil/standalone_soak.sh 0855 300
#
# Usage: standalone_soak.sh [END_HHMM] [SEG_SECONDS] [DELAY_MS]
# standalone_soak.sh 0855 300 500
set -u
cd "$(dirname "$0")/.."
END_HHMM=${1:-0855}
SEG=${2:-300}
DELAY_MS=${3:-500} # must match the DUT's persisted repeat_delay_ms
PY=./result-py/bin/python3
WMBUS=./result-rtl-wmbus/bin/rtl_wmbus
[ -x "$PY" ] || { echo "run 'nix build .#hil-python -o result-py' first" >&2; exit 2; }
[ -x "$WMBUS" ] || { echo "missing result-rtl-wmbus" >&2; exit 2; }
OUT="captures/$(date +%Y-%m-%dT%H-%M-%S)-standalone-soak"
mkdir -p "$OUT"
END_EPOCH=$(date -d "$END_HHMM" +%s 2>/dev/null) || { echo "bad end time: $END_HHMM" >&2; exit 2; }
if [ "$END_EPOCH" -le "$(date +%s)" ]; then
END_EPOCH=$(date -d "tomorrow $END_HHMM" +%s)
fi
echo "soak until $(date -d "@$END_EPOCH" '+%F %T'), segment ${SEG}s -> $OUT"
command -v usbreset >/dev/null || echo "WARNING: usbreset not on PATH; dongle recovery disabled" >&2
low_bursts=0
short_caps=0
i=0
while [ $(( $(date +%s) + SEG )) -le "$END_EPOCH" ]; do
i=$((i + 1))
D="$OUT/seg$i"
mkdir -p "$D"
timeout $((SEG + 60)) rtl_sdr -f 868950000 -s 1600000 -g 42.1 \
-n $((SEG * 1600000)) "$D/iq.raw" 2> "$D/rtl_sdr.err"
SZ=$(stat -c %s "$D/iq.raw" 2>/dev/null || echo 0)
T=$(date +%s)
if [ "$SZ" -lt $((SEG * 3200000 * 8 / 10)) ]; then
echo "{\"seg\":$i,\"error\":\"short_capture\",\"bytes\":$SZ,\"t\":$T}" >> "$OUT/soak.jsonl"
rm -f "$D/iq.raw"
usbreset 0bda:2838 >/dev/null 2>&1
short_caps=$((short_caps + 1))
# A de-enumerated dongle (needs physical replug) is unrecoverable
# from software: back off instead of hammering, keep probing in
# case it re-appears.
if [ "$short_caps" -ge 5 ]; then sleep 300; else sleep 20; fi
continue
fi
short_caps=0
V=$($PY hil/verify_standalone.py --iq "$D/iq.raw" --delay-ms "$DELAY_MS" \
--decodes "$D/decodes.log" --json 2> "$D/verdict.err")
RC=$?
if [ -n "$V" ]; then
printf '%s\n' "$V" | $PY -c \
"import json,sys; d=json.load(sys.stdin); d['seg']=$i; d['rc']=$RC; d['t']=$T; print(json.dumps(d))" \
>> "$OUT/soak.jsonl"
else
echo "{\"seg\":$i,\"rc\":$RC,\"error\":\"no_verdict\",\"t\":$T}" >> "$OUT/soak.jsonl"
fi
if grep -q "lost at least" "$D/rtl_sdr.err" 2>/dev/null; then
echo "{\"seg\":$i,\"note\":\"sample_drops\",\"t\":$T}" >> "$OUT/soak.jsonl"
fi
B=$(printf '%s\n' "$V" | $PY -c \
"import json,sys; print(json.load(sys.stdin).get('bursts_total',0))" \
2>/dev/null || echo 0)
if [ "${B:-0}" -lt 3 ]; then low_bursts=$((low_bursts + 1)); else low_bursts=0; fi
if [ "$low_bursts" -ge 2 ]; then
echo "{\"seg\":$i,\"note\":\"observer_suspect_reset\",\"t\":$(date +%s)}" >> "$OUT/soak.jsonl"
usbreset 0bda:2838 >/dev/null 2>&1
low_bursts=0
fi
if printf '%s' "$V" | grep -q '"verdict": "confirmed"'; then
rm -f "$D/iq.raw"
fi
ls -1t "$OUT"/seg*/iq.raw 2>/dev/null | tail -n +3 | xargs -r rm -f
sleep 15
done
echo "soak complete: $OUT/soak.jsonl"
+172
View File
@@ -0,0 +1,172 @@
#!/usr/bin/env python3
"""Verify standalone beacon TX from RF evidence alone (no serial log).
Use case: DUT on battery/wall power with no USB host, transmitting the
synthetic OMS beacon every N ms. Two independent lines of evidence from
one recorded IQ stream:
1. Content: rtl_wmbus decodes of the known beacon frame
(mfr 0x4D8E "SLN", ID 00000001). Decode timestamps are clustered into
airings (two demodulators emit 1-2 lines per airing).
2. Timing: short bursts in the IQ envelope forming a comb at the beacon
period. The effective period is fitted from the data, because the
firmware schedules the next beacon after the previous TX completes
(configured period + a few ms), and a beacon can be delayed by RX
activity.
Ambient (non-beacon) decodes and long bursts are reported as the observer
health control.
Usage:
verify_beacon_standalone.py --iq iq.raw --decodes decodes.log \
--period-ms 3000 [--rate 1600000] [--json]
Exit: 0 = beacon TX confirmed, 1 = not observed, 2 = insufficient data.
"""
import argparse
import json
import re
import sys
import numpy as np
BEACON_HEX = "448e4d010000000107" # C + mfr + ID + ver + type, L-agnostic
DECODE_RE = re.compile(
r"^(?P<mode>[TC]1);\d+;\d+;(?P<ts>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+);"
r".*0x(?P<hex>[0-9a-fA-F]+)")
def parse_decodes(path):
# Distinct airings are counted by distinct payload: the beacon's
# access number increments per airing, so each airing has unique hex,
# while rtl_wmbus's two demodulators emitting 1-2 lines per airing
# collapse to one. Timestamp-independent, so it works for both live
# captures and offline `rtl_wmbus < iq.raw` (compressed wall clock).
# acc wraps at 256: valid for captures up to 256 beacon periods.
beacon_hex, beacon_lines, ambient = set(), 0, 0
for line in open(path, errors="replace"):
m = DECODE_RE.match(line.strip())
if not m:
continue
if BEACON_HEX in m.group("hex").lower():
beacon_hex.add(m.group("hex").lower())
beacon_lines += 1
else:
ambient += 1
return len(beacon_hex), beacon_lines, ambient
def envelope_bursts(path, rate):
win = rate // 1000 # 1 ms bins
env = []
with open(path, "rb") as f:
while True:
raw = f.read(rate * 2)
if len(raw) < win * 2:
break
u8 = np.frombuffer(raw, dtype=np.uint8)
b = u8.astype(np.float32) - 127.5
iq = b[0::2] + 1j * b[1::2]
p = np.abs(iq) ** 2
p = p[: (len(p) // win) * win].reshape(-1, win).mean(axis=1)
env.append(p)
if not env:
return None, 0.0, []
db = 10 * np.log10(np.concatenate(env) + 1e-9)
floor = float(np.median(db))
idx = np.where(db > floor + 6.0)[0]
bursts = []
if len(idx):
start = prev = idx[0]
for i in idx[1:]:
if i > prev + 3:
bursts.append((int(start), int(prev - start + 1),
float(db[start:prev + 1].max() - floor)))
start = i
prev = i
bursts.append((int(start), int(prev - start + 1),
float(db[start:prev + 1].max() - floor)))
return db, floor, bursts
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--iq", required=True)
ap.add_argument("--decodes")
ap.add_argument("--period-ms", type=float, required=True,
help="configured beacon_period_ms")
ap.add_argument("--rate", type=int, default=1_600_000)
ap.add_argument("--json", action="store_true")
args = ap.parse_args()
db, floor, bursts = envelope_bursts(args.iq, args.rate)
if db is None:
print("IQ empty or unreadable", file=sys.stderr)
sys.exit(2)
# Beacon candidates: short bursts (L=19 beacon airtime is 2.7 ms;
# envelope smearing widens that). Longer bursts are ambient meters.
cand = [b for b in bursts if b[1] <= 8]
long_bursts = [b for b in bursts if b[1] > 8]
# Fit the effective period from consecutive candidate intervals near
# the configured period, then count on-grid intervals (a missed or
# RX-delayed slot shows up as ~2x the period and is counted as a gap).
starts = np.array([b[0] for b in cand], dtype=float)
iv = np.diff(starts)
p = args.period_ms
near = iv[(iv > 0.8 * p) & (iv < 1.35 * p)]
fitted = float(np.median(near)) if len(near) else None
on_grid = int(len(near))
gaps = int(np.count_nonzero((iv > 1.6 * p) & (iv < 2.6 * p)))
cand_levels = [b[2] for b in cand]
airings, beacon_lines, ambient_decodes = 0, 0, 0
if args.decodes:
airings, beacon_lines, ambient_decodes = parse_decodes(args.decodes)
decode_ok = airings >= 5
comb_ok = on_grid >= 5 and fitted is not None
if decode_ok or comb_ok:
verdict = "confirmed"
rc = 0
elif len(cand) or airings:
verdict = "insufficient"
rc = 2
else:
verdict = "not_observed"
rc = 1
report = {
"iq_seconds": round(len(db) / 1000.0, 1),
"noise_floor_db": round(floor, 1),
"beacon_candidate_bursts": len(cand),
"candidate_median_db": (round(float(np.median(cand_levels)), 1)
if cand_levels else None),
"period_configured_ms": p,
"period_fitted_ms": round(fitted, 1) if fitted else None,
"intervals_on_grid": on_grid,
"missed_slot_gaps": gaps,
"beacon_decoded_airings": airings,
"beacon_decode_lines": beacon_lines,
"ambient_decodes": ambient_decodes,
"ambient_long_bursts": len(long_bursts),
"verdict": verdict,
}
if args.json:
print(json.dumps(report))
else:
print(f"IQ: {report['iq_seconds']} s, floor {report['noise_floor_db']} dB")
print(f"beacon bursts: {len(cand)} (median "
f"+{report['candidate_median_db']} dB), period fitted "
f"{report['period_fitted_ms']} ms vs configured {p:.0f} ms, "
f"{on_grid} intervals on grid, {gaps} missed-slot gaps")
print(f"decoded beacon airings: {airings} ({beacon_lines} lines); "
f"ambient decodes: {ambient_decodes}; "
f"ambient long bursts: {len(long_bursts)}")
print(f"VERDICT: {verdict}")
sys.exit(rc)
if __name__ == "__main__":
main()
-223
View File
@@ -1,223 +0,0 @@
#!/usr/bin/env python3
"""Verify repeater transmissions from raw IQ, independent of packet decoding.
Why this exists: on this bench the Heltec transmission arrives at the SDR
far stronger than real meter traffic, and the dongle's own sensitivity
drifts by ~17 dB over minutes. rtl-wmbus therefore often receives our
repeat with a couple of byte errors (CRC fails) even though the RF is
present and correctly timed. Burst-envelope analysis is immune to that.
Method: extract RF bursts from the IQ envelope, then find the single clock
offset between the firmware's monotonic microsecond timestamps and the IQ
timeline that maximises the number of firmware TX events landing on a
burst. No amplitude threshold is assumed for attribution -- the alignment
itself is the evidence, and a wrong alignment cannot match many events.
Reports:
- how many firmware TX events have corresponding RF in the IQ
- the measured original -> repeat spacing for repeats whose source RX
event is known, compared against the configured delay
- the received level of our transmissions vs meter traffic
Usage:
verify_repeats_iq.py --iq iq.raw --serial serial.log --delay-ms 200
[--rate 1600000] [--tol-ms 15] [--json]
"""
import argparse
import json
import sys
import numpy as np
ap = argparse.ArgumentParser()
ap.add_argument("--iq", required=True)
ap.add_argument("--serial", required=True)
ap.add_argument("--delay-ms", type=float, required=True)
ap.add_argument("--rate", type=int, default=1_600_000)
ap.add_argument("--tol-ms", type=float, default=15.0)
ap.add_argument("--json", action="store_true")
args = ap.parse_args()
WIN = args.rate // 1000 # 1 ms resolution
# ---------------------------------------------------------------- IQ bursts
def load_envelope(path):
env, clipped = [], 0
with open(path, "rb") as f:
while True:
raw = f.read(args.rate * 2)
if len(raw) < WIN * 2:
break
u8 = np.frombuffer(raw, dtype=np.uint8)
clipped += int(np.count_nonzero((u8 == 0) | (u8 == 255)))
b = u8.astype(np.float32) - 127.5
iq = b[0::2] + 1j * b[1::2]
p = np.abs(iq) ** 2
p = p[: (len(p) // WIN) * WIN].reshape(-1, WIN).mean(axis=1)
env.append(p)
if not env:
return None, 0
return np.concatenate(env), clipped
power, clipped_samples = load_envelope(args.iq)
if power is None:
msg = {"error": "iq too short"}
print(json.dumps(msg) if args.json else "ERROR: iq too short")
sys.exit(3)
db = 10 * np.log10(power + 1e-9)
floor = float(np.median(db))
idx = np.where(db > floor + 6.0)[0]
bursts = []
if len(idx):
start = prev = idx[0]
for i in idx[1:]:
if i > prev + 3:
bursts.append((start, prev))
start = i
prev = i
bursts.append((start, prev))
burst_start = np.array([s for s, _ in bursts], dtype=np.int64)
burst_info = [{
"t_ms": int(s),
"duration_ms": int(e - s + 1),
"peak_db": float(db[s:e + 1].max() - floor),
} for s, e in bursts]
# ------------------------------------------------------------ firmware log
tx_events, rx_events = [], []
for line in open(args.serial, errors="replace"):
line = line.strip()
if not line.startswith("{"):
continue
try:
ev = json.loads(line)
except json.JSONDecodeError:
continue
if ev.get("event") == "wmbus_tx" and str(ev.get("result", "")).startswith("ok"):
tx_events.append(ev)
elif ev.get("event") == "wmbus_rx" and ev.get("crc_ok"):
rx_events.append(ev)
if not tx_events or not bursts:
result = {
"iq_seconds": round(len(db) / 1000.0, 1),
"noise_floor_db": round(floor, 1),
"bursts_total": len(burst_info),
"firmware_tx_ok": len(tx_events),
"tx_events_with_rf": 0,
"note": "no TX events or no bursts in this window",
}
print(json.dumps(result, indent=2) if args.json
else f"IQ_REPEAT_PROOF: no TX events or no bursts "
f"(tx={len(tx_events)} bursts={len(burst_info)})")
sys.exit(1)
tx_ms = np.array([ev["timestamp_us"] / 1000.0 for ev in tx_events])
# ------------------------------------------------- align firmware to IQ
# Candidate offsets: every (burst - tx_event) difference. The true offset
# appears once per genuinely observed transmission, so it wins by count.
cands = np.subtract.outer(burst_start.astype(float), tx_ms).ravel()
best_off, best_hits = 0.0, -1
for off in cands:
d = np.abs(burst_start[:, None] - (tx_ms + off)[None, :])
hits = int(np.count_nonzero(d.min(axis=0) <= args.tol_ms))
if hits > best_hits:
best_hits, best_off = hits, float(off)
# Match each TX event to its nearest burst under the winning offset.
# The IQ may cover only part of the serial log (bounded IQ segments, or a
# tuner fault mid-run), so TX events whose aligned time falls outside the
# IQ timeline are "not observable" rather than "not observed" -- counting
# them as failures would understate the result.
iq_span_ms = len(db)
matched, our_peaks, in_window = [], [], 0
for ev, t in zip(tx_events, tx_ms):
target = t + best_off
if target < 0 or target > iq_span_ms:
continue
in_window += 1
if not len(burst_start):
continue
j = int(np.argmin(np.abs(burst_start - target)))
err = float(burst_start[j] - target)
if abs(err) <= args.tol_ms:
matched.append({
"hash": ev.get("hash"),
"frame_len": ev.get("length"),
"iq_t_ms": int(burst_start[j]),
"align_error_ms": round(err, 1),
"burst_duration_ms": burst_info[j]["duration_ms"],
"burst_peak_db": round(burst_info[j]["peak_db"], 1),
"delay_us": ev.get("delay_us"),
})
our_peaks.append(burst_info[j]["peak_db"])
ours_t = {m["iq_t_ms"] for m in matched}
other_peaks = [b["peak_db"] for b in burst_info if b["t_ms"] not in ours_t]
# expected airtime: (frame + 2 mode bytes) * 80 us + preamble/sync ~0.64 ms
def expected_ms(n):
return ((n + 2) * 80 + 640) / 1000.0 if n else None
dur_err = [m["burst_duration_ms"] - expected_ms(m["frame_len"])
for m in matched if m["frame_len"]]
delays = sorted(m["delay_us"] / 1000.0 for m in matched
if m.get("delay_us"))
def stat(v):
if not v:
return {"min": None, "median": None, "max": None}
v = sorted(v)
return {"min": round(v[0], 1),
"median": round(v[len(v) // 2], 1),
"max": round(v[-1], 1)}
result = {
"iq_seconds": round(len(db) / 1000.0, 1),
"noise_floor_db": round(floor, 1),
"clipped_sample_pct": round(100.0 * clipped_samples /
max(1, len(db) * WIN * 2), 4),
"bursts_total": len(burst_info),
"firmware_tx_ok": len(tx_events),
"firmware_rx_crc_ok": len(rx_events),
"tx_events_in_iq_window": in_window,
"tx_events_with_rf": len(matched),
"tx_rf_confirmed_pct": round(100.0 * len(matched) / in_window, 1) if in_window else None,
"clock_offset_ms": round(best_off, 1),
"align_error_ms": stat([abs(m["align_error_ms"]) for m in matched]),
"burst_duration_error_ms": stat(dur_err),
"our_tx_peak_db_over_floor": stat(our_peaks),
"other_burst_peak_db_over_floor": stat(other_peaks),
"firmware_repeat_delay_ms": stat(delays),
"expected_delay_ms": args.delay_ms,
"matched": matched[:50],
}
if args.json:
print(json.dumps(result, indent=2))
else:
print(f"IQ_REPEAT_PROOF iq={result['iq_seconds']}s floor={floor:.1f}dB "
f"clipped={result['clipped_sample_pct']}%")
print(f" bursts in window: {len(burst_info)}")
print(f" firmware TX ok / RX crc-ok: {len(tx_events)} / {len(rx_events)}")
print(f" TX events inside IQ window: {in_window} of {len(tx_events)}")
print(f" TX events with RF in IQ: {len(matched)}/{in_window} "
f"({result['tx_rf_confirmed_pct']} %)")
print(f" clock offset: {best_off:.0f} ms")
print(f" alignment error min/med/max: {result['align_error_ms']}")
print(f" duration error vs airtime: {result['burst_duration_error_ms']} ms")
print(f" our TX level over floor: {result['our_tx_peak_db_over_floor']} dB")
print(f" other bursts over floor: {result['other_burst_peak_db_over_floor']} dB")
print(f" firmware repeat delay: {result['firmware_repeat_delay_ms']} ms "
f"(configured {args.delay_ms:.0f})")
sys.exit(0 if matched else 1)
+33 -20
View File
@@ -8,18 +8,18 @@ SDR observes. This is also the honest test of "power it up and leave it be".
Two independent lines of evidence, both derived from ONE recorded IQ file so Two independent lines of evidence, both derived from ONE recorded IQ file so
they cannot disagree about gain or time: they cannot disagree about gain or time:
1. Decoded repeats - rtl_wmbus run over the recording; a repeat is the 1. Decoded repeats - a supported H=0 original followed by an H=1 frame
same cooked payload decoded twice within the delay window. with every other cooked byte unchanged.
2. RF-level repeats - burst-envelope pairs separated by the delay window 2. RF-level repeats - burst-envelope pairs separated by the delay window
where the second burst is at least as strong as the first. This catches where the second burst is at least as strong as the first. This catches
repeats that fail CRC because our transmission overloads the dongle repeats that fail CRC because our transmission overloads the dongle
(docs/RF_FRONTEND_TESTS.md), which is the common case on this bench. (docs/RF_FRONTEND_TESTS.md), which is the common case on this bench.
Usage: Usage:
verify_standalone.py --iq iq.raw --decodes decodes.log verify_standalone.py --iq iq.raw --decodes decodes.log [--delay-ms 500]
[--min-ms 20] [--max-ms 400] [--rate 1600000] [--json] [--delay-tol-ms 25] [--rate 1600000] [--json]
Exit 0 if at least one repeat was observed by either method. Exit 0 only if a decoded hop transition confirms repeater activity.
""" """
import argparse import argparse
import json import json
@@ -30,7 +30,7 @@ from datetime import datetime
import numpy as np import numpy as np
sys.path.insert(0, __file__.rsplit("/", 1)[0]) sys.path.insert(0, __file__.rsplit("/", 1)[0])
from correlate import SDR_RE # noqa: E402 from correlate import SDR_RE, expected_repeat # noqa: E402
ap = argparse.ArgumentParser() ap = argparse.ArgumentParser()
ap.add_argument("--iq", required=True) ap.add_argument("--iq", required=True)
@@ -57,23 +57,35 @@ for line in open(args.decodes, errors="replace"):
"crc_ok": m["crc"] == "1", "crc_ok": m["crc"] == "1",
"t": datetime.strptime(m["ts"].strip(), "%Y-%m-%d %H:%M:%S.%f").timestamp(), "t": datetime.strptime(m["ts"].strip(), "%Y-%m-%d %H:%M:%S.%f").timestamp(),
"serial": m["serial"].lower(), "serial": m["serial"].lower(),
"hex": m["hex"].lower(), "cooked": bytes.fromhex(m["hex"]),
}) })
rows.sort(key=lambda r: r["t"]) rows.sort(key=lambda r: r["t"])
c1 = [r for r in rows if r["mode"] == "C1" and r["crc_ok"]] decoded = [r for r in rows if r["mode"] in ("C1", "T1") and r["crc_ok"]]
# rtl_wmbus runs two demodulators; collapse identical bytes <5 ms apart # rtl_wmbus runs two demodulators; collapse identical bytes <5 ms apart
dedup = [] dedup = []
for r in c1: for r in decoded:
if dedup and dedup[-1]["hex"] == r["hex"] and r["t"] - dedup[-1]["t"] < 0.005: if (dedup and dedup[-1]["mode"] == r["mode"]
and dedup[-1]["cooked"] == r["cooked"]
and r["t"] - dedup[-1]["t"] < 0.005):
continue continue
dedup.append(r) dedup.append(r)
# A decoded repeat: identical payload seen twice. rtl_sdr delivers ~82 ms # rtl_sdr delivers ~82 ms buffers, so decode timestamps are quantised.
# buffers so decode timestamps are quantised; count occurrences rather than # Preserve order rather than trusting sub-100 ms timing (docs/FINDINGS.md).
# trusting sub-100 ms timing (docs/FINDINGS.md). # Each supported H=0 original permits one later H=1 counterpart; never
occ = Counter(r["hex"] for r in dedup) # normalize H on both sides or count identical originals as repeats.
decoded_repeats = sum(v - 1 for v in occ.values() if v > 1) pending = Counter()
decoded_repeats = 0
for r in dedup:
key = (r["mode"], r["cooked"])
if pending[key]:
pending[key] -= 1
decoded_repeats += 1
else:
marked = expected_repeat(r["cooked"])
if marked is not None:
pending[(r["mode"], marked)] += 1
meters = Counter(r["serial"] for r in dedup) meters = Counter(r["serial"] for r in dedup)
# ---------------------------------------------------------------- IQ bursts # ---------------------------------------------------------------- IQ bursts
@@ -123,7 +135,7 @@ binfo = [b for b in binfo_all
# RF-level repeat: a burst followed by a comparable-or-stronger burst of # RF-level repeat: a burst followed by a comparable-or-stronger burst of
# similar duration inside the delay window. Duration similarity matters -- # similar duration inside the delay window. Duration similarity matters --
# a repeat carries the same frame, so its airtime should match. # setting H leaves the frame length unchanged, so its airtime should match.
pairs = [] pairs = []
for i, a in enumerate(binfo): for i, a in enumerate(binfo):
a_end = a["t_ms"] + a["dur_ms"] a_end = a["t_ms"] + a["dur_ms"]
@@ -131,7 +143,7 @@ for i, a in enumerate(binfo):
gap = b["t_ms"] - a_end gap = b["t_ms"] - a_end
if gap > args.delay_ms + args.delay_tol_ms: if gap > args.delay_ms + args.delay_tol_ms:
break break
# A repeat re-sends the SAME frame, so it must (a) sit at the # A hop-marked repeat retains the frame length, so it must (a) sit at the
# configured delay, (b) have an airtime consistent with a C1 frame, # configured delay, (b) have an airtime consistent with a C1 frame,
# and (c) match the original's duration closely. Pairs that merely # and (c) match the original's duration closely. Pairs that merely
# fall in a wide window are usually unrelated 868 MHz traffic -- # fall in a wide window are usually unrelated 868 MHz traffic --
@@ -153,7 +165,8 @@ result = {
"noise_floor_db": round(floor, 1), "noise_floor_db": round(floor, 1),
"bursts_total": len(binfo_all), "bursts_total": len(binfo_all),
"frame_like_bursts": len(binfo), "frame_like_bursts": len(binfo),
"c1_decodes": len(dedup), "c1_decodes": sum(r["mode"] == "C1" for r in dedup),
"t1_decodes": sum(r["mode"] == "T1" for r in dedup),
"unique_meters": len(meters), "unique_meters": len(meters),
"decoded_repeats": decoded_repeats, "decoded_repeats": decoded_repeats,
"rf_repeat_pairs": len(pairs), "rf_repeat_pairs": len(pairs),
@@ -176,7 +189,7 @@ if args.json:
print(json.dumps(result, indent=2)) print(json.dumps(result, indent=2))
else: else:
print(f"STANDALONE CHECK iq={result['iq_seconds']}s floor={floor:.1f}dB") print(f"STANDALONE CHECK iq={result['iq_seconds']}s floor={floor:.1f}dB")
print(f" C1 telegrams decoded: {len(dedup)} from {len(meters)} meters") print(f" C1/T1 telegrams decoded: {len(dedup)} from {len(meters)} meters")
print(f" RF bursts (all): {len(binfo_all)}") print(f" RF bursts (all): {len(binfo_all)}")
print(f" frame-like bursts: {len(binfo)} (>= {MIN_FRAME_MS} ms, >= +{MIN_FRAME_DB:.0f} dB)") print(f" frame-like bursts: {len(binfo)} (>= {MIN_FRAME_MS} ms, >= +{MIN_FRAME_DB:.0f} dB)")
print(f" decoded repeats: {decoded_repeats}") print(f" decoded repeats: {decoded_repeats}")
@@ -196,7 +209,7 @@ else:
print(" looks like a singleton. Read the firmware counters over USB") print(" looks like a singleton. Read the firmware counters over USB")
print(" for an unambiguous answer.)") print(" for an unambiguous answer.)")
elif v == "not observed": elif v == "not observed":
print(" (No frame-like burst pairs and no decoded duplicates. Either") print(" (No frame-like burst pairs and no decoded hop transitions. Either")
print(" the device is not transmitting, or it transmitted nothing in") print(" the device is not transmitting, or it transmitted nothing in")
print(" this window. Firmware counters over USB are decisive.)") print(" this window. Firmware counters over USB are decisive.)")
+9
View File
@@ -0,0 +1,9 @@
#!/usr/bin/env python3
"""Compress one embedded HTTP asset without runtime work or timestamp churn."""
import gzip
import sys
from pathlib import Path
source, destination = map(Path, sys.argv[1:])
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(gzip.compress(source.read_bytes(), compresslevel=9, mtime=0))
+7
View File
@@ -1,6 +1,8 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Deterministic discovery of DUT (Heltec V4.3) and SDR (Nooelec NESDR SMArt v5). # Deterministic discovery of DUT (Heltec V4.3) and SDR (Nooelec NESDR SMArt v5).
# Prints selected devices; exits nonzero on ambiguity or absence. # Prints selected devices; exits nonzero on ambiguity or absence.
# ./tools/discover_devices.sh [--dut-only|-v]
# --dut-only prints only DUT_PORT and does not inspect or require an SDR.
# #
# Output (eval-able): # Output (eval-able):
# DUT_PORT=/dev/serial/by-id/... # DUT_PORT=/dev/serial/by-id/...
@@ -22,6 +24,11 @@ fi
DUT_PORT="${dut_candidates[0]}" DUT_PORT="${dut_candidates[0]}"
[ -e "$DUT_PORT" ] || fail "DUT path vanished: $DUT_PORT" [ -e "$DUT_PORT" ] || fail "DUT path vanished: $DUT_PORT"
if [ "${1:-}" = "--dut-only" ]; then
printf 'DUT_PORT=%q\n' "$DUT_PORT"
exit 0
fi
# --- SDR: Nooelec NESDR SMArt v5 (RTL2832U, USB 0bda:2838) --- # --- SDR: Nooelec NESDR SMArt v5 (RTL2832U, USB 0bda:2838) ---
# Identity is established by USB VID:PID, because this dongle sometimes # Identity is established by USB VID:PID, because this dongle sometimes
# enumerates with empty descriptor strings after a fault (rtl_test then # enumerates with empty descriptor strings after a fault (rtl_test then
+5 -3
View File
@@ -1,12 +1,13 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Build + flash a firmware profile to the discovered DUT. # Build + flash a firmware profile to the discovered DUT; no SDR required.
# ./tools/flash.sh [test|debug] # ./tools/flash.sh [test|debug]
# Refuses ambiguous serial devices (see tools/discover_devices.sh). # Refuses ambiguous serial devices (see tools/discover_devices.sh).
set -euo pipefail set -euo pipefail
cd "$(dirname "$0")/.." cd "$(dirname "$0")/.."
PROFILE=${1:-test} PROFILE=${1:-test}
eval "$(./tools/discover_devices.sh)" devices=$(./tools/discover_devices.sh --dut-only)
eval "$devices"
echo "Flashing profile '$PROFILE' to: $DUT_PORT" >&2 echo "Flashing profile '$PROFILE' to: $DUT_PORT" >&2
./tools/build.sh "$PROFILE" ./tools/build.sh "$PROFILE"
@@ -16,7 +17,8 @@ if ! idf.py -B "build-$PROFILE" -p "$DUT_PORT" flash; then
# entering download mode re-enumerates the USB device; rediscover once # entering download mode re-enumerates the USB device; rediscover once
sleep 2 sleep 2
cd .. cd ..
eval "$(./tools/discover_devices.sh)" devices=$(./tools/discover_devices.sh --dut-only)
eval "$devices"
cd firmware cd firmware
idf.py -B "build-$PROFILE" -p "$DUT_PORT" flash idf.py -B "build-$PROFILE" -p "$DUT_PORT" flash
fi fi