Update HIL verification for hop-marked repeats

This commit is contained in:
Lars Nolden
2026-09-21 12:31:19 +02:00
parent a11a4b3b89
commit 8c44c54131
13 changed files with 525 additions and 309 deletions
+11 -3
View File
@@ -33,15 +33,23 @@ for i in $(seq 1 "$SEGS"); do
V=$(./result-py/bin/python3 hil/meter_anchored.py \
--iq "$D/iq.raw" --serial "$D/serial.log" --json 2>"$D/verdict.err")
RC=$?
STATE=""
if [ -n "$V" ]; then
echo "$V" | ./result-py/bin/python3 -c \
STATE=$(printf '%s\n' "$V" | ./result-py/bin/python3 -c \
"import json,sys; print(json.load(sys.stdin).get('rf_state','unknown'))")
printf '%s\n' "$V" | ./result-py/bin/python3 -c \
"import json,sys,time; d=json.load(sys.stdin); d['seg']=$i; d['rc']=$RC; d['t']=int(time.time()); print(json.dumps(d))" \
>> "$OUT/soak.jsonl"
else
echo "{\"seg\":$i,\"rc\":$RC,\"error\":\"no_verdict\",\"t\":$(date +%s)}" >> "$OUT/soak.jsonl"
fi
# keep disk bounded: drop IQ after verdict, keep serial
rm -f "$D/iq.raw"
# Strong segments are summarized and discarded. Retain every weak,
# absent, or unclassified segment so a hardware conclusion is auditable.
if [ "$STATE" = "strong" ] && [ "$RC" = "0" ]; then
rm -f "$D/iq.raw"
else
echo "{\"seg\":$i,\"note\":\"iq_retained\",\"rf_state\":\"$STATE\",\"t\":$(date +%s)}" >> "$OUT/soak.jsonl"
fi
if grep -q "lost at least" "$D/rtl_sdr.err" 2>/dev/null; then
echo "{\"seg\":$i,\"note\":\"sample_drops\",\"t\":$(date +%s)}" >> "$OUT/soak.jsonl"
fi
+55 -13
View File
@@ -10,8 +10,8 @@ Output: JSON report on stdout.
Correlation model (developer repeat delay ~50 ms):
SDR original <-> Heltec RX : same cooked bytes, |dt| < rx_window
Heltec RX <-> Heltec TX : same hash (firmware log)
Heltec TX <-> SDR repeat : same cooked bytes, dt in [min_delay, max_delay]
after the SDR original
SDR original <-> SDR repeat : only H changes 0 -> 1 on a supported header,
dt in [min_delay, max_delay]
The Heltec logs raw over-the-air frames (with CRCs); rtl_wmbus prints
"cooked" frames (CRCs stripped). cook() below mirrors rtl-wmbus.
@@ -69,6 +69,42 @@ def cook_b(frame: bytes):
return bytes(out)
def expected_repeat(cooked: bytes):
"""Return the H-marked counterpart of a supported CRC-stripped original.
Both format A and rtl-wmbus-normalized format B use L = cooked length - 1.
ELL hop control takes precedence over any encapsulated transport header.
Unsupported layouts, H=1 and R=1 cannot originate a received repeat.
"""
if len(cooked) < 11 or cooked[0] != len(cooked) - 1:
return None
if cooked[1] not in (0x44, 0x46):
return None
ci = cooked[10]
if ci in (0x8c, 0x8e):
# CI + CC + ACC, optionally eight address bytes, then another CI.
if len(cooked) < (14 if ci == 0x8c else 22):
return None
pos, h_mask = 11, 0x10
else:
if ci in (0x67, 0x6e, 0x74, 0x7a, 0x7d, 0x7f, 0x88, 0x8a,
0x9e, 0xc1, 0xc4):
pos = 13
elif ci in (0x68, 0x6f, 0x72, 0x75, 0x7c, 0x7e, 0x87, 0x8b,
0x9f, 0xc2, 0xc5):
pos = 21
else:
return None
if len(cooked) <= pos + 1 or cooked[pos + 1] & 0x1f not in (0, 5):
return None
h_mask = 0x01
if cooked[pos] & (h_mask | 0x02):
return None
marked = bytearray(cooked)
marked[pos] |= h_mask
return bytes(marked)
def parse_serial(path):
"""Heltec JSON-lines -> (rx_events, tx_events, other)."""
rx, tx, other = [], [], []
@@ -133,31 +169,37 @@ def main():
heltec_rx, heltec_tx, other = parse_serial(args.serial)
sdr = parse_sdr(args.sdr)
sdr_c1 = [s for s in sdr if s["mode"] == "C1" and s["crc_ok"]]
sdr_frames = [s for s in sdr if s["mode"] in ("C1", "T1") and s["crc_ok"]]
# rtl_wmbus runs two demodulator algorithms concurrently; a single burst
# can be reported twice within ~1 ms. Collapse identical bytes < 5 ms apart.
deduped = []
for s in sorted(sdr_c1, key=lambda x: x["t"]):
if deduped and deduped[-1]["cooked"] == s["cooked"] \
for s in sorted(sdr_frames, key=lambda x: x["t"]):
if deduped and deduped[-1]["mode"] == s["mode"] \
and deduped[-1]["cooked"] == s["cooked"] \
and s["t"] - deduped[-1]["t"] < 0.005:
continue
deduped.append(s)
sdr_c1 = deduped
sdr_frames = deduped
expected = [expected_repeat(s["cooked"]) for s in sdr_frames]
# --- classify SDR frames into originals vs repeats ---------------------
# A repeat is an SDR frame with identical cooked bytes to an earlier SDR
# frame within [min_delay, max_delay]. Everything else is an original.
# A repeat changes only H from 0 to 1 on a supported original header
# within [min_delay, max_delay]. Same-H duplicates are not repeats.
originals, repeats = [], []
for i, s in enumerate(sdr_c1):
used_originals = set()
for i, s in enumerate(sdr_frames):
matched = None
for j in range(i - 1, -1, -1):
prev = sdr_c1[j]
prev = sdr_frames[j]
dt = s["t"] - prev["t"]
if dt > args.max_delay:
break
if prev["cooked"] == s["cooked"] and args.min_delay <= dt <= args.max_delay:
if (j not in used_originals and prev["mode"] == s["mode"]
and expected[j] == s["cooked"]
and args.min_delay <= dt <= args.max_delay):
matched = (j, dt)
used_originals.add(j)
break
if matched is not None:
repeats.append({"idx": i, "orig_idx": matched[0], "dt": matched[1], **s})
@@ -197,7 +239,6 @@ def main():
verified_repeats = 0
for rep in repeats:
orig = sdr_c1[rep["orig_idx"]]
o_entry = next((o for o in originals if o["idx"] == rep["orig_idx"]), None)
hr = o_entry.get("heltec_rx") if o_entry else None
if hr and hr.get("hash") in tx_by_hash:
@@ -213,7 +254,8 @@ def main():
report = {
"sdr_frames_total": len(sdr),
"sdr_c1_crc_ok": len(sdr_c1),
"sdr_c1_crc_ok": sum(s["mode"] == "C1" for s in sdr_frames),
"sdr_t1_crc_ok": sum(s["mode"] == "T1" for s in sdr_frames),
"sdr_originals": len(originals),
"sdr_repeats": len(repeats),
"heltec_rx": len(heltec_rx),
+94 -14
View File
@@ -9,10 +9,10 @@ The only trustworthy way we have found to verify DUT transmissions in IQ:
3. Report ambient meter levels alongside -- they are the in-band health
control for the observer (deaf dongle => meters compressed => run void).
Why not fit the clock on TX events (hil/verify_repeats_iq.py, single-burst
mode): repeats fire RX + delay by construction, so an offset shifted by
-delay "matches" every repeat to its own original's burst. That artifact
produced fake "7/7 at +37 dB" confirmations. See docs/FINDINGS.md.
Why not fit the clock on TX events (the removed verify_repeats_iq.py did):
repeats fire RX + delay by construction, so an offset shifted by -delay
"matches" every repeat to its own original's burst. That artifact produced
fake "7/7 at +37 dB" confirmations. See docs/FINDINGS.md.
Usage: meter_anchored.py --iq FILE --serial FILE [--json]
"""
@@ -71,6 +71,62 @@ def fit_clock(bursts, anchors_ms, span_ms):
best = (off, m, e)
return best # offset, matched, median_err
def fsk_spectral_excess(path, db, floor, event_times_ms):
"""Return event-correlated 45 kHz FSK excess over nearby control windows."""
nfft = 2048
fft_window = np.hanning(nfft)
raw = np.memmap(path, dtype=np.uint8, mode="r")
sample_count = raw.size // 2
event_times = np.asarray(event_times_ms, dtype=float)
event_psd, control_psd = [], []
def spectrum(center):
lo = center - nfft // 2
hi = lo + nfft
if lo < 0 or hi > sample_count:
return None
values = raw[2 * lo:2 * hi].astype(np.float32) - 127.5
iq = values[0::2] + 1j * values[1::2]
return np.abs(np.fft.fftshift(np.fft.fft(iq * fft_window))) ** 2
for ts_ms in event_times:
a0, a1 = int(ts_ms) - 2, int(ts_ms) + 12
if not (0 <= a0 < a1 < len(db)):
continue
peak_bin = a0 + int(np.argmax(db[a0:a1] - floor))
event = spectrum(peak_bin * WIN + WIN // 2)
if event is None:
continue
control = None
for delta_ms in (100, 200, -100, -200):
control_bin = peak_bin + delta_ms
if not (0 <= control_bin < len(db)):
continue
if np.any(np.abs(event_times - control_bin) < 25):
continue
control = spectrum(control_bin * WIN + WIN // 2)
if control is not None:
break
if control is None:
continue
event_psd.append(event)
control_psd.append(control)
if len(event_psd) < 5:
return None, len(event_psd)
event_med = np.median(np.stack(event_psd), axis=0)
control_med = np.median(np.stack(control_psd), axis=0)
freq = np.fft.fftshift(np.fft.fftfreq(nfft, 1 / FS))
signal = (np.abs(freq) >= 20000) & (np.abs(freq) < 80000)
wide = (np.abs(freq) >= 250000) & (np.abs(freq) < 700000)
signal_db = 10 * np.log10(event_med[signal].sum()
/ control_med[signal].sum())
wide_db = 10 * np.log10(event_med[wide].sum()
/ control_med[wide].sum())
return float(signal_db - wide_db), len(event_psd)
def main():
ap = argparse.ArgumentParser()
@@ -124,19 +180,35 @@ def main():
return None
return float((db[a0:a1] - floor).max())
events = []
events, mapped_tx = [], []
for t in tx:
w = winmax(t["timestamp_us"] / 1000.0 - off)
mapped = t["timestamp_us"] / 1000.0 - off
w = winmax(mapped)
if w is None:
continue
mapped_tx.append(mapped)
events.append({"event": t.get("event", "wmbus_tx"),
"result": t.get("result"),
"window_db": round(w, 1),
"rf": bool(w >= 6)})
"above_6db": bool(w >= 6)})
meter_med = float(np.median(meter_lv)) if meter_lv else 0.0
observer = "ok" if meter_med >= 15 else ("degraded" if meter_med >= 8 else "deaf")
n_rf = sum(1 for e in events if e["rf"])
n_above = sum(1 for e in events if e["above_6db"])
levels = [e["window_db"] for e in events]
tx_med = float(np.median(levels)) if levels else None
controls = [winmax(t + 100) for t in mapped_tx]
controls = [v for v in controls if v is not None]
control_med = float(np.median(controls)) if controls else None
fsk_db, fsk_windows = fsk_spectral_excess(
args.iq, db, floor, mapped_tx)
if not events:
rf_state = "no_events"
elif fsk_db is not None and fsk_db >= 3.0:
rf_state = "strong" if tx_med >= 6.0 else "weak"
else:
rf_state = "not_observed"
report = {
"clock_matched": f"{matched}/{len(strong)}",
"clock_err_ms": round(err, 1),
@@ -144,8 +216,14 @@ def main():
"meter_median_db": round(meter_med, 1),
"observer": observer,
"tx_events": len(events),
"tx_with_rf": n_rf,
"tx_levels_db": [e["window_db"] for e in events],
"tx_above_6db": n_above,
"tx_median_db": round(tx_med, 1) if tx_med is not None else None,
"control_median_db": (round(control_med, 1)
if control_med is not None else None),
"fsk_excess_db": round(fsk_db, 1) if fsk_db is not None else None,
"fsk_windows": fsk_windows,
"rf_state": rf_state,
"tx_levels_db": levels,
}
if args.json:
print(json.dumps(report))
@@ -153,14 +231,16 @@ def main():
print(f"clock: {report['clock_matched']} anchors, err {report['clock_err_ms']} ms")
print(f"observer: {observer} (meter median +{report['meter_median_db']} dB, "
f"floor {report['floor_db']})")
print(f"TX with RF: {n_rf}/{len(events)}")
print(f"TX state: {rf_state}; median +{report['tx_median_db']} dB, "
f"FSK excess {report['fsk_excess_db']} dB")
for e in events:
print(f" {e['event']:>9} {e['result'] or '':>10} "
f"+{e['window_db']:>5.1f} dB {'RF' if e['rf'] else '--'}")
# exit 3 = observer unusable, 1 = TX dead while observer ok
f"+{e['window_db']:>5.1f} dB "
f"{'>=6dB' if e['above_6db'] else 'weak'}")
# exit 3 = observer unusable, 1 = no event-correlated FSK while observer ok
if observer == "deaf":
sys.exit(3)
sys.exit(0 if (n_rf or not events) else 1)
sys.exit(0 if rf_state in ("strong", "weak", "no_events") else 1)
if __name__ == "__main__":
+23 -21
View File
@@ -20,7 +20,7 @@ ap.add_argument("--tx-power", default="?")
ap.add_argument("--repeat-delay", default="?")
ap.add_argument("--sdr-streamed", default="?")
ap.add_argument("--tx-proof", default=None)
ap.add_argument("--iq-proof", default=None)
ap.add_argument("--rf-proof", default=None)
ap.add_argument("--sdr-sensitivity", default="?")
args = ap.parse_args()
@@ -38,7 +38,7 @@ def load_json(path):
proof = load_json(args.tx_proof)
iqp = load_json(args.iq_proof)
rfp = load_json(args.rf_proof)
def fmt(v, unit=""):
@@ -63,11 +63,13 @@ if fw.get("boots", 0) > 1:
if proof is not None and proof.get("bursts_with_sdr_decode", 0) == 0:
fail.append("SDR could not decode any Heltec C1 transmission")
# A decoded repeat is preferred, but a decode-independent RF observation of
# a repeat at the configured delay also satisfies "the observer
# independently saw the retransmission" (see docs/RF_FRONTEND_TESTS.md for
# why decoding our own TX is bench-limited).
if iqp is not None and iqp.get("tx_events_with_rf", 0) > 0:
# Decoded H=0 -> H=1 transitions are preferred. Meter-anchored, spectrally
# classified FSK at firmware repeat times is an independent RF proof when
# the nearby transmission overloads the decoder.
rf_confirmed = (rfp is not None
and rfp.get("rf_state") in ("strong", "weak")
and rfp.get("tx_events", 0) >= args.min_repeats)
if rf_confirmed:
fail = [f for f in fail if not f.startswith("SDR-verified repeats")]
verdict = "PASS" if not fail else "FAIL"
@@ -99,25 +101,25 @@ TX DECODABILITY PROOF (synthetic C1, independent of meters):
SDR decoded as C1: {proof.get('bursts_with_sdr_decode')}
both airings decoded: {proof.get('bursts_with_both_airings_decoded')}""")
if iqp is not None:
if rfp is not None:
print(f"""
RF-LEVEL TX PROOF (same stream's raw IQ, decode-independent):
IQ analysed: {iqp.get('iq_seconds')} s
clipped samples: {iqp.get('clipped_sample_pct')} %
bursts in window: {iqp.get('bursts_total')}
firmware TX ok: {iqp.get('firmware_tx_ok')}
TX events in IQ window: {iqp.get('tx_events_in_iq_window')}
TX events with RF: {iqp.get('tx_events_with_rf')} ({iqp.get('tx_rf_confirmed_pct')} %)
alignment error: {iqp.get('align_error_ms')} ms
airtime error: {iqp.get('burst_duration_error_ms')} ms
our TX over floor: {iqp.get('our_tx_peak_db_over_floor')} dB
meter bursts over floor: {iqp.get('other_burst_peak_db_over_floor')} dB
firmware repeat delay: {iqp.get('firmware_repeat_delay_ms')} ms""")
RF-LEVEL TX PROOF (meter-anchored IQ + non-TX controls):
clock anchors: {rfp.get('clock_matched')}
clock median error: {rfp.get('clock_err_ms')} ms
observer state: {rfp.get('observer')}
meter level over floor: {rfp.get('meter_median_db')} dB
firmware TX events: {rfp.get('tx_events')}
RF state: {rfp.get('rf_state')}
TX median over floor: {rfp.get('tx_median_db')} dB
control median: {rfp.get('control_median_db')} dB
FSK-selective excess: {rfp.get('fsk_excess_db')} dB
spectral windows: {rfp.get('fsk_windows')}""")
print(f"""
SDR (independent observer):
decode lines: {r.get('sdr_frames_total')}
C1 CRC-ok bursts: {r.get('sdr_c1_crc_ok')}
T1 CRC-ok bursts: {r.get('sdr_t1_crc_ok', 0)}
classified originals: {r.get('sdr_originals')}
classified repeats: {r.get('sdr_repeats')}
unique meters: {len(r.get('per_meter') or {})}
@@ -133,7 +135,7 @@ HELTEC TX:
reported success: {r.get('heltec_tx_ok')}
SDR-VERIFIED REPEATS:
byte-identical pairs: {r.get('sdr_repeats')}
H=0 -> H=1 pairs: {r.get('sdr_repeats')}
fully correlated: {r.get('fully_correlated_repeats')}
vs TX attempts: {fmt(r.get('repeat_success_rate_pct'), ' %')}
+23 -10
View File
@@ -37,6 +37,11 @@ while [ $# -gt 0 ]; do
esac
done
if ! [[ "$TXPOW" =~ ^-?[0-9]+$ ]] || (( TXPOW < -9 || TXPOW > 2 )); then
echo "--tx-power must be an integer in -9..2" >&2
exit 2
fi
PY=./result-py/bin/python3
[ -x "$PY" ] || { echo "run 'nix build .#hil-python -o result-py' first" >&2; exit 2; }
@@ -154,21 +159,29 @@ cat "$RUN_DIR/final_stats.log" >> "$RUN_DIR/serial.log"
# --- RF-level repeat proof from the same stream's IQ ----------------------
if [ -s "$IQ" ]; then
echo "RF-level repeat proof from captured IQ"
$PY hil/verify_repeats_iq.py --iq "$IQ" --serial "$RUN_DIR/serial.log" \
--delay-ms "$DELAY" > "$RUN_DIR/iq_proof.txt" 2>&1 || true
$PY hil/verify_repeats_iq.py --iq "$IQ" --serial "$RUN_DIR/serial.log" \
--delay-ms "$DELAY" --json > "$RUN_DIR/iq_proof.json" 2>&1 || true
cat "$RUN_DIR/iq_proof.txt"
# IQ is large: keep it only when the proof failed, for offline inspection
if grep -q '"tx_events_with_rf": [1-9]' "$RUN_DIR/iq_proof.json" 2>/dev/null; then
echo "RF-level repeat proof from meter-anchored IQ"
RF_RC=0
if ! $PY hil/meter_anchored.py --iq "$IQ" \
--serial "$RUN_DIR/serial.log" --json \
> "$RUN_DIR/rf_proof.json" 2> "$RUN_DIR/rf_proof.err"; then
RF_RC=$?
fi
if [ -s "$RUN_DIR/rf_proof.json" ]; then
cat "$RUN_DIR/rf_proof.json"
else
cat "$RUN_DIR/rf_proof.err"
fi
# Preserve weak, absent and unclassifiable RF for offline audit.
if [ "$RF_RC" = "0" ] \
&& grep -q '"rf_state": "strong"' "$RUN_DIR/rf_proof.json" 2>/dev/null; then
rm -f "$IQ"
fi
fi
# --- 13-14. correlate ------------------------------------------------------
MAXDELAY=$(awk "BEGIN{printf \"%.3f\", ($DELAY + 200) / 1000}")
$PY hil/correlate.py --serial "$RUN_DIR/serial.log" --sdr "$RUN_DIR/rtlwmbus.log" \
--min-delay 0.010 --max-delay 0.400 > "$RUN_DIR/report.json"
--min-delay 0.010 --max-delay "$MAXDELAY" > "$RUN_DIR/report.json"
# --- 15. lockup / reset detection -----------------------------------------
HB=$(grep -c '"event":"heartbeat"' "$RUN_DIR/serial.log" || true)
@@ -189,7 +202,7 @@ $PY hil/report.py "$RUN_DIR/report.json" \
--sdr-streamed "$STREAMED" \
--sdr-sensitivity "${SENS:-unknown}" \
--tx-proof "$RUN_DIR/tx_proof.json" \
--iq-proof "$RUN_DIR/iq_proof.json" \
--rf-proof "$RUN_DIR/rf_proof.json" \
| tee "$RUN_DIR/report.txt"
RC=${PIPESTATUS[0]}
echo "artifacts: $RUN_DIR"
+2 -2
View File
@@ -69,8 +69,8 @@ if not tx:
print("TX: no repeat events recorded")
print(f" result breakdown: {dict(results)}")
print(f" repeat delay: {s(delays, ' ms')}")
print(f" TX airtime: {s(air, ' ms')}")
print(f" TX-routine delay: {s(delays, ' ms')}")
print(f" computed RF airtime: {s(air, ' ms')}")
print(f"""
HEALTH:
radio recoveries: {len(rec)}
+91
View File
@@ -0,0 +1,91 @@
#!/usr/bin/env bash
# Overnight segmented RF soak for a STANDALONE repeater (no serial link).
# Per segment: bounded rtl_sdr capture -> offline rtl_wmbus decode ->
# hil/verify_standalone.py verdict appended to soak.jsonl.
#
# Designed around the known observer faults (docs/HARDWARE.md): bounded
# segments, usbreset recovery on short captures, and a deaf-observer
# heuristic -- the neighbouring meter rails the dongle at a steady rate,
# so consecutive near-empty segments mean a dead receiver, not a quiet
# night. Disk is bounded: confirmed segments drop their IQ; only the two
# newest unconfirmed segments keep raw IQ for offline audit.
#
# Run via the devshell so usbreset is on PATH:
# nix develop --command ./hil/standalone_soak.sh 0855 300
#
# Usage: standalone_soak.sh [END_HHMM] [SEG_SECONDS] [DELAY_MS]
# standalone_soak.sh 0855 300 500
set -u
cd "$(dirname "$0")/.."
END_HHMM=${1:-0855}
SEG=${2:-300}
DELAY_MS=${3:-500} # must match the DUT's persisted repeat_delay_ms
PY=./result-py/bin/python3
WMBUS=./result-rtl-wmbus/bin/rtl_wmbus
[ -x "$PY" ] || { echo "run 'nix build .#hil-python -o result-py' first" >&2; exit 2; }
[ -x "$WMBUS" ] || { echo "missing result-rtl-wmbus" >&2; exit 2; }
OUT="captures/$(date +%Y-%m-%dT%H-%M-%S)-standalone-soak"
mkdir -p "$OUT"
END_EPOCH=$(date -d "$END_HHMM" +%s 2>/dev/null) || { echo "bad end time: $END_HHMM" >&2; exit 2; }
if [ "$END_EPOCH" -le "$(date +%s)" ]; then
END_EPOCH=$(date -d "tomorrow $END_HHMM" +%s)
fi
echo "soak until $(date -d "@$END_EPOCH" '+%F %T'), segment ${SEG}s -> $OUT"
command -v usbreset >/dev/null || echo "WARNING: usbreset not on PATH; dongle recovery disabled" >&2
low_bursts=0
short_caps=0
i=0
while [ $(( $(date +%s) + SEG )) -le "$END_EPOCH" ]; do
i=$((i + 1))
D="$OUT/seg$i"
mkdir -p "$D"
timeout $((SEG + 60)) rtl_sdr -f 868950000 -s 1600000 -g 42.1 \
-n $((SEG * 1600000)) "$D/iq.raw" 2> "$D/rtl_sdr.err"
SZ=$(stat -c %s "$D/iq.raw" 2>/dev/null || echo 0)
T=$(date +%s)
if [ "$SZ" -lt $((SEG * 3200000 * 8 / 10)) ]; then
echo "{\"seg\":$i,\"error\":\"short_capture\",\"bytes\":$SZ,\"t\":$T}" >> "$OUT/soak.jsonl"
rm -f "$D/iq.raw"
usbreset 0bda:2838 >/dev/null 2>&1
short_caps=$((short_caps + 1))
# A de-enumerated dongle (needs physical replug) is unrecoverable
# from software: back off instead of hammering, keep probing in
# case it re-appears.
if [ "$short_caps" -ge 5 ]; then sleep 300; else sleep 20; fi
continue
fi
short_caps=0
V=$($PY hil/verify_standalone.py --iq "$D/iq.raw" --delay-ms "$DELAY_MS" \
--decodes "$D/decodes.log" --json 2> "$D/verdict.err")
RC=$?
if [ -n "$V" ]; then
printf '%s\n' "$V" | $PY -c \
"import json,sys; d=json.load(sys.stdin); d['seg']=$i; d['rc']=$RC; d['t']=$T; print(json.dumps(d))" \
>> "$OUT/soak.jsonl"
else
echo "{\"seg\":$i,\"rc\":$RC,\"error\":\"no_verdict\",\"t\":$T}" >> "$OUT/soak.jsonl"
fi
if grep -q "lost at least" "$D/rtl_sdr.err" 2>/dev/null; then
echo "{\"seg\":$i,\"note\":\"sample_drops\",\"t\":$T}" >> "$OUT/soak.jsonl"
fi
B=$(printf '%s\n' "$V" | $PY -c \
"import json,sys; print(json.load(sys.stdin).get('bursts_total',0))" \
2>/dev/null || echo 0)
if [ "${B:-0}" -lt 3 ]; then low_bursts=$((low_bursts + 1)); else low_bursts=0; fi
if [ "$low_bursts" -ge 2 ]; then
echo "{\"seg\":$i,\"note\":\"observer_suspect_reset\",\"t\":$(date +%s)}" >> "$OUT/soak.jsonl"
usbreset 0bda:2838 >/dev/null 2>&1
low_bursts=0
fi
if printf '%s' "$V" | grep -q '"verdict": "confirmed"'; then
rm -f "$D/iq.raw"
fi
ls -1t "$OUT"/seg*/iq.raw 2>/dev/null | tail -n +3 | xargs -r rm -f
sleep 15
done
echo "soak complete: $OUT/soak.jsonl"
+172
View File
@@ -0,0 +1,172 @@
#!/usr/bin/env python3
"""Verify standalone beacon TX from RF evidence alone (no serial log).
Use case: DUT on battery/wall power with no USB host, transmitting the
synthetic OMS beacon every N ms. Two independent lines of evidence from
one recorded IQ stream:
1. Content: rtl_wmbus decodes of the known beacon frame
(mfr 0x4D8E "SLN", ID 00000001). Decode timestamps are clustered into
airings (two demodulators emit 1-2 lines per airing).
2. Timing: short bursts in the IQ envelope forming a comb at the beacon
period. The effective period is fitted from the data, because the
firmware schedules the next beacon after the previous TX completes
(configured period + a few ms), and a beacon can be delayed by RX
activity.
Ambient (non-beacon) decodes and long bursts are reported as the observer
health control.
Usage:
verify_beacon_standalone.py --iq iq.raw --decodes decodes.log \
--period-ms 3000 [--rate 1600000] [--json]
Exit: 0 = beacon TX confirmed, 1 = not observed, 2 = insufficient data.
"""
import argparse
import json
import re
import sys
import numpy as np
BEACON_HEX = "448e4d010000000107" # C + mfr + ID + ver + type, L-agnostic
DECODE_RE = re.compile(
r"^(?P<mode>[TC]1);\d+;\d+;(?P<ts>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+);"
r".*0x(?P<hex>[0-9a-fA-F]+)")
def parse_decodes(path):
# Distinct airings are counted by distinct payload: the beacon's
# access number increments per airing, so each airing has unique hex,
# while rtl_wmbus's two demodulators emitting 1-2 lines per airing
# collapse to one. Timestamp-independent, so it works for both live
# captures and offline `rtl_wmbus < iq.raw` (compressed wall clock).
# acc wraps at 256: valid for captures up to 256 beacon periods.
beacon_hex, beacon_lines, ambient = set(), 0, 0
for line in open(path, errors="replace"):
m = DECODE_RE.match(line.strip())
if not m:
continue
if BEACON_HEX in m.group("hex").lower():
beacon_hex.add(m.group("hex").lower())
beacon_lines += 1
else:
ambient += 1
return len(beacon_hex), beacon_lines, ambient
def envelope_bursts(path, rate):
win = rate // 1000 # 1 ms bins
env = []
with open(path, "rb") as f:
while True:
raw = f.read(rate * 2)
if len(raw) < win * 2:
break
u8 = np.frombuffer(raw, dtype=np.uint8)
b = u8.astype(np.float32) - 127.5
iq = b[0::2] + 1j * b[1::2]
p = np.abs(iq) ** 2
p = p[: (len(p) // win) * win].reshape(-1, win).mean(axis=1)
env.append(p)
if not env:
return None, 0.0, []
db = 10 * np.log10(np.concatenate(env) + 1e-9)
floor = float(np.median(db))
idx = np.where(db > floor + 6.0)[0]
bursts = []
if len(idx):
start = prev = idx[0]
for i in idx[1:]:
if i > prev + 3:
bursts.append((int(start), int(prev - start + 1),
float(db[start:prev + 1].max() - floor)))
start = i
prev = i
bursts.append((int(start), int(prev - start + 1),
float(db[start:prev + 1].max() - floor)))
return db, floor, bursts
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--iq", required=True)
ap.add_argument("--decodes")
ap.add_argument("--period-ms", type=float, required=True,
help="configured beacon_period_ms")
ap.add_argument("--rate", type=int, default=1_600_000)
ap.add_argument("--json", action="store_true")
args = ap.parse_args()
db, floor, bursts = envelope_bursts(args.iq, args.rate)
if db is None:
print("IQ empty or unreadable", file=sys.stderr)
sys.exit(2)
# Beacon candidates: short bursts (L=19 beacon airtime is 2.7 ms;
# envelope smearing widens that). Longer bursts are ambient meters.
cand = [b for b in bursts if b[1] <= 8]
long_bursts = [b for b in bursts if b[1] > 8]
# Fit the effective period from consecutive candidate intervals near
# the configured period, then count on-grid intervals (a missed or
# RX-delayed slot shows up as ~2x the period and is counted as a gap).
starts = np.array([b[0] for b in cand], dtype=float)
iv = np.diff(starts)
p = args.period_ms
near = iv[(iv > 0.8 * p) & (iv < 1.35 * p)]
fitted = float(np.median(near)) if len(near) else None
on_grid = int(len(near))
gaps = int(np.count_nonzero((iv > 1.6 * p) & (iv < 2.6 * p)))
cand_levels = [b[2] for b in cand]
airings, beacon_lines, ambient_decodes = 0, 0, 0
if args.decodes:
airings, beacon_lines, ambient_decodes = parse_decodes(args.decodes)
decode_ok = airings >= 5
comb_ok = on_grid >= 5 and fitted is not None
if decode_ok or comb_ok:
verdict = "confirmed"
rc = 0
elif len(cand) or airings:
verdict = "insufficient"
rc = 2
else:
verdict = "not_observed"
rc = 1
report = {
"iq_seconds": round(len(db) / 1000.0, 1),
"noise_floor_db": round(floor, 1),
"beacon_candidate_bursts": len(cand),
"candidate_median_db": (round(float(np.median(cand_levels)), 1)
if cand_levels else None),
"period_configured_ms": p,
"period_fitted_ms": round(fitted, 1) if fitted else None,
"intervals_on_grid": on_grid,
"missed_slot_gaps": gaps,
"beacon_decoded_airings": airings,
"beacon_decode_lines": beacon_lines,
"ambient_decodes": ambient_decodes,
"ambient_long_bursts": len(long_bursts),
"verdict": verdict,
}
if args.json:
print(json.dumps(report))
else:
print(f"IQ: {report['iq_seconds']} s, floor {report['noise_floor_db']} dB")
print(f"beacon bursts: {len(cand)} (median "
f"+{report['candidate_median_db']} dB), period fitted "
f"{report['period_fitted_ms']} ms vs configured {p:.0f} ms, "
f"{on_grid} intervals on grid, {gaps} missed-slot gaps")
print(f"decoded beacon airings: {airings} ({beacon_lines} lines); "
f"ambient decodes: {ambient_decodes}; "
f"ambient long bursts: {len(long_bursts)}")
print(f"VERDICT: {verdict}")
sys.exit(rc)
if __name__ == "__main__":
main()
-223
View File
@@ -1,223 +0,0 @@
#!/usr/bin/env python3
"""Verify repeater transmissions from raw IQ, independent of packet decoding.
Why this exists: on this bench the Heltec transmission arrives at the SDR
far stronger than real meter traffic, and the dongle's own sensitivity
drifts by ~17 dB over minutes. rtl-wmbus therefore often receives our
repeat with a couple of byte errors (CRC fails) even though the RF is
present and correctly timed. Burst-envelope analysis is immune to that.
Method: extract RF bursts from the IQ envelope, then find the single clock
offset between the firmware's monotonic microsecond timestamps and the IQ
timeline that maximises the number of firmware TX events landing on a
burst. No amplitude threshold is assumed for attribution -- the alignment
itself is the evidence, and a wrong alignment cannot match many events.
Reports:
- how many firmware TX events have corresponding RF in the IQ
- the measured original -> repeat spacing for repeats whose source RX
event is known, compared against the configured delay
- the received level of our transmissions vs meter traffic
Usage:
verify_repeats_iq.py --iq iq.raw --serial serial.log --delay-ms 200
[--rate 1600000] [--tol-ms 15] [--json]
"""
import argparse
import json
import sys
import numpy as np
ap = argparse.ArgumentParser()
ap.add_argument("--iq", required=True)
ap.add_argument("--serial", required=True)
ap.add_argument("--delay-ms", type=float, required=True)
ap.add_argument("--rate", type=int, default=1_600_000)
ap.add_argument("--tol-ms", type=float, default=15.0)
ap.add_argument("--json", action="store_true")
args = ap.parse_args()
WIN = args.rate // 1000 # 1 ms resolution
# ---------------------------------------------------------------- IQ bursts
def load_envelope(path):
env, clipped = [], 0
with open(path, "rb") as f:
while True:
raw = f.read(args.rate * 2)
if len(raw) < WIN * 2:
break
u8 = np.frombuffer(raw, dtype=np.uint8)
clipped += int(np.count_nonzero((u8 == 0) | (u8 == 255)))
b = u8.astype(np.float32) - 127.5
iq = b[0::2] + 1j * b[1::2]
p = np.abs(iq) ** 2
p = p[: (len(p) // WIN) * WIN].reshape(-1, WIN).mean(axis=1)
env.append(p)
if not env:
return None, 0
return np.concatenate(env), clipped
power, clipped_samples = load_envelope(args.iq)
if power is None:
msg = {"error": "iq too short"}
print(json.dumps(msg) if args.json else "ERROR: iq too short")
sys.exit(3)
db = 10 * np.log10(power + 1e-9)
floor = float(np.median(db))
idx = np.where(db > floor + 6.0)[0]
bursts = []
if len(idx):
start = prev = idx[0]
for i in idx[1:]:
if i > prev + 3:
bursts.append((start, prev))
start = i
prev = i
bursts.append((start, prev))
burst_start = np.array([s for s, _ in bursts], dtype=np.int64)
burst_info = [{
"t_ms": int(s),
"duration_ms": int(e - s + 1),
"peak_db": float(db[s:e + 1].max() - floor),
} for s, e in bursts]
# ------------------------------------------------------------ firmware log
tx_events, rx_events = [], []
for line in open(args.serial, errors="replace"):
line = line.strip()
if not line.startswith("{"):
continue
try:
ev = json.loads(line)
except json.JSONDecodeError:
continue
if ev.get("event") == "wmbus_tx" and str(ev.get("result", "")).startswith("ok"):
tx_events.append(ev)
elif ev.get("event") == "wmbus_rx" and ev.get("crc_ok"):
rx_events.append(ev)
if not tx_events or not bursts:
result = {
"iq_seconds": round(len(db) / 1000.0, 1),
"noise_floor_db": round(floor, 1),
"bursts_total": len(burst_info),
"firmware_tx_ok": len(tx_events),
"tx_events_with_rf": 0,
"note": "no TX events or no bursts in this window",
}
print(json.dumps(result, indent=2) if args.json
else f"IQ_REPEAT_PROOF: no TX events or no bursts "
f"(tx={len(tx_events)} bursts={len(burst_info)})")
sys.exit(1)
tx_ms = np.array([ev["timestamp_us"] / 1000.0 for ev in tx_events])
# ------------------------------------------------- align firmware to IQ
# Candidate offsets: every (burst - tx_event) difference. The true offset
# appears once per genuinely observed transmission, so it wins by count.
cands = np.subtract.outer(burst_start.astype(float), tx_ms).ravel()
best_off, best_hits = 0.0, -1
for off in cands:
d = np.abs(burst_start[:, None] - (tx_ms + off)[None, :])
hits = int(np.count_nonzero(d.min(axis=0) <= args.tol_ms))
if hits > best_hits:
best_hits, best_off = hits, float(off)
# Match each TX event to its nearest burst under the winning offset.
# The IQ may cover only part of the serial log (bounded IQ segments, or a
# tuner fault mid-run), so TX events whose aligned time falls outside the
# IQ timeline are "not observable" rather than "not observed" -- counting
# them as failures would understate the result.
iq_span_ms = len(db)
matched, our_peaks, in_window = [], [], 0
for ev, t in zip(tx_events, tx_ms):
target = t + best_off
if target < 0 or target > iq_span_ms:
continue
in_window += 1
if not len(burst_start):
continue
j = int(np.argmin(np.abs(burst_start - target)))
err = float(burst_start[j] - target)
if abs(err) <= args.tol_ms:
matched.append({
"hash": ev.get("hash"),
"frame_len": ev.get("length"),
"iq_t_ms": int(burst_start[j]),
"align_error_ms": round(err, 1),
"burst_duration_ms": burst_info[j]["duration_ms"],
"burst_peak_db": round(burst_info[j]["peak_db"], 1),
"delay_us": ev.get("delay_us"),
})
our_peaks.append(burst_info[j]["peak_db"])
ours_t = {m["iq_t_ms"] for m in matched}
other_peaks = [b["peak_db"] for b in burst_info if b["t_ms"] not in ours_t]
# expected airtime: (frame + 2 mode bytes) * 80 us + preamble/sync ~0.64 ms
def expected_ms(n):
return ((n + 2) * 80 + 640) / 1000.0 if n else None
dur_err = [m["burst_duration_ms"] - expected_ms(m["frame_len"])
for m in matched if m["frame_len"]]
delays = sorted(m["delay_us"] / 1000.0 for m in matched
if m.get("delay_us"))
def stat(v):
if not v:
return {"min": None, "median": None, "max": None}
v = sorted(v)
return {"min": round(v[0], 1),
"median": round(v[len(v) // 2], 1),
"max": round(v[-1], 1)}
result = {
"iq_seconds": round(len(db) / 1000.0, 1),
"noise_floor_db": round(floor, 1),
"clipped_sample_pct": round(100.0 * clipped_samples /
max(1, len(db) * WIN * 2), 4),
"bursts_total": len(burst_info),
"firmware_tx_ok": len(tx_events),
"firmware_rx_crc_ok": len(rx_events),
"tx_events_in_iq_window": in_window,
"tx_events_with_rf": len(matched),
"tx_rf_confirmed_pct": round(100.0 * len(matched) / in_window, 1) if in_window else None,
"clock_offset_ms": round(best_off, 1),
"align_error_ms": stat([abs(m["align_error_ms"]) for m in matched]),
"burst_duration_error_ms": stat(dur_err),
"our_tx_peak_db_over_floor": stat(our_peaks),
"other_burst_peak_db_over_floor": stat(other_peaks),
"firmware_repeat_delay_ms": stat(delays),
"expected_delay_ms": args.delay_ms,
"matched": matched[:50],
}
if args.json:
print(json.dumps(result, indent=2))
else:
print(f"IQ_REPEAT_PROOF iq={result['iq_seconds']}s floor={floor:.1f}dB "
f"clipped={result['clipped_sample_pct']}%")
print(f" bursts in window: {len(burst_info)}")
print(f" firmware TX ok / RX crc-ok: {len(tx_events)} / {len(rx_events)}")
print(f" TX events inside IQ window: {in_window} of {len(tx_events)}")
print(f" TX events with RF in IQ: {len(matched)}/{in_window} "
f"({result['tx_rf_confirmed_pct']} %)")
print(f" clock offset: {best_off:.0f} ms")
print(f" alignment error min/med/max: {result['align_error_ms']}")
print(f" duration error vs airtime: {result['burst_duration_error_ms']} ms")
print(f" our TX level over floor: {result['our_tx_peak_db_over_floor']} dB")
print(f" other bursts over floor: {result['other_burst_peak_db_over_floor']} dB")
print(f" firmware repeat delay: {result['firmware_repeat_delay_ms']} ms "
f"(configured {args.delay_ms:.0f})")
sys.exit(0 if matched else 1)
+33 -20
View File
@@ -8,18 +8,18 @@ SDR observes. This is also the honest test of "power it up and leave it be".
Two independent lines of evidence, both derived from ONE recorded IQ file so
they cannot disagree about gain or time:
1. Decoded repeats - rtl_wmbus run over the recording; a repeat is the
same cooked payload decoded twice within the delay window.
1. Decoded repeats - a supported H=0 original followed by an H=1 frame
with every other cooked byte unchanged.
2. RF-level repeats - burst-envelope pairs separated by the delay window
where the second burst is at least as strong as the first. This catches
repeats that fail CRC because our transmission overloads the dongle
(docs/RF_FRONTEND_TESTS.md), which is the common case on this bench.
Usage:
verify_standalone.py --iq iq.raw --decodes decodes.log
[--min-ms 20] [--max-ms 400] [--rate 1600000] [--json]
verify_standalone.py --iq iq.raw --decodes decodes.log [--delay-ms 500]
[--delay-tol-ms 25] [--rate 1600000] [--json]
Exit 0 if at least one repeat was observed by either method.
Exit 0 only if a decoded hop transition confirms repeater activity.
"""
import argparse
import json
@@ -30,7 +30,7 @@ from datetime import datetime
import numpy as np
sys.path.insert(0, __file__.rsplit("/", 1)[0])
from correlate import SDR_RE # noqa: E402
from correlate import SDR_RE, expected_repeat # noqa: E402
ap = argparse.ArgumentParser()
ap.add_argument("--iq", required=True)
@@ -57,23 +57,35 @@ for line in open(args.decodes, errors="replace"):
"crc_ok": m["crc"] == "1",
"t": datetime.strptime(m["ts"].strip(), "%Y-%m-%d %H:%M:%S.%f").timestamp(),
"serial": m["serial"].lower(),
"hex": m["hex"].lower(),
"cooked": bytes.fromhex(m["hex"]),
})
rows.sort(key=lambda r: r["t"])
c1 = [r for r in rows if r["mode"] == "C1" and r["crc_ok"]]
decoded = [r for r in rows if r["mode"] in ("C1", "T1") and r["crc_ok"]]
# rtl_wmbus runs two demodulators; collapse identical bytes <5 ms apart
dedup = []
for r in c1:
if dedup and dedup[-1]["hex"] == r["hex"] and r["t"] - dedup[-1]["t"] < 0.005:
for r in decoded:
if (dedup and dedup[-1]["mode"] == r["mode"]
and dedup[-1]["cooked"] == r["cooked"]
and r["t"] - dedup[-1]["t"] < 0.005):
continue
dedup.append(r)
# A decoded repeat: identical payload seen twice. rtl_sdr delivers ~82 ms
# buffers so decode timestamps are quantised; count occurrences rather than
# trusting sub-100 ms timing (docs/FINDINGS.md).
occ = Counter(r["hex"] for r in dedup)
decoded_repeats = sum(v - 1 for v in occ.values() if v > 1)
# rtl_sdr delivers ~82 ms buffers, so decode timestamps are quantised.
# Preserve order rather than trusting sub-100 ms timing (docs/FINDINGS.md).
# Each supported H=0 original permits one later H=1 counterpart; never
# normalize H on both sides or count identical originals as repeats.
pending = Counter()
decoded_repeats = 0
for r in dedup:
key = (r["mode"], r["cooked"])
if pending[key]:
pending[key] -= 1
decoded_repeats += 1
else:
marked = expected_repeat(r["cooked"])
if marked is not None:
pending[(r["mode"], marked)] += 1
meters = Counter(r["serial"] for r in dedup)
# ---------------------------------------------------------------- IQ bursts
@@ -123,7 +135,7 @@ binfo = [b for b in binfo_all
# RF-level repeat: a burst followed by a comparable-or-stronger burst of
# similar duration inside the delay window. Duration similarity matters --
# a repeat carries the same frame, so its airtime should match.
# setting H leaves the frame length unchanged, so its airtime should match.
pairs = []
for i, a in enumerate(binfo):
a_end = a["t_ms"] + a["dur_ms"]
@@ -131,7 +143,7 @@ for i, a in enumerate(binfo):
gap = b["t_ms"] - a_end
if gap > args.delay_ms + args.delay_tol_ms:
break
# A repeat re-sends the SAME frame, so it must (a) sit at the
# A hop-marked repeat retains the frame length, so it must (a) sit at the
# configured delay, (b) have an airtime consistent with a C1 frame,
# and (c) match the original's duration closely. Pairs that merely
# fall in a wide window are usually unrelated 868 MHz traffic --
@@ -153,7 +165,8 @@ result = {
"noise_floor_db": round(floor, 1),
"bursts_total": len(binfo_all),
"frame_like_bursts": len(binfo),
"c1_decodes": len(dedup),
"c1_decodes": sum(r["mode"] == "C1" for r in dedup),
"t1_decodes": sum(r["mode"] == "T1" for r in dedup),
"unique_meters": len(meters),
"decoded_repeats": decoded_repeats,
"rf_repeat_pairs": len(pairs),
@@ -176,7 +189,7 @@ if args.json:
print(json.dumps(result, indent=2))
else:
print(f"STANDALONE CHECK iq={result['iq_seconds']}s floor={floor:.1f}dB")
print(f" C1 telegrams decoded: {len(dedup)} from {len(meters)} meters")
print(f" C1/T1 telegrams decoded: {len(dedup)} from {len(meters)} meters")
print(f" RF bursts (all): {len(binfo_all)}")
print(f" frame-like bursts: {len(binfo)} (>= {MIN_FRAME_MS} ms, >= +{MIN_FRAME_DB:.0f} dB)")
print(f" decoded repeats: {decoded_repeats}")
@@ -196,7 +209,7 @@ else:
print(" looks like a singleton. Read the firmware counters over USB")
print(" for an unambiguous answer.)")
elif v == "not observed":
print(" (No frame-like burst pairs and no decoded duplicates. Either")
print(" (No frame-like burst pairs and no decoded hop transitions. Either")
print(" the device is not transmitting, or it transmitted nothing in")
print(" this window. Firmware counters over USB are decisive.)")