From 3a2e23e2042acbaf66b2856f048b48b4236e3c68 Mon Sep 17 00:00:00 2001 From: Lars Nolden Date: Tue, 1 Sep 2026 10:06:32 +0200 Subject: [PATCH] hil: segmented capture with tuner-fault recovery, in-window IQ accounting --- hil/capture_both.sh | 94 +++++++++++++++++++++++++++++----------- hil/report.py | 3 +- hil/run.sh | 2 +- hil/verify_repeats_iq.py | 18 ++++++-- 4 files changed, 85 insertions(+), 32 deletions(-) diff --git a/hil/capture_both.sh b/hil/capture_both.sh index 20062da..02c956f 100755 --- a/hil/capture_both.sh +++ b/hil/capture_both.sh @@ -1,40 +1,82 @@ #!/usr/bin/env bash -# Capture rtl_wmbus decodes AND raw IQ from a SINGLE rtl_sdr stream. +# Capture rtl_wmbus decodes AND raw IQ from a single rtl_sdr stream, in +# bounded segments with recovery between them. # -# Why one stream: this dongle's sensitivity drifts by ~17 dB over minutes, -# so measurements taken in separate captures at "the same gain" are not -# comparable. Decoding meter originals and measuring our own transmissions -# in the IQ must therefore share one stream, one gain, one instant. +# Two hardware facts drive this design (docs/FINDINGS.md): +# 1. This dongle's R820T tuner dies after a few minutes of continuous +# streaming ("r82xx_write: i2c wr failed"). rtl_sdr does NOT exit -- it +# keeps producing nothing, so a naive long capture silently goes deaf. +# Bounded segments with -n make it exit cleanly, and we usbreset between +# segments instead of relying on failure detection. +# 2. Sensitivity drifts ~17 dB over minutes, so meter decodes and our own +# TX measurements must come from the same stream to be comparable. # -# Usage: capture_both.sh DURATION_S DECODE_LOG IQ_FILE [GAIN] +# Usage: capture_both.sh DURATION_S DECODE_LOG IQ_FILE [GAIN] [IQ_SEGMENTS] set -euo pipefail cd "$(dirname "$0")/.." -DUR=$1; OUT=$2; IQ=$3; GAIN=${4:-42.1} +DUR=$1; OUT=$2; IQ=$3; GAIN=${4:-42.1}; IQ_SEGMENTS=${5:-1} + +SEG=${SEG_SECONDS:-140} # below the observed tuner death point +RATE=1600000 : > "$OUT" +: > "$IQ" ERR="${OUT%.log}.err" STATS="${OUT%.log}.stats" : > "$ERR" -health_ok=0 -for i in 1 2 3; do - if ./result-py/bin/python3 hil/sdr_health.py >> "$ERR" 2>&1; then - health_ok=1; break +health() { + ./result-py/bin/python3 hil/sdr_health.py >> "$ERR" 2>&1 +} + +usable=0 +segments=0 +recovered=0 +iq_seconds=0 +deadline=$(( $(date +%s) + DUR )) + +while :; do + remaining=$(( deadline - $(date +%s) )) + [ "$remaining" -le 10 ] && break + this=$(( remaining < SEG ? remaining : SEG )) + segments=$((segments + 1)) + + if ! health; then + echo "--- segment $segments: SDR unhealthy, usbreset" >> "$ERR" + usbreset 0bda:2838 >> "$ERR" 2>&1 || true + recovered=$((recovered + 1)) + sleep 5 + health || { echo "--- still unhealthy, skipping segment" >> "$ERR"; sleep 2; continue; } + fi + + echo "--- segment $segments: ${this}s (gain $GAIN)" >> "$ERR" + t0=$(date +%s) + if [ "$segments" -le "$IQ_SEGMENTS" ]; then + # keep IQ for the first segments only; bounded so disk use is known + timeout $((this + 20)) rtl_sdr -f 868950000 -s "$RATE" -g "$GAIN" \ + -n $((this * RATE)) - 2>>"$ERR" \ + | tee -a "$IQ" \ + | rtl_wmbus -f >> "$OUT" 2>/dev/null || true + iq_seconds=$((iq_seconds + this)) + else + timeout $((this + 20)) rtl_sdr -f 868950000 -s "$RATE" -g "$GAIN" \ + -n $((this * RATE)) - 2>>"$ERR" \ + | rtl_wmbus -f >> "$OUT" 2>/dev/null || true + fi + t1=$(date +%s) + usable=$(( usable + t1 - t0 )) + + # tuner commonly dies during a segment; reset before the next one + if grep -q "i2c wr failed" "$ERR"; then + echo "--- tuner fault seen; usbreset" >> "$ERR" + usbreset 0bda:2838 >> "$ERR" 2>&1 || true + recovered=$((recovered + 1)) + : > "$ERR.tmp"; tail -c 20000 "$ERR" > "$ERR.tmp"; mv "$ERR.tmp" "$ERR" + sleep 5 fi - echo "--- SDR unhealthy; usbreset (try $i)" >> "$ERR" - usbreset 0bda:2838 >> "$ERR" 2>&1 || true - sleep $((i * 5)) done -echo "sdr_health_ok=$health_ok" > "${OUT%.log}.health" -t0=$(date +%s) -# tee duplicates the sample stream: one copy to disk, one to the decoder -timeout "$DUR" rtl_sdr -f 868950000 -s 1600000 -g "$GAIN" - 2>>"$ERR" \ - | tee "$IQ" \ - | rtl_wmbus -f >> "$OUT" 2>/dev/null || true -t1=$(date +%s) - -printf 'streamed_seconds=%s\nrequested_seconds=%s\ngain=%s\niq_bytes=%s\n' \ - "$((t1 - t0))" "$DUR" "$GAIN" "$(stat -c%s "$IQ" 2>/dev/null || echo 0)" \ - > "$STATS" -echo "--- streamed $((t1 - t0))s of ${DUR}s" >> "$ERR" +printf 'usable_seconds=%s\nrequested_seconds=%s\nsegments=%s\nrecoveries=%s\ngain=%s\niq_seconds=%s\niq_bytes=%s\n' \ + "$usable" "$DUR" "$segments" "$recovered" "$GAIN" "$iq_seconds" \ + "$(stat -c%s "$IQ" 2>/dev/null || echo 0)" > "$STATS" +echo "--- usable ${usable}s of ${DUR}s in $segments segment(s), $recovered recovery(ies)" >> "$ERR" diff --git a/hil/report.py b/hil/report.py index 3bcf4bb..d9c546b 100755 --- a/hil/report.py +++ b/hil/report.py @@ -88,7 +88,7 @@ RF: TEST: duration: {args.duration:.0f} s - SDR streaming time: {args.sdr_streamed} s + SDR usable time: {args.sdr_streamed} s SDR sensitivity: {args.sdr_sensitivity}""") if proof is not None: @@ -106,6 +106,7 @@ RF-LEVEL TX PROOF (same stream's raw IQ, decode-independent): clipped samples: {iqp.get('clipped_sample_pct')} % bursts in window: {iqp.get('bursts_total')} firmware TX ok: {iqp.get('firmware_tx_ok')} + TX events in IQ window: {iqp.get('tx_events_in_iq_window')} TX events with RF: {iqp.get('tx_events_with_rf')} ({iqp.get('tx_rf_confirmed_pct')} %) alignment error: {iqp.get('align_error_ms')} ms airtime error: {iqp.get('burst_duration_error_ms')} ms diff --git a/hil/run.sh b/hil/run.sh index 50dce3d..c954299 100755 --- a/hil/run.sh +++ b/hil/run.sh @@ -152,7 +152,7 @@ if [ "$HB" -lt "$EXPECTED_HB" ]; then fi # --- 16-18. report --------------------------------------------------------- -STREAMED=$(sed -n 's/^streamed_seconds=//p' "$RUN_DIR/rtlwmbus.stats" 2>/dev/null || echo "?") +STREAMED=$(sed -n 's/^usable_seconds=//p' "$RUN_DIR/rtlwmbus.stats" 2>/dev/null || sed -n 's/^streamed_seconds=//p' "$RUN_DIR/rtlwmbus.stats" 2>/dev/null || echo "?") SENS=$(sed -n 's/^SDR_SENSITIVITY=//p' "$RUN_DIR/sdr_health.txt" 2>/dev/null || echo unknown) $PY hil/report.py "$RUN_DIR/report.json" \ --min-repeats "$MIN_REPEATS" --duration "$DURATION" \ diff --git a/hil/verify_repeats_iq.py b/hil/verify_repeats_iq.py index 2ca87a5..23fa5c5 100755 --- a/hil/verify_repeats_iq.py +++ b/hil/verify_repeats_iq.py @@ -130,10 +130,18 @@ for off in cands: if hits > best_hits: best_hits, best_off = hits, float(off) -# match each TX event to its nearest burst under the winning offset -matched, our_peaks = [], [] +# Match each TX event to its nearest burst under the winning offset. +# The IQ may cover only part of the serial log (bounded IQ segments, or a +# tuner fault mid-run), so TX events whose aligned time falls outside the +# IQ timeline are "not observable" rather than "not observed" -- counting +# them as failures would understate the result. +iq_span_ms = len(db) +matched, our_peaks, in_window = [], [], 0 for ev, t in zip(tx_events, tx_ms): target = t + best_off + if target < 0 or target > iq_span_ms: + continue + in_window += 1 if not len(burst_start): continue j = int(np.argmin(np.abs(burst_start - target))) @@ -181,8 +189,9 @@ result = { "bursts_total": len(burst_info), "firmware_tx_ok": len(tx_events), "firmware_rx_crc_ok": len(rx_events), + "tx_events_in_iq_window": in_window, "tx_events_with_rf": len(matched), - "tx_rf_confirmed_pct": round(100.0 * len(matched) / len(tx_events), 1), + "tx_rf_confirmed_pct": round(100.0 * len(matched) / in_window, 1) if in_window else None, "clock_offset_ms": round(best_off, 1), "align_error_ms": stat([abs(m["align_error_ms"]) for m in matched]), "burst_duration_error_ms": stat(dur_err), @@ -200,7 +209,8 @@ else: f"clipped={result['clipped_sample_pct']}%") print(f" bursts in window: {len(burst_info)}") print(f" firmware TX ok / RX crc-ok: {len(tx_events)} / {len(rx_events)}") - print(f" TX events with RF in IQ: {len(matched)}/{len(tx_events)} " + print(f" TX events inside IQ window: {in_window} of {len(tx_events)}") + print(f" TX events with RF in IQ: {len(matched)}/{in_window} " f"({result['tx_rf_confirmed_pct']} %)") print(f" clock offset: {best_off:.0f} ms") print(f" alignment error min/med/max: {result['align_error_ms']}")