hil: segmented capture with tuner-fault recovery, in-window IQ accounting

This commit is contained in:
Lars Nolden
2026-09-01 10:06:32 +02:00
parent 9a65fa2040
commit 3a2e23e204
4 changed files with 85 additions and 32 deletions
+65 -23
View File
@@ -1,40 +1,82 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Capture rtl_wmbus decodes AND raw IQ from a SINGLE rtl_sdr stream. # Capture rtl_wmbus decodes AND raw IQ from a single rtl_sdr stream, in
# bounded segments with recovery between them.
# #
# Why one stream: this dongle's sensitivity drifts by ~17 dB over minutes, # Two hardware facts drive this design (docs/FINDINGS.md):
# so measurements taken in separate captures at "the same gain" are not # 1. This dongle's R820T tuner dies after a few minutes of continuous
# comparable. Decoding meter originals and measuring our own transmissions # streaming ("r82xx_write: i2c wr failed"). rtl_sdr does NOT exit -- it
# in the IQ must therefore share one stream, one gain, one instant. # keeps producing nothing, so a naive long capture silently goes deaf.
# Bounded segments with -n make it exit cleanly, and we usbreset between
# segments instead of relying on failure detection.
# 2. Sensitivity drifts ~17 dB over minutes, so meter decodes and our own
# TX measurements must come from the same stream to be comparable.
# #
# Usage: capture_both.sh DURATION_S DECODE_LOG IQ_FILE [GAIN] # Usage: capture_both.sh DURATION_S DECODE_LOG IQ_FILE [GAIN] [IQ_SEGMENTS]
set -euo pipefail set -euo pipefail
cd "$(dirname "$0")/.." cd "$(dirname "$0")/.."
DUR=$1; OUT=$2; IQ=$3; GAIN=${4:-42.1} DUR=$1; OUT=$2; IQ=$3; GAIN=${4:-42.1}; IQ_SEGMENTS=${5:-1}
SEG=${SEG_SECONDS:-140} # below the observed tuner death point
RATE=1600000
: > "$OUT" : > "$OUT"
: > "$IQ"
ERR="${OUT%.log}.err" ERR="${OUT%.log}.err"
STATS="${OUT%.log}.stats" STATS="${OUT%.log}.stats"
: > "$ERR" : > "$ERR"
health_ok=0 health() {
for i in 1 2 3; do ./result-py/bin/python3 hil/sdr_health.py >> "$ERR" 2>&1
if ./result-py/bin/python3 hil/sdr_health.py >> "$ERR" 2>&1; then }
health_ok=1; break
fi usable=0
echo "--- SDR unhealthy; usbreset (try $i)" >> "$ERR" segments=0
recovered=0
iq_seconds=0
deadline=$(( $(date +%s) + DUR ))
while :; do
remaining=$(( deadline - $(date +%s) ))
[ "$remaining" -le 10 ] && break
this=$(( remaining < SEG ? remaining : SEG ))
segments=$((segments + 1))
if ! health; then
echo "--- segment $segments: SDR unhealthy, usbreset" >> "$ERR"
usbreset 0bda:2838 >> "$ERR" 2>&1 || true usbreset 0bda:2838 >> "$ERR" 2>&1 || true
sleep $((i * 5)) recovered=$((recovered + 1))
done sleep 5
echo "sdr_health_ok=$health_ok" > "${OUT%.log}.health" health || { echo "--- still unhealthy, skipping segment" >> "$ERR"; sleep 2; continue; }
fi
echo "--- segment $segments: ${this}s (gain $GAIN)" >> "$ERR"
t0=$(date +%s) t0=$(date +%s)
# tee duplicates the sample stream: one copy to disk, one to the decoder if [ "$segments" -le "$IQ_SEGMENTS" ]; then
timeout "$DUR" rtl_sdr -f 868950000 -s 1600000 -g "$GAIN" - 2>>"$ERR" \ # keep IQ for the first segments only; bounded so disk use is known
| tee "$IQ" \ timeout $((this + 20)) rtl_sdr -f 868950000 -s "$RATE" -g "$GAIN" \
-n $((this * RATE)) - 2>>"$ERR" \
| tee -a "$IQ" \
| rtl_wmbus -f >> "$OUT" 2>/dev/null || true | rtl_wmbus -f >> "$OUT" 2>/dev/null || true
iq_seconds=$((iq_seconds + this))
else
timeout $((this + 20)) rtl_sdr -f 868950000 -s "$RATE" -g "$GAIN" \
-n $((this * RATE)) - 2>>"$ERR" \
| rtl_wmbus -f >> "$OUT" 2>/dev/null || true
fi
t1=$(date +%s) t1=$(date +%s)
usable=$(( usable + t1 - t0 ))
printf 'streamed_seconds=%s\nrequested_seconds=%s\ngain=%s\niq_bytes=%s\n' \ # tuner commonly dies during a segment; reset before the next one
"$((t1 - t0))" "$DUR" "$GAIN" "$(stat -c%s "$IQ" 2>/dev/null || echo 0)" \ if grep -q "i2c wr failed" "$ERR"; then
> "$STATS" echo "--- tuner fault seen; usbreset" >> "$ERR"
echo "--- streamed $((t1 - t0))s of ${DUR}s" >> "$ERR" usbreset 0bda:2838 >> "$ERR" 2>&1 || true
recovered=$((recovered + 1))
: > "$ERR.tmp"; tail -c 20000 "$ERR" > "$ERR.tmp"; mv "$ERR.tmp" "$ERR"
sleep 5
fi
done
printf 'usable_seconds=%s\nrequested_seconds=%s\nsegments=%s\nrecoveries=%s\ngain=%s\niq_seconds=%s\niq_bytes=%s\n' \
"$usable" "$DUR" "$segments" "$recovered" "$GAIN" "$iq_seconds" \
"$(stat -c%s "$IQ" 2>/dev/null || echo 0)" > "$STATS"
echo "--- usable ${usable}s of ${DUR}s in $segments segment(s), $recovered recovery(ies)" >> "$ERR"
+2 -1
View File
@@ -88,7 +88,7 @@ RF:
TEST: TEST:
duration: {args.duration:.0f} s duration: {args.duration:.0f} s
SDR streaming time: {args.sdr_streamed} s SDR usable time: {args.sdr_streamed} s
SDR sensitivity: {args.sdr_sensitivity}""") SDR sensitivity: {args.sdr_sensitivity}""")
if proof is not None: if proof is not None:
@@ -106,6 +106,7 @@ RF-LEVEL TX PROOF (same stream's raw IQ, decode-independent):
clipped samples: {iqp.get('clipped_sample_pct')} % clipped samples: {iqp.get('clipped_sample_pct')} %
bursts in window: {iqp.get('bursts_total')} bursts in window: {iqp.get('bursts_total')}
firmware TX ok: {iqp.get('firmware_tx_ok')} firmware TX ok: {iqp.get('firmware_tx_ok')}
TX events in IQ window: {iqp.get('tx_events_in_iq_window')}
TX events with RF: {iqp.get('tx_events_with_rf')} ({iqp.get('tx_rf_confirmed_pct')} %) TX events with RF: {iqp.get('tx_events_with_rf')} ({iqp.get('tx_rf_confirmed_pct')} %)
alignment error: {iqp.get('align_error_ms')} ms alignment error: {iqp.get('align_error_ms')} ms
airtime error: {iqp.get('burst_duration_error_ms')} ms airtime error: {iqp.get('burst_duration_error_ms')} ms
+1 -1
View File
@@ -152,7 +152,7 @@ if [ "$HB" -lt "$EXPECTED_HB" ]; then
fi fi
# --- 16-18. report --------------------------------------------------------- # --- 16-18. report ---------------------------------------------------------
STREAMED=$(sed -n 's/^streamed_seconds=//p' "$RUN_DIR/rtlwmbus.stats" 2>/dev/null || echo "?") STREAMED=$(sed -n 's/^usable_seconds=//p' "$RUN_DIR/rtlwmbus.stats" 2>/dev/null || sed -n 's/^streamed_seconds=//p' "$RUN_DIR/rtlwmbus.stats" 2>/dev/null || echo "?")
SENS=$(sed -n 's/^SDR_SENSITIVITY=//p' "$RUN_DIR/sdr_health.txt" 2>/dev/null || echo unknown) SENS=$(sed -n 's/^SDR_SENSITIVITY=//p' "$RUN_DIR/sdr_health.txt" 2>/dev/null || echo unknown)
$PY hil/report.py "$RUN_DIR/report.json" \ $PY hil/report.py "$RUN_DIR/report.json" \
--min-repeats "$MIN_REPEATS" --duration "$DURATION" \ --min-repeats "$MIN_REPEATS" --duration "$DURATION" \
+14 -4
View File
@@ -130,10 +130,18 @@ for off in cands:
if hits > best_hits: if hits > best_hits:
best_hits, best_off = hits, float(off) best_hits, best_off = hits, float(off)
# match each TX event to its nearest burst under the winning offset # Match each TX event to its nearest burst under the winning offset.
matched, our_peaks = [], [] # The IQ may cover only part of the serial log (bounded IQ segments, or a
# tuner fault mid-run), so TX events whose aligned time falls outside the
# IQ timeline are "not observable" rather than "not observed" -- counting
# them as failures would understate the result.
iq_span_ms = len(db)
matched, our_peaks, in_window = [], [], 0
for ev, t in zip(tx_events, tx_ms): for ev, t in zip(tx_events, tx_ms):
target = t + best_off target = t + best_off
if target < 0 or target > iq_span_ms:
continue
in_window += 1
if not len(burst_start): if not len(burst_start):
continue continue
j = int(np.argmin(np.abs(burst_start - target))) j = int(np.argmin(np.abs(burst_start - target)))
@@ -181,8 +189,9 @@ result = {
"bursts_total": len(burst_info), "bursts_total": len(burst_info),
"firmware_tx_ok": len(tx_events), "firmware_tx_ok": len(tx_events),
"firmware_rx_crc_ok": len(rx_events), "firmware_rx_crc_ok": len(rx_events),
"tx_events_in_iq_window": in_window,
"tx_events_with_rf": len(matched), "tx_events_with_rf": len(matched),
"tx_rf_confirmed_pct": round(100.0 * len(matched) / len(tx_events), 1), "tx_rf_confirmed_pct": round(100.0 * len(matched) / in_window, 1) if in_window else None,
"clock_offset_ms": round(best_off, 1), "clock_offset_ms": round(best_off, 1),
"align_error_ms": stat([abs(m["align_error_ms"]) for m in matched]), "align_error_ms": stat([abs(m["align_error_ms"]) for m in matched]),
"burst_duration_error_ms": stat(dur_err), "burst_duration_error_ms": stat(dur_err),
@@ -200,7 +209,8 @@ else:
f"clipped={result['clipped_sample_pct']}%") f"clipped={result['clipped_sample_pct']}%")
print(f" bursts in window: {len(burst_info)}") print(f" bursts in window: {len(burst_info)}")
print(f" firmware TX ok / RX crc-ok: {len(tx_events)} / {len(rx_events)}") print(f" firmware TX ok / RX crc-ok: {len(tx_events)} / {len(rx_events)}")
print(f" TX events with RF in IQ: {len(matched)}/{len(tx_events)} " print(f" TX events inside IQ window: {in_window} of {len(tx_events)}")
print(f" TX events with RF in IQ: {len(matched)}/{in_window} "
f"({result['tx_rf_confirmed_pct']} %)") f"({result['tx_rf_confirmed_pct']} %)")
print(f" clock offset: {best_off:.0f} ms") print(f" clock offset: {best_off:.0f} ms")
print(f" alignment error min/med/max: {result['align_error_ms']}") print(f" alignment error min/med/max: {result['align_error_ms']}")