diff --git a/firmware/main/console.c b/firmware/main/console.c index b4de002..7dca3a1 100644 --- a/firmware/main/console.c +++ b/firmware/main/console.c @@ -105,6 +105,12 @@ static void console_task(void *arg) repeater_request_probe(); ack("probe", true, "queued"); } + else if (strncmp(line, "inject ", 7) == 0) { + const char *w = line + 7; + if (strcmp(w, "standby") == 0) { repeater_inject_fault(1); ack("inject standby", true, "queued"); } + else if (strcmp(w, "sleep") == 0) { repeater_inject_fault(2); ack("inject sleep", true, "queued"); } + else ack("inject", false, "use standby|sleep"); + } else if (strcmp(line, "test_burst") == 0) { repeater_request_test_burst(); ack("test_burst", true, "queued"); diff --git a/firmware/main/repeater.c b/firmware/main/repeater.c index e852548..2fc0969 100644 --- a/firmware/main/repeater.c +++ b/firmware/main/repeater.c @@ -34,6 +34,7 @@ static volatile bool s_want_radio_reset; static volatile bool s_want_test_tx; static volatile bool s_want_probe; static volatile bool s_want_test_burst; +static volatile int s_inject; // 1 = standby, 2 = sleep (config loss) static uint8_t s_test_access_nr; static uint8_t s_test_lfield = 19; static uint32_t s_seq; @@ -95,20 +96,29 @@ static bool recover(const char *reason, int level) bool ok = false; switch (level) { case 1: + // Cheapest step: clear IRQ state and re-arm RX. It must only report + // success if the radio really is configured and receiving, otherwise + // the ladder would stop at a deaf radio. sx126x_clear_irq(SX126X_IRQ_ALL); sx126x_standby(); sx126x_restore_rx_payload_len(); - ok = sx126x_set_rx(); + ok = sx126x_set_rx() && sx126x_config_intact(); + if (ok) { + vTaskDelay(pdMS_TO_TICKS(2)); + ok = (((sx126x_get_status() >> 4) & 0x7) == 0x5); + } break; case 2: g_stats.radio_reinit++; - ok = radio_full_init() && sx126x_set_rx(); + ok = radio_full_init() && sx126x_set_rx() && sx126x_config_intact(); break; case 3: g_stats.fem_reinit++; fem_power(false); vTaskDelay(pdMS_TO_TICKS(20)); - ok = radio_full_init() && sx126x_set_rx(); + fem_power(true); + vTaskDelay(pdMS_TO_TICKS(5)); + ok = radio_full_init() && sx126x_set_rx() && sx126x_config_intact(); break; default: LOGJ("\"event\":\"radio_recovery\",\"reason\":\"%s\",\"level\":%d," @@ -496,6 +506,19 @@ static void repeater_task(void *arg) s_want_probe = false; run_probe(); } + if (s_inject) { + int what = s_inject; + s_inject = 0; + // Deliberately break the radio to exercise detection + the + // recovery ladder. The heartbeat probe is expected to notice. + if (what == 1) { + sx126x_standby(); + } else { + sx126x_inject_sleep(); + } + LOGJ("\"event\":\"fault_injected\",\"fault\":\"%s\"", + what == 1 ? "standby" : "sleep"); + } if (ulTaskNotifyTake(pdTRUE, pdMS_TO_TICKS(1000)) > 0) { uint16_t irq = sx126x_get_irq(); @@ -511,13 +534,21 @@ static void repeater_task(void *arg) int64_t now = esp_timer_get_time(); if (now - last_heartbeat > 10000000LL) { last_heartbeat = now; - // consistency probe: chip should be in RX (0x5) + // Consistency probe: the chip must be in RX (0x5) AND still + // hold its wM-Bus configuration. Checking the mode alone is not + // enough: a radio that lost its configuration (reset, sleep, + // glitch) can sit in RX looking healthy while deaf to C1. uint8_t st = sx126x_get_status(); uint8_t mode = (st >> 4) & 0x7; + bool cfg_ok = sx126x_config_intact(); LOGJ("\"event\":\"heartbeat\",\"uptime_us\":%lld,\"chip_mode\":%u," - "\"rssi_dbm\":%d", (long long)now, mode, sx126x_get_rssi_inst()); + "\"config_intact\":%s,\"rssi_dbm\":%d", + (long long)now, mode, cfg_ok ? "true" : "false", + sx126x_get_rssi_inst()); if (mode != 0x5) { recover_ladder("not_in_rx"); + } else if (!cfg_ok) { + recover_ladder("config_lost"); } } } @@ -533,6 +564,7 @@ void repeater_request_test_tx(void) { s_want_test_tx = true; } void repeater_request_probe(void) { s_want_probe = true; } void repeater_set_test_lfield(uint8_t l) { s_test_lfield = l; } void repeater_request_test_burst(void) { s_want_test_burst = true; } +void repeater_inject_fault(int what) { s_inject = what; } // ------------------------------------------------------------------ status diff --git a/firmware/main/repeater.h b/firmware/main/repeater.h index 7651927..29ee30a 100644 --- a/firmware/main/repeater.h +++ b/firmware/main/repeater.h @@ -39,6 +39,7 @@ void repeater_request_test_tx(void); void repeater_request_probe(void); void repeater_set_test_lfield(uint8_t l); void repeater_request_test_burst(void); +void repeater_inject_fault(int what); // 1 = standby, 2 = sleep void repeater_print_status(void); void repeater_print_stats(void); void repeater_print_radio_status(void); diff --git a/firmware/main/sx126x.c b/firmware/main/sx126x.c index 5a11424..8c5f81b 100644 --- a/firmware/main/sx126x.c +++ b/firmware/main/sx126x.c @@ -385,6 +385,25 @@ bool sx126x_set_tx_power(int8_t dbm) uint8_t tp[2] = {(uint8_t)dbm, 0x02}; // ramp 40 us return cmd(OP_SET_TX_PARAMS, tp, 2); } +// Integrity check: the sync word registers must still hold the wM-Bus +// access code. A radio that was reset, slept, or glitched loses its +// configuration while still reporting a plausible chip mode, so checking +// the mode alone cannot detect that failure. +bool sx126x_config_intact(void) +{ + uint8_t v[2] = {0, 0}; + if (!read_regs(REG_SYNCWORD_0, v, 2)) return false; + return v[0] == 0x54 && v[1] == 0x3D; +} + +// Fault injection for exercising the recovery ladder (see console `inject`). +bool sx126x_inject_sleep(void) +{ + // cold start: configuration is lost on wake + uint8_t cfg = 0x00; + return cmd(OP_SET_SLEEP, &cfg, 1); +} + bool sx126x_standby(void) { diff --git a/firmware/main/sx126x.h b/firmware/main/sx126x.h index d2d3a6d..86e1efb 100644 --- a/firmware/main/sx126x.h +++ b/firmware/main/sx126x.h @@ -34,6 +34,8 @@ bool sx126x_config_wmbus_c1(uint32_t freq_hz, uint8_t rx_payload_len); bool sx126x_set_tx_power(int8_t dbm); // -9 .. +14 (PA config capped) bool sx126x_set_rx(void); // continuous RX, FEM to RX path bool sx126x_standby(void); +bool sx126x_config_intact(void); // sync-word readback still correct +bool sx126x_inject_sleep(void); // fault injection: SLEEP, config lost // Transmit buf as GFSK payload (sync word prepended by radio). // Handles FEM CTX switching. Blocking until commands issued; TX-done is diff --git a/hil/soak.sh b/hil/soak.sh new file mode 100755 index 0000000..c05fb52 --- /dev/null +++ b/hil/soak.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Long unattended reliability soak of the repeater firmware (DUT only). +# +# No SDR: this measures firmware-side reliability (TX success, recoveries, +# MCU resets, timing stability) over hours. The RTL-SDR cannot stream that +# long without faulting (docs/HARDWARE.md), and SDR verification of the RF +# is demonstrated separately by hil/run.sh. +# +# Usage: soak.sh [SECONDS] +set -euo pipefail +cd "$(dirname "$0")/.." +DUR=${1:-9000} +PY=./result-py/bin/python3 + +eval "$(./tools/discover_devices.sh)" +OUT="captures/$(date +%Y-%m-%dT%H-%M-%S)-soak" +mkdir -p "$OUT" +echo "DUT: $DUT_PORT" +echo "duration: ${DUR}s -> $OUT" + +$PY hil/dut_cmd.py "$DUT_PORT" --listen 3 \ + "set repeat_enabled 1" "set repeat_delay_ms 200" "clear_stats" \ + > "$OUT/setup.log" 2>&1 + +$PY hil/dut_cmd.py "$DUT_PORT" --listen "$DUR" status > "$OUT/serial.log" 2>&1 +$PY hil/dut_cmd.py "$DUT_PORT" --listen 5 stats >> "$OUT/serial.log" 2>&1 + +$PY hil/soak_report.py "$OUT/serial.log" "$DUR" | tee "$OUT/report.txt" +echo "artifacts: $OUT" diff --git a/hil/soak_report.py b/hil/soak_report.py new file mode 100755 index 0000000..85b4097 --- /dev/null +++ b/hil/soak_report.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Summarise a long firmware reliability soak (hil/soak.sh).""" +import json +import sys +from collections import Counter + +path = sys.argv[1] +duration = float(sys.argv[2]) if len(sys.argv) > 2 else 0 + +rx, tx, hb, rec, boots = [], [], [], [], 0 +stats = None +for line in open(path, errors="replace"): + line = line.strip() + if not line.startswith("{"): + continue + try: + ev = json.loads(line) + except json.JSONDecodeError: + continue + e = ev.get("event") + if e == "wmbus_rx": + rx.append(ev) + elif e == "wmbus_tx": + tx.append(ev) + elif e == "heartbeat": + hb.append(ev) + elif e == "radio_recovery": + rec.append(ev) + elif e == "boot": + boots += 1 + elif e == "stats": + stats = ev + +tx_ok = [t for t in tx if str(t.get("result", "")).startswith("ok")] +delays = sorted(t["delay_us"] / 1000.0 for t in tx_ok if t.get("delay_us")) +air = sorted(t["airtime_us"] / 1000.0 for t in tx_ok if t.get("airtime_us")) +results = Counter(t.get("result") for t in tx) +uptimes = [h.get("uptime_us", 0) for h in hb] +cfg_bad = sum(1 for h in hb if h.get("config_intact") is False) +mode_bad = sum(1 for h in hb if h.get("chip_mode") not in (5, None)) + + +def s(v, unit=""): + if not v: + return "n/a" + return (f"min={v[0]:.3f}{unit} median={v[len(v)//2]:.3f}{unit} " + f"max={v[-1]:.3f}{unit}") + + +print("=" * 56) +print("REPEATER RELIABILITY SOAK") +print("=" * 56) +print(f""" +requested duration: {duration:.0f} s +heartbeats: {len(hb)} +firmware uptime at end: {max(uptimes)/1e6 if uptimes else 0:.0f} s +MCU boots seen: {boots} (>1 means an unexpected reset) + +RX: + candidate frames: {len(rx)} + CRC-valid frames: {sum(1 for r in rx if r.get('crc_ok'))} + +TX (repeats): + attempts: {len(tx)} + success: {len(tx_ok)} + success rate: {100*len(tx_ok)/len(tx):.1f} %""" if tx else "") + +if not tx: + print("TX: no repeat events recorded") + +print(f" result breakdown: {dict(results)}") +print(f" repeat delay: {s(delays, ' ms')}") +print(f" TX airtime: {s(air, ' ms')}") +print(f""" +HEALTH: + radio recoveries: {len(rec)} + heartbeats not in RX: {mode_bad} + heartbeats config lost:{cfg_bad}""") +if rec: + print(f" recovery reasons: {dict(Counter(r.get('reason') for r in rec))}") + print(f" recovery results: {dict(Counter(r.get('result') for r in rec))}") + +if stats: + print("\nFIRMWARE COUNTERS (final):") + for k in ("rx_candidates", "rx_valid", "rx_crc_fail", "rx_not_c1", + "rx_len_invalid", "rx_bytes", "tx_attempts", "tx_success", + "tx_timeout", "tx_failures", "tx_suppressed_dup", + "radio_reinit", "fem_reinit", "recoveries"): + if k in stats: + print(f" {k:22s} {stats[k]}") + +# milestones from the task description +n = len(tx_ok) +print("\nMILESTONES (firmware-reported repeats, no restart):") +for label, target in (("A", 10), ("B", 100), ("C", 500), ("D", 1000)): + ok = n >= target and boots <= 1 + print(f" {label} (>= {target:4d}): {'REACHED' if ok else 'not reached'} ({n})")