firmware: config-integrity check, fault injection, verified recovery ladder; soak harness
This commit is contained in:
@@ -105,6 +105,12 @@ static void console_task(void *arg)
|
|||||||
repeater_request_probe();
|
repeater_request_probe();
|
||||||
ack("probe", true, "queued");
|
ack("probe", true, "queued");
|
||||||
}
|
}
|
||||||
|
else if (strncmp(line, "inject ", 7) == 0) {
|
||||||
|
const char *w = line + 7;
|
||||||
|
if (strcmp(w, "standby") == 0) { repeater_inject_fault(1); ack("inject standby", true, "queued"); }
|
||||||
|
else if (strcmp(w, "sleep") == 0) { repeater_inject_fault(2); ack("inject sleep", true, "queued"); }
|
||||||
|
else ack("inject", false, "use standby|sleep");
|
||||||
|
}
|
||||||
else if (strcmp(line, "test_burst") == 0) {
|
else if (strcmp(line, "test_burst") == 0) {
|
||||||
repeater_request_test_burst();
|
repeater_request_test_burst();
|
||||||
ack("test_burst", true, "queued");
|
ack("test_burst", true, "queued");
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ static volatile bool s_want_radio_reset;
|
|||||||
static volatile bool s_want_test_tx;
|
static volatile bool s_want_test_tx;
|
||||||
static volatile bool s_want_probe;
|
static volatile bool s_want_probe;
|
||||||
static volatile bool s_want_test_burst;
|
static volatile bool s_want_test_burst;
|
||||||
|
static volatile int s_inject; // 1 = standby, 2 = sleep (config loss)
|
||||||
static uint8_t s_test_access_nr;
|
static uint8_t s_test_access_nr;
|
||||||
static uint8_t s_test_lfield = 19;
|
static uint8_t s_test_lfield = 19;
|
||||||
static uint32_t s_seq;
|
static uint32_t s_seq;
|
||||||
@@ -95,20 +96,29 @@ static bool recover(const char *reason, int level)
|
|||||||
bool ok = false;
|
bool ok = false;
|
||||||
switch (level) {
|
switch (level) {
|
||||||
case 1:
|
case 1:
|
||||||
|
// Cheapest step: clear IRQ state and re-arm RX. It must only report
|
||||||
|
// success if the radio really is configured and receiving, otherwise
|
||||||
|
// the ladder would stop at a deaf radio.
|
||||||
sx126x_clear_irq(SX126X_IRQ_ALL);
|
sx126x_clear_irq(SX126X_IRQ_ALL);
|
||||||
sx126x_standby();
|
sx126x_standby();
|
||||||
sx126x_restore_rx_payload_len();
|
sx126x_restore_rx_payload_len();
|
||||||
ok = sx126x_set_rx();
|
ok = sx126x_set_rx() && sx126x_config_intact();
|
||||||
|
if (ok) {
|
||||||
|
vTaskDelay(pdMS_TO_TICKS(2));
|
||||||
|
ok = (((sx126x_get_status() >> 4) & 0x7) == 0x5);
|
||||||
|
}
|
||||||
break;
|
break;
|
||||||
case 2:
|
case 2:
|
||||||
g_stats.radio_reinit++;
|
g_stats.radio_reinit++;
|
||||||
ok = radio_full_init() && sx126x_set_rx();
|
ok = radio_full_init() && sx126x_set_rx() && sx126x_config_intact();
|
||||||
break;
|
break;
|
||||||
case 3:
|
case 3:
|
||||||
g_stats.fem_reinit++;
|
g_stats.fem_reinit++;
|
||||||
fem_power(false);
|
fem_power(false);
|
||||||
vTaskDelay(pdMS_TO_TICKS(20));
|
vTaskDelay(pdMS_TO_TICKS(20));
|
||||||
ok = radio_full_init() && sx126x_set_rx();
|
fem_power(true);
|
||||||
|
vTaskDelay(pdMS_TO_TICKS(5));
|
||||||
|
ok = radio_full_init() && sx126x_set_rx() && sx126x_config_intact();
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
LOGJ("\"event\":\"radio_recovery\",\"reason\":\"%s\",\"level\":%d,"
|
LOGJ("\"event\":\"radio_recovery\",\"reason\":\"%s\",\"level\":%d,"
|
||||||
@@ -496,6 +506,19 @@ static void repeater_task(void *arg)
|
|||||||
s_want_probe = false;
|
s_want_probe = false;
|
||||||
run_probe();
|
run_probe();
|
||||||
}
|
}
|
||||||
|
if (s_inject) {
|
||||||
|
int what = s_inject;
|
||||||
|
s_inject = 0;
|
||||||
|
// Deliberately break the radio to exercise detection + the
|
||||||
|
// recovery ladder. The heartbeat probe is expected to notice.
|
||||||
|
if (what == 1) {
|
||||||
|
sx126x_standby();
|
||||||
|
} else {
|
||||||
|
sx126x_inject_sleep();
|
||||||
|
}
|
||||||
|
LOGJ("\"event\":\"fault_injected\",\"fault\":\"%s\"",
|
||||||
|
what == 1 ? "standby" : "sleep");
|
||||||
|
}
|
||||||
|
|
||||||
if (ulTaskNotifyTake(pdTRUE, pdMS_TO_TICKS(1000)) > 0) {
|
if (ulTaskNotifyTake(pdTRUE, pdMS_TO_TICKS(1000)) > 0) {
|
||||||
uint16_t irq = sx126x_get_irq();
|
uint16_t irq = sx126x_get_irq();
|
||||||
@@ -511,13 +534,21 @@ static void repeater_task(void *arg)
|
|||||||
int64_t now = esp_timer_get_time();
|
int64_t now = esp_timer_get_time();
|
||||||
if (now - last_heartbeat > 10000000LL) {
|
if (now - last_heartbeat > 10000000LL) {
|
||||||
last_heartbeat = now;
|
last_heartbeat = now;
|
||||||
// consistency probe: chip should be in RX (0x5)
|
// Consistency probe: the chip must be in RX (0x5) AND still
|
||||||
|
// hold its wM-Bus configuration. Checking the mode alone is not
|
||||||
|
// enough: a radio that lost its configuration (reset, sleep,
|
||||||
|
// glitch) can sit in RX looking healthy while deaf to C1.
|
||||||
uint8_t st = sx126x_get_status();
|
uint8_t st = sx126x_get_status();
|
||||||
uint8_t mode = (st >> 4) & 0x7;
|
uint8_t mode = (st >> 4) & 0x7;
|
||||||
|
bool cfg_ok = sx126x_config_intact();
|
||||||
LOGJ("\"event\":\"heartbeat\",\"uptime_us\":%lld,\"chip_mode\":%u,"
|
LOGJ("\"event\":\"heartbeat\",\"uptime_us\":%lld,\"chip_mode\":%u,"
|
||||||
"\"rssi_dbm\":%d", (long long)now, mode, sx126x_get_rssi_inst());
|
"\"config_intact\":%s,\"rssi_dbm\":%d",
|
||||||
|
(long long)now, mode, cfg_ok ? "true" : "false",
|
||||||
|
sx126x_get_rssi_inst());
|
||||||
if (mode != 0x5) {
|
if (mode != 0x5) {
|
||||||
recover_ladder("not_in_rx");
|
recover_ladder("not_in_rx");
|
||||||
|
} else if (!cfg_ok) {
|
||||||
|
recover_ladder("config_lost");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -533,6 +564,7 @@ void repeater_request_test_tx(void) { s_want_test_tx = true; }
|
|||||||
void repeater_request_probe(void) { s_want_probe = true; }
|
void repeater_request_probe(void) { s_want_probe = true; }
|
||||||
void repeater_set_test_lfield(uint8_t l) { s_test_lfield = l; }
|
void repeater_set_test_lfield(uint8_t l) { s_test_lfield = l; }
|
||||||
void repeater_request_test_burst(void) { s_want_test_burst = true; }
|
void repeater_request_test_burst(void) { s_want_test_burst = true; }
|
||||||
|
void repeater_inject_fault(int what) { s_inject = what; }
|
||||||
|
|
||||||
// ------------------------------------------------------------------ status
|
// ------------------------------------------------------------------ status
|
||||||
|
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ void repeater_request_test_tx(void);
|
|||||||
void repeater_request_probe(void);
|
void repeater_request_probe(void);
|
||||||
void repeater_set_test_lfield(uint8_t l);
|
void repeater_set_test_lfield(uint8_t l);
|
||||||
void repeater_request_test_burst(void);
|
void repeater_request_test_burst(void);
|
||||||
|
void repeater_inject_fault(int what); // 1 = standby, 2 = sleep
|
||||||
void repeater_print_status(void);
|
void repeater_print_status(void);
|
||||||
void repeater_print_stats(void);
|
void repeater_print_stats(void);
|
||||||
void repeater_print_radio_status(void);
|
void repeater_print_radio_status(void);
|
||||||
|
|||||||
@@ -385,6 +385,25 @@ bool sx126x_set_tx_power(int8_t dbm)
|
|||||||
uint8_t tp[2] = {(uint8_t)dbm, 0x02}; // ramp 40 us
|
uint8_t tp[2] = {(uint8_t)dbm, 0x02}; // ramp 40 us
|
||||||
return cmd(OP_SET_TX_PARAMS, tp, 2);
|
return cmd(OP_SET_TX_PARAMS, tp, 2);
|
||||||
}
|
}
|
||||||
|
// Integrity check: the sync word registers must still hold the wM-Bus
|
||||||
|
// access code. A radio that was reset, slept, or glitched loses its
|
||||||
|
// configuration while still reporting a plausible chip mode, so checking
|
||||||
|
// the mode alone cannot detect that failure.
|
||||||
|
bool sx126x_config_intact(void)
|
||||||
|
{
|
||||||
|
uint8_t v[2] = {0, 0};
|
||||||
|
if (!read_regs(REG_SYNCWORD_0, v, 2)) return false;
|
||||||
|
return v[0] == 0x54 && v[1] == 0x3D;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fault injection for exercising the recovery ladder (see console `inject`).
|
||||||
|
bool sx126x_inject_sleep(void)
|
||||||
|
{
|
||||||
|
// cold start: configuration is lost on wake
|
||||||
|
uint8_t cfg = 0x00;
|
||||||
|
return cmd(OP_SET_SLEEP, &cfg, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
bool sx126x_standby(void)
|
bool sx126x_standby(void)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -34,6 +34,8 @@ bool sx126x_config_wmbus_c1(uint32_t freq_hz, uint8_t rx_payload_len);
|
|||||||
bool sx126x_set_tx_power(int8_t dbm); // -9 .. +14 (PA config capped)
|
bool sx126x_set_tx_power(int8_t dbm); // -9 .. +14 (PA config capped)
|
||||||
bool sx126x_set_rx(void); // continuous RX, FEM to RX path
|
bool sx126x_set_rx(void); // continuous RX, FEM to RX path
|
||||||
bool sx126x_standby(void);
|
bool sx126x_standby(void);
|
||||||
|
bool sx126x_config_intact(void); // sync-word readback still correct
|
||||||
|
bool sx126x_inject_sleep(void); // fault injection: SLEEP, config lost
|
||||||
|
|
||||||
// Transmit buf as GFSK payload (sync word prepended by radio).
|
// Transmit buf as GFSK payload (sync word prepended by radio).
|
||||||
// Handles FEM CTX switching. Blocking until commands issued; TX-done is
|
// Handles FEM CTX switching. Blocking until commands issued; TX-done is
|
||||||
|
|||||||
Executable
+29
@@ -0,0 +1,29 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Long unattended reliability soak of the repeater firmware (DUT only).
|
||||||
|
#
|
||||||
|
# No SDR: this measures firmware-side reliability (TX success, recoveries,
|
||||||
|
# MCU resets, timing stability) over hours. The RTL-SDR cannot stream that
|
||||||
|
# long without faulting (docs/HARDWARE.md), and SDR verification of the RF
|
||||||
|
# is demonstrated separately by hil/run.sh.
|
||||||
|
#
|
||||||
|
# Usage: soak.sh [SECONDS]
|
||||||
|
set -euo pipefail
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
DUR=${1:-9000}
|
||||||
|
PY=./result-py/bin/python3
|
||||||
|
|
||||||
|
eval "$(./tools/discover_devices.sh)"
|
||||||
|
OUT="captures/$(date +%Y-%m-%dT%H-%M-%S)-soak"
|
||||||
|
mkdir -p "$OUT"
|
||||||
|
echo "DUT: $DUT_PORT"
|
||||||
|
echo "duration: ${DUR}s -> $OUT"
|
||||||
|
|
||||||
|
$PY hil/dut_cmd.py "$DUT_PORT" --listen 3 \
|
||||||
|
"set repeat_enabled 1" "set repeat_delay_ms 200" "clear_stats" \
|
||||||
|
> "$OUT/setup.log" 2>&1
|
||||||
|
|
||||||
|
$PY hil/dut_cmd.py "$DUT_PORT" --listen "$DUR" status > "$OUT/serial.log" 2>&1
|
||||||
|
$PY hil/dut_cmd.py "$DUT_PORT" --listen 5 stats >> "$OUT/serial.log" 2>&1
|
||||||
|
|
||||||
|
$PY hil/soak_report.py "$OUT/serial.log" "$DUR" | tee "$OUT/report.txt"
|
||||||
|
echo "artifacts: $OUT"
|
||||||
Executable
+97
@@ -0,0 +1,97 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Summarise a long firmware reliability soak (hil/soak.sh)."""
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from collections import Counter
|
||||||
|
|
||||||
|
path = sys.argv[1]
|
||||||
|
duration = float(sys.argv[2]) if len(sys.argv) > 2 else 0
|
||||||
|
|
||||||
|
rx, tx, hb, rec, boots = [], [], [], [], 0
|
||||||
|
stats = None
|
||||||
|
for line in open(path, errors="replace"):
|
||||||
|
line = line.strip()
|
||||||
|
if not line.startswith("{"):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
ev = json.loads(line)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
e = ev.get("event")
|
||||||
|
if e == "wmbus_rx":
|
||||||
|
rx.append(ev)
|
||||||
|
elif e == "wmbus_tx":
|
||||||
|
tx.append(ev)
|
||||||
|
elif e == "heartbeat":
|
||||||
|
hb.append(ev)
|
||||||
|
elif e == "radio_recovery":
|
||||||
|
rec.append(ev)
|
||||||
|
elif e == "boot":
|
||||||
|
boots += 1
|
||||||
|
elif e == "stats":
|
||||||
|
stats = ev
|
||||||
|
|
||||||
|
tx_ok = [t for t in tx if str(t.get("result", "")).startswith("ok")]
|
||||||
|
delays = sorted(t["delay_us"] / 1000.0 for t in tx_ok if t.get("delay_us"))
|
||||||
|
air = sorted(t["airtime_us"] / 1000.0 for t in tx_ok if t.get("airtime_us"))
|
||||||
|
results = Counter(t.get("result") for t in tx)
|
||||||
|
uptimes = [h.get("uptime_us", 0) for h in hb]
|
||||||
|
cfg_bad = sum(1 for h in hb if h.get("config_intact") is False)
|
||||||
|
mode_bad = sum(1 for h in hb if h.get("chip_mode") not in (5, None))
|
||||||
|
|
||||||
|
|
||||||
|
def s(v, unit=""):
|
||||||
|
if not v:
|
||||||
|
return "n/a"
|
||||||
|
return (f"min={v[0]:.3f}{unit} median={v[len(v)//2]:.3f}{unit} "
|
||||||
|
f"max={v[-1]:.3f}{unit}")
|
||||||
|
|
||||||
|
|
||||||
|
print("=" * 56)
|
||||||
|
print("REPEATER RELIABILITY SOAK")
|
||||||
|
print("=" * 56)
|
||||||
|
print(f"""
|
||||||
|
requested duration: {duration:.0f} s
|
||||||
|
heartbeats: {len(hb)}
|
||||||
|
firmware uptime at end: {max(uptimes)/1e6 if uptimes else 0:.0f} s
|
||||||
|
MCU boots seen: {boots} (>1 means an unexpected reset)
|
||||||
|
|
||||||
|
RX:
|
||||||
|
candidate frames: {len(rx)}
|
||||||
|
CRC-valid frames: {sum(1 for r in rx if r.get('crc_ok'))}
|
||||||
|
|
||||||
|
TX (repeats):
|
||||||
|
attempts: {len(tx)}
|
||||||
|
success: {len(tx_ok)}
|
||||||
|
success rate: {100*len(tx_ok)/len(tx):.1f} %""" if tx else "")
|
||||||
|
|
||||||
|
if not tx:
|
||||||
|
print("TX: no repeat events recorded")
|
||||||
|
|
||||||
|
print(f" result breakdown: {dict(results)}")
|
||||||
|
print(f" repeat delay: {s(delays, ' ms')}")
|
||||||
|
print(f" TX airtime: {s(air, ' ms')}")
|
||||||
|
print(f"""
|
||||||
|
HEALTH:
|
||||||
|
radio recoveries: {len(rec)}
|
||||||
|
heartbeats not in RX: {mode_bad}
|
||||||
|
heartbeats config lost:{cfg_bad}""")
|
||||||
|
if rec:
|
||||||
|
print(f" recovery reasons: {dict(Counter(r.get('reason') for r in rec))}")
|
||||||
|
print(f" recovery results: {dict(Counter(r.get('result') for r in rec))}")
|
||||||
|
|
||||||
|
if stats:
|
||||||
|
print("\nFIRMWARE COUNTERS (final):")
|
||||||
|
for k in ("rx_candidates", "rx_valid", "rx_crc_fail", "rx_not_c1",
|
||||||
|
"rx_len_invalid", "rx_bytes", "tx_attempts", "tx_success",
|
||||||
|
"tx_timeout", "tx_failures", "tx_suppressed_dup",
|
||||||
|
"radio_reinit", "fem_reinit", "recoveries"):
|
||||||
|
if k in stats:
|
||||||
|
print(f" {k:22s} {stats[k]}")
|
||||||
|
|
||||||
|
# milestones from the task description
|
||||||
|
n = len(tx_ok)
|
||||||
|
print("\nMILESTONES (firmware-reported repeats, no restart):")
|
||||||
|
for label, target in (("A", 10), ("B", 100), ("C", 500), ("D", 1000)):
|
||||||
|
ok = n >= target and boots <= 1
|
||||||
|
print(f" {label} (>= {target:4d}): {'REACHED' if ok else 'not reached'} ({n})")
|
||||||
Reference in New Issue
Block a user