fix: guard etcd watch teardown against detached-thread use-after-free

.gitignore

@@ -7,6 +7,7 @@ build
 **/fastrg_northbound
 **/.libs
 unit_test/unit-tester
+northbound/controller/test/test_etcd_watch_teardown
 **/*.gcda
 **/*.gcno
 coverage/

e2e_test/phases/local_validation_fixtures.txt

@@ -998,3 +998,73 @@ Error: 401 Unauthorized
 # expect-output: err
 # expect-status: 1
 ---
+==== fixture: node_shutdown_clean
+# expect-output: pass
+# expect-status: 0
+---
+log: ok
+log_last: FastRG[INFO][2026-10-09 03:17:05]: src/fastrg.c:915> bye!
+apport: ok
+kernel: ok
+==== fixture: node_shutdown_abort_without_bye
+# expect-output: no_bye
+# expect-status: 1
+---
+log: ok
+log_last: malloc_consolidate(): unaligned fastbin chunk detected
+apport: ok
+apport_new: INFO: apport (pid 80952) 2026-10-09 00:00:10,762: executable: /root/fastrg/fastrg-node/fastrg (command line "/root/fastrg/fastrg-node/fastrg -l 1-8 -n 4 --socket-mem 17408 -a 0000:07:00.0 -a 0000:08:00.0")
+kernel: ok
+==== fixture: node_shutdown_bye_not_last
+# expect-output: no_bye
+# expect-status: 1
+---
+log: ok
+log_last: free(): invalid pointer
+apport: ok
+kernel: ok
+==== fixture: node_shutdown_apport_record
+# expect-output: apport_record
+# expect-status: 1
+---
+log: ok
+log_last: FastRG[INFO][2026-10-09 03:17:05]: src/fastrg.c:915> bye!
+apport: ok
+apport_new: INFO: apport (pid 80952) 2026-10-09 00:00:10,762: executable: /root/fastrg/fastrg-node/fastrg (command line "/root/fastrg/fastrg-node/fastrg -l 1-8 -n 4 --socket-mem 17408 -a 0000:07:00.0 -a 0000:08:00.0")
+kernel: ok
+==== fixture: node_shutdown_kernel_trap
+# expect-output: kernel_trap
+# expect-status: 1
+---
+log: ok
+log_last: FastRG[INFO][2026-10-09 03:17:05]: src/fastrg.c:915> bye!
+apport: ok
+kernel: ok
+kernel_new: Oct 08 23:33:50 openrg-node kernel: traps: fastrg[51080] general protection fault ip:75335c8a9d3d sp:7ffe0e411340 error:0 in libc.so.6[75335c828000+189000]
+==== fixture: node_shutdown_log_unreadable
+# expect-output: collector_failed
+# expect-status: 1
+---
+log: unreadable
+apport: ok
+kernel: ok
+==== fixture: node_shutdown_apport_unhooked
+# expect-output: collector_failed
+# expect-status: 1
+---
+log: ok
+log_last: FastRG[INFO][2026-10-09 03:17:05]: src/fastrg.c:915> bye!
+apport: unhooked
+kernel: ok
+==== fixture: node_shutdown_kernel_unreadable
+# expect-output: collector_failed
+# expect-status: 1
+---
+log: ok
+log_last: FastRG[INFO][2026-10-09 03:17:05]: src/fastrg.c:915> bye!
+apport: ok
+kernel: unreadable
+==== fixture: node_shutdown_no_evidence
+# expect-output: collector_failed
+# expect-status: 1
+---

e2e_test/phases/node_shutdown_lib.sh

@@ -0,0 +1,163 @@
+#!/usr/bin/env bash
+# shellcheck shell=bash
+# ---------------------------------------------------------------------------
+# Crash check for a SIGTERM stop of the node, shared by every step that stops
+# fastrg. The process being gone does not prove a clean stop: a crash during
+# teardown ends it too. A clean stop ends the node's stdout log with "bye!"
+# and leaves no new apport record and no new kernel trap line.
+#
+# Usage: take a mark before the SIGTERM, check it once the process is gone.
+#   _mark=$(e2e_node_stop_mark)
+#   ssh_node "pkill -x fastrg"; <wait for the process to exit>
+#   _crash=$(e2e_node_shutdown_check "$_mark") || _issue="... ${_crash}"
+# ---------------------------------------------------------------------------
+
+# Run on the node before the SIGTERM. Prints "<stdout file> <apport inode>
+# <apport size> <epoch>"; the stdout file is "-" when fastrg is not running
+# or its stdout is not a regular file.
+_E2E_NODE_STOP_MARK_SCRIPT='
+pid=$(pgrep -xo fastrg)
+out=-
+[ -n "$pid" ] && out=$(readlink -f "/proc/$pid/fd/1" 2>/dev/null)
+[ -f "$out" ] || out=-
+printf "%s %s %s\n" "$out" "$(stat -c "%i %s" /var/log/apport.log 2>/dev/null || echo "- 0")" "$(date +%s)"
+'
+
+# Run on the node after the stop, with LOG/INODE/SIZE/SINCE set from the mark.
+# Each source reports "<name>: ok" only when it could really be read, so a
+# broken collector never reads as "no crash"; apport counts only while it is
+# the core handler, and the kernel source must hold at least one line.
+_E2E_NODE_SHUTDOWN_EVIDENCE_SCRIPT='
+if [ -f "$LOG" ] && [ -r "$LOG" ]; then
+    echo "log: ok"
+    printf "log_last: %s\n" "$(tail -n 50 "$LOG" | grep -av "^[[:space:]]*$" | tail -n 1)"
+else
+    echo "log: unreadable"
+fi
+A=/var/log/apport.log
+case "$(cat /proc/sys/kernel/core_pattern 2>/dev/null)" in
+*apport*)
+    if [ -e "$A" ] && [ ! -r "$A" ]; then
+        echo "apport: unreadable"
+    else
+        echo "apport: ok"
+        if [ "$(stat -c %i "$A" 2>/dev/null || echo -)" = "$INODE" ]; then
+            tail -c +"$((SIZE + 1))" "$A" 2>/dev/null
+        else
+            [ "$INODE" = - ] || tail -c +"$((SIZE + 1))" "$A.1" 2>/dev/null
+            cat "$A" 2>/dev/null
+        fi | grep -aE "executable: [^ ]*/fastrg( |$)" | sed "s/^/apport_new: /"
+    fi
+    ;;
+*)
+    echo "apport: unhooked"
+    ;;
+esac
+if journalctl -k -n 1 -q --no-pager 2>/dev/null | grep -q . &&
+        KLOG=$(journalctl -k -q --no-pager --since "@$SINCE" 2>/dev/null); then
+    echo "kernel: ok"
+    printf "%s\n" "$KLOG" | grep -aE "traps: |segfault at |general protection" | sed "s/^/kernel_new: /"
+else
+    echo "kernel: unreadable"
+fi
+'
+
+# Where the coming stop's crash evidence starts. Empty when the node could not
+# be reached, which the check reports as a collector failure.
+e2e_node_stop_mark() {
+    ssh_node "$_E2E_NODE_STOP_MARK_SCRIPT" 2>/dev/null | tail -n 1 || true
+}
+
+# The evidence a stop left on the node since a mark: the "<source>: ok" lines
+# plus log_last / apport_new / kernel_new lines. Wrapped so a drill can alter it.
+_e2e_node_shutdown_evidence() {
+    local _log="$1" _inode="$2" _size="$3" _since="$4"
+
+    ssh_node "LOG=$(printf '%q' "$_log"); INODE=$(printf '%q' "$_inode");
+        SIZE=$(printf '%q' "$_size"); SINCE=$(printf '%q' "$_since");
+        ${_E2E_NODE_SHUTDOWN_EVIDENCE_SCRIPT}" 2>/dev/null || true
+}
+
+# Verdict on a stop's evidence: pass | collector_failed | no_bye |
+# apport_record | kernel_trap.
+e2e_node_shutdown_verdict() {
+    local _evidence="${1:-}" _source _last
+
+    for _source in log apport kernel; do
+        if ! printf '%s\n' "$_evidence" | grep -qx "${_source}: ok"; then
+            printf 'collector_failed'
+            return 1
+        fi
+    done
+    _last=$(printf '%s\n' "$_evidence" | sed -n 's/^log_last: //p' | tail -n 1)
+    if [[ "$_last" != *'bye!' ]]; then
+        printf 'no_bye'
+        return 1
+    fi
+    if printf '%s\n' "$_evidence" | grep -q '^apport_new: '; then
+        printf 'apport_record'
+        return 1
+    fi
+    if printf '%s\n' "$_evidence" | grep -q '^kernel_new: '; then
+        printf 'kernel_trap'
+        return 1
+    fi
+    printf 'pass'
+    return 0
+}
+
+local_validation_register node_shutdown_verdict e2e_node_shutdown_verdict \
+    node_shutdown_clean \
+    node_shutdown_abort_without_bye \
+    node_shutdown_bye_not_last \
+    node_shutdown_apport_record \
+    node_shutdown_kernel_trap \
+    node_shutdown_log_unreadable \
+    node_shutdown_apport_unhooked \
+    node_shutdown_kernel_unreadable \
+    node_shutdown_no_evidence
+
+# Prints "pass" after a clean stop; otherwise the verdict with the evidence
+# behind it, and returns non-zero. Read-only on the node.
+e2e_node_shutdown_check() {
+    local _mark="${1:-}" _log="" _inode="" _size="" _since="" _evidence="" _verdict=""
+
+    read -r _log _inode _size _since <<< "$_mark" || true
+    if [[ -z "$_log" || ! "$_inode" =~ ^([0-9]+|-)$ || ! "$_size" =~ ^[0-9]+$ || \
+          ! "$_since" =~ ^[0-9]+$ ]]; then
+        printf "collector_failed (no usable stop mark: '%s')" "$_mark"
+        return 1
+    fi
+    _evidence=$(_e2e_node_shutdown_evidence "$_log" "$_inode" "$_size" "$_since")
+    if _verdict=$(e2e_node_shutdown_verdict "$_evidence"); then
+        printf 'pass'
+        return 0
+    fi
+    case "$_verdict" in
+        no_bye)
+            printf "no_bye (%s last line: '%s')" "$_log" \
+                "$(printf '%s\n' "$_evidence" | sed -n 's/^log_last: //p' | tail -n 1 | cut -c 1-200)" ;;
+        apport_record)
+            printf "apport_record (%s)" \
+                "$(printf '%s\n' "$_evidence" | sed -n 's/^apport_new: //p' | head -n 1 | cut -c 1-200)" ;;
+        kernel_trap)
+            printf "kernel_trap (%s)" \
+                "$(printf '%s\n' "$_evidence" | sed -n 's/^kernel_new: //p' | head -n 1 | cut -c 1-200)" ;;
+        *)
+            printf "%s (stdout log %s; evidence: '%s')" "${_verdict:-collector_failed}" "$_log" \
+                "$(printf '%s' "$_evidence" | grep -E '^(log|apport|kernel): ' | tr '\n' ' ')" ;;
+    esac
+    return 1
+}
+
+# Drill: the stop's stdout log loses its "bye!", as a crash in teardown leaves
+# it. The phase's own cleanup restores the real collector via the function below.
+_e2e_inject_node_shutdown_no_bye() {
+    sabotage_copy_function _e2e_node_shutdown_evidence _e2e_node_shutdown_evidence_real
+    sabotage_override_function _e2e_node_shutdown_evidence \
+        '_e2e_node_shutdown_evidence_real "$@" | sed "/^log_last: /s/bye!/(no bye)/"'
+}
+
+_e2e_restore_node_shutdown_functions() {
+    restore_phase_functions node_shutdown_lib.sh
+}

e2e_test/phases/phase17_etcd_offline_queue.sh

@@ -52,6 +52,15 @@ _cleanup_phase17_etcd_offline_queue() {
     fi
 }
 
+# Drill: the Step 70c stop leaves a stdout log without "bye!".
+_p17_cleanup_shutdown_drill() {
+    _e2e_restore_node_shutdown_functions
+    _cleanup_phase17_etcd_offline_queue
+}
+
+case_validation_register shutdown_bye_missing_step70c phase17_etcd_offline_queue \
+    _e2e_inject_node_shutdown_no_bye _p17_cleanup_shutdown_drill 'Step 70c:'
+
 # Raw node gRPC ApplyConfig call (bypasses the controller entirely — same
 # invocation shape as phase9 Step 38, but with full DHCP fields so a
 # non-SDN-guard ApplyConfig can actually succeed).
@@ -449,7 +458,8 @@ phase17_etcd_offline_queue() {
 
     info "Step 70c: gracefully stopping fastrg with the dirty edit persisted..."
     _P17C_RESTART_NEEDED=1
-    local _p17c_stopped=0
+    local _p17c_stopped=0 _p17c_stop_mark="" _p17c_stop_crash=""
+    _p17c_stop_mark=$(e2e_node_stop_mark)
     ssh_node "pkill -x fastrg" >/dev/null 2>&1 || true
     for _i in $(seq 1 30); do
         if ! ssh_node "pgrep -x fastrg >/dev/null 2>&1"; then
@@ -460,6 +470,8 @@ phase17_etcd_offline_queue() {
     done
     if [[ $_p17c_stopped -ne 1 ]]; then
         _p17c_issue="${_p17c_issue:+${_p17c_issue}; }fastrg did not exit within 30s of SIGTERM"
+    elif ! _p17c_stop_crash=$(e2e_node_shutdown_check "$_p17c_stop_mark"); then
+        _p17c_issue="${_p17c_issue:+${_p17c_issue}; }shutdown_crash=${_p17c_stop_crash}"
     fi
 
     # Kafka baseline while the node is down: every message after this offset

e2e_test/phases/phase19_node_restart.sh

@@ -174,6 +174,15 @@ case_validation_register registration_host_os_rest_missing phase19_node_restart
     _p19_inject_registration_host_os_rest_missing _p19_cleanup_registration_drill \
     'Step 77a:'
 
+# Drill: the Step 76 stop leaves a stdout log without "bye!".
+_p19_cleanup_shutdown_drill() {
+    _e2e_restore_node_shutdown_functions
+    _cleanup_phase19_node_restart
+}
+
+case_validation_register shutdown_bye_missing_step76 phase19_node_restart \
+    _e2e_inject_node_shutdown_no_bye _p19_cleanup_shutdown_drill 'Step 76:'
+
 phase19_node_restart() {
     local _hsi1_key="configs/${NODE_UUID}/hsi/1"
     local _hsi2_key="configs/${NODE_UUID}/hsi/2"
@@ -226,6 +235,8 @@ phase19_node_restart() {
     local _step74_issue=""
     local _step76_issue=""
     local _step77a_issue=""
+    local _p19_stop_mark=""
+    local _p19_stop_crash=""
     local _p19_reg_before=""
     local _p19_reg_record=""
     local _p19_rest_record=""
@@ -311,6 +322,7 @@ phase19_node_restart() {
 
     info "  Sending SIGTERM to fastrg and waiting up to 30s for a clean exit..."
     _P19_RESTART_NEEDED=1
+    _p19_stop_mark=$(e2e_node_stop_mark)
     if ! ssh_node "pkill -x fastrg" >/dev/null 2>&1; then
         _step73_issue="${_step73_issue} SIGTERM_delivery=failed"
     fi
@@ -323,11 +335,13 @@ phase19_node_restart() {
     done
     if [[ $_shutdown_done -ne 1 ]]; then
         _step73_issue="${_step73_issue} shutdown_timeout=30s"
+    elif ! _p19_stop_crash=$(e2e_node_shutdown_check "$_p19_stop_mark"); then
+        _step73_issue="${_step73_issue} shutdown_crash=${_p19_stop_crash}"
     fi
 
     if [[ -z "$_step73_issue" ]]; then
         pass "Step 76: Snapshot + graceful shutdown" \
-            "users 1/2 Data phase; desire_status=connect; revisions=${_hsi1_rev_before}/${_hsi2_rev_before}/${_count_rev_before}; clean SIGTERM exit"
+            "users 1/2 Data phase; desire_status=connect; revisions=${_hsi1_rev_before}/${_hsi2_rev_before}/${_count_rev_before}; clean SIGTERM exit (log ends with bye!, no crash record)"
     else
         fail "Step 76: Snapshot + graceful shutdown" "${_step73_issue# }"
     fi
@@ -554,6 +568,7 @@ phase19_node_restart() {
 
     # Stop gracefully, then block etcd BEFORE the cold start.
     _P19_RESTART_NEEDED=1
+    _p19_stop_mark=$(e2e_node_stop_mark)
     ssh_node "pkill -x fastrg" >/dev/null 2>&1 || true
     for _i in $(seq 1 20); do
         [[ "$(_p19_process_state)" == "stopped" ]] && break
@@ -563,6 +578,8 @@ phase19_node_restart() {
         fail "Step 80: snapshot is the boot base while etcd is down" "fastrg did not stop within 20s"
         return
     fi
+    # Reported with the step's verdict; the restart below still has to run.
+    _p19_stop_crash=$(e2e_node_shutdown_check "$_p19_stop_mark") || true
     ssh_node "iptables -I OUTPUT 1 -p tcp -d ${_p19_etcd_host} --dport ${_p19_etcd_port} -j REJECT --reject-with tcp-reset" \
         >/dev/null 2>&1 && _P19_ETCD_BLOCKED=1
     if [[ $_P19_ETCD_BLOCKED -ne 1 ]]; then
@@ -621,6 +638,10 @@ phase19_node_restart() {
     done
     [[ $_p79_sync -eq 1 ]] || _step79_issue="${_step79_issue:+${_step79_issue}; }no etcd watch event observed within 150s of unblocking (watchers did not recover)"
 
+    if [[ "$_p19_stop_crash" != "pass" ]]; then
+        _step79_issue="${_step79_issue:+${_step79_issue}; }shutdown_crash=${_p19_stop_crash}"
+    fi
+
     if [[ -z "$_step79_issue" ]]; then
         _P19_RESTART_NEEDED=0
         pass "Step 80: snapshot is the boot base while etcd is down" \