test(ha-heartbeat-liveness): SIGSTOP/SIGCONT tolerance integration test (task 4.5)
Add src/tools/test-heartbeat-tolerance.sh: automated bash script that deploys a 3-node cluster and runs two SIGSTOP/SIGCONT scenarios: Scenario A (1 × heartbeat_interval_ms pause): verifies no RDHUP fires on standbys — liveness_poll_fn must not trigger failover for a short silence within the tolerance window (Req 2.2). Scenario B (heartbeat_timeout_ms + 500ms pause): verifies exactly one RDHUP fires on each standby — the activity-based poll fires after sustained silence and does not loop (Req 2.3, Req 5.4). Add .kiro/specs/ha-heartbeat-liveness/manual-tests.md: step-by-step manual procedure, expected log patterns, and results template for both scenarios. Mark task 4.5 complete in tasks.md.
This commit is contained in:
parent
529552c127
commit
e56b2064de
3 changed files with 512 additions and 1 deletions
289
src/tools/test-heartbeat-tolerance.sh
Executable file
289
src/tools/test-heartbeat-tolerance.sh
Executable file
|
|
@ -0,0 +1,289 @@
|
|||
#!/bin/bash
|
||||
# test-heartbeat-tolerance.sh — Task 4.5: missed-heartbeat tolerance via SIGSTOP/SIGCONT
|
||||
#
|
||||
# Requirements: 2.2, 2.3
|
||||
#
|
||||
# Two scenarios:
|
||||
# Scenario A — short pause (1 × heartbeat_interval_ms): no failover fires
|
||||
# Scenario B — long pause (heartbeat_timeout_ms + 500ms): failover fires exactly once
|
||||
#
|
||||
# Usage:
|
||||
# bash src/tools/test-heartbeat-tolerance.sh [deploy-dir]
|
||||
#
|
||||
# deploy-dir defaults to /tmp/xce-deploy. The script expects a deployed
|
||||
# 3-node cluster (via deploy-ha.sh + start.sh). It deploys one if the
|
||||
# binary exists at the default location.
|
||||
set -e
|
||||
|
||||
WORKTREE=$(cd "$(dirname "$0")/../.." && pwd)
|
||||
BINARY="$WORKTREE/src/xce.build/controller/xce_controller"
|
||||
DATADIR="$WORKTREE/ha/data"
|
||||
DEPLOY_DIR="${1:-/tmp/xce-deploy}"
|
||||
PEERS="localhost:9991,localhost:9992,localhost:9993"
|
||||
LOGDIR="$DEPLOY_DIR/logs"
|
||||
|
||||
# Defaults from ha.h
|
||||
HB_INTERVAL_MS=3000
|
||||
HB_TIMEOUT_MS=10000
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
# ── colour helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
green() { printf '\033[32m%s\033[0m\n' "$*"; }
|
||||
red() { printf '\033[31m%s\033[0m\n' "$*"; }
|
||||
yellow() { printf '\033[33m%s\033[0m\n' "$*"; }
|
||||
|
||||
pass() { green " PASS: $*"; PASS=$((PASS+1)); }
|
||||
fail() { red " FAIL: $*"; FAIL=$((FAIL+1)); }
|
||||
|
||||
# ── cluster lifecycle ─────────────────────────────────────────────────────────
|
||||
|
||||
deploy_cluster() {
|
||||
if [ ! -f "$BINARY" ]; then
|
||||
red "Binary not found: $BINARY"
|
||||
red "Run /xce-build controller first."
|
||||
exit 1
|
||||
fi
|
||||
yellow "Deploying 3-node cluster to $DEPLOY_DIR ..."
|
||||
bash "$WORKTREE/src/tools/deploy-ha.sh" \
|
||||
"$BINARY" "$DATADIR" "$DEPLOY_DIR" "$PEERS"
|
||||
}
|
||||
|
||||
start_cluster() {
|
||||
kill_cluster 2>/dev/null || true
|
||||
sleep 0.3
|
||||
VERBOSE=-vvv "$DEPLOY_DIR/scripts/start.sh" ctrl-1 \
|
||||
> "$LOGDIR/ctrl-1.log" 2>&1 &
|
||||
sleep 0.5
|
||||
VERBOSE=-vvv "$DEPLOY_DIR/scripts/start.sh" ctrl-2 \
|
||||
> "$LOGDIR/ctrl-2.log" 2>&1 &
|
||||
sleep 0.5
|
||||
VERBOSE=-vvv "$DEPLOY_DIR/scripts/start.sh" ctrl-3 \
|
||||
> "$LOGDIR/ctrl-3.log" 2>&1 &
|
||||
sleep 0.3
|
||||
}
|
||||
|
||||
kill_cluster() {
|
||||
pkill -f "xce_controller" 2>/dev/null || true
|
||||
sleep 0.5
|
||||
}
|
||||
|
||||
wait_for_primary() {
|
||||
local deadline=$(($(date +%s) + 30))
|
||||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
if grep -q "CTRL_PRIMARY" "$LOGDIR/ctrl-1.log" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_for_standbys() {
|
||||
# Both ctrl-2 and ctrl-3 must be in CTRL_STANDBY
|
||||
local deadline=$(($(date +%s) + 30))
|
||||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
local c2 c3
|
||||
c2=$(grep -c "CTRL_STANDBY" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
|
||||
c3=$(grep -c "CTRL_STANDBY" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
|
||||
if [ "$c2" -gt 0 ] && [ "$c3" -gt 0 ]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
ctrl1_pid() {
|
||||
pgrep -f "xce_controller.*ctrl-1" 2>/dev/null | head -1
|
||||
}
|
||||
|
||||
# ── setup ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
if [ ! -d "$DEPLOY_DIR" ]; then
|
||||
deploy_cluster
|
||||
fi
|
||||
|
||||
echo ""
|
||||
yellow "=== Task 4.5: Missed-Heartbeat Tolerance (SIGSTOP/SIGCONT) ==="
|
||||
echo ""
|
||||
|
||||
# ── Scenario A: short pause — no failover ─────────────────────────────────────
|
||||
|
||||
echo "--- Scenario A: SIGSTOP for 1 × heartbeat_interval_ms (${HB_INTERVAL_MS}ms) ---"
|
||||
echo " Expected: no failover fires on standbys"
|
||||
echo ""
|
||||
|
||||
bash "$DEPLOY_DIR/scripts/cleanup.sh"
|
||||
start_cluster
|
||||
|
||||
if ! wait_for_primary; then
|
||||
fail "Scenario A: cluster did not elect a primary within 30s"
|
||||
kill_cluster
|
||||
exit 1
|
||||
fi
|
||||
if ! wait_for_standbys; then
|
||||
fail "Scenario A: standbys did not reach CTRL_STANDBY within 30s"
|
||||
kill_cluster
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo " Cluster stable. Primary: ctrl-1"
|
||||
echo " SIGSTOP ctrl-1 for ${HB_INTERVAL_MS}ms ..."
|
||||
|
||||
PID=$(ctrl1_pid)
|
||||
if [ -z "$PID" ]; then
|
||||
fail "Scenario A: ctrl-1 process not found"
|
||||
kill_cluster
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Snapshot log lengths before pause
|
||||
PRE_RDHUP_2=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
|
||||
PRE_RDHUP_3=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
|
||||
|
||||
kill -STOP "$PID"
|
||||
sleep "$(echo "scale=1; $HB_INTERVAL_MS / 1000" | bc)s"
|
||||
kill -CONT "$PID"
|
||||
|
||||
# Give the cluster a moment to react (if it mistakenly fires failover)
|
||||
sleep 3
|
||||
|
||||
POST_RDHUP_2=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
|
||||
POST_RDHUP_3=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
|
||||
|
||||
NEW_RDHUP_2=$((POST_RDHUP_2 - PRE_RDHUP_2))
|
||||
NEW_RDHUP_3=$((POST_RDHUP_3 - PRE_RDHUP_3))
|
||||
|
||||
if [ "$NEW_RDHUP_2" -eq 0 ] && [ "$NEW_RDHUP_3" -eq 0 ]; then
|
||||
pass "Scenario A: no RDHUP fired on ctrl-2 or ctrl-3 during short pause (Req 2.2)"
|
||||
else
|
||||
fail "Scenario A: unexpected RDHUP fires (ctrl-2: +${NEW_RDHUP_2}, ctrl-3: +${NEW_RDHUP_3})"
|
||||
fi
|
||||
|
||||
# Verify cluster is still stable
|
||||
sleep 2
|
||||
if grep -q "CTRL_PRIMARY" "$LOGDIR/ctrl-1.log" && kill -0 "$PID" 2>/dev/null; then
|
||||
pass "Scenario A: ctrl-1 still primary after SIGCONT"
|
||||
else
|
||||
fail "Scenario A: ctrl-1 lost primary role unexpectedly"
|
||||
fi
|
||||
|
||||
kill_cluster
|
||||
sleep 1
|
||||
|
||||
# ── Scenario B: long pause — failover fires exactly once ─────────────────────
|
||||
|
||||
LONG_PAUSE_MS=$((HB_TIMEOUT_MS + 500))
|
||||
echo ""
|
||||
echo "--- Scenario B: SIGSTOP for heartbeat_timeout_ms + 500ms (${LONG_PAUSE_MS}ms) ---"
|
||||
echo " Expected: failover fires exactly once on each standby"
|
||||
echo ""
|
||||
|
||||
bash "$DEPLOY_DIR/scripts/cleanup.sh"
|
||||
start_cluster
|
||||
|
||||
if ! wait_for_primary; then
|
||||
fail "Scenario B: cluster did not elect a primary within 30s"
|
||||
kill_cluster
|
||||
exit 1
|
||||
fi
|
||||
if ! wait_for_standbys; then
|
||||
fail "Scenario B: standbys did not reach CTRL_STANDBY within 30s"
|
||||
kill_cluster
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Extra settle time so heartbeat clocks are steady
|
||||
sleep 5
|
||||
|
||||
echo " Cluster stable. Primary: ctrl-1"
|
||||
echo " SIGSTOP ctrl-1 for ${LONG_PAUSE_MS}ms ..."
|
||||
|
||||
PID=$(ctrl1_pid)
|
||||
if [ -z "$PID" ]; then
|
||||
fail "Scenario B: ctrl-1 process not found"
|
||||
kill_cluster
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PRE_RDHUP_2=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
|
||||
PRE_RDHUP_3=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
|
||||
PRE_ELECTING_2=$(grep -c "CTRL_ELECTING" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
|
||||
PRE_ELECTING_3=$(grep -c "CTRL_ELECTING" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
|
||||
|
||||
kill -STOP "$PID"
|
||||
sleep "$(echo "scale=1; $LONG_PAUSE_MS / 1000" | bc)s"
|
||||
|
||||
# Do NOT SIGCONT: leave ctrl-1 stopped so it cannot win re-election.
|
||||
# Standbys should detect the silence and start failover.
|
||||
|
||||
# Wait for failover to complete (timeout + election + committing + margin)
|
||||
FAILOVER_WAIT=$(( (HB_TIMEOUT_MS + 5000 + 10000 + 5000) / 1000 ))
|
||||
echo " Waiting up to ${FAILOVER_WAIT}s for failover ..."
|
||||
deadline=$(($(date +%s) + FAILOVER_WAIT))
|
||||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
NEW_ELECTING=$(( \
|
||||
$(grep -c "CTRL_ELECTING" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0) \
|
||||
- PRE_ELECTING_2 \
|
||||
))
|
||||
if [ "$NEW_ELECTING" -gt 0 ]; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Now SIGCONT ctrl-1 (it should lose since a new primary was elected)
|
||||
kill -CONT "$PID" 2>/dev/null || true
|
||||
|
||||
# Collect final counts
|
||||
sleep 5
|
||||
POST_RDHUP_2=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
|
||||
POST_RDHUP_3=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
|
||||
|
||||
NEW_RDHUP_2=$((POST_RDHUP_2 - PRE_RDHUP_2))
|
||||
NEW_RDHUP_3=$((POST_RDHUP_3 - PRE_RDHUP_3))
|
||||
|
||||
# Each standby should show exactly 1 new RDHUP (not 0, not >1)
|
||||
if [ "$NEW_RDHUP_2" -eq 1 ]; then
|
||||
pass "Scenario B: ctrl-2 fired exactly 1 RDHUP during long pause (Req 2.3)"
|
||||
elif [ "$NEW_RDHUP_2" -eq 0 ]; then
|
||||
fail "Scenario B: ctrl-2 fired 0 RDHUP — liveness poll did not trigger failover"
|
||||
else
|
||||
fail "Scenario B: ctrl-2 fired ${NEW_RDHUP_2} RDHUPs — duplicate failover loop (Req 5.4)"
|
||||
fi
|
||||
|
||||
if [ "$NEW_RDHUP_3" -eq 1 ]; then
|
||||
pass "Scenario B: ctrl-3 fired exactly 1 RDHUP during long pause (Req 2.3)"
|
||||
elif [ "$NEW_RDHUP_3" -eq 0 ]; then
|
||||
fail "Scenario B: ctrl-3 fired 0 RDHUP — liveness poll did not trigger failover"
|
||||
else
|
||||
fail "Scenario B: ctrl-3 fired ${NEW_RDHUP_3} RDHUPs — duplicate failover loop (Req 5.4)"
|
||||
fi
|
||||
|
||||
# One of ctrl-2 or ctrl-3 must have become PRIMARY
|
||||
NEW_PRIMARY=""
|
||||
if grep -q "CTRL_PRIMARY" "$LOGDIR/ctrl-2.log" 2>/dev/null; then
|
||||
NEW_PRIMARY="ctrl-2"
|
||||
elif grep -q "CTRL_PRIMARY" "$LOGDIR/ctrl-3.log" 2>/dev/null; then
|
||||
NEW_PRIMARY="ctrl-3"
|
||||
fi
|
||||
|
||||
if [ -n "$NEW_PRIMARY" ]; then
|
||||
pass "Scenario B: $NEW_PRIMARY promoted to CTRL_PRIMARY after failover"
|
||||
else
|
||||
fail "Scenario B: no standby reached CTRL_PRIMARY — election did not complete"
|
||||
fi
|
||||
|
||||
kill_cluster
|
||||
|
||||
# ── summary ───────────────────────────────────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "========================="
|
||||
echo "Results: ${PASS} passed, ${FAIL} failed"
|
||||
echo "========================="
|
||||
|
||||
[ "$FAIL" -eq 0 ] && exit 0 || exit 1
|
||||
Loading…
Add table
Add a link
Reference in a new issue