Rongsong Shen
e56b2064de
Add src/tools/test-heartbeat-tolerance.sh: automated bash script that deploys a 3-node cluster and runs two SIGSTOP/SIGCONT scenarios: Scenario A (1 × heartbeat_interval_ms pause): verifies no RDHUP fires on standbys — liveness_poll_fn must not trigger failover for a short silence within the tolerance window (Req 2.2). Scenario B (heartbeat_timeout_ms + 500ms pause): verifies exactly one RDHUP fires on each standby — the activity-based poll fires after sustained silence and does not loop (Req 2.3, Req 5.4). Add .kiro/specs/ha-heartbeat-liveness/manual-tests.md: step-by-step manual procedure, expected log patterns, and results template for both scenarios. Mark task 4.5 complete in tasks.md.
289 lines
9.4 KiB
Bash
Executable file
289 lines
9.4 KiB
Bash
Executable file
#!/bin/bash
|
||
# test-heartbeat-tolerance.sh — Task 4.5: missed-heartbeat tolerance via SIGSTOP/SIGCONT
|
||
#
|
||
# Requirements: 2.2, 2.3
|
||
#
|
||
# Two scenarios:
|
||
# Scenario A — short pause (1 × heartbeat_interval_ms): no failover fires
|
||
# Scenario B — long pause (heartbeat_timeout_ms + 500ms): failover fires exactly once
|
||
#
|
||
# Usage:
|
||
# bash src/tools/test-heartbeat-tolerance.sh [deploy-dir]
|
||
#
|
||
# deploy-dir defaults to /tmp/xce-deploy. The script expects a deployed
|
||
# 3-node cluster (via deploy-ha.sh + start.sh). It deploys one if the
|
||
# binary exists at the default location.
|
||
set -e
|
||
|
||
WORKTREE=$(cd "$(dirname "$0")/../.." && pwd)
|
||
BINARY="$WORKTREE/src/xce.build/controller/xce_controller"
|
||
DATADIR="$WORKTREE/ha/data"
|
||
DEPLOY_DIR="${1:-/tmp/xce-deploy}"
|
||
PEERS="localhost:9991,localhost:9992,localhost:9993"
|
||
LOGDIR="$DEPLOY_DIR/logs"
|
||
|
||
# Defaults from ha.h
|
||
HB_INTERVAL_MS=3000
|
||
HB_TIMEOUT_MS=10000
|
||
|
||
PASS=0
|
||
FAIL=0
|
||
|
||
# ── colour helpers ────────────────────────────────────────────────────────────
|
||
|
||
green() { printf '\033[32m%s\033[0m\n' "$*"; }
|
||
red() { printf '\033[31m%s\033[0m\n' "$*"; }
|
||
yellow() { printf '\033[33m%s\033[0m\n' "$*"; }
|
||
|
||
pass() { green " PASS: $*"; PASS=$((PASS+1)); }
|
||
fail() { red " FAIL: $*"; FAIL=$((FAIL+1)); }
|
||
|
||
# ── cluster lifecycle ─────────────────────────────────────────────────────────
|
||
|
||
deploy_cluster() {
|
||
if [ ! -f "$BINARY" ]; then
|
||
red "Binary not found: $BINARY"
|
||
red "Run /xce-build controller first."
|
||
exit 1
|
||
fi
|
||
yellow "Deploying 3-node cluster to $DEPLOY_DIR ..."
|
||
bash "$WORKTREE/src/tools/deploy-ha.sh" \
|
||
"$BINARY" "$DATADIR" "$DEPLOY_DIR" "$PEERS"
|
||
}
|
||
|
||
start_cluster() {
|
||
kill_cluster 2>/dev/null || true
|
||
sleep 0.3
|
||
VERBOSE=-vvv "$DEPLOY_DIR/scripts/start.sh" ctrl-1 \
|
||
> "$LOGDIR/ctrl-1.log" 2>&1 &
|
||
sleep 0.5
|
||
VERBOSE=-vvv "$DEPLOY_DIR/scripts/start.sh" ctrl-2 \
|
||
> "$LOGDIR/ctrl-2.log" 2>&1 &
|
||
sleep 0.5
|
||
VERBOSE=-vvv "$DEPLOY_DIR/scripts/start.sh" ctrl-3 \
|
||
> "$LOGDIR/ctrl-3.log" 2>&1 &
|
||
sleep 0.3
|
||
}
|
||
|
||
kill_cluster() {
|
||
pkill -f "xce_controller" 2>/dev/null || true
|
||
sleep 0.5
|
||
}
|
||
|
||
wait_for_primary() {
|
||
local deadline=$(($(date +%s) + 30))
|
||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||
if grep -q "CTRL_PRIMARY" "$LOGDIR/ctrl-1.log" 2>/dev/null; then
|
||
return 0
|
||
fi
|
||
sleep 1
|
||
done
|
||
return 1
|
||
}
|
||
|
||
wait_for_standbys() {
|
||
# Both ctrl-2 and ctrl-3 must be in CTRL_STANDBY
|
||
local deadline=$(($(date +%s) + 30))
|
||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||
local c2 c3
|
||
c2=$(grep -c "CTRL_STANDBY" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
|
||
c3=$(grep -c "CTRL_STANDBY" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
|
||
if [ "$c2" -gt 0 ] && [ "$c3" -gt 0 ]; then
|
||
return 0
|
||
fi
|
||
sleep 1
|
||
done
|
||
return 1
|
||
}
|
||
|
||
ctrl1_pid() {
|
||
pgrep -f "xce_controller.*ctrl-1" 2>/dev/null | head -1
|
||
}
|
||
|
||
# ── setup ─────────────────────────────────────────────────────────────────────
|
||
|
||
if [ ! -d "$DEPLOY_DIR" ]; then
|
||
deploy_cluster
|
||
fi
|
||
|
||
echo ""
|
||
yellow "=== Task 4.5: Missed-Heartbeat Tolerance (SIGSTOP/SIGCONT) ==="
|
||
echo ""
|
||
|
||
# ── Scenario A: short pause — no failover ─────────────────────────────────────
|
||
|
||
echo "--- Scenario A: SIGSTOP for 1 × heartbeat_interval_ms (${HB_INTERVAL_MS}ms) ---"
|
||
echo " Expected: no failover fires on standbys"
|
||
echo ""
|
||
|
||
bash "$DEPLOY_DIR/scripts/cleanup.sh"
|
||
start_cluster
|
||
|
||
if ! wait_for_primary; then
|
||
fail "Scenario A: cluster did not elect a primary within 30s"
|
||
kill_cluster
|
||
exit 1
|
||
fi
|
||
if ! wait_for_standbys; then
|
||
fail "Scenario A: standbys did not reach CTRL_STANDBY within 30s"
|
||
kill_cluster
|
||
exit 1
|
||
fi
|
||
|
||
echo " Cluster stable. Primary: ctrl-1"
|
||
echo " SIGSTOP ctrl-1 for ${HB_INTERVAL_MS}ms ..."
|
||
|
||
PID=$(ctrl1_pid)
|
||
if [ -z "$PID" ]; then
|
||
fail "Scenario A: ctrl-1 process not found"
|
||
kill_cluster
|
||
exit 1
|
||
fi
|
||
|
||
# Snapshot log lengths before pause
|
||
PRE_RDHUP_2=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
|
||
PRE_RDHUP_3=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
|
||
|
||
kill -STOP "$PID"
|
||
sleep "$(echo "scale=1; $HB_INTERVAL_MS / 1000" | bc)s"
|
||
kill -CONT "$PID"
|
||
|
||
# Give the cluster a moment to react (if it mistakenly fires failover)
|
||
sleep 3
|
||
|
||
POST_RDHUP_2=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
|
||
POST_RDHUP_3=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
|
||
|
||
NEW_RDHUP_2=$((POST_RDHUP_2 - PRE_RDHUP_2))
|
||
NEW_RDHUP_3=$((POST_RDHUP_3 - PRE_RDHUP_3))
|
||
|
||
if [ "$NEW_RDHUP_2" -eq 0 ] && [ "$NEW_RDHUP_3" -eq 0 ]; then
|
||
pass "Scenario A: no RDHUP fired on ctrl-2 or ctrl-3 during short pause (Req 2.2)"
|
||
else
|
||
fail "Scenario A: unexpected RDHUP fires (ctrl-2: +${NEW_RDHUP_2}, ctrl-3: +${NEW_RDHUP_3})"
|
||
fi
|
||
|
||
# Verify cluster is still stable
|
||
sleep 2
|
||
if grep -q "CTRL_PRIMARY" "$LOGDIR/ctrl-1.log" && kill -0 "$PID" 2>/dev/null; then
|
||
pass "Scenario A: ctrl-1 still primary after SIGCONT"
|
||
else
|
||
fail "Scenario A: ctrl-1 lost primary role unexpectedly"
|
||
fi
|
||
|
||
kill_cluster
|
||
sleep 1
|
||
|
||
# ── Scenario B: long pause — failover fires exactly once ─────────────────────
|
||
|
||
LONG_PAUSE_MS=$((HB_TIMEOUT_MS + 500))
|
||
echo ""
|
||
echo "--- Scenario B: SIGSTOP for heartbeat_timeout_ms + 500ms (${LONG_PAUSE_MS}ms) ---"
|
||
echo " Expected: failover fires exactly once on each standby"
|
||
echo ""
|
||
|
||
bash "$DEPLOY_DIR/scripts/cleanup.sh"
|
||
start_cluster
|
||
|
||
if ! wait_for_primary; then
|
||
fail "Scenario B: cluster did not elect a primary within 30s"
|
||
kill_cluster
|
||
exit 1
|
||
fi
|
||
if ! wait_for_standbys; then
|
||
fail "Scenario B: standbys did not reach CTRL_STANDBY within 30s"
|
||
kill_cluster
|
||
exit 1
|
||
fi
|
||
|
||
# Extra settle time so heartbeat clocks are steady
|
||
sleep 5
|
||
|
||
echo " Cluster stable. Primary: ctrl-1"
|
||
echo " SIGSTOP ctrl-1 for ${LONG_PAUSE_MS}ms ..."
|
||
|
||
PID=$(ctrl1_pid)
|
||
if [ -z "$PID" ]; then
|
||
fail "Scenario B: ctrl-1 process not found"
|
||
kill_cluster
|
||
exit 1
|
||
fi
|
||
|
||
PRE_RDHUP_2=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
|
||
PRE_RDHUP_3=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
|
||
PRE_ELECTING_2=$(grep -c "CTRL_ELECTING" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
|
||
PRE_ELECTING_3=$(grep -c "CTRL_ELECTING" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
|
||
|
||
kill -STOP "$PID"
|
||
sleep "$(echo "scale=1; $LONG_PAUSE_MS / 1000" | bc)s"
|
||
|
||
# Do NOT SIGCONT: leave ctrl-1 stopped so it cannot win re-election.
|
||
# Standbys should detect the silence and start failover.
|
||
|
||
# Wait for failover to complete (timeout + election + committing + margin)
|
||
FAILOVER_WAIT=$(( (HB_TIMEOUT_MS + 5000 + 10000 + 5000) / 1000 ))
|
||
echo " Waiting up to ${FAILOVER_WAIT}s for failover ..."
|
||
deadline=$(($(date +%s) + FAILOVER_WAIT))
|
||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||
NEW_ELECTING=$(( \
|
||
$(grep -c "CTRL_ELECTING" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0) \
|
||
- PRE_ELECTING_2 \
|
||
))
|
||
if [ "$NEW_ELECTING" -gt 0 ]; then
|
||
break
|
||
fi
|
||
sleep 1
|
||
done
|
||
|
||
# Now SIGCONT ctrl-1 (it should lose since a new primary was elected)
|
||
kill -CONT "$PID" 2>/dev/null || true
|
||
|
||
# Collect final counts
|
||
sleep 5
|
||
POST_RDHUP_2=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
|
||
POST_RDHUP_3=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
|
||
|
||
NEW_RDHUP_2=$((POST_RDHUP_2 - PRE_RDHUP_2))
|
||
NEW_RDHUP_3=$((POST_RDHUP_3 - PRE_RDHUP_3))
|
||
|
||
# Each standby should show exactly 1 new RDHUP (not 0, not >1)
|
||
if [ "$NEW_RDHUP_2" -eq 1 ]; then
|
||
pass "Scenario B: ctrl-2 fired exactly 1 RDHUP during long pause (Req 2.3)"
|
||
elif [ "$NEW_RDHUP_2" -eq 0 ]; then
|
||
fail "Scenario B: ctrl-2 fired 0 RDHUP — liveness poll did not trigger failover"
|
||
else
|
||
fail "Scenario B: ctrl-2 fired ${NEW_RDHUP_2} RDHUPs — duplicate failover loop (Req 5.4)"
|
||
fi
|
||
|
||
if [ "$NEW_RDHUP_3" -eq 1 ]; then
|
||
pass "Scenario B: ctrl-3 fired exactly 1 RDHUP during long pause (Req 2.3)"
|
||
elif [ "$NEW_RDHUP_3" -eq 0 ]; then
|
||
fail "Scenario B: ctrl-3 fired 0 RDHUP — liveness poll did not trigger failover"
|
||
else
|
||
fail "Scenario B: ctrl-3 fired ${NEW_RDHUP_3} RDHUPs — duplicate failover loop (Req 5.4)"
|
||
fi
|
||
|
||
# One of ctrl-2 or ctrl-3 must have become PRIMARY
|
||
NEW_PRIMARY=""
|
||
if grep -q "CTRL_PRIMARY" "$LOGDIR/ctrl-2.log" 2>/dev/null; then
|
||
NEW_PRIMARY="ctrl-2"
|
||
elif grep -q "CTRL_PRIMARY" "$LOGDIR/ctrl-3.log" 2>/dev/null; then
|
||
NEW_PRIMARY="ctrl-3"
|
||
fi
|
||
|
||
if [ -n "$NEW_PRIMARY" ]; then
|
||
pass "Scenario B: $NEW_PRIMARY promoted to CTRL_PRIMARY after failover"
|
||
else
|
||
fail "Scenario B: no standby reached CTRL_PRIMARY — election did not complete"
|
||
fi
|
||
|
||
kill_cluster
|
||
|
||
# ── summary ───────────────────────────────────────────────────────────────────
|
||
|
||
echo ""
|
||
echo "========================="
|
||
echo "Results: ${PASS} passed, ${FAIL} failed"
|
||
echo "========================="
|
||
|
||
[ "$FAIL" -eq 0 ] && exit 0 || exit 1
|