Rongsong Shen
ee8473bd99
deploy-ha.sh no longer takes a <data-dir> argument or --build-config
flag. Every deploy now generates a fresh self-signed TLS cert/key
(openssl, RSA 2048, 10y validity) directly into the deploy dir's
certs/, and builds cluster.bin directly into controller/ctrl-1/conf/
via xceadmin -- no intermediate staging directory, nothing preserved
across runs. run-ha-soak.sh (/xce-test-ha) updated to match: dropped
its own XCE_HA_DATADIR-based precondition entirely. Updated 4 callers
(run-ha-soak.sh, test-heartbeat-tolerance.sh, test-commit-durability.sh,
both xce-deploy-ha/xce-test-ha SKILL.md docs).
Separately: xcecommon.wire.connect() was loading a client-side TLS
cert, diverging from the real reference client -- netio.c's
create_ssl_context() presents no certificate at all when acting as a
client (cert_file/key_file both NULL, bare TLS_client_method()).
Fixed connect() to match, and since nothing else consumed cert_dir
(verify_mode=CERT_NONE throughout, no CA check either), removed it
end-to-end: discovery.follow_redirects(), push.push_config()/
probe_config(), and the --cert-dir CLI flag.
Server-side cert loading (mock TLS test servers) is unaffected --
moved into a shared session-scoped tls_certs fixture (new
tests/conftest.py, openssl-generated) replacing 3 duplicated
REPO_HA_DATA_DIR/skipif blocks. Net result: those tests no longer skip
under a hermetic nix build sandbox (which never had ha/data/) --
verified 165/165 passed, 0 skipped, via both meson compile and a full
nix build --option sandbox false .#xcecore_debug rebuild (previously
128 passed / 37 skipped there).
ha/data/{xce.crt,xce.key} removed from git tracking. Requirements
2.3/2.4 and 8.3/8.4/8.5 marked superseded in requirements.md with
dated correction notes.
288 lines
9.3 KiB
Bash
Executable file
288 lines
9.3 KiB
Bash
Executable file
#!/bin/bash
|
||
# test-heartbeat-tolerance.sh — Task 4.5: missed-heartbeat tolerance via SIGSTOP/SIGCONT
|
||
#
|
||
# Requirements: 2.2, 2.3
|
||
#
|
||
# Two scenarios:
|
||
# Scenario A — short pause (1 × heartbeat_interval_ms): no failover fires
|
||
# Scenario B — long pause (heartbeat_timeout_ms + 500ms): failover fires exactly once
|
||
#
|
||
# Usage:
|
||
# bash src/tools/test-heartbeat-tolerance.sh [deploy-dir]
|
||
#
|
||
# deploy-dir defaults to /tmp/xce-deploy. The script expects a deployed
|
||
# 3-node cluster (via deploy-ha.sh + start.sh). It deploys one if the
|
||
# binary exists at the default location.
|
||
set -e
|
||
|
||
WORKTREE=$(cd "$(dirname "$0")/../.." && pwd)
|
||
BINARY="$WORKTREE/src/xce.build/controller/xce_controller"
|
||
DEPLOY_DIR="${1:-/tmp/xce-deploy}"
|
||
PEERS="localhost:9991,localhost:9992,localhost:9993"
|
||
LOGDIR="$DEPLOY_DIR/logs"
|
||
|
||
# Defaults from ha.h
|
||
HB_INTERVAL_MS=3000
|
||
HB_TIMEOUT_MS=10000
|
||
|
||
PASS=0
|
||
FAIL=0
|
||
|
||
# ── colour helpers ────────────────────────────────────────────────────────────
|
||
|
||
green() { printf '\033[32m%s\033[0m\n' "$*"; }
|
||
red() { printf '\033[31m%s\033[0m\n' "$*"; }
|
||
yellow() { printf '\033[33m%s\033[0m\n' "$*"; }
|
||
|
||
pass() { green " PASS: $*"; PASS=$((PASS+1)); }
|
||
fail() { red " FAIL: $*"; FAIL=$((FAIL+1)); }
|
||
|
||
# ── cluster lifecycle ─────────────────────────────────────────────────────────
|
||
|
||
deploy_cluster() {
|
||
if [ ! -f "$BINARY" ]; then
|
||
red "Binary not found: $BINARY"
|
||
red "Run /xce-build controller first."
|
||
exit 1
|
||
fi
|
||
yellow "Deploying 3-node cluster to $DEPLOY_DIR ..."
|
||
bash "$WORKTREE/src/tools/deploy-ha.sh" \
|
||
"$BINARY" "$DEPLOY_DIR" "$PEERS"
|
||
}
|
||
|
||
start_cluster() {
|
||
kill_cluster 2>/dev/null || true
|
||
sleep 0.3
|
||
VERBOSE=-vvv "$DEPLOY_DIR/scripts/start.sh" ctrl-1 \
|
||
> "$LOGDIR/ctrl-1.log" 2>&1 &
|
||
sleep 0.5
|
||
VERBOSE=-vvv "$DEPLOY_DIR/scripts/start.sh" ctrl-2 \
|
||
> "$LOGDIR/ctrl-2.log" 2>&1 &
|
||
sleep 0.5
|
||
VERBOSE=-vvv "$DEPLOY_DIR/scripts/start.sh" ctrl-3 \
|
||
> "$LOGDIR/ctrl-3.log" 2>&1 &
|
||
sleep 0.3
|
||
}
|
||
|
||
kill_cluster() {
|
||
pkill -f "xce_controller" 2>/dev/null || true
|
||
sleep 0.5
|
||
}
|
||
|
||
wait_for_primary() {
|
||
local deadline=$(($(date +%s) + 30))
|
||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||
if grep -q "CTRL_PRIMARY" "$LOGDIR/ctrl-1.log" 2>/dev/null; then
|
||
return 0
|
||
fi
|
||
sleep 1
|
||
done
|
||
return 1
|
||
}
|
||
|
||
wait_for_standbys() {
|
||
# Both ctrl-2 and ctrl-3 must be in CTRL_STANDBY
|
||
local deadline=$(($(date +%s) + 30))
|
||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||
local c2 c3
|
||
c2=$(grep -c "CTRL_STANDBY" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
|
||
c3=$(grep -c "CTRL_STANDBY" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
|
||
if [ "$c2" -gt 0 ] && [ "$c3" -gt 0 ]; then
|
||
return 0
|
||
fi
|
||
sleep 1
|
||
done
|
||
return 1
|
||
}
|
||
|
||
ctrl1_pid() {
|
||
pgrep -f "xce_controller.*ctrl-1" 2>/dev/null | head -1
|
||
}
|
||
|
||
# ── setup ─────────────────────────────────────────────────────────────────────
|
||
|
||
if [ ! -d "$DEPLOY_DIR" ]; then
|
||
deploy_cluster
|
||
fi
|
||
|
||
echo ""
|
||
yellow "=== Task 4.5: Missed-Heartbeat Tolerance (SIGSTOP/SIGCONT) ==="
|
||
echo ""
|
||
|
||
# ── Scenario A: short pause — no failover ─────────────────────────────────────
|
||
|
||
echo "--- Scenario A: SIGSTOP for 1 × heartbeat_interval_ms (${HB_INTERVAL_MS}ms) ---"
|
||
echo " Expected: no failover fires on standbys"
|
||
echo ""
|
||
|
||
bash "$DEPLOY_DIR/scripts/cleanup.sh"
|
||
start_cluster
|
||
|
||
if ! wait_for_primary; then
|
||
fail "Scenario A: cluster did not elect a primary within 30s"
|
||
kill_cluster
|
||
exit 1
|
||
fi
|
||
if ! wait_for_standbys; then
|
||
fail "Scenario A: standbys did not reach CTRL_STANDBY within 30s"
|
||
kill_cluster
|
||
exit 1
|
||
fi
|
||
|
||
echo " Cluster stable. Primary: ctrl-1"
|
||
echo " SIGSTOP ctrl-1 for ${HB_INTERVAL_MS}ms ..."
|
||
|
||
PID=$(ctrl1_pid)
|
||
if [ -z "$PID" ]; then
|
||
fail "Scenario A: ctrl-1 process not found"
|
||
kill_cluster
|
||
exit 1
|
||
fi
|
||
|
||
# Snapshot log lengths before pause
|
||
PRE_RDHUP_2=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
|
||
PRE_RDHUP_3=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
|
||
|
||
kill -STOP "$PID"
|
||
sleep "$(echo "scale=1; $HB_INTERVAL_MS / 1000" | bc)s"
|
||
kill -CONT "$PID"
|
||
|
||
# Give the cluster a moment to react (if it mistakenly fires failover)
|
||
sleep 3
|
||
|
||
POST_RDHUP_2=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
|
||
POST_RDHUP_3=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
|
||
|
||
NEW_RDHUP_2=$((POST_RDHUP_2 - PRE_RDHUP_2))
|
||
NEW_RDHUP_3=$((POST_RDHUP_3 - PRE_RDHUP_3))
|
||
|
||
if [ "$NEW_RDHUP_2" -eq 0 ] && [ "$NEW_RDHUP_3" -eq 0 ]; then
|
||
pass "Scenario A: no RDHUP fired on ctrl-2 or ctrl-3 during short pause (Req 2.2)"
|
||
else
|
||
fail "Scenario A: unexpected RDHUP fires (ctrl-2: +${NEW_RDHUP_2}, ctrl-3: +${NEW_RDHUP_3})"
|
||
fi
|
||
|
||
# Verify cluster is still stable
|
||
sleep 2
|
||
if grep -q "CTRL_PRIMARY" "$LOGDIR/ctrl-1.log" && kill -0 "$PID" 2>/dev/null; then
|
||
pass "Scenario A: ctrl-1 still primary after SIGCONT"
|
||
else
|
||
fail "Scenario A: ctrl-1 lost primary role unexpectedly"
|
||
fi
|
||
|
||
kill_cluster
|
||
sleep 1
|
||
|
||
# ── Scenario B: long pause — failover fires exactly once ─────────────────────
|
||
|
||
LONG_PAUSE_MS=$((HB_TIMEOUT_MS + 500))
|
||
echo ""
|
||
echo "--- Scenario B: SIGSTOP for heartbeat_timeout_ms + 500ms (${LONG_PAUSE_MS}ms) ---"
|
||
echo " Expected: failover fires exactly once on each standby"
|
||
echo ""
|
||
|
||
bash "$DEPLOY_DIR/scripts/cleanup.sh"
|
||
start_cluster
|
||
|
||
if ! wait_for_primary; then
|
||
fail "Scenario B: cluster did not elect a primary within 30s"
|
||
kill_cluster
|
||
exit 1
|
||
fi
|
||
if ! wait_for_standbys; then
|
||
fail "Scenario B: standbys did not reach CTRL_STANDBY within 30s"
|
||
kill_cluster
|
||
exit 1
|
||
fi
|
||
|
||
# Extra settle time so heartbeat clocks are steady
|
||
sleep 5
|
||
|
||
echo " Cluster stable. Primary: ctrl-1"
|
||
echo " SIGSTOP ctrl-1 for ${LONG_PAUSE_MS}ms ..."
|
||
|
||
PID=$(ctrl1_pid)
|
||
if [ -z "$PID" ]; then
|
||
fail "Scenario B: ctrl-1 process not found"
|
||
kill_cluster
|
||
exit 1
|
||
fi
|
||
|
||
PRE_RDHUP_2=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
|
||
PRE_RDHUP_3=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
|
||
PRE_ELECTING_2=$(grep -c "CTRL_ELECTING" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
|
||
PRE_ELECTING_3=$(grep -c "CTRL_ELECTING" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
|
||
|
||
kill -STOP "$PID"
|
||
sleep "$(echo "scale=1; $LONG_PAUSE_MS / 1000" | bc)s"
|
||
|
||
# Do NOT SIGCONT: leave ctrl-1 stopped so it cannot win re-election.
|
||
# Standbys should detect the silence and start failover.
|
||
|
||
# Wait for failover to complete (timeout + election + committing + margin)
|
||
FAILOVER_WAIT=$(( (HB_TIMEOUT_MS + 5000 + 10000 + 5000) / 1000 ))
|
||
echo " Waiting up to ${FAILOVER_WAIT}s for failover ..."
|
||
deadline=$(($(date +%s) + FAILOVER_WAIT))
|
||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||
NEW_ELECTING=$(( \
|
||
$(grep -c "CTRL_ELECTING" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0) \
|
||
- PRE_ELECTING_2 \
|
||
))
|
||
if [ "$NEW_ELECTING" -gt 0 ]; then
|
||
break
|
||
fi
|
||
sleep 1
|
||
done
|
||
|
||
# Now SIGCONT ctrl-1 (it should lose since a new primary was elected)
|
||
kill -CONT "$PID" 2>/dev/null || true
|
||
|
||
# Collect final counts
|
||
sleep 5
|
||
POST_RDHUP_2=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
|
||
POST_RDHUP_3=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
|
||
|
||
NEW_RDHUP_2=$((POST_RDHUP_2 - PRE_RDHUP_2))
|
||
NEW_RDHUP_3=$((POST_RDHUP_3 - PRE_RDHUP_3))
|
||
|
||
# Each standby should show exactly 1 new RDHUP (not 0, not >1)
|
||
if [ "$NEW_RDHUP_2" -eq 1 ]; then
|
||
pass "Scenario B: ctrl-2 fired exactly 1 RDHUP during long pause (Req 2.3)"
|
||
elif [ "$NEW_RDHUP_2" -eq 0 ]; then
|
||
fail "Scenario B: ctrl-2 fired 0 RDHUP — liveness poll did not trigger failover"
|
||
else
|
||
fail "Scenario B: ctrl-2 fired ${NEW_RDHUP_2} RDHUPs — duplicate failover loop (Req 5.4)"
|
||
fi
|
||
|
||
if [ "$NEW_RDHUP_3" -eq 1 ]; then
|
||
pass "Scenario B: ctrl-3 fired exactly 1 RDHUP during long pause (Req 2.3)"
|
||
elif [ "$NEW_RDHUP_3" -eq 0 ]; then
|
||
fail "Scenario B: ctrl-3 fired 0 RDHUP — liveness poll did not trigger failover"
|
||
else
|
||
fail "Scenario B: ctrl-3 fired ${NEW_RDHUP_3} RDHUPs — duplicate failover loop (Req 5.4)"
|
||
fi
|
||
|
||
# One of ctrl-2 or ctrl-3 must have become PRIMARY
|
||
NEW_PRIMARY=""
|
||
if grep -q "CTRL_PRIMARY" "$LOGDIR/ctrl-2.log" 2>/dev/null; then
|
||
NEW_PRIMARY="ctrl-2"
|
||
elif grep -q "CTRL_PRIMARY" "$LOGDIR/ctrl-3.log" 2>/dev/null; then
|
||
NEW_PRIMARY="ctrl-3"
|
||
fi
|
||
|
||
if [ -n "$NEW_PRIMARY" ]; then
|
||
pass "Scenario B: $NEW_PRIMARY promoted to CTRL_PRIMARY after failover"
|
||
else
|
||
fail "Scenario B: no standby reached CTRL_PRIMARY — election did not complete"
|
||
fi
|
||
|
||
kill_cluster
|
||
|
||
# ── summary ───────────────────────────────────────────────────────────────────
|
||
|
||
echo ""
|
||
echo "========================="
|
||
echo "Results: ${PASS} passed, ${FAIL} failed"
|
||
echo "========================="
|
||
|
||
[ "$FAIL" -eq 0 ] && exit 0 || exit 1
|