#!/bin/bash
# test-heartbeat-tolerance.sh — Task 4.5: missed-heartbeat tolerance via SIGSTOP/SIGCONT
#
# Requirements: 2.2, 2.3
#
# Two scenarios:
#   Scenario A — short pause (1 × heartbeat_interval_ms): no failover fires
#   Scenario B — long pause (heartbeat_timeout_ms + 500ms): failover fires exactly once
#
# Usage:
#   bash src/tools/test-heartbeat-tolerance.sh [deploy-dir]
#
# deploy-dir defaults to /tmp/xce-deploy.  The script expects a deployed
# 3-node cluster (via deploy-ha.sh + start.sh).  It deploys one if the
# binary exists at the default location.
set -e

WORKTREE=$(cd "$(dirname "$0")/../.." && pwd)
BINARY="$WORKTREE/src/xce.build/controller/xce_controller"
DATADIR="$WORKTREE/ha/data"
DEPLOY_DIR="${1:-/tmp/xce-deploy}"
PEERS="localhost:9991,localhost:9992,localhost:9993"
LOGDIR="$DEPLOY_DIR/logs"

# Defaults from ha.h
HB_INTERVAL_MS=3000
HB_TIMEOUT_MS=10000

PASS=0
FAIL=0

# ── colour helpers ────────────────────────────────────────────────────────────

green()  { printf '\033[32m%s\033[0m\n' "$*"; }
red()    { printf '\033[31m%s\033[0m\n' "$*"; }
yellow() { printf '\033[33m%s\033[0m\n' "$*"; }

pass() { green "  PASS: $*"; PASS=$((PASS+1)); }
fail() { red   "  FAIL: $*"; FAIL=$((FAIL+1)); }

# ── cluster lifecycle ─────────────────────────────────────────────────────────

deploy_cluster() {
    if [ ! -f "$BINARY" ]; then
        red "Binary not found: $BINARY"
        red "Run /xce-build controller first."
        exit 1
    fi
    yellow "Deploying 3-node cluster to $DEPLOY_DIR ..."
    bash "$WORKTREE/src/tools/deploy-ha.sh" \
        "$BINARY" "$DATADIR" "$DEPLOY_DIR" "$PEERS"
}

start_cluster() {
    kill_cluster 2>/dev/null || true
    sleep 0.3
    VERBOSE=-vvv "$DEPLOY_DIR/scripts/start.sh" ctrl-1 \
        > "$LOGDIR/ctrl-1.log" 2>&1 &
    sleep 0.5
    VERBOSE=-vvv "$DEPLOY_DIR/scripts/start.sh" ctrl-2 \
        > "$LOGDIR/ctrl-2.log" 2>&1 &
    sleep 0.5
    VERBOSE=-vvv "$DEPLOY_DIR/scripts/start.sh" ctrl-3 \
        > "$LOGDIR/ctrl-3.log" 2>&1 &
    sleep 0.3
}

kill_cluster() {
    pkill -f "xce_controller" 2>/dev/null || true
    sleep 0.5
}

wait_for_primary() {
    local deadline=$(($(date +%s) + 30))
    while [ "$(date +%s)" -lt "$deadline" ]; do
        if grep -q "CTRL_PRIMARY" "$LOGDIR/ctrl-1.log" 2>/dev/null; then
            return 0
        fi
        sleep 1
    done
    return 1
}

wait_for_standbys() {
    # Both ctrl-2 and ctrl-3 must be in CTRL_STANDBY
    local deadline=$(($(date +%s) + 30))
    while [ "$(date +%s)" -lt "$deadline" ]; do
        local c2 c3
        c2=$(grep -c "CTRL_STANDBY" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
        c3=$(grep -c "CTRL_STANDBY" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
        if [ "$c2" -gt 0 ] && [ "$c3" -gt 0 ]; then
            return 0
        fi
        sleep 1
    done
    return 1
}

ctrl1_pid() {
    pgrep -f "xce_controller.*ctrl-1" 2>/dev/null | head -1
}

# ── setup ─────────────────────────────────────────────────────────────────────

if [ ! -d "$DEPLOY_DIR" ]; then
    deploy_cluster
fi

echo ""
yellow "=== Task 4.5: Missed-Heartbeat Tolerance (SIGSTOP/SIGCONT) ==="
echo ""

# ── Scenario A: short pause — no failover ─────────────────────────────────────

echo "--- Scenario A: SIGSTOP for 1 × heartbeat_interval_ms (${HB_INTERVAL_MS}ms) ---"
echo "    Expected: no failover fires on standbys"
echo ""

bash "$DEPLOY_DIR/scripts/cleanup.sh"
start_cluster

if ! wait_for_primary; then
    fail "Scenario A: cluster did not elect a primary within 30s"
    kill_cluster
    exit 1
fi
if ! wait_for_standbys; then
    fail "Scenario A: standbys did not reach CTRL_STANDBY within 30s"
    kill_cluster
    exit 1
fi

echo "  Cluster stable. Primary: ctrl-1"
echo "  SIGSTOP ctrl-1 for ${HB_INTERVAL_MS}ms ..."

PID=$(ctrl1_pid)
if [ -z "$PID" ]; then
    fail "Scenario A: ctrl-1 process not found"
    kill_cluster
    exit 1
fi

# Snapshot log lengths before pause
PRE_RDHUP_2=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
PRE_RDHUP_3=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)

kill -STOP "$PID"
sleep "$(echo "scale=1; $HB_INTERVAL_MS / 1000" | bc)s"
kill -CONT "$PID"

# Give the cluster a moment to react (if it mistakenly fires failover)
sleep 3

POST_RDHUP_2=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
POST_RDHUP_3=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)

NEW_RDHUP_2=$((POST_RDHUP_2 - PRE_RDHUP_2))
NEW_RDHUP_3=$((POST_RDHUP_3 - PRE_RDHUP_3))

if [ "$NEW_RDHUP_2" -eq 0 ] && [ "$NEW_RDHUP_3" -eq 0 ]; then
    pass "Scenario A: no RDHUP fired on ctrl-2 or ctrl-3 during short pause (Req 2.2)"
else
    fail "Scenario A: unexpected RDHUP fires (ctrl-2: +${NEW_RDHUP_2}, ctrl-3: +${NEW_RDHUP_3})"
fi

# Verify cluster is still stable
sleep 2
if grep -q "CTRL_PRIMARY" "$LOGDIR/ctrl-1.log" && kill -0 "$PID" 2>/dev/null; then
    pass "Scenario A: ctrl-1 still primary after SIGCONT"
else
    fail "Scenario A: ctrl-1 lost primary role unexpectedly"
fi

kill_cluster
sleep 1

# ── Scenario B: long pause — failover fires exactly once ─────────────────────

LONG_PAUSE_MS=$((HB_TIMEOUT_MS + 500))
echo ""
echo "--- Scenario B: SIGSTOP for heartbeat_timeout_ms + 500ms (${LONG_PAUSE_MS}ms) ---"
echo "    Expected: failover fires exactly once on each standby"
echo ""

bash "$DEPLOY_DIR/scripts/cleanup.sh"
start_cluster

if ! wait_for_primary; then
    fail "Scenario B: cluster did not elect a primary within 30s"
    kill_cluster
    exit 1
fi
if ! wait_for_standbys; then
    fail "Scenario B: standbys did not reach CTRL_STANDBY within 30s"
    kill_cluster
    exit 1
fi

# Extra settle time so heartbeat clocks are steady
sleep 5

echo "  Cluster stable. Primary: ctrl-1"
echo "  SIGSTOP ctrl-1 for ${LONG_PAUSE_MS}ms ..."

PID=$(ctrl1_pid)
if [ -z "$PID" ]; then
    fail "Scenario B: ctrl-1 process not found"
    kill_cluster
    exit 1
fi

PRE_RDHUP_2=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
PRE_RDHUP_3=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)
PRE_ELECTING_2=$(grep -c "CTRL_ELECTING" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
PRE_ELECTING_3=$(grep -c "CTRL_ELECTING" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)

kill -STOP "$PID"
sleep "$(echo "scale=1; $LONG_PAUSE_MS / 1000" | bc)s"

# Do NOT SIGCONT: leave ctrl-1 stopped so it cannot win re-election.
# Standbys should detect the silence and start failover.

# Wait for failover to complete (timeout + election + committing + margin)
FAILOVER_WAIT=$(( (HB_TIMEOUT_MS + 5000 + 10000 + 5000) / 1000 ))
echo "  Waiting up to ${FAILOVER_WAIT}s for failover ..."
deadline=$(($(date +%s) + FAILOVER_WAIT))
while [ "$(date +%s)" -lt "$deadline" ]; do
    NEW_ELECTING=$(( \
        $(grep -c "CTRL_ELECTING" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0) \
        - PRE_ELECTING_2 \
    ))
    if [ "$NEW_ELECTING" -gt 0 ]; then
        break
    fi
    sleep 1
done

# Now SIGCONT ctrl-1 (it should lose since a new primary was elected)
kill -CONT "$PID" 2>/dev/null || true

# Collect final counts
sleep 5
POST_RDHUP_2=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-2.log" 2>/dev/null || echo 0)
POST_RDHUP_3=$(grep -c "ha_on_primary_ctrl_fd_rdhup" "$LOGDIR/ctrl-3.log" 2>/dev/null || echo 0)

NEW_RDHUP_2=$((POST_RDHUP_2 - PRE_RDHUP_2))
NEW_RDHUP_3=$((POST_RDHUP_3 - PRE_RDHUP_3))

# Each standby should show exactly 1 new RDHUP (not 0, not >1)
if [ "$NEW_RDHUP_2" -eq 1 ]; then
    pass "Scenario B: ctrl-2 fired exactly 1 RDHUP during long pause (Req 2.3)"
elif [ "$NEW_RDHUP_2" -eq 0 ]; then
    fail "Scenario B: ctrl-2 fired 0 RDHUP — liveness poll did not trigger failover"
else
    fail "Scenario B: ctrl-2 fired ${NEW_RDHUP_2} RDHUPs — duplicate failover loop (Req 5.4)"
fi

if [ "$NEW_RDHUP_3" -eq 1 ]; then
    pass "Scenario B: ctrl-3 fired exactly 1 RDHUP during long pause (Req 2.3)"
elif [ "$NEW_RDHUP_3" -eq 0 ]; then
    fail "Scenario B: ctrl-3 fired 0 RDHUP — liveness poll did not trigger failover"
else
    fail "Scenario B: ctrl-3 fired ${NEW_RDHUP_3} RDHUPs — duplicate failover loop (Req 5.4)"
fi

# One of ctrl-2 or ctrl-3 must have become PRIMARY
NEW_PRIMARY=""
if grep -q "CTRL_PRIMARY" "$LOGDIR/ctrl-2.log" 2>/dev/null; then
    NEW_PRIMARY="ctrl-2"
elif grep -q "CTRL_PRIMARY" "$LOGDIR/ctrl-3.log" 2>/dev/null; then
    NEW_PRIMARY="ctrl-3"
fi

if [ -n "$NEW_PRIMARY" ]; then
    pass "Scenario B: $NEW_PRIMARY promoted to CTRL_PRIMARY after failover"
else
    fail "Scenario B: no standby reached CTRL_PRIMARY — election did not complete"
fi

kill_cluster

# ── summary ───────────────────────────────────────────────────────────────────

echo ""
echo "========================="
echo "Results: ${PASS} passed, ${FAIL} failed"
echo "========================="

[ "$FAIL" -eq 0 ] && exit 0 || exit 1