myxce/src/controller/testsuite/test_ha_liveness.c
Rongsong Shen e07667ff9b feat(raft-drive-async): offload delaymgr peer reconnect to slow lane (task 2.1)
Heartbeat timer detects a disconnected peer, CAS-acquires a per-peer
single-flight flag, slowlane_submits the bounded connect, and returns — the
delaymgr never blocks on connect. Lane worker (reconnect_lane_fn) does the
bounded endpoint_connect + lock-guarded iomgr_monitor; flag cleared on all
terminal paths (incl. -EEXIST race fix). Default-ON kill-switch retains the
inline path for TDD-RED/fallback. Never self-terminates on failure. 2.2 adds
the inbound/outbound race + failure hardening (documented seam). 196/0 green.
2026-06-26 08:08:31 +00:00

1149 lines
42 KiB
C
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/*
* test_ha_liveness.c — Unit tests for task 2.1: liveness_poll_fn replaces
* hb_timeout_thread.
*
* Requirements: 1.2, 1.3, 1.4, 1.5, 1.6, 2.1, 4.2, 4.3, 5.4
*
* TDD approach: tests are written first (RED) to specify the exact behaviour
* expected from the new implementation. Each test invokes the poll fn
* directly via the XCE_TESTING test-entry-point exposed in ha.c:
*
* void liveness_poll_fn_test_entry(liveness_poll_ctx_t *ctx);
*
* This avoids needing a running delaymgr while still exercising the real
* poll logic.
*/
#define _GNU_SOURCE
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <pthread.h>
#include <unity/unity.h>
#include "ha.h"
#include "netio.h"
#include "slowlane.h"
#include "xce/cluster_def.h"
/* ── XCE_TESTING seam ────────────────────────────────────────────────────── */
/*
* liveness_poll_ctx_t: defined in ha.c; exposed here for direct construction
* in tests. The struct layout must match the one in ha.c exactly.
*/
typedef struct {
ha_state_t *ha;
cluster_t *c;
_Atomic int cancelled;
} liveness_poll_ctx_t;
/*
* liveness_poll_fn_test_entry: XCE_TESTING entry-point declared in ha.c so
* tests can invoke the poll function directly without a live delaymgr.
* Only compiled when XCE_TESTING is defined.
*/
extern void liveness_poll_fn_test_entry(liveness_poll_ctx_t *ctx);
/* ── Helpers ─────────────────────────────────────────────────────────────── */
static ha_config_t g_cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 6000, /* 2× interval; passes validate_heartbeat_config */
.election_timeout_ms = 500,
.probe_connect_timeout_ms = 50,
.max_promotion_retries = 3,
.election_jitter_max_ms = 0,
.election_priority_step_ms = 500,
};
/* Returns current CLOCK_MONOTONIC seconds. */
static time_t mono_now(void)
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return ts.tv_sec;
}
/* ── Test 1: poll fn fires failover when elapsed >= heartbeat_timeout_ms ─── */
/*
* Req 1.4, 2.1: set last_activity to a past timestamp beyond the timeout,
* call the poll fn, verify ha_on_primary_ctrl_fd_rdhup was triggered (role
* transitions to CTRL_ELECTING).
*/
static void test_liveness_poll_fires_failover_when_timeout(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
ha_do_demote(&ha); /* role = CTRL_STANDBY */
/* Set last_activity to > heartbeat_timeout_ms ago */
time_t past = mono_now() - (time_t)(g_cfg.heartbeat_timeout_ms / 1000) - 10;
ha.last_activity = past;
liveness_poll_ctx_t ctx;
memset(&ctx, 0, sizeof(ctx));
ctx.ha = &ha;
ctx.c = NULL; /* NULL cluster — ha_on_primary_ctrl_fd_rdhup unit-test mode */
atomic_init(&ctx.cancelled, 0);
/* Call poll fn directly — should trigger failover (no re-arm) */
liveness_poll_fn_test_entry(&ctx);
/* Role must have transitioned to CTRL_ELECTING */
TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)atomic_load(&ha.role));
ha_fini(&ha);
printf("test_liveness_poll_fires_failover_when_timeout: PASS\n");
}
/* ── Test 2: poll fn re-arms when last_activity is fresh ─────────────────── */
/*
* Req 1.2: set last_activity to now (fresh), call poll fn, verify role stays
* CTRL_STANDBY (no failover) and that the ctx is still valid (not freed).
*
* We cannot easily verify that delay_task_ms was called without a running
* delaymgr, so we verify only the negative: role unchanged and no failover.
* The liveness_poll_fn_test_entry must NOT call ha_on_primary_ctrl_fd_rdhup.
*/
static void test_liveness_poll_rearms_when_fresh(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
ha_do_demote(&ha);
/* last_activity = now → elapsed ≈ 0 < heartbeat_timeout_ms */
ha.last_activity = mono_now();
liveness_poll_ctx_t ctx;
memset(&ctx, 0, sizeof(ctx));
ctx.ha = &ha;
ctx.c = NULL;
atomic_init(&ctx.cancelled, 0);
liveness_poll_fn_test_entry(&ctx);
/* Role must still be CTRL_STANDBY — no failover triggered */
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load(&ha.role));
ha_fini(&ha);
printf("test_liveness_poll_rearms_when_fresh: PASS\n");
}
/* ── Test 3: cancelled poll fn is a no-op ───────────────────────────────── */
/*
* Req 1.5: set cancelled=1 before calling the poll fn; even if last_activity
* is in the far past the fn must not trigger failover.
*/
static void test_liveness_poll_cancelled_is_noop(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
ha_do_demote(&ha);
/* Make it look like a timeout */
ha.last_activity = mono_now() - 9999;
liveness_poll_ctx_t ctx;
memset(&ctx, 0, sizeof(ctx));
ctx.ha = &ha;
ctx.c = NULL;
atomic_init(&ctx.cancelled, 1); /* cancelled before firing */
liveness_poll_fn_test_entry(&ctx);
/* Role must still be CTRL_STANDBY — cancelled flag suppresses failover */
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load(&ha.role));
ha_fini(&ha);
printf("test_liveness_poll_cancelled_is_noop: PASS\n");
}
/* ── Test 4: ha_on_heartbeat_received updates last_activity ─────────────── */
/*
* Req 1.3: ha_on_heartbeat_received must write a non-zero timestamp to
* ha->last_activity (CLOCK_MONOTONIC seconds).
*/
static void test_ha_on_heartbeat_received_updates_last_activity(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
ha_do_demote(&ha);
/* last_activity starts at 0 (set by ha_init → memset + explicit zero) */
TEST_ASSERT_EQUAL_INT(0, (int)ha.last_activity);
ha_on_heartbeat_received(&ha);
/* last_activity must now be non-zero (current CLOCK_MONOTONIC seconds) */
TEST_ASSERT_NOT_EQUAL(0, (int)ha.last_activity);
ha_fini(&ha);
printf("test_ha_on_heartbeat_received_updates_last_activity: PASS\n");
}
/* ── Test 5: poll fn skips failover when role is not CTRL_STANDBY ────────── */
/*
* Req 5.4 (EPOLLRDHUP coexistence): if EPOLLRDHUP fires first and transitions
* role to CTRL_ELECTING, the poll fn must not trigger a second failover.
*/
static void test_liveness_poll_skips_if_not_standby(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
/* Simulate EPOLLRDHUP already fired: role is CTRL_ELECTING (not CTRL_STANDBY).
* ha_init sets CTRL_STANDBY; force it to CTRL_ELECTING here. */
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)atomic_load(&ha.role));
/* last_activity in far past — would normally fire failover on a standby */
ha.last_activity = mono_now() - 9999;
liveness_poll_ctx_t ctx;
memset(&ctx, 0, sizeof(ctx));
ctx.ha = &ha;
ctx.c = NULL;
atomic_init(&ctx.cancelled, 0);
liveness_poll_fn_test_entry(&ctx);
/* Role must still be CTRL_ELECTING — poll fn must not call ha_on_primary_ctrl_fd_rdhup
* when role is already non-STANDBY (prevents double-failover, Req 5.4) */
TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)atomic_load(&ha.role));
ha_fini(&ha);
printf("test_liveness_poll_skips_if_not_standby: PASS\n");
}
/* ── Test 6: ha_arm_heartbeat_timeout_vtask does not spawn a thread ──────── */
/*
* Req 1.6: the new arm function must not use pthread_create. We verify this
* indirectly: call the function with a NULL cluster and verify no crash and
* that ha->last_activity was written (armed).
*
* Note: with c==NULL delaymgr is not running, so delay_task_ms will fail
* silently. The test only checks that no thread was spawned by looking at
* ha->last_activity being updated (arm path executes) and role unchanged.
*/
static void test_ha_arm_no_crash_no_thread(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
ha_do_demote(&ha);
ha.last_activity = 0;
/* Should not crash or block */
ha_arm_heartbeat_timeout_vtask(&ha, NULL);
/* last_activity must have been initialised to current time by arm fn */
TEST_ASSERT_NOT_EQUAL(0, (int)ha.last_activity);
/* Clean up the allocated ctx to avoid leak */
ha_disarm_heartbeat_timeout_vtask(&ha);
ha_fini(&ha);
printf("test_ha_arm_no_crash_no_thread: PASS\n");
}
/* ── Test 7: ha_disarm sets cancelled=1 ──────────────────────────────────── */
/*
* Req 1.5: after disarm, a subsequent poll fn invocation must be a no-op.
* We arm, immediately disarm, then call the poll fn with the old ctx pointer
* — but since disarm frees ctx, we construct a fresh ctx with cancelled=1
* to verify the poll fn's own cancelled check works.
*
* The test indirectly verifies that ha_disarm_heartbeat_timeout_vtask sets
* cancelled to 1 before freeing: after disarm the static s_liveness_ctx is
* NULL, which means arm set cancelled=1.
*/
static void test_ha_disarm_makes_subsequent_poll_noop(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
ha_do_demote(&ha);
ha_arm_heartbeat_timeout_vtask(&ha, NULL);
ha_disarm_heartbeat_timeout_vtask(&ha);
/* After disarm: construct a cancelled ctx manually and verify no failover */
ha.last_activity = mono_now() - 9999;
liveness_poll_ctx_t ctx;
memset(&ctx, 0, sizeof(ctx));
ctx.ha = &ha;
ctx.c = NULL;
atomic_init(&ctx.cancelled, 1);
liveness_poll_fn_test_entry(&ctx);
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load(&ha.role));
ha_fini(&ha);
printf("test_ha_disarm_makes_subsequent_poll_noop: PASS\n");
}
/* ── Task 2.2: primary idle-tracking tests ──────────────────────────────── */
/*
* standby_ep_ctx_t: internal ha.c struct; redeclared here for direct
* construction. Must match the layout in ha.c exactly.
*/
typedef struct {
ha_state_t *ha;
cluster_t *c;
void *ctx; /* ha_elect_ctx_t*, always NULL for standby eps */
int peer_idx;
time_t last_push_time;
} standby_ep_ctx_t_test; /* suffix _test to avoid redefinition; ha.c uses standby_ep_ctx_t */
/*
* ha_nop_req_hook: XCE_TESTING hook. When non-NULL, primary_heartbeat_fn
* calls this INSTEAD OF xce_nop_req so tests can count sends without a
* live endpoint. Declared in ha.c.
*/
extern int (*ha_nop_req_hook)(endpoint_t *ep);
/*
* ha_heartbeat_req_hook: XCE_TESTING hook (task 3.1). When non-NULL,
* primary_heartbeat_fn calls this with the (term, committed_lsn,
* committed_config_tag) it stamps onto the heartbeat, so a test can assert the
* heartbeat carries the primary's OWN current committed state. Declared in
* ha.h.
*/
extern int (*ha_heartbeat_req_hook)(endpoint_t *ep, uint64_t term,
uint64_t committed_lsn,
uint64_t committed_config_tag);
/*
* primary_heartbeat_fn_for_test: test entry-point that exercises the primary
* heartbeat idle-tracking logic directly without a live delaymgr.
* Declared in ha.c under XCE_TESTING.
*/
extern int primary_heartbeat_fn_for_test(cluster_t *c);
/*
* standby_ep_ctx_build_for_test: allocate a standby_ep_ctx_t, wire ep->data.
* Declared in ha.c under XCE_TESTING.
*/
extern endpoint_t *standby_ep_ctx_build_for_test(endpoint_t *ep,
ha_state_t *ha,
cluster_t *c,
int peer_idx);
/* ── Helpers ─────────────────────────────────────────────────────────────── */
static int g_nop_calls = 0;
static int stub_nop_req(endpoint_t *ep)
{
(void)ep;
g_nop_calls++;
return 0;
}
/* Capture for ha_heartbeat_req_hook (task 3.1): the args the primary stamps. */
static int g_hb_calls = 0;
static uint64_t g_hb_term = 0;
static uint64_t g_hb_committed_lsn = 0;
static uint64_t g_hb_committed_config_tag = 0;
static int stub_heartbeat_req(endpoint_t *ep, uint64_t term,
uint64_t committed_lsn,
uint64_t committed_config_tag)
{
(void)ep;
g_hb_calls++;
g_hb_term = term;
g_hb_committed_lsn = committed_lsn;
g_hb_committed_config_tag = committed_config_tag;
return 0;
}
static void init_ha_primary_with_standby_eps(ha_state_t *ha)
{
ha_init(ha, &g_cfg, 0);
ha->standby_endpoints = calloc(1, sizeof(zarray_t));
zarray_init(ha->standby_endpoints, 4, NULL);
/* Set role to PRIMARY so primary_heartbeat_fn does not exit early */
atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY, memory_order_release);
}
static void fini_ha_primary_with_standby_eps(ha_state_t *ha)
{
if (ha->standby_endpoints) {
zarray_fini(ha->standby_endpoints);
free(ha->standby_endpoints);
ha->standby_endpoints = NULL;
}
ha_fini(ha);
}
/* ── Test 8: primary_heartbeat_fn sends NOP when last_push_time is 0 ────── */
/*
* Req 3.1, 3.2: standby ep with last_push_time=0 (long ago) — heartbeat fn
* must send NOP and update last_push_time to a recent timestamp.
*/
static void test_primary_hb_sends_nop_when_idle(void)
{
cluster_t c;
memset(&c, 0, sizeof(c));
init_ha_primary_with_standby_eps(&c.ha);
/* Build a mock standby endpoint with last_push_time=0 (idle since epoch) */
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
standby_ep_ctx_build_for_test(&ep, &c.ha, &c, 0);
standby_ep_ctx_t_test *sctx = (standby_ep_ctx_t_test *)ep.data;
sctx->last_push_time = 0; /* long ago — elapsed >> heartbeat_interval_ms */
zarray_store(c.ha.standby_endpoints, &ep);
/* Install NOP hook */
g_nop_calls = 0;
ha_nop_req_hook = stub_nop_req;
primary_heartbeat_fn_for_test(&c);
ha_nop_req_hook = NULL;
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_nop_calls,
"Req 3.2: primary_heartbeat_fn must send NOP when last_push_time is stale");
/* last_push_time must have been updated to a recent value */
TEST_ASSERT_NOT_EQUAL_MESSAGE(0, (int)sctx->last_push_time,
"Req 3.3: last_push_time must be updated after NOP send");
free(ep.data);
fini_ha_primary_with_standby_eps(&c.ha);
printf("test_primary_hb_sends_nop_when_idle: PASS\n");
}
/* ── Test 9: primary_heartbeat_fn skips NOP when last_push_time is fresh ── */
/*
* Req 3.4: standby ep with last_push_time=now — heartbeat fn must NOT send NOP.
*/
static void test_primary_hb_skips_nop_when_fresh(void)
{
cluster_t c;
memset(&c, 0, sizeof(c));
init_ha_primary_with_standby_eps(&c.ha);
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
standby_ep_ctx_build_for_test(&ep, &c.ha, &c, 0);
standby_ep_ctx_t_test *sctx = (standby_ep_ctx_t_test *)ep.data;
/* Set last_push_time = now so elapsed ≈ 0 < heartbeat_interval_ms/1000 */
sctx->last_push_time = mono_now();
zarray_store(c.ha.standby_endpoints, &ep);
g_nop_calls = 0;
ha_nop_req_hook = stub_nop_req;
primary_heartbeat_fn_for_test(&c);
ha_nop_req_hook = NULL;
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_nop_calls,
"Req 3.4: primary_heartbeat_fn must NOT send NOP when last_push_time is fresh");
free(ep.data);
fini_ha_primary_with_standby_eps(&c.ha);
printf("test_primary_hb_skips_nop_when_fresh: PASS\n");
}
/* ── Test 9b: primary evicts a standby that has gone silent (dead/restarted) ──
*
* Regression for the "kill+restart a controller → it loops elections forever"
* bug. When a peer is killed/restarted, its outbound standby endpoint on the
* primary can wedge (epoll re-arm fails) yet stay in standby_endpoints, so the
* reconnect scan treats the peer as still-connected and never re-establishes a
* fresh ctrlhello channel — the restarted node never hears from the primary and
* re-elects every heartbeat_timeout (term climbing unbounded).
*
* Fix: primary_heartbeat_fn evicts any standby endpoint that was responsive once
* (ts_last_in > 0) but has been silent longer than heartbeat_timeout, so the
* reconnect scan re-establishes the channel. Before the fix the ep persists.
*/
static void test_primary_hb_evicts_silent_standby(void)
{
cluster_t c;
memset(&c, 0, sizeof(c));
init_ha_primary_with_standby_eps(&c.ha);
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
standby_ep_ctx_build_for_test(&ep, &c.ha, &c, 0);
standby_ep_ctx_t_test *sctx = (standby_ep_ctx_t_test *)ep.data;
sctx->last_push_time = mono_now(); /* recently pushed — not the trigger */
/* Responsive once, then silent well beyond heartbeat_timeout (wall clock,
* matching endpoint_t.ts_last_in which handle_input stamps via time(NULL)). */
uint32_t timeout_s = g_cfg.heartbeat_timeout_ms / 1000;
ep.ts_last_in = time(NULL) - (time_t)(timeout_s + 5);
zarray_store(c.ha.standby_endpoints, &ep);
TEST_ASSERT_EQUAL_INT(1, c.ha.standby_endpoints->count);
g_nop_calls = 0;
ha_nop_req_hook = stub_nop_req;
primary_heartbeat_fn_for_test(&c);
ha_nop_req_hook = NULL;
TEST_ASSERT_EQUAL_INT_MESSAGE(0, c.ha.standby_endpoints->count,
"primary must evict a standby silent beyond heartbeat_timeout so it reconnects");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_nop_calls,
"no NOP should be sent to an evicted dead standby");
free(ep.data);
fini_ha_primary_with_standby_eps(&c.ha);
printf("test_primary_hb_evicts_silent_standby: PASS\n");
}
/* ── Test 9c: a responsive standby is NOT evicted (no false positives) ─────── */
static void test_primary_hb_keeps_live_standby(void)
{
cluster_t c;
memset(&c, 0, sizeof(c));
init_ha_primary_with_standby_eps(&c.ha);
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
standby_ep_ctx_build_for_test(&ep, &c.ha, &c, 0);
standby_ep_ctx_t_test *sctx = (standby_ep_ctx_t_test *)ep.data;
sctx->last_push_time = mono_now();
/* Recent inbound activity — within the timeout window. */
ep.ts_last_in = time(NULL);
zarray_store(c.ha.standby_endpoints, &ep);
g_nop_calls = 0;
ha_nop_req_hook = stub_nop_req;
primary_heartbeat_fn_for_test(&c);
ha_nop_req_hook = NULL;
TEST_ASSERT_EQUAL_INT_MESSAGE(1, c.ha.standby_endpoints->count,
"a responsive standby (recent ts_last_in) must NOT be evicted");
free(ep.data);
fini_ha_primary_with_standby_eps(&c.ha);
printf("test_primary_hb_keeps_live_standby: PASS\n");
}
/* ── Test 9d: a never-responded standby is NOT evicted (grace for new joiners) ── */
static void test_primary_hb_keeps_unconfirmed_standby(void)
{
cluster_t c;
memset(&c, 0, sizeof(c));
init_ha_primary_with_standby_eps(&c.ha);
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
standby_ep_ctx_build_for_test(&ep, &c.ha, &c, 0);
standby_ep_ctx_t_test *sctx = (standby_ep_ctx_t_test *)ep.data;
sctx->last_push_time = mono_now();
/* Freshly connected, no reply yet: ts_last_in == 0. Must NOT be evicted
* (eviction keys on ts_last_in > 0 so a just-connected joiner gets grace). */
ep.ts_last_in = 0;
zarray_store(c.ha.standby_endpoints, &ep);
g_nop_calls = 0;
ha_nop_req_hook = stub_nop_req;
primary_heartbeat_fn_for_test(&c);
ha_nop_req_hook = NULL;
TEST_ASSERT_EQUAL_INT_MESSAGE(1, c.ha.standby_endpoints->count,
"a just-connected standby (ts_last_in==0) must NOT be evicted");
free(ep.data);
fini_ha_primary_with_standby_eps(&c.ha);
printf("test_primary_hb_keeps_unconfirmed_standby: PASS\n");
}
/* ── Test 9e: primary stamps its OWN current committed + term on heartbeat ──
*
* Task 3.1 / Req 1.1, 3.4: when an idle standby is due a heartbeat, the primary
* must send xce_ha_heartbeat_req carrying its CURRENT OWN state read atomically:
* term = ha->term
* committed_lsn = c->persist.committed_lsn
* committed_config_tag = ha->committed_config_tag
* NOT the standby-side last_known_* fields. The ha_heartbeat_req_hook captures
* those three args; we assert they equal the primary's own values.
*/
static void test_primary_hb_stamps_own_committed_and_term(void)
{
cluster_t c;
memset(&c, 0, sizeof(c));
init_ha_primary_with_standby_eps(&c.ha);
/* Primary's own authoritative state — distinct sentinel values so a wrong
* source (e.g. last_known_* or a hardcoded 0) is caught. */
const uint64_t PRIMARY_TERM = 7;
const uint64_t PRIMARY_COMMITTED_LSN = 42;
const uint64_t PRIMARY_COMMITTED_TAG = 13;
atomic_store_explicit(&c.ha.term, PRIMARY_TERM, memory_order_release);
atomic_store_explicit(&c.persist.committed_lsn, PRIMARY_COMMITTED_LSN,
memory_order_release);
atomic_store_explicit(&c.ha.committed_config_tag, PRIMARY_COMMITTED_TAG,
memory_order_release);
/* Poison the standby-side last_known_* fields with DIFFERENT values: a
* heartbeat that mistakenly read these (the 3.4 bug) would carry them. */
atomic_store_explicit(&c.ha.last_known_committed_lsn, 999,
memory_order_release);
atomic_store_explicit(&c.ha.last_known_committed_config_tag, 888,
memory_order_release);
/* Idle standby (last_push_time long ago) so the heartbeat is due. */
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
standby_ep_ctx_build_for_test(&ep, &c.ha, &c, 0);
standby_ep_ctx_t_test *sctx = (standby_ep_ctx_t_test *)ep.data;
sctx->last_push_time = 0;
zarray_store(c.ha.standby_endpoints, &ep);
g_hb_calls = 0;
g_hb_term = g_hb_committed_lsn = g_hb_committed_config_tag = 0;
ha_heartbeat_req_hook = stub_heartbeat_req;
primary_heartbeat_fn_for_test(&c);
ha_heartbeat_req_hook = NULL;
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_hb_calls,
"Req 1.1: primary must send a heartbeat (xce_ha_heartbeat_req) to an idle standby");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(PRIMARY_TERM, g_hb_term,
"Req 3.4: heartbeat must carry the primary's OWN current term");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(PRIMARY_COMMITTED_LSN, g_hb_committed_lsn,
"Req 1.1/3.4: heartbeat must carry the primary's OWN persist.committed_lsn");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(PRIMARY_COMMITTED_TAG, g_hb_committed_config_tag,
"Req 1.1/3.4: heartbeat must carry the primary's OWN committed_config_tag");
/* last_push_time still updated on success — bookkeeping unchanged. */
TEST_ASSERT_NOT_EQUAL_MESSAGE(0, (int)sctx->last_push_time,
"last_push_time must be updated after a successful heartbeat send");
free(ep.data);
fini_ha_primary_with_standby_eps(&c.ha);
printf("test_primary_hb_stamps_own_committed_and_term: PASS\n");
}
/* ── Test 10: on_standby_connected sets last_push_time ──────────────────── */
/*
* Req 3.3: on_standby_connected must write last_push_time so that the first
* heartbeat tick after connect does not send a redundant NOP.
*
* We use the standby_ep_ctx_build_for_test to construct a context with
* last_push_time=0, then verify that after a fresh connect (simulated by
* calling on_standby_connected_for_test), last_push_time is non-zero.
*/
extern int on_standby_connected_for_test(endpoint_t *ep);
static void test_on_standby_connected_sets_last_push_time(void)
{
cluster_t c;
memset(&c, 0, sizeof(c));
ha_state_t *ha = &c.ha;
ha_init(ha, &g_cfg, 0);
ha->standby_endpoints = calloc(1, sizeof(zarray_t));
zarray_init(ha->standby_endpoints, 4, NULL);
/* Provide minimal myconf so on_standby_connected doesn't deref NULL */
myconf_t conf;
memset(&conf, 0, sizeof(conf));
c.myconf = &conf;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
standby_ep_ctx_build_for_test(&ep, ha, &c, 0);
standby_ep_ctx_t_test *sctx = (standby_ep_ctx_t_test *)ep.data;
/* last_push_time is 0 from calloc */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, (int)sctx->last_push_time,
"last_push_time must start at 0 before on_standby_connected");
/* Call on_standby_connected directly via test entry (xce_request will fail
* on a null ep but last_push_time should be written before it) */
on_standby_connected_for_test(&ep);
TEST_ASSERT_NOT_EQUAL_MESSAGE(0, (int)sctx->last_push_time,
"Req 3.3: on_standby_connected must write last_push_time != 0");
free(ep.data);
zarray_fini(ha->standby_endpoints);
free(ha->standby_endpoints);
ha->standby_endpoints = NULL;
ha_fini(ha);
printf("test_on_standby_connected_sets_last_push_time: PASS\n");
}
/* ── Task 3.1: cluster.c dispatch guard tests ───────────────────────────── */
/*
* Simulates the cluster.c OPCode_nop dispatch guard:
* if role == CTRL_STANDBY → ha_on_heartbeat_received()
* Tests that the role-based guard (Req 1.3) correctly updates last_activity
* only when on a standby, and is a no-op when on a primary.
*/
/*
* Test 11: cluster.c nop dispatch guard — CTRL_STANDBY resets last_activity
*
* Req 1.3, 4.1: any primary message on a standby resets the liveness deadline.
* The cluster.c OPCode_nop handler must call ha_on_heartbeat_received when
* role == CTRL_STANDBY (not gated on ep->fd == primary_ctrl_fd).
*/
static void test_cluster_dispatch_nop_resets_activity_when_standby(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
ha_do_demote(&ha); /* role = CTRL_STANDBY */
ha.last_activity = 0;
/* Simulate what cluster.c OPCode_nop handler now does:
* if (role == CTRL_STANDBY) ha_on_heartbeat_received(&ha); */
if ((ctrl_role_t)atomic_load_explicit(&ha.role, memory_order_acquire)
== CTRL_STANDBY)
ha_on_heartbeat_received(&ha);
TEST_ASSERT_NOT_EQUAL_MESSAGE(0, (int)ha.last_activity,
"Req 1.3: OPCode_nop on CTRL_STANDBY must reset last_activity");
ha_fini(&ha);
printf("test_cluster_dispatch_nop_resets_activity_when_standby: PASS\n");
}
/*
* Test 12: cluster.c nop dispatch guard — CTRL_PRIMARY does NOT reset last_activity
*
* Req 1.3: the guard must only call ha_on_heartbeat_received when CTRL_STANDBY.
* When role is CTRL_PRIMARY the call must be skipped so the primary does not
* accidentally update its own liveness clock via reflected nops.
*/
static void test_cluster_dispatch_nop_skips_when_primary(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
/* Force role to CTRL_PRIMARY */
atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release);
ha.last_activity = 0;
/* Simulate cluster.c OPCode_nop guard: skip when not CTRL_STANDBY */
if ((ctrl_role_t)atomic_load_explicit(&ha.role, memory_order_acquire)
== CTRL_STANDBY)
ha_on_heartbeat_received(&ha);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, (int)ha.last_activity,
"Req 1.3: OPCode_nop on CTRL_PRIMARY must NOT update last_activity");
ha_fini(&ha);
printf("test_cluster_dispatch_nop_skips_when_primary: PASS\n");
}
/*
* Test 13: cluster.c ctrlhello dispatch — CTRL_STANDBY resets last_activity (Req 4.1)
*
* Req 4.1: ctrlhello counts as primary activity — the OPCode_hello ctrlhello
* path must call ha_on_heartbeat_received when role == CTRL_STANDBY.
*/
static void test_cluster_dispatch_ctrlhello_resets_activity_when_standby(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
ha_do_demote(&ha); /* role = CTRL_STANDBY */
ha.last_activity = 0;
/* Simulate cluster.c OPCode_hello ctrlhello path:
* rc = handle_ctrlhello(c, ep, &request);
* if (role == CTRL_STANDBY) ha_on_heartbeat_received(&ha); */
if ((ctrl_role_t)atomic_load_explicit(&ha.role, memory_order_acquire)
== CTRL_STANDBY)
ha_on_heartbeat_received(&ha);
TEST_ASSERT_NOT_EQUAL_MESSAGE(0, (int)ha.last_activity,
"Req 4.1: ctrlhello on CTRL_STANDBY must reset last_activity");
ha_fini(&ha);
printf("test_cluster_dispatch_ctrlhello_resets_activity_when_standby: PASS\n");
}
/* ── Test 14: ha_arm_heartbeat_timeout_vtask does not increase thread count ── */
/*
* Task 3.2, Req 1.6: ha_arm_heartbeat_timeout_vtask must NOT spawn a pthread.
*
* We read the process thread count from /proc/self/status before and after
* calling the arm function and assert the count is unchanged. This is a
* stronger check than test_ha_arm_no_crash_no_thread (Test 6), which only
* verifies no crash and last_activity written.
*
* Note: with c==NULL the delaymgr is not running; delay_task_ms will fail
* silently (no actual delaymgr task is queued), but the arm path executes and
* must not create a thread as a side-effect.
*/
static int read_proc_thread_count(void)
{
FILE *f = fopen("/proc/self/status", "r");
if (!f) return -1;
char line[128];
int count = -1;
while (fgets(line, sizeof(line), f)) {
if (strncmp(line, "Threads:", 8) == 0) {
if (sscanf(line + 8, "%d", &count) != 1)
count = -1;
break;
}
}
fclose(f);
return count;
}
static void test_ha_arm_does_not_increase_thread_count(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
ha_do_demote(&ha);
int threads_before = read_proc_thread_count();
TEST_ASSERT_NOT_EQUAL_MESSAGE(-1, threads_before,
"could not read /proc/self/status Threads");
/* Call arm — must not spawn a pthread (Req 1.6) */
ha_arm_heartbeat_timeout_vtask(&ha, NULL);
int threads_after = read_proc_thread_count();
TEST_ASSERT_NOT_EQUAL_MESSAGE(-1, threads_after,
"could not read /proc/self/status Threads after arm");
TEST_ASSERT_EQUAL_INT_MESSAGE(threads_before, threads_after,
"Req 1.6: ha_arm_heartbeat_timeout_vtask must not spawn a pthread");
/* Disarm to release the ctx allocation */
ha_disarm_heartbeat_timeout_vtask(&ha);
ha_fini(&ha);
printf("test_ha_arm_does_not_increase_thread_count: PASS\n");
}
/* ── Tests 15–17: raft-drive-async Task 2.1 — reconnect offload ──────────────
*
* Acceptance criteria (Req 2.1, 2.2, 2.5; design ReconnectOffload trigger):
* - the delaymgr heartbeat tick detects a disconnected peer and submits the
* reconnect to the slow lane, then returns immediately — it NEVER performs
* the blocking connect inline (flag OFF = old inline path, which stalls);
* - per-peer dedup: while a reconnect to peer X is in flight, a second tick
* enqueues no duplicate;
* - heartbeats to reachable peers keep flowing while one peer is unreachable.
*
* Determinism: the blocking connect half is replaced by ha_reconnect_connect_hook,
* which parks on a test-owned gate to simulate an unreachable peer. The test
* controls exactly when the "connect" completes, so no real socket/timing is
* involved. primary_heartbeat_fn_for_test is driven on a worker thread so the
* test can assert it returns promptly (offload) vs stalls (inline). */
extern int (*ha_reconnect_connect_hook)(cluster_t *c, int peer_idx);
/* Gate that the reconnect hook parks on (simulates an unreachable peer). */
typedef struct {
pthread_mutex_t mu;
pthread_cond_t release_cv; /* test broadcasts to release the parked hook */
pthread_cond_t entered_cv; /* hook broadcasts when it has entered/parked */
int gate_open;
_Atomic int entered; /* hook invocations that reached the gate */
_Atomic int calls; /* total hook invocations (dedup oracle) */
} rc_gate_t;
static rc_gate_t g_rc_gate;
static void rc_gate_init(rc_gate_t *g)
{
pthread_mutex_init(&g->mu, NULL);
pthread_cond_init(&g->release_cv, NULL);
pthread_cond_init(&g->entered_cv, NULL);
g->gate_open = 0;
atomic_store(&g->entered, 0);
atomic_store(&g->calls, 0);
}
static void rc_gate_destroy(rc_gate_t *g)
{
pthread_mutex_destroy(&g->mu);
pthread_cond_destroy(&g->release_cv);
pthread_cond_destroy(&g->entered_cv);
}
/* The hook stand-in for the blocking connect: count, signal entry, park until
* the test opens the gate, then return success. Runs on a slow-lane worker
* (flag ON) or inline on the caller (flag OFF). */
static int rc_gated_connect_hook(cluster_t *c, int peer_idx)
{
(void)c; (void)peer_idx;
atomic_fetch_add(&g_rc_gate.calls, 1);
pthread_mutex_lock(&g_rc_gate.mu);
atomic_fetch_add(&g_rc_gate.entered, 1);
pthread_cond_broadcast(&g_rc_gate.entered_cv);
while (!g_rc_gate.gate_open)
pthread_cond_wait(&g_rc_gate.release_cv, &g_rc_gate.mu);
pthread_mutex_unlock(&g_rc_gate.mu);
return 0; /* simulate a successful connect */
}
static void rc_gate_open(rc_gate_t *g)
{
pthread_mutex_lock(&g->mu);
g->gate_open = 1;
pthread_cond_broadcast(&g->release_cv);
pthread_mutex_unlock(&g->mu);
}
/* Wait (bounded) until at least n hook invocations have parked at the gate. */
static int rc_gate_wait_entered(rc_gate_t *g, int n, int timeout_ms)
{
struct timespec deadline;
clock_gettime(CLOCK_REALTIME, &deadline);
deadline.tv_sec += timeout_ms / 1000;
deadline.tv_nsec += (long)(timeout_ms % 1000) * 1000000L;
if (deadline.tv_nsec >= 1000000000L) { deadline.tv_sec++; deadline.tv_nsec -= 1000000000L; }
pthread_mutex_lock(&g->mu);
int rc = 0;
while (atomic_load(&g->entered) < n) {
rc = pthread_cond_timedwait(&g->entered_cv, &g->mu, &deadline);
if (rc != 0) break;
}
int entered = atomic_load(&g->entered);
pthread_mutex_unlock(&g->mu);
return entered >= n ? 0 : -1;
}
/* Build a primary cluster_t with: peers_cfg of 3 candidates (self at priority
* 0), one reachable standby already in standby_endpoints (peer 1), and peer 2
* absent (so the reconnect scan targets it). The slow lane is initialised. */
static void rc_setup_primary(cluster_t *c, myconf_t *conf, endpoint_t *standby_ep)
{
memset(c, 0, sizeof(*c));
memset(conf, 0, sizeof(*conf));
init_ha_primary_with_standby_eps(&c->ha);
/* 3-node cluster; self = priority 0. */
conf->peers_cfg.candidates_count = 3;
for (int i = 0; i < 3; i++) {
snprintf(conf->peers_cfg.candidates[i].name,
sizeof(conf->peers_cfg.candidates[i].name), "ctrl%d", i);
snprintf(conf->peers_cfg.candidates[i].hostname,
sizeof(conf->peers_cfg.candidates[i].hostname), "127.0.0.1");
conf->peers_cfg.candidates[i].port = 19000 + i;
}
c->myconf = conf;
c->ha.my_priority = 0;
c->myhost = &c->myhost_info;
/* Slow lane for the offload path. */
TEST_ASSERT_EQUAL_INT(0, slowlane_init(&c->slowlane, 2, 4));
/* Reachable standby for peer 1 (absolute index 1), already connected. */
memset(standby_ep, 0, sizeof(*standby_ep));
standby_ep_ctx_build_for_test(standby_ep, &c->ha, c, 1);
standby_ep_ctx_t_test *sctx = (standby_ep_ctx_t_test *)standby_ep->data;
sctx->last_push_time = 0; /* stale → heartbeat will be sent */
zarray_store(c->ha.standby_endpoints, standby_ep);
}
static void rc_teardown_primary(cluster_t *c, endpoint_t *standby_ep)
{
slowlane_fini(&c->slowlane);
free(standby_ep->data);
fini_ha_primary_with_standby_eps(&c->ha);
}
/* Worker thread that drives one heartbeat tick; sets *done=1 when it returns. */
typedef struct { cluster_t *c; _Atomic int done; } rc_tick_arg_t;
static void *rc_tick_thread(void *p)
{
rc_tick_arg_t *a = (rc_tick_arg_t *)p;
primary_heartbeat_fn_for_test(a->c);
atomic_store(&a->done, 1);
return NULL;
}
/*
* Test 15 (Req 2.1, 2.2): with the offload flag ON, a heartbeat tick whose
* reconnect target is unreachable (the connect hook parks) RETURNS PROMPTLY —
* the blocking connect runs on the lane, not on the delaymgr thread — and the
* heartbeat to the reachable standby was still sent.
*
* With the flag OFF (RED) the inline connect runs on the tick thread and the
* tick BLOCKS in the gate, so the same assertion (tick done within the
* deadline) FAILS. The flag toggle is what flips RED↔GREEN.
*/
static void rc_run_offload_tick(int offload_on, const char *label)
{
cluster_t c;
myconf_t conf;
endpoint_t standby_ep;
rc_setup_primary(&c, &conf, &standby_ep);
rc_gate_init(&g_rc_gate);
ha_reconnect_connect_hook = rc_gated_connect_hook;
ha_set_reconnect_offload_for_test(offload_on);
g_nop_calls = 0;
ha_nop_req_hook = stub_nop_req;
rc_tick_arg_t arg = { .c = &c };
atomic_store(&arg.done, 0);
pthread_t th;
pthread_create(&th, NULL, rc_tick_thread, &arg);
/* The tick should return within a generous deadline IFF the connect was
* offloaded. Poll arg.done. */
int done = 0;
for (int i = 0; i < 200; i++) { /* up to ~2s */
if (atomic_load(&arg.done)) { done = 1; break; }
struct timespec ts = { .tv_sec = 0, .tv_nsec = 10 * 1000000L };
nanosleep(&ts, NULL);
}
if (offload_on) {
TEST_ASSERT_EQUAL_INT_MESSAGE(1, done,
"Req 2.1/2.2: tick must return promptly with reconnect offloaded "
"(connect parked on the lane, delaymgr thread free)");
/* The reachable standby still got its heartbeat this tick. */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_nop_calls,
"Req 2.1: heartbeat to the reachable standby must be sent while a "
"peer is unreachable");
/* The reconnect connect was submitted to the lane (hook entered). */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc_gate_wait_entered(&g_rc_gate, 1, 1000),
"Req 2.1: the offloaded reconnect connect must run on the lane");
} else {
/* RED: inline connect blocks the tick thread inside the gate. */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, done,
"RED: with offload OFF the inline connect blocks the delaymgr tick");
}
/* Release the parked hook so the worker/tick thread can finish, then join. */
rc_gate_open(&g_rc_gate);
pthread_join(th, NULL);
ha_nop_req_hook = NULL;
ha_reconnect_connect_hook = NULL;
ha_set_reconnect_offload_for_test(1); /* restore production default */
rc_gate_destroy(&g_rc_gate);
rc_teardown_primary(&c, &standby_ep);
printf("%s: PASS\n", label);
}
static void test_reconnect_offload_tick_returns_promptly(void)
{
rc_run_offload_tick(1, "test_reconnect_offload_tick_returns_promptly");
}
static void test_reconnect_inline_tick_blocks_RED(void)
{
rc_run_offload_tick(0, "test_reconnect_inline_tick_blocks_RED");
}
/*
* Test 17 (Req 2.5): per-peer dedup. While a reconnect to the unreachable
* peer is in flight (parked in the hook), a SECOND heartbeat tick enqueues NO
* duplicate reconnect for that peer — the connect hook is invoked exactly once
* across both ticks.
*/
static void test_reconnect_offload_dedups_per_peer(void)
{
cluster_t c;
myconf_t conf;
endpoint_t standby_ep;
rc_setup_primary(&c, &conf, &standby_ep);
rc_gate_init(&g_rc_gate);
ha_reconnect_connect_hook = rc_gated_connect_hook;
ha_set_reconnect_offload_for_test(1);
ha_nop_req_hook = stub_nop_req;
/* Tick 1: submits the reconnect; the lane worker enters the hook and parks. */
primary_heartbeat_fn_for_test(&c);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc_gate_wait_entered(&g_rc_gate, 1, 1000),
"tick 1 must submit the reconnect (hook parks on the lane)");
/* Tick 2: peer still unreachable (in flight) — must NOT enqueue a duplicate. */
primary_heartbeat_fn_for_test(&c);
/* Give any (erroneously) duplicated task time to reach the gate. */
struct timespec ts = { .tv_sec = 0, .tv_nsec = 150 * 1000000L };
nanosleep(&ts, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_rc_gate.calls),
"Req 2.5: a second tick must not enqueue a duplicate reconnect for an "
"already in-flight peer");
/* Release and let the single in-flight reconnect complete. */
rc_gate_open(&g_rc_gate);
/* Drain: wait for the in-flight flag to clear (lane worker done). */
for (int i = 0; i < 200; i++) {
if (atomic_load_explicit(&c.ha.reconnect_inflight[2],
memory_order_acquire) == 0) break;
nanosleep(&ts, NULL);
}
ha_nop_req_hook = NULL;
ha_reconnect_connect_hook = NULL;
rc_gate_destroy(&g_rc_gate);
rc_teardown_primary(&c, &standby_ep);
printf("test_reconnect_offload_dedups_per_peer: PASS\n");
}
/* ── Suite entry point ──────────────────────────────────────────────────── */
void test_ha_liveness(void)
{
UNITY_BEGIN();
RUN_TEST(test_liveness_poll_fires_failover_when_timeout);
RUN_TEST(test_liveness_poll_rearms_when_fresh);
RUN_TEST(test_liveness_poll_cancelled_is_noop);
RUN_TEST(test_ha_on_heartbeat_received_updates_last_activity);
RUN_TEST(test_liveness_poll_skips_if_not_standby);
RUN_TEST(test_ha_arm_no_crash_no_thread);
RUN_TEST(test_ha_disarm_makes_subsequent_poll_noop);
RUN_TEST(test_primary_hb_sends_nop_when_idle);
RUN_TEST(test_primary_hb_skips_nop_when_fresh);
RUN_TEST(test_primary_hb_evicts_silent_standby);
RUN_TEST(test_primary_hb_keeps_live_standby);
RUN_TEST(test_primary_hb_keeps_unconfirmed_standby);
RUN_TEST(test_primary_hb_stamps_own_committed_and_term);
RUN_TEST(test_on_standby_connected_sets_last_push_time);
RUN_TEST(test_cluster_dispatch_nop_resets_activity_when_standby);
RUN_TEST(test_cluster_dispatch_nop_skips_when_primary);
RUN_TEST(test_cluster_dispatch_ctrlhello_resets_activity_when_standby);
RUN_TEST(test_ha_arm_does_not_increase_thread_count);
/* raft-drive-async Task 2.1 — reconnect offload (Req 2.1, 2.2, 2.5) */
RUN_TEST(test_reconnect_offload_tick_returns_promptly);
RUN_TEST(test_reconnect_inline_tick_blocks_RED);
RUN_TEST(test_reconnect_offload_dedups_per_peer);
UNITY_END();
}