myxce/src/controller/testsuite/test_ha_liveness.c
Rongsong Shen d937c1c1bb feat(ha-heartbeat-liveness): role-based liveness reset in opcode dispatch (task 3.1)
Replace fd-based guard in OPCode_nop with role==CTRL_STANDBY check.
Add ha_on_heartbeat_received call to OPCode_hello ctrlhello path when standby.
Satisfies Req 1.3 (any message resets deadline) and Req 4.1 (ctrlhello counts).
3 new dispatch guard tests added to test_ha_liveness.c.
2026-06-01 15:31:15 +00:00

598 lines
21 KiB
C
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/*
* test_ha_liveness.c — Unit tests for task 2.1: liveness_poll_fn replaces
* hb_timeout_thread.
*
* Requirements: 1.2, 1.3, 1.4, 1.5, 1.6, 2.1, 4.2, 4.3, 5.4
*
* TDD approach: tests are written first (RED) to specify the exact behaviour
* expected from the new implementation. Each test invokes the poll fn
* directly via the XCE_TESTING test-entry-point exposed in ha.c:
*
* void liveness_poll_fn_test_entry(liveness_poll_ctx_t *ctx);
*
* This avoids needing a running delaymgr while still exercising the real
* poll logic.
*/
#define _GNU_SOURCE
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <unity/unity.h>
#include "ha.h"
#include "netio.h"
#include "xce/cluster_def.h"
/* ── XCE_TESTING seam ────────────────────────────────────────────────────── */
/*
* liveness_poll_ctx_t: defined in ha.c; exposed here for direct construction
* in tests. The struct layout must match the one in ha.c exactly.
*/
typedef struct {
ha_state_t *ha;
cluster_t *c;
_Atomic int cancelled;
} liveness_poll_ctx_t;
/*
* liveness_poll_fn_test_entry: XCE_TESTING entry-point declared in ha.c so
* tests can invoke the poll function directly without a live delaymgr.
* Only compiled when XCE_TESTING is defined.
*/
extern void liveness_poll_fn_test_entry(liveness_poll_ctx_t *ctx);
/* ── Helpers ─────────────────────────────────────────────────────────────── */
static ha_config_t g_cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 6000, /* 2× interval; passes validate_heartbeat_config */
.election_timeout_ms = 500,
.probe_connect_timeout_ms = 50,
.max_promotion_retries = 3,
.election_jitter_max_ms = 0,
.election_priority_step_ms = 500,
};
/* Returns current CLOCK_MONOTONIC seconds. */
static time_t mono_now(void)
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return ts.tv_sec;
}
/* ── Test 1: poll fn fires failover when elapsed >= heartbeat_timeout_ms ─── */
/*
* Req 1.4, 2.1: set last_activity to a past timestamp beyond the timeout,
* call the poll fn, verify ha_on_primary_ctrl_fd_rdhup was triggered (role
* transitions to CTRL_ELECTING).
*/
static void test_liveness_poll_fires_failover_when_timeout(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
ha_do_demote(&ha); /* role = CTRL_STANDBY */
/* Set last_activity to > heartbeat_timeout_ms ago */
time_t past = mono_now() - (time_t)(g_cfg.heartbeat_timeout_ms / 1000) - 10;
ha.last_activity = past;
liveness_poll_ctx_t ctx;
memset(&ctx, 0, sizeof(ctx));
ctx.ha = &ha;
ctx.c = NULL; /* NULL cluster — ha_on_primary_ctrl_fd_rdhup unit-test mode */
atomic_init(&ctx.cancelled, 0);
/* Call poll fn directly — should trigger failover (no re-arm) */
liveness_poll_fn_test_entry(&ctx);
/* Role must have transitioned to CTRL_ELECTING */
TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)atomic_load(&ha.role));
ha_fini(&ha);
printf("test_liveness_poll_fires_failover_when_timeout: PASS\n");
}
/* ── Test 2: poll fn re-arms when last_activity is fresh ─────────────────── */
/*
* Req 1.2: set last_activity to now (fresh), call poll fn, verify role stays
* CTRL_STANDBY (no failover) and that the ctx is still valid (not freed).
*
* We cannot easily verify that delay_task_ms was called without a running
* delaymgr, so we verify only the negative: role unchanged and no failover.
* The liveness_poll_fn_test_entry must NOT call ha_on_primary_ctrl_fd_rdhup.
*/
static void test_liveness_poll_rearms_when_fresh(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
ha_do_demote(&ha);
/* last_activity = now → elapsed ≈ 0 < heartbeat_timeout_ms */
ha.last_activity = mono_now();
liveness_poll_ctx_t ctx;
memset(&ctx, 0, sizeof(ctx));
ctx.ha = &ha;
ctx.c = NULL;
atomic_init(&ctx.cancelled, 0);
liveness_poll_fn_test_entry(&ctx);
/* Role must still be CTRL_STANDBY — no failover triggered */
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load(&ha.role));
ha_fini(&ha);
printf("test_liveness_poll_rearms_when_fresh: PASS\n");
}
/* ── Test 3: cancelled poll fn is a no-op ───────────────────────────────── */
/*
* Req 1.5: set cancelled=1 before calling the poll fn; even if last_activity
* is in the far past the fn must not trigger failover.
*/
static void test_liveness_poll_cancelled_is_noop(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
ha_do_demote(&ha);
/* Make it look like a timeout */
ha.last_activity = mono_now() - 9999;
liveness_poll_ctx_t ctx;
memset(&ctx, 0, sizeof(ctx));
ctx.ha = &ha;
ctx.c = NULL;
atomic_init(&ctx.cancelled, 1); /* cancelled before firing */
liveness_poll_fn_test_entry(&ctx);
/* Role must still be CTRL_STANDBY — cancelled flag suppresses failover */
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load(&ha.role));
ha_fini(&ha);
printf("test_liveness_poll_cancelled_is_noop: PASS\n");
}
/* ── Test 4: ha_on_heartbeat_received updates last_activity ─────────────── */
/*
* Req 1.3: ha_on_heartbeat_received must write a non-zero timestamp to
* ha->last_activity (CLOCK_MONOTONIC seconds).
*/
static void test_ha_on_heartbeat_received_updates_last_activity(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
ha_do_demote(&ha);
/* last_activity starts at 0 (set by ha_init → memset + explicit zero) */
TEST_ASSERT_EQUAL_INT(0, (int)ha.last_activity);
ha_on_heartbeat_received(&ha);
/* last_activity must now be non-zero (current CLOCK_MONOTONIC seconds) */
TEST_ASSERT_NOT_EQUAL(0, (int)ha.last_activity);
ha_fini(&ha);
printf("test_ha_on_heartbeat_received_updates_last_activity: PASS\n");
}
/* ── Test 5: poll fn skips failover when role is not CTRL_STANDBY ────────── */
/*
* Req 5.4 (EPOLLRDHUP coexistence): if EPOLLRDHUP fires first and transitions
* role to CTRL_ELECTING, the poll fn must not trigger a second failover.
*/
static void test_liveness_poll_skips_if_not_standby(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
/* Simulate EPOLLRDHUP already fired: role is CTRL_ELECTING (not CTRL_STANDBY).
* ha_init sets CTRL_STANDBY; force it to CTRL_ELECTING here. */
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)atomic_load(&ha.role));
/* last_activity in far past — would normally fire failover on a standby */
ha.last_activity = mono_now() - 9999;
liveness_poll_ctx_t ctx;
memset(&ctx, 0, sizeof(ctx));
ctx.ha = &ha;
ctx.c = NULL;
atomic_init(&ctx.cancelled, 0);
liveness_poll_fn_test_entry(&ctx);
/* Role must still be CTRL_ELECTING — poll fn must not call ha_on_primary_ctrl_fd_rdhup
* when role is already non-STANDBY (prevents double-failover, Req 5.4) */
TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)atomic_load(&ha.role));
ha_fini(&ha);
printf("test_liveness_poll_skips_if_not_standby: PASS\n");
}
/* ── Test 6: ha_arm_heartbeat_timeout_vtask does not spawn a thread ──────── */
/*
* Req 1.6: the new arm function must not use pthread_create. We verify this
* indirectly: call the function with a NULL cluster and verify no crash and
* that ha->last_activity was written (armed).
*
* Note: with c==NULL delaymgr is not running, so delay_task_ms will fail
* silently. The test only checks that no thread was spawned by looking at
* ha->last_activity being updated (arm path executes) and role unchanged.
*/
static void test_ha_arm_no_crash_no_thread(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
ha_do_demote(&ha);
ha.last_activity = 0;
/* Should not crash or block */
ha_arm_heartbeat_timeout_vtask(&ha, NULL);
/* last_activity must have been initialised to current time by arm fn */
TEST_ASSERT_NOT_EQUAL(0, (int)ha.last_activity);
/* Clean up the allocated ctx to avoid leak */
ha_disarm_heartbeat_timeout_vtask(&ha);
ha_fini(&ha);
printf("test_ha_arm_no_crash_no_thread: PASS\n");
}
/* ── Test 7: ha_disarm sets cancelled=1 ──────────────────────────────────── */
/*
* Req 1.5: after disarm, a subsequent poll fn invocation must be a no-op.
* We arm, immediately disarm, then call the poll fn with the old ctx pointer
* — but since disarm frees ctx, we construct a fresh ctx with cancelled=1
* to verify the poll fn's own cancelled check works.
*
* The test indirectly verifies that ha_disarm_heartbeat_timeout_vtask sets
* cancelled to 1 before freeing: after disarm the static s_liveness_ctx is
* NULL, which means arm set cancelled=1.
*/
static void test_ha_disarm_makes_subsequent_poll_noop(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
ha_do_demote(&ha);
ha_arm_heartbeat_timeout_vtask(&ha, NULL);
ha_disarm_heartbeat_timeout_vtask(&ha);
/* After disarm: construct a cancelled ctx manually and verify no failover */
ha.last_activity = mono_now() - 9999;
liveness_poll_ctx_t ctx;
memset(&ctx, 0, sizeof(ctx));
ctx.ha = &ha;
ctx.c = NULL;
atomic_init(&ctx.cancelled, 1);
liveness_poll_fn_test_entry(&ctx);
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load(&ha.role));
ha_fini(&ha);
printf("test_ha_disarm_makes_subsequent_poll_noop: PASS\n");
}
/* ── Task 2.2: primary idle-tracking tests ──────────────────────────────── */
/*
* standby_ep_ctx_t: internal ha.c struct; redeclared here for direct
* construction. Must match the layout in ha.c exactly.
*/
typedef struct {
ha_state_t *ha;
cluster_t *c;
void *ctx; /* ha_elect_ctx_t*, always NULL for standby eps */
int peer_idx;
time_t last_push_time;
} standby_ep_ctx_t_test; /* suffix _test to avoid redefinition; ha.c uses standby_ep_ctx_t */
/*
* ha_nop_req_hook: XCE_TESTING hook. When non-NULL, primary_heartbeat_fn
* calls this INSTEAD OF xce_nop_req so tests can count sends without a
* live endpoint. Declared in ha.c.
*/
extern int (*ha_nop_req_hook)(endpoint_t *ep);
/*
* primary_heartbeat_fn_for_test: test entry-point that exercises the primary
* heartbeat idle-tracking logic directly without a live delaymgr.
* Declared in ha.c under XCE_TESTING.
*/
extern int primary_heartbeat_fn_for_test(cluster_t *c);
/*
* standby_ep_ctx_build_for_test: allocate a standby_ep_ctx_t, wire ep->data.
* Declared in ha.c under XCE_TESTING.
*/
extern endpoint_t *standby_ep_ctx_build_for_test(endpoint_t *ep,
ha_state_t *ha,
cluster_t *c,
int peer_idx);
/* ── Helpers ─────────────────────────────────────────────────────────────── */
static int g_nop_calls = 0;
static int stub_nop_req(endpoint_t *ep)
{
(void)ep;
g_nop_calls++;
return 0;
}
static void init_ha_primary_with_standby_eps(ha_state_t *ha)
{
ha_init(ha, &g_cfg, 0);
ha->standby_endpoints = calloc(1, sizeof(zarray_t));
zarray_init(ha->standby_endpoints, 4, NULL);
/* Set role to PRIMARY so primary_heartbeat_fn does not exit early */
atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY, memory_order_release);
}
static void fini_ha_primary_with_standby_eps(ha_state_t *ha)
{
if (ha->standby_endpoints) {
zarray_fini(ha->standby_endpoints);
free(ha->standby_endpoints);
ha->standby_endpoints = NULL;
}
ha_fini(ha);
}
/* ── Test 8: primary_heartbeat_fn sends NOP when last_push_time is 0 ────── */
/*
* Req 3.1, 3.2: standby ep with last_push_time=0 (long ago) — heartbeat fn
* must send NOP and update last_push_time to a recent timestamp.
*/
static void test_primary_hb_sends_nop_when_idle(void)
{
cluster_t c;
memset(&c, 0, sizeof(c));
init_ha_primary_with_standby_eps(&c.ha);
/* Build a mock standby endpoint with last_push_time=0 (idle since epoch) */
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
standby_ep_ctx_build_for_test(&ep, &c.ha, &c, 0);
standby_ep_ctx_t_test *sctx = (standby_ep_ctx_t_test *)ep.data;
sctx->last_push_time = 0; /* long ago — elapsed >> heartbeat_interval_ms */
zarray_store(c.ha.standby_endpoints, &ep);
/* Install NOP hook */
g_nop_calls = 0;
ha_nop_req_hook = stub_nop_req;
primary_heartbeat_fn_for_test(&c);
ha_nop_req_hook = NULL;
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_nop_calls,
"Req 3.2: primary_heartbeat_fn must send NOP when last_push_time is stale");
/* last_push_time must have been updated to a recent value */
TEST_ASSERT_NOT_EQUAL_MESSAGE(0, (int)sctx->last_push_time,
"Req 3.3: last_push_time must be updated after NOP send");
free(ep.data);
fini_ha_primary_with_standby_eps(&c.ha);
printf("test_primary_hb_sends_nop_when_idle: PASS\n");
}
/* ── Test 9: primary_heartbeat_fn skips NOP when last_push_time is fresh ── */
/*
* Req 3.4: standby ep with last_push_time=now — heartbeat fn must NOT send NOP.
*/
static void test_primary_hb_skips_nop_when_fresh(void)
{
cluster_t c;
memset(&c, 0, sizeof(c));
init_ha_primary_with_standby_eps(&c.ha);
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
standby_ep_ctx_build_for_test(&ep, &c.ha, &c, 0);
standby_ep_ctx_t_test *sctx = (standby_ep_ctx_t_test *)ep.data;
/* Set last_push_time = now so elapsed ≈ 0 < heartbeat_interval_ms/1000 */
sctx->last_push_time = mono_now();
zarray_store(c.ha.standby_endpoints, &ep);
g_nop_calls = 0;
ha_nop_req_hook = stub_nop_req;
primary_heartbeat_fn_for_test(&c);
ha_nop_req_hook = NULL;
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_nop_calls,
"Req 3.4: primary_heartbeat_fn must NOT send NOP when last_push_time is fresh");
free(ep.data);
fini_ha_primary_with_standby_eps(&c.ha);
printf("test_primary_hb_skips_nop_when_fresh: PASS\n");
}
/* ── Test 10: on_standby_connected sets last_push_time ──────────────────── */
/*
* Req 3.3: on_standby_connected must write last_push_time so that the first
* heartbeat tick after connect does not send a redundant NOP.
*
* We use the standby_ep_ctx_build_for_test to construct a context with
* last_push_time=0, then verify that after a fresh connect (simulated by
* calling on_standby_connected_for_test), last_push_time is non-zero.
*/
extern int on_standby_connected_for_test(endpoint_t *ep);
static void test_on_standby_connected_sets_last_push_time(void)
{
cluster_t c;
memset(&c, 0, sizeof(c));
ha_state_t *ha = &c.ha;
ha_init(ha, &g_cfg, 0);
ha->standby_endpoints = calloc(1, sizeof(zarray_t));
zarray_init(ha->standby_endpoints, 4, NULL);
/* Provide minimal myconf so on_standby_connected doesn't deref NULL */
myconf_t conf;
memset(&conf, 0, sizeof(conf));
c.myconf = &conf;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
standby_ep_ctx_build_for_test(&ep, ha, &c, 0);
standby_ep_ctx_t_test *sctx = (standby_ep_ctx_t_test *)ep.data;
/* last_push_time is 0 from calloc */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, (int)sctx->last_push_time,
"last_push_time must start at 0 before on_standby_connected");
/* Call on_standby_connected directly via test entry (xce_request will fail
* on a null ep but last_push_time should be written before it) */
on_standby_connected_for_test(&ep);
TEST_ASSERT_NOT_EQUAL_MESSAGE(0, (int)sctx->last_push_time,
"Req 3.3: on_standby_connected must write last_push_time != 0");
free(ep.data);
zarray_fini(ha->standby_endpoints);
free(ha->standby_endpoints);
ha->standby_endpoints = NULL;
ha_fini(ha);
printf("test_on_standby_connected_sets_last_push_time: PASS\n");
}
/* ── Task 3.1: cluster.c dispatch guard tests ───────────────────────────── */
/*
* Simulates the cluster.c OPCode_nop dispatch guard:
* if role == CTRL_STANDBY → ha_on_heartbeat_received()
* Tests that the role-based guard (Req 1.3) correctly updates last_activity
* only when on a standby, and is a no-op when on a primary.
*/
/*
* Test 11: cluster.c nop dispatch guard — CTRL_STANDBY resets last_activity
*
* Req 1.3, 4.1: any primary message on a standby resets the liveness deadline.
* The cluster.c OPCode_nop handler must call ha_on_heartbeat_received when
* role == CTRL_STANDBY (not gated on ep->fd == primary_ctrl_fd).
*/
static void test_cluster_dispatch_nop_resets_activity_when_standby(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
ha_do_demote(&ha); /* role = CTRL_STANDBY */
ha.last_activity = 0;
/* Simulate what cluster.c OPCode_nop handler now does:
* if (role == CTRL_STANDBY) ha_on_heartbeat_received(&ha); */
if ((ctrl_role_t)atomic_load_explicit(&ha.role, memory_order_acquire)
== CTRL_STANDBY)
ha_on_heartbeat_received(&ha);
TEST_ASSERT_NOT_EQUAL_MESSAGE(0, (int)ha.last_activity,
"Req 1.3: OPCode_nop on CTRL_STANDBY must reset last_activity");
ha_fini(&ha);
printf("test_cluster_dispatch_nop_resets_activity_when_standby: PASS\n");
}
/*
* Test 12: cluster.c nop dispatch guard — CTRL_PRIMARY does NOT reset last_activity
*
* Req 1.3: the guard must only call ha_on_heartbeat_received when CTRL_STANDBY.
* When role is CTRL_PRIMARY the call must be skipped so the primary does not
* accidentally update its own liveness clock via reflected nops.
*/
static void test_cluster_dispatch_nop_skips_when_primary(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
/* Force role to CTRL_PRIMARY */
atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release);
ha.last_activity = 0;
/* Simulate cluster.c OPCode_nop guard: skip when not CTRL_STANDBY */
if ((ctrl_role_t)atomic_load_explicit(&ha.role, memory_order_acquire)
== CTRL_STANDBY)
ha_on_heartbeat_received(&ha);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, (int)ha.last_activity,
"Req 1.3: OPCode_nop on CTRL_PRIMARY must NOT update last_activity");
ha_fini(&ha);
printf("test_cluster_dispatch_nop_skips_when_primary: PASS\n");
}
/*
* Test 13: cluster.c ctrlhello dispatch — CTRL_STANDBY resets last_activity (Req 4.1)
*
* Req 4.1: ctrlhello counts as primary activity — the OPCode_hello ctrlhello
* path must call ha_on_heartbeat_received when role == CTRL_STANDBY.
*/
static void test_cluster_dispatch_ctrlhello_resets_activity_when_standby(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
ha_do_demote(&ha); /* role = CTRL_STANDBY */
ha.last_activity = 0;
/* Simulate cluster.c OPCode_hello ctrlhello path:
* rc = handle_ctrlhello(c, ep, &request);
* if (role == CTRL_STANDBY) ha_on_heartbeat_received(&ha); */
if ((ctrl_role_t)atomic_load_explicit(&ha.role, memory_order_acquire)
== CTRL_STANDBY)
ha_on_heartbeat_received(&ha);
TEST_ASSERT_NOT_EQUAL_MESSAGE(0, (int)ha.last_activity,
"Req 4.1: ctrlhello on CTRL_STANDBY must reset last_activity");
ha_fini(&ha);
printf("test_cluster_dispatch_ctrlhello_resets_activity_when_standby: PASS\n");
}
/* ── Suite entry point ──────────────────────────────────────────────────── */
void test_ha_liveness(void)
{
UNITY_BEGIN();
RUN_TEST(test_liveness_poll_fires_failover_when_timeout);
RUN_TEST(test_liveness_poll_rearms_when_fresh);
RUN_TEST(test_liveness_poll_cancelled_is_noop);
RUN_TEST(test_ha_on_heartbeat_received_updates_last_activity);
RUN_TEST(test_liveness_poll_skips_if_not_standby);
RUN_TEST(test_ha_arm_no_crash_no_thread);
RUN_TEST(test_ha_disarm_makes_subsequent_poll_noop);
RUN_TEST(test_primary_hb_sends_nop_when_idle);
RUN_TEST(test_primary_hb_skips_nop_when_fresh);
RUN_TEST(test_on_standby_connected_sets_last_push_time);
RUN_TEST(test_cluster_dispatch_nop_resets_activity_when_standby);
RUN_TEST(test_cluster_dispatch_nop_skips_when_primary);
RUN_TEST(test_cluster_dispatch_ctrlhello_resets_activity_when_standby);
UNITY_END();
}