feat(ha-config-bootstrap): implement EPOLLRDHUP failover and heartbeat timeout vtask (task 9.4)
This commit is contained in:
parent
9bbff7b016
commit
0d5f2cb1bb
6 changed files with 434 additions and 1 deletions
|
|
@ -42,6 +42,7 @@ extern void test_repl_majority(void);
|
|||
extern void test_persist_truncate(void);
|
||||
extern void test_ha_connect(void);
|
||||
extern void test_ha_ctrlhello(void);
|
||||
extern void test_ha_failover(void);
|
||||
|
||||
void setUp(void) { return; }
|
||||
|
||||
|
|
@ -153,6 +154,7 @@ int main(int argc, char* argv[])
|
|||
TEST(test_persist_truncate);
|
||||
TEST(test_ha_connect);
|
||||
TEST(test_ha_ctrlhello);
|
||||
TEST(test_ha_failover);
|
||||
|
||||
return UNITY_END();
|
||||
}
|
||||
|
|
|
|||
246
src/controller/testsuite/test_ha_failover.c
Normal file
246
src/controller/testsuite/test_ha_failover.c
Normal file
|
|
@ -0,0 +1,246 @@
|
|||
/*
|
||||
* test_ha_failover.c — Unit tests for task 9.4: EPOLLRDHUP failover and
|
||||
* heartbeat timeout vtask.
|
||||
*
|
||||
* Tests:
|
||||
* 1. ha_on_primary_ctrl_fd_rdhup closes fd, sets to -1, transitions to CTRL_ELECTING
|
||||
* 2. ha_on_primary_ctrl_fd_rdhup calls persist_truncate_wal_above
|
||||
* 3. ha_arm_heartbeat_timeout_vtask fires failover after silence
|
||||
* 4. ha_on_heartbeat_received resets the timeout (no failover on heartbeat)
|
||||
* 5. FSM: CTRL_STANDBY after election → primary_ctrl_fd closed
|
||||
* 6. INFO log on FSM transition (Req 10.5)
|
||||
*
|
||||
* Requirements: 6.2, 6.3, 6.4, 6.8, 10.5
|
||||
*/
|
||||
#define _GNU_SOURCE
|
||||
#include <errno.h>
|
||||
#include <pthread.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <sys/socket.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <unity/unity.h>
|
||||
#include "ha.h"
|
||||
|
||||
/* ── Forward declarations — function-under-test (not yet implemented) ────── */
|
||||
|
||||
/*
|
||||
* ha_on_primary_ctrl_fd_rdhup: EPOLLRDHUP event handler for primary_ctrl_fd.
|
||||
*
|
||||
* Steps (Req 6.2, 6.3):
|
||||
* 1. Close and null ha->primary_ctrl_fd
|
||||
* 2. Transition ha role to CTRL_ELECTING
|
||||
* 3. Call persist_truncate_wal_above(ha->last_known_committed_lsn) via cluster
|
||||
* 4. Log INFO: FSM transition (Req 10.5)
|
||||
*
|
||||
* Pass c == NULL to skip persist_truncate and am_i_primary_quorum (unit test mode).
|
||||
*/
|
||||
extern void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c);
|
||||
|
||||
/*
|
||||
* ha_arm_heartbeat_timeout_vtask: start background thread monitoring heartbeat
|
||||
* silence on primary_ctrl_fd (Req 6.3, 6.4).
|
||||
*
|
||||
* The thread sleeps for election_timeout_ms + rand_jitter. If no heartbeat
|
||||
* is received (ha_on_heartbeat_received not called) within that window,
|
||||
* it fires ha_on_primary_ctrl_fd_rdhup.
|
||||
*
|
||||
* Subsequent calls re-arm the timeout (same thread, reset deadline).
|
||||
* Calling ha_on_heartbeat_received() resets the deadline.
|
||||
*/
|
||||
extern void ha_arm_heartbeat_timeout_vtask(ha_state_t *ha, cluster_t *c);
|
||||
|
||||
/*
|
||||
* ha_on_heartbeat_received: reset the heartbeat timeout deadline.
|
||||
* Called whenever a heartbeat arrives on primary_ctrl_fd (Req 6.3).
|
||||
*/
|
||||
extern void ha_on_heartbeat_received(ha_state_t *ha);
|
||||
|
||||
/*
|
||||
* ha_disarm_heartbeat_timeout_vtask: stop the heartbeat timeout thread.
|
||||
* Called when transitioning away from CTRL_STANDBY.
|
||||
*/
|
||||
extern void ha_disarm_heartbeat_timeout_vtask(ha_state_t *ha);
|
||||
|
||||
/* ── Helpers ─────────────────────────────────────────────────────────────── */
|
||||
|
||||
static ha_config_t g_cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 500, /* minimum allowed by ha_init validation */
|
||||
.probe_connect_timeout_ms = 50,
|
||||
.max_promotion_retries = 3,
|
||||
.election_jitter_max_ms = 0, /* no jitter: deterministic timeout */
|
||||
.election_priority_step_ms = 500,
|
||||
};
|
||||
|
||||
/* ── Test 1: RDHUP closes fd and sets primary_ctrl_fd = -1 ─────────────── */
|
||||
|
||||
static void test_rdhup_closes_and_nulls_fd(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
|
||||
|
||||
/* Set up a socketpair as the "primary_ctrl_fd" */
|
||||
int sv[2];
|
||||
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
|
||||
ha.primary_ctrl_fd = sv[0];
|
||||
|
||||
/* Fire RDHUP with NULL cluster (unit test mode — skips persist/election) */
|
||||
ha_on_primary_ctrl_fd_rdhup(&ha, NULL);
|
||||
|
||||
/* fd must be closed and set to -1 */
|
||||
TEST_ASSERT_EQUAL_INT(-1, ha.primary_ctrl_fd);
|
||||
/* Verify sv[0] is really closed by trying to write to it */
|
||||
ssize_t n = write(sv[0], "x", 1);
|
||||
TEST_ASSERT_LESS_THAN_INT(0, (int)n);
|
||||
TEST_ASSERT_EQUAL_INT(EBADF, errno);
|
||||
|
||||
close(sv[1]);
|
||||
ha_fini(&ha);
|
||||
printf("test_rdhup_closes_and_nulls_fd: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 2: RDHUP transitions role to CTRL_ELECTING ───────────────────── */
|
||||
|
||||
static void test_rdhup_transitions_to_ctrl_electing(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
|
||||
|
||||
/* Start as CTRL_STANDBY */
|
||||
ha_do_demote(&ha);
|
||||
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load(&ha.role));
|
||||
|
||||
/* Simulate having a primary fd */
|
||||
int sv[2];
|
||||
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
|
||||
ha.primary_ctrl_fd = sv[0];
|
||||
|
||||
ha_on_primary_ctrl_fd_rdhup(&ha, NULL);
|
||||
|
||||
/* Must transition to CTRL_ELECTING */
|
||||
TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)atomic_load(&ha.role));
|
||||
|
||||
close(sv[1]);
|
||||
ha_fini(&ha);
|
||||
printf("test_rdhup_transitions_to_ctrl_electing: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 3: heartbeat received before timeout → no failover ───────────── */
|
||||
|
||||
static void test_heartbeat_received_resets_timeout(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
|
||||
ha_do_demote(&ha);
|
||||
|
||||
int sv[2];
|
||||
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
|
||||
ha.primary_ctrl_fd = sv[0];
|
||||
|
||||
/* Arm the timeout (200ms + jitter=0) */
|
||||
ha_arm_heartbeat_timeout_vtask(&ha, NULL);
|
||||
|
||||
/* Immediately receive heartbeat to reset the deadline */
|
||||
ha_on_heartbeat_received(&ha);
|
||||
|
||||
/* Wait briefly — less than one full election_timeout_ms */
|
||||
struct timespec ts = { .tv_sec = 0, .tv_nsec = 50 * 1000000L };
|
||||
nanosleep(&ts, NULL);
|
||||
|
||||
/* Role must still be CTRL_STANDBY (no failover yet) */
|
||||
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load(&ha.role));
|
||||
TEST_ASSERT_NOT_EQUAL(-1, ha.primary_ctrl_fd);
|
||||
|
||||
ha_disarm_heartbeat_timeout_vtask(&ha);
|
||||
close(sv[0]);
|
||||
close(sv[1]);
|
||||
ha.primary_ctrl_fd = -1;
|
||||
ha_fini(&ha);
|
||||
printf("test_heartbeat_received_resets_timeout: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 4: heartbeat timeout fires failover ───────────────────────────── */
|
||||
|
||||
static void test_heartbeat_timeout_fires_failover(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
|
||||
ha_do_demote(&ha);
|
||||
|
||||
int sv[2];
|
||||
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
|
||||
ha.primary_ctrl_fd = sv[0];
|
||||
|
||||
/* Arm with 200ms timeout, NO heartbeat received */
|
||||
ha_arm_heartbeat_timeout_vtask(&ha, NULL);
|
||||
|
||||
/* Wait 800ms — well past the 500ms election_timeout_ms */
|
||||
struct timespec ts = { .tv_sec = 0, .tv_nsec = 800 * 1000000L };
|
||||
nanosleep(&ts, NULL);
|
||||
|
||||
/* Failover must have fired: role = CTRL_ELECTING, fd = -1 */
|
||||
TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)atomic_load(&ha.role));
|
||||
TEST_ASSERT_EQUAL_INT(-1, ha.primary_ctrl_fd);
|
||||
|
||||
ha_disarm_heartbeat_timeout_vtask(&ha);
|
||||
close(sv[1]); /* sv[0] was closed by failover handler */
|
||||
ha_fini(&ha);
|
||||
printf("test_heartbeat_timeout_fires_failover: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 5: ha_on_primary_ctrl_fd_rdhup on already-closed fd is safe ───── */
|
||||
|
||||
static void test_rdhup_on_minus_one_fd_is_safe(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
|
||||
ha.primary_ctrl_fd = -1;
|
||||
|
||||
/* Should not crash or abort */
|
||||
ha_on_primary_ctrl_fd_rdhup(&ha, NULL);
|
||||
TEST_ASSERT_EQUAL_INT(-1, ha.primary_ctrl_fd);
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_rdhup_on_minus_one_fd_is_safe: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 6: symbol existence check ─────────────────────────────────────── */
|
||||
|
||||
static void test_ha_failover_symbols_exist(void)
|
||||
{
|
||||
typedef void (*rdhup_fn)(ha_state_t *, cluster_t *);
|
||||
typedef void (*arm_fn)(ha_state_t *, cluster_t *);
|
||||
typedef void (*hb_fn)(ha_state_t *);
|
||||
typedef void (*disarm_fn)(ha_state_t *);
|
||||
|
||||
rdhup_fn f1 = ha_on_primary_ctrl_fd_rdhup;
|
||||
arm_fn f2 = ha_arm_heartbeat_timeout_vtask;
|
||||
hb_fn f3 = ha_on_heartbeat_received;
|
||||
disarm_fn f4 = ha_disarm_heartbeat_timeout_vtask;
|
||||
|
||||
TEST_ASSERT_NOT_NULL((void*)f1);
|
||||
TEST_ASSERT_NOT_NULL((void*)f2);
|
||||
TEST_ASSERT_NOT_NULL((void*)f3);
|
||||
TEST_ASSERT_NOT_NULL((void*)f4);
|
||||
printf("test_ha_failover_symbols_exist: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Suite entry point ──────────────────────────────────────────────────── */
|
||||
|
||||
void test_ha_failover(void)
|
||||
{
|
||||
UNITY_BEGIN();
|
||||
RUN_TEST(test_rdhup_closes_and_nulls_fd);
|
||||
RUN_TEST(test_rdhup_transitions_to_ctrl_electing);
|
||||
RUN_TEST(test_heartbeat_received_resets_timeout);
|
||||
RUN_TEST(test_heartbeat_timeout_fires_failover);
|
||||
RUN_TEST(test_rdhup_on_minus_one_fd_is_safe);
|
||||
RUN_TEST(test_ha_failover_symbols_exist);
|
||||
UNITY_END();
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue