lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore: checkpoint all prior work for clean state

Browse source 
Captures all uncommitted work from ha-config-bootstrap,
ha-heartbeat-liveness, and associated spec/tooling updates:

- src: ha.c, ha.h, cluster.c, config_repl.c, myapp.c, raft.c —
  heartbeat liveness, RDHUP integration, committing phase fixes
- tests: test_ha_role.c, test_config_repl.c, test_ha_rdhup_integration.c,
  test_p1_integration.c, test_p3_hardening.c, scheduler test
- specs: election-iomgr-refactor, ha-heartbeat-liveness,
  ha-peer-connection-reuse, raft-library (new); ha-config-bootstrap
  and election-iomgr-refactor task/research updates
- tooling: .wolf/ state, ha/data cluster fixtures, raft/raft.tla,
  src/build symlink, xce-build skill update
This commit is contained in:
Rongsong Shen 2026-06-04 13:04:16 +00:00
parent 15bd291bdd
commit e31673dd88
66 changed files with 122422 additions and 860 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/controller/testsuite/controller_test.c
View file

@ -20,7 +20,6 @@ extern void test_e2e_hooks(void);
extern void test_e2e_gc(void);
extern void test_e2e_capnp(void);
extern void test_ha_role(void);
extern void test_am_i_primary(void);
extern void test_ha_cluster(void);
extern void test_ha_redirect(void);
extern void test_p1_integration(void);
@ -140,7 +139,6 @@ int main(int argc, char* argv[])
TEST(test_e2e_gc);
TEST(test_e2e_capnp);
TEST(test_ha_role);
TEST(test_am_i_primary);
TEST(test_ha_cluster);
TEST(test_ha_redirect);
TEST(test_p1_integration);

5
src/controller/testsuite/test_config_repl.c
View file

@ -886,13 +886,14 @@ static void test_upgrade_config_req_extract_raw(void)
free(out_sha256);
free(buf);
/* The decoded-form variant must always return -ENOTSUP */
/* xce_upgrade_config_req_extract: returns -EINVAL on a zeroed request
* (body_which is 0, not Request_body_upgradeconfigrequestmsg). */
{
xce_request_t dummy_req;
memset(&dummy_req, 0, sizeof(dummy_req));
uint8_t *b = NULL, *s = NULL;
size_t l = 0;
TEST_ASSERT_EQUAL_INT(-ENOTSUP,
TEST_ASSERT_EQUAL_INT(-EINVAL,
xce_upgrade_config_req_extract(&dummy_req, &b, &l, &s));
}

440
src/controller/testsuite/test_ha_rdhup_integration.c Normal file
View file

@ -0,0 +1,440 @@
/*
* test_ha_rdhup_integration.c — Task 6.4 integration test: RDHUP failover
* end-to-end role transition.
*
* Tests verify the full RDHUP → CTRL_ELECTING → CTRL_PRIMARY_COMMITTING →
* CTRL_PRIMARY state-machine chain using XCE_TESTING hook seams. No live
* sockets or real cluster_t required.
*
* Requirements: 6.3, 6.5, 9.2, 9.3 (Req numbers from tasks.md 5.2 / 6.4)
*
* Hook seams used:
* ha_rdhup_election_hook — intercepts ha_start_election_async in RDHUP path
* ha_phase2_hook — stubs cluster_init_phase2 (already done = 1)
* ha_push_config_hook — stubs push_config_to_standbys
* ha_send_ctrlhello_hook — stubs ha_send_ctrlhello_to_all_standbys
*
* TDD: tests written first (RED); implementation verified GREEN.
*/
#define _GNU_SOURCE
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <unistd.h>
#include <unity/unity.h>
#include "ha.h"
#include "myconf.h"
#include "xce/cluster_def.h"
/* ── XCE_TESTING hook seam externs ──────────────────────────────────────── */
extern int (*ha_phase2_hook)(void *c);
extern int (*ha_push_config_hook)(void *c);
extern int (*ha_send_ctrlhello_hook)(void *c);
extern void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c,
int n_peers, xce_candidate_t **peers,
uint64_t my_config_tag, uint64_t my_lsn,
const char *my_name);
/* ── Stub counters ───────────────────────────────────────────────────────── */
static int g_phase2_calls = 0;
static int g_push_config_calls = 0;
static int g_ctrlhello_calls = 0;
static int stub_phase2(void *c) { (void)c; g_phase2_calls++; return 0; }
static int stub_push_config(void *c) { (void)c; g_push_config_calls++; return 0; }
static int stub_ctrlhello(void *c) { (void)c; g_ctrlhello_calls++; return 0; }
/* ── Shared HA config ───────────────────────────────────────────────────── */
static ha_config_t g_cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.election_jitter_max_ms = 0,
.election_priority_step_ms = 500,
.committing_timeout_ms = 10000,
};
/* ── install_stubs / clear_stubs ─────────────────────────────────────────── */
static void install_stubs(void)
{
g_phase2_calls = 0;
g_push_config_calls = 0;
g_ctrlhello_calls = 0;
ha_phase2_hook = stub_phase2;
ha_push_config_hook = stub_push_config;
ha_send_ctrlhello_hook = stub_ctrlhello;
}
static void clear_stubs(void)
{
ha_rdhup_election_hook = NULL;
ha_phase2_hook = NULL;
ha_push_config_hook = NULL;
ha_send_ctrlhello_hook = NULL;
}
/* ── Inline RDHUP election hook: drives evaluate immediately ─────────────
*
* Simulates ha_start_election_async completing synchronously:
* allocates a minimal ha_elect_ctx_t with self-vote won, calls
* ha_elect_evaluate_and_signal, then frees ctx.
*
* This models the single-node shortcircuit path (Req 9.1) adapted to
* a multi-node cluster where the surviving standby immediately wins —
* the key assertion is role progression, not peer negotiation.
*/
typedef struct {
cluster_t *c;
} rdhup_hook_state_t;
static rdhup_hook_state_t g_rdhup_state;
static void rdhup_election_drives_evaluate(ha_state_t *ha, cluster_t *c,
int n_peers, xce_candidate_t **peers,
uint64_t my_config_tag,
uint64_t my_lsn,
const char *my_name)
{
(void)peers; (void)my_config_tag; (void)my_lsn; (void)my_name;
/* Allocate minimal ctx with quorum already satisfied (self-vote wins) */
int n_other = (n_peers > 1) ? n_peers - 1 : 0;
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) +
(size_t)n_other * sizeof(ctx->votes[0]));
if (!ctx) return;
ctx->n_peers = n_other;
ctx->votes_granted = 1; /* self-vote */
ctx->n_replied = n_other;
ctx->result = -1;
ctx->done_posted = 0;
ctx->c = g_rdhup_state.c;
pthread_mutex_init(&ctx->lock, NULL);
sem_init(&ctx->done, 0, 0);
ha_elect_evaluate_and_signal(ha, ctx);
pthread_mutex_destroy(&ctx->lock);
sem_destroy(&ctx->done);
free(ctx);
}
/* ── Helper: make a minimal 2-candidate cluster_t ───────────────────────── */
static void make_cluster(cluster_t *c, myconf_t *myconf, const char *my_name)
{
memset(c, 0, sizeof(*c));
memset(myconf, 0, sizeof(*myconf));
myconf->name = (char *)my_name;
myconf->peers_cfg.candidates_count = 2;
snprintf(myconf->peers_cfg.candidates[0].name,
sizeof(myconf->peers_cfg.candidates[0].name), "%s", my_name);
myconf->peers_cfg.candidates[0].port = 7001;
snprintf(myconf->peers_cfg.candidates[1].name,
sizeof(myconf->peers_cfg.candidates[1].name), "ctrl-b");
myconf->peers_cfg.candidates[1].port = 7002;
c->myconf = myconf;
}
/* ═══════════════════════════════════════════════════════════════════════════
* Test 1: RDHUP → ELECTING → COMMITTING (no standbys present → via hook chain)
*
* Req 6.3: RDHUP triggers CTRL_ELECTING + async election
* Req 9.3: RDHUP election uses ha_start_election_async (not am_i_primary_quorum)
* Req 9.2: phase2_done=1 suppresses cluster_init_phase2 on RDHUP re-election
* ═════════════════════════════════════════════════════════════════════════ */
static void test_rdhup_full_path_reaches_committing(void)
{
install_stubs();
cluster_t c;
myconf_t myconf;
make_cluster(&c, &myconf, "ctrl-a");
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
ha_do_demote(&c.ha);
c.ha.phase2_done = 1; /* surviving standby — phase2 already done */
c.ha.n_peers = 1; /* 1 peer (2-node cluster) */
/* Wire the hook that drives evaluate immediately */
g_rdhup_state.c = &c;
ha_rdhup_election_hook = rdhup_election_drives_evaluate;
/* Give RDHUP handler a socketpair fd to close */
int sv[2];
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
c.ha.primary_ctrl_fd = sv[0];
/* Fire RDHUP */
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
clear_stubs();
close(sv[1]);
/* phase2 must NOT have been called (phase2_done=1 → skip) */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_phase2_calls,
"RDHUP re-election must not call cluster_init_phase2 when phase2_done=1 (Req 9.2)");
/* Role must be CTRL_PRIMARY_COMMITTING (quorum won → committing gate) */
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, role,
"RDHUP full path must reach CTRL_PRIMARY_COMMITTING after winning election (Req 6.3)");
ha_fini(&c.ha);
printf("test_rdhup_full_path_reaches_committing: PASS (role=%d==CTRL_PRIMARY_COMMITTING)\n",
role);
}
/* ═══════════════════════════════════════════════════════════════════════════
* Test 2: RDHUP → COMMITTING → PRIMARY via ha_repl_ack_received
*
* Req 8.2: majority ack in COMMITTING → CTRL_PRIMARY
* Req 6.5: promoted standby serves traffic (dispatch gate opens)
* ═════════════════════════════════════════════════════════════════════════ */
static void test_rdhup_committing_to_primary_on_ack(void)
{
install_stubs();
cluster_t c;
myconf_t myconf;
make_cluster(&c, &myconf, "ctrl-a");
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
ha_do_demote(&c.ha);
c.ha.phase2_done = 1;
c.ha.n_peers = 1; /* 2-node cluster: quorum=1, threshold=0 */
g_rdhup_state.c = &c;
ha_rdhup_election_hook = rdhup_election_drives_evaluate;
int sv[2];
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
c.ha.primary_ctrl_fd = sv[0];
/* Fire RDHUP → should reach CTRL_PRIMARY_COMMITTING */
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
close(sv[1]);
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT((int)CTRL_PRIMARY_COMMITTING, role);
/* n_peers=1 → quorum=(1+2)/2=1 → threshold=0 → first ack promotes.
* ha_repl_ack_received increments count; acks(1) >= quorum-1(0) → promote. */
ha_repl_ack_received(&c.ha, &c);
role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
"ha_repl_ack_received must drive COMMITTING → CTRL_PRIMARY on quorum ack (Req 8.2)");
clear_stubs();
ha_fini(&c.ha);
printf("test_rdhup_committing_to_primary_on_ack: PASS (role=%d==CTRL_PRIMARY)\n", role);
}
/* ═══════════════════════════════════════════════════════════════════════════
* Test 3: RDHUP in single-node cluster → immediate PRIMARY (no COMMITTING)
*
* Req 9.1: single-node self-elects without network
* n_peers=0 → quorum=1, threshold=0 → acks(0) >= 0 → promotes immediately
* in ha_elect_evaluate_and_signal (no peers to wait for acks from)
* ═════════════════════════════════════════════════════════════════════════ */
static void test_rdhup_single_node_reaches_primary(void)
{
install_stubs();
cluster_t c;
myconf_t myconf;
make_cluster(&c, &myconf, "ctrl-a");
/* Override to single-node */
myconf.peers_cfg.candidates_count = 1;
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
ha_do_demote(&c.ha);
c.ha.phase2_done = 1;
c.ha.n_peers = 0; /* single node: no peers */
g_rdhup_state.c = &c;
ha_rdhup_election_hook = rdhup_election_drives_evaluate;
int sv[2];
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
c.ha.primary_ctrl_fd = sv[0];
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
close(sv[1]);
/* n_peers=0 → quorum=(0+2)/2=1 → threshold=0 → acks(0) >= 0 → promoted inline */
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
/* Single-node: no standbys to ack, committing_timeout degrades to PRIMARY.
* The committing timeout is 10s; for single-node we call ha_repl_ack_received
* once to simulate the degraded-mode zero-threshold promotion (threshold=0). */
if (role == (int)CTRL_PRIMARY_COMMITTING) {
ha_repl_ack_received(&c.ha, &c);
role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
}
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
"single-node RDHUP path must reach CTRL_PRIMARY (Req 9.1)");
clear_stubs();
ha_fini(&c.ha);
printf("test_rdhup_single_node_reaches_primary: PASS (role=%d==CTRL_PRIMARY)\n", role);
}
/* ═══════════════════════════════════════════════════════════════════════════
* Test 4: 3-node cluster RDHUP — two acks needed, premature ack stays COMMITTING
*
* Req 8.2: acks < quorum-1 → stays COMMITTING; acks >= quorum-1 → PRIMARY
* 3-node (n_peers=2): quorum=2, threshold=1 → need 1 ack to promote
* ═════════════════════════════════════════════════════════════════════════ */
static void test_rdhup_3node_one_ack_promotes(void)
{
install_stubs();
cluster_t c;
myconf_t myconf;
make_cluster(&c, &myconf, "ctrl-a");
myconf.peers_cfg.candidates_count = 3;
snprintf(myconf.peers_cfg.candidates[2].name,
sizeof(myconf.peers_cfg.candidates[2].name), "ctrl-c");
myconf.peers_cfg.candidates[2].port = 7003;
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
ha_do_demote(&c.ha);
c.ha.phase2_done = 1;
c.ha.n_peers = 2; /* 3-node: quorum=2, threshold=1 */
g_rdhup_state.c = &c;
ha_rdhup_election_hook = rdhup_election_drives_evaluate;
int sv[2];
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
c.ha.primary_ctrl_fd = sv[0];
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
close(sv[1]);
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, role,
"3-node RDHUP must enter CTRL_PRIMARY_COMMITTING before acks");
/* First ack: count=1 >= threshold=1 → promotes */
ha_repl_ack_received(&c.ha, &c);
role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
"3-node: first ack (count=1 >= threshold=1) must promote to CTRL_PRIMARY (Req 8.2)");
clear_stubs();
ha_fini(&c.ha);
printf("test_rdhup_3node_one_ack_promotes: PASS (role=%d==CTRL_PRIMARY)\n", role);
}
/* ═══════════════════════════════════════════════════════════════════════════
* Test 5: config push and ctrlhello hooks fire during COMMITTING entry
*
* Req 7.1: push_config_to_standbys called after winning election
* Req 7.2: ha_send_ctrlhello_to_all_standbys called after winning election
* ═════════════════════════════════════════════════════════════════════════ */
static void test_rdhup_committing_fires_replication_hooks(void)
{
install_stubs();
cluster_t c;
myconf_t myconf;
make_cluster(&c, &myconf, "ctrl-a");
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
ha_do_demote(&c.ha);
c.ha.phase2_done = 1;
c.ha.n_peers = 1;
g_rdhup_state.c = &c;
ha_rdhup_election_hook = rdhup_election_drives_evaluate;
int sv[2];
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
c.ha.primary_ctrl_fd = sv[0];
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
close(sv[1]);
TEST_ASSERT_GREATER_THAN_INT_MESSAGE(0, g_push_config_calls,
"push_config_to_standbys hook must fire when entering COMMITTING (Req 7.1)");
TEST_ASSERT_GREATER_THAN_INT_MESSAGE(0, g_ctrlhello_calls,
"ha_send_ctrlhello hook must fire when entering COMMITTING (Req 7.2)");
clear_stubs();
ha_fini(&c.ha);
printf("test_rdhup_committing_fires_replication_hooks: PASS "
"(push=%d, ctrlhello=%d)\n",
g_push_config_calls, g_ctrlhello_calls);
}
/* ═══════════════════════════════════════════════════════════════════════════
* Test 6: dispatch gate opens after CTRL_PRIMARY (job ops allowed)
*
* Req 6.5: after RDHUP promotion, primary accepts job dispatch (Req 5.3)
* ═════════════════════════════════════════════════════════════════════════ */
static void test_rdhup_primary_dispatch_gate_opens(void)
{
install_stubs();
cluster_t c;
myconf_t myconf;
make_cluster(&c, &myconf, "ctrl-a");
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
ha_do_demote(&c.ha);
c.ha.phase2_done = 1;
c.ha.n_peers = 1;
g_rdhup_state.c = &c;
ha_rdhup_election_hook = rdhup_election_drives_evaluate;
int sv[2];
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
c.ha.primary_ctrl_fd = sv[0];
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
close(sv[1]);
/* Drive to CTRL_PRIMARY via ack */
ha_repl_ack_received(&c.ha, &c);
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT((int)CTRL_PRIMARY, role);
/* Verify dispatch gate allows job operations in CTRL_PRIMARY */
int rc = ha_dispatch_gate(&c.ha, OPCode_jobsub);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"OPCode_jobsub must be allowed by dispatch gate in CTRL_PRIMARY (Req 5.3)");
rc = ha_dispatch_gate(&c.ha, OPCode_jobdispatch);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"OPCode_jobdispatch must be allowed by dispatch gate in CTRL_PRIMARY (Req 5.3)");
clear_stubs();
ha_fini(&c.ha);
printf("test_rdhup_primary_dispatch_gate_opens: PASS (gate accepts job ops)\n");
}
/* ── Suite entry point ──────────────────────────────────────────────────── */
void test_ha_rdhup_integration(void)
{
RUN_TEST(test_rdhup_full_path_reaches_committing);
RUN_TEST(test_rdhup_committing_to_primary_on_ack);
RUN_TEST(test_rdhup_single_node_reaches_primary);
RUN_TEST(test_rdhup_3node_one_ack_promotes);
RUN_TEST(test_rdhup_committing_fires_replication_hooks);
RUN_TEST(test_rdhup_primary_dispatch_gate_opens);
}

197
src/controller/testsuite/test_ha_role.c
View file

@ -88,72 +88,8 @@ static void test_ha_promote_sets_primary_role(void)
printf("test_ha_promote_sets_primary_role: PASS\n");
}
/* ── P1-3 watchdog tests ────────────────────────────────────── */
/* ha_start_watchdog() must be callable after ha_init() */
static void test_watchdog_starts_and_stops(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
TEST_ASSERT_EQUAL_INT(0, ha_start_watchdog(&ha));
/* brief sleep to let the thread wake at least once */
struct timespec ts = { 0, 50 * 1000000L }; /* 50ms */
nanosleep(&ts, NULL);
ha_stop_watchdog(&ha); /* sets stop flag + pthread_join */
ha_fini(&ha);
printf("test_watchdog_starts_and_stops: PASS\n");
}
/* ha_check_heartbeat_nocluster(): when in CTRL_STANDBY with no primary_ep,
* must not promote */
static void test_heartbeat_no_action_when_no_primary(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
ha_init(&ha, &cfg, 1 /* not highest priority */);
/* No primary_host set — watchdog must not promote */
ha_check_heartbeat_nocluster(&ha); /* test-only accessor */
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
ha_fini(&ha);
printf("test_heartbeat_no_action_when_no_primary: PASS\n");
}
/* ha_check_heartbeat_nocluster(): when last_heartbeat is stale (timeout exceeded)
* and primary_host is empty (no known primary), must not crash or change role */
static void test_heartbeat_stale_no_crash_without_primary(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 2000, /* minimum valid */
.heartbeat_timeout_ms = 3000, /* >= interval */
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
/* Staleness is injected manually below; timeout value only needs to be valid */
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
/* Set last_heartbeat to a time far in the past to simulate timeout */
clock_gettime(CLOCK_MONOTONIC, &ha.last_heartbeat);
ha.last_heartbeat.tv_sec -= 10; /* 10 seconds ago */
ha_check_heartbeat_nocluster(&ha);
/* Should not crash. Role may or may not change — but no segfault. */
ha_fini(&ha);
printf("test_heartbeat_stale_no_crash_without_primary: PASS\n");
}
/* Watchdog / SIGUSR1 tests deleted — watchdog pthread replaced by
* ha_arm_heartbeat_timeout_vtask (delaymgr); failover_requested removed */
/* ── P1-3a tests ─────────────────────────────────────────────── */
@ -206,120 +142,9 @@ static void test_demote_sets_standby_role(void)
printf("test_demote_sets_standby_role: PASS\n");
}
/* SIGUSR1 must set failover_requested via async-signal-safe atomic store */
static void test_sigusr1_sets_failover_requested(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
ha_init(&ha, &cfg, 0);
/* test_sigusr1_sets_failover_requested deleted — SIGUSR1 / failover_requested removed */
/* Install real SIGUSR1 handler that uses only the atomic flag */
ha_install_sigusr1(&ha);
/* Send SIGUSR1 to self */
raise(SIGUSR1);
/* Handler must have set failover_requested */
TEST_ASSERT_TRUE(
atomic_load_explicit(&ha.failover_requested, memory_order_acquire) != 0);
/* Restore default SIGUSR1 handler */
signal(SIGUSR1, SIG_DFL);
ha_fini(&ha);
printf("test_sigusr1_sets_failover_requested: PASS\n");
}
/* ── P1-6: am_i_primary() election tests ────────────────────────────── */
/* test: priority-0 with no peers → immediately primary (no probing needed) */
void test_am_i_primary_p0_no_peers(void)
{
cluster_t c;
memset(&c, 0, sizeof(c));
myconf_t myconf;
memset(&myconf, 0, sizeof(myconf));
c.myconf = &myconf;
ha_config_t cfg = {
.heartbeat_interval_ms = HA_DEFAULT_HEARTBEAT_INTERVAL_MS,
.heartbeat_timeout_ms = HA_DEFAULT_HEARTBEAT_TIMEOUT_MS,
.election_timeout_ms = 500, /* minimum valid value */
.probe_connect_timeout_ms = 100,
.max_promotion_retries = 1,
.election_jitter_max_ms = 0,
.election_priority_step_ms = 0,
};
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &cfg, 0));
TEST_ASSERT_EQUAL_INT(1, am_i_primary(&c));
ha_fini(&c.ha);
printf("test_am_i_primary_p0_no_peers: PASS\n");
}
/* test: priority-0 with peers → still primary (never probes higher-priority nodes) */
void test_am_i_primary_p0_with_peers(void)
{
cluster_t c;
memset(&c, 0, sizeof(c));
myconf_t myconf;
memset(&myconf, 0, sizeof(myconf));
myconf.peers_cfg.candidates_count = 2;
strncpy(myconf.peers_cfg.candidates[0].hostname, "127.0.0.1", XCE_MAXNAME_LEN - 1);
myconf.peers_cfg.candidates[0].port = 19900;
strncpy(myconf.peers_cfg.candidates[1].hostname, "127.0.0.1", XCE_MAXNAME_LEN - 1);
myconf.peers_cfg.candidates[1].port = 19901;
c.myconf = &myconf;
ha_config_t cfg = {
.heartbeat_interval_ms = HA_DEFAULT_HEARTBEAT_INTERVAL_MS,
.heartbeat_timeout_ms = HA_DEFAULT_HEARTBEAT_TIMEOUT_MS,
.election_timeout_ms = 500, /* minimum valid value */
.probe_connect_timeout_ms = 100,
.max_promotion_retries = 1,
.election_jitter_max_ms = 0,
.election_priority_step_ms = 0,
};
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &cfg, 0));
TEST_ASSERT_EQUAL_INT(1, am_i_primary(&c));
ha_fini(&c.ha);
printf("test_am_i_primary_p0_with_peers: PASS\n");
}
/* test: priority-1 with no higher-priority peer alive → deadline expires → primary */
void test_am_i_primary_p1_deadline_expires(void)
{
cluster_t c;
memset(&c, 0, sizeof(c));
myconf_t myconf;
memset(&myconf, 0, sizeof(myconf));
/* Point to ports where nothing is listening */
myconf.peers_cfg.candidates_count = 2;
strncpy(myconf.peers_cfg.candidates[0].hostname, "127.0.0.1", XCE_MAXNAME_LEN - 1);
myconf.peers_cfg.candidates[0].port = 19910; /* nothing listening */
strncpy(myconf.peers_cfg.candidates[1].hostname, "127.0.0.1", XCE_MAXNAME_LEN - 1);
myconf.peers_cfg.candidates[1].port = 19911;
c.myconf = &myconf;
ha_config_t cfg = {
.heartbeat_interval_ms = HA_DEFAULT_HEARTBEAT_INTERVAL_MS,
.heartbeat_timeout_ms = HA_DEFAULT_HEARTBEAT_TIMEOUT_MS,
.election_timeout_ms = 600, /* expires quickly; > 500ms floor */
.probe_connect_timeout_ms = 200,
.max_promotion_retries = 1,
.election_jitter_max_ms = 0,
.election_priority_step_ms = 0,
};
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &cfg, 1)); /* priority 1 — must probe priority 0 */
int result = am_i_primary(&c);
TEST_ASSERT_EQUAL_INT(1, result); /* no peer responded → self-declare primary */
ha_fini(&c.ha);
printf("test_am_i_primary_p1_deadline_expires: PASS\n");
}
/* am_i_primary() tests deleted — function removed (election-iomgr-refactor) */
/* ── Task 2.1: ha_state_t new fields zero-initialised by ha_init ──────── */
@ -376,22 +201,12 @@ void test_ha_role(void)
RUN_TEST(test_ha_init_rejects_short_election_timeout);
RUN_TEST(test_ha_init_rejects_timeout_less_than_interval);
RUN_TEST(test_ha_promote_sets_primary_role);
RUN_TEST(test_watchdog_starts_and_stops);
RUN_TEST(test_heartbeat_no_action_when_no_primary);
RUN_TEST(test_heartbeat_stale_no_crash_without_primary);
/* P1-3a */
RUN_TEST(test_promote_increments_term);
RUN_TEST(test_demote_sets_standby_role);
RUN_TEST(test_sigusr1_sets_failover_requested);
/* Watchdog / SIGUSR1 tests deleted */
/* Task 2.1: election-iomgr-refactor new fields */
RUN_TEST(test_ha_init_zeroes_new_fields);
}
/* ── P1-6 suite: am_i_primary() election ─────────────────────────────── */
void test_am_i_primary(void)
{
RUN_TEST(test_am_i_primary_p0_no_peers);
RUN_TEST(test_am_i_primary_p0_with_peers);
RUN_TEST(test_am_i_primary_p1_deadline_expires);
}
/* test_am_i_primary suite deleted — am_i_primary() removed (election-iomgr-refactor) */

22
src/controller/testsuite/test_p1_integration.c
View file

@ -53,27 +53,6 @@ static void test_term_fencing_monotonic(void)
printf("test_term_fencing_monotonic: PASS\n");
}
/* Test 3: Watchdog start+stop cycle — verify clean teardown */
static void test_watchdog_lifecycle_clean(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
ha_init(&ha, &cfg, 0);
TEST_ASSERT_EQUAL_INT(0, ha_start_watchdog(&ha));
struct timespec ts = { 0, 10 * 1000000L }; /* 10 ms */
nanosleep(&ts, NULL);
ha_stop_watchdog(&ha);
/* After stop, watchdog_thread must be reset to 0 */
TEST_ASSERT_EQUAL_INT(0, (int)ha.watchdog_thread);
ha_fini(&ha);
printf("test_watchdog_lifecycle_clean: PASS\n");
}
/* Test 4: Redirect guard — primary never redirects; standby with known
* primary redirects; standby with no known primary reports nomaster */
@ -145,7 +124,6 @@ void test_p1_integration(void)
{
RUN_TEST(test_single_node_starts_as_primary);
RUN_TEST(test_term_fencing_monotonic);
RUN_TEST(test_watchdog_lifecycle_clean);
RUN_TEST(test_redirect_guard_integration);
RUN_TEST(test_hooks_registered_guard_integration);
}

25
src/controller/testsuite/test_p3_hardening.c
View file

@ -142,30 +142,6 @@ static void test_p3_promote_demote_cycle_consistent_role(void)
printf("test_p3_promote_demote_cycle_consistent_role: PASS\n");
}
/* Heartbeat timeout with a known primary must set failover_requested
* (replicates the critical path exercised by the watchdog). */
static void test_p3_heartbeat_timeout_sets_failover_flag(void)
{
ha_state_t ha;
ha_config_t cfg = make_p3_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1 /* not highest priority */));
/* Seed a known primary so the timeout check fires */
strncpy(ha.primary_host, "dead-primary", XCE_MAXNAME_LEN - 1);
ha.primary_port = 7788;
/* Force an expired heartbeat timestamp */
clock_gettime(CLOCK_MONOTONIC, &ha.last_heartbeat);
ha.last_heartbeat.tv_sec -= 30; /* 30 s ago — far past 10 s timeout */
ha_check_heartbeat_nocluster(&ha);
TEST_ASSERT_TRUE(
atomic_load_explicit(&ha.failover_requested, memory_order_acquire) != 0);
ha_fini(&ha);
printf("test_p3_heartbeat_timeout_sets_failover_flag: PASS\n");
}
/* ================================================================
* P3-2: Redirect state-machine corner cases
@ -478,7 +454,6 @@ void test_p3_hardening(void)
/* P3-1: Failover state machine */
printf("-- P3-1: Failover state machine --\n");
RUN_TEST(test_p3_promote_demote_cycle_consistent_role);
RUN_TEST(test_p3_heartbeat_timeout_sets_failover_flag);
/* P3-2: Redirect corner cases */
printf("-- P3-2: Redirect --\n");