chore: checkpoint all prior work for clean state

Captures all uncommitted work from ha-config-bootstrap,
ha-heartbeat-liveness, and associated spec/tooling updates:

- src: ha.c, ha.h, cluster.c, config_repl.c, myapp.c, raft.c —
  heartbeat liveness, RDHUP integration, committing phase fixes
- tests: test_ha_role.c, test_config_repl.c, test_ha_rdhup_integration.c,
  test_p1_integration.c, test_p3_hardening.c, scheduler test
- specs: election-iomgr-refactor, ha-heartbeat-liveness,
  ha-peer-connection-reuse, raft-library (new); ha-config-bootstrap
  and election-iomgr-refactor task/research updates
- tooling: .wolf/ state, ha/data cluster fixtures, raft/raft.tla,
  src/build symlink, xce-build skill update
This commit is contained in:
Rongsong Shen 2026-06-04 13:04:16 +00:00
parent 15bd291bdd
commit e31673dd88
66 changed files with 122422 additions and 860 deletions

View file

@ -0,0 +1,440 @@
/*
* test_ha_rdhup_integration.c — Task 6.4 integration test: RDHUP failover
* end-to-end role transition.
*
* Tests verify the full RDHUP → CTRL_ELECTING → CTRL_PRIMARY_COMMITTING →
* CTRL_PRIMARY state-machine chain using XCE_TESTING hook seams. No live
* sockets or real cluster_t required.
*
* Requirements: 6.3, 6.5, 9.2, 9.3 (Req numbers from tasks.md 5.2 / 6.4)
*
* Hook seams used:
* ha_rdhup_election_hook — intercepts ha_start_election_async in RDHUP path
* ha_phase2_hook — stubs cluster_init_phase2 (already done = 1)
* ha_push_config_hook — stubs push_config_to_standbys
* ha_send_ctrlhello_hook — stubs ha_send_ctrlhello_to_all_standbys
*
* TDD: tests written first (RED); implementation verified GREEN.
*/
#define _GNU_SOURCE
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <unistd.h>
#include <unity/unity.h>
#include "ha.h"
#include "myconf.h"
#include "xce/cluster_def.h"
/* ── XCE_TESTING hook seam externs ──────────────────────────────────────── */
extern int (*ha_phase2_hook)(void *c);
extern int (*ha_push_config_hook)(void *c);
extern int (*ha_send_ctrlhello_hook)(void *c);
extern void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c,
int n_peers, xce_candidate_t **peers,
uint64_t my_config_tag, uint64_t my_lsn,
const char *my_name);
/* ── Stub counters ───────────────────────────────────────────────────────── */
static int g_phase2_calls = 0;
static int g_push_config_calls = 0;
static int g_ctrlhello_calls = 0;
static int stub_phase2(void *c) { (void)c; g_phase2_calls++; return 0; }
static int stub_push_config(void *c) { (void)c; g_push_config_calls++; return 0; }
static int stub_ctrlhello(void *c) { (void)c; g_ctrlhello_calls++; return 0; }
/* ── Shared HA config ───────────────────────────────────────────────────── */
static ha_config_t g_cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.election_jitter_max_ms = 0,
.election_priority_step_ms = 500,
.committing_timeout_ms = 10000,
};
/* ── install_stubs / clear_stubs ─────────────────────────────────────────── */
static void install_stubs(void)
{
g_phase2_calls = 0;
g_push_config_calls = 0;
g_ctrlhello_calls = 0;
ha_phase2_hook = stub_phase2;
ha_push_config_hook = stub_push_config;
ha_send_ctrlhello_hook = stub_ctrlhello;
}
static void clear_stubs(void)
{
ha_rdhup_election_hook = NULL;
ha_phase2_hook = NULL;
ha_push_config_hook = NULL;
ha_send_ctrlhello_hook = NULL;
}
/* ── Inline RDHUP election hook: drives evaluate immediately ─────────────
*
* Simulates ha_start_election_async completing synchronously:
* allocates a minimal ha_elect_ctx_t with self-vote won, calls
* ha_elect_evaluate_and_signal, then frees ctx.
*
* This models the single-node shortcircuit path (Req 9.1) adapted to
* a multi-node cluster where the surviving standby immediately wins —
* the key assertion is role progression, not peer negotiation.
*/
typedef struct {
cluster_t *c;
} rdhup_hook_state_t;
static rdhup_hook_state_t g_rdhup_state;
static void rdhup_election_drives_evaluate(ha_state_t *ha, cluster_t *c,
int n_peers, xce_candidate_t **peers,
uint64_t my_config_tag,
uint64_t my_lsn,
const char *my_name)
{
(void)peers; (void)my_config_tag; (void)my_lsn; (void)my_name;
/* Allocate minimal ctx with quorum already satisfied (self-vote wins) */
int n_other = (n_peers > 1) ? n_peers - 1 : 0;
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) +
(size_t)n_other * sizeof(ctx->votes[0]));
if (!ctx) return;
ctx->n_peers = n_other;
ctx->votes_granted = 1; /* self-vote */
ctx->n_replied = n_other;
ctx->result = -1;
ctx->done_posted = 0;
ctx->c = g_rdhup_state.c;
pthread_mutex_init(&ctx->lock, NULL);
sem_init(&ctx->done, 0, 0);
ha_elect_evaluate_and_signal(ha, ctx);
pthread_mutex_destroy(&ctx->lock);
sem_destroy(&ctx->done);
free(ctx);
}
/* ── Helper: make a minimal 2-candidate cluster_t ───────────────────────── */
static void make_cluster(cluster_t *c, myconf_t *myconf, const char *my_name)
{
memset(c, 0, sizeof(*c));
memset(myconf, 0, sizeof(*myconf));
myconf->name = (char *)my_name;
myconf->peers_cfg.candidates_count = 2;
snprintf(myconf->peers_cfg.candidates[0].name,
sizeof(myconf->peers_cfg.candidates[0].name), "%s", my_name);
myconf->peers_cfg.candidates[0].port = 7001;
snprintf(myconf->peers_cfg.candidates[1].name,
sizeof(myconf->peers_cfg.candidates[1].name), "ctrl-b");
myconf->peers_cfg.candidates[1].port = 7002;
c->myconf = myconf;
}
/* ═══════════════════════════════════════════════════════════════════════════
* Test 1: RDHUP → ELECTING → COMMITTING (no standbys present → via hook chain)
*
* Req 6.3: RDHUP triggers CTRL_ELECTING + async election
* Req 9.3: RDHUP election uses ha_start_election_async (not am_i_primary_quorum)
* Req 9.2: phase2_done=1 suppresses cluster_init_phase2 on RDHUP re-election
* ═════════════════════════════════════════════════════════════════════════ */
static void test_rdhup_full_path_reaches_committing(void)
{
install_stubs();
cluster_t c;
myconf_t myconf;
make_cluster(&c, &myconf, "ctrl-a");
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
ha_do_demote(&c.ha);
c.ha.phase2_done = 1; /* surviving standby — phase2 already done */
c.ha.n_peers = 1; /* 1 peer (2-node cluster) */
/* Wire the hook that drives evaluate immediately */
g_rdhup_state.c = &c;
ha_rdhup_election_hook = rdhup_election_drives_evaluate;
/* Give RDHUP handler a socketpair fd to close */
int sv[2];
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
c.ha.primary_ctrl_fd = sv[0];
/* Fire RDHUP */
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
clear_stubs();
close(sv[1]);
/* phase2 must NOT have been called (phase2_done=1 → skip) */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_phase2_calls,
"RDHUP re-election must not call cluster_init_phase2 when phase2_done=1 (Req 9.2)");
/* Role must be CTRL_PRIMARY_COMMITTING (quorum won → committing gate) */
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, role,
"RDHUP full path must reach CTRL_PRIMARY_COMMITTING after winning election (Req 6.3)");
ha_fini(&c.ha);
printf("test_rdhup_full_path_reaches_committing: PASS (role=%d==CTRL_PRIMARY_COMMITTING)\n",
role);
}
/* ═══════════════════════════════════════════════════════════════════════════
* Test 2: RDHUP → COMMITTING → PRIMARY via ha_repl_ack_received
*
* Req 8.2: majority ack in COMMITTING → CTRL_PRIMARY
* Req 6.5: promoted standby serves traffic (dispatch gate opens)
* ═════════════════════════════════════════════════════════════════════════ */
static void test_rdhup_committing_to_primary_on_ack(void)
{
install_stubs();
cluster_t c;
myconf_t myconf;
make_cluster(&c, &myconf, "ctrl-a");
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
ha_do_demote(&c.ha);
c.ha.phase2_done = 1;
c.ha.n_peers = 1; /* 2-node cluster: quorum=1, threshold=0 */
g_rdhup_state.c = &c;
ha_rdhup_election_hook = rdhup_election_drives_evaluate;
int sv[2];
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
c.ha.primary_ctrl_fd = sv[0];
/* Fire RDHUP → should reach CTRL_PRIMARY_COMMITTING */
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
close(sv[1]);
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT((int)CTRL_PRIMARY_COMMITTING, role);
/* n_peers=1 → quorum=(1+2)/2=1 → threshold=0 → first ack promotes.
* ha_repl_ack_received increments count; acks(1) >= quorum-1(0) → promote. */
ha_repl_ack_received(&c.ha, &c);
role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
"ha_repl_ack_received must drive COMMITTING → CTRL_PRIMARY on quorum ack (Req 8.2)");
clear_stubs();
ha_fini(&c.ha);
printf("test_rdhup_committing_to_primary_on_ack: PASS (role=%d==CTRL_PRIMARY)\n", role);
}
/* ═══════════════════════════════════════════════════════════════════════════
* Test 3: RDHUP in single-node cluster → immediate PRIMARY (no COMMITTING)
*
* Req 9.1: single-node self-elects without network
* n_peers=0 → quorum=1, threshold=0 → acks(0) >= 0 → promotes immediately
* in ha_elect_evaluate_and_signal (no peers to wait for acks from)
* ═════════════════════════════════════════════════════════════════════════ */
static void test_rdhup_single_node_reaches_primary(void)
{
install_stubs();
cluster_t c;
myconf_t myconf;
make_cluster(&c, &myconf, "ctrl-a");
/* Override to single-node */
myconf.peers_cfg.candidates_count = 1;
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
ha_do_demote(&c.ha);
c.ha.phase2_done = 1;
c.ha.n_peers = 0; /* single node: no peers */
g_rdhup_state.c = &c;
ha_rdhup_election_hook = rdhup_election_drives_evaluate;
int sv[2];
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
c.ha.primary_ctrl_fd = sv[0];
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
close(sv[1]);
/* n_peers=0 → quorum=(0+2)/2=1 → threshold=0 → acks(0) >= 0 → promoted inline */
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
/* Single-node: no standbys to ack, committing_timeout degrades to PRIMARY.
* The committing timeout is 10s; for single-node we call ha_repl_ack_received
* once to simulate the degraded-mode zero-threshold promotion (threshold=0). */
if (role == (int)CTRL_PRIMARY_COMMITTING) {
ha_repl_ack_received(&c.ha, &c);
role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
}
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
"single-node RDHUP path must reach CTRL_PRIMARY (Req 9.1)");
clear_stubs();
ha_fini(&c.ha);
printf("test_rdhup_single_node_reaches_primary: PASS (role=%d==CTRL_PRIMARY)\n", role);
}
/* ═══════════════════════════════════════════════════════════════════════════
* Test 4: 3-node cluster RDHUP — two acks needed, premature ack stays COMMITTING
*
* Req 8.2: acks < quorum-1 → stays COMMITTING; acks >= quorum-1 → PRIMARY
* 3-node (n_peers=2): quorum=2, threshold=1 → need 1 ack to promote
* ═════════════════════════════════════════════════════════════════════════ */
static void test_rdhup_3node_one_ack_promotes(void)
{
install_stubs();
cluster_t c;
myconf_t myconf;
make_cluster(&c, &myconf, "ctrl-a");
myconf.peers_cfg.candidates_count = 3;
snprintf(myconf.peers_cfg.candidates[2].name,
sizeof(myconf.peers_cfg.candidates[2].name), "ctrl-c");
myconf.peers_cfg.candidates[2].port = 7003;
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
ha_do_demote(&c.ha);
c.ha.phase2_done = 1;
c.ha.n_peers = 2; /* 3-node: quorum=2, threshold=1 */
g_rdhup_state.c = &c;
ha_rdhup_election_hook = rdhup_election_drives_evaluate;
int sv[2];
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
c.ha.primary_ctrl_fd = sv[0];
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
close(sv[1]);
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, role,
"3-node RDHUP must enter CTRL_PRIMARY_COMMITTING before acks");
/* First ack: count=1 >= threshold=1 → promotes */
ha_repl_ack_received(&c.ha, &c);
role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
"3-node: first ack (count=1 >= threshold=1) must promote to CTRL_PRIMARY (Req 8.2)");
clear_stubs();
ha_fini(&c.ha);
printf("test_rdhup_3node_one_ack_promotes: PASS (role=%d==CTRL_PRIMARY)\n", role);
}
/* ═══════════════════════════════════════════════════════════════════════════
* Test 5: config push and ctrlhello hooks fire during COMMITTING entry
*
* Req 7.1: push_config_to_standbys called after winning election
* Req 7.2: ha_send_ctrlhello_to_all_standbys called after winning election
* ═════════════════════════════════════════════════════════════════════════ */
static void test_rdhup_committing_fires_replication_hooks(void)
{
install_stubs();
cluster_t c;
myconf_t myconf;
make_cluster(&c, &myconf, "ctrl-a");
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
ha_do_demote(&c.ha);
c.ha.phase2_done = 1;
c.ha.n_peers = 1;
g_rdhup_state.c = &c;
ha_rdhup_election_hook = rdhup_election_drives_evaluate;
int sv[2];
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
c.ha.primary_ctrl_fd = sv[0];
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
close(sv[1]);
TEST_ASSERT_GREATER_THAN_INT_MESSAGE(0, g_push_config_calls,
"push_config_to_standbys hook must fire when entering COMMITTING (Req 7.1)");
TEST_ASSERT_GREATER_THAN_INT_MESSAGE(0, g_ctrlhello_calls,
"ha_send_ctrlhello hook must fire when entering COMMITTING (Req 7.2)");
clear_stubs();
ha_fini(&c.ha);
printf("test_rdhup_committing_fires_replication_hooks: PASS "
"(push=%d, ctrlhello=%d)\n",
g_push_config_calls, g_ctrlhello_calls);
}
/* ═══════════════════════════════════════════════════════════════════════════
* Test 6: dispatch gate opens after CTRL_PRIMARY (job ops allowed)
*
* Req 6.5: after RDHUP promotion, primary accepts job dispatch (Req 5.3)
* ═════════════════════════════════════════════════════════════════════════ */
static void test_rdhup_primary_dispatch_gate_opens(void)
{
install_stubs();
cluster_t c;
myconf_t myconf;
make_cluster(&c, &myconf, "ctrl-a");
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
ha_do_demote(&c.ha);
c.ha.phase2_done = 1;
c.ha.n_peers = 1;
g_rdhup_state.c = &c;
ha_rdhup_election_hook = rdhup_election_drives_evaluate;
int sv[2];
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
c.ha.primary_ctrl_fd = sv[0];
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
close(sv[1]);
/* Drive to CTRL_PRIMARY via ack */
ha_repl_ack_received(&c.ha, &c);
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT((int)CTRL_PRIMARY, role);
/* Verify dispatch gate allows job operations in CTRL_PRIMARY */
int rc = ha_dispatch_gate(&c.ha, OPCode_jobsub);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"OPCode_jobsub must be allowed by dispatch gate in CTRL_PRIMARY (Req 5.3)");
rc = ha_dispatch_gate(&c.ha, OPCode_jobdispatch);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"OPCode_jobdispatch must be allowed by dispatch gate in CTRL_PRIMARY (Req 5.3)");
clear_stubs();
ha_fini(&c.ha);
printf("test_rdhup_primary_dispatch_gate_opens: PASS (gate accepts job ops)\n");
}
/* ── Suite entry point ──────────────────────────────────────────────────── */
void test_ha_rdhup_integration(void)
{
RUN_TEST(test_rdhup_full_path_reaches_committing);
RUN_TEST(test_rdhup_committing_to_primary_on_ack);
RUN_TEST(test_rdhup_single_node_reaches_primary);
RUN_TEST(test_rdhup_3node_one_ack_promotes);
RUN_TEST(test_rdhup_committing_fires_replication_hooks);
RUN_TEST(test_rdhup_primary_dispatch_gate_opens);
}