/* * test_ha_rdhup_integration.c — Task 6.4 integration test: RDHUP failover * end-to-end role transition. * * Tests verify the full RDHUP → CTRL_ELECTING → CTRL_PRIMARY state-machine * chain (the election winner becomes primary directly — single leader role, * ha-single-leader-role) using XCE_TESTING hook seams. No live sockets or * real cluster_t required. * * Requirements: 6.3, 6.5, 9.2, 9.3 (Req numbers from tasks.md 5.2 / 6.4) * * Hook seams used: * ha_rdhup_election_hook — intercepts ha_start_election_async in RDHUP path * ha_phase2_hook — stubs cluster_init_phase2 (already done = 1) * ha_push_config_hook — stubs push_config_to_standbys * ha_send_ctrlhello_hook — stubs ha_send_ctrlhello_to_all_standbys * * TDD: tests written first (RED); implementation verified GREEN. */ #define _GNU_SOURCE #include #include #include #include #include #include #include #include #include "ha.h" #include "myconf.h" #include "xce/cluster_def.h" /* ── XCE_TESTING hook seam externs ──────────────────────────────────────── */ extern int (*ha_phase2_hook)(void *c); extern int (*ha_push_config_hook)(void *c); extern int (*ha_send_ctrlhello_hook)(void *c); extern void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c, int n_peers, xce_candidate_t **peers, uint64_t my_config_tag, uint64_t my_lsn, const char *my_name); /* ── Stub counters ───────────────────────────────────────────────────────── */ static int g_phase2_calls = 0; static int g_push_config_calls = 0; static int g_ctrlhello_calls = 0; static int stub_phase2(void *c) { (void)c; g_phase2_calls++; return 0; } static int stub_push_config(void *c) { (void)c; g_push_config_calls++; return 0; } static int stub_ctrlhello(void *c) { (void)c; g_ctrlhello_calls++; return 0; } /* ha-single-leader-role task 4.1: the replication-ack handler was deleted (the * ack-counting force-promotion machinery is gone — promotion is driven solely * by the heartbeat decision loop + per-entry majority ack). No shim is added * (design §"Transitional-State Deletions": no replacements, no shims). The * ack-driven-promotion cases were deleted; the surviving state-entry cases no * longer call it (production ha_elect_evaluate_and_signal now lands the winner * directly per task 5.1 territory). */ /* A real (nonzero) config tag, so a node is treated as CONFIGURED and is * allowed to start an RDHUP re-election. The no-config voter-only guard in * ha_on_primary_ctrl_fd_rdhup only trips for config_version_tag == 0 with * candidates_count > 1 (ha-config-bootstrap Req 2.8 / ha-repl-hardening Req 10.5). */ #define TEST_CONFIG_TAG 0x1234 /* ── Shared HA config ───────────────────────────────────────────────────── */ static ha_config_t g_cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, .election_jitter_max_ms = 0, .election_priority_step_ms = 500, }; /* ── install_stubs / clear_stubs ─────────────────────────────────────────── */ static void install_stubs(void) { g_phase2_calls = 0; g_push_config_calls = 0; g_ctrlhello_calls = 0; ha_phase2_hook = stub_phase2; ha_push_config_hook = stub_push_config; ha_send_ctrlhello_hook = stub_ctrlhello; } static void clear_stubs(void) { ha_rdhup_election_hook = NULL; ha_phase2_hook = NULL; ha_push_config_hook = NULL; ha_send_ctrlhello_hook = NULL; } /* ── Inline RDHUP election hook: drives evaluate immediately ───────────── * * Simulates ha_start_election_async completing synchronously: * allocates a minimal ha_elect_ctx_t with self-vote won, calls * ha_elect_evaluate_and_signal, then frees ctx. * * This models the single-node shortcircuit path (Req 9.1) adapted to * a multi-node cluster where the surviving standby immediately wins — * the key assertion is role progression, not peer negotiation. */ typedef struct { cluster_t *c; } rdhup_hook_state_t; static rdhup_hook_state_t g_rdhup_state; static void rdhup_election_drives_evaluate(ha_state_t *ha, cluster_t *c, int n_peers, xce_candidate_t **peers, uint64_t my_config_tag, uint64_t my_lsn, const char *my_name) { (void)peers; (void)my_config_tag; (void)my_lsn; (void)my_name; /* Allocate minimal ctx with quorum already satisfied (self-vote wins) */ int n_other = (n_peers > 1) ? n_peers - 1 : 0; ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + (size_t)n_other * sizeof(ctx->votes[0])); if (!ctx) return; ctx->n_peers = n_other; ctx->votes_granted = 1; /* self-vote */ ctx->n_replied = n_other; ctx->result = -1; ctx->done_posted = 0; ctx->c = g_rdhup_state.c; pthread_mutex_init(&ctx->lock, NULL); sem_init(&ctx->done, 0, 0); ha_elect_evaluate_and_signal(ha, ctx); pthread_mutex_destroy(&ctx->lock); sem_destroy(&ctx->done); free(ctx); } /* ── Helper: make a minimal 2-candidate cluster_t ───────────────────────── */ static void make_cluster(cluster_t *c, myconf_t *myconf, const char *my_name) { memset(c, 0, sizeof(*c)); memset(myconf, 0, sizeof(*myconf)); myconf->name = (char *)my_name; /* Configured node by default: a real (nonzero) tag so the no-config * voter-only RDHUP guard does NOT trip — these fixtures exercise the * legitimate "a CONFIGURED node re-elects after RDHUP" path. The * config-less guard is exercised by its own dedicated test below. */ myconf->config_version_tag = TEST_CONFIG_TAG; myconf->peers_cfg.candidates_count = 2; snprintf(myconf->peers_cfg.candidates[0].name, sizeof(myconf->peers_cfg.candidates[0].name), "%s", my_name); myconf->peers_cfg.candidates[0].port = 7001; snprintf(myconf->peers_cfg.candidates[1].name, sizeof(myconf->peers_cfg.candidates[1].name), "ctrl-b"); myconf->peers_cfg.candidates[1].port = 7002; c->myconf = myconf; } /* ═══════════════════════════════════════════════════════════════════════════ * Test 1: RDHUP → ELECTING → PRIMARY (no standbys present → via hook chain) * * Req 6.3: RDHUP triggers CTRL_ELECTING + async election * Req 9.3: RDHUP election uses ha_start_election_async (not am_i_primary_quorum) * Req 9.2: phase2_done=1 suppresses cluster_init_phase2 on RDHUP re-election * Req 1.1: the election winner becomes CTRL_PRIMARY directly (single leader role) * ═════════════════════════════════════════════════════════════════════════ */ static void test_rdhup_full_path_reaches_primary(void) { install_stubs(); cluster_t c; myconf_t myconf; make_cluster(&c, &myconf, "ctrl-a"); TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0)); ha_do_demote(&c.ha); c.ha.phase2_done = 1; /* surviving standby — phase2 already done */ c.ha.n_peers = 1; /* 1 peer (2-node cluster) */ /* Wire the hook that drives evaluate immediately */ g_rdhup_state.c = &c; ha_rdhup_election_hook = rdhup_election_drives_evaluate; /* Give RDHUP handler a socketpair fd to close */ int sv[2]; TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); c.ha.primary_ctrl_fd = sv[0]; /* Fire RDHUP */ ha_on_primary_ctrl_fd_rdhup(&c.ha, &c); clear_stubs(); close(sv[1]); /* phase2 must NOT have been called (phase2_done=1 → skip) */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_phase2_calls, "RDHUP re-election must not call cluster_init_phase2 when phase2_done=1 (Req 9.2)"); /* Role must be CTRL_PRIMARY directly — the election winner is the primary * (single leader role; no transitional sub-state, Req 1.1). */ int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role, "RDHUP full path must reach CTRL_PRIMARY after winning election (Req 6.3, 1.1)"); ha_fini(&c.ha); printf("test_rdhup_full_path_reaches_primary: PASS (role=%d==CTRL_PRIMARY)\n", role); } /* ═══════════════════════════════════════════════════════════════════════════ * Test 2 (RDHUP → PRIMARY via replication-ack handler) DELETED — ha-single-leader-role * task 4.1. Subject was the deleted ack-driven force-promotion machinery * (the replication-ack handler); nothing left to verify. No shim, no retarget * (design §"Transitional-State Deletions"). * ═════════════════════════════════════════════════════════════════════════ */ /* ═══════════════════════════════════════════════════════════════════════════ * Test 3: RDHUP in single-node cluster → immediate PRIMARY (single leader role) * * Req 9.1: single-node self-elects without network * n_peers=0 → quorum=1, threshold=0 → acks(0) >= 0 → promotes immediately * in ha_elect_evaluate_and_signal (no peers to wait for acks from) * ═════════════════════════════════════════════════════════════════════════ */ static void test_rdhup_single_node_reaches_primary(void) { install_stubs(); cluster_t c; myconf_t myconf; make_cluster(&c, &myconf, "ctrl-a"); /* Override to single-node */ myconf.peers_cfg.candidates_count = 1; TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0)); ha_do_demote(&c.ha); c.ha.phase2_done = 1; c.ha.n_peers = 0; /* single node: no peers */ g_rdhup_state.c = &c; ha_rdhup_election_hook = rdhup_election_drives_evaluate; int sv[2]; TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); c.ha.primary_ctrl_fd = sv[0]; ha_on_primary_ctrl_fd_rdhup(&c.ha, &c); close(sv[1]); /* n_peers=0 → ha_elect_evaluate_and_signal single-node shortcircuit lands * CTRL_PRIMARY directly (no transitional leader sub-state, no ack machinery — * ha-single-leader-role). The old replication-ack fallback was excised * (function deleted task 4.1); the PRIMARY assertion stands on its own. */ int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role, "single-node RDHUP path must reach CTRL_PRIMARY (Req 9.1)"); clear_stubs(); ha_fini(&c.ha); printf("test_rdhup_single_node_reaches_primary: PASS (role=%d==CTRL_PRIMARY)\n", role); } /* ═══════════════════════════════════════════════════════════════════════════ * Test 4 (3-node RDHUP — ack-driven promotion) DELETED — ha-single-leader-role * task 4.1. Subject was the deleted ack-counting promotion machinery * (the replication-ack handler promoted a transitional leader state to PRIMARY * on threshold acks); the mechanism is gone, nothing left to verify. No shim * (design §"Transitional-State Deletions"). * ═════════════════════════════════════════════════════════════════════════ */ /* ═══════════════════════════════════════════════════════════════════════════ * Test 5: config push and ctrlhello hooks fire on promotion to primary * * Req 7.1: push_config_to_standbys called after winning election * Req 7.2: ha_send_ctrlhello_to_all_standbys called after winning election * ═════════════════════════════════════════════════════════════════════════ */ static void test_rdhup_promotion_fires_replication_hooks(void) { install_stubs(); cluster_t c; myconf_t myconf; make_cluster(&c, &myconf, "ctrl-a"); TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0)); ha_do_demote(&c.ha); c.ha.phase2_done = 1; c.ha.n_peers = 1; g_rdhup_state.c = &c; ha_rdhup_election_hook = rdhup_election_drives_evaluate; int sv[2]; TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); c.ha.primary_ctrl_fd = sv[0]; ha_on_primary_ctrl_fd_rdhup(&c.ha, &c); close(sv[1]); TEST_ASSERT_GREATER_THAN_INT_MESSAGE(0, g_push_config_calls, "push_config_to_standbys hook must fire on promotion to primary (Req 7.1)"); TEST_ASSERT_GREATER_THAN_INT_MESSAGE(0, g_ctrlhello_calls, "ha_send_ctrlhello hook must fire on promotion to primary (Req 7.2)"); clear_stubs(); ha_fini(&c.ha); printf("test_rdhup_promotion_fires_replication_hooks: PASS " "(push=%d, ctrlhello=%d)\n", g_push_config_calls, g_ctrlhello_calls); } /* ═══════════════════════════════════════════════════════════════════════════ * Test 6: dispatch gate opens after CTRL_PRIMARY (job ops allowed) * * Req 6.5: after RDHUP promotion, primary accepts job dispatch (Req 5.3) * ═════════════════════════════════════════════════════════════════════════ */ static void test_rdhup_primary_dispatch_gate_opens(void) { static int dummy_ep; /* stands in for the ACKing standby's endpoint */ install_stubs(); cluster_t c; myconf_t myconf; make_cluster(&c, &myconf, "ctrl-a"); TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0)); ha_do_demote(&c.ha); c.ha.phase2_done = 1; c.ha.n_peers = 1; /* FIXTURE FIX (ha-commit-engine task 6.1, bug-411 baseline failure): * ha_on_primary_ctrl_fd_rdhup re-derives ha->n_peers from * myconf->peers_cfg.candidates_count (= 2 here), so the CTRL_PRIMARY * dispatch gate requires write quorum: >= 1 connected standby. The * ack below models a standby that connected and ACKed; in production * it lives in ha->standby_endpoints (raft_cb_become_leader / * on_standby_connected). Register it so quorum legitimately holds on * this healthy primary — the gate itself must NOT be weakened. */ c.ha.standby_endpoints = calloc(1, sizeof(zarray_t)); TEST_ASSERT_NOT_NULL(c.ha.standby_endpoints); zarray_init(c.ha.standby_endpoints, 4, NULL); zarray_store(c.ha.standby_endpoints, &dummy_ep); g_rdhup_state.c = &c; ha_rdhup_election_hook = rdhup_election_drives_evaluate; int sv[2]; TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); c.ha.primary_ctrl_fd = sv[0]; ha_on_primary_ctrl_fd_rdhup(&c.ha, &c); close(sv[1]); /* ha-single-leader-role: ha_elect_evaluate_and_signal lands the winner in * CTRL_PRIMARY directly (no transitional leader sub-state, no ack machinery). * The old replication-ack drive-step was excised (function deleted task 4.1); * the dispatch-gate assertion below is the subject of this case. */ int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire); TEST_ASSERT_EQUAL_INT((int)CTRL_PRIMARY, role); /* Verify dispatch gate allows job operations in CTRL_PRIMARY */ int rc = ha_dispatch_gate(&c.ha, OPCode_jobsub); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "OPCode_jobsub must be allowed by dispatch gate in CTRL_PRIMARY (Req 5.3)"); rc = ha_dispatch_gate(&c.ha, OPCode_jobdispatch); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "OPCode_jobdispatch must be allowed by dispatch gate in CTRL_PRIMARY (Req 5.3)"); clear_stubs(); zarray_fini(c.ha.standby_endpoints); free(c.ha.standby_endpoints); c.ha.standby_endpoints = NULL; ha_fini(&c.ha); printf("test_rdhup_primary_dispatch_gate_opens: PASS (gate accepts job ops)\n"); } /* ═══════════════════════════════════════════════════════════════════════════ * Test 7: config-less node (tag=0) with peers MUST NOT start an RDHUP re-election * * Election-SAFETY (ha-config-bootstrap Req 2.8 / ha-repl-hardening Req 10.5): * a node with no local config (config_version_tag == 0) and >1 candidates must * NOT win an election while a configured peer exists. ha_on_primary_ctrl_fd_rdhup * must mirror the ha_start_election_async voter-only guard: skip the elect_ctx * alloc + the election trigger entirely and stay a passive voter (CTRL_STANDBY). * * Asserted via the ha_rdhup_election_hook seam: the hook must NOT fire (count==0), * and role must be CTRL_STANDBY (not CTRL_ELECTING / CTRL_PRIMARY). * Before the fix this FAILS: the hook fires and the election proceeds. * ═════════════════════════════════════════════════════════════════════════ */ static int g_guard_hook_calls = 0; static void counting_election_hook(ha_state_t *ha, cluster_t *c, int n_peers, xce_candidate_t **peers, uint64_t my_config_tag, uint64_t my_lsn, const char *my_name) { (void)ha; (void)c; (void)n_peers; (void)peers; (void)my_config_tag; (void)my_lsn; (void)my_name; g_guard_hook_calls++; } static void test_rdhup_noconfig_node_does_not_elect(void) { install_stubs(); g_guard_hook_calls = 0; cluster_t c; myconf_t myconf; make_cluster(&c, &myconf, "ctrl-a"); /* Config-less node: no local config tag, but part of a >1 node cluster. */ myconf.config_version_tag = 0; myconf.peers_cfg.candidates_count = 2; TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0)); ha_do_demote(&c.ha); c.ha.phase2_done = 1; c.ha.n_peers = 1; ha_rdhup_election_hook = counting_election_hook; int sv[2]; TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); c.ha.primary_ctrl_fd = sv[0]; ha_on_primary_ctrl_fd_rdhup(&c.ha, &c); clear_stubs(); close(sv[1]); /* The election hook must NOT have fired — no re-election started. */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_guard_hook_calls, "config-less node (tag=0, peers>1) must NOT start an RDHUP re-election " "(ha-config-bootstrap Req 2.8)"); /* Role must be CTRL_STANDBY — a passive voter awaiting the configured primary. */ int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY, role, "config-less RDHUP node must stay CTRL_STANDBY (voter only), not ELECTING/PRIMARY"); ha_fini(&c.ha); printf("test_rdhup_noconfig_node_does_not_elect: PASS " "(hook_calls=%d, role=%d==CTRL_STANDBY)\n", g_guard_hook_calls, role); } /* ═══════════════════════════════════════════════════════════════════════════ * Test 8 (contrast / non-regression): a CONFIGURED node (tag!=0) with peers * MUST still start the RDHUP re-election. * * Same shape as Test 7 but with a real config tag — the guard must NOT trip. * The election hook MUST fire exactly once. * ═════════════════════════════════════════════════════════════════════════ */ static void test_rdhup_configured_node_still_elects(void) { install_stubs(); g_guard_hook_calls = 0; cluster_t c; myconf_t myconf; make_cluster(&c, &myconf, "ctrl-a"); /* Configured node: real tag, >1 candidates — guard must NOT trip. */ myconf.config_version_tag = TEST_CONFIG_TAG; myconf.peers_cfg.candidates_count = 2; TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0)); ha_do_demote(&c.ha); c.ha.phase2_done = 1; c.ha.n_peers = 1; ha_rdhup_election_hook = counting_election_hook; int sv[2]; TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); c.ha.primary_ctrl_fd = sv[0]; ha_on_primary_ctrl_fd_rdhup(&c.ha, &c); clear_stubs(); close(sv[1]); /* Configured node: the election hook MUST fire (re-election proceeds). */ TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_guard_hook_calls, "configured node (tag!=0, peers>1) must still start RDHUP re-election"); ha_fini(&c.ha); printf("test_rdhup_configured_node_still_elects: PASS (hook_calls=%d)\n", g_guard_hook_calls); } /* ═══════════════════════════════════════════════════════════════════════════ * Test 9 (regression — infinite election loop after primary death): * The RDHUP candidate must advertise its EFFECTIVE freshness floor * (max(applied_lsn, last_known_committed_lsn)) as the RequestVote last_log_lsn, * NOT the stale persist.committed_lsn. * * Root cause of the observed bug: a standby never advances * persist.committed_lsn (only the primary's commit engine writes it); it * advances applied_lsn (WAL catch-up) and last_known_committed_lsn (heartbeat * replies). The voter-side freshness floor (cluster.c OPCode_requestVote) is * max(applied_lsn, last_known_committed_lsn). When the RDHUP candidate * advertised persist.committed_lsn (stale-low) while every voter compared * against a high applied_lsn floor, the freshness gate (cand_lsn < my_lsn) * tripped on BOTH surviving standbys — each denied the other's vote, no * candidate ever reached quorum, and the cluster spun in an infinite * RDHUP→ELECTING→STANDBY loop with terms climbing in lockstep. * * Captured via the ha_rdhup_election_hook my_lsn argument. * Before the fix: captured == 0 (stale committed_lsn). After: == 10. * ═════════════════════════════════════════════════════════════════════════ */ static uint64_t g_captured_my_lsn = 0; static uint64_t g_captured_my_tag = 0; static void capture_lsn_hook(ha_state_t *ha, cluster_t *c, int n_peers, xce_candidate_t **peers, uint64_t my_config_tag, uint64_t my_lsn, const char *my_name) { (void)ha; (void)c; (void)n_peers; (void)peers; (void)my_name; g_captured_my_tag = my_config_tag; g_captured_my_lsn = my_lsn; } static void test_rdhup_candidate_advertises_freshness_floor_not_stale_committed(void) { install_stubs(); g_captured_my_lsn = 0; g_captured_my_tag = 0; cluster_t c; myconf_t myconf; make_cluster(&c, &myconf, "ctrl-a"); TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0)); ha_do_demote(&c.ha); c.ha.phase2_done = 1; c.ha.n_peers = 1; /* Standby state after applying WAL + learning commits from the (now-dead) * primary: applied_lsn and last_known_committed_lsn advanced to 10, but * persist.committed_lsn is still 0 — a standby never advances it. * * ha-commit-durability task 4.2: the candidate now advertises the BARE * DURABLE WAL tip (persist_durable_floor), not the folded * max(applied_lsn, learned_committed). Model the standby holding its log * durably up to LSN 10 by priming the durable watermark — this is the * durable-backed analog of the original "advertise the freshness floor, not * the stale committed_lsn" guarantee (still NOT 0). */ atomic_store_explicit(&c.ha.applied_lsn, 10ULL, memory_order_release); atomic_store_explicit(&c.ha.last_known_committed_lsn, 10ULL, memory_order_release); atomic_store_explicit(&c.persist.committed_lsn, 0ULL, memory_order_release); atomic_store_explicit(&c.persist.durable_lsn, 10ULL, memory_order_release); ha_rdhup_election_hook = capture_lsn_hook; int sv[2]; TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); c.ha.primary_ctrl_fd = sv[0]; ha_on_primary_ctrl_fd_rdhup(&c.ha, &c); clear_stubs(); close(sv[1]); TEST_ASSERT_EQUAL_UINT64_MESSAGE(10ULL, g_captured_my_lsn, "RDHUP candidate must advertise its DURABLE WAL tip (durable_floor = 10), " "not the stale persist.committed_lsn (0); otherwise surviving standbys " "mutually deny votes -> infinite election loop (bug-339; task 4.2 makes " "this the durable tip, not max(applied,learned))"); TEST_ASSERT_EQUAL_UINT64_MESSAGE((uint64_t)TEST_CONFIG_TAG, g_captured_my_tag, "RDHUP candidate must advertise its config_version_tag"); ha_fini(&c.ha); printf("test_rdhup_candidate_advertises_freshness_floor_not_stale_committed: " "PASS (my_lsn=%llu)\n", (unsigned long long)g_captured_my_lsn); } /* ── Suite entry point ──────────────────────────────────────────────────── */ /* ═══════════════════════════════════════════════════════════════════════════ * 5.10b: an RDHUP arriving within the failover cooldown of a prior trigger must * be DEBOUNCED — no fresh failover/election, no term ladder. The round-4 soak * storm (48 begin_failovers, terms 6→24) came from a dead primary's fd RDHUPing * faster than an election settles; each RDHUP that landed after a lost election * (role briefly back at CTRL_STANDBY) re-triggered a failover. The single-node * fixture normally reaches CTRL_PRIMARY on RDHUP (see * test_rdhup_single_node_reaches_primary — that PASS is the "not within * cooldown → still elects" pin). With a RECENT last_failover_ms and a stale * heartbeat (so the 5.2 alive-suppression does not fire), the node must stay * CTRL_STANDBY. Pre-fix (no debounce) this reached CTRL_PRIMARY → RED. * ═════════════════════════════════════════════════════════════════════════ */ static void test_rdhup_within_cooldown_is_debounced(void) { install_stubs(); cluster_t c; myconf_t myconf; make_cluster(&c, &myconf, "ctrl-a"); myconf.peers_cfg.candidates_count = 1; /* single node → would self-elect */ TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0)); ha_do_demote(&c.ha); c.ha.phase2_done = 1; c.ha.n_peers = 0; c.ha.last_activity = 0; /* stale heartbeat → 5.2 alive-suppression will NOT fire */ g_rdhup_state.c = &c; ha_rdhup_election_hook = rdhup_election_drives_evaluate; /* A failover was triggered "just now" → this RDHUP is within the cooldown. */ struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); uint64_t now_ms = (uint64_t)ts.tv_sec * 1000u + (uint64_t)(ts.tv_nsec / 1000000L); atomic_store_explicit(&c.ha.last_failover_ms, now_ms, memory_order_release); int sv[2]; TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); c.ha.primary_ctrl_fd = sv[0]; ha_on_primary_ctrl_fd_rdhup(&c.ha, &c); close(sv[1]); int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY, role, "an RDHUP within the failover cooldown must be debounced — no fresh " "failover; node stays CTRL_STANDBY (5.10b RDHUP storm)"); TEST_ASSERT_EQUAL_INT_MESSAGE(-1, c.ha.primary_ctrl_fd, "the debounced RDHUP must still close the dead primary_ctrl_fd"); clear_stubs(); ha_fini(&c.ha); printf("test_rdhup_within_cooldown_is_debounced: PASS\n"); } void test_ha_rdhup_integration(void) { RUN_TEST(test_rdhup_candidate_advertises_freshness_floor_not_stale_committed); RUN_TEST(test_rdhup_within_cooldown_is_debounced); RUN_TEST(test_rdhup_full_path_reaches_primary); /* ack-driven promotion test deleted (ha-single-leader-role 4.1: * ack-driven promotion machinery removed, no shim) */ RUN_TEST(test_rdhup_single_node_reaches_primary); /* test_rdhup_3node_one_ack_promotes deleted (ha-single-leader-role 4.1: * ack-counting promotion machinery removed, no shim) */ RUN_TEST(test_rdhup_promotion_fires_replication_hooks); RUN_TEST(test_rdhup_primary_dispatch_gate_opens); /* Election-safety guard: config-less node must not re-elect; configured * node still does (ha-config-bootstrap Req 2.8 / ha-repl-hardening Req 10.5). */ RUN_TEST(test_rdhup_noconfig_node_does_not_elect); RUN_TEST(test_rdhup_configured_node_still_elects); }