604 lines
28 KiB
C
604 lines
28 KiB
C
/*
|
|
* test_ha_rdhup_integration.c — Task 6.4 integration test: RDHUP failover
|
|
* end-to-end role transition.
|
|
*
|
|
* Tests verify the full RDHUP → CTRL_ELECTING → CTRL_PRIMARY state-machine
|
|
* chain (the election winner becomes primary directly — single leader role,
|
|
* ha-single-leader-role) using XCE_TESTING hook seams. No live sockets or
|
|
* real cluster_t required.
|
|
*
|
|
* Requirements: 6.3, 6.5, 9.2, 9.3 (Req numbers from tasks.md 5.2 / 6.4)
|
|
*
|
|
* Hook seams used:
|
|
* ha_rdhup_election_hook — intercepts ha_start_election_async in RDHUP path
|
|
* ha_phase2_hook — stubs cluster_init_phase2 (already done = 1)
|
|
* ha_push_config_hook — stubs push_config_to_standbys
|
|
* ha_send_ctrlhello_hook — stubs ha_send_ctrlhello_to_all_standbys
|
|
*
|
|
* TDD: tests written first (RED); implementation verified GREEN.
|
|
*/
|
|
#define _GNU_SOURCE
|
|
#include <stdatomic.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <sys/socket.h>
|
|
#include <unistd.h>
|
|
|
|
#include <unity/unity.h>
|
|
|
|
#include "ha.h"
|
|
#include "myconf.h"
|
|
#include "xce/cluster_def.h"
|
|
|
|
/* ── XCE_TESTING hook seam externs ──────────────────────────────────────── */
|
|
|
|
extern int (*ha_phase2_hook)(void *c);
|
|
extern int (*ha_push_config_hook)(void *c);
|
|
extern int (*ha_send_ctrlhello_hook)(void *c);
|
|
extern void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c,
|
|
int n_peers, xce_candidate_t **peers,
|
|
uint64_t my_config_tag, uint64_t my_lsn,
|
|
const char *my_name);
|
|
|
|
/* ── Stub counters ───────────────────────────────────────────────────────── */
|
|
|
|
static int g_phase2_calls = 0;
|
|
static int g_push_config_calls = 0;
|
|
static int g_ctrlhello_calls = 0;
|
|
|
|
static int stub_phase2(void *c) { (void)c; g_phase2_calls++; return 0; }
|
|
static int stub_push_config(void *c) { (void)c; g_push_config_calls++; return 0; }
|
|
static int stub_ctrlhello(void *c) { (void)c; g_ctrlhello_calls++; return 0; }
|
|
|
|
/* ha-single-leader-role task 4.1: the replication-ack handler was deleted (the
|
|
* ack-counting force-promotion machinery is gone — promotion is driven solely
|
|
* by the heartbeat decision loop + per-entry majority ack). No shim is added
|
|
* (design §"Transitional-State Deletions": no replacements, no shims). The
|
|
* ack-driven-promotion cases were deleted; the surviving state-entry cases no
|
|
* longer call it (production ha_elect_evaluate_and_signal now lands the winner
|
|
* directly per task 5.1 territory). */
|
|
|
|
/* A real (nonzero) config tag, so a node is treated as CONFIGURED and is
|
|
* allowed to start an RDHUP re-election. The no-config voter-only guard in
|
|
* ha_on_primary_ctrl_fd_rdhup only trips for config_version_tag == 0 with
|
|
* candidates_count > 1 (ha-config-bootstrap Req 2.8 / ha-repl-hardening Req 10.5). */
|
|
#define TEST_CONFIG_TAG 0x1234
|
|
|
|
/* ── Shared HA config ───────────────────────────────────────────────────── */
|
|
|
|
static ha_config_t g_cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
.election_jitter_max_ms = 0,
|
|
.election_priority_step_ms = 500,
|
|
};
|
|
|
|
/* ── install_stubs / clear_stubs ─────────────────────────────────────────── */
|
|
|
|
static void install_stubs(void)
|
|
{
|
|
g_phase2_calls = 0;
|
|
g_push_config_calls = 0;
|
|
g_ctrlhello_calls = 0;
|
|
ha_phase2_hook = stub_phase2;
|
|
ha_push_config_hook = stub_push_config;
|
|
ha_send_ctrlhello_hook = stub_ctrlhello;
|
|
}
|
|
|
|
static void clear_stubs(void)
|
|
{
|
|
ha_rdhup_election_hook = NULL;
|
|
ha_phase2_hook = NULL;
|
|
ha_push_config_hook = NULL;
|
|
ha_send_ctrlhello_hook = NULL;
|
|
}
|
|
|
|
/* ── Inline RDHUP election hook: drives evaluate immediately ─────────────
|
|
*
|
|
* Simulates ha_start_election_async completing synchronously:
|
|
* allocates a minimal ha_elect_ctx_t with self-vote won, calls
|
|
* ha_elect_evaluate_and_signal, then frees ctx.
|
|
*
|
|
* This models the single-node shortcircuit path (Req 9.1) adapted to
|
|
* a multi-node cluster where the surviving standby immediately wins —
|
|
* the key assertion is role progression, not peer negotiation.
|
|
*/
|
|
typedef struct {
|
|
cluster_t *c;
|
|
} rdhup_hook_state_t;
|
|
|
|
static rdhup_hook_state_t g_rdhup_state;
|
|
|
|
static void rdhup_election_drives_evaluate(ha_state_t *ha, cluster_t *c,
|
|
int n_peers, xce_candidate_t **peers,
|
|
uint64_t my_config_tag,
|
|
uint64_t my_lsn,
|
|
const char *my_name)
|
|
{
|
|
(void)peers; (void)my_config_tag; (void)my_lsn; (void)my_name;
|
|
|
|
/* Allocate minimal ctx with quorum already satisfied (self-vote wins) */
|
|
int n_other = (n_peers > 1) ? n_peers - 1 : 0;
|
|
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) +
|
|
(size_t)n_other * sizeof(ctx->votes[0]));
|
|
if (!ctx) return;
|
|
|
|
ctx->n_peers = n_other;
|
|
ctx->votes_granted = 1; /* self-vote */
|
|
ctx->n_replied = n_other;
|
|
ctx->result = -1;
|
|
ctx->done_posted = 0;
|
|
ctx->c = g_rdhup_state.c;
|
|
pthread_mutex_init(&ctx->lock, NULL);
|
|
sem_init(&ctx->done, 0, 0);
|
|
|
|
ha_elect_evaluate_and_signal(ha, ctx);
|
|
|
|
pthread_mutex_destroy(&ctx->lock);
|
|
sem_destroy(&ctx->done);
|
|
free(ctx);
|
|
}
|
|
|
|
/* ── Helper: make a minimal 2-candidate cluster_t ───────────────────────── */
|
|
|
|
static void make_cluster(cluster_t *c, myconf_t *myconf, const char *my_name)
|
|
{
|
|
memset(c, 0, sizeof(*c));
|
|
memset(myconf, 0, sizeof(*myconf));
|
|
myconf->name = (char *)my_name;
|
|
/* Configured node by default: a real (nonzero) tag so the no-config
|
|
* voter-only RDHUP guard does NOT trip — these fixtures exercise the
|
|
* legitimate "a CONFIGURED node re-elects after RDHUP" path. The
|
|
* config-less guard is exercised by its own dedicated test below. */
|
|
myconf->config_version_tag = TEST_CONFIG_TAG;
|
|
myconf->peers_cfg.candidates_count = 2;
|
|
snprintf(myconf->peers_cfg.candidates[0].name,
|
|
sizeof(myconf->peers_cfg.candidates[0].name), "%s", my_name);
|
|
myconf->peers_cfg.candidates[0].port = 7001;
|
|
snprintf(myconf->peers_cfg.candidates[1].name,
|
|
sizeof(myconf->peers_cfg.candidates[1].name), "ctrl-b");
|
|
myconf->peers_cfg.candidates[1].port = 7002;
|
|
c->myconf = myconf;
|
|
}
|
|
|
|
/* ═══════════════════════════════════════════════════════════════════════════
|
|
* Test 1: RDHUP → ELECTING → PRIMARY (no standbys present → via hook chain)
|
|
*
|
|
* Req 6.3: RDHUP triggers CTRL_ELECTING + async election
|
|
* Req 9.3: RDHUP election uses ha_start_election_async (not am_i_primary_quorum)
|
|
* Req 9.2: phase2_done=1 suppresses cluster_init_phase2 on RDHUP re-election
|
|
* Req 1.1: the election winner becomes CTRL_PRIMARY directly (single leader role)
|
|
* ═════════════════════════════════════════════════════════════════════════ */
|
|
static void test_rdhup_full_path_reaches_primary(void)
|
|
{
|
|
install_stubs();
|
|
|
|
cluster_t c;
|
|
myconf_t myconf;
|
|
make_cluster(&c, &myconf, "ctrl-a");
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
|
|
ha_do_demote(&c.ha);
|
|
c.ha.phase2_done = 1; /* surviving standby — phase2 already done */
|
|
c.ha.n_peers = 1; /* 1 peer (2-node cluster) */
|
|
|
|
/* Wire the hook that drives evaluate immediately */
|
|
g_rdhup_state.c = &c;
|
|
ha_rdhup_election_hook = rdhup_election_drives_evaluate;
|
|
|
|
/* Give RDHUP handler a socketpair fd to close */
|
|
int sv[2];
|
|
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
|
|
c.ha.primary_ctrl_fd = sv[0];
|
|
|
|
/* Fire RDHUP */
|
|
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
|
|
|
|
clear_stubs();
|
|
close(sv[1]);
|
|
|
|
/* phase2 must NOT have been called (phase2_done=1 → skip) */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_phase2_calls,
|
|
"RDHUP re-election must not call cluster_init_phase2 when phase2_done=1 (Req 9.2)");
|
|
|
|
/* Role must be CTRL_PRIMARY directly — the election winner is the primary
|
|
* (single leader role; no transitional sub-state, Req 1.1). */
|
|
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
|
|
"RDHUP full path must reach CTRL_PRIMARY after winning election (Req 6.3, 1.1)");
|
|
|
|
ha_fini(&c.ha);
|
|
printf("test_rdhup_full_path_reaches_primary: PASS (role=%d==CTRL_PRIMARY)\n",
|
|
role);
|
|
}
|
|
|
|
/* ═══════════════════════════════════════════════════════════════════════════
|
|
* Test 2 (RDHUP → PRIMARY via replication-ack handler) DELETED — ha-single-leader-role
|
|
* task 4.1. Subject was the deleted ack-driven force-promotion machinery
|
|
* (the replication-ack handler); nothing left to verify. No shim, no retarget
|
|
* (design §"Transitional-State Deletions").
|
|
* ═════════════════════════════════════════════════════════════════════════ */
|
|
|
|
/* ═══════════════════════════════════════════════════════════════════════════
|
|
* Test 3: RDHUP in single-node cluster → immediate PRIMARY (single leader role)
|
|
*
|
|
* Req 9.1: single-node self-elects without network
|
|
* n_peers=0 → quorum=1, threshold=0 → acks(0) >= 0 → promotes immediately
|
|
* in ha_elect_evaluate_and_signal (no peers to wait for acks from)
|
|
* ═════════════════════════════════════════════════════════════════════════ */
|
|
static void test_rdhup_single_node_reaches_primary(void)
|
|
{
|
|
install_stubs();
|
|
|
|
cluster_t c;
|
|
myconf_t myconf;
|
|
make_cluster(&c, &myconf, "ctrl-a");
|
|
/* Override to single-node */
|
|
myconf.peers_cfg.candidates_count = 1;
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
|
|
ha_do_demote(&c.ha);
|
|
c.ha.phase2_done = 1;
|
|
c.ha.n_peers = 0; /* single node: no peers */
|
|
|
|
g_rdhup_state.c = &c;
|
|
ha_rdhup_election_hook = rdhup_election_drives_evaluate;
|
|
|
|
int sv[2];
|
|
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
|
|
c.ha.primary_ctrl_fd = sv[0];
|
|
|
|
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
|
|
close(sv[1]);
|
|
|
|
/* n_peers=0 → ha_elect_evaluate_and_signal single-node shortcircuit lands
|
|
* CTRL_PRIMARY directly (no transitional leader sub-state, no ack machinery —
|
|
* ha-single-leader-role). The old replication-ack fallback was excised
|
|
* (function deleted task 4.1); the PRIMARY assertion stands on its own. */
|
|
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
|
|
"single-node RDHUP path must reach CTRL_PRIMARY (Req 9.1)");
|
|
|
|
clear_stubs();
|
|
ha_fini(&c.ha);
|
|
printf("test_rdhup_single_node_reaches_primary: PASS (role=%d==CTRL_PRIMARY)\n", role);
|
|
}
|
|
|
|
/* ═══════════════════════════════════════════════════════════════════════════
|
|
* Test 4 (3-node RDHUP — ack-driven promotion) DELETED — ha-single-leader-role
|
|
* task 4.1. Subject was the deleted ack-counting promotion machinery
|
|
* (the replication-ack handler promoted a transitional leader state to PRIMARY
|
|
* on threshold acks); the mechanism is gone, nothing left to verify. No shim
|
|
* (design §"Transitional-State Deletions").
|
|
* ═════════════════════════════════════════════════════════════════════════ */
|
|
|
|
/* ═══════════════════════════════════════════════════════════════════════════
|
|
* Test 5: config push and ctrlhello hooks fire on promotion to primary
|
|
*
|
|
* Req 7.1: push_config_to_standbys called after winning election
|
|
* Req 7.2: ha_send_ctrlhello_to_all_standbys called after winning election
|
|
* ═════════════════════════════════════════════════════════════════════════ */
|
|
static void test_rdhup_promotion_fires_replication_hooks(void)
|
|
{
|
|
install_stubs();
|
|
|
|
cluster_t c;
|
|
myconf_t myconf;
|
|
make_cluster(&c, &myconf, "ctrl-a");
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
|
|
ha_do_demote(&c.ha);
|
|
c.ha.phase2_done = 1;
|
|
c.ha.n_peers = 1;
|
|
|
|
g_rdhup_state.c = &c;
|
|
ha_rdhup_election_hook = rdhup_election_drives_evaluate;
|
|
|
|
int sv[2];
|
|
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
|
|
c.ha.primary_ctrl_fd = sv[0];
|
|
|
|
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
|
|
close(sv[1]);
|
|
|
|
TEST_ASSERT_GREATER_THAN_INT_MESSAGE(0, g_push_config_calls,
|
|
"push_config_to_standbys hook must fire on promotion to primary (Req 7.1)");
|
|
TEST_ASSERT_GREATER_THAN_INT_MESSAGE(0, g_ctrlhello_calls,
|
|
"ha_send_ctrlhello hook must fire on promotion to primary (Req 7.2)");
|
|
|
|
clear_stubs();
|
|
ha_fini(&c.ha);
|
|
printf("test_rdhup_promotion_fires_replication_hooks: PASS "
|
|
"(push=%d, ctrlhello=%d)\n",
|
|
g_push_config_calls, g_ctrlhello_calls);
|
|
}
|
|
|
|
/* ═══════════════════════════════════════════════════════════════════════════
|
|
* Test 6: dispatch gate opens after CTRL_PRIMARY (job ops allowed)
|
|
*
|
|
* Req 6.5: after RDHUP promotion, primary accepts job dispatch (Req 5.3)
|
|
* ═════════════════════════════════════════════════════════════════════════ */
|
|
static void test_rdhup_primary_dispatch_gate_opens(void)
|
|
{
|
|
static int dummy_ep; /* stands in for the ACKing standby's endpoint */
|
|
|
|
install_stubs();
|
|
|
|
cluster_t c;
|
|
myconf_t myconf;
|
|
make_cluster(&c, &myconf, "ctrl-a");
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
|
|
ha_do_demote(&c.ha);
|
|
c.ha.phase2_done = 1;
|
|
c.ha.n_peers = 1;
|
|
|
|
/* FIXTURE FIX (ha-commit-engine task 6.1, bug-411 baseline failure):
|
|
* ha_on_primary_ctrl_fd_rdhup re-derives ha->n_peers from
|
|
* myconf->peers_cfg.candidates_count (= 2 here), so the CTRL_PRIMARY
|
|
* dispatch gate requires write quorum: >= 1 connected standby. The
|
|
* ack below models a standby that connected and ACKed; in production
|
|
* it lives in ha->standby_endpoints (raft_cb_become_leader /
|
|
* on_standby_connected). Register it so quorum legitimately holds on
|
|
* this healthy primary — the gate itself must NOT be weakened. */
|
|
c.ha.standby_endpoints = calloc(1, sizeof(zarray_t));
|
|
TEST_ASSERT_NOT_NULL(c.ha.standby_endpoints);
|
|
zarray_init(c.ha.standby_endpoints, 4, NULL);
|
|
zarray_store(c.ha.standby_endpoints, &dummy_ep);
|
|
|
|
g_rdhup_state.c = &c;
|
|
ha_rdhup_election_hook = rdhup_election_drives_evaluate;
|
|
|
|
int sv[2];
|
|
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
|
|
c.ha.primary_ctrl_fd = sv[0];
|
|
|
|
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
|
|
close(sv[1]);
|
|
|
|
/* ha-single-leader-role: ha_elect_evaluate_and_signal lands the winner in
|
|
* CTRL_PRIMARY directly (no transitional leader sub-state, no ack machinery).
|
|
* The old replication-ack drive-step was excised (function deleted task 4.1);
|
|
* the dispatch-gate assertion below is the subject of this case. */
|
|
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_INT((int)CTRL_PRIMARY, role);
|
|
|
|
/* Verify dispatch gate allows job operations in CTRL_PRIMARY */
|
|
int rc = ha_dispatch_gate(&c.ha, OPCode_jobsub);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
|
|
"OPCode_jobsub must be allowed by dispatch gate in CTRL_PRIMARY (Req 5.3)");
|
|
|
|
rc = ha_dispatch_gate(&c.ha, OPCode_jobdispatch);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
|
|
"OPCode_jobdispatch must be allowed by dispatch gate in CTRL_PRIMARY (Req 5.3)");
|
|
|
|
clear_stubs();
|
|
zarray_fini(c.ha.standby_endpoints);
|
|
free(c.ha.standby_endpoints);
|
|
c.ha.standby_endpoints = NULL;
|
|
ha_fini(&c.ha);
|
|
printf("test_rdhup_primary_dispatch_gate_opens: PASS (gate accepts job ops)\n");
|
|
}
|
|
|
|
/* ═══════════════════════════════════════════════════════════════════════════
|
|
* Test 7: config-less node (tag=0) with peers MUST NOT start an RDHUP re-election
|
|
*
|
|
* Election-SAFETY (ha-config-bootstrap Req 2.8 / ha-repl-hardening Req 10.5):
|
|
* a node with no local config (config_version_tag == 0) and >1 candidates must
|
|
* NOT win an election while a configured peer exists. ha_on_primary_ctrl_fd_rdhup
|
|
* must mirror the ha_start_election_async voter-only guard: skip the elect_ctx
|
|
* alloc + the election trigger entirely and stay a passive voter (CTRL_STANDBY).
|
|
*
|
|
* Asserted via the ha_rdhup_election_hook seam: the hook must NOT fire (count==0),
|
|
* and role must be CTRL_STANDBY (not CTRL_ELECTING / CTRL_PRIMARY).
|
|
* Before the fix this FAILS: the hook fires and the election proceeds.
|
|
* ═════════════════════════════════════════════════════════════════════════ */
|
|
static int g_guard_hook_calls = 0;
|
|
|
|
static void counting_election_hook(ha_state_t *ha, cluster_t *c,
|
|
int n_peers, xce_candidate_t **peers,
|
|
uint64_t my_config_tag, uint64_t my_lsn,
|
|
const char *my_name)
|
|
{
|
|
(void)ha; (void)c; (void)n_peers; (void)peers;
|
|
(void)my_config_tag; (void)my_lsn; (void)my_name;
|
|
g_guard_hook_calls++;
|
|
}
|
|
|
|
static void test_rdhup_noconfig_node_does_not_elect(void)
|
|
{
|
|
install_stubs();
|
|
g_guard_hook_calls = 0;
|
|
|
|
cluster_t c;
|
|
myconf_t myconf;
|
|
make_cluster(&c, &myconf, "ctrl-a");
|
|
/* Config-less node: no local config tag, but part of a >1 node cluster. */
|
|
myconf.config_version_tag = 0;
|
|
myconf.peers_cfg.candidates_count = 2;
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
|
|
ha_do_demote(&c.ha);
|
|
c.ha.phase2_done = 1;
|
|
c.ha.n_peers = 1;
|
|
|
|
ha_rdhup_election_hook = counting_election_hook;
|
|
|
|
int sv[2];
|
|
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
|
|
c.ha.primary_ctrl_fd = sv[0];
|
|
|
|
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
|
|
|
|
clear_stubs();
|
|
close(sv[1]);
|
|
|
|
/* The election hook must NOT have fired — no re-election started. */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_guard_hook_calls,
|
|
"config-less node (tag=0, peers>1) must NOT start an RDHUP re-election "
|
|
"(ha-config-bootstrap Req 2.8)");
|
|
|
|
/* Role must be CTRL_STANDBY — a passive voter awaiting the configured primary. */
|
|
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY, role,
|
|
"config-less RDHUP node must stay CTRL_STANDBY (voter only), not ELECTING/PRIMARY");
|
|
|
|
ha_fini(&c.ha);
|
|
printf("test_rdhup_noconfig_node_does_not_elect: PASS "
|
|
"(hook_calls=%d, role=%d==CTRL_STANDBY)\n", g_guard_hook_calls, role);
|
|
}
|
|
|
|
/* ═══════════════════════════════════════════════════════════════════════════
|
|
* Test 8 (contrast / non-regression): a CONFIGURED node (tag!=0) with peers
|
|
* MUST still start the RDHUP re-election.
|
|
*
|
|
* Same shape as Test 7 but with a real config tag — the guard must NOT trip.
|
|
* The election hook MUST fire exactly once.
|
|
* ═════════════════════════════════════════════════════════════════════════ */
|
|
static void test_rdhup_configured_node_still_elects(void)
|
|
{
|
|
install_stubs();
|
|
g_guard_hook_calls = 0;
|
|
|
|
cluster_t c;
|
|
myconf_t myconf;
|
|
make_cluster(&c, &myconf, "ctrl-a");
|
|
/* Configured node: real tag, >1 candidates — guard must NOT trip. */
|
|
myconf.config_version_tag = TEST_CONFIG_TAG;
|
|
myconf.peers_cfg.candidates_count = 2;
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
|
|
ha_do_demote(&c.ha);
|
|
c.ha.phase2_done = 1;
|
|
c.ha.n_peers = 1;
|
|
|
|
ha_rdhup_election_hook = counting_election_hook;
|
|
|
|
int sv[2];
|
|
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
|
|
c.ha.primary_ctrl_fd = sv[0];
|
|
|
|
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
|
|
|
|
clear_stubs();
|
|
close(sv[1]);
|
|
|
|
/* Configured node: the election hook MUST fire (re-election proceeds). */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_guard_hook_calls,
|
|
"configured node (tag!=0, peers>1) must still start RDHUP re-election");
|
|
|
|
ha_fini(&c.ha);
|
|
printf("test_rdhup_configured_node_still_elects: PASS (hook_calls=%d)\n",
|
|
g_guard_hook_calls);
|
|
}
|
|
|
|
/* ═══════════════════════════════════════════════════════════════════════════
|
|
* Test 9 (regression — infinite election loop after primary death):
|
|
* The RDHUP candidate must advertise its EFFECTIVE freshness floor
|
|
* (max(applied_lsn, last_known_committed_lsn)) as the RequestVote last_log_lsn,
|
|
* NOT the stale persist.committed_lsn.
|
|
*
|
|
* Root cause of the observed bug: a standby never advances
|
|
* persist.committed_lsn (only the primary's commit engine writes it); it
|
|
* advances applied_lsn (WAL catch-up) and last_known_committed_lsn (heartbeat
|
|
* replies). The voter-side freshness floor (cluster.c OPCode_requestVote) is
|
|
* max(applied_lsn, last_known_committed_lsn). When the RDHUP candidate
|
|
* advertised persist.committed_lsn (stale-low) while every voter compared
|
|
* against a high applied_lsn floor, the freshness gate (cand_lsn < my_lsn)
|
|
* tripped on BOTH surviving standbys — each denied the other's vote, no
|
|
* candidate ever reached quorum, and the cluster spun in an infinite
|
|
* RDHUP→ELECTING→STANDBY loop with terms climbing in lockstep.
|
|
*
|
|
* Captured via the ha_rdhup_election_hook my_lsn argument.
|
|
* Before the fix: captured == 0 (stale committed_lsn). After: == 10.
|
|
* ═════════════════════════════════════════════════════════════════════════ */
|
|
static uint64_t g_captured_my_lsn = 0;
|
|
static uint64_t g_captured_my_tag = 0;
|
|
|
|
static void capture_lsn_hook(ha_state_t *ha, cluster_t *c,
|
|
int n_peers, xce_candidate_t **peers,
|
|
uint64_t my_config_tag, uint64_t my_lsn,
|
|
const char *my_name)
|
|
{
|
|
(void)ha; (void)c; (void)n_peers; (void)peers; (void)my_name;
|
|
g_captured_my_tag = my_config_tag;
|
|
g_captured_my_lsn = my_lsn;
|
|
}
|
|
|
|
static void test_rdhup_candidate_advertises_freshness_floor_not_stale_committed(void)
|
|
{
|
|
install_stubs();
|
|
g_captured_my_lsn = 0;
|
|
g_captured_my_tag = 0;
|
|
|
|
cluster_t c;
|
|
myconf_t myconf;
|
|
make_cluster(&c, &myconf, "ctrl-a");
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
|
|
ha_do_demote(&c.ha);
|
|
c.ha.phase2_done = 1;
|
|
c.ha.n_peers = 1;
|
|
|
|
/* Standby state after applying WAL + learning commits from the (now-dead)
|
|
* primary: applied_lsn and last_known_committed_lsn advanced to 10, but
|
|
* persist.committed_lsn is still 0 — a standby never advances it.
|
|
*
|
|
* ha-commit-durability task 4.2: the candidate now advertises the BARE
|
|
* DURABLE WAL tip (persist_durable_floor), not the folded
|
|
* max(applied_lsn, learned_committed). Model the standby holding its log
|
|
* durably up to LSN 10 by priming the durable watermark — this is the
|
|
* durable-backed analog of the original "advertise the freshness floor, not
|
|
* the stale committed_lsn" guarantee (still NOT 0). */
|
|
atomic_store_explicit(&c.ha.applied_lsn, 10ULL, memory_order_release);
|
|
atomic_store_explicit(&c.ha.last_known_committed_lsn, 10ULL,
|
|
memory_order_release);
|
|
atomic_store_explicit(&c.persist.committed_lsn, 0ULL, memory_order_release);
|
|
atomic_store_explicit(&c.persist.durable_lsn, 10ULL, memory_order_release);
|
|
|
|
ha_rdhup_election_hook = capture_lsn_hook;
|
|
|
|
int sv[2];
|
|
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
|
|
c.ha.primary_ctrl_fd = sv[0];
|
|
|
|
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
|
|
|
|
clear_stubs();
|
|
close(sv[1]);
|
|
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(10ULL, g_captured_my_lsn,
|
|
"RDHUP candidate must advertise its DURABLE WAL tip (durable_floor = 10), "
|
|
"not the stale persist.committed_lsn (0); otherwise surviving standbys "
|
|
"mutually deny votes -> infinite election loop (bug-339; task 4.2 makes "
|
|
"this the durable tip, not max(applied,learned))");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE((uint64_t)TEST_CONFIG_TAG, g_captured_my_tag,
|
|
"RDHUP candidate must advertise its config_version_tag");
|
|
|
|
ha_fini(&c.ha);
|
|
printf("test_rdhup_candidate_advertises_freshness_floor_not_stale_committed: "
|
|
"PASS (my_lsn=%llu)\n", (unsigned long long)g_captured_my_lsn);
|
|
}
|
|
|
|
/* ── Suite entry point ──────────────────────────────────────────────────── */
|
|
|
|
void test_ha_rdhup_integration(void)
|
|
{
|
|
RUN_TEST(test_rdhup_candidate_advertises_freshness_floor_not_stale_committed);
|
|
RUN_TEST(test_rdhup_full_path_reaches_primary);
|
|
/* ack-driven promotion test deleted (ha-single-leader-role 4.1:
|
|
* ack-driven promotion machinery removed, no shim) */
|
|
RUN_TEST(test_rdhup_single_node_reaches_primary);
|
|
/* test_rdhup_3node_one_ack_promotes deleted (ha-single-leader-role 4.1:
|
|
* ack-counting promotion machinery removed, no shim) */
|
|
RUN_TEST(test_rdhup_promotion_fires_replication_hooks);
|
|
RUN_TEST(test_rdhup_primary_dispatch_gate_opens);
|
|
/* Election-safety guard: config-less node must not re-elect; configured
|
|
* node still does (ha-config-bootstrap Req 2.8 / ha-repl-hardening Req 10.5). */
|
|
RUN_TEST(test_rdhup_noconfig_node_does_not_elect);
|
|
RUN_TEST(test_rdhup_configured_node_still_elects);
|
|
}
|