myxce/src/controller/testsuite/test_ha_role.c
Rongsong Shen 566c91cd8b feat(ha-wal-replication): standby WAL ACK reply (tasks 3.1-3.3)
Standby sends WalAckReply via xce_reply after each successfully
applied WAL entry. ha_apply_wal_catchup_entry signature extended
with endpoint_t* and cluster_t* parameters. walackreplymsg union
arm added to RequestReply capnp schema and xce_reply_t. Error
entries skip ACK. Dispatch gate test confirms OPCode_walCatchupEntry
allowed in CTRL_STANDBY/ELECTING/INIT.

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
2026-06-10 09:14:25 +00:00

1102 lines
41 KiB
C

#include <signal.h>
#include <stdatomic.h>
#include <string.h>
#include <stdio.h>
#include <time.h>
#include <unity/unity.h>
#include "ha.h"
#include "myconf.h"
#include "xce/cluster_def.h"
static void test_ha_init_default_role_is_standby(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
ha_fini(&ha);
printf("test_ha_init_default_role_is_standby: PASS\n");
}
static void test_ha_init_rejects_fast_heartbeat(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 1999, /* below 2000ms floor */
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
TEST_ASSERT_NOT_EQUAL(0, ha_init(&ha, &cfg, 0));
printf("test_ha_init_rejects_fast_heartbeat: PASS\n");
}
static void test_ha_init_rejects_short_election_timeout(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 499, /* below 500ms floor */
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
TEST_ASSERT_NOT_EQUAL(0, ha_init(&ha, &cfg, 0));
printf("test_ha_init_rejects_short_election_timeout: PASS\n");
}
static void test_ha_init_rejects_timeout_less_than_interval(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 5000,
.heartbeat_timeout_ms = 4999, /* must be >= heartbeat_interval_ms */
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
TEST_ASSERT_NOT_EQUAL(0, ha_init(&ha, &cfg, 0));
printf("test_ha_init_rejects_timeout_less_than_interval: PASS\n");
}
static void test_ha_promote_sets_primary_role(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
ha_init(&ha, &cfg, 0);
/* Directly set role to PRIMARY (stub path — full promotion in P1-3a) */
atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release);
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
ha_fini(&ha);
printf("test_ha_promote_sets_primary_role: PASS\n");
}
/* Watchdog / SIGUSR1 tests deleted — watchdog pthread replaced by
* ha_arm_heartbeat_timeout_vtask (delaymgr); failover_requested removed */
/* ── P1-3a tests ─────────────────────────────────────────────── */
/* Term must increment on each promotion attempt */
static void test_promote_increments_term(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
ha_init(&ha, &cfg, 0);
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
ha_do_promote(&ha); /* internal promotion logic — no cluster_t needed */
uint64_t term_after = atomic_load_explicit(&ha.term, memory_order_acquire);
TEST_ASSERT_TRUE(term_after > term_before);
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
ha_fini(&ha);
printf("test_promote_increments_term: PASS\n");
}
/* Demotion must set role back to STANDBY */
static void test_demote_sets_standby_role(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
ha_init(&ha, &cfg, 0);
ha_do_promote(&ha);
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
ha_do_demote(&ha);
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
ha_fini(&ha);
printf("test_demote_sets_standby_role: PASS\n");
}
/* test_sigusr1_sets_failover_requested deleted — SIGUSR1 / failover_requested removed */
/* am_i_primary() tests deleted — function removed (election-iomgr-refactor) */
/* ── Task 2.1: ha_state_t new fields zero-initialised by ha_init ──────── */
/*
* Verify that ha_init zeroes the new fields added for election-iomgr-refactor:
* phase2_done, repl_ack_count, n_peers, client_ssl_ctx
* and that CTRL_PRIMARY_COMMITTING is a distinct enum value in ctrl_role_t.
* Also verifies ha_config_t.committing_timeout_ms default macro exists.
*
* Requirements: 5.2, 8.1 (election-iomgr-refactor spec)
*/
static void test_ha_init_zeroes_new_fields(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
};
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
/* phase2_done must be explicitly zero after ha_init */
TEST_ASSERT_EQUAL_INT(0, ha.phase2_done);
/* repl_ack_count is _Atomic int; must be zero after ha_init */
TEST_ASSERT_EQUAL_INT(0, atomic_load_explicit(&ha.repl_ack_count, memory_order_relaxed));
/* n_peers must be zero after ha_init */
TEST_ASSERT_EQUAL_INT(0, ha.n_peers);
/* client_ssl_ctx must be NULL after ha_init */
TEST_ASSERT_NULL(ha.client_ssl_ctx);
/* committing_timeout_ms stored in config */
TEST_ASSERT_EQUAL_UINT32(HA_DEFAULT_COMMITTING_TIMEOUT_MS, ha.config.committing_timeout_ms);
/* CTRL_PRIMARY_COMMITTING must be a distinct enum value */
TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY_COMMITTING, (int)CTRL_PRIMARY);
TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY_COMMITTING, (int)CTRL_STANDBY);
TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY_COMMITTING, (int)CTRL_INIT);
TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY_COMMITTING, (int)CTRL_ELECTING);
ha_fini(&ha);
printf("test_ha_init_zeroes_new_fields: PASS\n");
}
/*
* Task 1.4: per-standby convergence tracking fields zero-initialised by ha_init.
*
* Verifies:
* - standby_reported_tag[i] == 0 for all i after ha_init()
* - standby_reported_lsn[i] == 0 for all i after ha_init()
* - standby_converged[i] == false for all i after ha_init()
* - fields are writable from test context
*
* Requirements: 2.1 (ha-repl-hardening spec)
*/
static void test_ha_init_zeroes_standby_convergence_fields(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
};
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
/* All per-standby tracking arrays must be zero/false after ha_init */
for (int i = 0; i < MAX_HA_CANDIDATES; i++) {
TEST_ASSERT_EQUAL_UINT64(0, ha.standby_reported_tag[i]);
TEST_ASSERT_EQUAL_UINT64(0, ha.standby_reported_lsn[i]);
TEST_ASSERT_FALSE(ha.standby_converged[i]);
}
/* Fields must be writable */
ha.standby_reported_tag[0] = 42ULL;
ha.standby_reported_lsn[0] = 100ULL;
ha.standby_converged[0] = true;
TEST_ASSERT_EQUAL_UINT64(42ULL, ha.standby_reported_tag[0]);
TEST_ASSERT_EQUAL_UINT64(100ULL, ha.standby_reported_lsn[0]);
TEST_ASSERT_TRUE(ha.standby_converged[0]);
ha_fini(&ha);
printf("test_ha_init_zeroes_standby_convergence_fields: PASS\n");
}
/* ── Task 3.1: ha_process_standby_state convergence tracking ─────────────── */
/*
* Declare the function under test. It is defined in ha.c and exposed only in
* XCE_TESTING builds (forward declaration matches the implementation).
*/
#ifdef XCE_TESTING
void ha_process_standby_state(ha_state_t *ha, int peer_idx,
uint64_t reported_tag, uint64_t reported_lsn,
uint64_t committed_tag, uint64_t committed_lsn);
#endif
/* Helper: build a ha_state_t with all convergence fields zeroed. */
static ha_state_t *build_ha_for_convergence_test(void)
{
static ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
};
ha_init(&ha, &cfg, 0);
return &ha;
}
/*
* Test: standby with lsn == committed_lsn and tag == committed_tag → converged.
* Fields standby_reported_tag[0] and standby_reported_lsn[0] updated correctly.
* Requirements: 1.2, 2.2 (ha-repl-hardening)
*/
static void test_process_standby_state_converged(void)
{
ha_state_t *ha = build_ha_for_convergence_test();
ha_process_standby_state(ha, 0,
/*reported_tag=*/5, /*reported_lsn=*/100,
/*committed_tag=*/5, /*committed_lsn=*/100);
TEST_ASSERT_EQUAL_UINT64(5, ha->standby_reported_tag[0]);
TEST_ASSERT_EQUAL_UINT64(100, ha->standby_reported_lsn[0]);
TEST_ASSERT_TRUE(ha->standby_converged[0]);
ha_fini(ha);
printf("test_process_standby_state_converged: PASS\n");
}
/*
* Test: standby lsn < committed_lsn → not converged.
* Requirements: 1.2, 2.2
*/
static void test_process_standby_state_lsn_behind(void)
{
ha_state_t *ha = build_ha_for_convergence_test();
ha_process_standby_state(ha, 0,
/*reported_tag=*/5, /*reported_lsn=*/50,
/*committed_tag=*/5, /*committed_lsn=*/100);
TEST_ASSERT_EQUAL_UINT64(5, ha->standby_reported_tag[0]);
TEST_ASSERT_EQUAL_UINT64(50, ha->standby_reported_lsn[0]);
TEST_ASSERT_FALSE(ha->standby_converged[0]);
ha_fini(ha);
printf("test_process_standby_state_lsn_behind: PASS\n");
}
/*
* Test: standby tag < committed_tag → not converged.
* Requirements: 1.2, 2.2
*/
static void test_process_standby_state_tag_behind(void)
{
ha_state_t *ha = build_ha_for_convergence_test();
ha_process_standby_state(ha, 0,
/*reported_tag=*/3, /*reported_lsn=*/100,
/*committed_tag=*/5, /*committed_lsn=*/100);
TEST_ASSERT_EQUAL_UINT64(3, ha->standby_reported_tag[0]);
TEST_ASSERT_EQUAL_UINT64(100, ha->standby_reported_lsn[0]);
TEST_ASSERT_FALSE(ha->standby_converged[0]);
ha_fini(ha);
printf("test_process_standby_state_tag_behind: PASS\n");
}
/*
* Test: convergence regression — standby was converged; primary advances;
* next update with old values should reset standby_converged to false.
* Requirements: 2.5 (converged standby regresses → mark not-converged)
*/
static void test_process_standby_state_regression(void)
{
ha_state_t *ha = build_ha_for_convergence_test();
/* First update: converged at (tag=5, lsn=100) */
ha_process_standby_state(ha, 0,
/*reported_tag=*/5, /*reported_lsn=*/100,
/*committed_tag=*/5, /*committed_lsn=*/100);
TEST_ASSERT_TRUE(ha->standby_converged[0]);
/* Primary advances committed_lsn to 200; standby still reports (5, 100) */
ha_process_standby_state(ha, 0,
/*reported_tag=*/5, /*reported_lsn=*/100,
/*committed_tag=*/5, /*committed_lsn=*/200);
TEST_ASSERT_FALSE(ha->standby_converged[0]);
ha_fini(ha);
printf("test_process_standby_state_regression: PASS\n");
}
/*
* Test: different standby indices updated independently.
* Requirements: 2.2
*/
static void test_process_standby_state_multiple_peers(void)
{
ha_state_t *ha = build_ha_for_convergence_test();
/* peer 0: converged */
ha_process_standby_state(ha, 0,
/*reported_tag=*/5, /*reported_lsn=*/100,
/*committed_tag=*/5, /*committed_lsn=*/100);
/* peer 1: not converged (tag behind) */
ha_process_standby_state(ha, 1,
/*reported_tag=*/3, /*reported_lsn=*/100,
/*committed_tag=*/5, /*committed_lsn=*/100);
TEST_ASSERT_TRUE(ha->standby_converged[0]);
TEST_ASSERT_FALSE(ha->standby_converged[1]);
TEST_ASSERT_EQUAL_UINT64(5, ha->standby_reported_tag[0]);
TEST_ASSERT_EQUAL_UINT64(3, ha->standby_reported_tag[1]);
ha_fini(ha);
printf("test_process_standby_state_multiple_peers: PASS\n");
}
/* ── Task 5.2: ha_check_majority_convergence → PRIMARY transition ────────────
*
* After ha_process_standby_state updates standby_converged[], the primary must
* count the number of converged standbys and promote to CTRL_PRIMARY when the
* majority threshold is met and the current role is CTRL_PRIMARY_COMMITTING.
*
* Threshold formula (Req 2.3): ceil(n_peers/2) - 1 converged standbys —
* i.e. primary + converged standbys form a majority of the cluster.
* Integer form: n_peers / 2 standbys (n=3 → 1, n=5 → 2).
* Single-node (n_peers<=1): threshold=0 → satisfied immediately.
* n_peers = total cluster size including self.
*
* Requirements: 2.3, 2.4, 6.1, 6.3, 6.5 (ha-repl-hardening)
*/
#ifdef XCE_TESTING
void ha_check_majority_convergence(ha_state_t *ha, cluster_t *c);
#endif
/*
* Test: 3-node cluster; no standbys converged (0/2)
* → role must remain CTRL_PRIMARY_COMMITTING.
* Requirements: 2.3, 6.3 (ha-repl-hardening)
*/
static void test_majority_convergence_3node_none_stays_committing(void)
{
#ifdef XCE_TESTING
cluster_t c;
memset(&c, 0, sizeof(c));
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
};
ha_init(&c.ha, &cfg, 0);
c.ha.n_peers = 3; /* 3-node cluster: primary + 2 standbys */
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
memory_order_release);
/* No standby converged */
c.ha.standby_converged[0] = false;
c.ha.standby_converged[1] = false;
ha_check_majority_convergence(&c.ha, &c);
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, role,
"3-node: 0/2 standbys converged → must remain CTRL_PRIMARY_COMMITTING (Req 2.3)");
ha_fini(&c.ha);
printf("test_majority_convergence_3node_none_stays_committing: PASS\n");
#else
printf("test_majority_convergence_3node_none_stays_committing: SKIP\n");
#endif
}
/*
* Test: 3-node cluster; 1/2 standbys converged → primary + 1 standby = 2/3
* nodes hold the state = majority → must promote to CTRL_PRIMARY.
* (Requirement 2.3 formula: ceil(3/2)-1 = 1 converged standby suffices.
* A dead minority standby must not force the degraded-mode timeout.)
* Requirements: 2.3, 2.4, 6.3 (ha-repl-hardening)
*/
static void test_majority_convergence_3node_one_standby_promotes(void)
{
#ifdef XCE_TESTING
cluster_t c;
memset(&c, 0, sizeof(c));
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
};
ha_init(&c.ha, &cfg, 0);
c.ha.n_peers = 3;
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
memory_order_release);
/* Only standby[0] converged; standby[1] not converged */
c.ha.standby_converged[0] = true;
c.ha.standby_converged[1] = false;
ha_check_majority_convergence(&c.ha, &c);
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
"3-node: 1/2 standbys converged → majority → CTRL_PRIMARY (Req 2.3)");
ha_fini(&c.ha);
printf("test_majority_convergence_3node_one_standby_promotes: PASS\n");
#else
printf("test_majority_convergence_3node_one_standby_promotes: SKIP\n");
#endif
}
/*
* Test: 5-node cluster; threshold = 5/2 = 2 converged standbys.
* 1/4 converged → stays COMMITTING; 2/4 converged → promotes.
* Requirements: 2.3, 6.3 (ha-repl-hardening)
*/
static void test_majority_convergence_5node_threshold(void)
{
#ifdef XCE_TESTING
cluster_t c;
memset(&c, 0, sizeof(c));
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
};
ha_init(&c.ha, &cfg, 0);
c.ha.n_peers = 5;
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
memory_order_release);
/* 1/4 converged: primary + 1 = 2 of 5 — not a majority → stays */
c.ha.standby_converged[0] = true;
ha_check_majority_convergence(&c.ha, &c);
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, role,
"5-node: 1/4 standbys converged → must remain COMMITTING (Req 2.3)");
/* 2/4 converged: primary + 2 = 3 of 5 — majority → promotes */
c.ha.standby_converged[1] = true;
ha_check_majority_convergence(&c.ha, &c);
role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
"5-node: 2/4 standbys converged → majority → CTRL_PRIMARY (Req 2.3)");
ha_fini(&c.ha);
printf("test_majority_convergence_5node_threshold: PASS\n");
#else
printf("test_majority_convergence_5node_threshold: SKIP\n");
#endif
}
/*
* Test: 3-node cluster (n_peers=2); both standbys converged (2/2)
* → role must transition to CTRL_PRIMARY.
* Requirements: 2.3, 2.4, 6.3 (ha-repl-hardening)
*/
static void test_majority_convergence_3node_all_standbys_promotes(void)
{
#ifdef XCE_TESTING
cluster_t c;
memset(&c, 0, sizeof(c));
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
};
ha_init(&c.ha, &cfg, 0);
c.ha.n_peers = 3; /* 3-node cluster */
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
memory_order_release);
/* Both standbys converged */
c.ha.standby_converged[0] = true;
c.ha.standby_converged[1] = true;
ha_check_majority_convergence(&c.ha, &c);
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
"3-node: 2/2 standbys converged → must transition to CTRL_PRIMARY (Req 2.3, 2.4)");
ha_fini(&c.ha);
printf("test_majority_convergence_3node_all_standbys_promotes: PASS\n");
#else
printf("test_majority_convergence_3node_all_standbys_promotes: SKIP\n");
#endif
}
/*
* Test: single-node cluster (n_peers=0)
* → threshold=0, satisfied immediately → must promote to CTRL_PRIMARY.
* Requirements: 6.5, 2.3 (ha-repl-hardening): single-node is always satisfied.
*/
static void test_majority_convergence_single_node_immediate(void)
{
#ifdef XCE_TESTING
cluster_t c;
memset(&c, 0, sizeof(c));
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
};
ha_init(&c.ha, &cfg, 0);
c.ha.n_peers = 1; /* single-node: just self */
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
memory_order_release);
/* No standbys to converge — threshold=0 must be satisfied immediately */
ha_check_majority_convergence(&c.ha, &c);
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
"single-node: n_peers=1 → threshold=0 satisfied → must promote to CTRL_PRIMARY (Req 6.5)");
ha_fini(&c.ha);
printf("test_majority_convergence_single_node_immediate: PASS\n");
#else
printf("test_majority_convergence_single_node_immediate: SKIP\n");
#endif
}
/* ── Task 3.3: ha_dispatch_repl_action_for_standby PUSH_CONFIG wiring ──────
*
* ha_dispatch_repl_action_for_standby reads the standby's stored
* (standby_reported_tag, standby_reported_lsn), calls ha_evaluate_repl_action,
* and if the result is HA_REPL_PUSH_CONFIG calls push_config_to_standbys (in
* production) or ha_repl_push_config_for_standby_hook (in XCE_TESTING).
*
* Requirements: 1.4 (ha-repl-hardening), design §Unified Heartbeat-Driven Protocol
*/
#ifdef XCE_TESTING
extern int (*ha_repl_push_config_for_standby_hook)(void *c, int peer_idx);
#endif
/* Call-counting state for the hook */
static int g_push_config_for_standby_count = 0;
static int g_push_config_for_standby_idx = -1;
static int stub_repl_push_config_for_standby(void *c, int peer_idx)
{
(void)c;
g_push_config_for_standby_count++;
g_push_config_for_standby_idx = peer_idx;
return 0;
}
/*
* Test: when standby_reported_tag < committed_tag, ha_dispatch_repl_action_for_standby
* must call push_config_to_standbys (via hook in test mode).
*
* Steps:
* 1. Set ha->standby_reported_tag[0] = 3 (behind committed_tag=5)
* 2. Set ha->standby_reported_lsn[0] = 100 (at committed_lsn)
* 3. Call ha_dispatch_repl_action_for_standby(ha, NULL, 0, committed_tag=5, ...)
* 4. Assert hook called once with peer_idx=0
*
* Requirements: 1.4 (push_config when standby tag < committed_tag)
*/
static void test_dispatch_repl_action_push_config_when_tag_behind(void)
{
#ifdef XCE_TESTING
ha_repl_push_config_for_standby_hook = stub_repl_push_config_for_standby;
g_push_config_for_standby_count = 0;
g_push_config_for_standby_idx = -1;
ha_state_t *ha = build_ha_for_convergence_test();
/* standby[0] has stale config tag */
ha->standby_reported_tag[0] = 3;
ha->standby_reported_lsn[0] = 100;
int rc = ha_dispatch_repl_action_for_standby(
ha, /*c=*/NULL, /*ep=*/NULL, /*peer_idx=*/0,
/*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/0);
(void)rc;
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_push_config_for_standby_count,
"PUSH_CONFIG: hook must fire once when standby_tag < committed_tag");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_push_config_for_standby_idx,
"PUSH_CONFIG: hook must receive correct peer_idx=0");
ha_fini(ha);
ha_repl_push_config_for_standby_hook = NULL;
printf("test_dispatch_repl_action_push_config_when_tag_behind: PASS\n");
#else
printf("test_dispatch_repl_action_push_config_when_tag_behind: SKIP (non-test build)\n");
#endif
}
/*
* Test: when standby is converged (tag == committed, lsn == committed),
* ha_dispatch_repl_action_for_standby must NOT call push_config hook.
*
* Requirements: decision tree — CONVERGED does not trigger push
*/
static void test_dispatch_repl_action_no_push_when_converged(void)
{
#ifdef XCE_TESTING
ha_repl_push_config_for_standby_hook = stub_repl_push_config_for_standby;
g_push_config_for_standby_count = 0;
g_push_config_for_standby_idx = -1;
ha_state_t *ha = build_ha_for_convergence_test();
/* standby[0] is fully converged */
ha->standby_reported_tag[0] = 5;
ha->standby_reported_lsn[0] = 100;
int rc = ha_dispatch_repl_action_for_standby(
ha, /*c=*/NULL, /*ep=*/NULL, /*peer_idx=*/0,
/*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/0);
(void)rc;
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_push_config_for_standby_count,
"CONVERGED: hook must NOT fire when standby is already converged");
ha_fini(ha);
ha_repl_push_config_for_standby_hook = NULL;
printf("test_dispatch_repl_action_no_push_when_converged: PASS\n");
#else
printf("test_dispatch_repl_action_no_push_when_converged: SKIP (non-test build)\n");
#endif
}
/* ── Task 3.5d: ha_dispatch_repl_action_for_standby SEND_SNAPSHOT wiring ────
*
* When standby_lsn < oldest_wal_lsn (standby predates WAL retention window),
* ha_dispatch_repl_action_for_standby must call ha_send_snapshot_to_standby
* (via ha_repl_send_snapshot_hook in XCE_TESTING builds).
*
* Requirements: 4.1 (ha-repl-hardening), design §Snapshot Transfer
*/
#ifdef XCE_TESTING
extern int (*ha_repl_send_snapshot_hook)(int peer_idx);
#endif
/* Call-counting state for the snapshot hook */
static int g_send_snapshot_count = 0;
static int g_send_snapshot_idx = -1;
static int stub_repl_send_snapshot(int peer_idx)
{
g_send_snapshot_count++;
g_send_snapshot_idx = peer_idx;
return 0;
}
/*
* Test: when standby_lsn < oldest_wal_lsn (and standby_tag == committed_tag),
* ha_dispatch_repl_action_for_standby must call ha_send_snapshot_to_standby
* (via hook in test mode).
*
* Steps:
* 1. Set ha->standby_reported_tag[0] = committed_tag (tag is current)
* 2. Set ha->standby_reported_lsn[0] = 10 (behind oldest_wal_lsn=50)
* 3. Call ha_dispatch_repl_action_for_standby(..., oldest_wal_lsn=50)
* 4. Assert hook called once with peer_idx=0
*
* Requirements: 4.1 (snapshot when standby_lsn < oldest_wal_lsn)
*/
static void test_dispatch_repl_action_send_snapshot(void)
{
#ifdef XCE_TESTING
ha_repl_send_snapshot_hook = stub_repl_send_snapshot;
g_send_snapshot_count = 0;
g_send_snapshot_idx = -1;
ha_state_t *ha = build_ha_for_convergence_test();
/* standby[0] has current config tag but LSN predates WAL window */
ha->standby_reported_tag[0] = 5;
ha->standby_reported_lsn[0] = 10; /* < oldest_wal_lsn=50 */
int rc = ha_dispatch_repl_action_for_standby(
ha, /*c=*/NULL, /*ep=*/NULL, /*peer_idx=*/0,
/*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/50);
(void)rc;
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_send_snapshot_count,
"SEND_SNAPSHOT: hook must fire once when standby_lsn < oldest_wal_lsn");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_send_snapshot_idx,
"SEND_SNAPSHOT: hook must receive correct peer_idx=0");
ha_fini(ha);
ha_repl_send_snapshot_hook = NULL;
printf("test_dispatch_repl_action_send_snapshot: PASS\n");
#else
printf("test_dispatch_repl_action_send_snapshot: SKIP (non-test build)\n");
#endif
}
/* ── Task 3.2: ha_evaluate_repl_action unified decision tree ─────────────── */
/*
* Declare the function under test. It is defined in ha.c.
* Pure function — no side effects — so no XCE_TESTING guard needed.
*/
ha_repl_action_t ha_evaluate_repl_action(uint64_t standby_tag,
uint64_t standby_lsn,
uint64_t committed_tag,
uint64_t committed_lsn,
uint64_t oldest_wal_lsn);
/*
* Test: (tag=0, lsn=0) with committed_tag=5, committed_lsn=100,
* oldest_wal_lsn=50 → PUSH_CONFIG (tag behind).
* Requirements: 2.2, 2.3 (ha-repl-hardening)
*/
static void test_evaluate_repl_action_push_config(void)
{
ha_repl_action_t action = ha_evaluate_repl_action(
/*standby_tag=*/0, /*standby_lsn=*/0,
/*committed_tag=*/5, /*committed_lsn=*/100,
/*oldest_wal_lsn=*/50);
TEST_ASSERT_EQUAL_INT(HA_REPL_PUSH_CONFIG, action);
printf("test_evaluate_repl_action_push_config: PASS\n");
}
/*
* Test: (tag=5, lsn=0) with committed_tag=5, committed_lsn=100,
* oldest_wal_lsn=50 → SEND_SNAPSHOT (lsn=0 is below WAL retention window).
* Requirements: 2.2, 2.3
*/
static void test_evaluate_repl_action_send_snapshot_lsn_zero(void)
{
ha_repl_action_t action = ha_evaluate_repl_action(
/*standby_tag=*/5, /*standby_lsn=*/0,
/*committed_tag=*/5, /*committed_lsn=*/100,
/*oldest_wal_lsn=*/50);
/* lsn=0 < oldest_wal_lsn=50 → SEND_SNAPSHOT */
TEST_ASSERT_EQUAL_INT(HA_REPL_SEND_SNAPSHOT, action);
printf("test_evaluate_repl_action_send_snapshot_lsn_zero: PASS\n");
}
/*
* Test: (tag=5, lsn=100) with committed_tag=5, committed_lsn=100 → CONVERGED.
* Requirements: 2.2, 2.4
*/
static void test_evaluate_repl_action_converged(void)
{
ha_repl_action_t action = ha_evaluate_repl_action(
/*standby_tag=*/5, /*standby_lsn=*/100,
/*committed_tag=*/5, /*committed_lsn=*/100,
/*oldest_wal_lsn=*/50);
TEST_ASSERT_EQUAL_INT(HA_REPL_CONVERGED, action);
printf("test_evaluate_repl_action_converged: PASS\n");
}
/*
* Test: (tag=5, lsn=10) with committed_tag=5, committed_lsn=100,
* oldest_wal_lsn=50 → SEND_SNAPSHOT (lsn < oldest_wal).
* Requirements: 2.2, 2.3
*/
static void test_evaluate_repl_action_send_snapshot(void)
{
ha_repl_action_t action = ha_evaluate_repl_action(
/*standby_tag=*/5, /*standby_lsn=*/10,
/*committed_tag=*/5, /*committed_lsn=*/100,
/*oldest_wal_lsn=*/50);
TEST_ASSERT_EQUAL_INT(HA_REPL_SEND_SNAPSHOT, action);
printf("test_evaluate_repl_action_send_snapshot: PASS\n");
}
/*
* Test: (tag=5, lsn=50) with committed_tag=5, committed_lsn=100,
* oldest_wal_lsn=50 → STREAM_WAL (lsn >= oldest_wal but < committed).
* Requirements: 2.2, 2.3
*/
static void test_evaluate_repl_action_stream_wal(void)
{
ha_repl_action_t action = ha_evaluate_repl_action(
/*standby_tag=*/5, /*standby_lsn=*/50,
/*committed_tag=*/5, /*committed_lsn=*/100,
/*oldest_wal_lsn=*/50);
TEST_ASSERT_EQUAL_INT(HA_REPL_STREAM_WAL, action);
printf("test_evaluate_repl_action_stream_wal: PASS\n");
}
/*
* Test: (tag=T, lsn=0) with oldest_wal_lsn=0 (fresh system, no WAL entries yet)
* → STREAM_WAL (lsn=0 >= oldest_wal_lsn=0, so not in snapshot territory;
* lsn=0 < committed_lsn → stream WAL from the beginning).
*
* This is the canonical Task 6.1 case: a standby that has caught up on config
* (tag matches) but has no WAL applied yet on a fresh system. When there is no
* WAL retention window to worry about (oldest_wal_lsn == 0), the correct action
* is STREAM_WAL, not SEND_SNAPSHOT.
*
* Requirements: 1.1, 1.2, 2.2, 2.3 (ha-repl-hardening)
*/
static void test_evaluate_repl_action_stream_wal_fresh_system(void)
{
ha_repl_action_t action = ha_evaluate_repl_action(
/*standby_tag=*/5, /*standby_lsn=*/0,
/*committed_tag=*/5, /*committed_lsn=*/100,
/*oldest_wal_lsn=*/0);
/* lsn=0 >= oldest_wal_lsn=0 → NOT snapshot; lsn=0 < committed → STREAM_WAL */
TEST_ASSERT_EQUAL_INT_MESSAGE(HA_REPL_STREAM_WAL, action,
"(tag=T, lsn=0) with oldest_wal_lsn=0 must route to STREAM_WAL (Req 1.1, 1.2)");
printf("test_evaluate_repl_action_stream_wal_fresh_system: PASS\n");
}
/* ── Task 5.3: committing timeout default value and degraded-mode WARN ───────
*
* Requirements: 6.4, 7.5 (ha-repl-hardening)
*
* Verifies:
* - HA_DEFAULT_COMMITTING_TIMEOUT_MS == 60000 (safety-net window is 60 s)
* - committing_timeout_fn_for_test still promotes to CTRL_PRIMARY in degraded
* mode even when only a subset of standbys have converged
* - The ha->standby_converged[] state is readable so the WARN log can include
* the converged/pending counts
*/
/*
* Test: HA_DEFAULT_COMMITTING_TIMEOUT_MS is 60000 (not 10000).
* Requirements: 6.4 (ha-repl-hardening) — "Change committing_timeout_ms default from 10000 to 60000"
*/
static void test_committing_timeout_default_is_60s(void)
{
TEST_ASSERT_EQUAL_UINT32_MESSAGE(60000, HA_DEFAULT_COMMITTING_TIMEOUT_MS,
"HA_DEFAULT_COMMITTING_TIMEOUT_MS must be 60000 (Req 6.4, Task 5.3)");
printf("test_committing_timeout_default_is_60s: PASS\n");
}
/*
* Test: committing_timeout_fn promotes to PRIMARY even when only some standbys
* have converged (degraded mode). The WARN log path fires when timeout expires
* before majority convergence. After promotion, role must be CTRL_PRIMARY.
*
* Requirements: 6.4, 7.5 (ha-repl-hardening)
*/
#ifdef XCE_TESTING
extern int committing_timeout_fn_for_test(ha_state_t *ha, cluster_t *c,
int cancelled_val);
#endif
static void test_committing_timeout_promotes_degraded_with_partial_convergence(void)
{
#ifdef XCE_TESTING
cluster_t c;
memset(&c, 0, sizeof(c));
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
};
ha_init(&c.ha, &cfg, 0);
c.ha.n_peers = 3; /* 3-node cluster; 2 standbys */
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
memory_order_release);
/* Only 1 of 2 standbys has converged — majority not yet reached */
c.ha.standby_converged[0] = true;
c.ha.standby_converged[1] = false;
/*
* Count converged/pending so the test mirrors what the WARN log reads.
* n_converged=1, n_pending=1 (Req 7.5).
*/
int n_converged = 0;
int n_pending = 0;
for (int i = 0; i < c.ha.n_peers - 1; i++) {
if (c.ha.standby_converged[i])
n_converged++;
else
n_pending++;
}
TEST_ASSERT_EQUAL_INT_MESSAGE(1, n_converged,
"degraded-mode: 1 standby converged before timeout");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, n_pending,
"degraded-mode: 1 standby still pending before timeout");
/* Fire the timeout — must promote despite partial convergence */
committing_timeout_fn_for_test(&c.ha, &c, 0 /* not cancelled */);
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
"committing timeout: must promote to CTRL_PRIMARY in degraded mode (Req 6.4)");
ha_fini(&c.ha);
printf("test_committing_timeout_promotes_degraded_with_partial_convergence: PASS "
"(n_converged=%d, n_pending=%d)\n", n_converged, n_pending);
#else
printf("test_committing_timeout_promotes_degraded_with_partial_convergence: SKIP\n");
#endif
}
/*
* Test: committing_timeout_fn promotes when NO standbys have converged
* (worst-case degraded mode — all standbys unreachable).
* Requirements: 6.4 (ha-repl-hardening)
*/
static void test_committing_timeout_promotes_degraded_zero_converged(void)
{
#ifdef XCE_TESTING
cluster_t c;
memset(&c, 0, sizeof(c));
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
};
ha_init(&c.ha, &cfg, 0);
c.ha.n_peers = 3; /* 3-node cluster; all standbys unreachable */
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
memory_order_release);
/* No standbys converged — n_converged=0, n_pending=2 (Req 7.5) */
c.ha.standby_converged[0] = false;
c.ha.standby_converged[1] = false;
committing_timeout_fn_for_test(&c.ha, &c, 0 /* not cancelled */);
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
"committing timeout: must promote even with 0 converged standbys (degraded, Req 6.4)");
ha_fini(&c.ha);
printf("test_committing_timeout_promotes_degraded_zero_converged: PASS\n");
#else
printf("test_committing_timeout_promotes_degraded_zero_converged: SKIP\n");
#endif
}
/* ── Task 3.3 (ha-wal-replication): dispatch gate allows OPCode_walCatchupEntry
* in CTRL_STANDBY, CTRL_ELECTING, and CTRL_INIT.
*
* Requirements: ha-wal-replication Req 3.3 (standby role guard).
* Verifies that ha_dispatch_gate(ha, OPCode_walCatchupEntry) returns 0
* (allow) for the three roles where a standby legitimately receives WAL
* catch-up entries from the primary. CTRL_PRIMARY and
* CTRL_PRIMARY_COMMITTING must NOT allow it (they never receive WAL pushes). */
static void test_dispatch_gate_allows_wal_catchup_in_standby_roles(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
};
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
/* CTRL_STANDBY — must allow */
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
ha_dispatch_gate(&ha, OPCode_walCatchupEntry),
"CTRL_STANDBY must allow OPCode_walCatchupEntry (Req 3.3)");
/* CTRL_ELECTING — must allow */
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
ha_dispatch_gate(&ha, OPCode_walCatchupEntry),
"CTRL_ELECTING must allow OPCode_walCatchupEntry (Req 3.3)");
/* CTRL_INIT — must allow */
atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release);
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
ha_dispatch_gate(&ha, OPCode_walCatchupEntry),
"CTRL_INIT must allow OPCode_walCatchupEntry (Req 3.3)");
ha_fini(&ha);
printf("test_dispatch_gate_allows_wal_catchup_in_standby_roles: PASS\n");
}
void test_ha_role(void)
{
RUN_TEST(test_ha_init_default_role_is_standby);
RUN_TEST(test_ha_init_rejects_fast_heartbeat);
RUN_TEST(test_ha_init_rejects_short_election_timeout);
RUN_TEST(test_ha_init_rejects_timeout_less_than_interval);
RUN_TEST(test_ha_promote_sets_primary_role);
/* P1-3a */
RUN_TEST(test_promote_increments_term);
RUN_TEST(test_demote_sets_standby_role);
/* Watchdog / SIGUSR1 tests deleted */
/* Task 2.1: election-iomgr-refactor new fields */
RUN_TEST(test_ha_init_zeroes_new_fields);
/* Task 1.4: ha-repl-hardening per-standby convergence fields */
RUN_TEST(test_ha_init_zeroes_standby_convergence_fields);
/* Task 3.1: ha_process_standby_state convergence tracking */
RUN_TEST(test_process_standby_state_converged);
RUN_TEST(test_process_standby_state_lsn_behind);
RUN_TEST(test_process_standby_state_tag_behind);
RUN_TEST(test_process_standby_state_regression);
RUN_TEST(test_process_standby_state_multiple_peers);
/* Task 3.2: ha_evaluate_repl_action unified decision tree */
RUN_TEST(test_evaluate_repl_action_push_config);
RUN_TEST(test_evaluate_repl_action_send_snapshot_lsn_zero);
RUN_TEST(test_evaluate_repl_action_converged);
RUN_TEST(test_evaluate_repl_action_send_snapshot);
RUN_TEST(test_evaluate_repl_action_stream_wal);
/* Task 6.1: (tag=T, lsn=0) with oldest_wal_lsn=0 → STREAM_WAL */
RUN_TEST(test_evaluate_repl_action_stream_wal_fresh_system);
/* Task 3.3: ha_dispatch_repl_action_for_standby PUSH_CONFIG wiring */
RUN_TEST(test_dispatch_repl_action_push_config_when_tag_behind);
RUN_TEST(test_dispatch_repl_action_no_push_when_converged);
/* Task 3.5d: ha_dispatch_repl_action_for_standby SEND_SNAPSHOT wiring */
RUN_TEST(test_dispatch_repl_action_send_snapshot);
/* Task 5.2: ha_check_majority_convergence → PRIMARY transition */
RUN_TEST(test_majority_convergence_3node_none_stays_committing);
RUN_TEST(test_majority_convergence_3node_one_standby_promotes);
RUN_TEST(test_majority_convergence_5node_threshold);
RUN_TEST(test_majority_convergence_3node_all_standbys_promotes);
RUN_TEST(test_majority_convergence_single_node_immediate);
/* Task 5.3: committing timeout default value and degraded-mode WARN */
RUN_TEST(test_committing_timeout_default_is_60s);
RUN_TEST(test_committing_timeout_promotes_degraded_with_partial_convergence);
RUN_TEST(test_committing_timeout_promotes_degraded_zero_converged);
/* Task 3.3 (ha-wal-replication): dispatch gate standby-role guard */
RUN_TEST(test_dispatch_gate_allows_wal_catchup_in_standby_roles);
}
/* test_am_i_primary suite deleted — am_i_primary() removed (election-iomgr-refactor) */