Standby sends WalAckReply via xce_reply after each successfully applied WAL entry. ha_apply_wal_catchup_entry signature extended with endpoint_t* and cluster_t* parameters. walackreplymsg union arm added to RequestReply capnp schema and xce_reply_t. Error entries skip ACK. Dispatch gate test confirms OPCode_walCatchupEntry allowed in CTRL_STANDBY/ELECTING/INIT. Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
1102 lines
41 KiB
C
1102 lines
41 KiB
C
#include <signal.h>
|
|
#include <stdatomic.h>
|
|
#include <string.h>
|
|
#include <stdio.h>
|
|
#include <time.h>
|
|
|
|
#include <unity/unity.h>
|
|
|
|
#include "ha.h"
|
|
#include "myconf.h"
|
|
#include "xce/cluster_def.h"
|
|
|
|
static void test_ha_init_default_role_is_standby(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
|
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
|
|
ha_fini(&ha);
|
|
printf("test_ha_init_default_role_is_standby: PASS\n");
|
|
}
|
|
|
|
static void test_ha_init_rejects_fast_heartbeat(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 1999, /* below 2000ms floor */
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_NOT_EQUAL(0, ha_init(&ha, &cfg, 0));
|
|
printf("test_ha_init_rejects_fast_heartbeat: PASS\n");
|
|
}
|
|
|
|
static void test_ha_init_rejects_short_election_timeout(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 499, /* below 500ms floor */
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_NOT_EQUAL(0, ha_init(&ha, &cfg, 0));
|
|
printf("test_ha_init_rejects_short_election_timeout: PASS\n");
|
|
}
|
|
|
|
static void test_ha_init_rejects_timeout_less_than_interval(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 5000,
|
|
.heartbeat_timeout_ms = 4999, /* must be >= heartbeat_interval_ms */
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_NOT_EQUAL(0, ha_init(&ha, &cfg, 0));
|
|
printf("test_ha_init_rejects_timeout_less_than_interval: PASS\n");
|
|
}
|
|
|
|
static void test_ha_promote_sets_primary_role(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
ha_init(&ha, &cfg, 0);
|
|
/* Directly set role to PRIMARY (stub path — full promotion in P1-3a) */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release);
|
|
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
|
|
ha_fini(&ha);
|
|
printf("test_ha_promote_sets_primary_role: PASS\n");
|
|
}
|
|
|
|
/* Watchdog / SIGUSR1 tests deleted — watchdog pthread replaced by
|
|
* ha_arm_heartbeat_timeout_vtask (delaymgr); failover_requested removed */
|
|
|
|
/* ── P1-3a tests ─────────────────────────────────────────────── */
|
|
|
|
/* Term must increment on each promotion attempt */
|
|
static void test_promote_increments_term(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
ha_init(&ha, &cfg, 0);
|
|
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
|
|
|
|
ha_do_promote(&ha); /* internal promotion logic — no cluster_t needed */
|
|
|
|
uint64_t term_after = atomic_load_explicit(&ha.term, memory_order_acquire);
|
|
TEST_ASSERT_TRUE(term_after > term_before);
|
|
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
|
|
|
|
ha_fini(&ha);
|
|
printf("test_promote_increments_term: PASS\n");
|
|
}
|
|
|
|
/* Demotion must set role back to STANDBY */
|
|
static void test_demote_sets_standby_role(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
ha_init(&ha, &cfg, 0);
|
|
ha_do_promote(&ha);
|
|
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
|
|
|
|
ha_do_demote(&ha);
|
|
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
|
|
|
|
ha_fini(&ha);
|
|
printf("test_demote_sets_standby_role: PASS\n");
|
|
}
|
|
|
|
/* test_sigusr1_sets_failover_requested deleted — SIGUSR1 / failover_requested removed */
|
|
|
|
/* am_i_primary() tests deleted — function removed (election-iomgr-refactor) */
|
|
|
|
/* ── Task 2.1: ha_state_t new fields zero-initialised by ha_init ──────── */
|
|
|
|
/*
|
|
* Verify that ha_init zeroes the new fields added for election-iomgr-refactor:
|
|
* phase2_done, repl_ack_count, n_peers, client_ssl_ctx
|
|
* and that CTRL_PRIMARY_COMMITTING is a distinct enum value in ctrl_role_t.
|
|
* Also verifies ha_config_t.committing_timeout_ms default macro exists.
|
|
*
|
|
* Requirements: 5.2, 8.1 (election-iomgr-refactor spec)
|
|
*/
|
|
static void test_ha_init_zeroes_new_fields(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
|
};
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
|
|
|
/* phase2_done must be explicitly zero after ha_init */
|
|
TEST_ASSERT_EQUAL_INT(0, ha.phase2_done);
|
|
|
|
/* repl_ack_count is _Atomic int; must be zero after ha_init */
|
|
TEST_ASSERT_EQUAL_INT(0, atomic_load_explicit(&ha.repl_ack_count, memory_order_relaxed));
|
|
|
|
/* n_peers must be zero after ha_init */
|
|
TEST_ASSERT_EQUAL_INT(0, ha.n_peers);
|
|
|
|
/* client_ssl_ctx must be NULL after ha_init */
|
|
TEST_ASSERT_NULL(ha.client_ssl_ctx);
|
|
|
|
/* committing_timeout_ms stored in config */
|
|
TEST_ASSERT_EQUAL_UINT32(HA_DEFAULT_COMMITTING_TIMEOUT_MS, ha.config.committing_timeout_ms);
|
|
|
|
/* CTRL_PRIMARY_COMMITTING must be a distinct enum value */
|
|
TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY_COMMITTING, (int)CTRL_PRIMARY);
|
|
TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY_COMMITTING, (int)CTRL_STANDBY);
|
|
TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY_COMMITTING, (int)CTRL_INIT);
|
|
TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY_COMMITTING, (int)CTRL_ELECTING);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_ha_init_zeroes_new_fields: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Task 1.4: per-standby convergence tracking fields zero-initialised by ha_init.
|
|
*
|
|
* Verifies:
|
|
* - standby_reported_tag[i] == 0 for all i after ha_init()
|
|
* - standby_reported_lsn[i] == 0 for all i after ha_init()
|
|
* - standby_converged[i] == false for all i after ha_init()
|
|
* - fields are writable from test context
|
|
*
|
|
* Requirements: 2.1 (ha-repl-hardening spec)
|
|
*/
|
|
static void test_ha_init_zeroes_standby_convergence_fields(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
|
};
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
|
|
|
/* All per-standby tracking arrays must be zero/false after ha_init */
|
|
for (int i = 0; i < MAX_HA_CANDIDATES; i++) {
|
|
TEST_ASSERT_EQUAL_UINT64(0, ha.standby_reported_tag[i]);
|
|
TEST_ASSERT_EQUAL_UINT64(0, ha.standby_reported_lsn[i]);
|
|
TEST_ASSERT_FALSE(ha.standby_converged[i]);
|
|
}
|
|
|
|
/* Fields must be writable */
|
|
ha.standby_reported_tag[0] = 42ULL;
|
|
ha.standby_reported_lsn[0] = 100ULL;
|
|
ha.standby_converged[0] = true;
|
|
TEST_ASSERT_EQUAL_UINT64(42ULL, ha.standby_reported_tag[0]);
|
|
TEST_ASSERT_EQUAL_UINT64(100ULL, ha.standby_reported_lsn[0]);
|
|
TEST_ASSERT_TRUE(ha.standby_converged[0]);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_ha_init_zeroes_standby_convergence_fields: PASS\n");
|
|
}
|
|
|
|
/* ── Task 3.1: ha_process_standby_state convergence tracking ─────────────── */
|
|
|
|
/*
|
|
* Declare the function under test. It is defined in ha.c and exposed only in
|
|
* XCE_TESTING builds (forward declaration matches the implementation).
|
|
*/
|
|
#ifdef XCE_TESTING
|
|
void ha_process_standby_state(ha_state_t *ha, int peer_idx,
|
|
uint64_t reported_tag, uint64_t reported_lsn,
|
|
uint64_t committed_tag, uint64_t committed_lsn);
|
|
#endif
|
|
|
|
/* Helper: build a ha_state_t with all convergence fields zeroed. */
|
|
static ha_state_t *build_ha_for_convergence_test(void)
|
|
{
|
|
static ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
|
};
|
|
ha_init(&ha, &cfg, 0);
|
|
return &ha;
|
|
}
|
|
|
|
/*
|
|
* Test: standby with lsn == committed_lsn and tag == committed_tag → converged.
|
|
* Fields standby_reported_tag[0] and standby_reported_lsn[0] updated correctly.
|
|
* Requirements: 1.2, 2.2 (ha-repl-hardening)
|
|
*/
|
|
static void test_process_standby_state_converged(void)
|
|
{
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
ha_process_standby_state(ha, 0,
|
|
/*reported_tag=*/5, /*reported_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100);
|
|
|
|
TEST_ASSERT_EQUAL_UINT64(5, ha->standby_reported_tag[0]);
|
|
TEST_ASSERT_EQUAL_UINT64(100, ha->standby_reported_lsn[0]);
|
|
TEST_ASSERT_TRUE(ha->standby_converged[0]);
|
|
|
|
ha_fini(ha);
|
|
printf("test_process_standby_state_converged: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: standby lsn < committed_lsn → not converged.
|
|
* Requirements: 1.2, 2.2
|
|
*/
|
|
static void test_process_standby_state_lsn_behind(void)
|
|
{
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
ha_process_standby_state(ha, 0,
|
|
/*reported_tag=*/5, /*reported_lsn=*/50,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100);
|
|
|
|
TEST_ASSERT_EQUAL_UINT64(5, ha->standby_reported_tag[0]);
|
|
TEST_ASSERT_EQUAL_UINT64(50, ha->standby_reported_lsn[0]);
|
|
TEST_ASSERT_FALSE(ha->standby_converged[0]);
|
|
|
|
ha_fini(ha);
|
|
printf("test_process_standby_state_lsn_behind: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: standby tag < committed_tag → not converged.
|
|
* Requirements: 1.2, 2.2
|
|
*/
|
|
static void test_process_standby_state_tag_behind(void)
|
|
{
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
ha_process_standby_state(ha, 0,
|
|
/*reported_tag=*/3, /*reported_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100);
|
|
|
|
TEST_ASSERT_EQUAL_UINT64(3, ha->standby_reported_tag[0]);
|
|
TEST_ASSERT_EQUAL_UINT64(100, ha->standby_reported_lsn[0]);
|
|
TEST_ASSERT_FALSE(ha->standby_converged[0]);
|
|
|
|
ha_fini(ha);
|
|
printf("test_process_standby_state_tag_behind: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: convergence regression — standby was converged; primary advances;
|
|
* next update with old values should reset standby_converged to false.
|
|
* Requirements: 2.5 (converged standby regresses → mark not-converged)
|
|
*/
|
|
static void test_process_standby_state_regression(void)
|
|
{
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
/* First update: converged at (tag=5, lsn=100) */
|
|
ha_process_standby_state(ha, 0,
|
|
/*reported_tag=*/5, /*reported_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100);
|
|
TEST_ASSERT_TRUE(ha->standby_converged[0]);
|
|
|
|
/* Primary advances committed_lsn to 200; standby still reports (5, 100) */
|
|
ha_process_standby_state(ha, 0,
|
|
/*reported_tag=*/5, /*reported_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/200);
|
|
TEST_ASSERT_FALSE(ha->standby_converged[0]);
|
|
|
|
ha_fini(ha);
|
|
printf("test_process_standby_state_regression: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: different standby indices updated independently.
|
|
* Requirements: 2.2
|
|
*/
|
|
static void test_process_standby_state_multiple_peers(void)
|
|
{
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
/* peer 0: converged */
|
|
ha_process_standby_state(ha, 0,
|
|
/*reported_tag=*/5, /*reported_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100);
|
|
/* peer 1: not converged (tag behind) */
|
|
ha_process_standby_state(ha, 1,
|
|
/*reported_tag=*/3, /*reported_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100);
|
|
|
|
TEST_ASSERT_TRUE(ha->standby_converged[0]);
|
|
TEST_ASSERT_FALSE(ha->standby_converged[1]);
|
|
TEST_ASSERT_EQUAL_UINT64(5, ha->standby_reported_tag[0]);
|
|
TEST_ASSERT_EQUAL_UINT64(3, ha->standby_reported_tag[1]);
|
|
|
|
ha_fini(ha);
|
|
printf("test_process_standby_state_multiple_peers: PASS\n");
|
|
}
|
|
|
|
/* ── Task 5.2: ha_check_majority_convergence → PRIMARY transition ────────────
|
|
*
|
|
* After ha_process_standby_state updates standby_converged[], the primary must
|
|
* count the number of converged standbys and promote to CTRL_PRIMARY when the
|
|
* majority threshold is met and the current role is CTRL_PRIMARY_COMMITTING.
|
|
*
|
|
* Threshold formula (Req 2.3): ceil(n_peers/2) - 1 converged standbys —
|
|
* i.e. primary + converged standbys form a majority of the cluster.
|
|
* Integer form: n_peers / 2 standbys (n=3 → 1, n=5 → 2).
|
|
* Single-node (n_peers<=1): threshold=0 → satisfied immediately.
|
|
* n_peers = total cluster size including self.
|
|
*
|
|
* Requirements: 2.3, 2.4, 6.1, 6.3, 6.5 (ha-repl-hardening)
|
|
*/
|
|
#ifdef XCE_TESTING
|
|
void ha_check_majority_convergence(ha_state_t *ha, cluster_t *c);
|
|
#endif
|
|
|
|
/*
|
|
* Test: 3-node cluster; no standbys converged (0/2)
|
|
* → role must remain CTRL_PRIMARY_COMMITTING.
|
|
* Requirements: 2.3, 6.3 (ha-repl-hardening)
|
|
*/
|
|
static void test_majority_convergence_3node_none_stays_committing(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
cluster_t c;
|
|
memset(&c, 0, sizeof(c));
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
|
};
|
|
ha_init(&c.ha, &cfg, 0);
|
|
c.ha.n_peers = 3; /* 3-node cluster: primary + 2 standbys */
|
|
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
|
memory_order_release);
|
|
|
|
/* No standby converged */
|
|
c.ha.standby_converged[0] = false;
|
|
c.ha.standby_converged[1] = false;
|
|
|
|
ha_check_majority_convergence(&c.ha, &c);
|
|
|
|
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, role,
|
|
"3-node: 0/2 standbys converged → must remain CTRL_PRIMARY_COMMITTING (Req 2.3)");
|
|
|
|
ha_fini(&c.ha);
|
|
printf("test_majority_convergence_3node_none_stays_committing: PASS\n");
|
|
#else
|
|
printf("test_majority_convergence_3node_none_stays_committing: SKIP\n");
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* Test: 3-node cluster; 1/2 standbys converged → primary + 1 standby = 2/3
|
|
* nodes hold the state = majority → must promote to CTRL_PRIMARY.
|
|
* (Requirement 2.3 formula: ceil(3/2)-1 = 1 converged standby suffices.
|
|
* A dead minority standby must not force the degraded-mode timeout.)
|
|
* Requirements: 2.3, 2.4, 6.3 (ha-repl-hardening)
|
|
*/
|
|
static void test_majority_convergence_3node_one_standby_promotes(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
cluster_t c;
|
|
memset(&c, 0, sizeof(c));
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
|
};
|
|
ha_init(&c.ha, &cfg, 0);
|
|
c.ha.n_peers = 3;
|
|
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
|
memory_order_release);
|
|
|
|
/* Only standby[0] converged; standby[1] not converged */
|
|
c.ha.standby_converged[0] = true;
|
|
c.ha.standby_converged[1] = false;
|
|
|
|
ha_check_majority_convergence(&c.ha, &c);
|
|
|
|
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
|
|
"3-node: 1/2 standbys converged → majority → CTRL_PRIMARY (Req 2.3)");
|
|
|
|
ha_fini(&c.ha);
|
|
printf("test_majority_convergence_3node_one_standby_promotes: PASS\n");
|
|
#else
|
|
printf("test_majority_convergence_3node_one_standby_promotes: SKIP\n");
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* Test: 5-node cluster; threshold = 5/2 = 2 converged standbys.
|
|
* 1/4 converged → stays COMMITTING; 2/4 converged → promotes.
|
|
* Requirements: 2.3, 6.3 (ha-repl-hardening)
|
|
*/
|
|
static void test_majority_convergence_5node_threshold(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
cluster_t c;
|
|
memset(&c, 0, sizeof(c));
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
|
};
|
|
ha_init(&c.ha, &cfg, 0);
|
|
c.ha.n_peers = 5;
|
|
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
|
memory_order_release);
|
|
|
|
/* 1/4 converged: primary + 1 = 2 of 5 — not a majority → stays */
|
|
c.ha.standby_converged[0] = true;
|
|
ha_check_majority_convergence(&c.ha, &c);
|
|
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, role,
|
|
"5-node: 1/4 standbys converged → must remain COMMITTING (Req 2.3)");
|
|
|
|
/* 2/4 converged: primary + 2 = 3 of 5 — majority → promotes */
|
|
c.ha.standby_converged[1] = true;
|
|
ha_check_majority_convergence(&c.ha, &c);
|
|
role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
|
|
"5-node: 2/4 standbys converged → majority → CTRL_PRIMARY (Req 2.3)");
|
|
|
|
ha_fini(&c.ha);
|
|
printf("test_majority_convergence_5node_threshold: PASS\n");
|
|
#else
|
|
printf("test_majority_convergence_5node_threshold: SKIP\n");
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* Test: 3-node cluster (n_peers=2); both standbys converged (2/2)
|
|
* → role must transition to CTRL_PRIMARY.
|
|
* Requirements: 2.3, 2.4, 6.3 (ha-repl-hardening)
|
|
*/
|
|
static void test_majority_convergence_3node_all_standbys_promotes(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
cluster_t c;
|
|
memset(&c, 0, sizeof(c));
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
|
};
|
|
ha_init(&c.ha, &cfg, 0);
|
|
c.ha.n_peers = 3; /* 3-node cluster */
|
|
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
|
memory_order_release);
|
|
|
|
/* Both standbys converged */
|
|
c.ha.standby_converged[0] = true;
|
|
c.ha.standby_converged[1] = true;
|
|
|
|
ha_check_majority_convergence(&c.ha, &c);
|
|
|
|
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
|
|
"3-node: 2/2 standbys converged → must transition to CTRL_PRIMARY (Req 2.3, 2.4)");
|
|
|
|
ha_fini(&c.ha);
|
|
printf("test_majority_convergence_3node_all_standbys_promotes: PASS\n");
|
|
#else
|
|
printf("test_majority_convergence_3node_all_standbys_promotes: SKIP\n");
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* Test: single-node cluster (n_peers=0)
|
|
* → threshold=0, satisfied immediately → must promote to CTRL_PRIMARY.
|
|
* Requirements: 6.5, 2.3 (ha-repl-hardening): single-node is always satisfied.
|
|
*/
|
|
static void test_majority_convergence_single_node_immediate(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
cluster_t c;
|
|
memset(&c, 0, sizeof(c));
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
|
};
|
|
ha_init(&c.ha, &cfg, 0);
|
|
c.ha.n_peers = 1; /* single-node: just self */
|
|
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
|
memory_order_release);
|
|
|
|
/* No standbys to converge — threshold=0 must be satisfied immediately */
|
|
ha_check_majority_convergence(&c.ha, &c);
|
|
|
|
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
|
|
"single-node: n_peers=1 → threshold=0 satisfied → must promote to CTRL_PRIMARY (Req 6.5)");
|
|
|
|
ha_fini(&c.ha);
|
|
printf("test_majority_convergence_single_node_immediate: PASS\n");
|
|
#else
|
|
printf("test_majority_convergence_single_node_immediate: SKIP\n");
|
|
#endif
|
|
}
|
|
|
|
/* ── Task 3.3: ha_dispatch_repl_action_for_standby PUSH_CONFIG wiring ──────
|
|
*
|
|
* ha_dispatch_repl_action_for_standby reads the standby's stored
|
|
* (standby_reported_tag, standby_reported_lsn), calls ha_evaluate_repl_action,
|
|
* and if the result is HA_REPL_PUSH_CONFIG calls push_config_to_standbys (in
|
|
* production) or ha_repl_push_config_for_standby_hook (in XCE_TESTING).
|
|
*
|
|
* Requirements: 1.4 (ha-repl-hardening), design §Unified Heartbeat-Driven Protocol
|
|
*/
|
|
#ifdef XCE_TESTING
|
|
extern int (*ha_repl_push_config_for_standby_hook)(void *c, int peer_idx);
|
|
#endif
|
|
|
|
/* Call-counting state for the hook */
|
|
static int g_push_config_for_standby_count = 0;
|
|
static int g_push_config_for_standby_idx = -1;
|
|
|
|
static int stub_repl_push_config_for_standby(void *c, int peer_idx)
|
|
{
|
|
(void)c;
|
|
g_push_config_for_standby_count++;
|
|
g_push_config_for_standby_idx = peer_idx;
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Test: when standby_reported_tag < committed_tag, ha_dispatch_repl_action_for_standby
|
|
* must call push_config_to_standbys (via hook in test mode).
|
|
*
|
|
* Steps:
|
|
* 1. Set ha->standby_reported_tag[0] = 3 (behind committed_tag=5)
|
|
* 2. Set ha->standby_reported_lsn[0] = 100 (at committed_lsn)
|
|
* 3. Call ha_dispatch_repl_action_for_standby(ha, NULL, 0, committed_tag=5, ...)
|
|
* 4. Assert hook called once with peer_idx=0
|
|
*
|
|
* Requirements: 1.4 (push_config when standby tag < committed_tag)
|
|
*/
|
|
static void test_dispatch_repl_action_push_config_when_tag_behind(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
ha_repl_push_config_for_standby_hook = stub_repl_push_config_for_standby;
|
|
g_push_config_for_standby_count = 0;
|
|
g_push_config_for_standby_idx = -1;
|
|
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
/* standby[0] has stale config tag */
|
|
ha->standby_reported_tag[0] = 3;
|
|
ha->standby_reported_lsn[0] = 100;
|
|
|
|
int rc = ha_dispatch_repl_action_for_standby(
|
|
ha, /*c=*/NULL, /*ep=*/NULL, /*peer_idx=*/0,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/0);
|
|
(void)rc;
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_push_config_for_standby_count,
|
|
"PUSH_CONFIG: hook must fire once when standby_tag < committed_tag");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_push_config_for_standby_idx,
|
|
"PUSH_CONFIG: hook must receive correct peer_idx=0");
|
|
|
|
ha_fini(ha);
|
|
ha_repl_push_config_for_standby_hook = NULL;
|
|
printf("test_dispatch_repl_action_push_config_when_tag_behind: PASS\n");
|
|
#else
|
|
printf("test_dispatch_repl_action_push_config_when_tag_behind: SKIP (non-test build)\n");
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* Test: when standby is converged (tag == committed, lsn == committed),
|
|
* ha_dispatch_repl_action_for_standby must NOT call push_config hook.
|
|
*
|
|
* Requirements: decision tree — CONVERGED does not trigger push
|
|
*/
|
|
static void test_dispatch_repl_action_no_push_when_converged(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
ha_repl_push_config_for_standby_hook = stub_repl_push_config_for_standby;
|
|
g_push_config_for_standby_count = 0;
|
|
g_push_config_for_standby_idx = -1;
|
|
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
/* standby[0] is fully converged */
|
|
ha->standby_reported_tag[0] = 5;
|
|
ha->standby_reported_lsn[0] = 100;
|
|
|
|
int rc = ha_dispatch_repl_action_for_standby(
|
|
ha, /*c=*/NULL, /*ep=*/NULL, /*peer_idx=*/0,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/0);
|
|
(void)rc;
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_push_config_for_standby_count,
|
|
"CONVERGED: hook must NOT fire when standby is already converged");
|
|
|
|
ha_fini(ha);
|
|
ha_repl_push_config_for_standby_hook = NULL;
|
|
printf("test_dispatch_repl_action_no_push_when_converged: PASS\n");
|
|
#else
|
|
printf("test_dispatch_repl_action_no_push_when_converged: SKIP (non-test build)\n");
|
|
#endif
|
|
}
|
|
|
|
/* ── Task 3.5d: ha_dispatch_repl_action_for_standby SEND_SNAPSHOT wiring ────
|
|
*
|
|
* When standby_lsn < oldest_wal_lsn (standby predates WAL retention window),
|
|
* ha_dispatch_repl_action_for_standby must call ha_send_snapshot_to_standby
|
|
* (via ha_repl_send_snapshot_hook in XCE_TESTING builds).
|
|
*
|
|
* Requirements: 4.1 (ha-repl-hardening), design §Snapshot Transfer
|
|
*/
|
|
#ifdef XCE_TESTING
|
|
extern int (*ha_repl_send_snapshot_hook)(int peer_idx);
|
|
#endif
|
|
|
|
/* Call-counting state for the snapshot hook */
|
|
static int g_send_snapshot_count = 0;
|
|
static int g_send_snapshot_idx = -1;
|
|
|
|
static int stub_repl_send_snapshot(int peer_idx)
|
|
{
|
|
g_send_snapshot_count++;
|
|
g_send_snapshot_idx = peer_idx;
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Test: when standby_lsn < oldest_wal_lsn (and standby_tag == committed_tag),
|
|
* ha_dispatch_repl_action_for_standby must call ha_send_snapshot_to_standby
|
|
* (via hook in test mode).
|
|
*
|
|
* Steps:
|
|
* 1. Set ha->standby_reported_tag[0] = committed_tag (tag is current)
|
|
* 2. Set ha->standby_reported_lsn[0] = 10 (behind oldest_wal_lsn=50)
|
|
* 3. Call ha_dispatch_repl_action_for_standby(..., oldest_wal_lsn=50)
|
|
* 4. Assert hook called once with peer_idx=0
|
|
*
|
|
* Requirements: 4.1 (snapshot when standby_lsn < oldest_wal_lsn)
|
|
*/
|
|
static void test_dispatch_repl_action_send_snapshot(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
ha_repl_send_snapshot_hook = stub_repl_send_snapshot;
|
|
g_send_snapshot_count = 0;
|
|
g_send_snapshot_idx = -1;
|
|
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
/* standby[0] has current config tag but LSN predates WAL window */
|
|
ha->standby_reported_tag[0] = 5;
|
|
ha->standby_reported_lsn[0] = 10; /* < oldest_wal_lsn=50 */
|
|
|
|
int rc = ha_dispatch_repl_action_for_standby(
|
|
ha, /*c=*/NULL, /*ep=*/NULL, /*peer_idx=*/0,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/50);
|
|
(void)rc;
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_send_snapshot_count,
|
|
"SEND_SNAPSHOT: hook must fire once when standby_lsn < oldest_wal_lsn");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_send_snapshot_idx,
|
|
"SEND_SNAPSHOT: hook must receive correct peer_idx=0");
|
|
|
|
ha_fini(ha);
|
|
ha_repl_send_snapshot_hook = NULL;
|
|
printf("test_dispatch_repl_action_send_snapshot: PASS\n");
|
|
#else
|
|
printf("test_dispatch_repl_action_send_snapshot: SKIP (non-test build)\n");
|
|
#endif
|
|
}
|
|
|
|
/* ── Task 3.2: ha_evaluate_repl_action unified decision tree ─────────────── */
|
|
|
|
/*
|
|
* Declare the function under test. It is defined in ha.c.
|
|
* Pure function — no side effects — so no XCE_TESTING guard needed.
|
|
*/
|
|
ha_repl_action_t ha_evaluate_repl_action(uint64_t standby_tag,
|
|
uint64_t standby_lsn,
|
|
uint64_t committed_tag,
|
|
uint64_t committed_lsn,
|
|
uint64_t oldest_wal_lsn);
|
|
|
|
/*
|
|
* Test: (tag=0, lsn=0) with committed_tag=5, committed_lsn=100,
|
|
* oldest_wal_lsn=50 → PUSH_CONFIG (tag behind).
|
|
* Requirements: 2.2, 2.3 (ha-repl-hardening)
|
|
*/
|
|
static void test_evaluate_repl_action_push_config(void)
|
|
{
|
|
ha_repl_action_t action = ha_evaluate_repl_action(
|
|
/*standby_tag=*/0, /*standby_lsn=*/0,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100,
|
|
/*oldest_wal_lsn=*/50);
|
|
TEST_ASSERT_EQUAL_INT(HA_REPL_PUSH_CONFIG, action);
|
|
printf("test_evaluate_repl_action_push_config: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: (tag=5, lsn=0) with committed_tag=5, committed_lsn=100,
|
|
* oldest_wal_lsn=50 → SEND_SNAPSHOT (lsn=0 is below WAL retention window).
|
|
* Requirements: 2.2, 2.3
|
|
*/
|
|
static void test_evaluate_repl_action_send_snapshot_lsn_zero(void)
|
|
{
|
|
ha_repl_action_t action = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/0,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100,
|
|
/*oldest_wal_lsn=*/50);
|
|
/* lsn=0 < oldest_wal_lsn=50 → SEND_SNAPSHOT */
|
|
TEST_ASSERT_EQUAL_INT(HA_REPL_SEND_SNAPSHOT, action);
|
|
printf("test_evaluate_repl_action_send_snapshot_lsn_zero: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: (tag=5, lsn=100) with committed_tag=5, committed_lsn=100 → CONVERGED.
|
|
* Requirements: 2.2, 2.4
|
|
*/
|
|
static void test_evaluate_repl_action_converged(void)
|
|
{
|
|
ha_repl_action_t action = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100,
|
|
/*oldest_wal_lsn=*/50);
|
|
TEST_ASSERT_EQUAL_INT(HA_REPL_CONVERGED, action);
|
|
printf("test_evaluate_repl_action_converged: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: (tag=5, lsn=10) with committed_tag=5, committed_lsn=100,
|
|
* oldest_wal_lsn=50 → SEND_SNAPSHOT (lsn < oldest_wal).
|
|
* Requirements: 2.2, 2.3
|
|
*/
|
|
static void test_evaluate_repl_action_send_snapshot(void)
|
|
{
|
|
ha_repl_action_t action = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/10,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100,
|
|
/*oldest_wal_lsn=*/50);
|
|
TEST_ASSERT_EQUAL_INT(HA_REPL_SEND_SNAPSHOT, action);
|
|
printf("test_evaluate_repl_action_send_snapshot: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: (tag=5, lsn=50) with committed_tag=5, committed_lsn=100,
|
|
* oldest_wal_lsn=50 → STREAM_WAL (lsn >= oldest_wal but < committed).
|
|
* Requirements: 2.2, 2.3
|
|
*/
|
|
static void test_evaluate_repl_action_stream_wal(void)
|
|
{
|
|
ha_repl_action_t action = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/50,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100,
|
|
/*oldest_wal_lsn=*/50);
|
|
TEST_ASSERT_EQUAL_INT(HA_REPL_STREAM_WAL, action);
|
|
printf("test_evaluate_repl_action_stream_wal: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: (tag=T, lsn=0) with oldest_wal_lsn=0 (fresh system, no WAL entries yet)
|
|
* → STREAM_WAL (lsn=0 >= oldest_wal_lsn=0, so not in snapshot territory;
|
|
* lsn=0 < committed_lsn → stream WAL from the beginning).
|
|
*
|
|
* This is the canonical Task 6.1 case: a standby that has caught up on config
|
|
* (tag matches) but has no WAL applied yet on a fresh system. When there is no
|
|
* WAL retention window to worry about (oldest_wal_lsn == 0), the correct action
|
|
* is STREAM_WAL, not SEND_SNAPSHOT.
|
|
*
|
|
* Requirements: 1.1, 1.2, 2.2, 2.3 (ha-repl-hardening)
|
|
*/
|
|
static void test_evaluate_repl_action_stream_wal_fresh_system(void)
|
|
{
|
|
ha_repl_action_t action = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/0,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100,
|
|
/*oldest_wal_lsn=*/0);
|
|
/* lsn=0 >= oldest_wal_lsn=0 → NOT snapshot; lsn=0 < committed → STREAM_WAL */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(HA_REPL_STREAM_WAL, action,
|
|
"(tag=T, lsn=0) with oldest_wal_lsn=0 must route to STREAM_WAL (Req 1.1, 1.2)");
|
|
printf("test_evaluate_repl_action_stream_wal_fresh_system: PASS\n");
|
|
}
|
|
|
|
/* ── Task 5.3: committing timeout default value and degraded-mode WARN ───────
|
|
*
|
|
* Requirements: 6.4, 7.5 (ha-repl-hardening)
|
|
*
|
|
* Verifies:
|
|
* - HA_DEFAULT_COMMITTING_TIMEOUT_MS == 60000 (safety-net window is 60 s)
|
|
* - committing_timeout_fn_for_test still promotes to CTRL_PRIMARY in degraded
|
|
* mode even when only a subset of standbys have converged
|
|
* - The ha->standby_converged[] state is readable so the WARN log can include
|
|
* the converged/pending counts
|
|
*/
|
|
|
|
/*
|
|
* Test: HA_DEFAULT_COMMITTING_TIMEOUT_MS is 60000 (not 10000).
|
|
* Requirements: 6.4 (ha-repl-hardening) — "Change committing_timeout_ms default from 10000 to 60000"
|
|
*/
|
|
static void test_committing_timeout_default_is_60s(void)
|
|
{
|
|
TEST_ASSERT_EQUAL_UINT32_MESSAGE(60000, HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
|
"HA_DEFAULT_COMMITTING_TIMEOUT_MS must be 60000 (Req 6.4, Task 5.3)");
|
|
printf("test_committing_timeout_default_is_60s: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: committing_timeout_fn promotes to PRIMARY even when only some standbys
|
|
* have converged (degraded mode). The WARN log path fires when timeout expires
|
|
* before majority convergence. After promotion, role must be CTRL_PRIMARY.
|
|
*
|
|
* Requirements: 6.4, 7.5 (ha-repl-hardening)
|
|
*/
|
|
#ifdef XCE_TESTING
|
|
extern int committing_timeout_fn_for_test(ha_state_t *ha, cluster_t *c,
|
|
int cancelled_val);
|
|
#endif
|
|
|
|
static void test_committing_timeout_promotes_degraded_with_partial_convergence(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
cluster_t c;
|
|
memset(&c, 0, sizeof(c));
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
|
};
|
|
ha_init(&c.ha, &cfg, 0);
|
|
c.ha.n_peers = 3; /* 3-node cluster; 2 standbys */
|
|
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
|
memory_order_release);
|
|
|
|
/* Only 1 of 2 standbys has converged — majority not yet reached */
|
|
c.ha.standby_converged[0] = true;
|
|
c.ha.standby_converged[1] = false;
|
|
|
|
/*
|
|
* Count converged/pending so the test mirrors what the WARN log reads.
|
|
* n_converged=1, n_pending=1 (Req 7.5).
|
|
*/
|
|
int n_converged = 0;
|
|
int n_pending = 0;
|
|
for (int i = 0; i < c.ha.n_peers - 1; i++) {
|
|
if (c.ha.standby_converged[i])
|
|
n_converged++;
|
|
else
|
|
n_pending++;
|
|
}
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, n_converged,
|
|
"degraded-mode: 1 standby converged before timeout");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, n_pending,
|
|
"degraded-mode: 1 standby still pending before timeout");
|
|
|
|
/* Fire the timeout — must promote despite partial convergence */
|
|
committing_timeout_fn_for_test(&c.ha, &c, 0 /* not cancelled */);
|
|
|
|
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
|
|
"committing timeout: must promote to CTRL_PRIMARY in degraded mode (Req 6.4)");
|
|
|
|
ha_fini(&c.ha);
|
|
printf("test_committing_timeout_promotes_degraded_with_partial_convergence: PASS "
|
|
"(n_converged=%d, n_pending=%d)\n", n_converged, n_pending);
|
|
#else
|
|
printf("test_committing_timeout_promotes_degraded_with_partial_convergence: SKIP\n");
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* Test: committing_timeout_fn promotes when NO standbys have converged
|
|
* (worst-case degraded mode — all standbys unreachable).
|
|
* Requirements: 6.4 (ha-repl-hardening)
|
|
*/
|
|
static void test_committing_timeout_promotes_degraded_zero_converged(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
cluster_t c;
|
|
memset(&c, 0, sizeof(c));
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
|
};
|
|
ha_init(&c.ha, &cfg, 0);
|
|
c.ha.n_peers = 3; /* 3-node cluster; all standbys unreachable */
|
|
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
|
memory_order_release);
|
|
|
|
/* No standbys converged — n_converged=0, n_pending=2 (Req 7.5) */
|
|
c.ha.standby_converged[0] = false;
|
|
c.ha.standby_converged[1] = false;
|
|
|
|
committing_timeout_fn_for_test(&c.ha, &c, 0 /* not cancelled */);
|
|
|
|
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
|
|
"committing timeout: must promote even with 0 converged standbys (degraded, Req 6.4)");
|
|
|
|
ha_fini(&c.ha);
|
|
printf("test_committing_timeout_promotes_degraded_zero_converged: PASS\n");
|
|
#else
|
|
printf("test_committing_timeout_promotes_degraded_zero_converged: SKIP\n");
|
|
#endif
|
|
}
|
|
|
|
/* ── Task 3.3 (ha-wal-replication): dispatch gate allows OPCode_walCatchupEntry
|
|
* in CTRL_STANDBY, CTRL_ELECTING, and CTRL_INIT.
|
|
*
|
|
* Requirements: ha-wal-replication Req 3.3 (standby role guard).
|
|
* Verifies that ha_dispatch_gate(ha, OPCode_walCatchupEntry) returns 0
|
|
* (allow) for the three roles where a standby legitimately receives WAL
|
|
* catch-up entries from the primary. CTRL_PRIMARY and
|
|
* CTRL_PRIMARY_COMMITTING must NOT allow it (they never receive WAL pushes). */
|
|
static void test_dispatch_gate_allows_wal_catchup_in_standby_roles(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
|
};
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
|
|
|
/* CTRL_STANDBY — must allow */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
ha_dispatch_gate(&ha, OPCode_walCatchupEntry),
|
|
"CTRL_STANDBY must allow OPCode_walCatchupEntry (Req 3.3)");
|
|
|
|
/* CTRL_ELECTING — must allow */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
ha_dispatch_gate(&ha, OPCode_walCatchupEntry),
|
|
"CTRL_ELECTING must allow OPCode_walCatchupEntry (Req 3.3)");
|
|
|
|
/* CTRL_INIT — must allow */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
ha_dispatch_gate(&ha, OPCode_walCatchupEntry),
|
|
"CTRL_INIT must allow OPCode_walCatchupEntry (Req 3.3)");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_dispatch_gate_allows_wal_catchup_in_standby_roles: PASS\n");
|
|
}
|
|
|
|
void test_ha_role(void)
|
|
{
|
|
RUN_TEST(test_ha_init_default_role_is_standby);
|
|
RUN_TEST(test_ha_init_rejects_fast_heartbeat);
|
|
RUN_TEST(test_ha_init_rejects_short_election_timeout);
|
|
RUN_TEST(test_ha_init_rejects_timeout_less_than_interval);
|
|
RUN_TEST(test_ha_promote_sets_primary_role);
|
|
/* P1-3a */
|
|
RUN_TEST(test_promote_increments_term);
|
|
RUN_TEST(test_demote_sets_standby_role);
|
|
/* Watchdog / SIGUSR1 tests deleted */
|
|
/* Task 2.1: election-iomgr-refactor new fields */
|
|
RUN_TEST(test_ha_init_zeroes_new_fields);
|
|
/* Task 1.4: ha-repl-hardening per-standby convergence fields */
|
|
RUN_TEST(test_ha_init_zeroes_standby_convergence_fields);
|
|
/* Task 3.1: ha_process_standby_state convergence tracking */
|
|
RUN_TEST(test_process_standby_state_converged);
|
|
RUN_TEST(test_process_standby_state_lsn_behind);
|
|
RUN_TEST(test_process_standby_state_tag_behind);
|
|
RUN_TEST(test_process_standby_state_regression);
|
|
RUN_TEST(test_process_standby_state_multiple_peers);
|
|
/* Task 3.2: ha_evaluate_repl_action unified decision tree */
|
|
RUN_TEST(test_evaluate_repl_action_push_config);
|
|
RUN_TEST(test_evaluate_repl_action_send_snapshot_lsn_zero);
|
|
RUN_TEST(test_evaluate_repl_action_converged);
|
|
RUN_TEST(test_evaluate_repl_action_send_snapshot);
|
|
RUN_TEST(test_evaluate_repl_action_stream_wal);
|
|
/* Task 6.1: (tag=T, lsn=0) with oldest_wal_lsn=0 → STREAM_WAL */
|
|
RUN_TEST(test_evaluate_repl_action_stream_wal_fresh_system);
|
|
/* Task 3.3: ha_dispatch_repl_action_for_standby PUSH_CONFIG wiring */
|
|
RUN_TEST(test_dispatch_repl_action_push_config_when_tag_behind);
|
|
RUN_TEST(test_dispatch_repl_action_no_push_when_converged);
|
|
/* Task 3.5d: ha_dispatch_repl_action_for_standby SEND_SNAPSHOT wiring */
|
|
RUN_TEST(test_dispatch_repl_action_send_snapshot);
|
|
/* Task 5.2: ha_check_majority_convergence → PRIMARY transition */
|
|
RUN_TEST(test_majority_convergence_3node_none_stays_committing);
|
|
RUN_TEST(test_majority_convergence_3node_one_standby_promotes);
|
|
RUN_TEST(test_majority_convergence_5node_threshold);
|
|
RUN_TEST(test_majority_convergence_3node_all_standbys_promotes);
|
|
RUN_TEST(test_majority_convergence_single_node_immediate);
|
|
/* Task 5.3: committing timeout default value and degraded-mode WARN */
|
|
RUN_TEST(test_committing_timeout_default_is_60s);
|
|
RUN_TEST(test_committing_timeout_promotes_degraded_with_partial_convergence);
|
|
RUN_TEST(test_committing_timeout_promotes_degraded_zero_converged);
|
|
/* Task 3.3 (ha-wal-replication): dispatch gate standby-role guard */
|
|
RUN_TEST(test_dispatch_gate_allows_wal_catchup_in_standby_roles);
|
|
}
|
|
|
|
/* test_am_i_primary suite deleted — am_i_primary() removed (election-iomgr-refactor) */
|