Rongsong Shen
102e263cce
Two independent call sites duplicated target = max(committed_lsn, last_written): ha.c's heartbeat/stateReport dispatch (repl_ack_decode_and_ dispatch) and config_repl.c's catchup_derive_target (used by the ACK-driven catch-up continuation path). Once config-upgrade-snapshot's marker-triggered checkpoint advances last_written one LSN past an already-committed committed_lsn (the checkpoint's own consumed, never-replicated LSN slot), the unconditional max() permanently wedges every standby's catch-up target at an unreachable value -- the primary re-streams an empty range every heartbeat forever. Extracted the shared, pure ha_compute_repl_target_lsn(committed_lsn, last_written) (ha.h/ha.c), which only falls back to last_written on genuine committed_lsn==0 bootstrap, and pointed both call sites at it so they can never disagree again. Fixing the second call site exposed three pre-existing tests in test_convergence_verification.c that hardcoded the old buggy formula's output as their expected target; updated their fixture values (committed_lsn 4->9, matching appended_lsn) to decouple them from the now-fixed edge case, which already has dedicated pure-function coverage in test_ha_role.c. GREEN: full controller_unit_test suite = 1040 tests, 0 failures, clean across multiple runs. Live-verified via run-ha-soak.sh --config-upgrade-wal-trim (18 jobs, zero job loss, ctrl-1 reconverged 3-of-3 using the preserved floor segment) -- unblocks config-upgrade-snapshot task 8.9.
977 lines
39 KiB
C
977 lines
39 KiB
C
#include <signal.h>
|
|
#include <stdatomic.h>
|
|
#include <string.h>
|
|
#include <stdio.h>
|
|
#include <time.h>
|
|
|
|
#include <unity/unity.h>
|
|
|
|
#include "ha.h"
|
|
#include "myconf.h"
|
|
#include "xce/cluster_def.h"
|
|
|
|
static void test_ha_init_default_role_is_standby(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
|
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
|
|
ha_fini(&ha);
|
|
printf("test_ha_init_default_role_is_standby: PASS\n");
|
|
}
|
|
|
|
static void test_ha_init_rejects_fast_heartbeat(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 1999, /* below 2000ms floor */
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_NOT_EQUAL(0, ha_init(&ha, &cfg, 0));
|
|
printf("test_ha_init_rejects_fast_heartbeat: PASS\n");
|
|
}
|
|
|
|
static void test_ha_init_rejects_short_election_timeout(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 499, /* below 500ms floor */
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_NOT_EQUAL(0, ha_init(&ha, &cfg, 0));
|
|
printf("test_ha_init_rejects_short_election_timeout: PASS\n");
|
|
}
|
|
|
|
static void test_ha_init_rejects_timeout_less_than_interval(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 5000,
|
|
.heartbeat_timeout_ms = 4999, /* must be >= heartbeat_interval_ms */
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_NOT_EQUAL(0, ha_init(&ha, &cfg, 0));
|
|
printf("test_ha_init_rejects_timeout_less_than_interval: PASS\n");
|
|
}
|
|
|
|
static void test_ha_promote_sets_primary_role(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
ha_init(&ha, &cfg, 0);
|
|
/* Directly set role to PRIMARY (stub path — full promotion in P1-3a) */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release);
|
|
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
|
|
ha_fini(&ha);
|
|
printf("test_ha_promote_sets_primary_role: PASS\n");
|
|
}
|
|
|
|
/* Watchdog / SIGUSR1 tests deleted — watchdog pthread replaced by
|
|
* ha_arm_heartbeat_timeout_vtask (delaymgr); failover_requested removed */
|
|
|
|
/* ── P1-3a tests ─────────────────────────────────────────────── */
|
|
|
|
/* Term must increment on each promotion attempt */
|
|
static void test_promote_increments_term(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
ha_init(&ha, &cfg, 0);
|
|
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
|
|
|
|
ha_do_promote(&ha); /* internal promotion logic — no cluster_t needed */
|
|
|
|
uint64_t term_after = atomic_load_explicit(&ha.term, memory_order_acquire);
|
|
TEST_ASSERT_TRUE(term_after > term_before);
|
|
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
|
|
|
|
ha_fini(&ha);
|
|
printf("test_promote_increments_term: PASS\n");
|
|
}
|
|
|
|
/* Demotion must set role back to STANDBY */
|
|
static void test_demote_sets_standby_role(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
ha_init(&ha, &cfg, 0);
|
|
ha_do_promote(&ha);
|
|
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
|
|
|
|
ha_do_demote(&ha);
|
|
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
|
|
|
|
ha_fini(&ha);
|
|
printf("test_demote_sets_standby_role: PASS\n");
|
|
}
|
|
|
|
/* test_sigusr1_sets_failover_requested deleted — SIGUSR1 / failover_requested removed */
|
|
|
|
/* am_i_primary() tests deleted — function removed (election-iomgr-refactor) */
|
|
|
|
/* ── Task 2.1: ha_state_t new fields zero-initialised by ha_init ──────── */
|
|
|
|
/*
|
|
* Verify that ha_init zeroes the new fields added for election-iomgr-refactor:
|
|
* phase2_done, n_peers, client_ssl_ctx
|
|
* and that the surviving leader role values are distinct in ctrl_role_t.
|
|
*
|
|
* Requirements: 5.2, 8.1 (election-iomgr-refactor spec); 1.2 (ha-single-leader-role:
|
|
* single leader role — no transitional leader state observable)
|
|
*/
|
|
static void test_ha_init_zeroes_new_fields(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
|
|
|
/* phase2_done must be explicitly zero after ha_init */
|
|
TEST_ASSERT_EQUAL_INT(0, ha.phase2_done);
|
|
|
|
/* n_peers must be zero after ha_init */
|
|
TEST_ASSERT_EQUAL_INT(0, ha.n_peers);
|
|
|
|
/* client_ssl_ctx must be NULL after ha_init */
|
|
TEST_ASSERT_NULL(ha.client_ssl_ctx);
|
|
|
|
/* The four surviving roles are distinct: the single leader role is
|
|
* CTRL_PRIMARY (no transitional leader sub-state, Req 1.2). */
|
|
TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY, (int)CTRL_STANDBY);
|
|
TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY, (int)CTRL_INIT);
|
|
TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY, (int)CTRL_ELECTING);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_ha_init_zeroes_new_fields: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Task 1.4: per-standby convergence tracking fields zero-initialised by ha_init.
|
|
*
|
|
* Verifies:
|
|
* - standby_reported_tag[i] == 0 for all i after ha_init()
|
|
* - standby_reported_lsn[i] == 0 for all i after ha_init()
|
|
* - standby_converged[i] == false for all i after ha_init()
|
|
* - fields are writable from test context
|
|
*
|
|
* Requirements: 2.1 (ha-repl-hardening spec)
|
|
*/
|
|
static void test_ha_init_zeroes_standby_convergence_fields(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
|
|
|
/* All per-standby tracking arrays must be zero/false after ha_init */
|
|
for (int i = 0; i < MAX_HA_CANDIDATES; i++) {
|
|
TEST_ASSERT_EQUAL_UINT64(0, ha.standby_reported_tag[i]);
|
|
TEST_ASSERT_EQUAL_UINT64(0, ha.standby_reported_lsn[i]);
|
|
TEST_ASSERT_FALSE(ha.standby_converged[i]);
|
|
}
|
|
|
|
/* Fields must be writable */
|
|
ha.standby_reported_tag[0] = 42ULL;
|
|
ha.standby_reported_lsn[0] = 100ULL;
|
|
ha.standby_converged[0] = true;
|
|
TEST_ASSERT_EQUAL_UINT64(42ULL, ha.standby_reported_tag[0]);
|
|
TEST_ASSERT_EQUAL_UINT64(100ULL, ha.standby_reported_lsn[0]);
|
|
TEST_ASSERT_TRUE(ha.standby_converged[0]);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_ha_init_zeroes_standby_convergence_fields: PASS\n");
|
|
}
|
|
|
|
/* ── Task 3.1: ha_process_standby_state convergence tracking ─────────────── */
|
|
|
|
/*
|
|
* Declare the function under test. It is defined in ha.c and exposed only in
|
|
* XCE_TESTING builds (forward declaration matches the implementation).
|
|
*/
|
|
#ifdef XCE_TESTING
|
|
void ha_process_standby_state(ha_state_t *ha, int peer_idx,
|
|
uint64_t reported_tag, uint64_t reported_lsn,
|
|
uint64_t committed_tag, uint64_t committed_lsn);
|
|
#endif
|
|
|
|
/* Helper: build a ha_state_t with all convergence fields zeroed. */
|
|
static ha_state_t *build_ha_for_convergence_test(void)
|
|
{
|
|
static ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
ha_init(&ha, &cfg, 0);
|
|
return &ha;
|
|
}
|
|
|
|
/*
|
|
* Test: standby with lsn == committed_lsn and tag == committed_tag → converged.
|
|
* Fields standby_reported_tag[0] and standby_reported_lsn[0] updated correctly.
|
|
* Requirements: 1.2, 2.2 (ha-repl-hardening)
|
|
*/
|
|
static void test_process_standby_state_converged(void)
|
|
{
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
ha_process_standby_state(ha, 0,
|
|
/*reported_tag=*/5, /*reported_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100);
|
|
|
|
TEST_ASSERT_EQUAL_UINT64(5, ha->standby_reported_tag[0]);
|
|
TEST_ASSERT_EQUAL_UINT64(100, ha->standby_reported_lsn[0]);
|
|
TEST_ASSERT_TRUE(ha->standby_converged[0]);
|
|
|
|
ha_fini(ha);
|
|
printf("test_process_standby_state_converged: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: standby lsn < committed_lsn → not converged.
|
|
* Requirements: 1.2, 2.2
|
|
*/
|
|
static void test_process_standby_state_lsn_behind(void)
|
|
{
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
ha_process_standby_state(ha, 0,
|
|
/*reported_tag=*/5, /*reported_lsn=*/50,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100);
|
|
|
|
TEST_ASSERT_EQUAL_UINT64(5, ha->standby_reported_tag[0]);
|
|
TEST_ASSERT_EQUAL_UINT64(50, ha->standby_reported_lsn[0]);
|
|
TEST_ASSERT_FALSE(ha->standby_converged[0]);
|
|
|
|
ha_fini(ha);
|
|
printf("test_process_standby_state_lsn_behind: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: standby tag < committed_tag → not converged.
|
|
* Requirements: 1.2, 2.2
|
|
*/
|
|
static void test_process_standby_state_tag_behind(void)
|
|
{
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
ha_process_standby_state(ha, 0,
|
|
/*reported_tag=*/3, /*reported_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100);
|
|
|
|
TEST_ASSERT_EQUAL_UINT64(3, ha->standby_reported_tag[0]);
|
|
TEST_ASSERT_EQUAL_UINT64(100, ha->standby_reported_lsn[0]);
|
|
TEST_ASSERT_FALSE(ha->standby_converged[0]);
|
|
|
|
ha_fini(ha);
|
|
printf("test_process_standby_state_tag_behind: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: convergence regression — standby was converged; primary advances;
|
|
* next update with old values should reset standby_converged to false.
|
|
* Requirements: 2.5 (converged standby regresses → mark not-converged)
|
|
*/
|
|
static void test_process_standby_state_regression(void)
|
|
{
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
/* First update: converged at (tag=5, lsn=100) */
|
|
ha_process_standby_state(ha, 0,
|
|
/*reported_tag=*/5, /*reported_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100);
|
|
TEST_ASSERT_TRUE(ha->standby_converged[0]);
|
|
|
|
/* Primary advances committed_lsn to 200; standby still reports (5, 100) */
|
|
ha_process_standby_state(ha, 0,
|
|
/*reported_tag=*/5, /*reported_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/200);
|
|
TEST_ASSERT_FALSE(ha->standby_converged[0]);
|
|
|
|
ha_fini(ha);
|
|
printf("test_process_standby_state_regression: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: different standby indices updated independently.
|
|
* Requirements: 2.2
|
|
*/
|
|
static void test_process_standby_state_multiple_peers(void)
|
|
{
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
/* peer 0: converged */
|
|
ha_process_standby_state(ha, 0,
|
|
/*reported_tag=*/5, /*reported_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100);
|
|
/* peer 1: not converged (tag behind) */
|
|
ha_process_standby_state(ha, 1,
|
|
/*reported_tag=*/3, /*reported_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100);
|
|
|
|
TEST_ASSERT_TRUE(ha->standby_converged[0]);
|
|
TEST_ASSERT_FALSE(ha->standby_converged[1]);
|
|
TEST_ASSERT_EQUAL_UINT64(5, ha->standby_reported_tag[0]);
|
|
TEST_ASSERT_EQUAL_UINT64(3, ha->standby_reported_tag[1]);
|
|
|
|
ha_fini(ha);
|
|
printf("test_process_standby_state_multiple_peers: PASS\n");
|
|
}
|
|
|
|
/* ── Task 5.2 convergence-checker tests REMOVED (ha-single-leader-role task 4.1)
|
|
*
|
|
* The majority-convergence checker was deleted in production (the convergence-
|
|
* counting force-promotion path is gone — promotion is driven solely by the
|
|
* heartbeat decision loop + per-entry majority ack). The machinery-subject
|
|
* cases that exercised it (3node_none / 3node_one / 5node_threshold /
|
|
* 3node_all / single_node_immediate) had nothing left to verify and were
|
|
* deleted per design §"Transitional-State Deletions" (no replacements, no
|
|
* shims). Their RUN_TEST registrations were removed as well.
|
|
*/
|
|
|
|
/* ── Task 3.3: ha_dispatch_repl_action_for_standby PUSH_CONFIG wiring ──────
|
|
*
|
|
* ha_dispatch_repl_action_for_standby reads the standby's stored
|
|
* (standby_reported_tag, standby_reported_lsn), calls ha_evaluate_repl_action,
|
|
* and if the result is HA_REPL_PUSH_CONFIG calls push_config_to_standbys (in
|
|
* production) or ha_repl_push_config_for_standby_hook (in XCE_TESTING).
|
|
*
|
|
* Requirements: 1.4 (ha-repl-hardening), design §Unified Heartbeat-Driven Protocol
|
|
*/
|
|
#ifdef XCE_TESTING
|
|
extern int (*ha_repl_push_config_for_standby_hook)(void *c, int peer_idx);
|
|
#endif
|
|
|
|
/* Call-counting state for the hook */
|
|
static int g_push_config_for_standby_count = 0;
|
|
static int g_push_config_for_standby_idx = -1;
|
|
|
|
static int stub_repl_push_config_for_standby(void *c, int peer_idx)
|
|
{
|
|
(void)c;
|
|
g_push_config_for_standby_count++;
|
|
g_push_config_for_standby_idx = peer_idx;
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Test: when standby_reported_tag < committed_tag, ha_dispatch_repl_action_for_standby
|
|
* must call push_config_to_standbys (via hook in test mode).
|
|
*
|
|
* Steps:
|
|
* 1. Set ha->standby_reported_tag[0] = 3 (behind committed_tag=5)
|
|
* 2. Set ha->standby_reported_lsn[0] = 100 (at committed_lsn)
|
|
* 3. Call ha_dispatch_repl_action_for_standby(ha, NULL, 0, committed_tag=5, ...)
|
|
* 4. Assert hook called once with peer_idx=0
|
|
*
|
|
* Requirements: 1.4 (push_config when standby tag < committed_tag)
|
|
*/
|
|
static void test_dispatch_repl_action_push_config_when_tag_behind(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
ha_repl_push_config_for_standby_hook = stub_repl_push_config_for_standby;
|
|
g_push_config_for_standby_count = 0;
|
|
g_push_config_for_standby_idx = -1;
|
|
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
/* standby[0] has stale config tag */
|
|
ha->standby_reported_tag[0] = 3;
|
|
ha->standby_reported_lsn[0] = 100;
|
|
|
|
int rc = ha_dispatch_repl_action_for_standby(
|
|
ha, /*c=*/NULL, /*ep=*/NULL, /*peer_idx=*/0,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/0);
|
|
(void)rc;
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_push_config_for_standby_count,
|
|
"PUSH_CONFIG: hook must fire once when standby_tag < committed_tag");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_push_config_for_standby_idx,
|
|
"PUSH_CONFIG: hook must receive correct peer_idx=0");
|
|
|
|
ha_fini(ha);
|
|
ha_repl_push_config_for_standby_hook = NULL;
|
|
printf("test_dispatch_repl_action_push_config_when_tag_behind: PASS\n");
|
|
#else
|
|
printf("test_dispatch_repl_action_push_config_when_tag_behind: SKIP (non-test build)\n");
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* Test: when standby is converged (tag == committed, lsn == committed),
|
|
* ha_dispatch_repl_action_for_standby must NOT call push_config hook.
|
|
*
|
|
* Requirements: decision tree — CONVERGED does not trigger push
|
|
*/
|
|
static void test_dispatch_repl_action_no_push_when_converged(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
ha_repl_push_config_for_standby_hook = stub_repl_push_config_for_standby;
|
|
g_push_config_for_standby_count = 0;
|
|
g_push_config_for_standby_idx = -1;
|
|
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
/* standby[0] is fully converged */
|
|
ha->standby_reported_tag[0] = 5;
|
|
ha->standby_reported_lsn[0] = 100;
|
|
|
|
int rc = ha_dispatch_repl_action_for_standby(
|
|
ha, /*c=*/NULL, /*ep=*/NULL, /*peer_idx=*/0,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/0);
|
|
(void)rc;
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_push_config_for_standby_count,
|
|
"CONVERGED: hook must NOT fire when standby is already converged");
|
|
|
|
ha_fini(ha);
|
|
ha_repl_push_config_for_standby_hook = NULL;
|
|
printf("test_dispatch_repl_action_no_push_when_converged: PASS\n");
|
|
#else
|
|
printf("test_dispatch_repl_action_no_push_when_converged: SKIP (non-test build)\n");
|
|
#endif
|
|
}
|
|
|
|
/* ── Task 3.5d: ha_dispatch_repl_action_for_standby SEND_SNAPSHOT wiring ────
|
|
*
|
|
* When standby_lsn < oldest_wal_lsn (standby predates WAL retention window),
|
|
* ha_dispatch_repl_action_for_standby must call ha_send_snapshot_to_standby
|
|
* (via ha_repl_send_snapshot_hook in XCE_TESTING builds).
|
|
*
|
|
* Requirements: 4.1 (ha-repl-hardening), design §Snapshot Transfer
|
|
*/
|
|
#ifdef XCE_TESTING
|
|
extern int (*ha_repl_send_snapshot_hook)(int peer_idx);
|
|
#endif
|
|
|
|
/* Call-counting state for the snapshot hook */
|
|
static int g_send_snapshot_count = 0;
|
|
static int g_send_snapshot_idx = -1;
|
|
|
|
static int stub_repl_send_snapshot(int peer_idx)
|
|
{
|
|
g_send_snapshot_count++;
|
|
g_send_snapshot_idx = peer_idx;
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Test: when standby_lsn < oldest_wal_lsn (and standby_tag == committed_tag),
|
|
* ha_dispatch_repl_action_for_standby must call ha_send_snapshot_to_standby
|
|
* (via hook in test mode).
|
|
*
|
|
* Steps:
|
|
* 1. Set ha->standby_reported_tag[0] = committed_tag (tag is current)
|
|
* 2. Set ha->standby_reported_lsn[0] = 10 (behind oldest_wal_lsn=50)
|
|
* 3. Call ha_dispatch_repl_action_for_standby(..., oldest_wal_lsn=50)
|
|
* 4. Assert hook called once with peer_idx=0
|
|
*
|
|
* Requirements: 4.1 (snapshot when standby_lsn < oldest_wal_lsn)
|
|
*/
|
|
static void test_dispatch_repl_action_send_snapshot(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
ha_repl_send_snapshot_hook = stub_repl_send_snapshot;
|
|
g_send_snapshot_count = 0;
|
|
g_send_snapshot_idx = -1;
|
|
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
/* standby[0] has current config tag but LSN predates WAL window */
|
|
ha->standby_reported_tag[0] = 5;
|
|
ha->standby_reported_lsn[0] = 10; /* < oldest_wal_lsn=50 */
|
|
|
|
int rc = ha_dispatch_repl_action_for_standby(
|
|
ha, /*c=*/NULL, /*ep=*/NULL, /*peer_idx=*/0,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/50);
|
|
(void)rc;
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_send_snapshot_count,
|
|
"SEND_SNAPSHOT: hook must fire once when standby_lsn < oldest_wal_lsn");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_send_snapshot_idx,
|
|
"SEND_SNAPSHOT: hook must receive correct peer_idx=0");
|
|
|
|
ha_fini(ha);
|
|
ha_repl_send_snapshot_hook = NULL;
|
|
printf("test_dispatch_repl_action_send_snapshot: PASS\n");
|
|
#else
|
|
printf("test_dispatch_repl_action_send_snapshot: SKIP (non-test build)\n");
|
|
#endif
|
|
}
|
|
|
|
/* ── Task 3.2: ha_evaluate_repl_action unified decision tree ─────────────── */
|
|
|
|
/*
|
|
* Declare the function under test. It is defined in ha.c.
|
|
* Pure function — no side effects — so no XCE_TESTING guard needed.
|
|
*/
|
|
ha_repl_action_t ha_evaluate_repl_action(uint64_t standby_tag,
|
|
uint64_t standby_lsn,
|
|
uint64_t committed_tag,
|
|
uint64_t committed_lsn,
|
|
uint64_t oldest_wal_lsn);
|
|
|
|
/*
|
|
* Test: (tag=0, lsn=0) with committed_tag=5, committed_lsn=100,
|
|
* oldest_wal_lsn=50 → PUSH_CONFIG (tag behind).
|
|
* Requirements: 2.2, 2.3 (ha-repl-hardening)
|
|
*/
|
|
static void test_evaluate_repl_action_push_config(void)
|
|
{
|
|
ha_repl_action_t action = ha_evaluate_repl_action(
|
|
/*standby_tag=*/0, /*standby_lsn=*/0,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100,
|
|
/*oldest_wal_lsn=*/50);
|
|
TEST_ASSERT_EQUAL_INT(HA_REPL_PUSH_CONFIG, action);
|
|
printf("test_evaluate_repl_action_push_config: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: (tag=5, lsn=0) with committed_tag=5, committed_lsn=100,
|
|
* oldest_wal_lsn=50 → SEND_SNAPSHOT (lsn=0 is below WAL retention window).
|
|
* Requirements: 2.2, 2.3
|
|
*/
|
|
static void test_evaluate_repl_action_send_snapshot_lsn_zero(void)
|
|
{
|
|
ha_repl_action_t action = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/0,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100,
|
|
/*oldest_wal_lsn=*/50);
|
|
/* lsn=0 < oldest_wal_lsn=50 → SEND_SNAPSHOT */
|
|
TEST_ASSERT_EQUAL_INT(HA_REPL_SEND_SNAPSHOT, action);
|
|
printf("test_evaluate_repl_action_send_snapshot_lsn_zero: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: (tag=5, lsn=100) with committed_tag=5, committed_lsn=100 → CONVERGED.
|
|
* Requirements: 2.2, 2.4
|
|
*/
|
|
static void test_evaluate_repl_action_converged(void)
|
|
{
|
|
ha_repl_action_t action = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100,
|
|
/*oldest_wal_lsn=*/50);
|
|
TEST_ASSERT_EQUAL_INT(HA_REPL_CONVERGED, action);
|
|
printf("test_evaluate_repl_action_converged: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: (tag=5, lsn=10) with committed_tag=5, committed_lsn=100,
|
|
* oldest_wal_lsn=50 → SEND_SNAPSHOT (lsn < oldest_wal).
|
|
* Requirements: 2.2, 2.3
|
|
*/
|
|
static void test_evaluate_repl_action_send_snapshot(void)
|
|
{
|
|
ha_repl_action_t action = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/10,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100,
|
|
/*oldest_wal_lsn=*/50);
|
|
TEST_ASSERT_EQUAL_INT(HA_REPL_SEND_SNAPSHOT, action);
|
|
printf("test_evaluate_repl_action_send_snapshot: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: (tag=5, lsn=50) with committed_tag=5, committed_lsn=100,
|
|
* oldest_wal_lsn=50 → STREAM_WAL (lsn >= oldest_wal but < committed).
|
|
* Requirements: 2.2, 2.3
|
|
*/
|
|
static void test_evaluate_repl_action_stream_wal(void)
|
|
{
|
|
ha_repl_action_t action = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/50,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100,
|
|
/*oldest_wal_lsn=*/50);
|
|
TEST_ASSERT_EQUAL_INT(HA_REPL_STREAM_WAL, action);
|
|
printf("test_evaluate_repl_action_stream_wal: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: (tag=T, lsn=0) with oldest_wal_lsn=0 (fresh system, no WAL entries yet)
|
|
* → STREAM_WAL (lsn=0 >= oldest_wal_lsn=0, so not in snapshot territory;
|
|
* lsn=0 < committed_lsn → stream WAL from the beginning).
|
|
*
|
|
* This is the canonical Task 6.1 case: a standby that has caught up on config
|
|
* (tag matches) but has no WAL applied yet on a fresh system. When there is no
|
|
* WAL retention window to worry about (oldest_wal_lsn == 0), the correct action
|
|
* is STREAM_WAL, not SEND_SNAPSHOT.
|
|
*
|
|
* Requirements: 1.1, 1.2, 2.2, 2.3 (ha-repl-hardening)
|
|
*/
|
|
static void test_evaluate_repl_action_stream_wal_fresh_system(void)
|
|
{
|
|
ha_repl_action_t action = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/0,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100,
|
|
/*oldest_wal_lsn=*/0);
|
|
/* lsn=0 >= oldest_wal_lsn=0 → NOT snapshot; lsn=0 < committed → STREAM_WAL */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(HA_REPL_STREAM_WAL, action,
|
|
"(tag=T, lsn=0) with oldest_wal_lsn=0 must route to STREAM_WAL (Req 1.1, 1.2)");
|
|
printf("test_evaluate_repl_action_stream_wal_fresh_system: PASS\n");
|
|
}
|
|
|
|
/* ── bug-1478: ha_compute_repl_target_lsn ─────────────────────────────────
|
|
*
|
|
* repl_ack_decode_and_dispatch (ha.c) computes the replication/convergence
|
|
* target passed into ha_dispatch_repl_action_for_standby as
|
|
* max(committed_lsn, last_written), where last_written is the raw locally-
|
|
* appended WAL tip (persist.current_lsn - 1). A live 3-node HA soak
|
|
* (config-upgrade-snapshot task 8.9, --config-upgrade-wal-trim) caught this
|
|
* unconditional max() permanently wedging catch-up: once a marker-triggered
|
|
* checkpoint (a local-only, NEVER-replicated WAL_CHECKPOINT entry —
|
|
* persist_write_checkpoint_marker bumps current_lsn directly, bypassing
|
|
* raft_log_append/the commit engine, bug-1466) trails an already-advancing
|
|
* committed_lsn, last_written sits permanently one LSN past it — a position
|
|
* no standby's applied_lsn can EVER legitimately reach (the checkpoint's own
|
|
* slot is, by design, never streamed or applied). ha_evaluate_repl_action's
|
|
* standby_lsn < committed_lsn check then never reaches HA_REPL_CONVERGED,
|
|
* and the primary re-streams the same empty range every heartbeat, forever.
|
|
* Ground-truthed via `xce_test waldump` on the live soak's preserved WAL
|
|
* (ctrl-2: lsn 21 = the WAL_CONFIG_MARKER, lsn 22 = WAL_CHECKPOINT, 0 bytes)
|
|
* and via ctrl-2's own commit_advance_locked log (the REAL, majority-gated
|
|
* committed_lsn never advanced past 21 — only the target_lsn fallback
|
|
* leaked 22 into the heartbeat decision).
|
|
*/
|
|
static void test_compute_repl_target_lsn_bootstrap_fallback(void)
|
|
{
|
|
/* committed_lsn genuinely stuck at 0 (no standby has ever acked) — the
|
|
* documented bootstrap exception: fall back to last_written so a
|
|
* first-ever connecting standby can catch up to real data. */
|
|
TEST_ASSERT_EQUAL_UINT64(42, ha_compute_repl_target_lsn(0, 42));
|
|
printf("test_compute_repl_target_lsn_bootstrap_fallback: PASS\n");
|
|
}
|
|
|
|
static void test_compute_repl_target_lsn_zero_zero(void)
|
|
{
|
|
/* Genuinely fresh cluster, nothing appended yet. */
|
|
TEST_ASSERT_EQUAL_UINT64(0, ha_compute_repl_target_lsn(0, 0));
|
|
printf("test_compute_repl_target_lsn_zero_zero: PASS\n");
|
|
}
|
|
|
|
static void test_compute_repl_target_lsn_ignores_trailing_checkpoint_lsn(void)
|
|
{
|
|
/* bug-1478's exact reproduction shape: committed_lsn=21 is already
|
|
* advancing (non-zero, real, majority-gated), but last_written=22
|
|
* reflects a local-only WAL_CHECKPOINT that trails it. The bootstrap
|
|
* fallback must NOT fire here — the target must stay at the real
|
|
* committed_lsn (21), never the unstreamable checkpoint slot (22). */
|
|
TEST_ASSERT_EQUAL_UINT64(21, ha_compute_repl_target_lsn(21, 22));
|
|
printf("test_compute_repl_target_lsn_ignores_trailing_checkpoint_lsn: PASS\n");
|
|
}
|
|
|
|
static void test_compute_repl_target_lsn_committed_already_ahead(void)
|
|
{
|
|
/* Normal steady-state case: committed_lsn is already the higher/current
|
|
* value; last_written (a possibly-stale, lock-free read) trails it.
|
|
* The target must simply be committed_lsn. */
|
|
TEST_ASSERT_EQUAL_UINT64(100, ha_compute_repl_target_lsn(100, 50));
|
|
printf("test_compute_repl_target_lsn_committed_already_ahead: PASS\n");
|
|
}
|
|
|
|
/* The transitional-leader timeout default test was removed: the config knob
|
|
* and its default macro are deleted by ha-single-leader-role (the wall-clock
|
|
* force-promotion watchdog it configured is gone — Req 1.5). */
|
|
|
|
/* ── Task 3.3 (ha-wal-replication): dispatch gate allows OPCode_walCatchupEntry
|
|
* in CTRL_STANDBY, CTRL_ELECTING, and CTRL_INIT.
|
|
*
|
|
* Requirements: ha-wal-replication Req 3.3 (standby role guard).
|
|
* Verifies that ha_dispatch_gate(ha, OPCode_walCatchupEntry) returns 0
|
|
* (allow) for the three roles where a standby legitimately receives WAL
|
|
* catch-up entries from the primary. CTRL_PRIMARY must NOT allow it
|
|
* (the leader never receives WAL pushes). */
|
|
static void test_dispatch_gate_allows_wal_catchup_in_standby_roles(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
|
|
|
/* CTRL_STANDBY — must allow */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
ha_dispatch_gate(&ha, OPCode_walCatchupEntry),
|
|
"CTRL_STANDBY must allow OPCode_walCatchupEntry (Req 3.3)");
|
|
|
|
/* CTRL_ELECTING — must allow */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
ha_dispatch_gate(&ha, OPCode_walCatchupEntry),
|
|
"CTRL_ELECTING must allow OPCode_walCatchupEntry (Req 3.3)");
|
|
|
|
/* CTRL_INIT — must allow */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
ha_dispatch_gate(&ha, OPCode_walCatchupEntry),
|
|
"CTRL_INIT must allow OPCode_walCatchupEntry (Req 3.3)");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_dispatch_gate_allows_wal_catchup_in_standby_roles: PASS\n");
|
|
}
|
|
|
|
/* ── ha-prevote: dispatch gate must admit OPCode_preVoteRequest wherever it
|
|
* admits OPCode_requestVote (CTRL_STANDBY, CTRL_ELECTING, CTRL_INIT).
|
|
*
|
|
* Regression (bug-605): the PreVote opcode + handler were added but never
|
|
* wired into ha_dispatch_gate, so a STANDBY voter rejected incoming PreVote
|
|
* straw-polls with -XCE_ERR_STANDBY before the handler ran. No preVoteReply
|
|
* was ever sent; the candidate's failed exchange was tallied as a denial,
|
|
* so a cold cluster looped PreVote LOST forever and never elected a primary.
|
|
*
|
|
* PreVote is strictly non-mutating, so it must be at least as admissible as a
|
|
* real requestVote in every role. This test asserts that parity. */
|
|
static void test_dispatch_gate_allows_prevote_in_voter_roles(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
|
|
|
const int32_t voter_roles[] = { CTRL_STANDBY, CTRL_ELECTING, CTRL_INIT };
|
|
for (size_t i = 0; i < sizeof(voter_roles) / sizeof(voter_roles[0]); i++) {
|
|
atomic_store_explicit(&ha.role, voter_roles[i], memory_order_release);
|
|
/* PreVote must be admitted exactly where the real vote is. */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
|
ha_dispatch_gate(&ha, OPCode_requestVote),
|
|
ha_dispatch_gate(&ha, OPCode_preVoteRequest),
|
|
"ha_dispatch_gate must admit OPCode_preVoteRequest wherever it "
|
|
"admits OPCode_requestVote (bug-605)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
ha_dispatch_gate(&ha, OPCode_preVoteRequest),
|
|
"voter role must admit OPCode_preVoteRequest (bug-605)");
|
|
}
|
|
|
|
ha_fini(&ha);
|
|
printf("test_dispatch_gate_allows_prevote_in_voter_roles: PASS\n");
|
|
}
|
|
|
|
/* ── ha-prevote/raft (bug-606): failover election stagger desyncs survivors.
|
|
*
|
|
* After a primary dies, two surviving standbys must NOT campaign in lock-step
|
|
* (split-vote livelock, term climbing unbounded). ha_election_stagger_ms gives
|
|
* each node a priority-ordered, jitter-bounded delay before starting its
|
|
* failover PreVote so one survivor wins before the other begins. Asserts:
|
|
* - bounds: stagger ∈ [my_priority*step, my_priority*step + jitter_max]
|
|
* - ordering: a higher-priority-index node's MINIMUM stagger exceeds a
|
|
* lower-index node's MAXIMUM (requires step > jitter_max — the desync
|
|
* guarantee), so the lower index always campaigns first. */
|
|
static void test_election_stagger_desyncs_by_priority(void)
|
|
{
|
|
const uint32_t step = 500, jmax = 200; /* step > jmax → strict ordering */
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 5000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
.election_priority_step_ms = step,
|
|
.election_jitter_max_ms = jmax,
|
|
};
|
|
|
|
for (int prio = 0; prio < 3; prio++) {
|
|
ha_state_t ha;
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, prio));
|
|
uint32_t lo = (uint32_t)prio * step;
|
|
uint32_t hi = lo + jmax;
|
|
/* Sample several times — jitter is re-randomized each call. */
|
|
for (int i = 0; i < 16; i++) {
|
|
uint32_t d = ha_election_stagger_ms(&ha);
|
|
TEST_ASSERT_TRUE_MESSAGE(d >= lo && d <= hi,
|
|
"stagger must be within [prio*step, prio*step+jitter_max] (bug-606)");
|
|
}
|
|
ha_fini(&ha);
|
|
}
|
|
|
|
/* Desync guarantee: prio 1's max (step+jmax) < prio 2's min (2*step). */
|
|
TEST_ASSERT_TRUE_MESSAGE((step + jmax) < (2u * step),
|
|
"election_priority_step_ms must exceed election_jitter_max_ms so "
|
|
"adjacent priorities never overlap (bug-606 desync guarantee)");
|
|
printf("test_election_stagger_desyncs_by_priority: PASS\n");
|
|
}
|
|
|
|
/* ── controller-status task 2.2: dispatch gate admits OPCode_ctrlstatus in
|
|
* EVERY role — STANDBY, ELECTING, INIT, and PRIMARY-without-quorum — so the
|
|
* read-only status query always answers (even fenced / without quorum),
|
|
* rather than being rejected as -XCE_ERR_STANDBY / -XCE_ERR_NOT_READY.
|
|
*
|
|
* Requirements: controller-status 7.1, 7.2 (failover-aware, any-node delivery);
|
|
* design §"Dispatch-gate allow-lists — Service contract".
|
|
*
|
|
* Scope guard: the change must be strictly scoped to OPCode_ctrlstatus — no
|
|
* other opcode's gating may widen. This test also asserts that a previously
|
|
* primary-only-when-serving read (OPCode_clusterconf) is STILL rejected on a
|
|
* STANDBY after the change, proving the allow-list was not broadened.
|
|
*
|
|
* Note on PRIMARY-without-quorum: ha_has_write_quorum() returns false when the
|
|
* connected-standby count is below n_peers/2. With n_peers == 0 (default
|
|
* ha_init) the threshold is 0 and quorum is trivially satisfied, so to exercise
|
|
* the no-quorum branch we set n_peers high enough that 0 connected standbys
|
|
* fail the >= n_peers/2 test. */
|
|
static void test_dispatch_gate_allows_ctrlstatus_in_every_role(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
|
|
|
/* CTRL_STANDBY — must allow ctrlstatus (Req 7.1, 7.2) */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
ha_dispatch_gate(&ha, OPCode_ctrlstatus),
|
|
"CTRL_STANDBY must allow OPCode_ctrlstatus (Req 7.1, 7.2)");
|
|
|
|
/* CTRL_ELECTING — must allow ctrlstatus */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
ha_dispatch_gate(&ha, OPCode_ctrlstatus),
|
|
"CTRL_ELECTING must allow OPCode_ctrlstatus (Req 7.1, 7.2)");
|
|
|
|
/* CTRL_INIT — must allow ctrlstatus */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
ha_dispatch_gate(&ha, OPCode_ctrlstatus),
|
|
"CTRL_INIT must allow OPCode_ctrlstatus (Req 7.1, 7.2)");
|
|
|
|
/* CTRL_PRIMARY without write quorum — must allow ctrlstatus.
|
|
* Force the no-quorum branch: with no replicator and zero connected
|
|
* standby_endpoints, ha_has_write_quorum() returns false once n_peers > 1
|
|
* (connected 0 < n_peers/2). */
|
|
ha.n_peers = 3; /* threshold = n_peers/2 = 1; connected = 0 → no quorum */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
ha_dispatch_gate(&ha, OPCode_ctrlstatus),
|
|
"CTRL_PRIMARY without quorum must allow OPCode_ctrlstatus (Req 7.1, 7.2)");
|
|
|
|
/* Scope guard (Req: change strictly scoped to ctrlstatus): a read that is
|
|
* primary-only-when-serving (OPCode_clusterconf) must STILL be rejected on
|
|
* a STANDBY — proving the allow-list was not widened for any other opcode. */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(-XCE_ERR_STANDBY,
|
|
ha_dispatch_gate(&ha, OPCode_clusterconf),
|
|
"CTRL_STANDBY must STILL reject OPCode_clusterconf — gate change must "
|
|
"be scoped strictly to OPCode_ctrlstatus");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_dispatch_gate_allows_ctrlstatus_in_every_role: PASS\n");
|
|
}
|
|
|
|
void test_ha_role(void)
|
|
{
|
|
UnitySetTestFile(__FILE__);
|
|
RUN_TEST(test_ha_init_default_role_is_standby);
|
|
RUN_TEST(test_ha_init_rejects_fast_heartbeat);
|
|
RUN_TEST(test_ha_init_rejects_short_election_timeout);
|
|
RUN_TEST(test_ha_init_rejects_timeout_less_than_interval);
|
|
RUN_TEST(test_ha_promote_sets_primary_role);
|
|
/* P1-3a */
|
|
RUN_TEST(test_promote_increments_term);
|
|
RUN_TEST(test_demote_sets_standby_role);
|
|
/* Watchdog / SIGUSR1 tests deleted */
|
|
/* Task 2.1: election-iomgr-refactor new fields */
|
|
RUN_TEST(test_ha_init_zeroes_new_fields);
|
|
/* Task 1.4: ha-repl-hardening per-standby convergence fields */
|
|
RUN_TEST(test_ha_init_zeroes_standby_convergence_fields);
|
|
/* Task 3.1: ha_process_standby_state convergence tracking */
|
|
RUN_TEST(test_process_standby_state_converged);
|
|
RUN_TEST(test_process_standby_state_lsn_behind);
|
|
RUN_TEST(test_process_standby_state_tag_behind);
|
|
RUN_TEST(test_process_standby_state_regression);
|
|
RUN_TEST(test_process_standby_state_multiple_peers);
|
|
/* Task 3.2: ha_evaluate_repl_action unified decision tree */
|
|
RUN_TEST(test_evaluate_repl_action_push_config);
|
|
RUN_TEST(test_evaluate_repl_action_send_snapshot_lsn_zero);
|
|
RUN_TEST(test_evaluate_repl_action_converged);
|
|
RUN_TEST(test_evaluate_repl_action_send_snapshot);
|
|
RUN_TEST(test_evaluate_repl_action_stream_wal);
|
|
/* Task 6.1: (tag=T, lsn=0) with oldest_wal_lsn=0 → STREAM_WAL */
|
|
RUN_TEST(test_evaluate_repl_action_stream_wal_fresh_system);
|
|
/* bug-1478: ha_compute_repl_target_lsn */
|
|
RUN_TEST(test_compute_repl_target_lsn_bootstrap_fallback);
|
|
RUN_TEST(test_compute_repl_target_lsn_zero_zero);
|
|
RUN_TEST(test_compute_repl_target_lsn_ignores_trailing_checkpoint_lsn);
|
|
RUN_TEST(test_compute_repl_target_lsn_committed_already_ahead);
|
|
/* Task 3.3: ha_dispatch_repl_action_for_standby PUSH_CONFIG wiring */
|
|
RUN_TEST(test_dispatch_repl_action_push_config_when_tag_behind);
|
|
RUN_TEST(test_dispatch_repl_action_no_push_when_converged);
|
|
/* Task 3.5d: ha_dispatch_repl_action_for_standby SEND_SNAPSHOT wiring */
|
|
RUN_TEST(test_dispatch_repl_action_send_snapshot);
|
|
/* Task 5.2 convergence-checker cases deleted (ha-single-leader-role 4.1:
|
|
* the majority-convergence checker removed — no shims, machinery-subject) */
|
|
/* transitional-leader timeout default test removed (ha-single-leader-role:
|
|
* config knob + wall-clock watchdog deleted, Req 1.5) */
|
|
/* Task 3.3 (ha-wal-replication): dispatch gate standby-role guard */
|
|
RUN_TEST(test_dispatch_gate_allows_wal_catchup_in_standby_roles);
|
|
/* ha-prevote (bug-605): dispatch gate must admit PreVote like the real vote */
|
|
RUN_TEST(test_dispatch_gate_allows_prevote_in_voter_roles);
|
|
/* raft/ha-prevote (bug-606): failover election stagger desyncs survivors */
|
|
RUN_TEST(test_election_stagger_desyncs_by_priority);
|
|
/* controller-status task 2.2: dispatch gate admits ctrlstatus in every role */
|
|
RUN_TEST(test_dispatch_gate_allows_ctrlstatus_in_every_role);
|
|
}
|
|
|
|
/* test_am_i_primary suite deleted — am_i_primary() removed (election-iomgr-refactor) */
|