751 lines
28 KiB
C
751 lines
28 KiB
C
#include <signal.h>
|
|
#include <stdatomic.h>
|
|
#include <string.h>
|
|
#include <stdio.h>
|
|
#include <time.h>
|
|
|
|
#include <unity/unity.h>
|
|
|
|
#include "ha.h"
|
|
#include "myconf.h"
|
|
#include "xce/cluster_def.h"
|
|
|
|
static void test_ha_init_default_role_is_standby(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
|
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
|
|
ha_fini(&ha);
|
|
printf("test_ha_init_default_role_is_standby: PASS\n");
|
|
}
|
|
|
|
static void test_ha_init_rejects_fast_heartbeat(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 1999, /* below 2000ms floor */
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_NOT_EQUAL(0, ha_init(&ha, &cfg, 0));
|
|
printf("test_ha_init_rejects_fast_heartbeat: PASS\n");
|
|
}
|
|
|
|
static void test_ha_init_rejects_short_election_timeout(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 499, /* below 500ms floor */
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_NOT_EQUAL(0, ha_init(&ha, &cfg, 0));
|
|
printf("test_ha_init_rejects_short_election_timeout: PASS\n");
|
|
}
|
|
|
|
static void test_ha_init_rejects_timeout_less_than_interval(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 5000,
|
|
.heartbeat_timeout_ms = 4999, /* must be >= heartbeat_interval_ms */
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_NOT_EQUAL(0, ha_init(&ha, &cfg, 0));
|
|
printf("test_ha_init_rejects_timeout_less_than_interval: PASS\n");
|
|
}
|
|
|
|
static void test_ha_promote_sets_primary_role(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
ha_init(&ha, &cfg, 0);
|
|
/* Directly set role to PRIMARY (stub path — full promotion in P1-3a) */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release);
|
|
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
|
|
ha_fini(&ha);
|
|
printf("test_ha_promote_sets_primary_role: PASS\n");
|
|
}
|
|
|
|
/* Watchdog / SIGUSR1 tests deleted — watchdog pthread replaced by
|
|
* ha_arm_heartbeat_timeout_vtask (delaymgr); failover_requested removed */
|
|
|
|
/* ── P1-3a tests ─────────────────────────────────────────────── */
|
|
|
|
/* Term must increment on each promotion attempt */
|
|
static void test_promote_increments_term(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
ha_init(&ha, &cfg, 0);
|
|
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
|
|
|
|
ha_do_promote(&ha); /* internal promotion logic — no cluster_t needed */
|
|
|
|
uint64_t term_after = atomic_load_explicit(&ha.term, memory_order_acquire);
|
|
TEST_ASSERT_TRUE(term_after > term_before);
|
|
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
|
|
|
|
ha_fini(&ha);
|
|
printf("test_promote_increments_term: PASS\n");
|
|
}
|
|
|
|
/* Demotion must set role back to STANDBY */
|
|
static void test_demote_sets_standby_role(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
ha_init(&ha, &cfg, 0);
|
|
ha_do_promote(&ha);
|
|
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
|
|
|
|
ha_do_demote(&ha);
|
|
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
|
|
|
|
ha_fini(&ha);
|
|
printf("test_demote_sets_standby_role: PASS\n");
|
|
}
|
|
|
|
/* test_sigusr1_sets_failover_requested deleted — SIGUSR1 / failover_requested removed */
|
|
|
|
/* am_i_primary() tests deleted — function removed (election-iomgr-refactor) */
|
|
|
|
/* ── Task 2.1: ha_state_t new fields zero-initialised by ha_init ──────── */
|
|
|
|
/*
|
|
* Verify that ha_init zeroes the new fields added for election-iomgr-refactor:
|
|
* phase2_done, n_peers, client_ssl_ctx
|
|
* and that the surviving leader role values are distinct in ctrl_role_t.
|
|
*
|
|
* Requirements: 5.2, 8.1 (election-iomgr-refactor spec); 1.2 (ha-single-leader-role:
|
|
* single leader role — no transitional leader state observable)
|
|
*/
|
|
static void test_ha_init_zeroes_new_fields(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
|
|
|
/* phase2_done must be explicitly zero after ha_init */
|
|
TEST_ASSERT_EQUAL_INT(0, ha.phase2_done);
|
|
|
|
/* n_peers must be zero after ha_init */
|
|
TEST_ASSERT_EQUAL_INT(0, ha.n_peers);
|
|
|
|
/* client_ssl_ctx must be NULL after ha_init */
|
|
TEST_ASSERT_NULL(ha.client_ssl_ctx);
|
|
|
|
/* The four surviving roles are distinct: the single leader role is
|
|
* CTRL_PRIMARY (no transitional leader sub-state, Req 1.2). */
|
|
TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY, (int)CTRL_STANDBY);
|
|
TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY, (int)CTRL_INIT);
|
|
TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY, (int)CTRL_ELECTING);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_ha_init_zeroes_new_fields: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Task 1.4: per-standby convergence tracking fields zero-initialised by ha_init.
|
|
*
|
|
* Verifies:
|
|
* - standby_reported_tag[i] == 0 for all i after ha_init()
|
|
* - standby_reported_lsn[i] == 0 for all i after ha_init()
|
|
* - standby_converged[i] == false for all i after ha_init()
|
|
* - fields are writable from test context
|
|
*
|
|
* Requirements: 2.1 (ha-repl-hardening spec)
|
|
*/
|
|
static void test_ha_init_zeroes_standby_convergence_fields(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
|
|
|
/* All per-standby tracking arrays must be zero/false after ha_init */
|
|
for (int i = 0; i < MAX_HA_CANDIDATES; i++) {
|
|
TEST_ASSERT_EQUAL_UINT64(0, ha.standby_reported_tag[i]);
|
|
TEST_ASSERT_EQUAL_UINT64(0, ha.standby_reported_lsn[i]);
|
|
TEST_ASSERT_FALSE(ha.standby_converged[i]);
|
|
}
|
|
|
|
/* Fields must be writable */
|
|
ha.standby_reported_tag[0] = 42ULL;
|
|
ha.standby_reported_lsn[0] = 100ULL;
|
|
ha.standby_converged[0] = true;
|
|
TEST_ASSERT_EQUAL_UINT64(42ULL, ha.standby_reported_tag[0]);
|
|
TEST_ASSERT_EQUAL_UINT64(100ULL, ha.standby_reported_lsn[0]);
|
|
TEST_ASSERT_TRUE(ha.standby_converged[0]);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_ha_init_zeroes_standby_convergence_fields: PASS\n");
|
|
}
|
|
|
|
/* ── Task 3.1: ha_process_standby_state convergence tracking ─────────────── */
|
|
|
|
/*
|
|
* Declare the function under test. It is defined in ha.c and exposed only in
|
|
* XCE_TESTING builds (forward declaration matches the implementation).
|
|
*/
|
|
#ifdef XCE_TESTING
|
|
void ha_process_standby_state(ha_state_t *ha, int peer_idx,
|
|
uint64_t reported_tag, uint64_t reported_lsn,
|
|
uint64_t committed_tag, uint64_t committed_lsn);
|
|
#endif
|
|
|
|
/* Helper: build a ha_state_t with all convergence fields zeroed. */
|
|
static ha_state_t *build_ha_for_convergence_test(void)
|
|
{
|
|
static ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
ha_init(&ha, &cfg, 0);
|
|
return &ha;
|
|
}
|
|
|
|
/*
|
|
* Test: standby with lsn == committed_lsn and tag == committed_tag → converged.
|
|
* Fields standby_reported_tag[0] and standby_reported_lsn[0] updated correctly.
|
|
* Requirements: 1.2, 2.2 (ha-repl-hardening)
|
|
*/
|
|
static void test_process_standby_state_converged(void)
|
|
{
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
ha_process_standby_state(ha, 0,
|
|
/*reported_tag=*/5, /*reported_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100);
|
|
|
|
TEST_ASSERT_EQUAL_UINT64(5, ha->standby_reported_tag[0]);
|
|
TEST_ASSERT_EQUAL_UINT64(100, ha->standby_reported_lsn[0]);
|
|
TEST_ASSERT_TRUE(ha->standby_converged[0]);
|
|
|
|
ha_fini(ha);
|
|
printf("test_process_standby_state_converged: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: standby lsn < committed_lsn → not converged.
|
|
* Requirements: 1.2, 2.2
|
|
*/
|
|
static void test_process_standby_state_lsn_behind(void)
|
|
{
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
ha_process_standby_state(ha, 0,
|
|
/*reported_tag=*/5, /*reported_lsn=*/50,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100);
|
|
|
|
TEST_ASSERT_EQUAL_UINT64(5, ha->standby_reported_tag[0]);
|
|
TEST_ASSERT_EQUAL_UINT64(50, ha->standby_reported_lsn[0]);
|
|
TEST_ASSERT_FALSE(ha->standby_converged[0]);
|
|
|
|
ha_fini(ha);
|
|
printf("test_process_standby_state_lsn_behind: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: standby tag < committed_tag → not converged.
|
|
* Requirements: 1.2, 2.2
|
|
*/
|
|
static void test_process_standby_state_tag_behind(void)
|
|
{
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
ha_process_standby_state(ha, 0,
|
|
/*reported_tag=*/3, /*reported_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100);
|
|
|
|
TEST_ASSERT_EQUAL_UINT64(3, ha->standby_reported_tag[0]);
|
|
TEST_ASSERT_EQUAL_UINT64(100, ha->standby_reported_lsn[0]);
|
|
TEST_ASSERT_FALSE(ha->standby_converged[0]);
|
|
|
|
ha_fini(ha);
|
|
printf("test_process_standby_state_tag_behind: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: convergence regression — standby was converged; primary advances;
|
|
* next update with old values should reset standby_converged to false.
|
|
* Requirements: 2.5 (converged standby regresses → mark not-converged)
|
|
*/
|
|
static void test_process_standby_state_regression(void)
|
|
{
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
/* First update: converged at (tag=5, lsn=100) */
|
|
ha_process_standby_state(ha, 0,
|
|
/*reported_tag=*/5, /*reported_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100);
|
|
TEST_ASSERT_TRUE(ha->standby_converged[0]);
|
|
|
|
/* Primary advances committed_lsn to 200; standby still reports (5, 100) */
|
|
ha_process_standby_state(ha, 0,
|
|
/*reported_tag=*/5, /*reported_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/200);
|
|
TEST_ASSERT_FALSE(ha->standby_converged[0]);
|
|
|
|
ha_fini(ha);
|
|
printf("test_process_standby_state_regression: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: different standby indices updated independently.
|
|
* Requirements: 2.2
|
|
*/
|
|
static void test_process_standby_state_multiple_peers(void)
|
|
{
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
/* peer 0: converged */
|
|
ha_process_standby_state(ha, 0,
|
|
/*reported_tag=*/5, /*reported_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100);
|
|
/* peer 1: not converged (tag behind) */
|
|
ha_process_standby_state(ha, 1,
|
|
/*reported_tag=*/3, /*reported_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100);
|
|
|
|
TEST_ASSERT_TRUE(ha->standby_converged[0]);
|
|
TEST_ASSERT_FALSE(ha->standby_converged[1]);
|
|
TEST_ASSERT_EQUAL_UINT64(5, ha->standby_reported_tag[0]);
|
|
TEST_ASSERT_EQUAL_UINT64(3, ha->standby_reported_tag[1]);
|
|
|
|
ha_fini(ha);
|
|
printf("test_process_standby_state_multiple_peers: PASS\n");
|
|
}
|
|
|
|
/* ── Task 5.2 convergence-checker tests REMOVED (ha-single-leader-role task 4.1)
|
|
*
|
|
* The majority-convergence checker was deleted in production (the convergence-
|
|
* counting force-promotion path is gone — promotion is driven solely by the
|
|
* heartbeat decision loop + per-entry majority ack). The machinery-subject
|
|
* cases that exercised it (3node_none / 3node_one / 5node_threshold /
|
|
* 3node_all / single_node_immediate) had nothing left to verify and were
|
|
* deleted per design §"Transitional-State Deletions" (no replacements, no
|
|
* shims). Their RUN_TEST registrations were removed as well.
|
|
*/
|
|
|
|
/* ── Task 3.3: ha_dispatch_repl_action_for_standby PUSH_CONFIG wiring ──────
|
|
*
|
|
* ha_dispatch_repl_action_for_standby reads the standby's stored
|
|
* (standby_reported_tag, standby_reported_lsn), calls ha_evaluate_repl_action,
|
|
* and if the result is HA_REPL_PUSH_CONFIG calls push_config_to_standbys (in
|
|
* production) or ha_repl_push_config_for_standby_hook (in XCE_TESTING).
|
|
*
|
|
* Requirements: 1.4 (ha-repl-hardening), design §Unified Heartbeat-Driven Protocol
|
|
*/
|
|
#ifdef XCE_TESTING
|
|
extern int (*ha_repl_push_config_for_standby_hook)(void *c, int peer_idx);
|
|
#endif
|
|
|
|
/* Call-counting state for the hook */
|
|
static int g_push_config_for_standby_count = 0;
|
|
static int g_push_config_for_standby_idx = -1;
|
|
|
|
static int stub_repl_push_config_for_standby(void *c, int peer_idx)
|
|
{
|
|
(void)c;
|
|
g_push_config_for_standby_count++;
|
|
g_push_config_for_standby_idx = peer_idx;
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Test: when standby_reported_tag < committed_tag, ha_dispatch_repl_action_for_standby
|
|
* must call push_config_to_standbys (via hook in test mode).
|
|
*
|
|
* Steps:
|
|
* 1. Set ha->standby_reported_tag[0] = 3 (behind committed_tag=5)
|
|
* 2. Set ha->standby_reported_lsn[0] = 100 (at committed_lsn)
|
|
* 3. Call ha_dispatch_repl_action_for_standby(ha, NULL, 0, committed_tag=5, ...)
|
|
* 4. Assert hook called once with peer_idx=0
|
|
*
|
|
* Requirements: 1.4 (push_config when standby tag < committed_tag)
|
|
*/
|
|
static void test_dispatch_repl_action_push_config_when_tag_behind(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
ha_repl_push_config_for_standby_hook = stub_repl_push_config_for_standby;
|
|
g_push_config_for_standby_count = 0;
|
|
g_push_config_for_standby_idx = -1;
|
|
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
/* standby[0] has stale config tag */
|
|
ha->standby_reported_tag[0] = 3;
|
|
ha->standby_reported_lsn[0] = 100;
|
|
|
|
int rc = ha_dispatch_repl_action_for_standby(
|
|
ha, /*c=*/NULL, /*ep=*/NULL, /*peer_idx=*/0,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/0);
|
|
(void)rc;
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_push_config_for_standby_count,
|
|
"PUSH_CONFIG: hook must fire once when standby_tag < committed_tag");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_push_config_for_standby_idx,
|
|
"PUSH_CONFIG: hook must receive correct peer_idx=0");
|
|
|
|
ha_fini(ha);
|
|
ha_repl_push_config_for_standby_hook = NULL;
|
|
printf("test_dispatch_repl_action_push_config_when_tag_behind: PASS\n");
|
|
#else
|
|
printf("test_dispatch_repl_action_push_config_when_tag_behind: SKIP (non-test build)\n");
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* Test: when standby is converged (tag == committed, lsn == committed),
|
|
* ha_dispatch_repl_action_for_standby must NOT call push_config hook.
|
|
*
|
|
* Requirements: decision tree — CONVERGED does not trigger push
|
|
*/
|
|
static void test_dispatch_repl_action_no_push_when_converged(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
ha_repl_push_config_for_standby_hook = stub_repl_push_config_for_standby;
|
|
g_push_config_for_standby_count = 0;
|
|
g_push_config_for_standby_idx = -1;
|
|
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
/* standby[0] is fully converged */
|
|
ha->standby_reported_tag[0] = 5;
|
|
ha->standby_reported_lsn[0] = 100;
|
|
|
|
int rc = ha_dispatch_repl_action_for_standby(
|
|
ha, /*c=*/NULL, /*ep=*/NULL, /*peer_idx=*/0,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/0);
|
|
(void)rc;
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_push_config_for_standby_count,
|
|
"CONVERGED: hook must NOT fire when standby is already converged");
|
|
|
|
ha_fini(ha);
|
|
ha_repl_push_config_for_standby_hook = NULL;
|
|
printf("test_dispatch_repl_action_no_push_when_converged: PASS\n");
|
|
#else
|
|
printf("test_dispatch_repl_action_no_push_when_converged: SKIP (non-test build)\n");
|
|
#endif
|
|
}
|
|
|
|
/* ── Task 3.5d: ha_dispatch_repl_action_for_standby SEND_SNAPSHOT wiring ────
|
|
*
|
|
* When standby_lsn < oldest_wal_lsn (standby predates WAL retention window),
|
|
* ha_dispatch_repl_action_for_standby must call ha_send_snapshot_to_standby
|
|
* (via ha_repl_send_snapshot_hook in XCE_TESTING builds).
|
|
*
|
|
* Requirements: 4.1 (ha-repl-hardening), design §Snapshot Transfer
|
|
*/
|
|
#ifdef XCE_TESTING
|
|
extern int (*ha_repl_send_snapshot_hook)(int peer_idx);
|
|
#endif
|
|
|
|
/* Call-counting state for the snapshot hook */
|
|
static int g_send_snapshot_count = 0;
|
|
static int g_send_snapshot_idx = -1;
|
|
|
|
static int stub_repl_send_snapshot(int peer_idx)
|
|
{
|
|
g_send_snapshot_count++;
|
|
g_send_snapshot_idx = peer_idx;
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Test: when standby_lsn < oldest_wal_lsn (and standby_tag == committed_tag),
|
|
* ha_dispatch_repl_action_for_standby must call ha_send_snapshot_to_standby
|
|
* (via hook in test mode).
|
|
*
|
|
* Steps:
|
|
* 1. Set ha->standby_reported_tag[0] = committed_tag (tag is current)
|
|
* 2. Set ha->standby_reported_lsn[0] = 10 (behind oldest_wal_lsn=50)
|
|
* 3. Call ha_dispatch_repl_action_for_standby(..., oldest_wal_lsn=50)
|
|
* 4. Assert hook called once with peer_idx=0
|
|
*
|
|
* Requirements: 4.1 (snapshot when standby_lsn < oldest_wal_lsn)
|
|
*/
|
|
static void test_dispatch_repl_action_send_snapshot(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
ha_repl_send_snapshot_hook = stub_repl_send_snapshot;
|
|
g_send_snapshot_count = 0;
|
|
g_send_snapshot_idx = -1;
|
|
|
|
ha_state_t *ha = build_ha_for_convergence_test();
|
|
|
|
/* standby[0] has current config tag but LSN predates WAL window */
|
|
ha->standby_reported_tag[0] = 5;
|
|
ha->standby_reported_lsn[0] = 10; /* < oldest_wal_lsn=50 */
|
|
|
|
int rc = ha_dispatch_repl_action_for_standby(
|
|
ha, /*c=*/NULL, /*ep=*/NULL, /*peer_idx=*/0,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/50);
|
|
(void)rc;
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_send_snapshot_count,
|
|
"SEND_SNAPSHOT: hook must fire once when standby_lsn < oldest_wal_lsn");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_send_snapshot_idx,
|
|
"SEND_SNAPSHOT: hook must receive correct peer_idx=0");
|
|
|
|
ha_fini(ha);
|
|
ha_repl_send_snapshot_hook = NULL;
|
|
printf("test_dispatch_repl_action_send_snapshot: PASS\n");
|
|
#else
|
|
printf("test_dispatch_repl_action_send_snapshot: SKIP (non-test build)\n");
|
|
#endif
|
|
}
|
|
|
|
/* ── Task 3.2: ha_evaluate_repl_action unified decision tree ─────────────── */
|
|
|
|
/*
|
|
* Declare the function under test. It is defined in ha.c.
|
|
* Pure function — no side effects — so no XCE_TESTING guard needed.
|
|
*/
|
|
ha_repl_action_t ha_evaluate_repl_action(uint64_t standby_tag,
|
|
uint64_t standby_lsn,
|
|
uint64_t committed_tag,
|
|
uint64_t committed_lsn,
|
|
uint64_t oldest_wal_lsn);
|
|
|
|
/*
|
|
* Test: (tag=0, lsn=0) with committed_tag=5, committed_lsn=100,
|
|
* oldest_wal_lsn=50 → PUSH_CONFIG (tag behind).
|
|
* Requirements: 2.2, 2.3 (ha-repl-hardening)
|
|
*/
|
|
static void test_evaluate_repl_action_push_config(void)
|
|
{
|
|
ha_repl_action_t action = ha_evaluate_repl_action(
|
|
/*standby_tag=*/0, /*standby_lsn=*/0,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100,
|
|
/*oldest_wal_lsn=*/50);
|
|
TEST_ASSERT_EQUAL_INT(HA_REPL_PUSH_CONFIG, action);
|
|
printf("test_evaluate_repl_action_push_config: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: (tag=5, lsn=0) with committed_tag=5, committed_lsn=100,
|
|
* oldest_wal_lsn=50 → SEND_SNAPSHOT (lsn=0 is below WAL retention window).
|
|
* Requirements: 2.2, 2.3
|
|
*/
|
|
static void test_evaluate_repl_action_send_snapshot_lsn_zero(void)
|
|
{
|
|
ha_repl_action_t action = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/0,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100,
|
|
/*oldest_wal_lsn=*/50);
|
|
/* lsn=0 < oldest_wal_lsn=50 → SEND_SNAPSHOT */
|
|
TEST_ASSERT_EQUAL_INT(HA_REPL_SEND_SNAPSHOT, action);
|
|
printf("test_evaluate_repl_action_send_snapshot_lsn_zero: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: (tag=5, lsn=100) with committed_tag=5, committed_lsn=100 → CONVERGED.
|
|
* Requirements: 2.2, 2.4
|
|
*/
|
|
static void test_evaluate_repl_action_converged(void)
|
|
{
|
|
ha_repl_action_t action = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/100,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100,
|
|
/*oldest_wal_lsn=*/50);
|
|
TEST_ASSERT_EQUAL_INT(HA_REPL_CONVERGED, action);
|
|
printf("test_evaluate_repl_action_converged: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: (tag=5, lsn=10) with committed_tag=5, committed_lsn=100,
|
|
* oldest_wal_lsn=50 → SEND_SNAPSHOT (lsn < oldest_wal).
|
|
* Requirements: 2.2, 2.3
|
|
*/
|
|
static void test_evaluate_repl_action_send_snapshot(void)
|
|
{
|
|
ha_repl_action_t action = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/10,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100,
|
|
/*oldest_wal_lsn=*/50);
|
|
TEST_ASSERT_EQUAL_INT(HA_REPL_SEND_SNAPSHOT, action);
|
|
printf("test_evaluate_repl_action_send_snapshot: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: (tag=5, lsn=50) with committed_tag=5, committed_lsn=100,
|
|
* oldest_wal_lsn=50 → STREAM_WAL (lsn >= oldest_wal but < committed).
|
|
* Requirements: 2.2, 2.3
|
|
*/
|
|
static void test_evaluate_repl_action_stream_wal(void)
|
|
{
|
|
ha_repl_action_t action = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/50,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100,
|
|
/*oldest_wal_lsn=*/50);
|
|
TEST_ASSERT_EQUAL_INT(HA_REPL_STREAM_WAL, action);
|
|
printf("test_evaluate_repl_action_stream_wal: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Test: (tag=T, lsn=0) with oldest_wal_lsn=0 (fresh system, no WAL entries yet)
|
|
* → STREAM_WAL (lsn=0 >= oldest_wal_lsn=0, so not in snapshot territory;
|
|
* lsn=0 < committed_lsn → stream WAL from the beginning).
|
|
*
|
|
* This is the canonical Task 6.1 case: a standby that has caught up on config
|
|
* (tag matches) but has no WAL applied yet on a fresh system. When there is no
|
|
* WAL retention window to worry about (oldest_wal_lsn == 0), the correct action
|
|
* is STREAM_WAL, not SEND_SNAPSHOT.
|
|
*
|
|
* Requirements: 1.1, 1.2, 2.2, 2.3 (ha-repl-hardening)
|
|
*/
|
|
static void test_evaluate_repl_action_stream_wal_fresh_system(void)
|
|
{
|
|
ha_repl_action_t action = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/0,
|
|
/*committed_tag=*/5, /*committed_lsn=*/100,
|
|
/*oldest_wal_lsn=*/0);
|
|
/* lsn=0 >= oldest_wal_lsn=0 → NOT snapshot; lsn=0 < committed → STREAM_WAL */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(HA_REPL_STREAM_WAL, action,
|
|
"(tag=T, lsn=0) with oldest_wal_lsn=0 must route to STREAM_WAL (Req 1.1, 1.2)");
|
|
printf("test_evaluate_repl_action_stream_wal_fresh_system: PASS\n");
|
|
}
|
|
|
|
/* The transitional-leader timeout default test was removed: the config knob
|
|
* and its default macro are deleted by ha-single-leader-role (the wall-clock
|
|
* force-promotion watchdog it configured is gone — Req 1.5). */
|
|
|
|
/* ── Task 3.3 (ha-wal-replication): dispatch gate allows OPCode_walCatchupEntry
|
|
* in CTRL_STANDBY, CTRL_ELECTING, and CTRL_INIT.
|
|
*
|
|
* Requirements: ha-wal-replication Req 3.3 (standby role guard).
|
|
* Verifies that ha_dispatch_gate(ha, OPCode_walCatchupEntry) returns 0
|
|
* (allow) for the three roles where a standby legitimately receives WAL
|
|
* catch-up entries from the primary. CTRL_PRIMARY must NOT allow it
|
|
* (the leader never receives WAL pushes). */
|
|
static void test_dispatch_gate_allows_wal_catchup_in_standby_roles(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
|
|
|
/* CTRL_STANDBY — must allow */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
ha_dispatch_gate(&ha, OPCode_walCatchupEntry),
|
|
"CTRL_STANDBY must allow OPCode_walCatchupEntry (Req 3.3)");
|
|
|
|
/* CTRL_ELECTING — must allow */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
ha_dispatch_gate(&ha, OPCode_walCatchupEntry),
|
|
"CTRL_ELECTING must allow OPCode_walCatchupEntry (Req 3.3)");
|
|
|
|
/* CTRL_INIT — must allow */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
ha_dispatch_gate(&ha, OPCode_walCatchupEntry),
|
|
"CTRL_INIT must allow OPCode_walCatchupEntry (Req 3.3)");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_dispatch_gate_allows_wal_catchup_in_standby_roles: PASS\n");
|
|
}
|
|
|
|
void test_ha_role(void)
|
|
{
|
|
RUN_TEST(test_ha_init_default_role_is_standby);
|
|
RUN_TEST(test_ha_init_rejects_fast_heartbeat);
|
|
RUN_TEST(test_ha_init_rejects_short_election_timeout);
|
|
RUN_TEST(test_ha_init_rejects_timeout_less_than_interval);
|
|
RUN_TEST(test_ha_promote_sets_primary_role);
|
|
/* P1-3a */
|
|
RUN_TEST(test_promote_increments_term);
|
|
RUN_TEST(test_demote_sets_standby_role);
|
|
/* Watchdog / SIGUSR1 tests deleted */
|
|
/* Task 2.1: election-iomgr-refactor new fields */
|
|
RUN_TEST(test_ha_init_zeroes_new_fields);
|
|
/* Task 1.4: ha-repl-hardening per-standby convergence fields */
|
|
RUN_TEST(test_ha_init_zeroes_standby_convergence_fields);
|
|
/* Task 3.1: ha_process_standby_state convergence tracking */
|
|
RUN_TEST(test_process_standby_state_converged);
|
|
RUN_TEST(test_process_standby_state_lsn_behind);
|
|
RUN_TEST(test_process_standby_state_tag_behind);
|
|
RUN_TEST(test_process_standby_state_regression);
|
|
RUN_TEST(test_process_standby_state_multiple_peers);
|
|
/* Task 3.2: ha_evaluate_repl_action unified decision tree */
|
|
RUN_TEST(test_evaluate_repl_action_push_config);
|
|
RUN_TEST(test_evaluate_repl_action_send_snapshot_lsn_zero);
|
|
RUN_TEST(test_evaluate_repl_action_converged);
|
|
RUN_TEST(test_evaluate_repl_action_send_snapshot);
|
|
RUN_TEST(test_evaluate_repl_action_stream_wal);
|
|
/* Task 6.1: (tag=T, lsn=0) with oldest_wal_lsn=0 → STREAM_WAL */
|
|
RUN_TEST(test_evaluate_repl_action_stream_wal_fresh_system);
|
|
/* Task 3.3: ha_dispatch_repl_action_for_standby PUSH_CONFIG wiring */
|
|
RUN_TEST(test_dispatch_repl_action_push_config_when_tag_behind);
|
|
RUN_TEST(test_dispatch_repl_action_no_push_when_converged);
|
|
/* Task 3.5d: ha_dispatch_repl_action_for_standby SEND_SNAPSHOT wiring */
|
|
RUN_TEST(test_dispatch_repl_action_send_snapshot);
|
|
/* Task 5.2 convergence-checker cases deleted (ha-single-leader-role 4.1:
|
|
* the majority-convergence checker removed — no shims, machinery-subject) */
|
|
/* transitional-leader timeout default test removed (ha-single-leader-role:
|
|
* config knob + wall-clock watchdog deleted, Req 1.5) */
|
|
/* Task 3.3 (ha-wal-replication): dispatch gate standby-role guard */
|
|
RUN_TEST(test_dispatch_gate_allows_wal_catchup_in_standby_roles);
|
|
}
|
|
|
|
/* test_am_i_primary suite deleted — am_i_primary() removed (election-iomgr-refactor) */
|