feat(ha-wal-replication): standby WAL ACK reply (tasks 3.1-3.3)
Standby sends WalAckReply via xce_reply after each successfully applied WAL entry. ha_apply_wal_catchup_entry signature extended with endpoint_t* and cluster_t* parameters. walackreplymsg union arm added to RequestReply capnp schema and xce_reply_t. Error entries skip ACK. Dispatch gate test confirms OPCode_walCatchupEntry allowed in CTRL_STANDBY/ELECTING/INIT. Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
81dbc4d7b4
commit
566c91cd8b
8 changed files with 304 additions and 60 deletions
|
|
@ -383,8 +383,10 @@ static void test_process_standby_state_multiple_peers(void)
|
|||
* count the number of converged standbys and promote to CTRL_PRIMARY when the
|
||||
* majority threshold is met and the current role is CTRL_PRIMARY_COMMITTING.
|
||||
*
|
||||
* Threshold formula: n_peers - 1 (all standbys must converge).
|
||||
* Single-node (n_peers=1): threshold=0 → satisfied immediately (matches raft convention).
|
||||
* Threshold formula (Req 2.3): ceil(n_peers/2) - 1 converged standbys —
|
||||
* i.e. primary + converged standbys form a majority of the cluster.
|
||||
* Integer form: n_peers / 2 standbys (n=3 → 1, n=5 → 2).
|
||||
* Single-node (n_peers<=1): threshold=0 → satisfied immediately.
|
||||
* n_peers = total cluster size including self.
|
||||
*
|
||||
* Requirements: 2.3, 2.4, 6.1, 6.3, 6.5 (ha-repl-hardening)
|
||||
|
|
@ -394,11 +396,11 @@ void ha_check_majority_convergence(ha_state_t *ha, cluster_t *c);
|
|||
#endif
|
||||
|
||||
/*
|
||||
* Test: 3-node cluster (n_peers=2); only 1/2 standbys converged
|
||||
* Test: 3-node cluster; no standbys converged (0/2)
|
||||
* → role must remain CTRL_PRIMARY_COMMITTING.
|
||||
* Requirements: 2.3, 6.3 (ha-repl-hardening)
|
||||
*/
|
||||
static void test_majority_convergence_3node_one_standby_stays_committing(void)
|
||||
static void test_majority_convergence_3node_none_stays_committing(void)
|
||||
{
|
||||
#ifdef XCE_TESTING
|
||||
cluster_t c;
|
||||
|
|
@ -416,6 +418,48 @@ static void test_majority_convergence_3node_one_standby_stays_committing(void)
|
|||
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
||||
memory_order_release);
|
||||
|
||||
/* No standby converged */
|
||||
c.ha.standby_converged[0] = false;
|
||||
c.ha.standby_converged[1] = false;
|
||||
|
||||
ha_check_majority_convergence(&c.ha, &c);
|
||||
|
||||
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, role,
|
||||
"3-node: 0/2 standbys converged → must remain CTRL_PRIMARY_COMMITTING (Req 2.3)");
|
||||
|
||||
ha_fini(&c.ha);
|
||||
printf("test_majority_convergence_3node_none_stays_committing: PASS\n");
|
||||
#else
|
||||
printf("test_majority_convergence_3node_none_stays_committing: SKIP\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* Test: 3-node cluster; 1/2 standbys converged → primary + 1 standby = 2/3
|
||||
* nodes hold the state = majority → must promote to CTRL_PRIMARY.
|
||||
* (Requirement 2.3 formula: ceil(3/2)-1 = 1 converged standby suffices.
|
||||
* A dead minority standby must not force the degraded-mode timeout.)
|
||||
* Requirements: 2.3, 2.4, 6.3 (ha-repl-hardening)
|
||||
*/
|
||||
static void test_majority_convergence_3node_one_standby_promotes(void)
|
||||
{
|
||||
#ifdef XCE_TESTING
|
||||
cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
ha_config_t cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 2000,
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
||||
};
|
||||
ha_init(&c.ha, &cfg, 0);
|
||||
c.ha.n_peers = 3;
|
||||
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
||||
memory_order_release);
|
||||
|
||||
/* Only standby[0] converged; standby[1] not converged */
|
||||
c.ha.standby_converged[0] = true;
|
||||
c.ha.standby_converged[1] = false;
|
||||
|
|
@ -423,13 +467,57 @@ static void test_majority_convergence_3node_one_standby_stays_committing(void)
|
|||
ha_check_majority_convergence(&c.ha, &c);
|
||||
|
||||
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, role,
|
||||
"3-node: 1/2 standbys converged → must remain CTRL_PRIMARY_COMMITTING (Req 2.3)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
|
||||
"3-node: 1/2 standbys converged → majority → CTRL_PRIMARY (Req 2.3)");
|
||||
|
||||
ha_fini(&c.ha);
|
||||
printf("test_majority_convergence_3node_one_standby_stays_committing: PASS\n");
|
||||
printf("test_majority_convergence_3node_one_standby_promotes: PASS\n");
|
||||
#else
|
||||
printf("test_majority_convergence_3node_one_standby_stays_committing: SKIP\n");
|
||||
printf("test_majority_convergence_3node_one_standby_promotes: SKIP\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* Test: 5-node cluster; threshold = 5/2 = 2 converged standbys.
|
||||
* 1/4 converged → stays COMMITTING; 2/4 converged → promotes.
|
||||
* Requirements: 2.3, 6.3 (ha-repl-hardening)
|
||||
*/
|
||||
static void test_majority_convergence_5node_threshold(void)
|
||||
{
|
||||
#ifdef XCE_TESTING
|
||||
cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
ha_config_t cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 2000,
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
||||
};
|
||||
ha_init(&c.ha, &cfg, 0);
|
||||
c.ha.n_peers = 5;
|
||||
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
||||
memory_order_release);
|
||||
|
||||
/* 1/4 converged: primary + 1 = 2 of 5 — not a majority → stays */
|
||||
c.ha.standby_converged[0] = true;
|
||||
ha_check_majority_convergence(&c.ha, &c);
|
||||
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, role,
|
||||
"5-node: 1/4 standbys converged → must remain COMMITTING (Req 2.3)");
|
||||
|
||||
/* 2/4 converged: primary + 2 = 3 of 5 — majority → promotes */
|
||||
c.ha.standby_converged[1] = true;
|
||||
ha_check_majority_convergence(&c.ha, &c);
|
||||
role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
|
||||
"5-node: 2/4 standbys converged → majority → CTRL_PRIMARY (Req 2.3)");
|
||||
|
||||
ha_fini(&c.ha);
|
||||
printf("test_majority_convergence_5node_threshold: PASS\n");
|
||||
#else
|
||||
printf("test_majority_convergence_5node_threshold: SKIP\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
|
|
@ -920,6 +1008,49 @@ static void test_committing_timeout_promotes_degraded_zero_converged(void)
|
|||
#endif
|
||||
}
|
||||
|
||||
/* ── Task 3.3 (ha-wal-replication): dispatch gate allows OPCode_walCatchupEntry
|
||||
* in CTRL_STANDBY, CTRL_ELECTING, and CTRL_INIT.
|
||||
*
|
||||
* Requirements: ha-wal-replication Req 3.3 (standby role guard).
|
||||
* Verifies that ha_dispatch_gate(ha, OPCode_walCatchupEntry) returns 0
|
||||
* (allow) for the three roles where a standby legitimately receives WAL
|
||||
* catch-up entries from the primary. CTRL_PRIMARY and
|
||||
* CTRL_PRIMARY_COMMITTING must NOT allow it (they never receive WAL pushes). */
|
||||
static void test_dispatch_gate_allows_wal_catchup_in_standby_roles(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
ha_config_t cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 2000,
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
||||
};
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
||||
|
||||
/* CTRL_STANDBY — must allow */
|
||||
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
||||
ha_dispatch_gate(&ha, OPCode_walCatchupEntry),
|
||||
"CTRL_STANDBY must allow OPCode_walCatchupEntry (Req 3.3)");
|
||||
|
||||
/* CTRL_ELECTING — must allow */
|
||||
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
||||
ha_dispatch_gate(&ha, OPCode_walCatchupEntry),
|
||||
"CTRL_ELECTING must allow OPCode_walCatchupEntry (Req 3.3)");
|
||||
|
||||
/* CTRL_INIT — must allow */
|
||||
atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
||||
ha_dispatch_gate(&ha, OPCode_walCatchupEntry),
|
||||
"CTRL_INIT must allow OPCode_walCatchupEntry (Req 3.3)");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_dispatch_gate_allows_wal_catchup_in_standby_roles: PASS\n");
|
||||
}
|
||||
|
||||
void test_ha_role(void)
|
||||
{
|
||||
RUN_TEST(test_ha_init_default_role_is_standby);
|
||||
|
|
@ -955,13 +1086,17 @@ void test_ha_role(void)
|
|||
/* Task 3.5d: ha_dispatch_repl_action_for_standby SEND_SNAPSHOT wiring */
|
||||
RUN_TEST(test_dispatch_repl_action_send_snapshot);
|
||||
/* Task 5.2: ha_check_majority_convergence → PRIMARY transition */
|
||||
RUN_TEST(test_majority_convergence_3node_one_standby_stays_committing);
|
||||
RUN_TEST(test_majority_convergence_3node_none_stays_committing);
|
||||
RUN_TEST(test_majority_convergence_3node_one_standby_promotes);
|
||||
RUN_TEST(test_majority_convergence_5node_threshold);
|
||||
RUN_TEST(test_majority_convergence_3node_all_standbys_promotes);
|
||||
RUN_TEST(test_majority_convergence_single_node_immediate);
|
||||
/* Task 5.3: committing timeout default value and degraded-mode WARN */
|
||||
RUN_TEST(test_committing_timeout_default_is_60s);
|
||||
RUN_TEST(test_committing_timeout_promotes_degraded_with_partial_convergence);
|
||||
RUN_TEST(test_committing_timeout_promotes_degraded_zero_converged);
|
||||
/* Task 3.3 (ha-wal-replication): dispatch gate standby-role guard */
|
||||
RUN_TEST(test_dispatch_gate_allows_wal_catchup_in_standby_roles);
|
||||
}
|
||||
|
||||
/* test_am_i_primary suite deleted — am_i_primary() removed (election-iomgr-refactor) */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue