wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity

feat(ha-wal-replication): standby WAL ACK reply (tasks 3.1-3.3)

Browse source 
Standby sends WalAckReply via xce_reply after each successfully
applied WAL entry. ha_apply_wal_catchup_entry signature extended
with endpoint_t* and cluster_t* parameters. walackreplymsg union
arm added to RequestReply capnp schema and xce_reply_t. Error
entries skip ACK. Dispatch gate test confirms OPCode_walCatchupEntry
allowed in CTRL_STANDBY/ELECTING/INIT.

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-06-10 09:14:25 +00:00
parent 81dbc4d7b4
commit 566c91cd8b
8 changed files with 304 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

28
src/controller/testsuite/test_config_repl.c
View file

@ -32,12 +32,12 @@
#include "xce/cluster_def.h"
#ifdef XCE_TESTING
/* Declared in cluster.c under #ifdef XCE_TESTING — test seam for
* ha_apply_wal_catchup_entry: applies a single WAL catch-up entry to
* persistent storage and advances ha->applied_lsn on success.
* On error: logs ERROR, returns non-zero, does not crash. */
/* Declared in ha.h — test seam for ha_apply_wal_catchup_entry.
* ep and c are passed as NULL in unit tests (ACK path is skipped when
* either is NULL — Task 3.1, ha-wal-replication). */
int ha_apply_wal_catchup_entry(ha_state_t *ha, persist_t *p,
wal_catchup_entry_t *entry);
wal_catchup_entry_t *entry,
struct endpoint_t *ep, cluster_t *c);
#endif
/* Per-test directory helpers — not Unity setUp/tearDown (those are in
@ -974,7 +974,7 @@ static void test_wal_catchup_entry_advances_applied_lsn(void)
entry.entries = NULL;
/* GREEN will implement ha_apply_wal_catchup_entry() */
int rc = ha_apply_wal_catchup_entry(&ha, p, &entry);
int rc = ha_apply_wal_catchup_entry(&ha, p, &entry, NULL, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
/* applied_lsn must advance to lsn+1 = 43 (mirrors persist->current_lsn) */
TEST_ASSERT_EQUAL_UINT64(43,
@ -1014,7 +1014,7 @@ static void test_wal_catchup_entry_error_skipped_no_crash(void)
entry.entries = NULL;
/* Handler must NOT crash; must log ERROR; applied_lsn must remain at 10 */
int rc = ha_apply_wal_catchup_entry(&ha, /*persist=*/NULL, &entry);
int rc = ha_apply_wal_catchup_entry(&ha, /*persist=*/NULL, &entry, NULL, NULL);
/* rc is non-zero (error propagated from apply_wal_entry(-EINVAL)) */
TEST_ASSERT_TRUE(rc != 0);
/* applied_lsn must not advance on failure */
@ -1038,7 +1038,7 @@ static void test_wal_catchup_entry_null_entry_no_crash(void)
ha_init(&ha, &cfg, 0);
/* NULL entry — must not crash, must return error */
int rc = ha_apply_wal_catchup_entry(&ha, NULL, NULL);
int rc = ha_apply_wal_catchup_entry(&ha, NULL, NULL, NULL, NULL);
TEST_ASSERT_TRUE(rc != 0);
ha_fini(&ha);
@ -1084,7 +1084,7 @@ static void test_wal_catchup_batch_advances_applied_lsn(void)
outer.n_entries = 3;
outer.entries = batch_entries;
int rc = ha_apply_wal_catchup_entry(&ha, p, &outer);
int rc = ha_apply_wal_catchup_entry(&ha, p, &outer, NULL, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
/* applied_lsn must advance to last_entry->lsn + 1 = 13 */
@ -1132,7 +1132,7 @@ static void test_wal_catchup_partial_batch(void)
outer.n_entries = 2;
outer.entries = batch_entries;
int rc = ha_apply_wal_catchup_entry(&ha, p, &outer);
int rc = ha_apply_wal_catchup_entry(&ha, p, &outer, NULL, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
/* applied_lsn must advance to 21+1=22 */
@ -1182,7 +1182,7 @@ static void test_wal_catchup_batch_error_entry_skipped(void)
outer.entries = batch_entries;
/* Handler must not crash; applied_lsn advances for successful entries */
int rc = ha_apply_wal_catchup_entry(&ha, p, &outer);
int rc = ha_apply_wal_catchup_entry(&ha, p, &outer, NULL, NULL);
/* Overall rc == 0: outer batch call succeeds even with inner skips */
TEST_ASSERT_EQUAL_INT(0, rc);
@ -1223,7 +1223,7 @@ static void test_wal_catchup_batch_zero_entries_fallback(void)
entry.n_entries = 0;
entry.entries = NULL;
int rc = ha_apply_wal_catchup_entry(&ha, p, &entry);
int rc = ha_apply_wal_catchup_entry(&ha, p, &entry, NULL, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Single-entry path: applied_lsn = 50+1 = 51 */
TEST_ASSERT_EQUAL_UINT64(51,
@ -1281,7 +1281,7 @@ static void test_wal_catchup_batch_256_entries(void)
TEST_ASSERT_EQUAL_UINT64(0,
(uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire));
int rc = ha_apply_wal_catchup_entry(&ha, p, &outer);
int rc = ha_apply_wal_catchup_entry(&ha, p, &outer, NULL, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
/* applied_lsn must advance to last_entry->lsn + 1 = 1000+255+1 = 1256 */
@ -1343,7 +1343,7 @@ static void test_wal_catchup_batch_applied_lsn_only_after_full_batch(void)
outer.n_entries = 4;
outer.entries = ptrs;
int rc = ha_apply_wal_catchup_entry(&ha, p, &outer);
int rc = ha_apply_wal_catchup_entry(&ha, p, &outer, NULL, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
/* applied_lsn must be exactly 204 (203+1) — the post-batch value.

153
src/controller/testsuite/test_ha_role.c
View file

@ -383,8 +383,10 @@ static void test_process_standby_state_multiple_peers(void)
* count the number of converged standbys and promote to CTRL_PRIMARY when the
* majority threshold is met and the current role is CTRL_PRIMARY_COMMITTING.
*
* Threshold formula: n_peers - 1 (all standbys must converge).
* Single-node (n_peers=1): threshold=0 → satisfied immediately (matches raft convention).
* Threshold formula (Req 2.3): ceil(n_peers/2) - 1 converged standbys —
* i.e. primary + converged standbys form a majority of the cluster.
* Integer form: n_peers / 2 standbys (n=3 → 1, n=5 → 2).
* Single-node (n_peers<=1): threshold=0 → satisfied immediately.
* n_peers = total cluster size including self.
*
* Requirements: 2.3, 2.4, 6.1, 6.3, 6.5 (ha-repl-hardening)
@ -394,11 +396,11 @@ void ha_check_majority_convergence(ha_state_t *ha, cluster_t *c);
#endif
/*
* Test: 3-node cluster (n_peers=2); only 1/2 standbys converged
* Test: 3-node cluster; no standbys converged (0/2)
* → role must remain CTRL_PRIMARY_COMMITTING.
* Requirements: 2.3, 6.3 (ha-repl-hardening)
*/
static void test_majority_convergence_3node_one_standby_stays_committing(void)
static void test_majority_convergence_3node_none_stays_committing(void)
{
#ifdef XCE_TESTING
cluster_t c;
@ -416,6 +418,48 @@ static void test_majority_convergence_3node_one_standby_stays_committing(void)
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
memory_order_release);
/* No standby converged */
c.ha.standby_converged[0] = false;
c.ha.standby_converged[1] = false;
ha_check_majority_convergence(&c.ha, &c);
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, role,
"3-node: 0/2 standbys converged → must remain CTRL_PRIMARY_COMMITTING (Req 2.3)");
ha_fini(&c.ha);
printf("test_majority_convergence_3node_none_stays_committing: PASS\n");
#else
printf("test_majority_convergence_3node_none_stays_committing: SKIP\n");
#endif
}
/*
* Test: 3-node cluster; 1/2 standbys converged → primary + 1 standby = 2/3
* nodes hold the state = majority → must promote to CTRL_PRIMARY.
* (Requirement 2.3 formula: ceil(3/2)-1 = 1 converged standby suffices.
* A dead minority standby must not force the degraded-mode timeout.)
* Requirements: 2.3, 2.4, 6.3 (ha-repl-hardening)
*/
static void test_majority_convergence_3node_one_standby_promotes(void)
{
#ifdef XCE_TESTING
cluster_t c;
memset(&c, 0, sizeof(c));
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
};
ha_init(&c.ha, &cfg, 0);
c.ha.n_peers = 3;
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
memory_order_release);
/* Only standby[0] converged; standby[1] not converged */
c.ha.standby_converged[0] = true;
c.ha.standby_converged[1] = false;
@ -423,13 +467,57 @@ static void test_majority_convergence_3node_one_standby_stays_committing(void)
ha_check_majority_convergence(&c.ha, &c);
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, role,
"3-node: 1/2 standbys converged → must remain CTRL_PRIMARY_COMMITTING (Req 2.3)");
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
"3-node: 1/2 standbys converged → majority → CTRL_PRIMARY (Req 2.3)");
ha_fini(&c.ha);
printf("test_majority_convergence_3node_one_standby_stays_committing: PASS\n");
printf("test_majority_convergence_3node_one_standby_promotes: PASS\n");
#else
printf("test_majority_convergence_3node_one_standby_stays_committing: SKIP\n");
printf("test_majority_convergence_3node_one_standby_promotes: SKIP\n");
#endif
}
/*
* Test: 5-node cluster; threshold = 5/2 = 2 converged standbys.
* 1/4 converged → stays COMMITTING; 2/4 converged → promotes.
* Requirements: 2.3, 6.3 (ha-repl-hardening)
*/
static void test_majority_convergence_5node_threshold(void)
{
#ifdef XCE_TESTING
cluster_t c;
memset(&c, 0, sizeof(c));
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
};
ha_init(&c.ha, &cfg, 0);
c.ha.n_peers = 5;
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
memory_order_release);
/* 1/4 converged: primary + 1 = 2 of 5 — not a majority → stays */
c.ha.standby_converged[0] = true;
ha_check_majority_convergence(&c.ha, &c);
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, role,
"5-node: 1/4 standbys converged → must remain COMMITTING (Req 2.3)");
/* 2/4 converged: primary + 2 = 3 of 5 — majority → promotes */
c.ha.standby_converged[1] = true;
ha_check_majority_convergence(&c.ha, &c);
role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
"5-node: 2/4 standbys converged → majority → CTRL_PRIMARY (Req 2.3)");
ha_fini(&c.ha);
printf("test_majority_convergence_5node_threshold: PASS\n");
#else
printf("test_majority_convergence_5node_threshold: SKIP\n");
#endif
}
@ -920,6 +1008,49 @@ static void test_committing_timeout_promotes_degraded_zero_converged(void)
#endif
}
/* ── Task 3.3 (ha-wal-replication): dispatch gate allows OPCode_walCatchupEntry
* in CTRL_STANDBY, CTRL_ELECTING, and CTRL_INIT.
*
* Requirements: ha-wal-replication Req 3.3 (standby role guard).
* Verifies that ha_dispatch_gate(ha, OPCode_walCatchupEntry) returns 0
* (allow) for the three roles where a standby legitimately receives WAL
* catch-up entries from the primary. CTRL_PRIMARY and
* CTRL_PRIMARY_COMMITTING must NOT allow it (they never receive WAL pushes). */
static void test_dispatch_gate_allows_wal_catchup_in_standby_roles(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
};
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
/* CTRL_STANDBY — must allow */
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
ha_dispatch_gate(&ha, OPCode_walCatchupEntry),
"CTRL_STANDBY must allow OPCode_walCatchupEntry (Req 3.3)");
/* CTRL_ELECTING — must allow */
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
ha_dispatch_gate(&ha, OPCode_walCatchupEntry),
"CTRL_ELECTING must allow OPCode_walCatchupEntry (Req 3.3)");
/* CTRL_INIT — must allow */
atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release);
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
ha_dispatch_gate(&ha, OPCode_walCatchupEntry),
"CTRL_INIT must allow OPCode_walCatchupEntry (Req 3.3)");
ha_fini(&ha);
printf("test_dispatch_gate_allows_wal_catchup_in_standby_roles: PASS\n");
}
void test_ha_role(void)
{
RUN_TEST(test_ha_init_default_role_is_standby);
@ -955,13 +1086,17 @@ void test_ha_role(void)
/* Task 3.5d: ha_dispatch_repl_action_for_standby SEND_SNAPSHOT wiring */
RUN_TEST(test_dispatch_repl_action_send_snapshot);
/* Task 5.2: ha_check_majority_convergence → PRIMARY transition */
RUN_TEST(test_majority_convergence_3node_one_standby_stays_committing);
RUN_TEST(test_majority_convergence_3node_none_stays_committing);
RUN_TEST(test_majority_convergence_3node_one_standby_promotes);
RUN_TEST(test_majority_convergence_5node_threshold);
RUN_TEST(test_majority_convergence_3node_all_standbys_promotes);
RUN_TEST(test_majority_convergence_single_node_immediate);
/* Task 5.3: committing timeout default value and degraded-mode WARN */
RUN_TEST(test_committing_timeout_default_is_60s);
RUN_TEST(test_committing_timeout_promotes_degraded_with_partial_convergence);
RUN_TEST(test_committing_timeout_promotes_degraded_zero_converged);
/* Task 3.3 (ha-wal-replication): dispatch gate standby-role guard */
RUN_TEST(test_dispatch_gate_allows_wal_catchup_in_standby_roles);
}
/* test_am_i_primary suite deleted — am_i_primary() removed (election-iomgr-refactor) */