forked from lei.chen/myxce
feat(ha-single-leader-role): delete force-promotion watchdog + convergence/ack machinery (task 4.1)
This commit is contained in:
parent
b7b951b9e7
commit
a5bf86112a
10 changed files with 95 additions and 1869 deletions
|
|
@ -377,226 +377,16 @@ static void test_process_standby_state_multiple_peers(void)
|
|||
printf("test_process_standby_state_multiple_peers: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Task 5.2: ha_check_majority_convergence → PRIMARY transition ────────────
|
||||
/* ── Task 5.2 convergence-checker tests REMOVED (ha-single-leader-role task 4.1)
|
||||
*
|
||||
* After ha_process_standby_state updates standby_converged[], the primary must
|
||||
* count the number of converged standbys and promote to CTRL_PRIMARY when the
|
||||
* majority threshold is met and the current role is CTRL_PRIMARY_COMMITTING.
|
||||
*
|
||||
* Threshold formula (Req 2.3): ceil(n_peers/2) - 1 converged standbys —
|
||||
* i.e. primary + converged standbys form a majority of the cluster.
|
||||
* Integer form: n_peers / 2 standbys (n=3 → 1, n=5 → 2).
|
||||
* Single-node (n_peers<=1): threshold=0 → satisfied immediately.
|
||||
* n_peers = total cluster size including self.
|
||||
*
|
||||
* Requirements: 2.3, 2.4, 6.1, 6.3, 6.5 (ha-repl-hardening)
|
||||
* The majority-convergence checker was deleted in production (the convergence-
|
||||
* counting force-promotion path is gone — promotion is driven solely by the
|
||||
* heartbeat decision loop + per-entry majority ack). The machinery-subject
|
||||
* cases that exercised it (3node_none / 3node_one / 5node_threshold /
|
||||
* 3node_all / single_node_immediate) had nothing left to verify and were
|
||||
* deleted per design §"Transitional-State Deletions" (no replacements, no
|
||||
* shims). Their RUN_TEST registrations were removed as well.
|
||||
*/
|
||||
#ifdef XCE_TESTING
|
||||
void ha_check_majority_convergence(ha_state_t *ha, cluster_t *c);
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Test: 3-node cluster; no standbys converged (0/2)
|
||||
* → role must remain CTRL_PRIMARY_COMMITTING.
|
||||
* Requirements: 2.3, 6.3 (ha-repl-hardening)
|
||||
*/
|
||||
static void test_majority_convergence_3node_none_stays_committing(void)
|
||||
{
|
||||
#ifdef XCE_TESTING
|
||||
cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
ha_config_t cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 2000,
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
||||
};
|
||||
ha_init(&c.ha, &cfg, 0);
|
||||
c.ha.n_peers = 3; /* 3-node cluster: primary + 2 standbys */
|
||||
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
||||
memory_order_release);
|
||||
|
||||
/* No standby converged */
|
||||
c.ha.standby_converged[0] = false;
|
||||
c.ha.standby_converged[1] = false;
|
||||
|
||||
ha_check_majority_convergence(&c.ha, &c);
|
||||
|
||||
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, role,
|
||||
"3-node: 0/2 standbys converged → must remain CTRL_PRIMARY_COMMITTING (Req 2.3)");
|
||||
|
||||
ha_fini(&c.ha);
|
||||
printf("test_majority_convergence_3node_none_stays_committing: PASS\n");
|
||||
#else
|
||||
printf("test_majority_convergence_3node_none_stays_committing: SKIP\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* Test: 3-node cluster; 1/2 standbys converged → primary + 1 standby = 2/3
|
||||
* nodes hold the state = majority → must promote to CTRL_PRIMARY.
|
||||
* (Requirement 2.3 formula: ceil(3/2)-1 = 1 converged standby suffices.
|
||||
* A dead minority standby must not force the degraded-mode timeout.)
|
||||
* Requirements: 2.3, 2.4, 6.3 (ha-repl-hardening)
|
||||
*/
|
||||
static void test_majority_convergence_3node_one_standby_promotes(void)
|
||||
{
|
||||
#ifdef XCE_TESTING
|
||||
cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
ha_config_t cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 2000,
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
||||
};
|
||||
ha_init(&c.ha, &cfg, 0);
|
||||
c.ha.n_peers = 3;
|
||||
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
||||
memory_order_release);
|
||||
|
||||
/* Only standby[0] converged; standby[1] not converged */
|
||||
c.ha.standby_converged[0] = true;
|
||||
c.ha.standby_converged[1] = false;
|
||||
|
||||
ha_check_majority_convergence(&c.ha, &c);
|
||||
|
||||
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
|
||||
"3-node: 1/2 standbys converged → majority → CTRL_PRIMARY (Req 2.3)");
|
||||
|
||||
ha_fini(&c.ha);
|
||||
printf("test_majority_convergence_3node_one_standby_promotes: PASS\n");
|
||||
#else
|
||||
printf("test_majority_convergence_3node_one_standby_promotes: SKIP\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* Test: 5-node cluster; threshold = 5/2 = 2 converged standbys.
|
||||
* 1/4 converged → stays COMMITTING; 2/4 converged → promotes.
|
||||
* Requirements: 2.3, 6.3 (ha-repl-hardening)
|
||||
*/
|
||||
static void test_majority_convergence_5node_threshold(void)
|
||||
{
|
||||
#ifdef XCE_TESTING
|
||||
cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
ha_config_t cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 2000,
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
||||
};
|
||||
ha_init(&c.ha, &cfg, 0);
|
||||
c.ha.n_peers = 5;
|
||||
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
||||
memory_order_release);
|
||||
|
||||
/* 1/4 converged: primary + 1 = 2 of 5 — not a majority → stays */
|
||||
c.ha.standby_converged[0] = true;
|
||||
ha_check_majority_convergence(&c.ha, &c);
|
||||
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, role,
|
||||
"5-node: 1/4 standbys converged → must remain COMMITTING (Req 2.3)");
|
||||
|
||||
/* 2/4 converged: primary + 2 = 3 of 5 — majority → promotes */
|
||||
c.ha.standby_converged[1] = true;
|
||||
ha_check_majority_convergence(&c.ha, &c);
|
||||
role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
|
||||
"5-node: 2/4 standbys converged → majority → CTRL_PRIMARY (Req 2.3)");
|
||||
|
||||
ha_fini(&c.ha);
|
||||
printf("test_majority_convergence_5node_threshold: PASS\n");
|
||||
#else
|
||||
printf("test_majority_convergence_5node_threshold: SKIP\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* Test: 3-node cluster (n_peers=2); both standbys converged (2/2)
|
||||
* → role must transition to CTRL_PRIMARY.
|
||||
* Requirements: 2.3, 2.4, 6.3 (ha-repl-hardening)
|
||||
*/
|
||||
static void test_majority_convergence_3node_all_standbys_promotes(void)
|
||||
{
|
||||
#ifdef XCE_TESTING
|
||||
cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
ha_config_t cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 2000,
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
||||
};
|
||||
ha_init(&c.ha, &cfg, 0);
|
||||
c.ha.n_peers = 3; /* 3-node cluster */
|
||||
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
||||
memory_order_release);
|
||||
|
||||
/* Both standbys converged */
|
||||
c.ha.standby_converged[0] = true;
|
||||
c.ha.standby_converged[1] = true;
|
||||
|
||||
ha_check_majority_convergence(&c.ha, &c);
|
||||
|
||||
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
|
||||
"3-node: 2/2 standbys converged → must transition to CTRL_PRIMARY (Req 2.3, 2.4)");
|
||||
|
||||
ha_fini(&c.ha);
|
||||
printf("test_majority_convergence_3node_all_standbys_promotes: PASS\n");
|
||||
#else
|
||||
printf("test_majority_convergence_3node_all_standbys_promotes: SKIP\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* Test: single-node cluster (n_peers=0)
|
||||
* → threshold=0, satisfied immediately → must promote to CTRL_PRIMARY.
|
||||
* Requirements: 6.5, 2.3 (ha-repl-hardening): single-node is always satisfied.
|
||||
*/
|
||||
static void test_majority_convergence_single_node_immediate(void)
|
||||
{
|
||||
#ifdef XCE_TESTING
|
||||
cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
ha_config_t cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 2000,
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
||||
};
|
||||
ha_init(&c.ha, &cfg, 0);
|
||||
c.ha.n_peers = 1; /* single-node: just self */
|
||||
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
||||
memory_order_release);
|
||||
|
||||
/* No standbys to converge — threshold=0 must be satisfied immediately */
|
||||
ha_check_majority_convergence(&c.ha, &c);
|
||||
|
||||
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
|
||||
"single-node: n_peers=1 → threshold=0 satisfied → must promote to CTRL_PRIMARY (Req 6.5)");
|
||||
|
||||
ha_fini(&c.ha);
|
||||
printf("test_majority_convergence_single_node_immediate: PASS\n");
|
||||
#else
|
||||
printf("test_majority_convergence_single_node_immediate: SKIP\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
/* ── Task 3.3: ha_dispatch_repl_action_for_standby PUSH_CONFIG wiring ──────
|
||||
*
|
||||
|
|
@ -879,16 +669,16 @@ static void test_evaluate_repl_action_stream_wal_fresh_system(void)
|
|||
printf("test_evaluate_repl_action_stream_wal_fresh_system: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Task 5.3: committing timeout default value and degraded-mode WARN ───────
|
||||
/* ── Task 5.3: committing timeout default value ──────────────────────────────
|
||||
*
|
||||
* Requirements: 6.4, 7.5 (ha-repl-hardening)
|
||||
* Requirements: 6.4 (ha-repl-hardening)
|
||||
*
|
||||
* Verifies:
|
||||
* - HA_DEFAULT_COMMITTING_TIMEOUT_MS == 60000 (safety-net window is 60 s)
|
||||
* - committing_timeout_fn_for_test still promotes to CTRL_PRIMARY in degraded
|
||||
* mode even when only a subset of standbys have converged
|
||||
* - The ha->standby_converged[] state is readable so the WARN log can include
|
||||
* the converged/pending counts
|
||||
* The committing_timeout_ms CONFIG knob survives until task 5.2 (it is a
|
||||
* different token from the deleted committing-timeout watchdog callback). The
|
||||
* degraded-mode force-promotion cases that drove the watchdog test wrapper
|
||||
* were machinery-subject and were deleted (ha-single-leader-role task 4.1,
|
||||
* design §"Transitional-State Deletions": no shims). Only the config-default
|
||||
* assertion remains.
|
||||
*/
|
||||
|
||||
/*
|
||||
|
|
@ -902,112 +692,6 @@ static void test_committing_timeout_default_is_60s(void)
|
|||
printf("test_committing_timeout_default_is_60s: PASS\n");
|
||||
}
|
||||
|
||||
/*
|
||||
* Test: committing_timeout_fn promotes to PRIMARY even when only some standbys
|
||||
* have converged (degraded mode). The WARN log path fires when timeout expires
|
||||
* before majority convergence. After promotion, role must be CTRL_PRIMARY.
|
||||
*
|
||||
* Requirements: 6.4, 7.5 (ha-repl-hardening)
|
||||
*/
|
||||
#ifdef XCE_TESTING
|
||||
extern int committing_timeout_fn_for_test(ha_state_t *ha, cluster_t *c,
|
||||
int cancelled_val);
|
||||
#endif
|
||||
|
||||
static void test_committing_timeout_promotes_degraded_with_partial_convergence(void)
|
||||
{
|
||||
#ifdef XCE_TESTING
|
||||
cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
ha_config_t cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 2000,
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
||||
};
|
||||
ha_init(&c.ha, &cfg, 0);
|
||||
c.ha.n_peers = 3; /* 3-node cluster; 2 standbys */
|
||||
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
||||
memory_order_release);
|
||||
|
||||
/* Only 1 of 2 standbys has converged — majority not yet reached */
|
||||
c.ha.standby_converged[0] = true;
|
||||
c.ha.standby_converged[1] = false;
|
||||
|
||||
/*
|
||||
* Count converged/pending so the test mirrors what the WARN log reads.
|
||||
* n_converged=1, n_pending=1 (Req 7.5).
|
||||
*/
|
||||
int n_converged = 0;
|
||||
int n_pending = 0;
|
||||
for (int i = 0; i < c.ha.n_peers - 1; i++) {
|
||||
if (c.ha.standby_converged[i])
|
||||
n_converged++;
|
||||
else
|
||||
n_pending++;
|
||||
}
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, n_converged,
|
||||
"degraded-mode: 1 standby converged before timeout");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, n_pending,
|
||||
"degraded-mode: 1 standby still pending before timeout");
|
||||
|
||||
/* Fire the timeout — must promote despite partial convergence */
|
||||
committing_timeout_fn_for_test(&c.ha, &c, 0 /* not cancelled */);
|
||||
|
||||
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
|
||||
"committing timeout: must promote to CTRL_PRIMARY in degraded mode (Req 6.4)");
|
||||
|
||||
ha_fini(&c.ha);
|
||||
printf("test_committing_timeout_promotes_degraded_with_partial_convergence: PASS "
|
||||
"(n_converged=%d, n_pending=%d)\n", n_converged, n_pending);
|
||||
#else
|
||||
printf("test_committing_timeout_promotes_degraded_with_partial_convergence: SKIP\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* Test: committing_timeout_fn promotes when NO standbys have converged
|
||||
* (worst-case degraded mode — all standbys unreachable).
|
||||
* Requirements: 6.4 (ha-repl-hardening)
|
||||
*/
|
||||
static void test_committing_timeout_promotes_degraded_zero_converged(void)
|
||||
{
|
||||
#ifdef XCE_TESTING
|
||||
cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
ha_config_t cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 2000,
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
||||
};
|
||||
ha_init(&c.ha, &cfg, 0);
|
||||
c.ha.n_peers = 3; /* 3-node cluster; all standbys unreachable */
|
||||
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
||||
memory_order_release);
|
||||
|
||||
/* No standbys converged — n_converged=0, n_pending=2 (Req 7.5) */
|
||||
c.ha.standby_converged[0] = false;
|
||||
c.ha.standby_converged[1] = false;
|
||||
|
||||
committing_timeout_fn_for_test(&c.ha, &c, 0 /* not cancelled */);
|
||||
|
||||
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
|
||||
"committing timeout: must promote even with 0 converged standbys (degraded, Req 6.4)");
|
||||
|
||||
ha_fini(&c.ha);
|
||||
printf("test_committing_timeout_promotes_degraded_zero_converged: PASS\n");
|
||||
#else
|
||||
printf("test_committing_timeout_promotes_degraded_zero_converged: SKIP\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
/* ── Task 3.3 (ha-wal-replication): dispatch gate allows OPCode_walCatchupEntry
|
||||
* in CTRL_STANDBY, CTRL_ELECTING, and CTRL_INIT.
|
||||
*
|
||||
|
|
@ -1085,16 +769,12 @@ void test_ha_role(void)
|
|||
RUN_TEST(test_dispatch_repl_action_no_push_when_converged);
|
||||
/* Task 3.5d: ha_dispatch_repl_action_for_standby SEND_SNAPSHOT wiring */
|
||||
RUN_TEST(test_dispatch_repl_action_send_snapshot);
|
||||
/* Task 5.2: ha_check_majority_convergence → PRIMARY transition */
|
||||
RUN_TEST(test_majority_convergence_3node_none_stays_committing);
|
||||
RUN_TEST(test_majority_convergence_3node_one_standby_promotes);
|
||||
RUN_TEST(test_majority_convergence_5node_threshold);
|
||||
RUN_TEST(test_majority_convergence_3node_all_standbys_promotes);
|
||||
RUN_TEST(test_majority_convergence_single_node_immediate);
|
||||
/* Task 5.3: committing timeout default value and degraded-mode WARN */
|
||||
/* Task 5.2 convergence-checker cases deleted (ha-single-leader-role 4.1:
|
||||
* the majority-convergence checker removed — no shims, machinery-subject) */
|
||||
/* Task 5.3: committing timeout config-default knob (committing_timeout_ms
|
||||
* survives until task 5.2; the degraded-mode committing-timeout watchdog
|
||||
* cases were deleted — ha-single-leader-role 4.1, machinery-subject) */
|
||||
RUN_TEST(test_committing_timeout_default_is_60s);
|
||||
RUN_TEST(test_committing_timeout_promotes_degraded_with_partial_convergence);
|
||||
RUN_TEST(test_committing_timeout_promotes_degraded_zero_converged);
|
||||
/* Task 3.3 (ha-wal-replication): dispatch gate standby-role guard */
|
||||
RUN_TEST(test_dispatch_gate_allows_wal_catchup_in_standby_roles);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue