forked from lei.chen/myxce
feat(ha-repl-hardening): committing timeout 60s degraded-mode safety net (task 5.3)
- Change HA_DEFAULT_COMMITTING_TIMEOUT_MS from 10000 to 60000 - committing_timeout_fn: count converged/pending standbys from standby_converged[], log WARN with counts, then promote (degraded mode) - Replace hardcoded 10000 in test_ha_evaluate_committing.c with macro - Three new tests: default=60000, degrade-promote with 1/2 and 0/2 converged
This commit is contained in:
parent
534bfd68fe
commit
0762577bf7
5 changed files with 160 additions and 9 deletions
|
|
@ -766,6 +766,135 @@ static void test_evaluate_repl_action_stream_wal(void)
|
|||
printf("test_evaluate_repl_action_stream_wal: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Task 5.3: committing timeout default value and degraded-mode WARN ───────
|
||||
*
|
||||
* Requirements: 6.4, 7.5 (ha-repl-hardening)
|
||||
*
|
||||
* Verifies:
|
||||
* - HA_DEFAULT_COMMITTING_TIMEOUT_MS == 60000 (safety-net window is 60 s)
|
||||
* - committing_timeout_fn_for_test still promotes to CTRL_PRIMARY in degraded
|
||||
* mode even when only a subset of standbys have converged
|
||||
* - The ha->standby_converged[] state is readable so the WARN log can include
|
||||
* the converged/pending counts
|
||||
*/
|
||||
|
||||
/*
|
||||
* Test: HA_DEFAULT_COMMITTING_TIMEOUT_MS is 60000 (not 10000).
|
||||
* Requirements: 6.4 (ha-repl-hardening) — "Change committing_timeout_ms default from 10000 to 60000"
|
||||
*/
|
||||
static void test_committing_timeout_default_is_60s(void)
|
||||
{
|
||||
TEST_ASSERT_EQUAL_UINT32_MESSAGE(60000, HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
||||
"HA_DEFAULT_COMMITTING_TIMEOUT_MS must be 60000 (Req 6.4, Task 5.3)");
|
||||
printf("test_committing_timeout_default_is_60s: PASS\n");
|
||||
}
|
||||
|
||||
/*
|
||||
* Test: committing_timeout_fn promotes to PRIMARY even when only some standbys
|
||||
* have converged (degraded mode). The WARN log path fires when timeout expires
|
||||
* before majority convergence. After promotion, role must be CTRL_PRIMARY.
|
||||
*
|
||||
* Requirements: 6.4, 7.5 (ha-repl-hardening)
|
||||
*/
|
||||
#ifdef XCE_TESTING
|
||||
extern int committing_timeout_fn_for_test(ha_state_t *ha, cluster_t *c,
|
||||
int cancelled_val);
|
||||
#endif
|
||||
|
||||
static void test_committing_timeout_promotes_degraded_with_partial_convergence(void)
|
||||
{
|
||||
#ifdef XCE_TESTING
|
||||
cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
ha_config_t cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 2000,
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
||||
};
|
||||
ha_init(&c.ha, &cfg, 0);
|
||||
c.ha.n_peers = 2; /* 3-node cluster; 2 standbys */
|
||||
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
||||
memory_order_release);
|
||||
|
||||
/* Only 1 of 2 standbys has converged — majority not yet reached */
|
||||
c.ha.standby_converged[0] = true;
|
||||
c.ha.standby_converged[1] = false;
|
||||
|
||||
/*
|
||||
* Count converged/pending so the test mirrors what the WARN log reads.
|
||||
* n_converged=1, n_pending=1 (Req 7.5).
|
||||
*/
|
||||
int n_converged = 0;
|
||||
int n_pending = 0;
|
||||
for (int i = 0; i < c.ha.n_peers; i++) {
|
||||
if (c.ha.standby_converged[i])
|
||||
n_converged++;
|
||||
else
|
||||
n_pending++;
|
||||
}
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, n_converged,
|
||||
"degraded-mode: 1 standby converged before timeout");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, n_pending,
|
||||
"degraded-mode: 1 standby still pending before timeout");
|
||||
|
||||
/* Fire the timeout — must promote despite partial convergence */
|
||||
committing_timeout_fn_for_test(&c.ha, &c, 0 /* not cancelled */);
|
||||
|
||||
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
|
||||
"committing timeout: must promote to CTRL_PRIMARY in degraded mode (Req 6.4)");
|
||||
|
||||
ha_fini(&c.ha);
|
||||
printf("test_committing_timeout_promotes_degraded_with_partial_convergence: PASS "
|
||||
"(n_converged=%d, n_pending=%d)\n", n_converged, n_pending);
|
||||
#else
|
||||
printf("test_committing_timeout_promotes_degraded_with_partial_convergence: SKIP\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* Test: committing_timeout_fn promotes when NO standbys have converged
|
||||
* (worst-case degraded mode — all standbys unreachable).
|
||||
* Requirements: 6.4 (ha-repl-hardening)
|
||||
*/
|
||||
static void test_committing_timeout_promotes_degraded_zero_converged(void)
|
||||
{
|
||||
#ifdef XCE_TESTING
|
||||
cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
ha_config_t cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 2000,
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
||||
};
|
||||
ha_init(&c.ha, &cfg, 0);
|
||||
c.ha.n_peers = 2; /* 3-node cluster; all standbys unreachable */
|
||||
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
||||
memory_order_release);
|
||||
|
||||
/* No standbys converged — n_converged=0, n_pending=2 (Req 7.5) */
|
||||
c.ha.standby_converged[0] = false;
|
||||
c.ha.standby_converged[1] = false;
|
||||
|
||||
committing_timeout_fn_for_test(&c.ha, &c, 0 /* not cancelled */);
|
||||
|
||||
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
|
||||
"committing timeout: must promote even with 0 converged standbys (degraded, Req 6.4)");
|
||||
|
||||
ha_fini(&c.ha);
|
||||
printf("test_committing_timeout_promotes_degraded_zero_converged: PASS\n");
|
||||
#else
|
||||
printf("test_committing_timeout_promotes_degraded_zero_converged: SKIP\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
void test_ha_role(void)
|
||||
{
|
||||
RUN_TEST(test_ha_init_default_role_is_standby);
|
||||
|
|
@ -802,6 +931,10 @@ void test_ha_role(void)
|
|||
RUN_TEST(test_majority_convergence_3node_one_standby_stays_committing);
|
||||
RUN_TEST(test_majority_convergence_3node_all_standbys_promotes);
|
||||
RUN_TEST(test_majority_convergence_single_node_immediate);
|
||||
/* Task 5.3: committing timeout default value and degraded-mode WARN */
|
||||
RUN_TEST(test_committing_timeout_default_is_60s);
|
||||
RUN_TEST(test_committing_timeout_promotes_degraded_with_partial_convergence);
|
||||
RUN_TEST(test_committing_timeout_promotes_degraded_zero_converged);
|
||||
}
|
||||
|
||||
/* test_am_i_primary suite deleted — am_i_primary() removed (election-iomgr-refactor) */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue