feat(ha-repl-hardening): committing timeout 60s degraded-mode safety net (task 5.3)

- Change HA_DEFAULT_COMMITTING_TIMEOUT_MS from 10000 to 60000
- committing_timeout_fn: count converged/pending standbys from
  standby_converged[], log WARN with counts, then promote (degraded mode)
- Replace hardcoded 10000 in test_ha_evaluate_committing.c with macro
- Three new tests: default=60000, degrade-promote with 1/2 and 0/2 converged
This commit is contained in:
Rongsong Shen 2026-06-05 05:03:40 +00:00
parent 534bfd68fe
commit 0762577bf7
5 changed files with 160 additions and 9 deletions

View file

@ -185,7 +185,7 @@
- _Boundary: Controller FSM (ha.c)_
- _Depends: 3.1_
- [ ] 5.3 Extend committing timeout to degraded-mode safety net
- [x] 5.3 Extend committing timeout to degraded-mode safety net
- Change `committing_timeout_ms` default from `10000` to `60000`
- Timeout handler logs WARN with per-standby convergence state, promotes in degraded mode
- Majority-convergence path (task 5.2) is the PRIMARY promotion mechanism; timeout is the fallback

View file

@ -2853,10 +2853,16 @@ void raft_cb_become_follower_for_test(ha_state_t *ha, uint64_t term)
}
#endif /* XCE_TESTING */
/* ── Task 4.1: committing_timeout_fn — delaymgr callback ───────────────────
/* ── Task 4.1 / Task 5.3: committing_timeout_fn — delaymgr callback ─────────
*
* Fires ha_promote_to_primary (degraded mode) if majority convergence has not
* been reached within committing_timeout_ms. The majority-convergence path
* (ha_check_majority_convergence) is the PRIMARY promotion mechanism; this
* timeout is the safety-net fallback (Req 6.4, 8.4).
*
* WARN log includes the number of standbys that have converged and the number
* still pending, satisfying Req 7.5.
*
* Fires ha_promote_to_primary (degraded mode) if the majority ack threshold
* has not been reached within committing_timeout_ms. Req 8.4.
* Owns the committing_timeout_ctx_t — frees it on exit.
*/
static int committing_timeout_fn(void *arg)
@ -2869,8 +2875,20 @@ static int committing_timeout_fn(void *arg)
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&t->ha->role,
memory_order_acquire);
if (role == CTRL_PRIMARY_COMMITTING) {
WARN("HA: committing timeout — majority ack not received within timeout; "
"promoting to PRIMARY (degraded mode, Req 8.4)");
/* Count converged vs pending standbys for the WARN log (Req 7.5). */
int n_converged = 0;
int n_pending = 0;
int n_peers = t->ha->n_peers;
for (int i = 0; i < n_peers; i++) {
if (t->ha->standby_converged[i])
n_converged++;
else
n_pending++;
}
WARN("HA: committing timeout — majority convergence not reached within timeout; "
"%d/%d standbys converged, %d still pending; "
"promoting to PRIMARY in degraded mode (Req 6.4, 7.5)",
n_converged, n_peers, n_pending);
if (t->c)
ha_promote_to_primary(t->c);
}

View file

@ -121,7 +121,7 @@ typedef struct {
#define HA_DEFAULT_MAX_PROMOTION_RETRIES 5
#define HA_DEFAULT_ELECTION_JITTER_MAX_MS 200
#define HA_DEFAULT_ELECTION_PRIORITY_STEP_MS 500
#define HA_DEFAULT_COMMITTING_TIMEOUT_MS 10000
#define HA_DEFAULT_COMMITTING_TIMEOUT_MS 60000
typedef struct {
uint32_t heartbeat_interval_ms; /* Primary heartbeat send interval (min 2000 ms) */
@ -131,7 +131,7 @@ typedef struct {
uint32_t max_promotion_retries; /* Max consecutive failed promotion attempts */
uint32_t election_jitter_max_ms; /* random jitter added to election timeout (default 200) */
uint32_t election_priority_step_ms; /* extra ms per priority level (default 500) */
uint32_t committing_timeout_ms; /* Quorum-ACK deadline for CTRL_PRIMARY_COMMITTING (default 10000) */
uint32_t committing_timeout_ms; /* Quorum-ACK deadline for CTRL_PRIMARY_COMMITTING (default 60000) */
} ha_config_t;
/* probe_ctx_t deleted — only used by am_i_primary() (election-iomgr-refactor) */

View file

@ -60,7 +60,7 @@ static ha_config_t g_cfg = {
.max_promotion_retries = 5,
.election_jitter_max_ms = 0,
.election_priority_step_ms = 500,
.committing_timeout_ms = 10000,
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
};
/* ── Helper: build a minimal ha_elect_ctx_t for n_other peers ───────────── */

View file

@ -766,6 +766,135 @@ static void test_evaluate_repl_action_stream_wal(void)
printf("test_evaluate_repl_action_stream_wal: PASS\n");
}
/* ── Task 5.3: committing timeout default value and degraded-mode WARN ───────
*
* Requirements: 6.4, 7.5 (ha-repl-hardening)
*
* Verifies:
* - HA_DEFAULT_COMMITTING_TIMEOUT_MS == 60000 (safety-net window is 60 s)
* - committing_timeout_fn_for_test still promotes to CTRL_PRIMARY in degraded
* mode even when only a subset of standbys have converged
* - The ha->standby_converged[] state is readable so the WARN log can include
* the converged/pending counts
*/
/*
* Test: HA_DEFAULT_COMMITTING_TIMEOUT_MS is 60000 (not 10000).
* Requirements: 6.4 (ha-repl-hardening) — "Change committing_timeout_ms default from 10000 to 60000"
*/
static void test_committing_timeout_default_is_60s(void)
{
TEST_ASSERT_EQUAL_UINT32_MESSAGE(60000, HA_DEFAULT_COMMITTING_TIMEOUT_MS,
"HA_DEFAULT_COMMITTING_TIMEOUT_MS must be 60000 (Req 6.4, Task 5.3)");
printf("test_committing_timeout_default_is_60s: PASS\n");
}
/*
* Test: committing_timeout_fn promotes to PRIMARY even when only some standbys
* have converged (degraded mode). The WARN log path fires when timeout expires
* before majority convergence. After promotion, role must be CTRL_PRIMARY.
*
* Requirements: 6.4, 7.5 (ha-repl-hardening)
*/
#ifdef XCE_TESTING
extern int committing_timeout_fn_for_test(ha_state_t *ha, cluster_t *c,
int cancelled_val);
#endif
static void test_committing_timeout_promotes_degraded_with_partial_convergence(void)
{
#ifdef XCE_TESTING
cluster_t c;
memset(&c, 0, sizeof(c));
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
};
ha_init(&c.ha, &cfg, 0);
c.ha.n_peers = 2; /* 3-node cluster; 2 standbys */
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
memory_order_release);
/* Only 1 of 2 standbys has converged — majority not yet reached */
c.ha.standby_converged[0] = true;
c.ha.standby_converged[1] = false;
/*
* Count converged/pending so the test mirrors what the WARN log reads.
* n_converged=1, n_pending=1 (Req 7.5).
*/
int n_converged = 0;
int n_pending = 0;
for (int i = 0; i < c.ha.n_peers; i++) {
if (c.ha.standby_converged[i])
n_converged++;
else
n_pending++;
}
TEST_ASSERT_EQUAL_INT_MESSAGE(1, n_converged,
"degraded-mode: 1 standby converged before timeout");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, n_pending,
"degraded-mode: 1 standby still pending before timeout");
/* Fire the timeout — must promote despite partial convergence */
committing_timeout_fn_for_test(&c.ha, &c, 0 /* not cancelled */);
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
"committing timeout: must promote to CTRL_PRIMARY in degraded mode (Req 6.4)");
ha_fini(&c.ha);
printf("test_committing_timeout_promotes_degraded_with_partial_convergence: PASS "
"(n_converged=%d, n_pending=%d)\n", n_converged, n_pending);
#else
printf("test_committing_timeout_promotes_degraded_with_partial_convergence: SKIP\n");
#endif
}
/*
* Test: committing_timeout_fn promotes when NO standbys have converged
* (worst-case degraded mode — all standbys unreachable).
* Requirements: 6.4 (ha-repl-hardening)
*/
static void test_committing_timeout_promotes_degraded_zero_converged(void)
{
#ifdef XCE_TESTING
cluster_t c;
memset(&c, 0, sizeof(c));
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
};
ha_init(&c.ha, &cfg, 0);
c.ha.n_peers = 2; /* 3-node cluster; all standbys unreachable */
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
memory_order_release);
/* No standbys converged — n_converged=0, n_pending=2 (Req 7.5) */
c.ha.standby_converged[0] = false;
c.ha.standby_converged[1] = false;
committing_timeout_fn_for_test(&c.ha, &c, 0 /* not cancelled */);
int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role,
"committing timeout: must promote even with 0 converged standbys (degraded, Req 6.4)");
ha_fini(&c.ha);
printf("test_committing_timeout_promotes_degraded_zero_converged: PASS\n");
#else
printf("test_committing_timeout_promotes_degraded_zero_converged: SKIP\n");
#endif
}
void test_ha_role(void)
{
RUN_TEST(test_ha_init_default_role_is_standby);
@ -802,6 +931,10 @@ void test_ha_role(void)
RUN_TEST(test_majority_convergence_3node_one_standby_stays_committing);
RUN_TEST(test_majority_convergence_3node_all_standbys_promotes);
RUN_TEST(test_majority_convergence_single_node_immediate);
/* Task 5.3: committing timeout default value and degraded-mode WARN */
RUN_TEST(test_committing_timeout_default_is_60s);
RUN_TEST(test_committing_timeout_promotes_degraded_with_partial_convergence);
RUN_TEST(test_committing_timeout_promotes_degraded_zero_converged);
}
/* test_am_i_primary suite deleted — am_i_primary() removed (election-iomgr-refactor) */