From 0762577bf7f76eb30c0d493d42fc2f9f4e123228 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Fri, 5 Jun 2026 05:03:40 +0000 Subject: [PATCH] feat(ha-repl-hardening): committing timeout 60s degraded-mode safety net (task 5.3) - Change HA_DEFAULT_COMMITTING_TIMEOUT_MS from 10000 to 60000 - committing_timeout_fn: count converged/pending standbys from standby_converged[], log WARN with counts, then promote (degraded mode) - Replace hardcoded 10000 in test_ha_evaluate_committing.c with macro - Three new tests: default=60000, degrade-promote with 1/2 and 0/2 converged --- .kiro/specs/ha-repl-hardening/tasks.md | 2 +- src/controller/ha.c | 28 +++- src/controller/ha.h | 4 +- .../testsuite/test_ha_evaluate_committing.c | 2 +- src/controller/testsuite/test_ha_role.c | 133 ++++++++++++++++++ 5 files changed, 160 insertions(+), 9 deletions(-) diff --git a/.kiro/specs/ha-repl-hardening/tasks.md b/.kiro/specs/ha-repl-hardening/tasks.md index d25114be..6b516de8 100644 --- a/.kiro/specs/ha-repl-hardening/tasks.md +++ b/.kiro/specs/ha-repl-hardening/tasks.md @@ -185,7 +185,7 @@ - _Boundary: Controller FSM (ha.c)_ - _Depends: 3.1_ -- [ ] 5.3 Extend committing timeout to degraded-mode safety net +- [x] 5.3 Extend committing timeout to degraded-mode safety net - Change `committing_timeout_ms` default from `10000` to `60000` - Timeout handler logs WARN with per-standby convergence state, promotes in degraded mode - Majority-convergence path (task 5.2) is the PRIMARY promotion mechanism; timeout is the fallback diff --git a/src/controller/ha.c b/src/controller/ha.c index c3adbf3d..a6c36b06 100644 --- a/src/controller/ha.c +++ b/src/controller/ha.c @@ -2853,10 +2853,16 @@ void raft_cb_become_follower_for_test(ha_state_t *ha, uint64_t term) } #endif /* XCE_TESTING */ -/* ── Task 4.1: committing_timeout_fn — delaymgr callback ─────────────────── +/* ── Task 4.1 / Task 5.3: committing_timeout_fn — delaymgr callback ───────── + * + * Fires ha_promote_to_primary (degraded mode) if majority convergence has not + * been reached within committing_timeout_ms. The majority-convergence path + * (ha_check_majority_convergence) is the PRIMARY promotion mechanism; this + * timeout is the safety-net fallback (Req 6.4, 8.4). + * + * WARN log includes the number of standbys that have converged and the number + * still pending, satisfying Req 7.5. * - * Fires ha_promote_to_primary (degraded mode) if the majority ack threshold - * has not been reached within committing_timeout_ms. Req 8.4. * Owns the committing_timeout_ctx_t — frees it on exit. */ static int committing_timeout_fn(void *arg) @@ -2869,8 +2875,20 @@ static int committing_timeout_fn(void *arg) ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&t->ha->role, memory_order_acquire); if (role == CTRL_PRIMARY_COMMITTING) { - WARN("HA: committing timeout — majority ack not received within timeout; " - "promoting to PRIMARY (degraded mode, Req 8.4)"); + /* Count converged vs pending standbys for the WARN log (Req 7.5). */ + int n_converged = 0; + int n_pending = 0; + int n_peers = t->ha->n_peers; + for (int i = 0; i < n_peers; i++) { + if (t->ha->standby_converged[i]) + n_converged++; + else + n_pending++; + } + WARN("HA: committing timeout — majority convergence not reached within timeout; " + "%d/%d standbys converged, %d still pending; " + "promoting to PRIMARY in degraded mode (Req 6.4, 7.5)", + n_converged, n_peers, n_pending); if (t->c) ha_promote_to_primary(t->c); } diff --git a/src/controller/ha.h b/src/controller/ha.h index 9731c5b6..4e7dbce3 100644 --- a/src/controller/ha.h +++ b/src/controller/ha.h @@ -121,7 +121,7 @@ typedef struct { #define HA_DEFAULT_MAX_PROMOTION_RETRIES 5 #define HA_DEFAULT_ELECTION_JITTER_MAX_MS 200 #define HA_DEFAULT_ELECTION_PRIORITY_STEP_MS 500 -#define HA_DEFAULT_COMMITTING_TIMEOUT_MS 10000 +#define HA_DEFAULT_COMMITTING_TIMEOUT_MS 60000 typedef struct { uint32_t heartbeat_interval_ms; /* Primary heartbeat send interval (min 2000 ms) */ @@ -131,7 +131,7 @@ typedef struct { uint32_t max_promotion_retries; /* Max consecutive failed promotion attempts */ uint32_t election_jitter_max_ms; /* random jitter added to election timeout (default 200) */ uint32_t election_priority_step_ms; /* extra ms per priority level (default 500) */ - uint32_t committing_timeout_ms; /* Quorum-ACK deadline for CTRL_PRIMARY_COMMITTING (default 10000) */ + uint32_t committing_timeout_ms; /* Quorum-ACK deadline for CTRL_PRIMARY_COMMITTING (default 60000) */ } ha_config_t; /* probe_ctx_t deleted — only used by am_i_primary() (election-iomgr-refactor) */ diff --git a/src/controller/testsuite/test_ha_evaluate_committing.c b/src/controller/testsuite/test_ha_evaluate_committing.c index f8af96f2..b6cab067 100644 --- a/src/controller/testsuite/test_ha_evaluate_committing.c +++ b/src/controller/testsuite/test_ha_evaluate_committing.c @@ -60,7 +60,7 @@ static ha_config_t g_cfg = { .max_promotion_retries = 5, .election_jitter_max_ms = 0, .election_priority_step_ms = 500, - .committing_timeout_ms = 10000, + .committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS, }; /* ── Helper: build a minimal ha_elect_ctx_t for n_other peers ───────────── */ diff --git a/src/controller/testsuite/test_ha_role.c b/src/controller/testsuite/test_ha_role.c index a69223a6..6a8444c6 100644 --- a/src/controller/testsuite/test_ha_role.c +++ b/src/controller/testsuite/test_ha_role.c @@ -766,6 +766,135 @@ static void test_evaluate_repl_action_stream_wal(void) printf("test_evaluate_repl_action_stream_wal: PASS\n"); } +/* ── Task 5.3: committing timeout default value and degraded-mode WARN ─────── + * + * Requirements: 6.4, 7.5 (ha-repl-hardening) + * + * Verifies: + * - HA_DEFAULT_COMMITTING_TIMEOUT_MS == 60000 (safety-net window is 60 s) + * - committing_timeout_fn_for_test still promotes to CTRL_PRIMARY in degraded + * mode even when only a subset of standbys have converged + * - The ha->standby_converged[] state is readable so the WARN log can include + * the converged/pending counts + */ + +/* + * Test: HA_DEFAULT_COMMITTING_TIMEOUT_MS is 60000 (not 10000). + * Requirements: 6.4 (ha-repl-hardening) — "Change committing_timeout_ms default from 10000 to 60000" + */ +static void test_committing_timeout_default_is_60s(void) +{ + TEST_ASSERT_EQUAL_UINT32_MESSAGE(60000, HA_DEFAULT_COMMITTING_TIMEOUT_MS, + "HA_DEFAULT_COMMITTING_TIMEOUT_MS must be 60000 (Req 6.4, Task 5.3)"); + printf("test_committing_timeout_default_is_60s: PASS\n"); +} + +/* + * Test: committing_timeout_fn promotes to PRIMARY even when only some standbys + * have converged (degraded mode). The WARN log path fires when timeout expires + * before majority convergence. After promotion, role must be CTRL_PRIMARY. + * + * Requirements: 6.4, 7.5 (ha-repl-hardening) + */ +#ifdef XCE_TESTING +extern int committing_timeout_fn_for_test(ha_state_t *ha, cluster_t *c, + int cancelled_val); +#endif + +static void test_committing_timeout_promotes_degraded_with_partial_convergence(void) +{ +#ifdef XCE_TESTING + cluster_t c; + memset(&c, 0, sizeof(c)); + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, + .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, + .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + .committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS, + }; + ha_init(&c.ha, &cfg, 0); + c.ha.n_peers = 2; /* 3-node cluster; 2 standbys */ + atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING, + memory_order_release); + + /* Only 1 of 2 standbys has converged — majority not yet reached */ + c.ha.standby_converged[0] = true; + c.ha.standby_converged[1] = false; + + /* + * Count converged/pending so the test mirrors what the WARN log reads. + * n_converged=1, n_pending=1 (Req 7.5). + */ + int n_converged = 0; + int n_pending = 0; + for (int i = 0; i < c.ha.n_peers; i++) { + if (c.ha.standby_converged[i]) + n_converged++; + else + n_pending++; + } + TEST_ASSERT_EQUAL_INT_MESSAGE(1, n_converged, + "degraded-mode: 1 standby converged before timeout"); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, n_pending, + "degraded-mode: 1 standby still pending before timeout"); + + /* Fire the timeout — must promote despite partial convergence */ + committing_timeout_fn_for_test(&c.ha, &c, 0 /* not cancelled */); + + int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire); + TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role, + "committing timeout: must promote to CTRL_PRIMARY in degraded mode (Req 6.4)"); + + ha_fini(&c.ha); + printf("test_committing_timeout_promotes_degraded_with_partial_convergence: PASS " + "(n_converged=%d, n_pending=%d)\n", n_converged, n_pending); +#else + printf("test_committing_timeout_promotes_degraded_with_partial_convergence: SKIP\n"); +#endif +} + +/* + * Test: committing_timeout_fn promotes when NO standbys have converged + * (worst-case degraded mode — all standbys unreachable). + * Requirements: 6.4 (ha-repl-hardening) + */ +static void test_committing_timeout_promotes_degraded_zero_converged(void) +{ +#ifdef XCE_TESTING + cluster_t c; + memset(&c, 0, sizeof(c)); + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, + .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, + .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + .committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS, + }; + ha_init(&c.ha, &cfg, 0); + c.ha.n_peers = 2; /* 3-node cluster; all standbys unreachable */ + atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING, + memory_order_release); + + /* No standbys converged — n_converged=0, n_pending=2 (Req 7.5) */ + c.ha.standby_converged[0] = false; + c.ha.standby_converged[1] = false; + + committing_timeout_fn_for_test(&c.ha, &c, 0 /* not cancelled */); + + int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire); + TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role, + "committing timeout: must promote even with 0 converged standbys (degraded, Req 6.4)"); + + ha_fini(&c.ha); + printf("test_committing_timeout_promotes_degraded_zero_converged: PASS\n"); +#else + printf("test_committing_timeout_promotes_degraded_zero_converged: SKIP\n"); +#endif +} + void test_ha_role(void) { RUN_TEST(test_ha_init_default_role_is_standby); @@ -802,6 +931,10 @@ void test_ha_role(void) RUN_TEST(test_majority_convergence_3node_one_standby_stays_committing); RUN_TEST(test_majority_convergence_3node_all_standbys_promotes); RUN_TEST(test_majority_convergence_single_node_immediate); + /* Task 5.3: committing timeout default value and degraded-mode WARN */ + RUN_TEST(test_committing_timeout_default_is_60s); + RUN_TEST(test_committing_timeout_promotes_degraded_with_partial_convergence); + RUN_TEST(test_committing_timeout_promotes_degraded_zero_converged); } /* test_am_i_primary suite deleted — am_i_primary() removed (election-iomgr-refactor) */