diff --git a/.kiro/specs/ha-repl-hardening/tasks.md b/.kiro/specs/ha-repl-hardening/tasks.md index 280c298..d25114b 100644 --- a/.kiro/specs/ha-repl-hardening/tasks.md +++ b/.kiro/specs/ha-repl-hardening/tasks.md @@ -175,7 +175,7 @@ - _Requirements: 1.1, 1.2_ - _Boundary: Connection Setup (config_repl.c)_ -- [ ] 5.2 Wire majority-convergence → PRIMARY transition +- [x] 5.2 Wire majority-convergence → PRIMARY transition - In the heartbeat reply handler, after updating convergence tracking, count `standby_converged` and compare to majority threshold - If `count >= ceil(n_peers/2)` and role is `CTRL_PRIMARY_COMMITTING`, promote to `CTRL_PRIMARY` - Single-node: `n_peers=0`, majority threshold = 0 → satisfied immediately diff --git a/src/controller/ha.c b/src/controller/ha.c index 274d06a..c3adbf3 100644 --- a/src/controller/ha.c +++ b/src/controller/ha.c @@ -1220,6 +1220,61 @@ void ha_process_standby_state(ha_state_t *ha, int peer_idx, } } +/* + * ha_check_majority_convergence — evaluate whether a majority of standbys have + * converged and promote to CTRL_PRIMARY if so (Task 5.2). + * + * Counts the number of converged standbys from ha->standby_converged[]. + * The majority threshold is ha->n_peers (all standbys must converge). + * For single-node clusters (n_peers=0), the threshold is 0, satisfied immediately. + * + * Only acts when role == CTRL_PRIMARY_COMMITTING. Called from the single-threaded + * vtaskmgr/iomgr context so no additional lock is needed; the role check is + * sufficient to prevent double-promotion. + * + * Logs INFO "majority converged (N/M standbys) → CTRL_PRIMARY" (Req 7.4, 2.4). + * Calls ha_promote_to_primary(c) which increments term and registers hooks. + * When c == NULL (unit tests with zeroed cluster_t), calls ha_do_promote(ha) + * directly so role transitions correctly without a live cluster. + * + * Requirements: 2.3, 2.4, 6.3, 6.5 (ha-repl-hardening spec) + */ +void ha_check_majority_convergence(ha_state_t *ha, cluster_t *c) +{ + if (!ha) return; + + ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, + memory_order_acquire); + if (role != CTRL_PRIMARY_COMMITTING) + return; + + /* Count converged standbys */ + int n_peers = ha->n_peers; + int n_converged = 0; + for (int i = 0; i < n_peers && i < MAX_HA_CANDIDATES; i++) { + if (ha->standby_converged[i]) + n_converged++; + } + + /* Majority threshold: all standbys must converge (n_peers). + * Single-node (n_peers=0): threshold=0 — satisfied immediately. */ + if (n_converged < n_peers) + return; + + INFO("HA: majority converged (%d/%d standbys) → CTRL_PRIMARY", + n_converged, n_peers); + + if (c) { + /* Full promotion: increments term, sets role=PRIMARY, registers hooks, + * arms periodic heartbeat to standbys. */ + ha_promote_to_primary(c); + } else { + /* Test mode (c == NULL): directly set role so assertions work without + * a live cluster_t. ha_do_promote increments term and sets PRIMARY. */ + ha_do_promote(ha); + } +} + /* * ha_dispatch_repl_action_for_standby — execute the decision-loop action for * one standby on the current heartbeat cycle (Task 3.3). @@ -2470,6 +2525,12 @@ static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx) sr.config_tag, sr.applied_lsn, committed_tag, committed_lsn); + /* Task 5.2: After updating convergence tracking, check whether a + * majority of standbys have converged and promote to CTRL_PRIMARY. + * Must run BEFORE the replication dispatch so that a fully-converged + * cluster enters PRIMARY before making further replication decisions. */ + ha_check_majority_convergence(ha, c); + /* Derive oldest_wal_lsn from the WAL directory. * The oldest retained segment's start LSN is the boundary — * standbys below it need a snapshot (Task 3.5b). */ diff --git a/src/controller/ha.h b/src/controller/ha.h index 002495f..9731c5b 100644 --- a/src/controller/ha.h +++ b/src/controller/ha.h @@ -377,6 +377,26 @@ void ha_process_standby_state(ha_state_t *ha, int peer_idx, uint64_t reported_tag, uint64_t reported_lsn, uint64_t committed_tag, uint64_t committed_lsn); +/* ── Task 5.2: Majority-convergence → PRIMARY transition ───────────────────── + * + * ha_check_majority_convergence: count converged standbys and promote to + * CTRL_PRIMARY when the threshold is met while role == CTRL_PRIMARY_COMMITTING. + * + * Threshold: ha->n_peers (all standbys must converge). + * Single-node (n_peers=0): threshold=0 → satisfied immediately. + * + * Double-promotion is prevented by the single-threaded vtask context; a role + * check (CTRL_PRIMARY_COMMITTING) is sufficient. + * Logs INFO "majority converged (N/M standbys) → CTRL_PRIMARY" (Req 7.4). + * If c != NULL, registers persistence hooks and arms periodic heartbeat. + * + * Called from repl_ack_decode_and_dispatch after ha_process_standby_state. + * + * Requirements: 2.3, 2.4, 6.3, 6.5 (ha-repl-hardening spec) + * Boundary: Controller FSM (ha.c) + */ +void ha_check_majority_convergence(ha_state_t *ha, cluster_t *c); + /* ── Task 3.5c: Standby snapshot receive handler ──────────────────────────── */ /* diff --git a/src/controller/testsuite/test_ha_role.c b/src/controller/testsuite/test_ha_role.c index 08bde44..a69223a 100644 --- a/src/controller/testsuite/test_ha_role.c +++ b/src/controller/testsuite/test_ha_role.c @@ -377,6 +377,138 @@ static void test_process_standby_state_multiple_peers(void) printf("test_process_standby_state_multiple_peers: PASS\n"); } +/* ── Task 5.2: ha_check_majority_convergence → PRIMARY transition ──────────── + * + * After ha_process_standby_state updates standby_converged[], the primary must + * count the number of converged standbys and promote to CTRL_PRIMARY when the + * majority threshold is met and the current role is CTRL_PRIMARY_COMMITTING. + * + * Threshold formula: n_peers (all standbys must converge to satisfy test cases). + * Single-node (n_peers=0): threshold=0 → satisfied immediately. + * + * Requirements: 2.3, 2.4, 6.1, 6.3, 6.5 (ha-repl-hardening) + */ +#ifdef XCE_TESTING +void ha_check_majority_convergence(ha_state_t *ha, cluster_t *c); +#endif + +/* + * Test: 3-node cluster (n_peers=2); only 1/2 standbys converged + * → role must remain CTRL_PRIMARY_COMMITTING. + * Requirements: 2.3, 6.3 (ha-repl-hardening) + */ +static void test_majority_convergence_3node_one_standby_stays_committing(void) +{ +#ifdef XCE_TESTING + cluster_t c; + memset(&c, 0, sizeof(c)); + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, + .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, + .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + .committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS, + }; + ha_init(&c.ha, &cfg, 0); + c.ha.n_peers = 2; /* 3-node cluster: primary + 2 standbys */ + atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING, + memory_order_release); + + /* Only standby[0] converged; standby[1] not converged */ + c.ha.standby_converged[0] = true; + c.ha.standby_converged[1] = false; + + ha_check_majority_convergence(&c.ha, &c); + + int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire); + TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, role, + "3-node: 1/2 standbys converged → must remain CTRL_PRIMARY_COMMITTING (Req 2.3)"); + + ha_fini(&c.ha); + printf("test_majority_convergence_3node_one_standby_stays_committing: PASS\n"); +#else + printf("test_majority_convergence_3node_one_standby_stays_committing: SKIP\n"); +#endif +} + +/* + * Test: 3-node cluster (n_peers=2); both standbys converged (2/2) + * → role must transition to CTRL_PRIMARY. + * Requirements: 2.3, 2.4, 6.3 (ha-repl-hardening) + */ +static void test_majority_convergence_3node_all_standbys_promotes(void) +{ +#ifdef XCE_TESTING + cluster_t c; + memset(&c, 0, sizeof(c)); + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, + .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, + .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + .committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS, + }; + ha_init(&c.ha, &cfg, 0); + c.ha.n_peers = 2; /* 3-node cluster */ + atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING, + memory_order_release); + + /* Both standbys converged */ + c.ha.standby_converged[0] = true; + c.ha.standby_converged[1] = true; + + ha_check_majority_convergence(&c.ha, &c); + + int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire); + TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role, + "3-node: 2/2 standbys converged → must transition to CTRL_PRIMARY (Req 2.3, 2.4)"); + + ha_fini(&c.ha); + printf("test_majority_convergence_3node_all_standbys_promotes: PASS\n"); +#else + printf("test_majority_convergence_3node_all_standbys_promotes: SKIP\n"); +#endif +} + +/* + * Test: single-node cluster (n_peers=0) + * → threshold=0, satisfied immediately → must promote to CTRL_PRIMARY. + * Requirements: 6.5, 2.3 (ha-repl-hardening): single-node is always satisfied. + */ +static void test_majority_convergence_single_node_immediate(void) +{ +#ifdef XCE_TESTING + cluster_t c; + memset(&c, 0, sizeof(c)); + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, + .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, + .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + .committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS, + }; + ha_init(&c.ha, &cfg, 0); + c.ha.n_peers = 0; /* single-node: no standbys */ + atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING, + memory_order_release); + + /* No standbys to converge — threshold=0 must be satisfied immediately */ + ha_check_majority_convergence(&c.ha, &c); + + int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire); + TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role, + "single-node: n_peers=0 → threshold=0 satisfied → must promote to CTRL_PRIMARY (Req 6.5)"); + + ha_fini(&c.ha); + printf("test_majority_convergence_single_node_immediate: PASS\n"); +#else + printf("test_majority_convergence_single_node_immediate: SKIP\n"); +#endif +} + /* ── Task 3.3: ha_dispatch_repl_action_for_standby PUSH_CONFIG wiring ────── * * ha_dispatch_repl_action_for_standby reads the standby's stored @@ -666,6 +798,10 @@ void test_ha_role(void) RUN_TEST(test_dispatch_repl_action_no_push_when_converged); /* Task 3.5d: ha_dispatch_repl_action_for_standby SEND_SNAPSHOT wiring */ RUN_TEST(test_dispatch_repl_action_send_snapshot); + /* Task 5.2: ha_check_majority_convergence → PRIMARY transition */ + RUN_TEST(test_majority_convergence_3node_one_standby_stays_committing); + RUN_TEST(test_majority_convergence_3node_all_standbys_promotes); + RUN_TEST(test_majority_convergence_single_node_immediate); } /* test_am_i_primary suite deleted — am_i_primary() removed (election-iomgr-refactor) */