From 21ae00b37574595a1cf87db4c1a328c75d6787e1 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Wed, 16 Sep 2026 04:54:25 +0000 Subject: [PATCH] feat(controller-status): stamp election time on both promotion roads (task 7.2) Stamps ha_state_t.last_known_elected_at_ms (CLOCK_REALTIME, epoch-ms) in both ha_do_promote (non-raft road) and raft_cb_become_leader (raft road), placed at each road's genuine, final commit point to CTRL_PRIMARY -- past every early-return refusal path in raft_cb_become_leader (first-boot phase2 failure, the Live Apply Resource Fence re-check, the promotion eligibility gate, config-serve-fenced, upgrade-materialization-fenced). Purely observational: does not change whether/when a promotion succeeds or is refused. Three new tests (2 positive, 1 negative) prove placement: a refused promotion leaves the field at its pre-attempt value. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01RLSNJhNoULvVcUhXEFafQC --- .kiro/specs/controller-status/tasks.md | 2 +- src/controller/ha.c | 41 ++++++ src/controller/testsuite/test_ha_role.c | 168 ++++++++++++++++++++++++ 3 files changed, 210 insertions(+), 1 deletion(-) diff --git a/.kiro/specs/controller-status/tasks.md b/.kiro/specs/controller-status/tasks.md index ac702a5d..1df0bc14 100644 --- a/.kiro/specs/controller-status/tasks.md +++ b/.kiro/specs/controller-status/tasks.md @@ -100,7 +100,7 @@ - _Requirements: 12.1_ - _Boundary: schema + typemap + ha_state_t_ -- [ ] 7.2 Stamp the election time when a node becomes primary +- [x] 7.2 Stamp the election time when a node becomes primary - TDD (red first): a promotion via the non-election-protocol path and a promotion via the leader-election path both record a non-zero election-time at the moment the node commits to the primary role; a promotion attempt that is refused (fenced, ineligible, or otherwise rejected) records NO election-time and leaves any prior value untouched. - Stamp the wall-clock election time exactly once, at the point each promotion path irrevocably commits to the primary role — after any eligibility/refusal checks that path performs, never before. - Observable: the new tests go red → green; a refused promotion attempt (either path) leaves the election-time state unchanged, proving the stamp is not placed before the refusal checks. diff --git a/src/controller/ha.c b/src/controller/ha.c index 606ee210..62f94690 100644 --- a/src/controller/ha.c +++ b/src/controller/ha.c @@ -1960,6 +1960,28 @@ static void raft_cb_become_leader(void *ctx, uint64_t term) * (raft_cb_become_leader_for_test) reaches it. */ atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY, memory_order_release); + + /* controller-status delta task 7.2 (Req 12.1, 12.5): stamp the wall-clock + * election time HERE — this is the raft road's genuine, final commit + * point to CTRL_PRIMARY, past every early-return refusal above (the + * first-boot cluster_init_phase2 failure, the first-boot Live Apply + * Resource Fence re-check, the ha_promotion_gate_decision eligibility + * gate, config_serve_fenced, and upgrade_materialization_fenced). Do + * NOT move this earlier — in particular NOT to the + * last_known_committed_lsn self-observe point above (inside + * `if (promo_ectx)`, ~line 1665): that point precedes the eligibility + * gate and can still be followed by a refusal, which must never produce + * a bogus election time. CLOCK_REALTIME (wall clock), same epoch-ms + * conversion as ctrlstatus_fill's responder_time_ms (Req 11 precedent, + * ctrlstatus.c). Purely observational: does not affect whether/when + * this promotion is refused. */ + struct timespec elected_ts; + clock_gettime(CLOCK_REALTIME, &elected_ts); + atomic_store_explicit(&ha->last_known_elected_at_ms, + (uint64_t)elected_ts.tv_sec * 1000u + + (uint64_t)elected_ts.tv_nsec / 1000000u, + memory_order_release); + INFO("HA FSM (raft): → CTRL_PRIMARY (term=%" PRIu64 ", first_boot=%d)", term, is_first_boot); @@ -2931,6 +2953,25 @@ void ha_do_promote(ha_state_t *ha) atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY, memory_order_release); /* COH-5: new primary starts a fresh CRC chain; standbys reset their chain on catch-up */ ha->last_received_crc = WAL_CRC_NONE; + + /* controller-status delta task 7.2 (Req 12.1, 12.5): stamp the wall-clock + * election time now that this (non-raft) road has irrevocably committed + * to CTRL_PRIMARY. This function has no refusal path of its own — any + * eligibility/refusal gating for this road lives in its caller, + * ha_promote_to_primary, outside this task's boundary — so the stamp + * placement here is trivially "after every refusal check this function + * performs" (there are none). CLOCK_REALTIME (wall clock, NOT + * CLOCK_MONOTONIC — comparable to controller log timestamps), same + * epoch-ms conversion ctrlstatus_fill already uses for responder_time_ms + * (Req 11 precedent, ctrlstatus.c). Purely observational: does not affect + * whether/when promotion succeeds. */ + struct timespec elected_ts; + clock_gettime(CLOCK_REALTIME, &elected_ts); + atomic_store_explicit(&ha->last_known_elected_at_ms, + (uint64_t)elected_ts.tv_sec * 1000u + + (uint64_t)elected_ts.tv_nsec / 1000000u, + memory_order_release); + /* Req 10.5: log FSM transition to PRIMARY */ INFO("HA FSM: → CTRL_PRIMARY (term=%" PRIu64 ")", atomic_load_explicit(&ha->term, memory_order_acquire)); diff --git a/src/controller/testsuite/test_ha_role.c b/src/controller/testsuite/test_ha_role.c index ae77700f..fbc0e0a4 100644 --- a/src/controller/testsuite/test_ha_role.c +++ b/src/controller/testsuite/test_ha_role.c @@ -10,6 +10,21 @@ #include "myconf.h" #include "xce/cluster_def.h" +/* ── Task 7.2 (controller-status delta, Req 12.1/12.5) test seams ───────── + * raft_cb_become_leader_for_test: the XCE_TESTING shim (ha.c) that forces + * ha->raft_c = NULL and drives the raft road with no real cluster_t behind + * it -- mirrors test_ha_role_lifecycle.c's usage. + * ha_phase2_hook: the XCE_TESTING test-only seam (ha.c) that stands in for + * cluster_init_phase2 on first boot; used here to force the phase2-failure + * refusal path (one of raft_cb_become_leader's documented early-return + * refusal paths) without any real cluster_t/persist_t fixture. */ +#ifdef XCE_TESTING +extern void raft_cb_become_leader_for_test(ha_state_t *ha, uint64_t term); +extern int (*ha_phase2_hook)(void *c); + +static int stub_phase2_fail(void *c) { (void)c; return -1; } +#endif + static void test_ha_init_default_role_is_standby(void) { ha_state_t ha; @@ -267,6 +282,155 @@ static void test_ha_init_zeroes_last_known_elected_at_ms(void) printf("test_ha_init_zeroes_last_known_elected_at_ms: PASS\n"); } +/* ── Task 7.2 (controller-status delta, Req 12.1, 12.5): stamp the election + * time when a node becomes primary ──────────────────────────────────────── + * + * Design.md "Design Delta — Requirement 12" Locked Decision 2: both + * promotion roads write ha->last_known_elected_at_ms exactly once, at the + * instant each road COMMITS to CTRL_PRIMARY -- past every refusal/ + * eligibility gate, never at function entry. These tests cover: + * (1) ha_do_promote (non-raft road) -- stamps a non-zero, wall-clock- + * plausible election time on every call (this function has no + * refusal path of its own; any eligibility gating for this road lives + * in its caller, ha_promote_to_primary, outside this task's boundary). + * (2) raft_cb_become_leader (raft road) -- stamps on a SUCCESSFUL + * promotion (test 2) but NOT on a REFUSED one (test 3, driven via the + * ha_phase2_hook seam to force the documented first-boot + * cluster_init_phase2-failure early return) -- proving the stamp sits + * past that refusal, not before it (Req 12.5). + */ + +/* Return the current wall-clock time in epoch-ms, same conversion as + * ctrlstatus_fill's responder_time_ms stamp (ctrlstatus.c) -- used here only + * to bracket [before, after] around the call under test. */ +static uint64_t test_wallclock_now_ms(void) +{ + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + return (uint64_t)ts.tv_sec * 1000u + (uint64_t)ts.tv_nsec / 1000000u; +} + +static void test_ha_do_promote_stamps_elected_at_ms(void) +{ + ha_state_t ha; + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, + .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, + .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + }; + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); + TEST_ASSERT_EQUAL_UINT64( + 0, atomic_load_explicit(&ha.last_known_elected_at_ms, + memory_order_acquire)); + + uint64_t before_ms = test_wallclock_now_ms(); + ha_do_promote(&ha); + uint64_t after_ms = test_wallclock_now_ms(); + + uint64_t stamped = atomic_load_explicit(&ha.last_known_elected_at_ms, + memory_order_acquire); + TEST_ASSERT_TRUE_MESSAGE(stamped != 0, + "ha_do_promote must stamp a non-zero election time (Req 12.1)"); + TEST_ASSERT_TRUE_MESSAGE(stamped >= before_ms && stamped <= after_ms, + "stamped election time must reflect the current wall clock at the " + "moment ha_do_promote commits to CTRL_PRIMARY"); + + ha_fini(&ha); + printf("test_ha_do_promote_stamps_elected_at_ms: PASS\n"); +} + +static void test_raft_cb_become_leader_stamps_elected_at_ms(void) +{ +#ifdef XCE_TESTING + ha_state_t ha; + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, + .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, + .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + }; + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); + ha.phase2_done = 1; /* skip phase2 -- no real cluster behind the shim */ + atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release); + TEST_ASSERT_EQUAL_UINT64( + 0, atomic_load_explicit(&ha.last_known_elected_at_ms, + memory_order_acquire)); + + uint64_t before_ms = test_wallclock_now_ms(); + raft_cb_become_leader_for_test(&ha, /*term=*/1); + uint64_t after_ms = test_wallclock_now_ms(); + + TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire), + "a successful raft-road promotion must land in CTRL_PRIMARY"); + uint64_t stamped = atomic_load_explicit(&ha.last_known_elected_at_ms, + memory_order_acquire); + TEST_ASSERT_TRUE_MESSAGE(stamped != 0, + "raft_cb_become_leader must stamp a non-zero election time on a " + "successful promotion (Req 12.1)"); + TEST_ASSERT_TRUE_MESSAGE(stamped >= before_ms && stamped <= after_ms, + "stamped election time must reflect the current wall clock at the " + "moment raft_cb_become_leader commits to CTRL_PRIMARY"); + + ha_fini(&ha); + printf("test_raft_cb_become_leader_stamps_elected_at_ms: PASS\n"); +#else + printf("test_raft_cb_become_leader_stamps_elected_at_ms: SKIP (non-test build)\n"); +#endif +} + +static void test_raft_cb_become_leader_refused_promotion_leaves_elected_at_ms_unchanged(void) +{ +#ifdef XCE_TESTING + ha_state_t ha; + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, + .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, + .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + }; + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); + /* phase2_done stays 0 (ha_init default) so is_first_boot is taken. */ + atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release); + + /* Seed a sentinel PRIOR value -- proves the refused attempt leaves any + * prior value UNTOUCHED, not merely that it stays at the ha_init zero + * (the task's acceptance criteria covers both: 0 on a first attempt, or + * the prior value on a repeat attempt). */ + atomic_store_explicit(&ha.last_known_elected_at_ms, + UINT64_C(999999), memory_order_release); + + /* Force the documented first-boot cluster_init_phase2-failure refusal + * path (one of raft_cb_become_leader's early-return refusal paths) via + * the ha_phase2_hook test seam -- no real cluster_t/persist_t fixture + * needed since raft_cb_become_leader_for_test forces ha->raft_c = NULL, + * which this refusal path (unlike the later eligibility gate) does not + * require. */ + ha_phase2_hook = stub_phase2_fail; + raft_cb_become_leader_for_test(&ha, /*term=*/1); + ha_phase2_hook = NULL; /* restore -- shared global test seam */ + + TEST_ASSERT_NOT_EQUAL_MESSAGE((int)CTRL_PRIMARY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire), + "a refused promotion (phase2 failure) must NOT become CTRL_PRIMARY"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(UINT64_C(999999), + atomic_load_explicit(&ha.last_known_elected_at_ms, + memory_order_acquire), + "a refused raft-road promotion must leave last_known_elected_at_ms " + "UNCHANGED (Req 12.5) -- proves the stamp sits past this refusal, " + "not before it"); + + ha_fini(&ha); + printf("test_raft_cb_become_leader_refused_promotion_leaves_elected_at_ms_unchanged: PASS\n"); +#else + printf("test_raft_cb_become_leader_refused_promotion_leaves_elected_at_ms_unchanged: SKIP (non-test build)\n"); +#endif +} + /* ── Task 3.1: ha_process_standby_state convergence tracking ─────────────── */ /* @@ -1030,6 +1194,10 @@ void test_ha_role(void) /* Task 1.4: ha-repl-hardening per-standby convergence fields */ RUN_TEST(test_ha_init_zeroes_standby_convergence_fields); RUN_TEST(test_ha_init_zeroes_last_known_elected_at_ms); + /* Task 7.2: stamp last_known_elected_at_ms on promotion (both roads) */ + RUN_TEST(test_ha_do_promote_stamps_elected_at_ms); + RUN_TEST(test_raft_cb_become_leader_stamps_elected_at_ms); + RUN_TEST(test_raft_cb_become_leader_refused_promotion_leaves_elected_at_ms_unchanged); /* Task 3.1: ha_process_standby_state convergence tracking */ RUN_TEST(test_process_standby_state_converged); RUN_TEST(test_process_standby_state_lsn_behind);