feat(controller-status): stamp election time on both promotion roads (task 7.2)
Stamps ha_state_t.last_known_elected_at_ms (CLOCK_REALTIME, epoch-ms) in both ha_do_promote (non-raft road) and raft_cb_become_leader (raft road), placed at each road's genuine, final commit point to CTRL_PRIMARY -- past every early-return refusal path in raft_cb_become_leader (first-boot phase2 failure, the Live Apply Resource Fence re-check, the promotion eligibility gate, config-serve-fenced, upgrade-materialization-fenced). Purely observational: does not change whether/when a promotion succeeds or is refused. Three new tests (2 positive, 1 negative) prove placement: a refused promotion leaves the field at its pre-attempt value. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01RLSNJhNoULvVcUhXEFafQC
This commit is contained in:
parent
cae1039ed7
commit
21ae00b375
3 changed files with 210 additions and 1 deletions
|
|
@ -100,7 +100,7 @@
|
|||
- _Requirements: 12.1_
|
||||
- _Boundary: schema + typemap + ha_state_t_
|
||||
|
||||
- [ ] 7.2 Stamp the election time when a node becomes primary
|
||||
- [x] 7.2 Stamp the election time when a node becomes primary
|
||||
- TDD (red first): a promotion via the non-election-protocol path and a promotion via the leader-election path both record a non-zero election-time at the moment the node commits to the primary role; a promotion attempt that is refused (fenced, ineligible, or otherwise rejected) records NO election-time and leaves any prior value untouched.
|
||||
- Stamp the wall-clock election time exactly once, at the point each promotion path irrevocably commits to the primary role — after any eligibility/refusal checks that path performs, never before.
|
||||
- Observable: the new tests go red → green; a refused promotion attempt (either path) leaves the election-time state unchanged, proving the stamp is not placed before the refusal checks.
|
||||
|
|
|
|||
|
|
@ -1960,6 +1960,28 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
|
|||
* (raft_cb_become_leader_for_test) reaches it. */
|
||||
atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY,
|
||||
memory_order_release);
|
||||
|
||||
/* controller-status delta task 7.2 (Req 12.1, 12.5): stamp the wall-clock
|
||||
* election time HERE — this is the raft road's genuine, final commit
|
||||
* point to CTRL_PRIMARY, past every early-return refusal above (the
|
||||
* first-boot cluster_init_phase2 failure, the first-boot Live Apply
|
||||
* Resource Fence re-check, the ha_promotion_gate_decision eligibility
|
||||
* gate, config_serve_fenced, and upgrade_materialization_fenced). Do
|
||||
* NOT move this earlier — in particular NOT to the
|
||||
* last_known_committed_lsn self-observe point above (inside
|
||||
* `if (promo_ectx)`, ~line 1665): that point precedes the eligibility
|
||||
* gate and can still be followed by a refusal, which must never produce
|
||||
* a bogus election time. CLOCK_REALTIME (wall clock), same epoch-ms
|
||||
* conversion as ctrlstatus_fill's responder_time_ms (Req 11 precedent,
|
||||
* ctrlstatus.c). Purely observational: does not affect whether/when
|
||||
* this promotion is refused. */
|
||||
struct timespec elected_ts;
|
||||
clock_gettime(CLOCK_REALTIME, &elected_ts);
|
||||
atomic_store_explicit(&ha->last_known_elected_at_ms,
|
||||
(uint64_t)elected_ts.tv_sec * 1000u
|
||||
+ (uint64_t)elected_ts.tv_nsec / 1000000u,
|
||||
memory_order_release);
|
||||
|
||||
INFO("HA FSM (raft): → CTRL_PRIMARY (term=%" PRIu64
|
||||
", first_boot=%d)", term, is_first_boot);
|
||||
|
||||
|
|
@ -2931,6 +2953,25 @@ void ha_do_promote(ha_state_t *ha)
|
|||
atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY, memory_order_release);
|
||||
/* COH-5: new primary starts a fresh CRC chain; standbys reset their chain on catch-up */
|
||||
ha->last_received_crc = WAL_CRC_NONE;
|
||||
|
||||
/* controller-status delta task 7.2 (Req 12.1, 12.5): stamp the wall-clock
|
||||
* election time now that this (non-raft) road has irrevocably committed
|
||||
* to CTRL_PRIMARY. This function has no refusal path of its own — any
|
||||
* eligibility/refusal gating for this road lives in its caller,
|
||||
* ha_promote_to_primary, outside this task's boundary — so the stamp
|
||||
* placement here is trivially "after every refusal check this function
|
||||
* performs" (there are none). CLOCK_REALTIME (wall clock, NOT
|
||||
* CLOCK_MONOTONIC — comparable to controller log timestamps), same
|
||||
* epoch-ms conversion ctrlstatus_fill already uses for responder_time_ms
|
||||
* (Req 11 precedent, ctrlstatus.c). Purely observational: does not affect
|
||||
* whether/when promotion succeeds. */
|
||||
struct timespec elected_ts;
|
||||
clock_gettime(CLOCK_REALTIME, &elected_ts);
|
||||
atomic_store_explicit(&ha->last_known_elected_at_ms,
|
||||
(uint64_t)elected_ts.tv_sec * 1000u
|
||||
+ (uint64_t)elected_ts.tv_nsec / 1000000u,
|
||||
memory_order_release);
|
||||
|
||||
/* Req 10.5: log FSM transition to PRIMARY */
|
||||
INFO("HA FSM: → CTRL_PRIMARY (term=%" PRIu64 ")",
|
||||
atomic_load_explicit(&ha->term, memory_order_acquire));
|
||||
|
|
|
|||
|
|
@ -10,6 +10,21 @@
|
|||
#include "myconf.h"
|
||||
#include "xce/cluster_def.h"
|
||||
|
||||
/* ── Task 7.2 (controller-status delta, Req 12.1/12.5) test seams ─────────
|
||||
* raft_cb_become_leader_for_test: the XCE_TESTING shim (ha.c) that forces
|
||||
* ha->raft_c = NULL and drives the raft road with no real cluster_t behind
|
||||
* it -- mirrors test_ha_role_lifecycle.c's usage.
|
||||
* ha_phase2_hook: the XCE_TESTING test-only seam (ha.c) that stands in for
|
||||
* cluster_init_phase2 on first boot; used here to force the phase2-failure
|
||||
* refusal path (one of raft_cb_become_leader's documented early-return
|
||||
* refusal paths) without any real cluster_t/persist_t fixture. */
|
||||
#ifdef XCE_TESTING
|
||||
extern void raft_cb_become_leader_for_test(ha_state_t *ha, uint64_t term);
|
||||
extern int (*ha_phase2_hook)(void *c);
|
||||
|
||||
static int stub_phase2_fail(void *c) { (void)c; return -1; }
|
||||
#endif
|
||||
|
||||
static void test_ha_init_default_role_is_standby(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
|
|
@ -267,6 +282,155 @@ static void test_ha_init_zeroes_last_known_elected_at_ms(void)
|
|||
printf("test_ha_init_zeroes_last_known_elected_at_ms: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Task 7.2 (controller-status delta, Req 12.1, 12.5): stamp the election
|
||||
* time when a node becomes primary ────────────────────────────────────────
|
||||
*
|
||||
* Design.md "Design Delta — Requirement 12" Locked Decision 2: both
|
||||
* promotion roads write ha->last_known_elected_at_ms exactly once, at the
|
||||
* instant each road COMMITS to CTRL_PRIMARY -- past every refusal/
|
||||
* eligibility gate, never at function entry. These tests cover:
|
||||
* (1) ha_do_promote (non-raft road) -- stamps a non-zero, wall-clock-
|
||||
* plausible election time on every call (this function has no
|
||||
* refusal path of its own; any eligibility gating for this road lives
|
||||
* in its caller, ha_promote_to_primary, outside this task's boundary).
|
||||
* (2) raft_cb_become_leader (raft road) -- stamps on a SUCCESSFUL
|
||||
* promotion (test 2) but NOT on a REFUSED one (test 3, driven via the
|
||||
* ha_phase2_hook seam to force the documented first-boot
|
||||
* cluster_init_phase2-failure early return) -- proving the stamp sits
|
||||
* past that refusal, not before it (Req 12.5).
|
||||
*/
|
||||
|
||||
/* Return the current wall-clock time in epoch-ms, same conversion as
|
||||
* ctrlstatus_fill's responder_time_ms stamp (ctrlstatus.c) -- used here only
|
||||
* to bracket [before, after] around the call under test. */
|
||||
static uint64_t test_wallclock_now_ms(void)
|
||||
{
|
||||
struct timespec ts;
|
||||
clock_gettime(CLOCK_REALTIME, &ts);
|
||||
return (uint64_t)ts.tv_sec * 1000u + (uint64_t)ts.tv_nsec / 1000000u;
|
||||
}
|
||||
|
||||
static void test_ha_do_promote_stamps_elected_at_ms(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
ha_config_t cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 2000,
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
};
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
||||
TEST_ASSERT_EQUAL_UINT64(
|
||||
0, atomic_load_explicit(&ha.last_known_elected_at_ms,
|
||||
memory_order_acquire));
|
||||
|
||||
uint64_t before_ms = test_wallclock_now_ms();
|
||||
ha_do_promote(&ha);
|
||||
uint64_t after_ms = test_wallclock_now_ms();
|
||||
|
||||
uint64_t stamped = atomic_load_explicit(&ha.last_known_elected_at_ms,
|
||||
memory_order_acquire);
|
||||
TEST_ASSERT_TRUE_MESSAGE(stamped != 0,
|
||||
"ha_do_promote must stamp a non-zero election time (Req 12.1)");
|
||||
TEST_ASSERT_TRUE_MESSAGE(stamped >= before_ms && stamped <= after_ms,
|
||||
"stamped election time must reflect the current wall clock at the "
|
||||
"moment ha_do_promote commits to CTRL_PRIMARY");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_ha_do_promote_stamps_elected_at_ms: PASS\n");
|
||||
}
|
||||
|
||||
static void test_raft_cb_become_leader_stamps_elected_at_ms(void)
|
||||
{
|
||||
#ifdef XCE_TESTING
|
||||
ha_state_t ha;
|
||||
ha_config_t cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 2000,
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
};
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
||||
ha.phase2_done = 1; /* skip phase2 -- no real cluster behind the shim */
|
||||
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
|
||||
TEST_ASSERT_EQUAL_UINT64(
|
||||
0, atomic_load_explicit(&ha.last_known_elected_at_ms,
|
||||
memory_order_acquire));
|
||||
|
||||
uint64_t before_ms = test_wallclock_now_ms();
|
||||
raft_cb_become_leader_for_test(&ha, /*term=*/1);
|
||||
uint64_t after_ms = test_wallclock_now_ms();
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY,
|
||||
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
|
||||
"a successful raft-road promotion must land in CTRL_PRIMARY");
|
||||
uint64_t stamped = atomic_load_explicit(&ha.last_known_elected_at_ms,
|
||||
memory_order_acquire);
|
||||
TEST_ASSERT_TRUE_MESSAGE(stamped != 0,
|
||||
"raft_cb_become_leader must stamp a non-zero election time on a "
|
||||
"successful promotion (Req 12.1)");
|
||||
TEST_ASSERT_TRUE_MESSAGE(stamped >= before_ms && stamped <= after_ms,
|
||||
"stamped election time must reflect the current wall clock at the "
|
||||
"moment raft_cb_become_leader commits to CTRL_PRIMARY");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_raft_cb_become_leader_stamps_elected_at_ms: PASS\n");
|
||||
#else
|
||||
printf("test_raft_cb_become_leader_stamps_elected_at_ms: SKIP (non-test build)\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
static void test_raft_cb_become_leader_refused_promotion_leaves_elected_at_ms_unchanged(void)
|
||||
{
|
||||
#ifdef XCE_TESTING
|
||||
ha_state_t ha;
|
||||
ha_config_t cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 2000,
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
};
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
||||
/* phase2_done stays 0 (ha_init default) so is_first_boot is taken. */
|
||||
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
|
||||
|
||||
/* Seed a sentinel PRIOR value -- proves the refused attempt leaves any
|
||||
* prior value UNTOUCHED, not merely that it stays at the ha_init zero
|
||||
* (the task's acceptance criteria covers both: 0 on a first attempt, or
|
||||
* the prior value on a repeat attempt). */
|
||||
atomic_store_explicit(&ha.last_known_elected_at_ms,
|
||||
UINT64_C(999999), memory_order_release);
|
||||
|
||||
/* Force the documented first-boot cluster_init_phase2-failure refusal
|
||||
* path (one of raft_cb_become_leader's early-return refusal paths) via
|
||||
* the ha_phase2_hook test seam -- no real cluster_t/persist_t fixture
|
||||
* needed since raft_cb_become_leader_for_test forces ha->raft_c = NULL,
|
||||
* which this refusal path (unlike the later eligibility gate) does not
|
||||
* require. */
|
||||
ha_phase2_hook = stub_phase2_fail;
|
||||
raft_cb_become_leader_for_test(&ha, /*term=*/1);
|
||||
ha_phase2_hook = NULL; /* restore -- shared global test seam */
|
||||
|
||||
TEST_ASSERT_NOT_EQUAL_MESSAGE((int)CTRL_PRIMARY,
|
||||
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
|
||||
"a refused promotion (phase2 failure) must NOT become CTRL_PRIMARY");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(UINT64_C(999999),
|
||||
atomic_load_explicit(&ha.last_known_elected_at_ms,
|
||||
memory_order_acquire),
|
||||
"a refused raft-road promotion must leave last_known_elected_at_ms "
|
||||
"UNCHANGED (Req 12.5) -- proves the stamp sits past this refusal, "
|
||||
"not before it");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_raft_cb_become_leader_refused_promotion_leaves_elected_at_ms_unchanged: PASS\n");
|
||||
#else
|
||||
printf("test_raft_cb_become_leader_refused_promotion_leaves_elected_at_ms_unchanged: SKIP (non-test build)\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
/* ── Task 3.1: ha_process_standby_state convergence tracking ─────────────── */
|
||||
|
||||
/*
|
||||
|
|
@ -1030,6 +1194,10 @@ void test_ha_role(void)
|
|||
/* Task 1.4: ha-repl-hardening per-standby convergence fields */
|
||||
RUN_TEST(test_ha_init_zeroes_standby_convergence_fields);
|
||||
RUN_TEST(test_ha_init_zeroes_last_known_elected_at_ms);
|
||||
/* Task 7.2: stamp last_known_elected_at_ms on promotion (both roads) */
|
||||
RUN_TEST(test_ha_do_promote_stamps_elected_at_ms);
|
||||
RUN_TEST(test_raft_cb_become_leader_stamps_elected_at_ms);
|
||||
RUN_TEST(test_raft_cb_become_leader_refused_promotion_leaves_elected_at_ms_unchanged);
|
||||
/* Task 3.1: ha_process_standby_state convergence tracking */
|
||||
RUN_TEST(test_process_standby_state_converged);
|
||||
RUN_TEST(test_process_standby_state_lsn_behind);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue