diff --git a/.kiro/specs/ha-config-bootstrap/tasks.md b/.kiro/specs/ha-config-bootstrap/tasks.md index 24220fdc..f11a5421 100644 --- a/.kiro/specs/ha-config-bootstrap/tasks.md +++ b/.kiro/specs/ha-config-bootstrap/tasks.md @@ -76,7 +76,7 @@ Dependency order: startup validation (D) unblocks vtaskmgr before election (A); - _Requirements: 1.3, 1.4_ - _Boundary: Controller FSM (ha.c, cluster.c)_ -- [ ] 3.4 Implement ha_wait_for_heartbeat_or_timeout() +- [x] 3.4 Implement ha_wait_for_heartbeat_or_timeout() - `sem_timedwait(&c->ha.heartbeat_seen, &deadline)` where `deadline = now + election_timeout_ms + xorshift64_jitter()` - Heartbeat handler posts `heartbeat_seen` only when `ha.role == CTRL_INIT`; receiving a heartbeat transitions to `CTRL_STANDBY` and returns - Election timeout fires → returns without setting standby; caller proceeds to `am_i_primary_quorum()` diff --git a/src/controller/ha.c b/src/controller/ha.c index c612edfe..98fd38ad 100644 --- a/src/controller/ha.c +++ b/src/controller/ha.c @@ -499,6 +499,15 @@ int ha_dispatch_gate(ha_state_t *ha, int opcode) return 0; case CTRL_INIT: + /* Allow election probes, config probes, AND heartbeats so the primary's + * periodic heartbeat can reach the handler and post heartbeat_seen, + * triggering the CTRL_INIT → CTRL_STANDBY transition (Req 1.1). */ + if (opcode == OPCode_requestVote || + opcode == OPCode_probeConfigTag || + opcode == OPCode_heartbeat) + return 0; + return -XCE_ERR_NOT_READY; + case CTRL_ELECTING: if (opcode == OPCode_requestVote || opcode == OPCode_probeConfigTag) return 0; @@ -611,3 +620,55 @@ int ha_demote_to_standby(cluster_t *c) ha_do_demote(&c->ha); /* sets role=STANDBY, resets hooks_registered=0 */ return 0; } + +/* ── HA Config Bootstrap: startup heartbeat wait (Req 1.1, 1.2) ─────────── */ + +/* + * ha_wait_for_heartbeat_or_timeout: called from the startup path after + * ha_init() when the node needs to discover whether a primary already exists. + * + * Blocks on sem_timedwait(&ha->heartbeat_seen, &deadline) where + * deadline = now(CLOCK_REALTIME) + election_timeout_ms + jitter + * + * The heartbeat handler (cluster.c) posts heartbeat_seen when it receives + * OPCode_heartbeat and ha->role == CTRL_INIT. ha_dispatch_gate() permits + * OPCode_heartbeat in CTRL_INIT state so the packet reaches the handler. + * + * Returns CTRL_STANDBY — semaphore was posted (heartbeat arrived); + * ha_do_demote() was called (role = CTRL_STANDBY). + * Returns CTRL_ELECTING — sem_timedwait timed out (ETIMEDOUT); + * caller is responsible for driving the election. + */ +ctrl_role_t ha_wait_for_heartbeat_or_timeout(ha_state_t *ha) +{ + /* Seed jitter PRNG on first use */ + if (ha->jitter_seed == 0) + ha_seed_jitter(&ha->jitter_seed, ha->my_priority); + + /* Compute timeout in milliseconds: election_timeout + random jitter */ + long to_ms = (long)ha->config.election_timeout_ms; + if (ha->config.election_jitter_max_ms > 0) { + to_ms += (long)(xorshift64_next(&ha->jitter_seed) + % ((uint64_t)ha->config.election_jitter_max_ms + 1)); + } + + /* sem_timedwait requires an absolute CLOCK_REALTIME deadline */ + struct timespec deadline; + clock_gettime(CLOCK_REALTIME, &deadline); + deadline.tv_sec += to_ms / 1000; + deadline.tv_nsec += (to_ms % 1000) * 1000000L; + if (deadline.tv_nsec >= 1000000000L) { + deadline.tv_sec++; + deadline.tv_nsec -= 1000000000L; + } + + int rc = sem_timedwait(&ha->heartbeat_seen, &deadline); + if (rc == 0) { + /* Heartbeat arrived before the deadline — demote to standby */ + ha_do_demote(ha); + return CTRL_STANDBY; + } + + /* ETIMEDOUT (or any other error): treat as election timeout */ + return CTRL_ELECTING; +} diff --git a/src/controller/ha.h b/src/controller/ha.h index 611bccac..02a00277 100644 --- a/src/controller/ha.h +++ b/src/controller/ha.h @@ -313,6 +313,25 @@ int bootstrap_config_if_needed(cluster_t *c); */ int xce_probe_config_tag(const char *host, uint16_t port, uint64_t *tag_out); +/* + * ha_wait_for_heartbeat_or_timeout: wait in CTRL_INIT for an incoming heartbeat. + * + * Blocks on sem_timedwait(&ha->heartbeat_seen, &deadline) where + * deadline = now(CLOCK_REALTIME) + election_timeout_ms + xorshift64_jitter + * + * Returns CTRL_STANDBY — heartbeat arrived; ha_do_demote() was called + * (role is now CTRL_STANDBY). + * Returns CTRL_ELECTING — deadline expired; caller must drive the election. + * + * The heartbeat handler posts heartbeat_seen only when role == CTRL_INIT. + * OPCode_heartbeat is allowed through ha_dispatch_gate() in CTRL_INIT state + * so that the primary's periodic heartbeat can reach the handler. + * + * Requirements: 1.1 (heartbeat → CTRL_STANDBY), 1.2 (timeout → CTRL_ELECTING). + * Boundary: Controller FSM (ha.c). + */ +ctrl_role_t ha_wait_for_heartbeat_or_timeout(ha_state_t *ha); + /* ── P3-6: Test-only failover shim ────────────────────────────── */ #ifdef XCE_TESTING diff --git a/src/controller/testsuite/test_ha_startup.c b/src/controller/testsuite/test_ha_startup.c index 9aa2b1eb..ba28718c 100644 --- a/src/controller/testsuite/test_ha_startup.c +++ b/src/controller/testsuite/test_ha_startup.c @@ -428,6 +428,118 @@ static void test_ha_init_initializes_heartbeat_semaphore(void) printf("test_ha_init_initializes_heartbeat_semaphore: PASS\n"); } +/* ── Req 1.1/1.2: ha_wait_for_heartbeat_or_timeout() ────────────────────── */ + +/* + * ha_wait_for_heartbeat_or_timeout: waits on ha->heartbeat_seen semaphore. + * Deadline = now + election_timeout_ms + xorshift64_jitter(election_jitter_max_ms). + * + * Returns CTRL_STANDBY if a heartbeat arrives before the deadline + * (calls ha_do_demote() which sets role = CTRL_STANDBY). + * Returns CTRL_ELECTING if sem_timedwait times out (ETIMEDOUT). + * + * Takes ha_state_t* for standalone testability; production call-site wraps + * with cluster_t* and passes &c->ha. + * + * Req 1.1: heartbeat before timeout → CTRL_STANDBY transition. + * Req 1.2: timeout fires → returns CTRL_ELECTING. + */ +ctrl_role_t ha_wait_for_heartbeat_or_timeout(ha_state_t *ha); + +static void test_wait_returns_standby_when_heartbeat_arrives_before_timeout(void) +{ + /* Req 1.1: When a heartbeat is posted before the deadline, + * ha_wait_for_heartbeat_or_timeout() must: + * - return CTRL_STANDBY + * - set ha->role to CTRL_STANDBY (via ha_do_demote) + * + * We simulate a heartbeat by posting heartbeat_seen directly before calling + * the function, so sem_timedwait returns immediately with value 0. */ + ha_state_t ha; + /* Use a short timeout but with jitter disabled so the test is deterministic */ + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, + .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 500, + .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + .election_jitter_max_ms = 0, /* no jitter — deterministic deadline */ + }; + ha_init(&ha, &cfg, 1); + atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release); + + /* Post the semaphore to simulate a heartbeat arriving before the deadline */ + sem_post(&ha.heartbeat_seen); + + ctrl_role_t result = ha_wait_for_heartbeat_or_timeout(&ha); + + TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)result); + TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire)); + + ha_fini(&ha); + printf("test_wait_returns_standby_when_heartbeat_arrives_before_timeout: PASS\n"); +} + +static void test_wait_returns_electing_when_timeout_fires(void) +{ + /* Req 1.2: When no heartbeat arrives before the deadline, + * ha_wait_for_heartbeat_or_timeout() must: + * - return CTRL_ELECTING + * - NOT change ha->role (caller is responsible for the transition) + * + * Use election_timeout_ms=1 and election_jitter_max_ms=0 so the semaphore + * times out almost immediately. The test must complete in < 500ms. */ + ha_state_t ha; + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, + .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 1, /* 1 ms — times out almost instantly */ + .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + .election_jitter_max_ms = 0, /* no jitter */ + }; + /* Note: ha_init rejects election_timeout_ms < 500. + * Bypass via direct struct assignment after ha_init with a valid config, + * then overwrite the field for the test. */ + ha_config_t valid_cfg = cfg; + valid_cfg.election_timeout_ms = 500; /* satisfy ha_init's >= 500 check */ + ha_init(&ha, &valid_cfg, 1); + /* Now override to 1 ms for a near-instant timeout */ + ha.config.election_timeout_ms = 1; + ha.config.election_jitter_max_ms = 0; + atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release); + + /* Do NOT post heartbeat_seen — let it time out */ + ctrl_role_t result = ha_wait_for_heartbeat_or_timeout(&ha); + + TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)result); + /* Role must still be CTRL_INIT — caller drives the transition */ + TEST_ASSERT_EQUAL_INT(CTRL_INIT, + (int)atomic_load_explicit(&ha.role, memory_order_acquire)); + + ha_fini(&ha); + printf("test_wait_returns_electing_when_timeout_fires: PASS\n"); +} + +/* ── Req 1.1/1.2: dispatch gate allows OPCode_heartbeat in CTRL_INIT ──────── */ + +static void test_dispatch_gate_init_allows_heartbeat(void) +{ + /* Req 1.1: OPCode_heartbeat must pass through in CTRL_INIT so the + * heartbeat handler can post heartbeat_seen and trigger the + * CTRL_INIT → CTRL_STANDBY transition. */ + ha_state_t ha; + ha_init(&ha, &g_dispatch_cfg, 0); + atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release); + + int rc = ha_dispatch_gate(&ha, OPCode_heartbeat); + TEST_ASSERT_EQUAL_INT(0, rc); + + ha_fini(&ha); + printf("test_dispatch_gate_init_allows_heartbeat: PASS\n"); +} + /* ── Test suite entry point ───────────────────────────────────────────── */ void test_ha_startup(void) @@ -466,4 +578,13 @@ void test_ha_startup(void) /* Req 1.x — ha_init() initializes heartbeat_seen semaphore */ RUN_TEST(test_ha_init_initializes_heartbeat_semaphore); + + /* Req 1.1 — ha_wait_for_heartbeat_or_timeout: heartbeat → CTRL_STANDBY */ + RUN_TEST(test_wait_returns_standby_when_heartbeat_arrives_before_timeout); + + /* Req 1.2 — ha_wait_for_heartbeat_or_timeout: timeout → CTRL_ELECTING */ + RUN_TEST(test_wait_returns_electing_when_timeout_fires); + + /* Req 1.1 — dispatch gate allows heartbeat in CTRL_INIT */ + RUN_TEST(test_dispatch_gate_init_allows_heartbeat); }