feat(ha-config-bootstrap): implement ha_wait_for_heartbeat_or_timeout (task 3.4)
- sem_timedwait on heartbeat_seen semaphore with CLOCK_REALTIME deadline - deadline = now + election_timeout_ms + xorshift64 jitter - Returns CTRL_STANDBY on heartbeat arrival (calls ha_do_demote) - Returns CTRL_ELECTING on timeout (caller drives election) - Allows OPCode_heartbeat through dispatch gate in CTRL_INIT state - 3 new unit tests: standby-on-heartbeat, electing-on-timeout, gate-allows-heartbeat - 24 Tests 0 Failures in test_ha_startup suite
This commit is contained in:
parent
72776f18ff
commit
f0029bf000
4 changed files with 202 additions and 1 deletions
|
|
@ -76,7 +76,7 @@ Dependency order: startup validation (D) unblocks vtaskmgr before election (A);
|
|||
- _Requirements: 1.3, 1.4_
|
||||
- _Boundary: Controller FSM (ha.c, cluster.c)_
|
||||
|
||||
- [ ] 3.4 Implement ha_wait_for_heartbeat_or_timeout()
|
||||
- [x] 3.4 Implement ha_wait_for_heartbeat_or_timeout()
|
||||
- `sem_timedwait(&c->ha.heartbeat_seen, &deadline)` where `deadline = now + election_timeout_ms + xorshift64_jitter()`
|
||||
- Heartbeat handler posts `heartbeat_seen` only when `ha.role == CTRL_INIT`; receiving a heartbeat transitions to `CTRL_STANDBY` and returns
|
||||
- Election timeout fires → returns without setting standby; caller proceeds to `am_i_primary_quorum()`
|
||||
|
|
|
|||
|
|
@ -499,6 +499,15 @@ int ha_dispatch_gate(ha_state_t *ha, int opcode)
|
|||
return 0;
|
||||
|
||||
case CTRL_INIT:
|
||||
/* Allow election probes, config probes, AND heartbeats so the primary's
|
||||
* periodic heartbeat can reach the handler and post heartbeat_seen,
|
||||
* triggering the CTRL_INIT → CTRL_STANDBY transition (Req 1.1). */
|
||||
if (opcode == OPCode_requestVote ||
|
||||
opcode == OPCode_probeConfigTag ||
|
||||
opcode == OPCode_heartbeat)
|
||||
return 0;
|
||||
return -XCE_ERR_NOT_READY;
|
||||
|
||||
case CTRL_ELECTING:
|
||||
if (opcode == OPCode_requestVote || opcode == OPCode_probeConfigTag)
|
||||
return 0;
|
||||
|
|
@ -611,3 +620,55 @@ int ha_demote_to_standby(cluster_t *c)
|
|||
ha_do_demote(&c->ha); /* sets role=STANDBY, resets hooks_registered=0 */
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ── HA Config Bootstrap: startup heartbeat wait (Req 1.1, 1.2) ─────────── */
|
||||
|
||||
/*
|
||||
* ha_wait_for_heartbeat_or_timeout: called from the startup path after
|
||||
* ha_init() when the node needs to discover whether a primary already exists.
|
||||
*
|
||||
* Blocks on sem_timedwait(&ha->heartbeat_seen, &deadline) where
|
||||
* deadline = now(CLOCK_REALTIME) + election_timeout_ms + jitter
|
||||
*
|
||||
* The heartbeat handler (cluster.c) posts heartbeat_seen when it receives
|
||||
* OPCode_heartbeat and ha->role == CTRL_INIT. ha_dispatch_gate() permits
|
||||
* OPCode_heartbeat in CTRL_INIT state so the packet reaches the handler.
|
||||
*
|
||||
* Returns CTRL_STANDBY — semaphore was posted (heartbeat arrived);
|
||||
* ha_do_demote() was called (role = CTRL_STANDBY).
|
||||
* Returns CTRL_ELECTING — sem_timedwait timed out (ETIMEDOUT);
|
||||
* caller is responsible for driving the election.
|
||||
*/
|
||||
ctrl_role_t ha_wait_for_heartbeat_or_timeout(ha_state_t *ha)
|
||||
{
|
||||
/* Seed jitter PRNG on first use */
|
||||
if (ha->jitter_seed == 0)
|
||||
ha_seed_jitter(&ha->jitter_seed, ha->my_priority);
|
||||
|
||||
/* Compute timeout in milliseconds: election_timeout + random jitter */
|
||||
long to_ms = (long)ha->config.election_timeout_ms;
|
||||
if (ha->config.election_jitter_max_ms > 0) {
|
||||
to_ms += (long)(xorshift64_next(&ha->jitter_seed)
|
||||
% ((uint64_t)ha->config.election_jitter_max_ms + 1));
|
||||
}
|
||||
|
||||
/* sem_timedwait requires an absolute CLOCK_REALTIME deadline */
|
||||
struct timespec deadline;
|
||||
clock_gettime(CLOCK_REALTIME, &deadline);
|
||||
deadline.tv_sec += to_ms / 1000;
|
||||
deadline.tv_nsec += (to_ms % 1000) * 1000000L;
|
||||
if (deadline.tv_nsec >= 1000000000L) {
|
||||
deadline.tv_sec++;
|
||||
deadline.tv_nsec -= 1000000000L;
|
||||
}
|
||||
|
||||
int rc = sem_timedwait(&ha->heartbeat_seen, &deadline);
|
||||
if (rc == 0) {
|
||||
/* Heartbeat arrived before the deadline — demote to standby */
|
||||
ha_do_demote(ha);
|
||||
return CTRL_STANDBY;
|
||||
}
|
||||
|
||||
/* ETIMEDOUT (or any other error): treat as election timeout */
|
||||
return CTRL_ELECTING;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -313,6 +313,25 @@ int bootstrap_config_if_needed(cluster_t *c);
|
|||
*/
|
||||
int xce_probe_config_tag(const char *host, uint16_t port, uint64_t *tag_out);
|
||||
|
||||
/*
|
||||
* ha_wait_for_heartbeat_or_timeout: wait in CTRL_INIT for an incoming heartbeat.
|
||||
*
|
||||
* Blocks on sem_timedwait(&ha->heartbeat_seen, &deadline) where
|
||||
* deadline = now(CLOCK_REALTIME) + election_timeout_ms + xorshift64_jitter
|
||||
*
|
||||
* Returns CTRL_STANDBY — heartbeat arrived; ha_do_demote() was called
|
||||
* (role is now CTRL_STANDBY).
|
||||
* Returns CTRL_ELECTING — deadline expired; caller must drive the election.
|
||||
*
|
||||
* The heartbeat handler posts heartbeat_seen only when role == CTRL_INIT.
|
||||
* OPCode_heartbeat is allowed through ha_dispatch_gate() in CTRL_INIT state
|
||||
* so that the primary's periodic heartbeat can reach the handler.
|
||||
*
|
||||
* Requirements: 1.1 (heartbeat → CTRL_STANDBY), 1.2 (timeout → CTRL_ELECTING).
|
||||
* Boundary: Controller FSM (ha.c).
|
||||
*/
|
||||
ctrl_role_t ha_wait_for_heartbeat_or_timeout(ha_state_t *ha);
|
||||
|
||||
/* ── P3-6: Test-only failover shim ────────────────────────────── */
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
|
|
|
|||
|
|
@ -428,6 +428,118 @@ static void test_ha_init_initializes_heartbeat_semaphore(void)
|
|||
printf("test_ha_init_initializes_heartbeat_semaphore: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Req 1.1/1.2: ha_wait_for_heartbeat_or_timeout() ────────────────────── */
|
||||
|
||||
/*
|
||||
* ha_wait_for_heartbeat_or_timeout: waits on ha->heartbeat_seen semaphore.
|
||||
* Deadline = now + election_timeout_ms + xorshift64_jitter(election_jitter_max_ms).
|
||||
*
|
||||
* Returns CTRL_STANDBY if a heartbeat arrives before the deadline
|
||||
* (calls ha_do_demote() which sets role = CTRL_STANDBY).
|
||||
* Returns CTRL_ELECTING if sem_timedwait times out (ETIMEDOUT).
|
||||
*
|
||||
* Takes ha_state_t* for standalone testability; production call-site wraps
|
||||
* with cluster_t* and passes &c->ha.
|
||||
*
|
||||
* Req 1.1: heartbeat before timeout → CTRL_STANDBY transition.
|
||||
* Req 1.2: timeout fires → returns CTRL_ELECTING.
|
||||
*/
|
||||
ctrl_role_t ha_wait_for_heartbeat_or_timeout(ha_state_t *ha);
|
||||
|
||||
static void test_wait_returns_standby_when_heartbeat_arrives_before_timeout(void)
|
||||
{
|
||||
/* Req 1.1: When a heartbeat is posted before the deadline,
|
||||
* ha_wait_for_heartbeat_or_timeout() must:
|
||||
* - return CTRL_STANDBY
|
||||
* - set ha->role to CTRL_STANDBY (via ha_do_demote)
|
||||
*
|
||||
* We simulate a heartbeat by posting heartbeat_seen directly before calling
|
||||
* the function, so sem_timedwait returns immediately with value 0. */
|
||||
ha_state_t ha;
|
||||
/* Use a short timeout but with jitter disabled so the test is deterministic */
|
||||
ha_config_t cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 500,
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
.election_jitter_max_ms = 0, /* no jitter — deterministic deadline */
|
||||
};
|
||||
ha_init(&ha, &cfg, 1);
|
||||
atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release);
|
||||
|
||||
/* Post the semaphore to simulate a heartbeat arriving before the deadline */
|
||||
sem_post(&ha.heartbeat_seen);
|
||||
|
||||
ctrl_role_t result = ha_wait_for_heartbeat_or_timeout(&ha);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)result);
|
||||
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
|
||||
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_wait_returns_standby_when_heartbeat_arrives_before_timeout: PASS\n");
|
||||
}
|
||||
|
||||
static void test_wait_returns_electing_when_timeout_fires(void)
|
||||
{
|
||||
/* Req 1.2: When no heartbeat arrives before the deadline,
|
||||
* ha_wait_for_heartbeat_or_timeout() must:
|
||||
* - return CTRL_ELECTING
|
||||
* - NOT change ha->role (caller is responsible for the transition)
|
||||
*
|
||||
* Use election_timeout_ms=1 and election_jitter_max_ms=0 so the semaphore
|
||||
* times out almost immediately. The test must complete in < 500ms. */
|
||||
ha_state_t ha;
|
||||
ha_config_t cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 1, /* 1 ms — times out almost instantly */
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
.election_jitter_max_ms = 0, /* no jitter */
|
||||
};
|
||||
/* Note: ha_init rejects election_timeout_ms < 500.
|
||||
* Bypass via direct struct assignment after ha_init with a valid config,
|
||||
* then overwrite the field for the test. */
|
||||
ha_config_t valid_cfg = cfg;
|
||||
valid_cfg.election_timeout_ms = 500; /* satisfy ha_init's >= 500 check */
|
||||
ha_init(&ha, &valid_cfg, 1);
|
||||
/* Now override to 1 ms for a near-instant timeout */
|
||||
ha.config.election_timeout_ms = 1;
|
||||
ha.config.election_jitter_max_ms = 0;
|
||||
atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release);
|
||||
|
||||
/* Do NOT post heartbeat_seen — let it time out */
|
||||
ctrl_role_t result = ha_wait_for_heartbeat_or_timeout(&ha);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)result);
|
||||
/* Role must still be CTRL_INIT — caller drives the transition */
|
||||
TEST_ASSERT_EQUAL_INT(CTRL_INIT,
|
||||
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_wait_returns_electing_when_timeout_fires: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Req 1.1/1.2: dispatch gate allows OPCode_heartbeat in CTRL_INIT ──────── */
|
||||
|
||||
static void test_dispatch_gate_init_allows_heartbeat(void)
|
||||
{
|
||||
/* Req 1.1: OPCode_heartbeat must pass through in CTRL_INIT so the
|
||||
* heartbeat handler can post heartbeat_seen and trigger the
|
||||
* CTRL_INIT → CTRL_STANDBY transition. */
|
||||
ha_state_t ha;
|
||||
ha_init(&ha, &g_dispatch_cfg, 0);
|
||||
atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release);
|
||||
|
||||
int rc = ha_dispatch_gate(&ha, OPCode_heartbeat);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_dispatch_gate_init_allows_heartbeat: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test suite entry point ───────────────────────────────────────────── */
|
||||
|
||||
void test_ha_startup(void)
|
||||
|
|
@ -466,4 +578,13 @@ void test_ha_startup(void)
|
|||
|
||||
/* Req 1.x — ha_init() initializes heartbeat_seen semaphore */
|
||||
RUN_TEST(test_ha_init_initializes_heartbeat_semaphore);
|
||||
|
||||
/* Req 1.1 — ha_wait_for_heartbeat_or_timeout: heartbeat → CTRL_STANDBY */
|
||||
RUN_TEST(test_wait_returns_standby_when_heartbeat_arrives_before_timeout);
|
||||
|
||||
/* Req 1.2 — ha_wait_for_heartbeat_or_timeout: timeout → CTRL_ELECTING */
|
||||
RUN_TEST(test_wait_returns_electing_when_timeout_fires);
|
||||
|
||||
/* Req 1.1 — dispatch gate allows heartbeat in CTRL_INIT */
|
||||
RUN_TEST(test_dispatch_gate_init_allows_heartbeat);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue