fix(ha): election-safety hardening — vote gates, restored-vote term, freshness source
Election protocol fixes validated against raft.tla invariants:
- ha_init_raft(): one-time raft state init at startup (re-init would lose
durable election progress); n_peers = full cluster size including self
- handle_request_vote(): higher-term step-down moved BEFORE the role gate
(split-brain fix); standby vote-deny now requires a live primary control
channel (primary_ctrl_fd >= 0) AND a fresh heartbeat
- ha_heartbeat_is_fresh(): derive freshness from ha->last_activity (the
field production actually writes); the orphaned last_heartbeat timespec
had no production writer, leaving the vote-deny gate inert (bug-337);
field removed from ha_state_t
- handle_request_vote(): restore same-candidate idempotent re-grant —
deny only a DIFFERENT candidate in the same term, matching raft.c and
Raft §5.2 votedFor ∈ {Nil, candidateId} (bug-338)
- raft_init(): restore voted_for_term alongside a persisted vote; leaving
it 0 allowed a double vote in the same term across a restart (bug-339)
- myapp_run(): leader_known check treats a vote recorded under this node's
own name as a self-vote — production persists the real candidate name,
the "self" placeholder only appears in test mode
- raft_on_vote_request(): leader steps down on higher-term vote request
instead of denying it
Tests: test_proto_H6_restored_vote_blocks_second_vote added; H1 restore
assertion strengthened; test_ha_vote_gate.c driven through last_activity +
primary_ctrl_fd (the real production fields). Full /xce-build all green
(lib + controller + scheduler + Go suites, 0 failures, ASAN clean).
deploy-ha.sh: start.sh now forwards extra args to the controller binary.
This commit is contained in:
parent
fa66d1ba9c
commit
9821ec4f3c
7 changed files with 207 additions and 91 deletions
|
|
@ -47,24 +47,16 @@ static ha_config_t g_gate_cfg = {
|
|||
.election_priority_step_ms = 500,
|
||||
};
|
||||
|
||||
/* ── Helper: set ha->last_heartbeat to "now - offset_ms milliseconds ago" ── */
|
||||
/* ── Helper: set ha->last_activity to "now - offset_ms milliseconds ago" ──
|
||||
*
|
||||
* last_activity is the production field written by ha_on_heartbeat_received
|
||||
* (whole CLOCK_MONOTONIC seconds); offset_ms is rounded down to seconds. */
|
||||
|
||||
static void set_last_heartbeat_age(ha_state_t *ha, long offset_ms)
|
||||
static void set_last_activity_age(ha_state_t *ha, long offset_ms)
|
||||
{
|
||||
struct timespec now;
|
||||
clock_gettime(CLOCK_MONOTONIC, &now);
|
||||
|
||||
long sec = offset_ms / 1000;
|
||||
long nsec = (offset_ms % 1000) * 1000000L;
|
||||
|
||||
ha->last_heartbeat.tv_sec = now.tv_sec - sec;
|
||||
ha->last_heartbeat.tv_nsec = now.tv_nsec - nsec;
|
||||
|
||||
/* Carry / borrow normalisation */
|
||||
if (ha->last_heartbeat.tv_nsec < 0) {
|
||||
ha->last_heartbeat.tv_sec -= 1;
|
||||
ha->last_heartbeat.tv_nsec += 1000000000L;
|
||||
}
|
||||
ha->last_activity = now.tv_sec - offset_ms / 1000;
|
||||
}
|
||||
|
||||
/* ── Req 6.1: CTRL_PRIMARY → deny regardless of candidate freshness ────── */
|
||||
|
|
@ -139,8 +131,13 @@ static void test_standby_fresh_heartbeat_denies_vote_request(void)
|
|||
atomic_store_explicit(&ha.term, 2ULL, memory_order_release);
|
||||
ha.voted_for_term = 0;
|
||||
|
||||
/* last_heartbeat = 500 ms ago (< 10 000 ms timeout → fresh) */
|
||||
set_last_heartbeat_age(&ha, 500L);
|
||||
/* Active primary control channel (handle_ctrlhello sets this on a real
|
||||
* standby; the deny gate requires a live connection AND a fresh heartbeat).
|
||||
* ha_fini does not close the fd, so any non-negative value is safe. */
|
||||
ha.primary_ctrl_fd = 999;
|
||||
|
||||
/* last_activity = 500 ms ago → rounds to "this second" (< 10 s → fresh) */
|
||||
set_last_activity_age(&ha, 500L);
|
||||
|
||||
int granted = handle_request_vote(&ha,
|
||||
2ULL,
|
||||
|
|
@ -174,8 +171,11 @@ static void test_standby_stale_heartbeat_evaluates_vote(void)
|
|||
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
||||
ha.voted_for_term = 0;
|
||||
|
||||
/* last_heartbeat = 15 000 ms ago (> 10 000 ms timeout → stale) */
|
||||
set_last_heartbeat_age(&ha, 15000L);
|
||||
/* Active primary control channel — staleness alone must open the gate */
|
||||
ha.primary_ctrl_fd = 999;
|
||||
|
||||
/* last_activity = 15 000 ms ago (> 10 000 ms timeout → stale) */
|
||||
set_last_activity_age(&ha, 15000L);
|
||||
|
||||
int granted = handle_request_vote(&ha,
|
||||
1ULL,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue