fix(ha): election-safety hardening — vote gates, restored-vote term, freshness source
Election protocol fixes validated against raft.tla invariants:
- ha_init_raft(): one-time raft state init at startup (re-init would lose
durable election progress); n_peers = full cluster size including self
- handle_request_vote(): higher-term step-down moved BEFORE the role gate
(split-brain fix); standby vote-deny now requires a live primary control
channel (primary_ctrl_fd >= 0) AND a fresh heartbeat
- ha_heartbeat_is_fresh(): derive freshness from ha->last_activity (the
field production actually writes); the orphaned last_heartbeat timespec
had no production writer, leaving the vote-deny gate inert (bug-337);
field removed from ha_state_t
- handle_request_vote(): restore same-candidate idempotent re-grant —
deny only a DIFFERENT candidate in the same term, matching raft.c and
Raft §5.2 votedFor ∈ {Nil, candidateId} (bug-338)
- raft_init(): restore voted_for_term alongside a persisted vote; leaving
it 0 allowed a double vote in the same term across a restart (bug-339)
- myapp_run(): leader_known check treats a vote recorded under this node's
own name as a self-vote — production persists the real candidate name,
the "self" placeholder only appears in test mode
- raft_on_vote_request(): leader steps down on higher-term vote request
instead of denying it
Tests: test_proto_H6_restored_vote_blocks_second_vote added; H1 restore
assertion strengthened; test_ha_vote_gate.c driven through last_activity +
primary_ctrl_fd (the real production fields). Full /xce-build all green
(lib + controller + scheduler + Go suites, 0 failures, ASAN clean).
deploy-ha.sh: start.sh now forwards extra args to the controller binary.
This commit is contained in:
parent
fa66d1ba9c
commit
9821ec4f3c
7 changed files with 207 additions and 91 deletions
|
|
@ -1753,21 +1753,23 @@ int ha_restore_term(ha_state_t *ha, const char *workdir)
|
|||
|
||||
/* ── ha_heartbeat_is_fresh (Req 6.2) ────────────────────────────────────────
|
||||
*
|
||||
* Returns 1 if the elapsed time since ha->last_heartbeat is strictly less than
|
||||
* ha->config.heartbeat_timeout_ms (i.e. the primary is still alive).
|
||||
* Returns 0 if the heartbeat is stale (timeout has expired).
|
||||
* Returns 1 if the elapsed time since the last primary message
|
||||
* (ha->last_activity, written by ha_on_heartbeat_received) is strictly less
|
||||
* than ha->config.heartbeat_timeout_ms (i.e. the primary is still alive).
|
||||
* Returns 0 if no primary message has been seen yet (last_activity == 0) or
|
||||
* the timeout has expired.
|
||||
*
|
||||
* Uses CLOCK_MONOTONIC to avoid wall-clock skew.
|
||||
* Uses CLOCK_MONOTONIC seconds to avoid wall-clock skew; whole-second
|
||||
* granularity is sufficient for the 10 s default timeout.
|
||||
*/
|
||||
static int ha_heartbeat_is_fresh(ha_state_t *ha)
|
||||
{
|
||||
struct timespec now, last;
|
||||
time_t last = ha->last_activity; /* time_t load is atomic on LP64 */
|
||||
if (last == 0)
|
||||
return 0; /* no primary message seen yet */
|
||||
struct timespec now;
|
||||
clock_gettime(CLOCK_MONOTONIC, &now);
|
||||
pthread_mutex_lock(&ha->lock);
|
||||
last = ha->last_heartbeat;
|
||||
pthread_mutex_unlock(&ha->lock);
|
||||
long ms = (now.tv_sec - last.tv_sec) * 1000L
|
||||
+ (now.tv_nsec - last.tv_nsec) / 1000000L;
|
||||
long ms = (long)(now.tv_sec - last) * 1000L;
|
||||
return ms < (long)ha->config.heartbeat_timeout_ms;
|
||||
}
|
||||
|
||||
|
|
@ -1837,8 +1839,9 @@ int handle_request_vote(ha_state_t *ha,
|
|||
memory_order_acquire);
|
||||
if (my_role == CTRL_PRIMARY || my_role == CTRL_PRIMARY_COMMITTING)
|
||||
return 0; /* deny — I am primary (or committing); protect against challengers */
|
||||
if (my_role == CTRL_STANDBY && ha_heartbeat_is_fresh(ha))
|
||||
return 0; /* deny — my primary is alive */
|
||||
if (my_role == CTRL_STANDBY && ha->primary_ctrl_fd >= 0 &&
|
||||
ha_heartbeat_is_fresh(ha))
|
||||
return 0; /* deny — my primary is alive (active primary connection) */
|
||||
|
||||
/* Req 2.5: catching-up standbys deny all votes */
|
||||
if (in_catch_up_flag)
|
||||
|
|
@ -1847,9 +1850,10 @@ int handle_request_vote(ha_state_t *ha,
|
|||
/* Protect voted_for check+set from concurrent vote requests on different vtasks */
|
||||
pthread_mutex_lock(&ha->lock);
|
||||
|
||||
/* Req 2.6: already voted for a different candidate in this term → deny */
|
||||
if (ha->voted_for_term == term &&
|
||||
ha->voted_for_name[0] != '\0' &&
|
||||
/* Req 2.6: already voted for a DIFFERENT candidate in this term → deny.
|
||||
* Same-candidate re-grant is idempotent (Raft §5.2: grant iff votedFor ∈
|
||||
* {Nil, candidateId}) — matches the duplicate-vote check in raft.c. */
|
||||
if (ha->voted_for_term == term && ha->voted_for_name[0] != '\0' &&
|
||||
strcmp(ha->voted_for_name, candidate_id) != 0) {
|
||||
pthread_mutex_unlock(&ha->lock);
|
||||
return 0;
|
||||
|
|
@ -2975,6 +2979,49 @@ static int vote_ep_err_handler(void *ep_raw)
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* ── ha_init_raft: one-time raft state machine initialization ────────────────
|
||||
*
|
||||
* Called once at startup after ha_restore_term() to wire the raft state
|
||||
* machine with the correct cluster size and callbacks. Subsequent elections
|
||||
* use raft_start_election() / raft_on_leader_lost() on this same state.
|
||||
*
|
||||
* n_peers: total candidates_count from cluster config (includes self).
|
||||
*/
|
||||
void ha_init_raft(ha_state_t *ha, cluster_t *c, int n_peers)
|
||||
{
|
||||
/* Map persisted voted_for_name back to a peer index for initial_voted_for.
|
||||
* This lets raft_check_and_start_election (RDHUP path) see the prior vote
|
||||
* without re-reading the file. */
|
||||
int restored_voted_for = -1;
|
||||
if (ha->voted_for_name[0] != '\0' && c && c->myconf) {
|
||||
const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg;
|
||||
for (int i = 0; i < pcfg->candidates_count; i++) {
|
||||
if (strcmp(pcfg->candidates[i].name, ha->voted_for_name) == 0) {
|
||||
restored_voted_for = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
raft_config_t rcfg;
|
||||
memset(&rcfg, 0, sizeof(rcfg));
|
||||
rcfg.n_peers = n_peers;
|
||||
rcfg.self_idx = ha->my_priority;
|
||||
rcfg.election_timeout_ms = ha->config.election_timeout_ms;
|
||||
rcfg.my_priority = ha->my_priority;
|
||||
rcfg.initial_term = atomic_load_explicit(&ha->term, memory_order_acquire);
|
||||
rcfg.initial_voted_for = restored_voted_for;
|
||||
rcfg.cbs = &s_real_cbs;
|
||||
rcfg.ctx = ha;
|
||||
raft_init(&ha->raft, &rcfg);
|
||||
|
||||
ha->n_peers = n_peers;
|
||||
ha->raft_c = c;
|
||||
INFO("HA: raft initialized (n_peers=%d, self=%d, term=%" PRIu64 ")",
|
||||
n_peers, ha->my_priority,
|
||||
atomic_load_explicit(&ha->term, memory_order_acquire));
|
||||
}
|
||||
|
||||
/* ── Task 6.3: ha_start_election_async — delegate to raft_start_election ────
|
||||
*
|
||||
* Non-blocking election initiator. Sets up the raft state machine and
|
||||
|
|
@ -3001,45 +3048,8 @@ void ha_start_election_async(ha_state_t *ha, cluster_t *c,
|
|||
(void)peers; /* peer connections driven by raft_cb_send_vote_request */
|
||||
(void)my_name; /* self-vote name stored in on_persist via raft_start_election */
|
||||
|
||||
/* Step 0: wire cluster pointer for raft callbacks.
|
||||
* Must be set before any raft_* call so callbacks can reach cluster_t. */
|
||||
ha->raft_c = c;
|
||||
|
||||
/* Step 0b: re-initialise raft_state_t with the real cluster size and
|
||||
* the term/voted_for restored from disk by ha_restore_term().
|
||||
*
|
||||
* ha->raft.term may be 0 from ha_init while ha->term was restored to a
|
||||
* higher value by ha_restore_term() — reconcile here before the election.
|
||||
*
|
||||
* self_idx = ha->my_priority (index of this node in candidates[]).
|
||||
* n_peers = total cluster size (candidates_count); passed directly.
|
||||
*
|
||||
* voted_for: map ha->voted_for_name back to a peer index (-1 if none).
|
||||
*/
|
||||
{
|
||||
int restored_voted_for = -1;
|
||||
if (ha->voted_for_name[0] != '\0' && c && c->myconf) {
|
||||
const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg;
|
||||
for (int i = 0; i < pcfg->candidates_count; i++) {
|
||||
if (strcmp(pcfg->candidates[i].name, ha->voted_for_name) == 0) {
|
||||
restored_voted_for = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
raft_config_t rcfg;
|
||||
memset(&rcfg, 0, sizeof(rcfg));
|
||||
rcfg.n_peers = n_peers; /* total candidates_count */
|
||||
rcfg.self_idx = ha->my_priority;
|
||||
rcfg.election_timeout_ms = ha->config.election_timeout_ms;
|
||||
rcfg.my_priority = ha->my_priority;
|
||||
rcfg.initial_term = atomic_load_explicit(&ha->term, memory_order_acquire);
|
||||
rcfg.initial_voted_for = restored_voted_for;
|
||||
rcfg.cbs = &s_real_cbs;
|
||||
rcfg.ctx = ha;
|
||||
raft_init(&ha->raft, &rcfg);
|
||||
}
|
||||
/* Step 0: raft is initialized once at startup by ha_init_raft(); nothing
|
||||
* to do here. ha->raft_c and ha->n_peers are already set. */
|
||||
|
||||
/* Step 1: transition to CTRL_ELECTING before firing RPCs.
|
||||
* raft_start_election fires on_become_leader / on_send_vote_request below;
|
||||
|
|
@ -3255,6 +3265,7 @@ void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c)
|
|||
/* Update n_peers tracking — total cluster size (including self) */
|
||||
ha->n_peers = n_peers;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue