fix(ha): election-safety hardening — vote gates, restored-vote term, freshness source

Election protocol fixes validated against raft.tla invariants:

- ha_init_raft(): one-time raft state init at startup (re-init would lose
  durable election progress); n_peers = full cluster size including self
- handle_request_vote(): higher-term step-down moved BEFORE the role gate
  (split-brain fix); standby vote-deny now requires a live primary control
  channel (primary_ctrl_fd >= 0) AND a fresh heartbeat
- ha_heartbeat_is_fresh(): derive freshness from ha->last_activity (the
  field production actually writes); the orphaned last_heartbeat timespec
  had no production writer, leaving the vote-deny gate inert (bug-337);
  field removed from ha_state_t
- handle_request_vote(): restore same-candidate idempotent re-grant —
  deny only a DIFFERENT candidate in the same term, matching raft.c and
  Raft §5.2 votedFor ∈ {Nil, candidateId} (bug-338)
- raft_init(): restore voted_for_term alongside a persisted vote; leaving
  it 0 allowed a double vote in the same term across a restart (bug-339)
- myapp_run(): leader_known check treats a vote recorded under this node's
  own name as a self-vote — production persists the real candidate name,
  the "self" placeholder only appears in test mode
- raft_on_vote_request(): leader steps down on higher-term vote request
  instead of denying it

Tests: test_proto_H6_restored_vote_blocks_second_vote added; H1 restore
assertion strengthened; test_ha_vote_gate.c driven through last_activity +
primary_ctrl_fd (the real production fields). Full /xce-build all green
(lib + controller + scheduler + Go suites, 0 failures, ASAN clean).

deploy-ha.sh: start.sh now forwards extra args to the controller binary.
This commit is contained in:
Rongsong Shen 2026-06-10 01:25:39 +00:00
parent fa66d1ba9c
commit 9821ec4f3c
7 changed files with 207 additions and 91 deletions

View file

@ -40,7 +40,11 @@ int raft_init(raft_state_t *s, const raft_config_t *cfg)
s->term = cfg->initial_term;
s->role = RAFT_FOLLOWER;
s->voted_for = cfg->initial_voted_for;
s->voted_for_term = 0;
/* A restored vote belongs to the restored term: persistence writes
* (term, voted_for) as a pair. Leaving voted_for_term at 0 would let the
* duplicate-vote check miss after restart, granting a second vote in the
* same term (Election Safety violation). */
s->voted_for_term = (cfg->initial_voted_for != -1) ? cfg->initial_term : 0;
s->n_peers = cfg->n_peers;
s->self_idx = cfg->self_idx;
s->votes_granted = 0;
@ -132,22 +136,35 @@ int raft_on_vote_request(raft_state_t *s, int from_idx, uint64_t term,
/* reply_term reflects our (possibly updated) current term from here on */
*reply_term = s->term;
/* Step 3: role gate — LEADER denies */
/* Step 3: role gate — same-term challenger denied by live leader.
*
* This only fires when term == s->term (the equal-term case). For
* term > s->term, Step 2 already downgraded s->role to RAFT_FOLLOWER
* before we reach here, so the check below is false and we fall through
* to grant/deny based on freshness.
*
* A RequestVote for the same term as the current leader means the sender
* started a competing candidacy in a term the leader already won — deny. */
if (s->role == RAFT_LEADER) {
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "vote_denied_role", NULL);
return RAFT_DENY;
}
/* Step 4: duplicate-vote deny — already voted for a *different* candidate
* this same term. If voted_for == from_idx and voted_for_term == term,
* fall through to idempotent re-grant path below. */
if (s->voted_for != -1 &&
s->voted_for != from_idx &&
s->voted_for_term == term) {
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "vote_denied_already_voted", NULL);
return RAFT_DENY;
/* Step 4: duplicate-vote check — already voted in this term.
* Idempotent re-grant: if this is the same candidate we already voted for
* in this term, fall through so we re-persist and re-grant (network retry
* path). This is safe: the candidate has already committed to this term
* and we already durably recorded the vote.
* Different candidate: deny — standard Raft invariant of at most one vote
* per node per term. */
if (s->voted_for != -1 && s->voted_for_term == term) {
if (s->voted_for != from_idx) {
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "vote_denied_already_voted", NULL);
return RAFT_DENY;
}
/* same candidate — fall through to re-persist and re-grant */
}
/* Step 5: application-level policy gate (freshness, log lsn, etc.) */

View file

@ -2029,6 +2029,9 @@ static void test_proto_H1_init_with_persisted_state(void)
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT64(5, s.term);
TEST_ASSERT_EQUAL_INT(1, s.voted_for);
/* The restored vote belongs to the restored term — without this, the
* duplicate-vote check cannot match after restart (see H6). */
TEST_ASSERT_EQUAL_UINT64(5, s.voted_for_term);
TEST_ASSERT_EQUAL_INT(RAFT_FOLLOWER, s.role);
}
@ -2061,6 +2064,51 @@ static void test_proto_H2_init_first_boot(void)
TEST_ASSERT_EQUAL_INT(RAFT_FOLLOWER, s.role);
}
/*
* test_proto_H6_restored_vote_blocks_second_vote
*
* Election Safety across restart: a node that voted for candidate 1 in
* term 5, then restarted (raft_init with initial_term=5, initial_voted_for=1),
* must DENY a vote request from a DIFFERENT candidate (idx=2) in the same
* term 5. Without restoring voted_for_term alongside voted_for, the
* duplicate-vote check cannot match the restored term and the node would
* grant a second vote in term 5 → two leaders in one term.
*/
static void test_proto_H6_restored_vote_blocks_second_vote(void)
{
raft_state_t s;
raft_callbacks_t cbs = make_mock_cbs();
uint64_t reply_term = 0;
reset_mock();
raft_config_t cfg;
memset(&s, 0, sizeof(s));
memset(&cfg, 0, sizeof(cfg));
cfg.n_peers = 3;
cfg.self_idx = 0;
cfg.election_timeout_ms = 1000;
cfg.initial_term = 5;
cfg.initial_voted_for = 1; /* persisted vote for candidate 1, term 5 */
cfg.cbs = &cbs;
TEST_ASSERT_EQUAL_INT(0, raft_init(&s, &cfg));
/* Different candidate (idx=2) requests a vote in the SAME term 5 */
int rc = raft_on_vote_request(&s, 2, 5, &reply_term);
TEST_ASSERT_EQUAL_INT(RAFT_DENY, rc);
TEST_ASSERT_EQUAL_UINT64(5, reply_term);
/* The restored vote must be untouched */
TEST_ASSERT_EQUAL_INT(1, s.voted_for);
TEST_ASSERT_EQUAL_UINT64(5, s.voted_for_term);
/* Same candidate (idx=1) retransmitting must still be re-granted */
rc = raft_on_vote_request(&s, 1, 5, &reply_term);
TEST_ASSERT_EQUAL_INT(RAFT_GRANT, rc);
}
/*
* test_proto_H3_persist_before_vote_recorded
*
@ -2964,6 +3012,7 @@ void test_raft(void)
RUN_TEST(test_proto_H3_persist_before_vote_recorded);
RUN_TEST(test_proto_H4_persist_fail_denies);
RUN_TEST(test_proto_H5_start_election_self_vote_persisted);
RUN_TEST(test_proto_H6_restored_vote_blocks_second_vote);
RUN_TEST(test_proto_Hnull_required_callback_einval);
/* Group D: Quorum and Result (task 5.3) */