fix(ha): election-safety hardening — vote gates, restored-vote term, freshness source
Election protocol fixes validated against raft.tla invariants:
- ha_init_raft(): one-time raft state init at startup (re-init would lose
durable election progress); n_peers = full cluster size including self
- handle_request_vote(): higher-term step-down moved BEFORE the role gate
(split-brain fix); standby vote-deny now requires a live primary control
channel (primary_ctrl_fd >= 0) AND a fresh heartbeat
- ha_heartbeat_is_fresh(): derive freshness from ha->last_activity (the
field production actually writes); the orphaned last_heartbeat timespec
had no production writer, leaving the vote-deny gate inert (bug-337);
field removed from ha_state_t
- handle_request_vote(): restore same-candidate idempotent re-grant —
deny only a DIFFERENT candidate in the same term, matching raft.c and
Raft §5.2 votedFor ∈ {Nil, candidateId} (bug-338)
- raft_init(): restore voted_for_term alongside a persisted vote; leaving
it 0 allowed a double vote in the same term across a restart (bug-339)
- myapp_run(): leader_known check treats a vote recorded under this node's
own name as a self-vote — production persists the real candidate name,
the "self" placeholder only appears in test mode
- raft_on_vote_request(): leader steps down on higher-term vote request
instead of denying it
Tests: test_proto_H6_restored_vote_blocks_second_vote added; H1 restore
assertion strengthened; test_ha_vote_gate.c driven through last_activity +
primary_ctrl_fd (the real production fields). Full /xce-build all green
(lib + controller + scheduler + Go suites, 0 failures, ASAN clean).
deploy-ha.sh: start.sh now forwards extra args to the controller binary.
This commit is contained in:
parent
fa66d1ba9c
commit
9821ec4f3c
7 changed files with 207 additions and 91 deletions
|
|
@ -40,7 +40,11 @@ int raft_init(raft_state_t *s, const raft_config_t *cfg)
|
|||
s->term = cfg->initial_term;
|
||||
s->role = RAFT_FOLLOWER;
|
||||
s->voted_for = cfg->initial_voted_for;
|
||||
s->voted_for_term = 0;
|
||||
/* A restored vote belongs to the restored term: persistence writes
|
||||
* (term, voted_for) as a pair. Leaving voted_for_term at 0 would let the
|
||||
* duplicate-vote check miss after restart, granting a second vote in the
|
||||
* same term (Election Safety violation). */
|
||||
s->voted_for_term = (cfg->initial_voted_for != -1) ? cfg->initial_term : 0;
|
||||
s->n_peers = cfg->n_peers;
|
||||
s->self_idx = cfg->self_idx;
|
||||
s->votes_granted = 0;
|
||||
|
|
@ -132,22 +136,35 @@ int raft_on_vote_request(raft_state_t *s, int from_idx, uint64_t term,
|
|||
/* reply_term reflects our (possibly updated) current term from here on */
|
||||
*reply_term = s->term;
|
||||
|
||||
/* Step 3: role gate — LEADER denies */
|
||||
/* Step 3: role gate — same-term challenger denied by live leader.
|
||||
*
|
||||
* This only fires when term == s->term (the equal-term case). For
|
||||
* term > s->term, Step 2 already downgraded s->role to RAFT_FOLLOWER
|
||||
* before we reach here, so the check below is false and we fall through
|
||||
* to grant/deny based on freshness.
|
||||
*
|
||||
* A RequestVote for the same term as the current leader means the sender
|
||||
* started a competing candidacy in a term the leader already won — deny. */
|
||||
if (s->role == RAFT_LEADER) {
|
||||
if (s->cbs->on_trace)
|
||||
s->cbs->on_trace(s->ctx, "vote_denied_role", NULL);
|
||||
return RAFT_DENY;
|
||||
}
|
||||
|
||||
/* Step 4: duplicate-vote deny — already voted for a *different* candidate
|
||||
* this same term. If voted_for == from_idx and voted_for_term == term,
|
||||
* fall through to idempotent re-grant path below. */
|
||||
if (s->voted_for != -1 &&
|
||||
s->voted_for != from_idx &&
|
||||
s->voted_for_term == term) {
|
||||
if (s->cbs->on_trace)
|
||||
s->cbs->on_trace(s->ctx, "vote_denied_already_voted", NULL);
|
||||
return RAFT_DENY;
|
||||
/* Step 4: duplicate-vote check — already voted in this term.
|
||||
* Idempotent re-grant: if this is the same candidate we already voted for
|
||||
* in this term, fall through so we re-persist and re-grant (network retry
|
||||
* path). This is safe: the candidate has already committed to this term
|
||||
* and we already durably recorded the vote.
|
||||
* Different candidate: deny — standard Raft invariant of at most one vote
|
||||
* per node per term. */
|
||||
if (s->voted_for != -1 && s->voted_for_term == term) {
|
||||
if (s->voted_for != from_idx) {
|
||||
if (s->cbs->on_trace)
|
||||
s->cbs->on_trace(s->ctx, "vote_denied_already_voted", NULL);
|
||||
return RAFT_DENY;
|
||||
}
|
||||
/* same candidate — fall through to re-persist and re-grant */
|
||||
}
|
||||
|
||||
/* Step 5: application-level policy gate (freshness, log lsn, etc.) */
|
||||
|
|
|
|||
|
|
@ -2029,6 +2029,9 @@ static void test_proto_H1_init_with_persisted_state(void)
|
|||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
TEST_ASSERT_EQUAL_UINT64(5, s.term);
|
||||
TEST_ASSERT_EQUAL_INT(1, s.voted_for);
|
||||
/* The restored vote belongs to the restored term — without this, the
|
||||
* duplicate-vote check cannot match after restart (see H6). */
|
||||
TEST_ASSERT_EQUAL_UINT64(5, s.voted_for_term);
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_FOLLOWER, s.role);
|
||||
}
|
||||
|
||||
|
|
@ -2061,6 +2064,51 @@ static void test_proto_H2_init_first_boot(void)
|
|||
TEST_ASSERT_EQUAL_INT(RAFT_FOLLOWER, s.role);
|
||||
}
|
||||
|
||||
/*
|
||||
* test_proto_H6_restored_vote_blocks_second_vote
|
||||
*
|
||||
* Election Safety across restart: a node that voted for candidate 1 in
|
||||
* term 5, then restarted (raft_init with initial_term=5, initial_voted_for=1),
|
||||
* must DENY a vote request from a DIFFERENT candidate (idx=2) in the same
|
||||
* term 5. Without restoring voted_for_term alongside voted_for, the
|
||||
* duplicate-vote check cannot match the restored term and the node would
|
||||
* grant a second vote in term 5 → two leaders in one term.
|
||||
*/
|
||||
static void test_proto_H6_restored_vote_blocks_second_vote(void)
|
||||
{
|
||||
raft_state_t s;
|
||||
raft_callbacks_t cbs = make_mock_cbs();
|
||||
uint64_t reply_term = 0;
|
||||
|
||||
reset_mock();
|
||||
|
||||
raft_config_t cfg;
|
||||
memset(&s, 0, sizeof(s));
|
||||
memset(&cfg, 0, sizeof(cfg));
|
||||
cfg.n_peers = 3;
|
||||
cfg.self_idx = 0;
|
||||
cfg.election_timeout_ms = 1000;
|
||||
cfg.initial_term = 5;
|
||||
cfg.initial_voted_for = 1; /* persisted vote for candidate 1, term 5 */
|
||||
cfg.cbs = &cbs;
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, raft_init(&s, &cfg));
|
||||
|
||||
/* Different candidate (idx=2) requests a vote in the SAME term 5 */
|
||||
int rc = raft_on_vote_request(&s, 2, 5, &reply_term);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_DENY, rc);
|
||||
TEST_ASSERT_EQUAL_UINT64(5, reply_term);
|
||||
|
||||
/* The restored vote must be untouched */
|
||||
TEST_ASSERT_EQUAL_INT(1, s.voted_for);
|
||||
TEST_ASSERT_EQUAL_UINT64(5, s.voted_for_term);
|
||||
|
||||
/* Same candidate (idx=1) retransmitting must still be re-granted */
|
||||
rc = raft_on_vote_request(&s, 1, 5, &reply_term);
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_GRANT, rc);
|
||||
}
|
||||
|
||||
/*
|
||||
* test_proto_H3_persist_before_vote_recorded
|
||||
*
|
||||
|
|
@ -2964,6 +3012,7 @@ void test_raft(void)
|
|||
RUN_TEST(test_proto_H3_persist_before_vote_recorded);
|
||||
RUN_TEST(test_proto_H4_persist_fail_denies);
|
||||
RUN_TEST(test_proto_H5_start_election_self_vote_persisted);
|
||||
RUN_TEST(test_proto_H6_restored_vote_blocks_second_vote);
|
||||
RUN_TEST(test_proto_Hnull_required_callback_einval);
|
||||
|
||||
/* Group D: Quorum and Result (task 5.3) */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue