fix(ha-config-bootstrap,ha-repl-hardening): config-less node must not win election
Root cause (found via ha-convergence-recovery task 8 live 3-node E2E): a controller with no local config (config_version_tag==0) could become PRIMARY and push an empty config, breaking replication. Two upstream election-safety gaps, both outside ha-convergence-recovery: 1. RDHUP re-election bypassed the no-config 'voter only' guard. The startup path (ha_start_election_async) refuses to start an election when my_config_tag==0 && n_peers>1, but the RDHUP failover path (ha_on_primary_ctrl_fd_rdhup) called raft_on_leader_lost unconditionally. Fix: mirror the guard before elect_ctx alloc / hook / raft_on_leader_lost; a config-less node stays CTRL_STANDBY (voter only) and grants the configured peer instead of starting (and winning) its own election. 2. Voter freshness gate compared the candidate only against the voter's LOCAL config, ignoring the cluster's known committed state. Fix: ha_vote_self_freshness floors the comparison at max(local, learned-committed config_tag / committed_lsn), so a voter denies any candidate whose config/cluster state is outdated vs what the cluster has committed. Strictly additive (never denies the freshest node — Leader Completeness preserves liveness). TDD: test_rdhup_noconfig_node_does_not_elect + configured-node contrast; test_ha_vote_gate learned-floor deny + liveness + backward-compat. Full controller suite green under ASAN. commit.c and ha-convergence-recovery files untouched.
This commit is contained in:
parent
513b965f98
commit
c97c7632e9
6 changed files with 439 additions and 9 deletions
|
|
@ -35,6 +35,15 @@ int handle_request_vote(ha_state_t *ha,
|
|||
uint64_t my_last_lsn,
|
||||
int in_catch_up_flag);
|
||||
|
||||
/* Pure helper under test (election-safety fix #2): computes the voter's
|
||||
* effective freshness floor as the max of its local state and its learned
|
||||
* COMMITTED cluster state. */
|
||||
void ha_vote_self_freshness(ha_state_t *ha,
|
||||
uint64_t local_config_tag,
|
||||
uint64_t local_applied_lsn,
|
||||
uint64_t *out_tag,
|
||||
uint64_t *out_lsn);
|
||||
|
||||
/* ── Shared config for vote gate tests ──────────────────────────────────── */
|
||||
|
||||
static ha_config_t g_gate_cfg = {
|
||||
|
|
@ -229,6 +238,199 @@ static void test_electing_role_evaluates_vote(void)
|
|||
printf("test_electing_role_evaluates_vote: PASS\n");
|
||||
}
|
||||
|
||||
/* ══ Election-safety fix #2: deny candidates outdated vs LEARNED committed state ══
|
||||
*
|
||||
* A voter must measure candidate freshness against the strongest knowledge it
|
||||
* has of the cluster's COMMITTED state — not merely its own local config file /
|
||||
* applied LSN. ha->committed_config_tag and ha->last_known_committed_lsn carry
|
||||
* that learned-committed floor (from prior primary heartbeats / quorum ACK /
|
||||
* post-election safe point). The gate floor is max(local, learned). */
|
||||
|
||||
/* ── Direct unit test of ha_vote_self_freshness (max logic both fields) ───── */
|
||||
|
||||
static void test_vote_self_freshness_takes_max_of_local_and_learned(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
ha_init(&ha, &g_gate_cfg, 0);
|
||||
|
||||
/* learned-committed > local on tag, local > learned on lsn → expect the
|
||||
* per-field max of each. */
|
||||
atomic_store_explicit(&ha.committed_config_tag, 100ULL, memory_order_release);
|
||||
atomic_store_explicit(&ha.last_known_committed_lsn, 20ULL, memory_order_release);
|
||||
|
||||
uint64_t out_tag = 0, out_lsn = 0;
|
||||
ha_vote_self_freshness(&ha, 40ULL /*local tag*/, 70ULL /*local lsn*/,
|
||||
&out_tag, &out_lsn);
|
||||
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(100ULL, out_tag,
|
||||
"freshness floor tag must be max(local=40, learned=100)");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(70ULL, out_lsn,
|
||||
"freshness floor lsn must be max(local=70, learned=20)");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_vote_self_freshness_takes_max_of_local_and_learned: PASS\n");
|
||||
}
|
||||
|
||||
/* ── RED: learned-committed config_tag floor denies an outdated candidate ─── */
|
||||
|
||||
static void test_learned_committed_tag_denies_outdated_candidate(void)
|
||||
{
|
||||
/* Voter local config is cold (my_config_tag=0) but it has LEARNED that the
|
||||
* cluster committed config_tag=100. A candidate offering config_tag=50
|
||||
* would roll the cluster back below the committed floor → must be DENIED.
|
||||
*
|
||||
* Before the fix the gate used the raw local my_config_tag=0, so cand=50 ≥ 0
|
||||
* granted (RED). After the fix the floor is max(0,100)=100 → deny (GREEN). */
|
||||
ha_state_t ha;
|
||||
ha_init(&ha, &g_gate_cfg, 0);
|
||||
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
||||
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
||||
ha.voted_for_term = 0;
|
||||
/* No active/fresh primary connection, not in catch-up → role gate opens. */
|
||||
ha.primary_ctrl_fd = -1;
|
||||
set_last_activity_age(&ha, 60000L); /* stale */
|
||||
|
||||
/* Learned committed floor: config_tag=100. */
|
||||
atomic_store_explicit(&ha.committed_config_tag, 100ULL, memory_order_release);
|
||||
|
||||
int granted = handle_request_vote(&ha,
|
||||
1ULL,
|
||||
"ctrl-stale",
|
||||
50ULL, /* candidate config_tag — below learned floor */
|
||||
999ULL, /* high lsn — irrelevant once tag is below */
|
||||
0ULL, /* my local config_tag (cold) */
|
||||
0ULL, /* my local applied lsn (cold) */
|
||||
0);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted,
|
||||
"candidate below learned-committed config_tag floor must be DENIED");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_learned_committed_tag_denies_outdated_candidate: PASS\n");
|
||||
}
|
||||
|
||||
/* ── RED: equal-tag, lower learned-committed LSN denies ──────────────────── */
|
||||
|
||||
static void test_learned_committed_lsn_denies_lower_lsn_candidate(void)
|
||||
{
|
||||
/* Voter has learned the cluster committed up to LSN=100 (tag stays 0).
|
||||
* Candidate matches the effective tag but offers last_lsn=50 < 100 →
|
||||
* would lose committed entries → must be DENIED. */
|
||||
ha_state_t ha;
|
||||
ha_init(&ha, &g_gate_cfg, 0);
|
||||
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
||||
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
||||
ha.voted_for_term = 0;
|
||||
ha.primary_ctrl_fd = -1;
|
||||
set_last_activity_age(&ha, 60000L); /* stale */
|
||||
|
||||
/* Learned committed floor: lsn=100 (tag unchanged at 0). */
|
||||
atomic_store_explicit(&ha.last_known_committed_lsn, 100ULL, memory_order_release);
|
||||
|
||||
int granted = handle_request_vote(&ha,
|
||||
1ULL,
|
||||
"ctrl-laggard",
|
||||
0ULL, /* cand tag == effective tag (0) */
|
||||
50ULL, /* cand lsn < learned floor (100) */
|
||||
0ULL, /* my local config_tag */
|
||||
0ULL, /* my local applied lsn (cold) */
|
||||
0);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted,
|
||||
"equal-tag candidate below learned-committed LSN floor must be DENIED");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_learned_committed_lsn_denies_lower_lsn_candidate: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Liveness: an up-to-date candidate is still GRANTED ──────────────────── */
|
||||
|
||||
static void test_uptodate_candidate_still_granted_with_learned_floor(void)
|
||||
{
|
||||
/* The freshest node must still be able to win — no liveness regression.
|
||||
* Candidate meets the learned floor exactly (tag=100, lsn=100). */
|
||||
ha_state_t ha;
|
||||
ha_init(&ha, &g_gate_cfg, 0);
|
||||
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
||||
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
||||
ha.voted_for_term = 0;
|
||||
ha.primary_ctrl_fd = -1;
|
||||
set_last_activity_age(&ha, 60000L); /* stale */
|
||||
|
||||
atomic_store_explicit(&ha.committed_config_tag, 100ULL, memory_order_release);
|
||||
atomic_store_explicit(&ha.last_known_committed_lsn, 100ULL, memory_order_release);
|
||||
|
||||
int granted = handle_request_vote(&ha,
|
||||
1ULL,
|
||||
"ctrl-fresh",
|
||||
100ULL, /* meets learned tag floor */
|
||||
100ULL, /* meets learned lsn floor */
|
||||
0ULL, /* local cold */
|
||||
0ULL,
|
||||
0);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, granted,
|
||||
"candidate meeting the learned-committed floor must still be GRANTED (liveness)");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_uptodate_candidate_still_granted_with_learned_floor: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Backward-compat: no learned state (cold) falls back to local-only gate ─ */
|
||||
|
||||
static void test_no_learned_state_falls_back_to_local_gate(void)
|
||||
{
|
||||
/* committed_config_tag=0 and last_known_committed_lsn=0 → floor is the raw
|
||||
* local state, so behaviour is identical to the original local-only gate.
|
||||
* Local floor tag=100/lsn=50; candidate is below it → deny (unchanged). */
|
||||
ha_state_t ha;
|
||||
ha_init(&ha, &g_gate_cfg, 0);
|
||||
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
||||
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
||||
ha.voted_for_term = 0;
|
||||
ha.primary_ctrl_fd = -1;
|
||||
set_last_activity_age(&ha, 60000L); /* stale */
|
||||
|
||||
/* No learned state. */
|
||||
atomic_store_explicit(&ha.committed_config_tag, 0ULL, memory_order_release);
|
||||
atomic_store_explicit(&ha.last_known_committed_lsn, 0ULL, memory_order_release);
|
||||
|
||||
int denied = handle_request_vote(&ha,
|
||||
1ULL,
|
||||
"ctrl-old",
|
||||
99ULL, /* below local tag 100 */
|
||||
999ULL,
|
||||
100ULL, /* local tag */
|
||||
50ULL, /* local lsn */
|
||||
0);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, denied,
|
||||
"with no learned state the gate must behave as the local-only gate (deny lower tag)");
|
||||
|
||||
/* And an equally-fresh candidate (local-only) still grants. */
|
||||
ha_state_t ha2;
|
||||
ha_init(&ha2, &g_gate_cfg, 0);
|
||||
atomic_store_explicit(&ha2.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
||||
atomic_store_explicit(&ha2.term, 1ULL, memory_order_release);
|
||||
ha2.voted_for_term = 0;
|
||||
ha2.primary_ctrl_fd = -1;
|
||||
set_last_activity_age(&ha2, 60000L);
|
||||
|
||||
int granted = handle_request_vote(&ha2,
|
||||
1ULL,
|
||||
"ctrl-eq",
|
||||
100ULL, /* == local tag */
|
||||
50ULL, /* == local lsn */
|
||||
100ULL,
|
||||
50ULL,
|
||||
0);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, granted,
|
||||
"with no learned state an equally-fresh candidate still grants (local-only)");
|
||||
|
||||
ha_fini(&ha2);
|
||||
ha_fini(&ha);
|
||||
printf("test_no_learned_state_falls_back_to_local_gate: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test suite entry point ───────────────────────────────────────────────── */
|
||||
|
||||
void test_ha_vote_gate(void)
|
||||
|
|
@ -247,4 +449,11 @@ void test_ha_vote_gate(void)
|
|||
|
||||
/* Req 6.4 — ELECTING evaluates */
|
||||
RUN_TEST(test_electing_role_evaluates_vote);
|
||||
|
||||
/* Election-safety fix #2 — learned-committed freshness floor */
|
||||
RUN_TEST(test_vote_self_freshness_takes_max_of_local_and_learned);
|
||||
RUN_TEST(test_learned_committed_tag_denies_outdated_candidate);
|
||||
RUN_TEST(test_learned_committed_lsn_denies_lower_lsn_candidate);
|
||||
RUN_TEST(test_uptodate_candidate_still_granted_with_learned_floor);
|
||||
RUN_TEST(test_no_learned_state_falls_back_to_local_gate);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue