forked from lei.chen/myxce
Only 21 of 86 test_*.c group functions called their own nested UNITY_BEGIN()/UNITY_END(). Each nested call reset Unity's global NumberOfTests/TestFailures counters, so controller_test.c's final `return UNITY_END();` -- the value meson test actually gates on -- only ever reflected whichever small window of tests ran since the LAST nested reset, silently discarding pass/fail evidence for everything before it (measured: 16 of 972 total tests in one run). The same nested-BEGIN calls also caused PASS/FAIL banners to show the wrong source file for tests in files that don't reset it themselves (Unity.TestFile just carries over from whichever group last called UNITY_BEGIN()). Fix: replace every nested UNITY_BEGIN() with UnitySetTestFile(__FILE__) (sets file attribution only, no counter reset) and drop the matching UNITY_END(); add the same UnitySetTestFile(__FILE__) as the first statement to the other group functions that had neither. The 2 genuinely standalone binaries (test_job_array_sched_push.c, test_job_array_apiserver_push.c -- each has its own int main(), not registered via controller_test.c's TEST() macro) are untouched. Verified: single consolidated summary now reads "972 Tests 0 Failures 0 Ignored OK", exactly matching the ground-truth RUN_TEST call-site count across all 83 registered group files (976 total minus 4 belonging to the 2 standalone binaries). Every PASS/FAIL banner now names its own true source file. Full xce-build.sh all regression pass clean (Ok: 9, Fail: 0). No production code touched -- test-harness plumbing only.
1164 lines
53 KiB
C
1164 lines
53 KiB
C
/* test_ha_vote_gate.c — Unity tests for Task 3.2: role-based vote-deny gate
|
|
* in handle_request_vote().
|
|
*
|
|
* Tests verify (Req 6.1, 6.2, 6.4):
|
|
* Req 6.1: CTRL_PRIMARY → deny all vote requests (return 0)
|
|
* Req 6.1: CTRL_PRIMARY + fresher same-term challenger → still deny (return 0)
|
|
* Req 6.2: CTRL_STANDBY + fresh heartbeat → deny (return 0)
|
|
* Req 6.4: CTRL_STANDBY + stale heartbeat → evaluate Raft gates (not denied by role gate)
|
|
* Req 6.4: CTRL_ELECTING → evaluate Raft gates (not denied by role gate)
|
|
*
|
|
* Implementation gated on:
|
|
* - ha_heartbeat_is_fresh(ha): returns 1 if elapsed < heartbeat_timeout_ms
|
|
* - handle_request_vote() prepends role-based deny before existing Raft gates
|
|
*/
|
|
#define _GNU_SOURCE
|
|
#include <stdatomic.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <time.h>
|
|
#include <unistd.h>
|
|
|
|
#include <unity/unity.h>
|
|
|
|
#include "controller.capnp.h"
|
|
#include "ha.h" /* pulls in raft/raft.h: raft_state_t, RAFT_GRANT,
|
|
* raft_on_pre_vote_request, RAFT_PRE_VOTE_* */
|
|
#include "persist.h" /* persist_t, persist_durable_floor, persist_wal_async,
|
|
* persist_flush — task 4.2 durable-backed floor */
|
|
#include "wal_types.h" /* WAL_JOB_UPDATE */
|
|
|
|
/* Functions under test — declared here to match their signatures in ha.c */
|
|
int handle_request_vote(ha_state_t *ha,
|
|
uint64_t term,
|
|
const char *candidate_id,
|
|
uint64_t config_tag,
|
|
uint64_t last_lsn,
|
|
uint64_t my_config_tag,
|
|
uint64_t my_last_lsn,
|
|
int in_catch_up_flag);
|
|
|
|
/* Pure helper under test (election-safety fix #2): computes the voter's
|
|
* effective freshness floor as the max of its local state and its learned
|
|
* COMMITTED cluster state. */
|
|
void ha_vote_self_freshness(ha_state_t *ha,
|
|
uint64_t local_config_tag,
|
|
uint64_t local_applied_lsn,
|
|
uint64_t *out_tag,
|
|
uint64_t *out_lsn);
|
|
|
|
/* ── Shared config for vote gate tests ──────────────────────────────────── */
|
|
|
|
static ha_config_t g_gate_cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000, /* 10 s — used for ha_heartbeat_is_fresh */
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
.election_jitter_max_ms = 0,
|
|
.election_priority_step_ms = 500,
|
|
};
|
|
|
|
/* ── Helper: set ha->last_activity to "now - offset_ms milliseconds ago" ──
|
|
*
|
|
* last_activity is the production field written by ha_on_heartbeat_received
|
|
* (whole CLOCK_MONOTONIC seconds); offset_ms is rounded down to seconds. */
|
|
|
|
static void set_last_activity_age(ha_state_t *ha, long offset_ms)
|
|
{
|
|
struct timespec now;
|
|
clock_gettime(CLOCK_MONOTONIC, &now);
|
|
ha->last_activity = now.tv_sec - offset_ms / 1000;
|
|
}
|
|
|
|
/* ── Req 6.1: CTRL_PRIMARY → deny regardless of candidate freshness ────── */
|
|
|
|
static void test_primary_role_denies_vote_request(void)
|
|
{
|
|
/* When the receiving node is CTRL_PRIMARY it must deny all vote requests
|
|
* without evaluating term, config_tag, or LSN. The candidate is perfectly
|
|
* fresh (same config_tag, same LSN, same term) — still deny. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release);
|
|
atomic_store_explicit(&ha.term, 5ULL, memory_order_release);
|
|
ha.voted_for_term = 0;
|
|
|
|
int granted = handle_request_vote(&ha,
|
|
5ULL, /* same term — would pass term check */
|
|
"ctrl-b",
|
|
100ULL, /* same config_tag as mine */
|
|
50ULL, /* same LSN as mine */
|
|
100ULL, /* my config_tag */
|
|
50ULL, /* my last_lsn */
|
|
0 /* not in catch_up */);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted,
|
|
"CTRL_PRIMARY must deny all incoming vote requests (Req 6.1)");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_primary_role_denies_vote_request: PASS\n");
|
|
}
|
|
|
|
/* ── Req 6.1: CTRL_PRIMARY denies a fresher same-term challenger ─────────── */
|
|
|
|
static void test_primary_role_denies_fresher_same_term_challenger(void)
|
|
{
|
|
/* The vote-deny contract lives entirely on CTRL_PRIMARY (single leader
|
|
* role — there is no transitional leader sub-state). An elected
|
|
* primary must deny a same-term challenger even when the challenger looks
|
|
* FRESHER (higher config_tag / LSN): the role gate fires before any
|
|
* freshness evaluation, protecting the live primary from displacement. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY,
|
|
memory_order_release);
|
|
atomic_store_explicit(&ha.term, 3ULL, memory_order_release);
|
|
ha.voted_for_term = 0;
|
|
|
|
int granted = handle_request_vote(&ha,
|
|
3ULL, /* same term */
|
|
"ctrl-c",
|
|
200ULL, /* candidate is fresher */
|
|
100ULL,
|
|
100ULL, /* my config_tag */
|
|
50ULL,
|
|
0);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted,
|
|
"CTRL_PRIMARY must deny a fresher same-term challenger (Req 6.1)");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_primary_role_denies_fresher_same_term_challenger: PASS\n");
|
|
}
|
|
|
|
/* ── Req 6.2: CTRL_STANDBY + fresh heartbeat → deny ────────────────────── */
|
|
|
|
static void test_standby_fresh_heartbeat_denies_vote_request(void)
|
|
{
|
|
/* When the receiving node is CTRL_STANDBY and received a heartbeat recently
|
|
* (within heartbeat_timeout_ms = 10 s), ha_heartbeat_is_fresh() returns 1
|
|
* and the function must deny the vote — my primary is alive. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
atomic_store_explicit(&ha.term, 2ULL, memory_order_release);
|
|
ha.voted_for_term = 0;
|
|
|
|
/* Active primary control channel (handle_ctrlhello sets this on a real
|
|
* standby; the deny gate requires a live connection AND a fresh heartbeat).
|
|
* ha_fini does not close the fd, so any non-negative value is safe. */
|
|
ha.primary_ctrl_fd = 999;
|
|
|
|
/* last_activity = 500 ms ago → rounds to "this second" (< 10 s → fresh) */
|
|
set_last_activity_age(&ha, 500L);
|
|
|
|
int granted = handle_request_vote(&ha,
|
|
2ULL,
|
|
"ctrl-b",
|
|
100ULL,
|
|
50ULL,
|
|
100ULL,
|
|
50ULL,
|
|
0);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted,
|
|
"CTRL_STANDBY with fresh heartbeat must deny vote requests (Req 6.2)");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_standby_fresh_heartbeat_denies_vote_request: PASS\n");
|
|
}
|
|
|
|
/* ── Req 6.4: CTRL_STANDBY + stale heartbeat → evaluate Raft gates ──────── */
|
|
|
|
static void test_standby_stale_heartbeat_evaluates_vote(void)
|
|
{
|
|
/* When the receiving node is CTRL_STANDBY but the heartbeat is older than
|
|
* heartbeat_timeout_ms (10 s), ha_heartbeat_is_fresh() returns 0.
|
|
* The role gate must NOT deny — fall through to evaluate Raft gates.
|
|
*
|
|
* We set up a candidate with the same config_tag and LSN (freshness tie →
|
|
* grant) and no prior vote — the Raft logic should grant the vote. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
|
ha.voted_for_term = 0;
|
|
|
|
/* Active primary control channel — staleness alone must open the gate */
|
|
ha.primary_ctrl_fd = 999;
|
|
|
|
/* last_activity = 15 000 ms ago (> 10 000 ms timeout → stale) */
|
|
set_last_activity_age(&ha, 15000L);
|
|
|
|
int granted = handle_request_vote(&ha,
|
|
1ULL,
|
|
"ctrl-a",
|
|
100ULL, /* same config_tag — freshness tie */
|
|
50ULL, /* same LSN — freshness tie */
|
|
100ULL,
|
|
50ULL,
|
|
0);
|
|
|
|
/* With stale heartbeat the role gate does not deny; Raft gates evaluate
|
|
* and should grant (equal freshness, no prior vote). */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, granted,
|
|
"CTRL_STANDBY with stale heartbeat must evaluate Raft gates (Req 6.4)");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_standby_stale_heartbeat_evaluates_vote: PASS\n");
|
|
}
|
|
|
|
/* ── Req 6.4: CTRL_ELECTING → evaluate Raft gates ───────────────────────── */
|
|
|
|
static void test_electing_role_evaluates_vote(void)
|
|
{
|
|
/* When the receiving node is CTRL_ELECTING (it is itself campaigning),
|
|
* the role gate must not deny — fall through to Raft term/freshness gates.
|
|
*
|
|
* A valid candidate (same term, fresh config_tag, no prior vote) must be
|
|
* granted, exercising the full Raft evaluation path. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
|
|
atomic_store_explicit(&ha.term, 4ULL, memory_order_release);
|
|
ha.voted_for_term = 0;
|
|
|
|
int granted = handle_request_vote(&ha,
|
|
4ULL,
|
|
"ctrl-c",
|
|
300ULL, /* candidate fresher (higher tag) */
|
|
200ULL,
|
|
100ULL, /* my config_tag (lower) */
|
|
50ULL,
|
|
0);
|
|
|
|
/* Role gate does not fire; Raft gates evaluate; candidate is fresh → grant */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, granted,
|
|
"CTRL_ELECTING must evaluate Raft gates, not deny via role gate (Req 6.4)");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_electing_role_evaluates_vote: PASS\n");
|
|
}
|
|
|
|
/* ══ Election-safety fix #2: deny candidates outdated vs LEARNED committed state ══
|
|
*
|
|
* A voter must measure candidate freshness against the strongest knowledge it
|
|
* has of the cluster's COMMITTED state — not merely its own local config file /
|
|
* applied LSN. ha->committed_config_tag and ha->last_known_committed_lsn carry
|
|
* that learned-committed floor (from prior primary heartbeats / quorum ACK /
|
|
* post-election safe point). The gate floor is max(local, learned). */
|
|
|
|
/* ── Direct unit test of ha_vote_self_freshness (max logic both fields) ───── */
|
|
|
|
static void test_vote_self_freshness_takes_max_of_local_and_learned(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
|
|
/* learned-committed > local on tag, local > learned on lsn → expect the
|
|
* per-field max of each. */
|
|
atomic_store_explicit(&ha.committed_config_tag, 100ULL, memory_order_release);
|
|
atomic_store_explicit(&ha.last_known_committed_lsn, 20ULL, memory_order_release);
|
|
|
|
uint64_t out_tag = 0, out_lsn = 0;
|
|
ha_vote_self_freshness(&ha, 40ULL /*local tag*/, 70ULL /*local lsn*/,
|
|
&out_tag, &out_lsn);
|
|
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(100ULL, out_tag,
|
|
"freshness floor tag must be max(local=40, learned=100)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(70ULL, out_lsn,
|
|
"freshness floor lsn must be max(local=70, learned=20)");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_vote_self_freshness_takes_max_of_local_and_learned: PASS\n");
|
|
}
|
|
|
|
/* ── RED: learned-committed config_tag floor denies an outdated candidate ─── */
|
|
|
|
static void test_learned_committed_tag_denies_outdated_candidate(void)
|
|
{
|
|
/* Voter local config is cold (my_config_tag=0) but it has LEARNED that the
|
|
* cluster committed config_tag=100. A candidate offering config_tag=50
|
|
* would roll the cluster back below the committed floor → must be DENIED.
|
|
*
|
|
* Before the fix the gate used the raw local my_config_tag=0, so cand=50 ≥ 0
|
|
* granted (RED). After the fix the floor is max(0,100)=100 → deny (GREEN). */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
|
ha.voted_for_term = 0;
|
|
/* No active/fresh primary connection, not in catch-up → role gate opens. */
|
|
ha.primary_ctrl_fd = -1;
|
|
set_last_activity_age(&ha, 60000L); /* stale */
|
|
|
|
/* Learned committed floor: config_tag=100. */
|
|
atomic_store_explicit(&ha.committed_config_tag, 100ULL, memory_order_release);
|
|
|
|
int granted = handle_request_vote(&ha,
|
|
1ULL,
|
|
"ctrl-stale",
|
|
50ULL, /* candidate config_tag — below learned floor */
|
|
999ULL, /* high lsn — irrelevant once tag is below */
|
|
0ULL, /* my local config_tag (cold) */
|
|
0ULL, /* my local applied lsn (cold) */
|
|
0);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted,
|
|
"candidate below learned-committed config_tag floor must be DENIED");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_learned_committed_tag_denies_outdated_candidate: PASS\n");
|
|
}
|
|
|
|
/* ── RED: equal-tag, lower learned-committed LSN denies ──────────────────── */
|
|
|
|
static void test_learned_committed_lsn_denies_lower_lsn_candidate(void)
|
|
{
|
|
/* Voter has learned the cluster committed up to LSN=100 (tag stays 0).
|
|
* Candidate matches the effective tag but offers last_lsn=50 < 100 →
|
|
* would lose committed entries → must be DENIED. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
|
ha.voted_for_term = 0;
|
|
ha.primary_ctrl_fd = -1;
|
|
set_last_activity_age(&ha, 60000L); /* stale */
|
|
|
|
/* Learned committed floor: lsn=100 (tag unchanged at 0). */
|
|
atomic_store_explicit(&ha.last_known_committed_lsn, 100ULL, memory_order_release);
|
|
|
|
int granted = handle_request_vote(&ha,
|
|
1ULL,
|
|
"ctrl-laggard",
|
|
0ULL, /* cand tag == effective tag (0) */
|
|
50ULL, /* cand lsn < learned floor (100) */
|
|
0ULL, /* my local config_tag */
|
|
0ULL, /* my local applied lsn (cold) */
|
|
0);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted,
|
|
"equal-tag candidate below learned-committed LSN floor must be DENIED");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_learned_committed_lsn_denies_lower_lsn_candidate: PASS\n");
|
|
}
|
|
|
|
/* ── Liveness: an up-to-date candidate is still GRANTED ──────────────────── */
|
|
|
|
static void test_uptodate_candidate_still_granted_with_learned_floor(void)
|
|
{
|
|
/* The freshest node must still be able to win — no liveness regression.
|
|
* Candidate meets the learned floor exactly (tag=100, lsn=100). */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
|
ha.voted_for_term = 0;
|
|
ha.primary_ctrl_fd = -1;
|
|
set_last_activity_age(&ha, 60000L); /* stale */
|
|
|
|
atomic_store_explicit(&ha.committed_config_tag, 100ULL, memory_order_release);
|
|
atomic_store_explicit(&ha.last_known_committed_lsn, 100ULL, memory_order_release);
|
|
|
|
int granted = handle_request_vote(&ha,
|
|
1ULL,
|
|
"ctrl-fresh",
|
|
100ULL, /* meets learned tag floor */
|
|
100ULL, /* meets learned lsn floor */
|
|
0ULL, /* local cold */
|
|
0ULL,
|
|
0);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, granted,
|
|
"candidate meeting the learned-committed floor must still be GRANTED (liveness)");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_uptodate_candidate_still_granted_with_learned_floor: PASS\n");
|
|
}
|
|
|
|
/* ── Backward-compat: no learned state (cold) falls back to local-only gate ─ */
|
|
|
|
static void test_no_learned_state_falls_back_to_local_gate(void)
|
|
{
|
|
/* committed_config_tag=0 and last_known_committed_lsn=0 → floor is the raw
|
|
* local state, so behaviour is identical to the original local-only gate.
|
|
* Local floor tag=100/lsn=50; candidate is below it → deny (unchanged). */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
|
ha.voted_for_term = 0;
|
|
ha.primary_ctrl_fd = -1;
|
|
set_last_activity_age(&ha, 60000L); /* stale */
|
|
|
|
/* No learned state. */
|
|
atomic_store_explicit(&ha.committed_config_tag, 0ULL, memory_order_release);
|
|
atomic_store_explicit(&ha.last_known_committed_lsn, 0ULL, memory_order_release);
|
|
|
|
int denied = handle_request_vote(&ha,
|
|
1ULL,
|
|
"ctrl-old",
|
|
99ULL, /* below local tag 100 */
|
|
999ULL,
|
|
100ULL, /* local tag */
|
|
50ULL, /* local lsn */
|
|
0);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, denied,
|
|
"with no learned state the gate must behave as the local-only gate (deny lower tag)");
|
|
|
|
/* And an equally-fresh candidate (local-only) still grants. */
|
|
ha_state_t ha2;
|
|
ha_init(&ha2, &g_gate_cfg, 0);
|
|
atomic_store_explicit(&ha2.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
atomic_store_explicit(&ha2.term, 1ULL, memory_order_release);
|
|
ha2.voted_for_term = 0;
|
|
ha2.primary_ctrl_fd = -1;
|
|
set_last_activity_age(&ha2, 60000L);
|
|
|
|
int granted = handle_request_vote(&ha2,
|
|
1ULL,
|
|
"ctrl-eq",
|
|
100ULL, /* == local tag */
|
|
50ULL, /* == local lsn */
|
|
100ULL,
|
|
50ULL,
|
|
0);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, granted,
|
|
"with no learned state an equally-fresh candidate still grants (local-only)");
|
|
|
|
ha_fini(&ha2);
|
|
ha_fini(&ha);
|
|
printf("test_no_learned_state_falls_back_to_local_gate: PASS\n");
|
|
}
|
|
|
|
/* ══ Task 3.1: voter-side PreVote dispatch gate (cluster.c OPCode_preVoteRequest) ══
|
|
*
|
|
* The PreVote dispatch case in cluster.c is a thin composition of three
|
|
* primitives that all live in the unit-test binary:
|
|
* 1. ha_vote_self_freshness() — the freshness floor (Req 2.1, 2.2)
|
|
* 2. candidate-name → peer index mapping
|
|
* 3. raft_on_pre_vote_request() — stale-term + the SHARED eligibility gate
|
|
* (raft_cb_vote_eligible: PRIMARY-deny / fresh-heartbeat-deny / catch-up-deny;
|
|
* Req 3.1, 3.2, 3.3)
|
|
* cluster.c is not linked into the unit-test binary (transport deps), so we
|
|
* exercise the EXACT decision the dispatch composes via this mirror helper, then
|
|
* assert the dispatch's observable contract: the reply term is the voter's
|
|
* CURRENT term UNCHANGED (no term adoption) and no persistent state mutates.
|
|
*
|
|
* pre_vote_dispatch_decide reproduces cluster.c's OPCode_preVoteRequest decision
|
|
* byte-for-byte: freshness floor, then raft_on_pre_vote_request, with the reply
|
|
* term pinned to the voter's current term on every path (grant OR deny). */
|
|
|
|
static int pre_vote_dispatch_decide(ha_state_t *ha,
|
|
uint64_t cand_term,
|
|
uint64_t cand_tag,
|
|
uint64_t cand_lsn,
|
|
uint64_t local_tag,
|
|
uint64_t local_lsn,
|
|
int from_idx,
|
|
uint64_t *out_reply_term,
|
|
uint64_t *out_my_tag,
|
|
uint64_t *out_my_lsn)
|
|
{
|
|
/* Freshness floor: max(local, learned-committed), identical to the real-vote
|
|
* dispatch case. */
|
|
uint64_t my_tag = 0, my_lsn = 0;
|
|
ha_vote_self_freshness(ha, local_tag, local_lsn, &my_tag, &my_lsn);
|
|
|
|
int freshness_ok = !(cand_tag < my_tag ||
|
|
(cand_tag == my_tag && cand_lsn < my_lsn));
|
|
|
|
int granted = 0;
|
|
if (freshness_ok && from_idx >= 0)
|
|
/* ha-term-safety task 1.2: pass candidate (lastLogTerm, lastLogIndex).
|
|
* These gate tests do not exercise the logOk freshness path, so the
|
|
* tuple is (0,0); the library's local last-log term also reads 0 (the
|
|
* on_get_entry_term stub), so logOk grants and pre-existing gate
|
|
* behavior is unchanged. */
|
|
granted = (raft_on_pre_vote_request(&ha->raft, from_idx, cand_term,
|
|
0, 0)
|
|
== RAFT_GRANT) ? 1 : 0;
|
|
|
|
/* PreVote NEVER adopts the candidate term: the reply carries the voter's
|
|
* CURRENT term unchanged, on grant OR deny. */
|
|
if (out_reply_term)
|
|
*out_reply_term =
|
|
atomic_load_explicit(&ha->term, memory_order_acquire);
|
|
if (out_my_tag) *out_my_tag = my_tag;
|
|
if (out_my_lsn) *out_my_lsn = my_lsn;
|
|
return granted;
|
|
}
|
|
|
|
/* Snapshot of every field the PreVote handler must leave UNCHANGED. */
|
|
typedef struct {
|
|
uint64_t term;
|
|
uint64_t raft_term;
|
|
uint64_t voted_for_term;
|
|
char voted_for_name[XCE_MAXNAME_LEN];
|
|
int raft_voted_for;
|
|
int32_t role;
|
|
} prevote_state_snapshot_t;
|
|
|
|
static void snapshot_prevote_state(ha_state_t *ha, prevote_state_snapshot_t *s)
|
|
{
|
|
s->term = atomic_load_explicit(&ha->term, memory_order_acquire);
|
|
s->raft_term = ha->raft.term;
|
|
s->voted_for_term = ha->voted_for_term;
|
|
memcpy(s->voted_for_name, ha->voted_for_name, sizeof(s->voted_for_name));
|
|
s->raft_voted_for = ha->raft.voted_for;
|
|
s->role = atomic_load_explicit(&ha->role, memory_order_acquire);
|
|
}
|
|
|
|
static void assert_prevote_state_unchanged(ha_state_t *ha,
|
|
const prevote_state_snapshot_t *before)
|
|
{
|
|
prevote_state_snapshot_t after;
|
|
snapshot_prevote_state(ha, &after);
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(before->term, after.term,
|
|
"PreVote must NOT change ha->term (no term adoption)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(before->raft_term, after.raft_term,
|
|
"PreVote must NOT change raft.term (no term adoption)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(before->voted_for_term, after.voted_for_term,
|
|
"PreVote must NOT change voted_for_term (records no vote)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
memcmp(before->voted_for_name, after.voted_for_name,
|
|
sizeof(after.voted_for_name)),
|
|
"PreVote must NOT change voted_for_name (records no vote)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(before->raft_voted_for, after.raft_voted_for,
|
|
"PreVote must NOT change raft.voted_for (records no vote)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(before->role, after.role,
|
|
"PreVote must NOT change role");
|
|
}
|
|
|
|
/* ── Req 3.3: CTRL_PRIMARY denies a PreVote regardless of candidate freshness ─ */
|
|
|
|
static void test_pre_vote_primary_role_denies(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release);
|
|
atomic_store_explicit(&ha.term, 5ULL, memory_order_release);
|
|
ha.raft.term = 5ULL;
|
|
ha.voted_for_term = 0;
|
|
|
|
prevote_state_snapshot_t before;
|
|
snapshot_prevote_state(&ha, &before);
|
|
|
|
uint64_t reply_term = 0;
|
|
int granted = pre_vote_dispatch_decide(&ha,
|
|
5ULL, /* cand_term */
|
|
100ULL, /* cand_tag — perfectly fresh */
|
|
50ULL, /* cand_lsn */
|
|
100ULL, /* local tag */
|
|
50ULL, /* local lsn */
|
|
0, /* from_idx */
|
|
&reply_term, NULL, NULL);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted,
|
|
"CTRL_PRIMARY must deny all PreVote requests (Req 3.3)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, reply_term,
|
|
"denied PreVote reply carries the voter's UNCHANGED term (no adoption)");
|
|
assert_prevote_state_unchanged(&ha, &before);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_pre_vote_primary_role_denies: PASS\n");
|
|
}
|
|
|
|
/* ── Req 3.1: CTRL_STANDBY + fresh heartbeat denies a fresh PreVote ─────────── */
|
|
|
|
static void test_pre_vote_standby_fresh_heartbeat_denies(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
atomic_store_explicit(&ha.term, 2ULL, memory_order_release);
|
|
ha.raft.term = 2ULL;
|
|
ha.voted_for_term = 0;
|
|
ha.primary_ctrl_fd = 999; /* live primary connection */
|
|
set_last_activity_age(&ha, 500L); /* fresh heartbeat (< 10 s) */
|
|
|
|
prevote_state_snapshot_t before;
|
|
snapshot_prevote_state(&ha, &before);
|
|
|
|
uint64_t reply_term = 0;
|
|
int granted = pre_vote_dispatch_decide(&ha,
|
|
3ULL, /* cand wants term+1 = 3 */
|
|
100ULL, 50ULL, /* candidate is fresh */
|
|
100ULL, 50ULL,
|
|
0, &reply_term, NULL, NULL);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted,
|
|
"STANDBY with fresh heartbeat must deny PreVote — leader believed alive (Req 3.1)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(2ULL, reply_term,
|
|
"denied PreVote reply carries the voter's UNCHANGED term (no adoption)");
|
|
assert_prevote_state_unchanged(&ha, &before);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_pre_vote_standby_fresh_heartbeat_denies: PASS\n");
|
|
}
|
|
|
|
/* ── Req 3.1/3.2 (post raft-ha-core-safety task 5.3): catch-up standby vote
|
|
* gate is scoped to a LIVE primary, not unconditional ──────────────────────
|
|
*
|
|
* commit bf88681 (task 5.3, bug-1272) deliberately relaxed the in_catch_up
|
|
* deny in raft_cb_vote_eligible (ha.c): it now denies ONLY while a primary is
|
|
* actually LIVE (primary_ctrl_fd>=0 AND ha_heartbeat_is_fresh) feeding this
|
|
* node. Once the primary is unreachable, a catching-up survivor that meets
|
|
* the freshness floor MUST be grantable — otherwise two lagging survivors
|
|
* whose primary just died mutually deny every PreVote forever (the round-4
|
|
* election-stall this task fixed; see test_ha_core_safety.c's
|
|
* test_5_3_catchup_survivor_grants_fresh_candidate_when_primary_gone /
|
|
* test_5_3_catchup_denies_while_live_primary_feeds, which pin the same
|
|
* contract against the real raft_on_pre_vote_request path). The two cases
|
|
* below split what used to be a single unconditional-deny test. */
|
|
|
|
/* Was test_pre_vote_catch_up_denies — that name asserted the PRE-5.3
|
|
* unconditional deny, which 5.3 deliberately relaxed for exactly this
|
|
* scenario (primary unreachable + candidate meets the floor). Renamed to
|
|
* assert the current, intended contract instead of reverting the fix. */
|
|
static void test_pre_vote_catch_up_grants_when_primary_unreachable(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
atomic_store_explicit(&ha.term, 4ULL, memory_order_release);
|
|
ha.raft.term = 4ULL;
|
|
ha.voted_for_term = 0;
|
|
ha.primary_ctrl_fd = -1; /* primary is gone — nothing to catch up FROM */
|
|
set_last_activity_age(&ha, 60000L); /* stale heartbeat → not sticky either */
|
|
atomic_store_explicit(&ha.in_catch_up, 1, memory_order_release);
|
|
|
|
prevote_state_snapshot_t before;
|
|
snapshot_prevote_state(&ha, &before);
|
|
|
|
uint64_t reply_term = 0;
|
|
int granted = pre_vote_dispatch_decide(&ha,
|
|
5ULL,
|
|
100ULL, 50ULL, /* candidate meets the floor */
|
|
100ULL, 50ULL,
|
|
0, &reply_term, NULL, NULL);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, granted,
|
|
"a catching-up standby with NO live primary must GRANT a PreVote from a "
|
|
"candidate that meets the freshness floor (task 5.3 / bug-1272) — the "
|
|
"in_catch_up deny is scoped to a LIVE primary, not unconditional");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(4ULL, reply_term,
|
|
"granted PreVote reply carries the voter's UNCHANGED term (no adoption)");
|
|
assert_prevote_state_unchanged(&ha, &before);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_pre_vote_catch_up_grants_when_primary_unreachable: PASS\n");
|
|
}
|
|
|
|
/* Companion case preserving the ORIGINAL Req 3.x protective intent, correctly
|
|
* scoped: in_catch_up while a primary is ACTIVELY feeding this node (live fd +
|
|
* fresh heartbeat) must still deny — task 5.3 only lifts the deny once the
|
|
* primary is gone; it must never disrupt a primary that is alive and
|
|
* replicating. */
|
|
static void test_pre_vote_catch_up_denies_while_primary_live(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
atomic_store_explicit(&ha.term, 4ULL, memory_order_release);
|
|
ha.raft.term = 4ULL;
|
|
ha.voted_for_term = 0;
|
|
ha.primary_ctrl_fd = 999; /* a live primary connection */
|
|
set_last_activity_age(&ha, 500L); /* fresh heartbeat (< 10 s) */
|
|
atomic_store_explicit(&ha.in_catch_up, 1, memory_order_release);
|
|
|
|
prevote_state_snapshot_t before;
|
|
snapshot_prevote_state(&ha, &before);
|
|
|
|
uint64_t reply_term = 0;
|
|
int granted = pre_vote_dispatch_decide(&ha,
|
|
5ULL,
|
|
100ULL, 50ULL, /* fresh candidate */
|
|
100ULL, 50ULL,
|
|
0, &reply_term, NULL, NULL);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted,
|
|
"a catching-up standby with a LIVE primary still feeding it must DENY "
|
|
"PreVote (Req 3.x) — the 5.3 relaxation applies ONLY once the primary "
|
|
"is unreachable");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(4ULL, reply_term,
|
|
"denied PreVote reply carries the voter's UNCHANGED term (no adoption)");
|
|
assert_prevote_state_unchanged(&ha, &before);
|
|
|
|
ha.primary_ctrl_fd = -1; /* defensive: never let ha_fini touch fd 999 */
|
|
ha_fini(&ha);
|
|
printf("test_pre_vote_catch_up_denies_while_primary_live: PASS\n");
|
|
}
|
|
|
|
/* ── Req 2.2: stale-freshness candidate denied even when stickiness lapsed ──── */
|
|
|
|
static void test_pre_vote_stale_freshness_denies(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
|
ha.raft.term = 1ULL;
|
|
ha.voted_for_term = 0;
|
|
ha.primary_ctrl_fd = -1;
|
|
set_last_activity_age(&ha, 60000L); /* stale — stickiness lapsed */
|
|
|
|
/* Learned-committed floor: config_tag=100. */
|
|
atomic_store_explicit(&ha.committed_config_tag, 100ULL, memory_order_release);
|
|
|
|
prevote_state_snapshot_t before;
|
|
snapshot_prevote_state(&ha, &before);
|
|
|
|
uint64_t reply_term = 0;
|
|
int granted = pre_vote_dispatch_decide(&ha,
|
|
2ULL,
|
|
50ULL, /* cand_tag below learned floor 100 */
|
|
999ULL, /* high lsn — irrelevant */
|
|
0ULL, /* local cold */
|
|
0ULL,
|
|
0, &reply_term, NULL, NULL);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted,
|
|
"candidate below the freshness floor must be DENIED (Req 2.2)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(1ULL, reply_term,
|
|
"denied PreVote reply carries the voter's UNCHANGED term (no adoption)");
|
|
assert_prevote_state_unchanged(&ha, &before);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_pre_vote_stale_freshness_denies: PASS\n");
|
|
}
|
|
|
|
/* ── Req 2.3: stale-term candidate denied, no term adoption ─────────────────── */
|
|
|
|
static void test_pre_vote_stale_term_denies_no_adoption(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
atomic_store_explicit(&ha.term, 5ULL, memory_order_release);
|
|
ha.raft.term = 5ULL;
|
|
ha.voted_for_term = 0;
|
|
ha.primary_ctrl_fd = -1;
|
|
set_last_activity_age(&ha, 60000L);
|
|
|
|
prevote_state_snapshot_t before;
|
|
snapshot_prevote_state(&ha, &before);
|
|
|
|
uint64_t reply_term = 0;
|
|
int granted = pre_vote_dispatch_decide(&ha,
|
|
3ULL, /* cand_term < raft.term (5) → stale */
|
|
100ULL, 50ULL,
|
|
100ULL, 50ULL,
|
|
0, &reply_term, NULL, NULL);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted,
|
|
"a stale-term PreVote candidate must be DENIED (Req 2.3)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, reply_term,
|
|
"stale-term PreVote reply carries the voter's UNCHANGED higher term");
|
|
assert_prevote_state_unchanged(&ha, &before);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_pre_vote_stale_term_denies_no_adoption: PASS\n");
|
|
}
|
|
|
|
/* ── Req 2.1/3.2/5.1: stickiness lapsed + fresh candidate → GRANT, no mutation ─ */
|
|
|
|
static void test_pre_vote_granted_when_stickiness_lapsed_and_fresh(void)
|
|
{
|
|
/* The genuine-failover case: no live primary, heartbeat stale (stickiness
|
|
* lapsed), candidate proposes term+1 and meets the freshness floor → GRANT.
|
|
* The grant must still mutate NOTHING (no voted_for, no term, no role). */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
atomic_store_explicit(&ha.term, 3ULL, memory_order_release);
|
|
ha.raft.term = 3ULL;
|
|
ha.voted_for_term = 0;
|
|
ha.primary_ctrl_fd = -1;
|
|
set_last_activity_age(&ha, 60000L); /* stale — stickiness lapsed */
|
|
|
|
prevote_state_snapshot_t before;
|
|
snapshot_prevote_state(&ha, &before);
|
|
|
|
uint64_t reply_term = 0, my_tag = 0, my_lsn = 0;
|
|
int granted = pre_vote_dispatch_decide(&ha,
|
|
4ULL, /* prospective term = current+1 */
|
|
100ULL, 50ULL, /* meets local floor */
|
|
100ULL, 50ULL,
|
|
0, &reply_term, &my_tag, &my_lsn);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, granted,
|
|
"fresh candidate after stickiness lapse must be GRANTED a PreVote (Req 5.1)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(3ULL, reply_term,
|
|
"granted PreVote reply STILL carries the voter's UNCHANGED term (no adoption)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(100ULL, my_tag,
|
|
"reply must carry the voter's freshness config_tag");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(50ULL, my_lsn,
|
|
"reply must carry the voter's freshness last_log_lsn");
|
|
assert_prevote_state_unchanged(&ha, &before);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_pre_vote_granted_when_stickiness_lapsed_and_fresh: PASS\n");
|
|
}
|
|
|
|
/* ── bug-660: dead primary (fd<0) must GRANT even when last_activity is fresh ── */
|
|
|
|
static void test_pre_vote_dead_primary_grants_despite_fresh_activity(void)
|
|
{
|
|
/* The convergence bug: after the primary DIED (primary_ctrl_fd < 0) the
|
|
* standby's last_activity stays falsely "fresh" because the failover
|
|
* watchdog (ha_arm_heartbeat_timeout_vtask) re-arms it to now on every retry.
|
|
* With NO live primary connection the node must GRANT a fresh higher-term
|
|
* PreVote so the cluster converges in ONE round instead of denying with
|
|
* reason=fresh_heartbeat for minutes (the observed 11-round / 3-min stall).
|
|
* Distinguishes from test_pre_vote_granted_when_stickiness_lapsed_and_fresh
|
|
* (which has a STALE heartbeat): here the heartbeat looks FRESH but there is
|
|
* no primary connection, so the fix's primary_ctrl_fd>=0 guard is what makes
|
|
* it grant. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
atomic_store_explicit(&ha.term, 3ULL, memory_order_release);
|
|
ha.raft.term = 3ULL;
|
|
ha.voted_for_term = 0;
|
|
ha.primary_ctrl_fd = -1; /* primary connection dropped (RDHUP) */
|
|
set_last_activity_age(&ha, 500L); /* last_activity FALSELY fresh (watchdog re-arm) */
|
|
|
|
prevote_state_snapshot_t before;
|
|
snapshot_prevote_state(&ha, &before);
|
|
|
|
uint64_t reply_term = 0;
|
|
int granted = pre_vote_dispatch_decide(&ha,
|
|
4ULL, /* prospective term = current+1 */
|
|
100ULL, 50ULL, /* candidate meets the floor */
|
|
100ULL, 50ULL,
|
|
0, &reply_term, NULL, NULL);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, granted,
|
|
"a STANDBY whose primary connection is gone (fd<0) must GRANT the PreVote "
|
|
"even if last_activity looks fresh — else convergence stalls (bug-660)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(3ULL, reply_term,
|
|
"granted PreVote reply carries the voter's UNCHANGED term (no adoption)");
|
|
assert_prevote_state_unchanged(&ha, &before);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_pre_vote_dead_primary_grants_despite_fresh_activity: PASS\n");
|
|
}
|
|
|
|
/* ══════════════════════════════════════════════════════════════════════════
|
|
* ha-commit-durability task 4.2 — durable-backed freshness floor (Req 2.3)
|
|
*
|
|
* Contract (design §1a/§1b/§5):
|
|
* - CANDIDATE advertises BARE persist_durable_floor on the LSN axis (NOT
|
|
* folded with the volatile last_known_committed_lsn) — a truncated-restart
|
|
* node cannot over-advertise a stale-high learned value and win.
|
|
* - VOTER floor = max(durable_floor, last_known_committed_lsn): the local
|
|
* state input to ha_vote_self_freshness becomes the DURABLE tip, the
|
|
* learned-committed backstop is retained on the voter axis (bug-339).
|
|
* ════════════════════════════════════════════════════════════════════════ */
|
|
|
|
uint64_t ha_candidate_advertise_lsn(persist_t *p);
|
|
|
|
/* Build a real persist whose durable floor == `target` by appending `target`
|
|
* entries (LSN 1..target) then explicitly flushing. wal_sync_ms is large so
|
|
* the ONLY durability point is the explicit persist_flush — the durable floor
|
|
* is deterministic, not racing the background timer. */
|
|
static persist_t *make_durable_persist(const char *tmpdir_template,
|
|
uint64_t target)
|
|
{
|
|
char tmp[256];
|
|
strncpy(tmp, tmpdir_template, sizeof(tmp) - 1);
|
|
tmp[sizeof(tmp) - 1] = '\0';
|
|
if (!mkdtemp(tmp))
|
|
return NULL;
|
|
|
|
persist_t *p = calloc(1, sizeof(persist_t));
|
|
if (!p)
|
|
return NULL;
|
|
|
|
persist_config_t cfg = persist_config_defaults();
|
|
cfg.wal_sync_ms = 5000;
|
|
cfg.snap_keep_n = 2;
|
|
|
|
persist_set_workdir(p, tmp, "ctrl-test");
|
|
if (persist_init(p, &cfg, NULL) != 0) {
|
|
free(p);
|
|
return NULL;
|
|
}
|
|
if (persist_start(p) != 0) {
|
|
persist_fini(p);
|
|
free(p);
|
|
return NULL;
|
|
}
|
|
for (uint64_t i = 0; i < target; i++) {
|
|
if (persist_wal_async(p, WAL_JOB_UPDATE, "x", 1) != 0) {
|
|
persist_stop(p);
|
|
persist_fini(p);
|
|
free(p);
|
|
return NULL;
|
|
}
|
|
}
|
|
if (target > 0)
|
|
persist_flush(p);
|
|
return p;
|
|
}
|
|
|
|
static void free_durable_persist(persist_t *p)
|
|
{
|
|
if (!p)
|
|
return;
|
|
persist_stop(p);
|
|
persist_fini(p);
|
|
free(p);
|
|
}
|
|
|
|
/* ── RED: candidate advertises BARE durable_floor, never the stale learned ──
|
|
*
|
|
* A node restarted with a TRUNCATED log holds durable_floor=4 but still has a
|
|
* stale volatile last_known_committed_lsn=8 in memory. The candidate MUST
|
|
* advertise 4 (its durable tip), NOT 8. If ha_candidate_advertise_lsn folded
|
|
* in learned_committed (the bug), it would return 8 and the truncated node
|
|
* could win — exactly the hole this task closes (design §5a). */
|
|
static void test_candidate_advertises_bare_durable_floor_not_learned(void)
|
|
{
|
|
persist_t *p = make_durable_persist("/tmp/xce_42cand_XXXXXX", 4);
|
|
TEST_ASSERT_NOT_NULL(p);
|
|
TEST_ASSERT_EQUAL_UINT64(4ULL, persist_durable_floor(p));
|
|
|
|
/* The advertised LSN must be the BARE durable tip — independent of any
|
|
* volatile learned-committed state a real ha_state_t might carry. */
|
|
uint64_t advertised = ha_candidate_advertise_lsn(p);
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(4ULL, advertised,
|
|
"candidate must advertise the BARE durable WAL tip (4), never a stale "
|
|
"volatile learned_committed value — over-advertising lets a truncated "
|
|
"node win (design §1a/§5a)");
|
|
|
|
free_durable_persist(p);
|
|
printf("test_candidate_advertises_bare_durable_floor_not_learned: PASS\n");
|
|
}
|
|
|
|
/* ── RED: voter denies a candidate advertising below the durable voter floor ─
|
|
*
|
|
* Voter holds durable_floor=8 (its real durable tip) at config_tag=10. A
|
|
* truncated candidate advertises its bare durable tip cand_lsn=4 at the same
|
|
* tag. The voter floor's local-state LSN is now the DURABLE tip (8), so
|
|
* cand_lsn(4) < floor(8) ⇒ DENY. Mirrors the production cluster.c gate using
|
|
* persist_durable_floor as the local input to ha_vote_self_freshness. */
|
|
static void test_voter_floor_uses_durable_tip_denies_truncated(void)
|
|
{
|
|
persist_t *p = make_durable_persist("/tmp/xce_42vdeny_XXXXXX", 8);
|
|
TEST_ASSERT_NOT_NULL(p);
|
|
TEST_ASSERT_EQUAL_UINT64(8ULL, persist_durable_floor(p));
|
|
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
/* No learned-committed state primed: the floor must come from the durable
|
|
* tip alone, proving the local-state input is durable-backed (not the
|
|
* volatile applied_lsn, which this fixture never advances). */
|
|
uint64_t my_tag = 0, my_lsn = 0;
|
|
ha_vote_self_freshness(&ha, 10ULL /*local tag*/,
|
|
persist_durable_floor(p) /*durable local LSN*/,
|
|
&my_tag, &my_lsn);
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(8ULL, my_lsn,
|
|
"voter floor LSN must reflect the DURABLE tip (8) fed as local input");
|
|
|
|
/* Truncated candidate advertises its bare durable tip = 4 at equal tag. */
|
|
uint64_t cand_tag = 10ULL, cand_lsn = 4ULL;
|
|
int freshness_ok = !(cand_tag < my_tag ||
|
|
(cand_tag == my_tag && cand_lsn < my_lsn));
|
|
TEST_ASSERT_FALSE_MESSAGE(freshness_ok,
|
|
"a truncated candidate (durable=4) must be DENIED against a voter whose "
|
|
"durable floor is 8 (design §5a)");
|
|
|
|
ha_fini(&ha);
|
|
free_durable_persist(p);
|
|
printf("test_voter_floor_uses_durable_tip_denies_truncated: PASS\n");
|
|
}
|
|
|
|
/* ── RED: longest-durable candidate still WINS — no livelock (design §5b/§4) ─
|
|
*
|
|
* Two survivors: A durable=8, B durable=7, equal config_tag. A advertises its
|
|
* bare durable tip 8; B's voter floor (durable 7, no higher learned) is 7, so
|
|
* A(8) >= B(7) ⇒ B GRANTS A. Symmetrically A denies B (B's 7 < A's floor 8),
|
|
* steering the election to the most-durable node — no mutual-deny livelock. */
|
|
static void test_longest_durable_candidate_grantable_no_livelock(void)
|
|
{
|
|
persist_t *a = make_durable_persist("/tmp/xce_42winA_XXXXXX", 8);
|
|
persist_t *b = make_durable_persist("/tmp/xce_42winB_XXXXXX", 7);
|
|
TEST_ASSERT_NOT_NULL(a);
|
|
TEST_ASSERT_NOT_NULL(b);
|
|
TEST_ASSERT_EQUAL_UINT64(8ULL, persist_durable_floor(a));
|
|
TEST_ASSERT_EQUAL_UINT64(7ULL, persist_durable_floor(b));
|
|
|
|
uint64_t cand_lsn_A = ha_candidate_advertise_lsn(a); /* 8 */
|
|
|
|
/* Voter B's floor from its own durable tip (no learned-committed primed). */
|
|
ha_state_t hb;
|
|
ha_init(&hb, &g_gate_cfg, 0);
|
|
uint64_t b_tag = 0, b_floor = 0;
|
|
ha_vote_self_freshness(&hb, 10ULL, persist_durable_floor(b), &b_tag, &b_floor);
|
|
|
|
int b_grants_A = !(10ULL < b_tag || (10ULL == b_tag && cand_lsn_A < b_floor));
|
|
TEST_ASSERT_TRUE_MESSAGE(b_grants_A,
|
|
"the longest-durable candidate A(8) must be grantable by B(floor 7) — "
|
|
"the floor is satisfiable by the most-durable reachable node, so no "
|
|
"livelock (design §4/§5b)");
|
|
|
|
/* Symmetric: A denies the less-durable B (steers election to A). */
|
|
uint64_t cand_lsn_B = ha_candidate_advertise_lsn(b); /* 7 */
|
|
ha_state_t haa;
|
|
ha_init(&haa, &g_gate_cfg, 0);
|
|
uint64_t a_tag = 0, a_floor = 0;
|
|
ha_vote_self_freshness(&haa, 10ULL, persist_durable_floor(a), &a_tag, &a_floor);
|
|
int a_grants_B = !(10ULL < a_tag || (10ULL == a_tag && cand_lsn_B < a_floor));
|
|
TEST_ASSERT_FALSE_MESSAGE(a_grants_B,
|
|
"the less-durable candidate B(7) must be DENIED by A(floor 8) — election "
|
|
"steers to the most-durable node");
|
|
|
|
ha_fini(&hb);
|
|
ha_fini(&haa);
|
|
free_durable_persist(a);
|
|
free_durable_persist(b);
|
|
printf("test_longest_durable_candidate_grantable_no_livelock: PASS\n");
|
|
}
|
|
|
|
/* ══════════════════════════════════════════════════════════════════════════
|
|
* bug-828 part 2 — a standby caught up POST-BOOT must advertise its LIVE durable
|
|
* tip, not its (lower) boot floor, and that live tip must DOMINATE a lagging
|
|
* learned_committed.
|
|
*
|
|
* Live failover evidence (fix828c_r1..r3): after a clean failover the two
|
|
* surviving voters advertised position 1 and ha_compute_safe_committed_lsn
|
|
* returned safe=1 — but in those runs the standbys were GENUINELY durable only
|
|
* at LSN 1 (their WAL physically held 1..2; the committed-prefix loss is the
|
|
* UPSTREAM replication/commit-barrier durability gap bug-795/661, where the
|
|
* primary advanced committed_lsn to 5 before the surviving majority was durable
|
|
* there — out of scope here).
|
|
*
|
|
* This test pins the IN-SCOPE contract: GIVEN a standby that IS durable at its
|
|
* live tip (durable_lsn=5, advanced via persist_wal_write_replicated's inline
|
|
* fdatasync → the part-1 secondary fix, mirrored here by make_durable_persist's
|
|
* explicit flush), even though its learned_committed lags at 1, it must:
|
|
* (a) ADVERTISE 5 as both the candidate self-LSN (ha_candidate_advertise_lsn)
|
|
* and the voter floor LSN (ha_vote_self_freshness over persist_durable_floor),
|
|
* (b) so two such granting voters drive ha_compute_safe_committed_lsn to 5,
|
|
* NOT to the lagging learned_committed (1).
|
|
*
|
|
* SAFETY: the advertised position is the DURABLE (fdatasync'd) tip — a position
|
|
* the node can back from its own WAL — never a merely-received/appended tip.
|
|
* persist_durable_lsn == persist_durable_floor (both read p->durable_lsn, the
|
|
* fsync watermark); make_durable_persist advances it only via persist_flush, so
|
|
* the advertised 5 is fully durable.
|
|
* ════════════════════════════════════════════════════════════════════════ */
|
|
|
|
static void test_live_durable_tip_advertised_over_lagging_learned(void)
|
|
{
|
|
/* Real persist durable at LSN 5 (explicit flush → durable_lsn=5). This is the
|
|
* caught-up standby: durable tip 5, learned_committed will lag at 1. */
|
|
persist_t *p = make_durable_persist("/tmp/xce_828p2_XXXXXX", 5);
|
|
TEST_ASSERT_NOT_NULL(p);
|
|
TEST_ASSERT_EQUAL_UINT64(5ULL, persist_durable_floor(p));
|
|
TEST_ASSERT_EQUAL_UINT64(5ULL, persist_durable_lsn(p)); /* same field */
|
|
|
|
/* (a) Candidate self-advertise = the BARE durable tip = 5 (never the boot
|
|
* floor, never the lagging learned value). */
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, ha_candidate_advertise_lsn(p),
|
|
"candidate must advertise its LIVE durable tip (5), the fsync'd watermark");
|
|
|
|
/* (a) Voter floor LSN over the durable tip, with learned_committed LAGGING
|
|
* at 1: max(durable_floor=5, learned=1) = 5. The durable tip dominates. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_gate_cfg, 0);
|
|
atomic_store_explicit(&ha.committed_config_tag, 100ULL, memory_order_release);
|
|
atomic_store_explicit(&ha.last_known_committed_lsn, 1ULL, memory_order_release);
|
|
|
|
uint64_t my_tag = 0, my_lsn = 0;
|
|
ha_vote_self_freshness(&ha, 100ULL /*local tag*/,
|
|
persist_durable_floor(p) /*durable local LSN = 5*/,
|
|
&my_tag, &my_lsn);
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, my_lsn,
|
|
"voter floor LSN must be the LIVE durable tip (5), dominating a lagging "
|
|
"learned_committed (1) — this is what RequestVoteReply.peerLastLogLsn "
|
|
"advertises");
|
|
|
|
/* (b) Two granting voters each advertising 5 must drive the post-election
|
|
* safe-commit to 5, NOT to the lagging learned (1). Build a 3-node election
|
|
* ctx: self (the catch-up candidate, durable 5) + 2 peers both advertising 5. */
|
|
ha_elect_ctx_t *ctx =
|
|
calloc(1, sizeof(*ctx) + 2 * sizeof(ctx->votes[0]));
|
|
TEST_ASSERT_NOT_NULL(ctx);
|
|
ctx->n_peers = 2;
|
|
ctx->self_lsn = ha_candidate_advertise_lsn(p); /* 5 */
|
|
ctx->self_tag = 100ULL;
|
|
ctx->votes[0].vote_granted = 1;
|
|
ctx->votes[0].peer_lsn = 5ULL;
|
|
ctx->votes[0].peer_tag = 100ULL;
|
|
ctx->votes[1].vote_granted = 1;
|
|
ctx->votes[1].peer_lsn = 5ULL;
|
|
ctx->votes[1].peer_tag = 100ULL;
|
|
|
|
uint64_t safe_lsn = 0, safe_tag = 0;
|
|
ha_compute_safe_committed_lsn(ctx, &safe_lsn, &safe_tag);
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, safe_lsn,
|
|
"safe_committed_lsn must reflect the majority-durable tip (5), not the "
|
|
"lagging learned_committed (1) — the durable tip dominates");
|
|
|
|
free(ctx);
|
|
ha_fini(&ha);
|
|
free_durable_persist(p);
|
|
printf("test_live_durable_tip_advertised_over_lagging_learned: PASS\n");
|
|
}
|
|
|
|
/* ── Test suite entry point ───────────────────────────────────────────────── */
|
|
|
|
void test_ha_vote_gate(void)
|
|
{
|
|
UnitySetTestFile(__FILE__);
|
|
/* Req 6.1 — PRIMARY denies */
|
|
RUN_TEST(test_primary_role_denies_vote_request);
|
|
|
|
/* Req 6.1 — PRIMARY denies a fresher same-term challenger */
|
|
RUN_TEST(test_primary_role_denies_fresher_same_term_challenger);
|
|
|
|
/* Req 6.2 — STANDBY + fresh heartbeat denies */
|
|
RUN_TEST(test_standby_fresh_heartbeat_denies_vote_request);
|
|
|
|
/* Req 6.4 — STANDBY + stale heartbeat evaluates */
|
|
RUN_TEST(test_standby_stale_heartbeat_evaluates_vote);
|
|
|
|
/* Req 6.4 — ELECTING evaluates */
|
|
RUN_TEST(test_electing_role_evaluates_vote);
|
|
|
|
/* Election-safety fix #2 — learned-committed freshness floor */
|
|
RUN_TEST(test_vote_self_freshness_takes_max_of_local_and_learned);
|
|
RUN_TEST(test_learned_committed_tag_denies_outdated_candidate);
|
|
RUN_TEST(test_learned_committed_lsn_denies_lower_lsn_candidate);
|
|
RUN_TEST(test_uptodate_candidate_still_granted_with_learned_floor);
|
|
RUN_TEST(test_no_learned_state_falls_back_to_local_gate);
|
|
|
|
/* ha-commit-durability task 4.2 — durable-backed freshness floor (Req 2.3) */
|
|
RUN_TEST(test_candidate_advertises_bare_durable_floor_not_learned);
|
|
RUN_TEST(test_voter_floor_uses_durable_tip_denies_truncated);
|
|
RUN_TEST(test_longest_durable_candidate_grantable_no_livelock);
|
|
|
|
/* bug-828 part 2 — a caught-up standby advertises its LIVE durable tip
|
|
* (dominating a lagging learned_committed); safe-commit-from-votes reaches it */
|
|
RUN_TEST(test_live_durable_tip_advertised_over_lagging_learned);
|
|
|
|
/* Task 3.1 — voter-side PreVote dispatch gate (stickiness + freshness,
|
|
* no mutation, reply term unchanged) */
|
|
RUN_TEST(test_pre_vote_primary_role_denies);
|
|
RUN_TEST(test_pre_vote_standby_fresh_heartbeat_denies);
|
|
RUN_TEST(test_pre_vote_catch_up_grants_when_primary_unreachable);
|
|
RUN_TEST(test_pre_vote_catch_up_denies_while_primary_live);
|
|
RUN_TEST(test_pre_vote_stale_freshness_denies);
|
|
RUN_TEST(test_pre_vote_stale_term_denies_no_adoption);
|
|
RUN_TEST(test_pre_vote_granted_when_stickiness_lapsed_and_fresh);
|
|
|
|
/* bug-660 — dead primary (fd<0) must GRANT even when last_activity is fresh */
|
|
RUN_TEST(test_pre_vote_dead_primary_grants_despite_fresh_activity);
|
|
}
|