Rongsong Shen
13c43c23a4
Adds two controller-level regression tests driving the production raft_on_vote_reply
via ha->raft (test_ha_election.c): a duplicated vote-grant from one peer must not
double-count toward quorum (bug-1095) and a stale-term reply must be dropped
(bug-1096) — both asserted on role/promotion (no two leaders per term). Formally
absorbs bug-1095/1096 into the ha-term-safety spec (Implementation Note) mapping to
its Requirement 1 / Objective; the raft.c fix (b5b7e01) is owned there with
protocol-suite regression at both the libxce (test_vote_reply_*) and controller
(test_ha_election) levels. No production source changed.
1285 lines
58 KiB
C
1285 lines
58 KiB
C
/* test_ha_election.c — Unity RED-phase tests for Raft-style quorum election logic.
|
|
*
|
|
* Tests verify:
|
|
* Req 2.1: 2/3 votes received → CTRL_PRIMARY
|
|
* Req 2.2: Lower-term vote request → deny
|
|
* Req 2.3: Higher-term vote request → update term, clear voted_for, grant
|
|
* Req 2.4: Freshness gate: peer has higher config_tag or equal-tag+higher LSN → deny
|
|
* Req 2.5: in_catch_up=1 standby votes are denied
|
|
* Req 2.6: voted_for enforcement: deny second candidate in same term
|
|
* Req 2.7: Fresher-peer yield gate: candidate wins votes but a responding peer is
|
|
* fresher → yield to that peer → CTRL_STANDBY
|
|
* Req 2.8: config_tag=0 candidate with peer having non-zero tag → fatal, not PRIMARY
|
|
* Req 2.9: Single-node self-elects immediately without any network calls
|
|
* Req 2.10: Split vote → retry with incremented term
|
|
* Req 2.11: Second election also fails quorum → CTRL_STANDBY
|
|
* Req 2.12: Identical tag/LSN peers → jitter tiebreaker (randomised, no priority)
|
|
*
|
|
* Functions under test:
|
|
* handle_request_vote() — per-request vote handler (not yet in ha.h)
|
|
* ha_elect_evaluate_and_signal() — evaluates accumulated votes; signals done sem
|
|
* am_i_primary_quorum() — top-level quorum election entrypoint
|
|
*
|
|
* These functions do NOT exist yet. Linking will fail with "undefined reference"
|
|
* which is the expected RED-phase proof for task 4.1.
|
|
*/
|
|
#define _GNU_SOURCE
|
|
#include <errno.h>
|
|
#include <semaphore.h>
|
|
#include <stdatomic.h>
|
|
#include <stdbool.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <time.h>
|
|
#include <unistd.h>
|
|
|
|
#include <unity/unity.h>
|
|
|
|
#include "controller.capnp.h"
|
|
#include "ha.h"
|
|
|
|
/* ── Forward declarations for functions not yet implemented ──────────────────
|
|
*
|
|
* These extern declarations cause linker-level "undefined reference" failures
|
|
* when the functions are absent, which is the required RED-phase proof.
|
|
*
|
|
* handle_request_vote:
|
|
* Called by the vtaskmgr dispatch loop when an OP_requestVote message arrives.
|
|
* Checks term, voted_for, and freshness gate; updates ha state; grants or
|
|
* denies the vote; persists term state via ha_persist_term().
|
|
*
|
|
* Parameters:
|
|
* ha — the receiving controller's ha_state_t (vote state modified)
|
|
* term — candidate's proposed term
|
|
* candidate_id — candidate's controller name (NUL-terminated)
|
|
* config_tag — candidate's config_tag at time of request
|
|
* last_lsn — candidate's highest applied WAL LSN
|
|
* my_config_tag — this controller's current config_tag
|
|
* my_last_lsn — this controller's current WAL LSN
|
|
* in_catch_up_flag — non-zero if this node is currently catching up
|
|
*
|
|
* Returns:
|
|
* 1 — vote granted
|
|
* 0 — vote denied (term too low, already voted, not fresh enough, catch-up)
|
|
* <0 — error
|
|
*
|
|
* ha_elect_evaluate_and_signal:
|
|
* Called by each per-peer vote-reply thread as replies accumulate in ctx.
|
|
* Evaluates the current vote tally; if quorum is reached (or all peers replied),
|
|
* sets ctx->result and posts ctx->done.
|
|
* Also applies the fresher-peer yield gate (req 2.7).
|
|
*
|
|
* Parameters:
|
|
* ha — the candidate's ha_state_t (term read for fencing)
|
|
* ctx — the election context accumulating vote state
|
|
*
|
|
* Returns:
|
|
* CTRL_PRIMARY — quorum won and no fresher responding peer
|
|
* CTRL_STANDBY — quorum lost or fresher peer yields (ctx->result already set)
|
|
*
|
|
* am_i_primary_quorum:
|
|
* Top-level election entrypoint called from myapp.c after the heartbeat
|
|
* timeout fires. Increments term, allocates ha_elect_ctx_t, spawns one
|
|
* pthread per peer to send OP_requestVote, blocks on ctx->done, evaluates
|
|
* the result, and returns the final role.
|
|
*
|
|
* For single-node clusters (n_peers == 1): immediately returns CTRL_PRIMARY
|
|
* without any network calls (req 2.9).
|
|
*
|
|
* Parameters:
|
|
* ha — the candidate's ha_state_t (term + votes modified)
|
|
* n_peers — total controller count INCLUDING self
|
|
* peers — array of xce_candidate_t* for the other controllers (n_peers-1)
|
|
* my_config_tag — local config_tag at election time
|
|
* my_lsn — local WAL LSN at election time
|
|
*
|
|
* Returns:
|
|
* CTRL_PRIMARY — won election and no fresher peer
|
|
* CTRL_STANDBY — lost election (quorum not reached, fresher peer, or fatal)
|
|
*/
|
|
int handle_request_vote(ha_state_t *ha, uint64_t term,
|
|
const char *candidate_id,
|
|
uint64_t config_tag, uint64_t last_lsn,
|
|
uint64_t my_config_tag, uint64_t my_last_lsn,
|
|
int in_catch_up_flag);
|
|
|
|
ctrl_role_t ha_elect_evaluate_and_signal(ha_state_t *ha, ha_elect_ctx_t *ctx);
|
|
|
|
ctrl_role_t am_i_primary_quorum(ha_state_t *ha, int n_peers,
|
|
xce_candidate_t **peers,
|
|
uint64_t my_config_tag, uint64_t my_lsn,
|
|
const char *my_name);
|
|
|
|
void handle_request_vote_reply(ha_state_t *ha, ha_elect_ctx_t *ctx,
|
|
int peer_idx, uint64_t term,
|
|
int vote_granted,
|
|
uint64_t peer_config_tag, uint64_t peer_lsn);
|
|
|
|
/* bug-828: record-only vote-reply seam. Records ctx->votes[peer_idx] (granted
|
|
* flag + peer_tag + peer_lsn) under ctx->lock with the idempotency guard, but
|
|
* does NOT call ha_elect_evaluate_and_signal (the raft library owns promotion
|
|
* in production). This is the helper the production vote_reply_dispatch path
|
|
* now invokes so the granting voter's advertised LSN reaches the election ctx
|
|
* and ha_compute_safe_committed_lsn can compute the true majority-held LSN. */
|
|
void ha_record_vote_reply(ha_state_t *ha, ha_elect_ctx_t *ctx,
|
|
int peer_idx, uint64_t term,
|
|
int vote_granted,
|
|
uint64_t peer_config_tag, uint64_t peer_lsn);
|
|
|
|
/* ── Minimal ha_config_t used across all election tests ─────────────────── */
|
|
|
|
static ha_config_t g_elect_cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
.election_jitter_max_ms = 0, /* no jitter — deterministic in tests */
|
|
.election_priority_step_ms = 500,
|
|
};
|
|
|
|
/* ── Helper: allocate a fresh ha_elect_ctx_t for n_other_peers ──────────── */
|
|
|
|
static ha_elect_ctx_t *alloc_elect_ctx(int n_other_peers)
|
|
{
|
|
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) +
|
|
(size_t)n_other_peers * sizeof(ctx->votes[0]));
|
|
if (!ctx) {
|
|
fprintf(stderr, "alloc_elect_ctx: calloc failed\n");
|
|
exit(1);
|
|
}
|
|
ctx->n_peers = n_other_peers;
|
|
ctx->votes_granted = 1; /* self-vote */
|
|
ctx->n_replied = 0;
|
|
ctx->max_resp_tag = 0;
|
|
ctx->max_resp_lsn = 0;
|
|
ctx->result = -1; /* not yet determined */
|
|
sem_init(&ctx->done, 0, 0);
|
|
return ctx;
|
|
}
|
|
|
|
static void free_elect_ctx(ha_elect_ctx_t *ctx)
|
|
{
|
|
sem_destroy(&ctx->done);
|
|
free(ctx);
|
|
}
|
|
|
|
/* ── Req 2.1: 2/3 votes → CTRL_PRIMARY ──────────────────────────────────── */
|
|
|
|
static void test_quorum_2_of_3_wins_election(void)
|
|
{
|
|
/* Scenario: 3-node cluster, self-vote + 2 peer grants = quorum (2 of 3).
|
|
* ha_elect_evaluate_and_signal with votes_granted=3 (self + 2 peers),
|
|
* no fresher peer, must return CTRL_PRIMARY and set ctx->result=1. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
|
|
|
ha_elect_ctx_t *ctx = alloc_elect_ctx(2); /* 2 other peers in 3-node cluster */
|
|
ctx->votes_granted = 3; /* self + both peers granted */
|
|
ctx->n_replied = 2;
|
|
ctx->max_resp_tag = 0; /* peers report same tag (fresh tie — no yield) */
|
|
ctx->max_resp_lsn = 0;
|
|
|
|
/* candidate's own tag/lsn must be >= max_resp to not yield */
|
|
ctrl_role_t role = ha_elect_evaluate_and_signal(&ha, ctx);
|
|
|
|
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, (int)role);
|
|
TEST_ASSERT_EQUAL_INT(1, ctx->result);
|
|
|
|
free_elect_ctx(ctx);
|
|
ha_fini(&ha);
|
|
printf("test_quorum_2_of_3_wins_election: PASS\n");
|
|
}
|
|
|
|
/* Req 2.1 boundary: exactly quorum (ceil(3/2)=2 grants needed beyond self):
|
|
* self-vote + 1 peer grant = 2 ≥ quorum(2). Must be PRIMARY. */
|
|
static void test_quorum_exactly_majority_wins(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 2ULL, memory_order_release);
|
|
|
|
ha_elect_ctx_t *ctx = alloc_elect_ctx(2);
|
|
ctx->votes_granted = 2; /* self + 1 of 2 peers — strict majority of 3 */
|
|
ctx->n_replied = 2;
|
|
ctx->max_resp_tag = 0;
|
|
ctx->max_resp_lsn = 0;
|
|
|
|
ctrl_role_t role = ha_elect_evaluate_and_signal(&ha, ctx);
|
|
|
|
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, (int)role);
|
|
|
|
free_elect_ctx(ctx);
|
|
ha_fini(&ha);
|
|
printf("test_quorum_exactly_majority_wins: PASS\n");
|
|
}
|
|
|
|
/* ── Req 2.1: only 1/3 vote (self-vote alone) → CTRL_STANDBY ────────────── */
|
|
|
|
static void test_quorum_1_of_3_loses_election(void)
|
|
{
|
|
/* Scenario: 3-node cluster. Both peers deny the vote.
|
|
* votes_granted=1 (self only), n_replied=2.
|
|
* ha_elect_evaluate_and_signal must return CTRL_STANDBY and set result=0. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
|
|
|
ha_elect_ctx_t *ctx = alloc_elect_ctx(2);
|
|
ctx->votes_granted = 1; /* self-vote only */
|
|
ctx->n_replied = 2;
|
|
ctx->max_resp_tag = 0;
|
|
ctx->max_resp_lsn = 0;
|
|
|
|
ctrl_role_t role = ha_elect_evaluate_and_signal(&ha, ctx);
|
|
|
|
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)role);
|
|
TEST_ASSERT_EQUAL_INT(0, ctx->result);
|
|
|
|
free_elect_ctx(ctx);
|
|
ha_fini(&ha);
|
|
printf("test_quorum_1_of_3_loses_election: PASS\n");
|
|
}
|
|
|
|
/* ── Req 2.2: lower-term vote request → deny ─────────────────────────────── */
|
|
|
|
static void test_vote_request_lower_term_denied(void)
|
|
{
|
|
/* The receiver is at term 5; candidate proposes term 3 (lower).
|
|
* handle_request_vote must deny (return 0) and NOT update ha->term. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 5ULL, memory_order_release);
|
|
ha.voted_for_term = 0; /* no prior vote */
|
|
|
|
int granted = handle_request_vote(&ha,
|
|
3ULL, /* candidate's term < my term */
|
|
"ctrl-b", /* candidate name */
|
|
100ULL, /* candidate config_tag */
|
|
50ULL, /* candidate last_lsn */
|
|
100ULL, /* my config_tag (same) */
|
|
50ULL, /* my last_lsn (same) */
|
|
0 /* not in catch-up */);
|
|
TEST_ASSERT_EQUAL_INT(0, granted);
|
|
|
|
/* Term must be unchanged at 5 */
|
|
TEST_ASSERT_EQUAL_UINT64(5ULL,
|
|
atomic_load_explicit(&ha.term, memory_order_acquire));
|
|
/* No vote should have been recorded */
|
|
TEST_ASSERT_EQUAL_INT(0, ha.voted_for_name[0]);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_vote_request_lower_term_denied: PASS\n");
|
|
}
|
|
|
|
/* ── Req 2.3: higher-term request → update term, clear voted_for ─────────── */
|
|
|
|
static void test_vote_request_higher_term_updates_term_and_clears_vote(void)
|
|
{
|
|
/* The receiver is at term 2 with a prior vote cast in term 2.
|
|
* Candidate arrives with term 5 (higher).
|
|
* handle_request_vote must:
|
|
* - update ha->term to 5
|
|
* - clear voted_for_term / voted_for_name
|
|
* - grant the vote (assuming freshness passes) */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 2ULL, memory_order_release);
|
|
ha.voted_for_term = 2;
|
|
strncpy(ha.voted_for_name, "ctrl-old", sizeof(ha.voted_for_name) - 1);
|
|
|
|
int granted = handle_request_vote(&ha,
|
|
5ULL, /* higher term */
|
|
"ctrl-new", /* new candidate */
|
|
200ULL, /* candidate config_tag */
|
|
100ULL, /* candidate last_lsn */
|
|
100ULL, /* my config_tag (lower — candidate fresher) */
|
|
50ULL, /* my last_lsn */
|
|
0 /* not in catch-up */);
|
|
TEST_ASSERT_EQUAL_INT(1, granted);
|
|
|
|
/* Term must be updated to 5 */
|
|
TEST_ASSERT_EQUAL_UINT64(5ULL,
|
|
atomic_load_explicit(&ha.term, memory_order_acquire));
|
|
|
|
/* voted_for_name must now be the new candidate */
|
|
TEST_ASSERT_EQUAL_STRING("ctrl-new", ha.voted_for_name);
|
|
TEST_ASSERT_EQUAL_UINT64(5ULL, ha.voted_for_term);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_vote_request_higher_term_updates_term_and_clears_vote: PASS\n");
|
|
}
|
|
|
|
/* ── Req 2.4: freshness gate — peer tag higher → deny vote ──────────────── */
|
|
|
|
static void test_vote_denied_when_candidate_config_tag_lower(void)
|
|
{
|
|
/* Receiver has config_tag=200; candidate has config_tag=100 (stale).
|
|
* Freshness gate: candidate not at least as fresh → deny. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
|
ha.voted_for_term = 0;
|
|
|
|
int granted = handle_request_vote(&ha,
|
|
1ULL, /* same term */
|
|
"ctrl-b",
|
|
100ULL, /* candidate config_tag < my 200 */
|
|
999ULL, /* candidate lsn (high but irrelevant) */
|
|
200ULL, /* my config_tag (fresher) */
|
|
500ULL, /* my last_lsn */
|
|
0);
|
|
TEST_ASSERT_EQUAL_INT(0, granted);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_vote_denied_when_candidate_config_tag_lower: PASS\n");
|
|
}
|
|
|
|
static void test_vote_denied_when_same_tag_but_candidate_lsn_lower(void)
|
|
{
|
|
/* Equal config_tag but candidate's LSN < my LSN → deny (req 2.4).
|
|
* The fresher WAL log should win. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 3ULL, memory_order_release);
|
|
ha.voted_for_term = 0;
|
|
|
|
int granted = handle_request_vote(&ha,
|
|
3ULL, /* same term */
|
|
"ctrl-c",
|
|
500ULL, /* same config_tag */
|
|
100ULL, /* candidate last_lsn < my 200 */
|
|
500ULL, /* my config_tag (equal) */
|
|
200ULL, /* my last_lsn (higher → I'm fresher) */
|
|
0);
|
|
TEST_ASSERT_EQUAL_INT(0, granted);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_vote_denied_when_same_tag_but_candidate_lsn_lower: PASS\n");
|
|
}
|
|
|
|
/* Req 2.4 inverse: candidate is fresher → grant */
|
|
static void test_vote_granted_when_candidate_fresher_than_receiver(void)
|
|
{
|
|
/* Candidate has higher config_tag — receiver should grant. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
|
ha.voted_for_term = 0;
|
|
|
|
int granted = handle_request_vote(&ha,
|
|
1ULL,
|
|
"ctrl-fresh",
|
|
999ULL, /* candidate config_tag > my 100 */
|
|
500ULL, /* candidate last_lsn */
|
|
100ULL, /* my config_tag (stale) */
|
|
50ULL, /* my last_lsn */
|
|
0);
|
|
TEST_ASSERT_EQUAL_INT(1, granted);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_vote_granted_when_candidate_fresher_than_receiver: PASS\n");
|
|
}
|
|
|
|
/* ── Req 2.5: in_catch_up=1 → deny vote ─────────────────────────────────── */
|
|
|
|
static void test_vote_denied_when_receiver_in_catch_up(void)
|
|
{
|
|
/* A node that is still catching up with the primary's committed log must
|
|
* deny all vote requests (req 2.5). The candidate may be fresh — still deny. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
|
ha.voted_for_term = 0;
|
|
atomic_store_explicit(&ha.in_catch_up, 1, memory_order_release);
|
|
|
|
int granted = handle_request_vote(&ha,
|
|
1ULL,
|
|
"ctrl-b",
|
|
1000ULL, /* candidate fresher than me */
|
|
5000ULL,
|
|
100ULL, /* my config_tag */
|
|
50ULL, /* my last_lsn */
|
|
1 /* in_catch_up_flag */);
|
|
TEST_ASSERT_EQUAL_INT(0, granted);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_vote_denied_when_receiver_in_catch_up: PASS\n");
|
|
}
|
|
|
|
/* ── Req 2.6: voted_for enforcement — deny second candidate in same term ─── */
|
|
|
|
static void test_vote_denied_for_second_candidate_same_term(void)
|
|
{
|
|
/* Node already voted for "ctrl-a" in term 3.
|
|
* A second candidate "ctrl-b" arrives with the same term.
|
|
* handle_request_vote must deny (return 0) to prevent split-vote betrayal. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 3ULL, memory_order_release);
|
|
ha.voted_for_term = 3;
|
|
strncpy(ha.voted_for_name, "ctrl-a", sizeof(ha.voted_for_name) - 1);
|
|
|
|
int granted = handle_request_vote(&ha,
|
|
3ULL, /* same term as prior vote */
|
|
"ctrl-b",/* DIFFERENT candidate */
|
|
200ULL, /* candidate is fresh */
|
|
100ULL,
|
|
100ULL, /* my tag */
|
|
50ULL,
|
|
0);
|
|
TEST_ASSERT_EQUAL_INT(0, granted);
|
|
|
|
/* voted_for_name must remain "ctrl-a" — no vote change */
|
|
TEST_ASSERT_EQUAL_STRING("ctrl-a", ha.voted_for_name);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_vote_denied_for_second_candidate_same_term: PASS\n");
|
|
}
|
|
|
|
/* Req 2.6 positive: same candidate gets the grant (idempotent re-vote) */
|
|
static void test_vote_granted_to_same_candidate_same_term(void)
|
|
{
|
|
/* Node already voted for "ctrl-a" in term 3.
|
|
* "ctrl-a" retransmits (network retransmission) — must be granted again. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 3ULL, memory_order_release);
|
|
ha.voted_for_term = 3;
|
|
strncpy(ha.voted_for_name, "ctrl-a", sizeof(ha.voted_for_name) - 1);
|
|
|
|
int granted = handle_request_vote(&ha,
|
|
3ULL, /* same term */
|
|
"ctrl-a",/* SAME candidate */
|
|
200ULL, /* fresh */
|
|
100ULL,
|
|
100ULL,
|
|
50ULL,
|
|
0);
|
|
TEST_ASSERT_EQUAL_INT(1, granted);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_vote_granted_to_same_candidate_same_term: PASS\n");
|
|
}
|
|
|
|
/* ── Req 2.7: fresher-peer yield gate ────────────────────────────────────── */
|
|
|
|
static void test_candidate_yields_to_fresher_responding_peer(void)
|
|
{
|
|
/* Candidate wins quorum (2/2 votes from both peers), but one responding
|
|
* peer reports a higher config_tag than the candidate's own tag.
|
|
* ha_elect_evaluate_and_signal must return CTRL_STANDBY (yield). */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
|
|
|
ha_elect_ctx_t *ctx = alloc_elect_ctx(2);
|
|
ctx->votes_granted = 3; /* quorum reached in a 3-node cluster */
|
|
ctx->n_replied = 2;
|
|
/* Peer reported a higher config_tag — trigger yield gate */
|
|
ctx->max_resp_tag = 9999ULL; /* peer's tag > candidate's tag below */
|
|
ctx->max_resp_lsn = 100ULL;
|
|
|
|
/* Candidate's own tag is 100 — lower than max_resp_tag → must yield */
|
|
/* The implementation must compare ctx->max_resp_tag vs the candidate's
|
|
* my_config_tag parameter stored in ctx or passed at evaluation time.
|
|
* We embed the candidate's my_config_tag in ctx via a direct write of
|
|
* my_tag on vote[0] (convention: index 0 represents self) for implementations
|
|
* that store it there. Alternatively, the caller passes it; see contract. */
|
|
ctx->votes[0].my_tag = 100ULL; /* candidate's own tag — lower than peer's 9999 */
|
|
ctx->votes[0].my_lsn = 50ULL;
|
|
|
|
ctrl_role_t role = ha_elect_evaluate_and_signal(&ha, ctx);
|
|
|
|
/* Must yield: fresher peer responds → CTRL_STANDBY */
|
|
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)role);
|
|
|
|
free_elect_ctx(ctx);
|
|
ha_fini(&ha);
|
|
printf("test_candidate_yields_to_fresher_responding_peer: PASS\n");
|
|
}
|
|
|
|
/* Req 2.7 negative: candidate is fresher than all responding peers → PRIMARY */
|
|
static void test_candidate_wins_when_fresher_than_all_peers(void)
|
|
{
|
|
/* Candidate has config_tag=9999 (highest); both responding peers have 100.
|
|
* Quorum won + no fresher peer → CTRL_PRIMARY. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
|
|
|
ha_elect_ctx_t *ctx = alloc_elect_ctx(2);
|
|
ctx->votes_granted = 3;
|
|
ctx->n_replied = 2;
|
|
ctx->max_resp_tag = 100ULL; /* peers' max tag — lower than candidate's 9999 */
|
|
ctx->max_resp_lsn = 50ULL;
|
|
|
|
ctx->votes[0].my_tag = 9999ULL; /* candidate's own tag (the highest) */
|
|
ctx->votes[0].my_lsn = 500ULL;
|
|
|
|
ctrl_role_t role = ha_elect_evaluate_and_signal(&ha, ctx);
|
|
|
|
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, (int)role);
|
|
|
|
free_elect_ctx(ctx);
|
|
ha_fini(&ha);
|
|
printf("test_candidate_wins_when_fresher_than_all_peers: PASS\n");
|
|
}
|
|
|
|
/* ── Req 2.9: single-node self-elects immediately ────────────────────────── */
|
|
|
|
static void test_single_node_self_elects_as_primary(void)
|
|
{
|
|
/* n_peers=1 means only this controller exists.
|
|
* am_i_primary_quorum must return CTRL_PRIMARY without making any network
|
|
* calls. No peers array is needed (NULL is safe for n_peers=1). */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 0ULL, memory_order_release);
|
|
|
|
ctrl_role_t role = am_i_primary_quorum(&ha,
|
|
1, /* n_peers = 1 (single node) */
|
|
NULL, /* no other peers */
|
|
500ULL, /* my_config_tag */
|
|
100ULL, /* my_lsn */
|
|
"test-node");
|
|
|
|
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, (int)role);
|
|
|
|
/* Term must have been incremented (self-election still bumps term) */
|
|
TEST_ASSERT_GREATER_THAN_UINT64(0ULL,
|
|
atomic_load_explicit(&ha.term, memory_order_acquire));
|
|
|
|
ha_fini(&ha);
|
|
printf("test_single_node_self_elects_as_primary: PASS\n");
|
|
}
|
|
|
|
/* ── Req 2.10: split vote → retry with incremented term ──────────────────── */
|
|
|
|
static void test_split_vote_triggers_retry_with_incremented_term(void)
|
|
{
|
|
/* A split vote occurs when neither candidate reaches quorum.
|
|
* ha_elect_evaluate_and_signal with votes_granted < quorum and all peers
|
|
* replied must return CTRL_STANDBY (split-vote detected).
|
|
*
|
|
* For the full retry path (am_i_primary_quorum doing two rounds with
|
|
* incremented term) we verify the simpler observable property:
|
|
* After the first failing round, if ha_elect_evaluate_and_signal is called
|
|
* with a second context at term+1 (simulating the retry), the term in ha
|
|
* must be monotonically higher than the initial round's term.
|
|
*
|
|
* Direct verification of the two-round retry loop is in the am_i_primary_quorum
|
|
* integration test below (test_split_vote_full_retry_increments_term). */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 7ULL, memory_order_release);
|
|
|
|
/* Round 1: split — 1 self-vote, 1 grant, 1 deny → total 2 of 3 = quorum met?
|
|
* Actually 3-node quorum = ceil(3/2)=2. self+1=2 IS quorum.
|
|
* To simulate a true split vote: use a 5-node cluster where quorum is 3
|
|
* and candidate only gets 2 (self + 1 of 4). */
|
|
ha_elect_ctx_t *ctx1 = alloc_elect_ctx(4); /* 5-node cluster: 4 other peers */
|
|
ctx1->votes_granted = 2; /* self + 1 peer — below quorum(3) */
|
|
ctx1->n_replied = 4; /* all 4 peers replied */
|
|
ctx1->max_resp_tag = 0;
|
|
ctx1->max_resp_lsn = 0;
|
|
|
|
ctx1->votes[0].my_tag = 100ULL;
|
|
ctx1->votes[0].my_lsn = 50ULL;
|
|
|
|
ctrl_role_t role1 = ha_elect_evaluate_and_signal(&ha, ctx1);
|
|
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)role1);
|
|
|
|
free_elect_ctx(ctx1);
|
|
|
|
/* The implementation must increment the term before the retry.
|
|
* Simulate term bump as the implementation would do, then run a second
|
|
* round that also fails — verify term is still higher. */
|
|
uint64_t term_after_round1 = atomic_load_explicit(&ha.term, memory_order_acquire);
|
|
/* Term should remain >= 7; the actual retry bump happens inside
|
|
* am_i_primary_quorum's retry loop — tested separately below. */
|
|
TEST_ASSERT_GREATER_OR_EQUAL_UINT64(7ULL, term_after_round1);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_split_vote_triggers_retry_with_incremented_term: PASS\n");
|
|
}
|
|
|
|
/* Req 2.10/2.11: am_i_primary_quorum full two-round retry with term increment.
|
|
*
|
|
* This test exercises the top-level election loop. Because the network is
|
|
* not available in unit tests we use n_peers=1 (single-node shortcircuit) to
|
|
* exercise at least the term-increment path, and a synthetic quorum failure
|
|
* path via the evaluate function.
|
|
*
|
|
* The full retry round-trip (two real election rounds with network peers) is
|
|
* validated by integration tests. Here we assert the observable contract:
|
|
* if am_i_primary_quorum is called on a single-node cluster, it must have
|
|
* incremented the term by at least 1 relative to the starting value. */
|
|
static void test_split_vote_full_retry_increments_term(void)
|
|
{
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
uint64_t initial_term = 4ULL;
|
|
atomic_store_explicit(&ha.term, initial_term, memory_order_release);
|
|
|
|
/* Single-node: am_i_primary_quorum must self-elect and bump the term.
|
|
* (Multi-node split-vote path is verified in integration tests.) */
|
|
ctrl_role_t role = am_i_primary_quorum(&ha, 1, NULL, 100ULL, 50ULL, "test-node");
|
|
|
|
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, (int)role);
|
|
TEST_ASSERT_GREATER_THAN_UINT64(initial_term,
|
|
atomic_load_explicit(&ha.term, memory_order_acquire));
|
|
|
|
ha_fini(&ha);
|
|
printf("test_split_vote_full_retry_increments_term: PASS\n");
|
|
}
|
|
|
|
/* ── Req 2.11: second election failure → CTRL_STANDBY ────────────────────── */
|
|
|
|
static void test_two_failed_elections_result_in_standby(void)
|
|
{
|
|
/* Two consecutive calls to ha_elect_evaluate_and_signal that both fail
|
|
* quorum — the second failure must produce CTRL_STANDBY (not a loop). */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
|
|
|
/* Round 1: no quorum (5-node cluster, only 2 of 5 votes) */
|
|
ha_elect_ctx_t *ctx1 = alloc_elect_ctx(4);
|
|
ctx1->votes_granted = 2;
|
|
ctx1->n_replied = 4;
|
|
ctx1->max_resp_tag = 0;
|
|
ctx1->max_resp_lsn = 0;
|
|
ctx1->votes[0].my_tag = 100ULL;
|
|
ctx1->votes[0].my_lsn = 50ULL;
|
|
ctrl_role_t role1 = ha_elect_evaluate_and_signal(&ha, ctx1);
|
|
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)role1);
|
|
free_elect_ctx(ctx1);
|
|
|
|
/* Round 2: same outcome — also fails quorum */
|
|
atomic_store_explicit(&ha.term, 2ULL, memory_order_release); /* simulate retry bump */
|
|
ha_elect_ctx_t *ctx2 = alloc_elect_ctx(4);
|
|
ctx2->votes_granted = 2;
|
|
ctx2->n_replied = 4;
|
|
ctx2->max_resp_tag = 0;
|
|
ctx2->max_resp_lsn = 0;
|
|
ctx2->votes[0].my_tag = 100ULL;
|
|
ctx2->votes[0].my_lsn = 50ULL;
|
|
ctrl_role_t role2 = ha_elect_evaluate_and_signal(&ha, ctx2);
|
|
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)role2);
|
|
free_elect_ctx(ctx2);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_two_failed_elections_result_in_standby: PASS\n");
|
|
}
|
|
|
|
/* ── Req 2.12: identical tag/LSN — jitter, not priority ──────────────────── */
|
|
|
|
static void test_identical_peers_use_jitter_not_priority(void)
|
|
{
|
|
/* When all responding peers have the same config_tag and LSN as the
|
|
* candidate, the tiebreaker must be randomised jitter — not ha->my_priority.
|
|
*
|
|
* This test verifies that ha_elect_evaluate_and_signal does NOT use
|
|
* ha.my_priority as a factor when tags and LSNs are identical.
|
|
*
|
|
* Observable: two separate ha_state_t instances with different my_priority
|
|
* values but identical tag/LSN context must BOTH be able to reach
|
|
* CTRL_PRIMARY (i.e., priority alone must not exclude any candidate).
|
|
* In practice only one will win on any run due to jitter, but neither is
|
|
* statically blocked. We verify that both result values (PRIMARY or STANDBY)
|
|
* are possible (not forced) by checking the result is one of the two valid
|
|
* values — not a hard deterministic outcome based on priority. */
|
|
ha_state_t ha_low; /* priority 0 — lowest index, highest LSF priority */
|
|
ha_state_t ha_high; /* priority 2 — higher index, lower LSF priority */
|
|
ha_init(&ha_low, &g_elect_cfg, 0);
|
|
ha_init(&ha_high, &g_elect_cfg, 2);
|
|
|
|
atomic_store_explicit(&ha_low.term, 1ULL, memory_order_release);
|
|
atomic_store_explicit(&ha_high.term, 1ULL, memory_order_release);
|
|
|
|
/* Identical context: same tag and LSN for both candidates */
|
|
ha_elect_ctx_t *ctx_low = alloc_elect_ctx(2);
|
|
ctx_low->votes_granted = 2;
|
|
ctx_low->n_replied = 2;
|
|
ctx_low->max_resp_tag = 500ULL; /* equal to my_tag below */
|
|
ctx_low->max_resp_lsn = 100ULL; /* equal to my_lsn below */
|
|
ctx_low->votes[0].my_tag = 500ULL;
|
|
ctx_low->votes[0].my_lsn = 100ULL;
|
|
|
|
ha_elect_ctx_t *ctx_high = alloc_elect_ctx(2);
|
|
ctx_high->votes_granted = 2;
|
|
ctx_high->n_replied = 2;
|
|
ctx_high->max_resp_tag = 500ULL;
|
|
ctx_high->max_resp_lsn = 100ULL;
|
|
ctx_high->votes[0].my_tag = 500ULL;
|
|
ctx_high->votes[0].my_lsn = 100ULL;
|
|
|
|
ctrl_role_t role_low = ha_elect_evaluate_and_signal(&ha_low, ctx_low);
|
|
ctrl_role_t role_high = ha_elect_evaluate_and_signal(&ha_high, ctx_high);
|
|
|
|
/* Both results must be valid role values (PRIMARY or STANDBY) */
|
|
TEST_ASSERT_TRUE((int)role_low == CTRL_PRIMARY || (int)role_low == CTRL_STANDBY);
|
|
TEST_ASSERT_TRUE((int)role_high == CTRL_PRIMARY || (int)role_high == CTRL_STANDBY);
|
|
|
|
/* Critical: my_priority alone must not determine the outcome.
|
|
* If priority were the tiebreaker, priority-0 (ha_low) would ALWAYS win.
|
|
* We cannot fully test randomness in a unit test, but we assert that the
|
|
* lower-priority candidate (ha_high) is not statically excluded.
|
|
* The implementation must NOT hard-code priority as a condition. */
|
|
|
|
free_elect_ctx(ctx_low);
|
|
free_elect_ctx(ctx_high);
|
|
ha_fini(&ha_low);
|
|
ha_fini(&ha_high);
|
|
printf("test_identical_peers_use_jitter_not_priority: PASS\n");
|
|
}
|
|
|
|
/* ── Additional edge-case tests ──────────────────────────────────────────── */
|
|
|
|
/* Req 2.4: same-tag candidate with equal LSN → grant (not denied by freshness) */
|
|
static void test_vote_granted_when_candidate_equally_fresh(void)
|
|
{
|
|
/* Exactly equal tag and LSN → candidate is as fresh as receiver → grant.
|
|
* The freshness gate is: deny if candidate LESS fresh; allow if equal or fresher. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
|
ha.voted_for_term = 0;
|
|
|
|
int granted = handle_request_vote(&ha,
|
|
1ULL,
|
|
"ctrl-equal",
|
|
500ULL, /* same config_tag as mine */
|
|
200ULL, /* same last_lsn as mine */
|
|
500ULL, /* my config_tag */
|
|
200ULL, /* my last_lsn */
|
|
0);
|
|
TEST_ASSERT_EQUAL_INT(1, granted);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_vote_granted_when_candidate_equally_fresh: PASS\n");
|
|
}
|
|
|
|
/* Req 2.8: config_tag=0 candidate, peer has non-zero tag → am_i_primary_quorum
|
|
* must NOT return CTRL_PRIMARY. In the design this triggers a fatal exit; for
|
|
* the unit test we verify that the return value is CTRL_STANDBY (not PRIMARY).
|
|
* Full fatal-exit behaviour is verified at the integration level. */
|
|
static void test_zero_config_tag_candidate_does_not_win_when_peer_has_config(void)
|
|
{
|
|
/* ha_elect_evaluate_and_signal sees ctx->max_resp_tag > 0 but candidate's
|
|
* my_tag = 0. This means the candidate has no config while at least one
|
|
* peer does. The implementation must treat this as a fatal freshness failure
|
|
* → return CTRL_STANDBY (or abort). */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
|
|
|
ha_elect_ctx_t *ctx = alloc_elect_ctx(2);
|
|
ctx->votes_granted = 3; /* quorum met: all 3 nodes voted for us */
|
|
ctx->n_replied = 2;
|
|
ctx->max_resp_tag = 777ULL; /* a peer has non-zero config tag */
|
|
ctx->max_resp_lsn = 10ULL;
|
|
|
|
ctx->votes[0].my_tag = 0ULL; /* candidate has NO config (tag=0) */
|
|
ctx->votes[0].my_lsn = 0ULL;
|
|
|
|
ctrl_role_t role = ha_elect_evaluate_and_signal(&ha, ctx);
|
|
|
|
/* Must NOT become primary when self has no config but peer does */
|
|
TEST_ASSERT_NOT_EQUAL(CTRL_PRIMARY, (int)role);
|
|
|
|
free_elect_ctx(ctx);
|
|
ha_fini(&ha);
|
|
printf("test_zero_config_tag_candidate_does_not_win_when_peer_has_config: PASS\n");
|
|
}
|
|
|
|
/* Req 2.2: vote in current term with self as candidate → granted (self-vote) */
|
|
static void test_vote_request_from_self_same_term_granted(void)
|
|
{
|
|
/* A node may re-receive its own OP_requestVote in loopback scenarios.
|
|
* Same term, name matches voted_for (or no prior vote): grant. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 2ULL, memory_order_release);
|
|
ha.voted_for_term = 0; /* no prior vote */
|
|
|
|
int granted = handle_request_vote(&ha,
|
|
2ULL,
|
|
"ctrl-self",
|
|
100ULL, /* my own fresh tag */
|
|
50ULL,
|
|
100ULL, /* my tag */
|
|
50ULL,
|
|
0);
|
|
TEST_ASSERT_EQUAL_INT(1, granted);
|
|
TEST_ASSERT_EQUAL_STRING("ctrl-self", ha.voted_for_name);
|
|
|
|
ha_fini(&ha);
|
|
printf("test_vote_request_from_self_same_term_granted: PASS\n");
|
|
}
|
|
|
|
/* ── handle_request_vote_reply tests ────────────────────────────────────────
|
|
*
|
|
* These tests verify the reply handler's counting and signalling logic.
|
|
* They use a 3-node cluster (n_peers=2 other peers) so quorum = ceil(3/2) = 2.
|
|
* self-vote starts at 1; receiving one granted reply raises votes_granted to 2,
|
|
* which meets quorum — the handler must post ctx->done at that point.
|
|
*/
|
|
|
|
/* Req 4.2: one granted reply in a 3-node cluster reaches quorum → done posted */
|
|
static void test_reply_handler_quorum_reached_posts_done(void)
|
|
{
|
|
/* 3-node cluster: n_peers=2. quorum=2. votes_granted starts at 1 (self). */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
|
|
|
ha_elect_ctx_t *ctx = alloc_elect_ctx(2);
|
|
/* Record candidate's own tag/lsn so the evaluator can apply the yield gate */
|
|
ctx->votes[0].my_tag = 100ULL;
|
|
ctx->votes[0].my_lsn = 50ULL;
|
|
|
|
/* Peer 0 grants the vote with equal freshness → votes_granted becomes 2 = quorum */
|
|
handle_request_vote_reply(&ha, ctx,
|
|
0, /* peer_idx */
|
|
1ULL, /* term */
|
|
1, /* vote_granted */
|
|
100ULL, /* peer_config_tag (equal to candidate) */
|
|
50ULL /* peer_lsn */);
|
|
|
|
/* ctx->done should have been posted — sem_trywait must succeed */
|
|
int rc = sem_trywait(&ctx->done);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
|
|
"ctx->done must be posted when quorum is reached after first granted reply");
|
|
|
|
/* Tallies must be updated correctly */
|
|
TEST_ASSERT_EQUAL_INT(2, ctx->votes_granted); /* self + peer 0 */
|
|
TEST_ASSERT_EQUAL_INT(1, ctx->n_replied);
|
|
TEST_ASSERT_EQUAL_INT(1, ctx->votes[0].replied);
|
|
TEST_ASSERT_EQUAL_INT(1, ctx->votes[0].vote_granted);
|
|
|
|
free_elect_ctx(ctx);
|
|
ha_fini(&ha);
|
|
printf("test_reply_handler_quorum_reached_posts_done: PASS\n");
|
|
}
|
|
|
|
/* Req 4.2: partial replies that do not yet reach quorum must NOT post done */
|
|
static void test_reply_handler_partial_replies_do_not_post_done(void)
|
|
{
|
|
/* 5-node cluster: n_peers=4. quorum = ceil(5/2) = 3.
|
|
* votes_granted starts at 1 (self). One granted reply → votes_granted=2 < 3.
|
|
* n_replied=1 < 4. Neither threshold met → done must NOT be posted. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 2ULL, memory_order_release);
|
|
|
|
ha_elect_ctx_t *ctx = alloc_elect_ctx(4);
|
|
ctx->votes[0].my_tag = 200ULL;
|
|
ctx->votes[0].my_lsn = 100ULL;
|
|
|
|
/* Peer 0 grants; only 1 of 4 peers have replied → still below both thresholds */
|
|
handle_request_vote_reply(&ha, ctx,
|
|
0, /* peer_idx */
|
|
2ULL, /* term */
|
|
1, /* vote_granted */
|
|
200ULL, /* peer_config_tag */
|
|
100ULL /* peer_lsn */);
|
|
|
|
/* ctx->done must NOT have been posted — sem_trywait must fail with EAGAIN */
|
|
int rc = sem_trywait(&ctx->done);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(-1, rc,
|
|
"ctx->done must NOT be posted when quorum is not yet reached");
|
|
TEST_ASSERT_EQUAL_INT(EAGAIN, errno);
|
|
|
|
/* Tallies should still be updated */
|
|
TEST_ASSERT_EQUAL_INT(2, ctx->votes_granted);
|
|
TEST_ASSERT_EQUAL_INT(1, ctx->n_replied);
|
|
|
|
free_elect_ctx(ctx);
|
|
ha_fini(&ha);
|
|
printf("test_reply_handler_partial_replies_do_not_post_done: PASS\n");
|
|
}
|
|
|
|
/* Req 4.2: all peers reply with denials → n_replied reaches n_peers → done posted */
|
|
static void test_reply_handler_all_denied_posts_done_on_last_reply(void)
|
|
{
|
|
/* 3-node cluster: n_peers=2. Both peers deny → votes_granted stays 1 < quorum(2).
|
|
* After second deny n_replied=2 = n_peers → done is posted (CTRL_STANDBY result). */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
|
|
|
ha_elect_ctx_t *ctx = alloc_elect_ctx(2);
|
|
ctx->votes[0].my_tag = 100ULL;
|
|
ctx->votes[0].my_lsn = 50ULL;
|
|
|
|
/* Peer 0 denies */
|
|
handle_request_vote_reply(&ha, ctx, 0, 1ULL, 0, 100ULL, 50ULL);
|
|
/* done must not be posted yet (1 of 2 replied, votes_granted=1 < quorum=2) */
|
|
int rc = sem_trywait(&ctx->done);
|
|
TEST_ASSERT_EQUAL_INT(-1, rc);
|
|
TEST_ASSERT_EQUAL_INT(EAGAIN, errno);
|
|
|
|
/* Peer 1 denies — now all 2 peers replied → done must be posted */
|
|
handle_request_vote_reply(&ha, ctx, 1, 1ULL, 0, 100ULL, 50ULL);
|
|
rc = sem_trywait(&ctx->done);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
|
|
"ctx->done must be posted when all peers have replied (even if all deny)");
|
|
|
|
TEST_ASSERT_EQUAL_INT(1, ctx->votes_granted); /* only self-vote */
|
|
TEST_ASSERT_EQUAL_INT(2, ctx->n_replied);
|
|
TEST_ASSERT_EQUAL_INT(0, ctx->result); /* CTRL_STANDBY */
|
|
|
|
free_elect_ctx(ctx);
|
|
ha_fini(&ha);
|
|
printf("test_reply_handler_all_denied_posts_done_on_last_reply: PASS\n");
|
|
}
|
|
|
|
/* ── bug-1095 / bug-1096: controller-level Election Safety guard ──────────────
|
|
*
|
|
* Ownership: ha-term-safety, Requirement 1 / Objective — "two leaders never
|
|
* share a term" (bug-851 family). The defects live in libxce raft.c
|
|
* `raft_on_vote_reply`, which is the SOLE production promotion driver: the
|
|
* controller's `vote_reply_dispatch` (ha.c) calls
|
|
* `raft_on_vote_reply(&ha->raft, idx, vote_granted, term)`.
|
|
*
|
|
* bug-1095 — duplicate grant double-count: a counter-based tally that ignored
|
|
* `from_idx` let one peer's DUPLICATED vote-response count twice
|
|
* toward quorum → two leaders at the same term. Fixed by a per-peer
|
|
* `vote_replies[RAFT_MAX_PEERS]` set (commit b5b7e01).
|
|
* bug-1096 — stale-term grant counted: a reply with `mterm < currentTerm` was
|
|
* tallied toward the CURRENT election. Fixed by a
|
|
* `term < s->term` DropStaleResponse that discards the reply WITHOUT
|
|
* stepping the candidate down (commit b5b7e01).
|
|
*
|
|
* These tests drive the REAL `raft_on_vote_reply` against the CONTROLLER's
|
|
* `ha->raft` instance — exactly the object production drives — NOT a bypassing
|
|
* `xce_vote_reply_hook` (the hook is NULL here, and we call the library entry
|
|
* point directly the way `vote_reply_dispatch` does). Assertions are on the
|
|
* resulting ROLE / promotion (Election Safety), not an internal counter, so a
|
|
* premature promotion on the duplicate (bug-1095) or any count/promotion on the
|
|
* stale reply (bug-1096) would FAIL the test. With the fix in place these are
|
|
* the GREEN over the fixed library.
|
|
*
|
|
* Protocol-level companions live in libxce:
|
|
* test_vote_reply_duplicate_grant_not_double_counted
|
|
* test_vote_reply_stale_term_dropped (src/lib/testsuite/test_raft.c)
|
|
*
|
|
* Surfaced by the raft-sim-conformance oracle (task 4.2); fixed cross-boundary
|
|
* with user authorization.
|
|
*
|
|
* Fixture note: ha_init leaves ha->raft with the placeholder n_peers=1 (which
|
|
* would self-elect immediately). We re-init ha->raft for a 5-node cluster by
|
|
* reusing the callbacks/ctx that ha_init already wired (s_real_cbs is file-
|
|
* static in ha.c, but the live pointer is preserved in ha->raft.cbs/.ctx).
|
|
* ha->raft_c stays NULL, so every callback takes its NULL-safe test-mode path
|
|
* (raft_cb_become_leader still sets ha->role = CTRL_PRIMARY outside its c-guard).
|
|
*/
|
|
static void reinit_ctrl_raft_n(ha_state_t *ha, int n_peers, uint64_t initial_term)
|
|
{
|
|
raft_config_t rcfg;
|
|
memset(&rcfg, 0, sizeof(rcfg));
|
|
rcfg.n_peers = n_peers;
|
|
rcfg.self_idx = ha->my_priority; /* self_idx == priority by convention */
|
|
rcfg.my_priority = ha->my_priority;
|
|
rcfg.initial_term = initial_term;
|
|
rcfg.initial_voted_for = -1;
|
|
/* Reuse the production callbacks/ctx ha_init already installed on ha->raft. */
|
|
rcfg.cbs = ha->raft.cbs;
|
|
rcfg.ctx = ha->raft.ctx;
|
|
TEST_ASSERT_EQUAL_INT(0, raft_init(&ha->raft, &rcfg));
|
|
}
|
|
|
|
/* bug-1095: a DUPLICATED vote grant from one peer must NOT count twice toward
|
|
* quorum, so it must NOT promote the candidate before a DISTINCT peer grants. */
|
|
static void test_vote_reply_duplicate_grant_does_not_promote(void)
|
|
{
|
|
/* 5-node cluster: quorum = 3. self-vote = 1. One distinct grant → 2 (< 3).
|
|
* A SECOND grant from the SAME peer must stay at 2 (set semantics) — the
|
|
* candidate must NOT become leader on the duplicate. Only a grant from a
|
|
* DISTINCT peer raises the tally to 3 = quorum → promotion. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0); /* self_idx = 0, raft_c = NULL */
|
|
reinit_ctrl_raft_n(&ha, 5, /*initial_term=*/4ULL);
|
|
|
|
/* Begin the election: term → 5, role → CANDIDATE, votes_granted = 1 (self). */
|
|
raft_start_election(&ha.raft);
|
|
raft_snapshot_t snap;
|
|
raft_snapshot(&ha.raft, &snap);
|
|
TEST_ASSERT_EQUAL_UINT64(5ULL, snap.term);
|
|
TEST_ASSERT_EQUAL_INT(RAFT_CANDIDATE, (int)snap.role);
|
|
|
|
/* Peer 1 GRANTS at the current term → votes_granted = 2 (< quorum 3). */
|
|
raft_on_vote_reply(&ha.raft, /*from_idx=*/1, /*granted=*/1, /*term=*/5ULL);
|
|
raft_snapshot(&ha.raft, &snap);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_CANDIDATE, (int)snap.role,
|
|
"one distinct grant (2 of 5) must NOT reach quorum (3)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int32_t)CTRL_STANDBY,
|
|
(int32_t)atomic_load_explicit(&ha.role, memory_order_acquire),
|
|
"controller must NOT be CTRL_PRIMARY after one grant");
|
|
|
|
/* Peer 1 GRANTS AGAIN — a duplicate (legal DuplicateMessage). bug-1095: this
|
|
* must be dropped by the per-peer vote_replies[] set and must NOT raise the
|
|
* tally to 3 / promote. WITHOUT the dedup this duplicate would push
|
|
* votes_granted to 3 and prematurely elect the candidate → two leaders. */
|
|
raft_on_vote_reply(&ha.raft, /*from_idx=*/1, /*granted=*/1, /*term=*/5ULL);
|
|
raft_snapshot(&ha.raft, &snap);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_CANDIDATE, (int)snap.role,
|
|
"bug-1095: a DUPLICATE grant from the same peer must NOT count twice / "
|
|
"must NOT promote (Election Safety: votesGranted is a SET)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int32_t)CTRL_STANDBY,
|
|
(int32_t)atomic_load_explicit(&ha.role, memory_order_acquire),
|
|
"controller role must still be CTRL_STANDBY after the duplicate grant");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(2, snap.votes_granted,
|
|
"duplicate grant must leave votes_granted at 2 (self + one distinct peer)");
|
|
|
|
/* A grant from a DISTINCT peer (peer 2) now reaches quorum (3 of 5) → leader.
|
|
* This proves the duplicate above was the ONLY thing withheld — quorum is
|
|
* reachable, just not by double-counting one peer. */
|
|
raft_on_vote_reply(&ha.raft, /*from_idx=*/2, /*granted=*/1, /*term=*/5ULL);
|
|
raft_snapshot(&ha.raft, &snap);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_LEADER, (int)snap.role,
|
|
"a DISTINCT third grant must reach quorum (3 of 5) and promote");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int32_t)CTRL_PRIMARY,
|
|
(int32_t)atomic_load_explicit(&ha.role, memory_order_acquire),
|
|
"controller must become CTRL_PRIMARY once quorum is reached by distinct grants");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_vote_reply_duplicate_grant_does_not_promote: PASS\n");
|
|
}
|
|
|
|
/* bug-1096: a vote reply carrying a STALE term (term < currentTerm) must be
|
|
* dropped — not counted toward this election and not a step-down to follower. */
|
|
static void test_vote_reply_stale_term_does_not_count_or_promote(void)
|
|
{
|
|
/* 5-node cluster: quorum = 3. Candidate at term 5. A grant advertising an
|
|
* OLD term (3) must be discarded by DropStaleResponse: it must NOT raise the
|
|
* tally, must NOT promote, and must NOT step the candidate down. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
reinit_ctrl_raft_n(&ha, 5, /*initial_term=*/4ULL);
|
|
|
|
raft_start_election(&ha.raft); /* term → 5, CANDIDATE, votes = 1 */
|
|
raft_snapshot_t snap;
|
|
raft_snapshot(&ha.raft, &snap);
|
|
TEST_ASSERT_EQUAL_UINT64(5ULL, snap.term);
|
|
TEST_ASSERT_EQUAL_INT(RAFT_CANDIDATE, (int)snap.role);
|
|
|
|
/* Stale grant: peer 1 replies GRANTED but with term 3 (< current term 5).
|
|
* bug-1096: this must be dropped — votes_granted must stay 1, role must stay
|
|
* CANDIDATE (no premature promotion, no spurious step-down). */
|
|
int rc = raft_on_vote_reply(&ha.raft, /*from_idx=*/1, /*granted=*/1, /*term=*/3ULL);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_PENDING, rc,
|
|
"bug-1096: a stale-term grant must be DROPPED (RAFT_PENDING), not counted");
|
|
|
|
raft_snapshot(&ha.raft, &snap);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_CANDIDATE, (int)snap.role,
|
|
"bug-1096: a stale-term reply must NOT promote AND must NOT step the "
|
|
"candidate down to follower (DropStaleResponse)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, snap.votes_granted,
|
|
"stale-term grant must NOT be counted — votes_granted stays at 1 (self)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, snap.term,
|
|
"a stale (lower) reply term must NOT lower the candidate's currentTerm");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int32_t)CTRL_STANDBY,
|
|
(int32_t)atomic_load_explicit(&ha.role, memory_order_acquire),
|
|
"controller must NOT be CTRL_PRIMARY after a stale-term grant");
|
|
|
|
/* Sanity: a fresh-term grant from a DISTINCT peer still works after the stale
|
|
* one was dropped — the election is alive and only the stale reply was lost. */
|
|
raft_on_vote_reply(&ha.raft, /*from_idx=*/1, /*granted=*/1, /*term=*/5ULL);
|
|
raft_on_vote_reply(&ha.raft, /*from_idx=*/2, /*granted=*/1, /*term=*/5ULL);
|
|
raft_snapshot(&ha.raft, &snap);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_LEADER, (int)snap.role,
|
|
"two fresh-term distinct grants reach quorum (3 of 5) after the stale drop");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_vote_reply_stale_term_does_not_count_or_promote: PASS\n");
|
|
}
|
|
|
|
/* ── bug-828: safe-commit-from-votes must reflect the voted majority ─────────
|
|
*
|
|
* Production root cause: vote_reply_dispatch decoded the granting voter's
|
|
* advertised LSN but discarded it, so ctx->votes[i].vote_granted stayed 0 and
|
|
* ha_compute_safe_committed_lsn saw only the self-vote (n_granted=1) →
|
|
* safe_committed_lsn = self_lsn. A standby promoting with self_lsn=1 then
|
|
* truncated committed entries 2..5.
|
|
*
|
|
* This test drives the record-only seam (ha_record_vote_reply) that the
|
|
* production path now invokes: two peers GRANT advertising peer_lsn=5; the
|
|
* candidate's own self_lsn=1. ha_compute_safe_committed_lsn must return the
|
|
* majority-held LSN (5), NOT self_lsn (1).
|
|
*
|
|
* RED before the fix: ha_record_vote_reply does not exist (undefined reference);
|
|
* and even if grants were not recorded, safe_committed would be self_lsn=1.
|
|
*/
|
|
static void test_safe_commit_reflects_granting_voter_lsn(void)
|
|
{
|
|
/* 3-node cluster: n_peers=2 other peers. */
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_elect_cfg, 0);
|
|
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
|
|
|
|
ha_elect_ctx_t *ctx = alloc_elect_ctx(2);
|
|
|
|
/* Candidate's own (boot) position: durable floor 1, the catch-up node case. */
|
|
ctx->self_lsn = 1ULL;
|
|
ctx->self_tag = 100ULL;
|
|
ctx->votes[0].my_tag = 100ULL;
|
|
ctx->votes[0].my_lsn = 1ULL;
|
|
|
|
/* Both peers GRANT, each advertising a durable position of LSN 5
|
|
* (they hold the committed prefix). Record via the production seam. */
|
|
ha_record_vote_reply(&ha, ctx, 0, 1ULL, 1, 100ULL, 5ULL);
|
|
ha_record_vote_reply(&ha, ctx, 1, 1ULL, 1, 100ULL, 5ULL);
|
|
|
|
/* The recording must NOT have signalled the legacy promotion path —
|
|
* ha_record_vote_reply records only; the raft library owns promotion.
|
|
* ctx->done must remain unposted (no double-promotion). */
|
|
int rc = sem_trywait(&ctx->done);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(-1, rc,
|
|
"ha_record_vote_reply must NOT post ctx->done (record-only; no double-promotion)");
|
|
TEST_ASSERT_EQUAL_INT(EAGAIN, errno);
|
|
|
|
/* Grants must be recorded into the election context. */
|
|
TEST_ASSERT_EQUAL_INT(1, ctx->votes[0].vote_granted);
|
|
TEST_ASSERT_EQUAL_UINT64(5ULL, ctx->votes[0].peer_lsn);
|
|
TEST_ASSERT_EQUAL_INT(1, ctx->votes[1].vote_granted);
|
|
TEST_ASSERT_EQUAL_UINT64(5ULL, ctx->votes[1].peer_lsn);
|
|
|
|
/* The binding assertion: safe_committed_lsn = majority-held LSN = 5,
|
|
* NOT self_lsn (1). granted={self=1, peer0=5, peer1=5} → median = 5. */
|
|
uint64_t safe_lsn = 0;
|
|
uint64_t safe_tag = 0;
|
|
ha_compute_safe_committed_lsn(ctx, &safe_lsn, &safe_tag);
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, safe_lsn,
|
|
"safe_committed_lsn must reflect the majority-durable LSN (5), not self_lsn (1)");
|
|
|
|
free_elect_ctx(ctx);
|
|
ha_fini(&ha);
|
|
printf("test_safe_commit_reflects_granting_voter_lsn: PASS\n");
|
|
}
|
|
|
|
/* ── bug-1060: a new leader must NEVER truncate its own committed entries ─────
|
|
*
|
|
* Leader Completeness (Raft §5.4): a server that wins an election must already
|
|
* hold every entry committed by a prior leader, and it must never truncate its
|
|
* own log to "catch down" to a slower majority.
|
|
*
|
|
* Production root cause (bug-1060): raft_cb_become_leader truncated its WAL down
|
|
* to ha_compute_safe_committed_lsn()'s majority-threshold element. When the new
|
|
* leader durably holds entries ABOVE that element (it acked them as a standby
|
|
* under the prior primary — so they were committed), truncating to the majority
|
|
* element physically destroyed those committed entries and rewound commitIndex.
|
|
* e.g. granted voters {self=47, peer=44} → median safe_lsn=44, but self holds
|
|
* committed entries 45..47 → truncate(44) deletes 45..47. The leader then
|
|
* wrote its leadership NOP at LSN 45 (term+1) while the diverged peers still
|
|
* held the old job at LSN 45 → permanent prevote deadlock + committed-job loss.
|
|
*
|
|
* Invariant under test: the promotion truncation floor is clamped to the new
|
|
* leader's own durable tip, so persist_truncate_wal_above is a no-op on the
|
|
* leader's durable log. safe_lsn only governs removal of entries STRICTLY ABOVE
|
|
* the leader's durable tip (non-durable, divergent tail).
|
|
*
|
|
* RED before the fix: ha_promotion_truncate_floor does not exist (undefined
|
|
* reference).
|
|
*/
|
|
static void test_promotion_never_truncates_own_committed_entries(void)
|
|
{
|
|
/* The bug-1060 case: majority element (44) is BELOW the leader's own durable
|
|
* tip (47). The truncation floor must clamp UP to the durable tip so the
|
|
* leader's committed entries 45..47 are never cut. */
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
|
47ULL, ha_promotion_truncate_floor(/*safe_lsn=*/44ULL, /*durable_tip=*/47ULL),
|
|
"truncation floor must never fall below the leader's own durable tip "
|
|
"(Leader Completeness: a new leader never truncates its own committed log)");
|
|
|
|
/* Equal: floor stays at the durable tip (no truncation). */
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
|
47ULL, ha_promotion_truncate_floor(47ULL, 47ULL),
|
|
"equal safe_lsn and durable tip → floor at the tip (no-op truncation)");
|
|
|
|
/* safe_lsn ABOVE the durable tip: only a divergent tail strictly above the
|
|
* leader's durable log can be removed — safe_lsn governs there. */
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
|
50ULL, ha_promotion_truncate_floor(50ULL, 47ULL),
|
|
"when safe_lsn exceeds the durable tip, the floor follows safe_lsn "
|
|
"(removes only the non-durable divergent tail above the leader's log)");
|
|
|
|
/* Single-node / fresh boot: durable tip 0 → floor follows safe_lsn. */
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
|
0ULL, ha_promotion_truncate_floor(0ULL, 0ULL),
|
|
"empty log: floor 0 (nothing to truncate)");
|
|
|
|
printf("test_promotion_never_truncates_own_committed_entries: PASS\n");
|
|
}
|
|
|
|
/* ── Test suite entry point ───────────────────────────────────────────────── */
|
|
|
|
void test_ha_election(void)
|
|
{
|
|
/* Req 2.1 — quorum victory and defeat */
|
|
RUN_TEST(test_quorum_2_of_3_wins_election);
|
|
RUN_TEST(test_quorum_exactly_majority_wins);
|
|
RUN_TEST(test_quorum_1_of_3_loses_election);
|
|
|
|
/* Req 2.2 — lower-term denial */
|
|
RUN_TEST(test_vote_request_lower_term_denied);
|
|
|
|
/* Req 2.3 — higher-term: update term, clear voted_for, grant */
|
|
RUN_TEST(test_vote_request_higher_term_updates_term_and_clears_vote);
|
|
|
|
/* Req 2.4 — freshness gate */
|
|
RUN_TEST(test_vote_denied_when_candidate_config_tag_lower);
|
|
RUN_TEST(test_vote_denied_when_same_tag_but_candidate_lsn_lower);
|
|
RUN_TEST(test_vote_granted_when_candidate_fresher_than_receiver);
|
|
RUN_TEST(test_vote_granted_when_candidate_equally_fresh);
|
|
|
|
/* Req 2.5 — in_catch_up standby vote denial */
|
|
RUN_TEST(test_vote_denied_when_receiver_in_catch_up);
|
|
|
|
/* Req 2.6 — voted_for enforcement */
|
|
RUN_TEST(test_vote_denied_for_second_candidate_same_term);
|
|
RUN_TEST(test_vote_granted_to_same_candidate_same_term);
|
|
|
|
/* Req 2.7 — fresher-peer yield gate */
|
|
RUN_TEST(test_candidate_yields_to_fresher_responding_peer);
|
|
RUN_TEST(test_candidate_wins_when_fresher_than_all_peers);
|
|
|
|
/* Req 2.8 — zero config_tag candidate must not win when peer has config */
|
|
RUN_TEST(test_zero_config_tag_candidate_does_not_win_when_peer_has_config);
|
|
|
|
/* Req 2.9 — single-node self-election */
|
|
RUN_TEST(test_single_node_self_elects_as_primary);
|
|
|
|
/* Req 2.10 — split vote triggers retry with incremented term */
|
|
RUN_TEST(test_split_vote_triggers_retry_with_incremented_term);
|
|
RUN_TEST(test_split_vote_full_retry_increments_term);
|
|
|
|
/* Req 2.11 — second failure → CTRL_STANDBY */
|
|
RUN_TEST(test_two_failed_elections_result_in_standby);
|
|
|
|
/* Req 2.12 — identical peers: jitter, not priority */
|
|
RUN_TEST(test_identical_peers_use_jitter_not_priority);
|
|
|
|
/* Misc / self-vote */
|
|
RUN_TEST(test_vote_request_from_self_same_term_granted);
|
|
|
|
/* handle_request_vote_reply — reply handler counting and signalling */
|
|
RUN_TEST(test_reply_handler_quorum_reached_posts_done);
|
|
RUN_TEST(test_reply_handler_partial_replies_do_not_post_done);
|
|
RUN_TEST(test_reply_handler_all_denied_posts_done_on_last_reply);
|
|
|
|
/* bug-1095 / bug-1096 — controller-level Election Safety guard over the real
|
|
* raft_on_vote_reply (ha-term-safety R1 / "two leaders never share a term") */
|
|
RUN_TEST(test_vote_reply_duplicate_grant_does_not_promote);
|
|
RUN_TEST(test_vote_reply_stale_term_does_not_count_or_promote);
|
|
|
|
/* bug-828 — safe-commit-from-votes must reflect the granting voter LSN */
|
|
RUN_TEST(test_safe_commit_reflects_granting_voter_lsn);
|
|
|
|
/* bug-1060 — a new leader must never truncate its own committed entries */
|
|
RUN_TEST(test_promotion_never_truncates_own_committed_entries);
|
|
}
|