lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-term-safety): drive ACK path through raft_on_append_entries_reply + advance_commit (task 3.3)

Browse source 
Includes fix for bug-960: endpoint-type-aware peer-index resolution
(promoted-vote relative vs heartbeat-standby absolute) with fail-loud
bounds check, preventing an out-of-bounds match_index[] write / wrong-peer
false-quorum on the replication ACK path.
This commit is contained in:
Rongsong Shen 2026-06-22 09:25:44 +00:00
parent 754e586f8f
commit f1783f7952
4 changed files with 466 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

318
src/controller/testsuite/test_term_safety_adapter.c
View file

@ -1,4 +1,5 @@
#define _GNU_SOURCE
#include <errno.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
@ -1276,6 +1277,317 @@ static void test_intake_logmismatch_rejected_no_false_convergence(void)
printf("test_intake_logmismatch_rejected_no_false_convergence: PASS\n");
}
/* ================================================================== */
/* Task 3.3 — drive raft_on_append_entries_reply + raft_advance_commit */
/* on the replication ACK path */
/* ================================================================== */
/* These tests drive the ACK seam ha_raft_on_repl_ack(c, abs_peer_idx, acked_lsn,
* reply_term): on a replication ACK the controller routes the reply through
* raft_on_append_entries_reply (updating match_index[]/next_index[]) then
* raft_advance_commit, which recomputes the current-term-gated commit and MAY
* fire on_commit_advanced (the task-2.4 callback that feeds the barrier).
*
* Peer-index contract (REMEDIATION round 1): ha_raft_on_repl_ack now takes the
* ABSOLUTE library peer index directly (the match_index[]/next_index[] index
* space, j != self_idx). The CALLER resolves it per endpoint type via
* ha_repl_ack_abs_peer_idx BEFORE this seam — promoted-vote eps carry a relative
* "votes" index (remapped), heartbeat standby eps carry an already-absolute
* index (passed through). The earlier blind remap inside the seam corrupted the
* heartbeat path (wrong-peer false quorum / OOB match_index[] write); the
* endpoint-type round-trip is pinned by the two remediation tests below.
*
* Fixture: make_leader(c, term, n_peers) brings the node up as the raft LEADER
* with self_idx == my_priority == 0 (make_cluster calloc's my_priority=0), so
* the only standby's absolute peer index is 1.
*
* Against the pre-3.3 ACK path (repl_on_ack durability only, no library reply),
* ha_raft_on_repl_ack does not exist (compile RED); once wired, a current-term
* quorum ACK MUST advance the library commit_index and fire on_commit_advanced
* (R4.1/R4.5), a prior-term agreed entry MUST hold, and a stale-term reply MUST
* be dropped. */
/* Seed `count` entries at the given term into the leader's local WAL via the
* wired on_persist_entry, growing the library log_len in lockstep. */
static void leader_seed_entries(cluster_t* c, uint64_t term, int count)
{
uint8_t blob[64];
for (int i = 1; i <= count; i++) {
size_t n = pack_entry(blob, WAL_JOB_UPDATE, "x", 1);
TEST_ASSERT_EQUAL_INT(0,
ha_cb_persist_entry_for_test(&c->ha, (uint64_t)i, term, blob, n));
c->ha.raft.log_len = (uint64_t)i;
}
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
}
/* (a) A current-term quorum ACK drives raft_on_append_entries_reply →
* raft_advance_commit → on_commit_advanced fires with the agreed commit_index,
* which raises the barrier ceiling. 3-peer cluster (self + 2 standbys, quorum=2):
* self holds LSN 1,2 at the current term; one standby ACKs LSN 2 → {self, peer1}
* is a quorum for LSN 2, the agreed entry is current-term, so commit advances to
* 2 and the library ceiling (commit_set_raft_commit_index effect) caps committed
* at the library point under a durable majority. */
static void test_repl_ack_currentterm_quorum_advances_commit(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_ack_adv_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
const uint64_t T = 6;
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
make_leader(c, T, /*n_peers*/3);
/* Init the cluster's commit engine over the same 3-peer cluster so a
* majority is self + 1 standby, and register one standby whose durable ACK
* supplies the durable-majority floor. */
TEST_ASSERT_EQUAL_INT(0,
commit_engine_init(&c->commit, &c->persist, /*n_peers*/3, /*n_vcpus*/8));
/* Seed LSN 1,2 at the current term T → durable floor reaches LSN 2. */
leader_seed_entries(c, T, /*count*/2);
TEST_ASSERT_EQUAL_INT(0,
repl_add_standby(&c->persist.repl, &g_ae_dummy_ep, "sb0", 7000));
/* Durable majority: the standby's durable position covers LSN 2 so the
* barrier ceiling is not held below the library commit point. */
atomic_store(&c->persist.repl.standbys[0].acked_lsn, 2);
commit_on_ack(&c->commit, /*peer_idx*/0, /*acked_lsn*/2);
/* Nothing committed yet: the library has not advanced commit_index (no ACK
* routed through the library) so the commit source is still inert at 0. */
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.commit_index);
/* The standby ACKs LSN 2 at the leader's current term. The caller has
* already resolved the absolute peer index (self_idx 0 → the only standby is
* absolute peer 1); the seam takes that absolute index directly. */
int rc = ha_raft_on_repl_ack(c, /*abs_peer_idx*/1, /*acked_lsn*/2, /*reply_term*/T);
TEST_ASSERT_EQUAL_INT(0, rc);
/* (1) The library recorded the peer's match_index at the absolute index. */
TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.match_index[1]);
TEST_ASSERT_EQUAL_UINT64(3, c->ha.raft.next_index[1]);
/* (2) raft_advance_commit advanced the current-term-gated commit_index to 2
* ({self, peer1} quorum, entry at LSN 2 is term T). */
TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.commit_index);
/* (3) on_commit_advanced fired → the barrier ceiling rose and committed_lsn
* advanced to the library point under the durable majority (R4.1). */
TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn));
commit_engine_fini(&c->commit);
teardown_cluster(c);
printf("test_repl_ack_currentterm_quorum_advances_commit: PASS\n");
}
/* (b) A quorum that agrees only on a PRIOR-term entry does NOT advance commit
* (the Figure-8 current-term rule, R4.1/R4.5). self holds LSN 1@term3 (a
* prior-term entry) and is now leader at term 6 with no current-term entry yet.
* A standby ACKs LSN 1 → {self, peer1} is a quorum for LSN 1, but the entry at
* LSN 1 is term 3 != current term 6, so raft_advance_commit holds commit_index
* at 0 and on_commit_advanced never fires. */
static void test_repl_ack_priorterm_quorum_holds_commit(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_ack_prior_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
const uint64_t PRIOR = 3;
const uint64_t CUR = 6;
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
make_leader(c, CUR, /*n_peers*/3);
/* Seed a single PRIOR-term entry at LSN 1 (replicated from an old leader). */
leader_seed_entries(c, PRIOR, /*count*/1);
/* A standby ACKs LSN 1 at the leader's current term (the wire reply term is
* the leader's term; the ENTRY's term is what gates the commit). */
int rc = ha_raft_on_repl_ack(c, /*abs_peer_idx*/1, /*acked_lsn*/1, /*reply_term*/CUR);
TEST_ASSERT_EQUAL_INT(0, rc);
/* match_index updated (the reply itself is current-term, not stale). */
TEST_ASSERT_EQUAL_UINT64(1, c->ha.raft.match_index[1]);
/* But commit is HELD: the agreed entry (LSN1@term3) is not current-term →
* Figure-8 rule blocks the advance, commit_index stays 0. */
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.commit_index);
teardown_cluster(c);
printf("test_repl_ack_priorterm_quorum_holds_commit: PASS\n");
}
/* (c) A stale-term reply is dropped: raft_on_append_entries_reply discards a
* reply whose term != s->term, so match_index[] is NOT updated and no commit
* advance occurs (R4.5 / raft.tla 415). self holds LSN 1,2 at the current term;
* a stale reply (reply_term < current term) for LSN 2 must leave match_index and
* commit_index untouched. */
static void test_repl_ack_staleterm_reply_dropped(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_ack_stale_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
const uint64_t T = 8;
const uint64_t STALE = 5;
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
make_leader(c, T, /*n_peers*/3);
leader_seed_entries(c, T, /*count*/2);
/* Stale-term ACK for LSN 2 → dropped by the library before any state change. */
int rc = ha_raft_on_repl_ack(c, /*abs_peer_idx*/1, /*acked_lsn*/2, /*reply_term*/STALE);
TEST_ASSERT_EQUAL_INT(0, rc);
/* No match_index update (the peer's slot stays at the post-init default 0). */
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.match_index[1]);
/* No commit advance. */
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.commit_index);
teardown_cluster(c);
printf("test_repl_ack_staleterm_reply_dropped: PASS\n");
}
/* ================================================================== */
/* Task 3.3 REMEDIATION round 1 — endpoint-type-correct peer mapping */
/* ================================================================== */
/* (d) The bug: the ACK walAck branch fires for BOTH promoted-vote eps (relative
* votes index) AND heartbeat standby eps (already-absolute index). The earlier
* seam unconditionally remapped, so a heartbeat ep's absolute index was mapped a
* SECOND time — crediting the WRONG peer (false quorum) or writing OUT OF BOUNDS.
* ha_repl_ack_abs_peer_idx must remap ONLY the vote ep and pass the standby ep
* through unchanged. We pin both endpoint types AND both my_priority values so
* the remap-vs-passthrough distinction actually round-trips (the pre-fix tests
* only used my_priority==0, which masked the bug).
*
* Round-trip oracle: for a vote ep, peer absolute index i is sent as votes_idx
* (i < self ? i : i-1); ha_repl_ack_abs_peer_idx must recover exactly i. For a
* standby ep, the stored peer_idx IS i and must pass through unchanged. */
static void test_repl_ack_abs_peer_idx_endpoint_type_roundtrip(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_ack_map_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
/* --- self_idx (my_priority) == 0: every peer is after self --- */
c->ha.my_priority = 0;
/* Heartbeat standby ep (ctx==NULL → ep_is_vote=0): absolute index passes
* through UNCHANGED. Absolute peer 1 must stay 1, absolute peer 2 stay 2.
* (The pre-fix blind remap would have turned 1→2 and 2→3 — the bug.) */
TEST_ASSERT_EQUAL_INT(1, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/0, /*peer*/1));
TEST_ASSERT_EQUAL_INT(2, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/0, /*peer*/2));
/* Promoted vote ep (ep_is_vote=1): votes_idx 0 → abs 1, votes_idx 1 → abs 2. */
TEST_ASSERT_EQUAL_INT(1, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/1, /*votes*/0));
TEST_ASSERT_EQUAL_INT(2, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/1, /*votes*/1));
/* --- self_idx (my_priority) == 1: a peer below self is NOT shifted --- */
c->ha.my_priority = 1;
/* Standby ep: absolute indices 0 and 2 (peer 1 is self) pass through. */
TEST_ASSERT_EQUAL_INT(0, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/0, /*peer*/0));
TEST_ASSERT_EQUAL_INT(2, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/0, /*peer*/2));
/* Vote ep with self_idx==1: votes_idx 0 (peer absolute 0, < self) → 0;
* votes_idx 1 (peer absolute 2, >= self) → 2. This is where remap and
* passthrough DIVERGE — the case the my_priority==0 tests could not reach. */
TEST_ASSERT_EQUAL_INT(0, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/1, /*votes*/0));
TEST_ASSERT_EQUAL_INT(2, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/1, /*votes*/1));
teardown_cluster(c);
printf("test_repl_ack_abs_peer_idx_endpoint_type_roundtrip: PASS\n");
}
/* (e) End-to-end heartbeat-standby ACK: a standby ep carries an ALREADY-ABSOLUTE
* peer index. Resolving it via ha_repl_ack_abs_peer_idx (ep_is_vote=0) and
* driving ha_raft_on_repl_ack must land match_index on the CORRECT absolute slot
* (not the +1 wrong slot the blind remap produced). 3-peer cluster, self=0; the
* standby's absolute index is 1. A current-term quorum ACK for LSN 2 advances
* the library commit_index to 2 — but ONLY because match_index[1] (not [2]) is
* credited. Under the broken blind remap the credit landed on match_index[2]
* (a phantom peer): no quorum at LSN 2, commit_index would stay 0. */
static void test_repl_ack_heartbeat_standby_absolute_slot(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_ack_hb_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
const uint64_t T = 6;
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
make_leader(c, T, /*n_peers*/3);
TEST_ASSERT_EQUAL_INT(0, c->ha.my_priority); /* self_idx 0 */
TEST_ASSERT_EQUAL_INT(0,
commit_engine_init(&c->commit, &c->persist, /*n_peers*/3, /*n_vcpus*/8));
leader_seed_entries(c, T, /*count*/2);
TEST_ASSERT_EQUAL_INT(0,
repl_add_standby(&c->persist.repl, &g_ae_dummy_ep, "sb0", 7000));
atomic_store(&c->persist.repl.standbys[0].acked_lsn, 2);
commit_on_ack(&c->commit, /*peer_idx*/0, /*acked_lsn*/2);
/* The standby ep stores the ABSOLUTE index 1 (standby_ep_ctx_t.peer_idx).
* Resolve it as a heartbeat ep (ep_is_vote=0) — it must pass through as 1. */
int abs_idx = ha_repl_ack_abs_peer_idx(&c->ha, /*ep_is_vote*/0, /*peer_idx*/1);
TEST_ASSERT_EQUAL_INT(1, abs_idx); /* NOT 2 — the blind-remap bug slot */
int rc = ha_raft_on_repl_ack(c, abs_idx, /*acked_lsn*/2, /*reply_term*/T);
TEST_ASSERT_EQUAL_INT(0, rc);
/* The CORRECT absolute slot was credited; the phantom slot 2 stayed empty. */
TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.match_index[1]);
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.match_index[2]);
/* {self, peer1} is a real quorum for LSN 2 → current-term commit advances. */
TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.commit_index);
TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn));
commit_engine_fini(&c->commit);
teardown_cluster(c);
printf("test_repl_ack_heartbeat_standby_absolute_slot: PASS\n");
}
/* (f) Defense-in-depth: an out-of-range / self absolute index is rejected loudly
* (returns -EINVAL) and performs NO library accounting — fencing the unchecked
* match_index[] write at the controller boundary rather than corrupting memory. */
static void test_repl_ack_out_of_range_peer_rejected(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_ack_oob_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
const uint64_t T = 6;
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
make_leader(c, T, /*n_peers*/3); /* valid peers are absolute 1,2 */
leader_seed_entries(c, T, /*count*/2);
/* abs index == n_peers (3) is out of range → rejected, no state change. */
TEST_ASSERT_EQUAL_INT(-EINVAL,
ha_raft_on_repl_ack(c, /*abs_peer_idx*/3, /*acked_lsn*/2, /*reply_term*/T));
/* self (0) is never a valid peer → rejected. */
TEST_ASSERT_EQUAL_INT(-EINVAL,
ha_raft_on_repl_ack(c, /*abs_peer_idx*/0, /*acked_lsn*/2, /*reply_term*/T));
/* Nothing was credited and commit did not advance. */
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.match_index[1]);
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.commit_index);
teardown_cluster(c);
printf("test_repl_ack_out_of_range_peer_rejected: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Suite entry point */
/* ------------------------------------------------------------------ */
@ -1308,4 +1620,10 @@ void test_term_safety_adapter(void)
RUN_TEST(test_intake_higher_term_heartbeat_adopts_term);
RUN_TEST(test_intake_divergent_tail_truncated_and_reconverged);
RUN_TEST(test_intake_logmismatch_rejected_no_false_convergence);
RUN_TEST(test_repl_ack_currentterm_quorum_advances_commit);
RUN_TEST(test_repl_ack_priorterm_quorum_holds_commit);
RUN_TEST(test_repl_ack_staleterm_reply_dropped);
RUN_TEST(test_repl_ack_abs_peer_idx_endpoint_type_roundtrip);
RUN_TEST(test_repl_ack_heartbeat_standby_absolute_slot);
RUN_TEST(test_repl_ack_out_of_range_peer_rejected);
}