lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(ha-term-safety): bug-828 partial — revive dead safe-commit-from-votes + advertise real durable tip

Browse source 
Clean failover was truncating committed jobs because the post-election
safe-commit-from-votes computation was dead: vote_reply_dispatch decoded each
granting voter's advertised peerLastLogLsn but discarded it, so ha_compute_safe
_committed_lsn saw only the self-vote (the 'from 1 granted voters' fingerprint)
and truncated to self_lsn.

Part 1: vote_reply_dispatch now records the granting voter's (lsn,tag,grant) into
ctx->votes[] via a record-only ha_record_vote_reply seam (no double-promotion;
raft_on_vote_reply stays the sole production promotion driver) — so safe-commit
reflects the true voted majority.
Part 2: the candidate/voter advertise the real DURABLE (fsync'd) tip; persist_wal
_write_replicated advances durable_lsn after its fail-loud inline fdatasync.

Reviewer-verified SAFE (advertises only durable positions; safe-commit bounded by
the durable majority, preserving more committed entries, never marking a
non-majority-durable LSN safe). 177 unit tests; new test_ha_vote_gate + election
tests green.

PARTIAL: does NOT yet make the live clean-failover pass. A deeper slice of
bug-828/795 remains — a catch-up'd standby still under-advertises its durable
position in the live run (the documented architectural catch-up/durability gap),
so a job-less node can still win and truncate. Tracked for a dedicated effort.
This commit is contained in:
Rongsong Shen 2026-06-23 03:13:02 +00:00
parent 14c41cb298
commit 91b9a445f0
6 changed files with 313 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

77
src/controller/testsuite/test_ha_election.c
View file

@ -116,6 +116,17 @@ void handle_request_vote_reply(ha_state_t *ha, ha_elect_ctx_t *ctx,
int vote_granted,
uint64_t peer_config_tag, uint64_t peer_lsn);
/* bug-828: record-only vote-reply seam. Records ctx->votes[peer_idx] (granted
* flag + peer_tag + peer_lsn) under ctx->lock with the idempotency guard, but
* does NOT call ha_elect_evaluate_and_signal (the raft library owns promotion
* in production). This is the helper the production vote_reply_dispatch path
* now invokes so the granting voter's advertised LSN reaches the election ctx
* and ha_compute_safe_committed_lsn can compute the true majority-held LSN. */
void ha_record_vote_reply(ha_state_t *ha, ha_elect_ctx_t *ctx,
int peer_idx, uint64_t term,
int vote_granted,
uint64_t peer_config_tag, uint64_t peer_lsn);
/* ── Minimal ha_config_t used across all election tests ─────────────────── */
static ha_config_t g_elect_cfg = {
@ -927,6 +938,69 @@ static void test_reply_handler_all_denied_posts_done_on_last_reply(void)
printf("test_reply_handler_all_denied_posts_done_on_last_reply: PASS\n");
}
/* ── bug-828: safe-commit-from-votes must reflect the voted majority ─────────
*
* Production root cause: vote_reply_dispatch decoded the granting voter's
* advertised LSN but discarded it, so ctx->votes[i].vote_granted stayed 0 and
* ha_compute_safe_committed_lsn saw only the self-vote (n_granted=1) →
* safe_committed_lsn = self_lsn. A standby promoting with self_lsn=1 then
* truncated committed entries 2..5.
*
* This test drives the record-only seam (ha_record_vote_reply) that the
* production path now invokes: two peers GRANT advertising peer_lsn=5; the
* candidate's own self_lsn=1. ha_compute_safe_committed_lsn must return the
* majority-held LSN (5), NOT self_lsn (1).
*
* RED before the fix: ha_record_vote_reply does not exist (undefined reference);
* and even if grants were not recorded, safe_committed would be self_lsn=1.
*/
static void test_safe_commit_reflects_granting_voter_lsn(void)
{
/* 3-node cluster: n_peers=2 other peers. */
ha_state_t ha;
ha_init(&ha, &g_elect_cfg, 0);
atomic_store_explicit(&ha.term, 1ULL, memory_order_release);
ha_elect_ctx_t *ctx = alloc_elect_ctx(2);
/* Candidate's own (boot) position: durable floor 1, the catch-up node case. */
ctx->self_lsn = 1ULL;
ctx->self_tag = 100ULL;
ctx->votes[0].my_tag = 100ULL;
ctx->votes[0].my_lsn = 1ULL;
/* Both peers GRANT, each advertising a durable position of LSN 5
* (they hold the committed prefix). Record via the production seam. */
ha_record_vote_reply(&ha, ctx, 0, 1ULL, 1, 100ULL, 5ULL);
ha_record_vote_reply(&ha, ctx, 1, 1ULL, 1, 100ULL, 5ULL);
/* The recording must NOT have signalled the legacy promotion path —
* ha_record_vote_reply records only; the raft library owns promotion.
* ctx->done must remain unposted (no double-promotion). */
int rc = sem_trywait(&ctx->done);
TEST_ASSERT_EQUAL_INT_MESSAGE(-1, rc,
"ha_record_vote_reply must NOT post ctx->done (record-only; no double-promotion)");
TEST_ASSERT_EQUAL_INT(EAGAIN, errno);
/* Grants must be recorded into the election context. */
TEST_ASSERT_EQUAL_INT(1, ctx->votes[0].vote_granted);
TEST_ASSERT_EQUAL_UINT64(5ULL, ctx->votes[0].peer_lsn);
TEST_ASSERT_EQUAL_INT(1, ctx->votes[1].vote_granted);
TEST_ASSERT_EQUAL_UINT64(5ULL, ctx->votes[1].peer_lsn);
/* The binding assertion: safe_committed_lsn = majority-held LSN = 5,
* NOT self_lsn (1). granted={self=1, peer0=5, peer1=5} → median = 5. */
uint64_t safe_lsn = 0;
uint64_t safe_tag = 0;
ha_compute_safe_committed_lsn(ctx, &safe_lsn, &safe_tag);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, safe_lsn,
"safe_committed_lsn must reflect the majority-durable LSN (5), not self_lsn (1)");
free_elect_ctx(ctx);
ha_fini(&ha);
printf("test_safe_commit_reflects_granting_voter_lsn: PASS\n");
}
/* ── Test suite entry point ───────────────────────────────────────────────── */
void test_ha_election(void)
@ -982,4 +1056,7 @@ void test_ha_election(void)
RUN_TEST(test_reply_handler_quorum_reached_posts_done);
RUN_TEST(test_reply_handler_partial_replies_do_not_post_done);
RUN_TEST(test_reply_handler_all_denied_posts_done_on_last_reply);
/* bug-828 — safe-commit-from-votes must reflect the granting voter LSN */
RUN_TEST(test_safe_commit_reflects_granting_voter_lsn);
}

91
src/controller/testsuite/test_ha_vote_gate.c
View file

@ -967,6 +967,93 @@ static void test_longest_durable_candidate_grantable_no_livelock(void)
printf("test_longest_durable_candidate_grantable_no_livelock: PASS\n");
}
/* ══════════════════════════════════════════════════════════════════════════
* bug-828 part 2 — a standby caught up POST-BOOT must advertise its LIVE durable
* tip, not its (lower) boot floor, and that live tip must DOMINATE a lagging
* learned_committed.
*
* Live failover evidence (fix828c_r1..r3): after a clean failover the two
* surviving voters advertised position 1 and ha_compute_safe_committed_lsn
* returned safe=1 — but in those runs the standbys were GENUINELY durable only
* at LSN 1 (their WAL physically held 1..2; the committed-prefix loss is the
* UPSTREAM replication/commit-barrier durability gap bug-795/661, where the
* primary advanced committed_lsn to 5 before the surviving majority was durable
* there — out of scope here).
*
* This test pins the IN-SCOPE contract: GIVEN a standby that IS durable at its
* live tip (durable_lsn=5, advanced via persist_wal_write_replicated's inline
* fdatasync → the part-1 secondary fix, mirrored here by make_durable_persist's
* explicit flush), even though its learned_committed lags at 1, it must:
* (a) ADVERTISE 5 as both the candidate self-LSN (ha_candidate_advertise_lsn)
* and the voter floor LSN (ha_vote_self_freshness over persist_durable_floor),
* (b) so two such granting voters drive ha_compute_safe_committed_lsn to 5,
* NOT to the lagging learned_committed (1).
*
* SAFETY: the advertised position is the DURABLE (fdatasync'd) tip — a position
* the node can back from its own WAL — never a merely-received/appended tip.
* persist_durable_lsn == persist_durable_floor (both read p->durable_lsn, the
* fsync watermark); make_durable_persist advances it only via persist_flush, so
* the advertised 5 is fully durable.
* ════════════════════════════════════════════════════════════════════════ */
static void test_live_durable_tip_advertised_over_lagging_learned(void)
{
/* Real persist durable at LSN 5 (explicit flush → durable_lsn=5). This is the
* caught-up standby: durable tip 5, learned_committed will lag at 1. */
persist_t *p = make_durable_persist("/tmp/xce_828p2_XXXXXX", 5);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_UINT64(5ULL, persist_durable_floor(p));
TEST_ASSERT_EQUAL_UINT64(5ULL, persist_durable_lsn(p)); /* same field */
/* (a) Candidate self-advertise = the BARE durable tip = 5 (never the boot
* floor, never the lagging learned value). */
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, ha_candidate_advertise_lsn(p),
"candidate must advertise its LIVE durable tip (5), the fsync'd watermark");
/* (a) Voter floor LSN over the durable tip, with learned_committed LAGGING
* at 1: max(durable_floor=5, learned=1) = 5. The durable tip dominates. */
ha_state_t ha;
ha_init(&ha, &g_gate_cfg, 0);
atomic_store_explicit(&ha.committed_config_tag, 100ULL, memory_order_release);
atomic_store_explicit(&ha.last_known_committed_lsn, 1ULL, memory_order_release);
uint64_t my_tag = 0, my_lsn = 0;
ha_vote_self_freshness(&ha, 100ULL /*local tag*/,
persist_durable_floor(p) /*durable local LSN = 5*/,
&my_tag, &my_lsn);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, my_lsn,
"voter floor LSN must be the LIVE durable tip (5), dominating a lagging "
"learned_committed (1) — this is what RequestVoteReply.peerLastLogLsn "
"advertises");
/* (b) Two granting voters each advertising 5 must drive the post-election
* safe-commit to 5, NOT to the lagging learned (1). Build a 3-node election
* ctx: self (the catch-up candidate, durable 5) + 2 peers both advertising 5. */
ha_elect_ctx_t *ctx =
calloc(1, sizeof(*ctx) + 2 * sizeof(ctx->votes[0]));
TEST_ASSERT_NOT_NULL(ctx);
ctx->n_peers = 2;
ctx->self_lsn = ha_candidate_advertise_lsn(p); /* 5 */
ctx->self_tag = 100ULL;
ctx->votes[0].vote_granted = 1;
ctx->votes[0].peer_lsn = 5ULL;
ctx->votes[0].peer_tag = 100ULL;
ctx->votes[1].vote_granted = 1;
ctx->votes[1].peer_lsn = 5ULL;
ctx->votes[1].peer_tag = 100ULL;
uint64_t safe_lsn = 0, safe_tag = 0;
ha_compute_safe_committed_lsn(ctx, &safe_lsn, &safe_tag);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, safe_lsn,
"safe_committed_lsn must reflect the majority-durable tip (5), not the "
"lagging learned_committed (1) — the durable tip dominates");
free(ctx);
ha_fini(&ha);
free_durable_persist(p);
printf("test_live_durable_tip_advertised_over_lagging_learned: PASS\n");
}
/* ── Test suite entry point ───────────────────────────────────────────────── */
void test_ha_vote_gate(void)
@ -998,6 +1085,10 @@ void test_ha_vote_gate(void)
RUN_TEST(test_voter_floor_uses_durable_tip_denies_truncated);
RUN_TEST(test_longest_durable_candidate_grantable_no_livelock);
/* bug-828 part 2 — a caught-up standby advertises its LIVE durable tip
* (dominating a lagging learned_committed); safe-commit-from-votes reaches it */
RUN_TEST(test_live_durable_tip_advertised_over_lagging_learned);
/* Task 3.1 — voter-side PreVote dispatch gate (stickiness + freshness,
* no mutation, reply term unchanged) */
RUN_TEST(test_pre_vote_primary_role_denies);

8
src/controller/testsuite/test_repl_apply.c
View file

@ -303,6 +303,14 @@ static void test_repl_write_persists_and_recovers(void)
/* current_lsn advanced to lsn+1 */
TEST_ASSERT_EQUAL_UINT64(2, p->current_lsn);
/* bug-828 secondary: the entry was fdatasync'd inline before returning, so
* durable_lsn must reflect the synced tip (lsn 1) IMMEDIATELY — not lag
* behind waiting for the cadence flusher. A standby advertises this as its
* durable floor (RequestVoteReply.peerLastLogLsn) during a vote; if it lags,
* a clean failover's safe-commit-from-votes under-counts. RED before fix. */
TEST_ASSERT_EQUAL_UINT64_MESSAGE(1ULL, persist_durable_lsn(p),
"durable_lsn must advance to the synced tip after persist_wal_write_replicated");
/* A .wal segment must now exist on disk. */
TEST_ASSERT_TRUE(count_replwr_wal_files(tmpdir) >= 1);
TEST_ASSERT_TRUE(total_replwr_wal_bytes(tmpdir) > 0);