fix(ha-term-safety): bug-828 partial — revive dead safe-commit-from-votes + advertise real durable tip
Clean failover was truncating committed jobs because the post-election safe-commit-from-votes computation was dead: vote_reply_dispatch decoded each granting voter's advertised peerLastLogLsn but discarded it, so ha_compute_safe _committed_lsn saw only the self-vote (the 'from 1 granted voters' fingerprint) and truncated to self_lsn. Part 1: vote_reply_dispatch now records the granting voter's (lsn,tag,grant) into ctx->votes[] via a record-only ha_record_vote_reply seam (no double-promotion; raft_on_vote_reply stays the sole production promotion driver) — so safe-commit reflects the true voted majority. Part 2: the candidate/voter advertise the real DURABLE (fsync'd) tip; persist_wal _write_replicated advances durable_lsn after its fail-loud inline fdatasync. Reviewer-verified SAFE (advertises only durable positions; safe-commit bounded by the durable majority, preserving more committed entries, never marking a non-majority-durable LSN safe). 177 unit tests; new test_ha_vote_gate + election tests green. PARTIAL: does NOT yet make the live clean-failover pass. A deeper slice of bug-828/795 remains — a catch-up'd standby still under-advertises its durable position in the live run (the documented architectural catch-up/durability gap), so a job-less node can still win and truncate. Tracked for a dedicated effort.
This commit is contained in:
parent
14c41cb298
commit
91b9a445f0
6 changed files with 313 additions and 20 deletions
|
|
@ -967,6 +967,93 @@ static void test_longest_durable_candidate_grantable_no_livelock(void)
|
|||
printf("test_longest_durable_candidate_grantable_no_livelock: PASS\n");
|
||||
}
|
||||
|
||||
/* ══════════════════════════════════════════════════════════════════════════
|
||||
* bug-828 part 2 — a standby caught up POST-BOOT must advertise its LIVE durable
|
||||
* tip, not its (lower) boot floor, and that live tip must DOMINATE a lagging
|
||||
* learned_committed.
|
||||
*
|
||||
* Live failover evidence (fix828c_r1..r3): after a clean failover the two
|
||||
* surviving voters advertised position 1 and ha_compute_safe_committed_lsn
|
||||
* returned safe=1 — but in those runs the standbys were GENUINELY durable only
|
||||
* at LSN 1 (their WAL physically held 1..2; the committed-prefix loss is the
|
||||
* UPSTREAM replication/commit-barrier durability gap bug-795/661, where the
|
||||
* primary advanced committed_lsn to 5 before the surviving majority was durable
|
||||
* there — out of scope here).
|
||||
*
|
||||
* This test pins the IN-SCOPE contract: GIVEN a standby that IS durable at its
|
||||
* live tip (durable_lsn=5, advanced via persist_wal_write_replicated's inline
|
||||
* fdatasync → the part-1 secondary fix, mirrored here by make_durable_persist's
|
||||
* explicit flush), even though its learned_committed lags at 1, it must:
|
||||
* (a) ADVERTISE 5 as both the candidate self-LSN (ha_candidate_advertise_lsn)
|
||||
* and the voter floor LSN (ha_vote_self_freshness over persist_durable_floor),
|
||||
* (b) so two such granting voters drive ha_compute_safe_committed_lsn to 5,
|
||||
* NOT to the lagging learned_committed (1).
|
||||
*
|
||||
* SAFETY: the advertised position is the DURABLE (fdatasync'd) tip — a position
|
||||
* the node can back from its own WAL — never a merely-received/appended tip.
|
||||
* persist_durable_lsn == persist_durable_floor (both read p->durable_lsn, the
|
||||
* fsync watermark); make_durable_persist advances it only via persist_flush, so
|
||||
* the advertised 5 is fully durable.
|
||||
* ════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
static void test_live_durable_tip_advertised_over_lagging_learned(void)
|
||||
{
|
||||
/* Real persist durable at LSN 5 (explicit flush → durable_lsn=5). This is the
|
||||
* caught-up standby: durable tip 5, learned_committed will lag at 1. */
|
||||
persist_t *p = make_durable_persist("/tmp/xce_828p2_XXXXXX", 5);
|
||||
TEST_ASSERT_NOT_NULL(p);
|
||||
TEST_ASSERT_EQUAL_UINT64(5ULL, persist_durable_floor(p));
|
||||
TEST_ASSERT_EQUAL_UINT64(5ULL, persist_durable_lsn(p)); /* same field */
|
||||
|
||||
/* (a) Candidate self-advertise = the BARE durable tip = 5 (never the boot
|
||||
* floor, never the lagging learned value). */
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, ha_candidate_advertise_lsn(p),
|
||||
"candidate must advertise its LIVE durable tip (5), the fsync'd watermark");
|
||||
|
||||
/* (a) Voter floor LSN over the durable tip, with learned_committed LAGGING
|
||||
* at 1: max(durable_floor=5, learned=1) = 5. The durable tip dominates. */
|
||||
ha_state_t ha;
|
||||
ha_init(&ha, &g_gate_cfg, 0);
|
||||
atomic_store_explicit(&ha.committed_config_tag, 100ULL, memory_order_release);
|
||||
atomic_store_explicit(&ha.last_known_committed_lsn, 1ULL, memory_order_release);
|
||||
|
||||
uint64_t my_tag = 0, my_lsn = 0;
|
||||
ha_vote_self_freshness(&ha, 100ULL /*local tag*/,
|
||||
persist_durable_floor(p) /*durable local LSN = 5*/,
|
||||
&my_tag, &my_lsn);
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, my_lsn,
|
||||
"voter floor LSN must be the LIVE durable tip (5), dominating a lagging "
|
||||
"learned_committed (1) — this is what RequestVoteReply.peerLastLogLsn "
|
||||
"advertises");
|
||||
|
||||
/* (b) Two granting voters each advertising 5 must drive the post-election
|
||||
* safe-commit to 5, NOT to the lagging learned (1). Build a 3-node election
|
||||
* ctx: self (the catch-up candidate, durable 5) + 2 peers both advertising 5. */
|
||||
ha_elect_ctx_t *ctx =
|
||||
calloc(1, sizeof(*ctx) + 2 * sizeof(ctx->votes[0]));
|
||||
TEST_ASSERT_NOT_NULL(ctx);
|
||||
ctx->n_peers = 2;
|
||||
ctx->self_lsn = ha_candidate_advertise_lsn(p); /* 5 */
|
||||
ctx->self_tag = 100ULL;
|
||||
ctx->votes[0].vote_granted = 1;
|
||||
ctx->votes[0].peer_lsn = 5ULL;
|
||||
ctx->votes[0].peer_tag = 100ULL;
|
||||
ctx->votes[1].vote_granted = 1;
|
||||
ctx->votes[1].peer_lsn = 5ULL;
|
||||
ctx->votes[1].peer_tag = 100ULL;
|
||||
|
||||
uint64_t safe_lsn = 0, safe_tag = 0;
|
||||
ha_compute_safe_committed_lsn(ctx, &safe_lsn, &safe_tag);
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, safe_lsn,
|
||||
"safe_committed_lsn must reflect the majority-durable tip (5), not the "
|
||||
"lagging learned_committed (1) — the durable tip dominates");
|
||||
|
||||
free(ctx);
|
||||
ha_fini(&ha);
|
||||
free_durable_persist(p);
|
||||
printf("test_live_durable_tip_advertised_over_lagging_learned: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test suite entry point ───────────────────────────────────────────────── */
|
||||
|
||||
void test_ha_vote_gate(void)
|
||||
|
|
@ -998,6 +1085,10 @@ void test_ha_vote_gate(void)
|
|||
RUN_TEST(test_voter_floor_uses_durable_tip_denies_truncated);
|
||||
RUN_TEST(test_longest_durable_candidate_grantable_no_livelock);
|
||||
|
||||
/* bug-828 part 2 — a caught-up standby advertises its LIVE durable tip
|
||||
* (dominating a lagging learned_committed); safe-commit-from-votes reaches it */
|
||||
RUN_TEST(test_live_durable_tip_advertised_over_lagging_learned);
|
||||
|
||||
/* Task 3.1 — voter-side PreVote dispatch gate (stickiness + freshness,
|
||||
* no mutation, reply term unchanged) */
|
||||
RUN_TEST(test_pre_vote_primary_role_denies);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue