forked from lei.chen/myxce
fix bug-1060 bug-1067
This commit is contained in:
parent
6067ba6c94
commit
61e1391cd0
28 changed files with 7021 additions and 2926 deletions
|
|
@ -336,7 +336,8 @@ static void test_h4_empty_standby_receives_lsn1_and_lsn2_no_hole(void)
|
|||
|
||||
/* Empty fresh standby = standby_lsn 0; primary committed_lsn = 2 (holds
|
||||
* [1,2]). Drive the REAL streamer. */
|
||||
int rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir,
|
||||
int rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir, /*p=*/NULL,
|
||||
/*leader_term=*/7,
|
||||
/*standby_lsn=*/0,
|
||||
/*committed_lsn=*/2);
|
||||
xce_wal_stream_request_hook = NULL;
|
||||
|
|
@ -384,7 +385,8 @@ static void test_h4_control_standby_with_lsn1_gets_only_lsn2(void)
|
|||
h4_capture_reset();
|
||||
|
||||
/* standby_lsn = 1 (already holds the first entry) catching up to committed 2. */
|
||||
int rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir,
|
||||
int rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir, /*p=*/NULL,
|
||||
/*leader_term=*/7,
|
||||
/*standby_lsn=*/1,
|
||||
/*committed_lsn=*/2);
|
||||
xce_wal_stream_request_hook = NULL;
|
||||
|
|
|
|||
|
|
@ -2586,7 +2586,8 @@ static void test_wal_stream_500_entries_two_batches(void)
|
|||
g_wal_stream_last_n_entries = 0;
|
||||
|
||||
/* Stream entries [1..500]: standby_lsn=0, committed_lsn=500 */
|
||||
rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir,
|
||||
rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir, /*p=*/NULL,
|
||||
/*leader_term=*/7,
|
||||
/*standby_lsn=*/0,
|
||||
/*committed_lsn=*/500);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "ha_stream_wal_to_standby failed");
|
||||
|
|
@ -2627,7 +2628,8 @@ static void test_wal_stream_partial_batch(void)
|
|||
g_wal_stream_last_n_entries = 0;
|
||||
|
||||
/* Stream entries [6..14]: standby_lsn=5, committed_lsn=14 */
|
||||
rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir,
|
||||
rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir, /*p=*/NULL,
|
||||
/*leader_term=*/7,
|
||||
/*standby_lsn=*/5,
|
||||
/*committed_lsn=*/14);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "ha_stream_wal_to_standby failed");
|
||||
|
|
@ -2661,7 +2663,8 @@ static void test_wal_stream_empty_range_no_batches(void)
|
|||
g_wal_stream_total_entries = 0;
|
||||
|
||||
/* standby_lsn == committed_lsn: nothing to stream */
|
||||
int rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir,
|
||||
int rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir, /*p=*/NULL,
|
||||
/*leader_term=*/7,
|
||||
/*standby_lsn=*/100,
|
||||
/*committed_lsn=*/100);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "Empty range must return 0");
|
||||
|
|
@ -2676,6 +2679,89 @@ static void test_wal_stream_empty_range_no_batches(void)
|
|||
printf("test_wal_stream_empty_range_no_batches: PASS\n");
|
||||
}
|
||||
|
||||
/*
|
||||
* bug-1028 — the catch-up streamer must source each streamed entry's term and
|
||||
* the AppendEntries prev_log_term from the AUTHORITATIVE per-LSN term index
|
||||
* (persist_entry_term), NOT the raw on-disk hdr.term. A legacy-0 entry has
|
||||
* hdr.term==0 on disk but persist_entry_term maps it to WAL_LEGACY_TERM(1) —
|
||||
* which is exactly what the standby's logOk and the library send path compute.
|
||||
* If the streamer sends the raw 0, prev_log_term(0) != standby's term(1) → logOk
|
||||
* rejects the new-term entry forever (the "stream WAL batches" wedge).
|
||||
*
|
||||
* Fixture: LSN 1 appended at term 0 (legacy), LSN 2 at term 3. Streaming
|
||||
* (standby_lsn=1, committed=2] must emit LSN 2 with term=3 and prev_log_term=1
|
||||
* (NOT 0). RED before the fix (streamer sent hdr.term=0 for the predecessor).
|
||||
*/
|
||||
static uint64_t g_cap_term_for_lsn2 = 0;
|
||||
static uint64_t g_cap_prevterm_for_lsn2 = 0;
|
||||
static int g_cap_saw_lsn2 = 0;
|
||||
|
||||
static int capture_terms_wal_stream_hook(endpoint_t *ep, xce_request_t *req)
|
||||
{
|
||||
(void)ep;
|
||||
if (!req || !req->body.walcatchupentrymsg) return 0;
|
||||
wal_catchup_entry_t *batch =
|
||||
(wal_catchup_entry_t *)req->body.walcatchupentrymsg;
|
||||
for (int i = 0; i < batch->n_entries; i++) {
|
||||
wal_catchup_entry_t *e = batch->entries[i];
|
||||
if (e && e->lsn == 2) {
|
||||
g_cap_saw_lsn2 = 1;
|
||||
g_cap_term_for_lsn2 = e->term;
|
||||
g_cap_prevterm_for_lsn2 = e->prev_log_term;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void test_wal_stream_term_from_index_not_raw_hdr(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr_streamterm_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
|
||||
/* LSN 1: legacy entry (term 0 on disk → persist_entry_term maps to 1). */
|
||||
persist_set_term(&c->persist, 0);
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(&c->persist, WAL_JOB_UPDATE,
|
||||
NULL, 0));
|
||||
/* LSN 2: real new-term entry (term 3). */
|
||||
persist_set_term(&c->persist, 3);
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(&c->persist, WAL_JOB_UPDATE,
|
||||
NULL, 0));
|
||||
|
||||
/* Sanity: the authoritative index normalises legacy-0 → 1, keeps 3. */
|
||||
TEST_ASSERT_EQUAL_UINT64(WAL_LEGACY_TERM, persist_entry_term(&c->persist, 1));
|
||||
TEST_ASSERT_EQUAL_UINT64(3, persist_entry_term(&c->persist, 2));
|
||||
|
||||
xce_wal_stream_request_hook = capture_terms_wal_stream_hook;
|
||||
g_cap_saw_lsn2 = 0; g_cap_term_for_lsn2 = 0; g_cap_prevterm_for_lsn2 = 0;
|
||||
|
||||
/* Standby holds LSN 1, catching up to committed 2 — streams just LSN 2,
|
||||
* whose prev tuple references LSN 1 (the legacy entry). */
|
||||
int rc = ha_stream_wal_to_standby(/*ep=*/NULL, c->persist.wal_path,
|
||||
&c->persist,
|
||||
/*leader_term=*/atomic_load_explicit(
|
||||
&c->ha.term, memory_order_acquire),
|
||||
/*standby_lsn=*/1, /*committed_lsn=*/2);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "ha_stream_wal_to_standby failed");
|
||||
xce_wal_stream_request_hook = NULL;
|
||||
|
||||
TEST_ASSERT_TRUE_MESSAGE(g_cap_saw_lsn2, "LSN 2 must be streamed");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(3, g_cap_term_for_lsn2,
|
||||
"streamed entry term must come from persist_entry_term (3)");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(WAL_LEGACY_TERM, g_cap_prevterm_for_lsn2,
|
||||
"bug-1028: prev_log_term must be the index-normalised legacy term (1), "
|
||||
"NOT raw on-disk hdr.term (0) — else the standby's logOk rejects forever");
|
||||
|
||||
t42_teardown(c);
|
||||
char cmd[512];
|
||||
snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
|
||||
system(cmd);
|
||||
printf("test_wal_stream_term_from_index_not_raw_hdr: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Task 3.5: Snapshot chunk transfer ──────────────────────── */
|
||||
|
||||
static snapshot_transfer_t g_snaptest_chunks[16];
|
||||
|
|
@ -3363,6 +3449,7 @@ void test_config_repl(void)
|
|||
RUN_TEST(test_wal_stream_500_entries_two_batches);
|
||||
RUN_TEST(test_wal_stream_partial_batch);
|
||||
RUN_TEST(test_wal_stream_empty_range_no_batches);
|
||||
RUN_TEST(test_wal_stream_term_from_index_not_raw_hdr); /* bug-1028 */
|
||||
/* Task 3.5: snapshot chunk transfer (sender side) */
|
||||
RUN_TEST(test_snapshot_transfer_two_chunks);
|
||||
RUN_TEST(test_snapshot_transfer_small_less_than_one_chunk);
|
||||
|
|
|
|||
|
|
@ -1001,6 +1001,60 @@ static void test_safe_commit_reflects_granting_voter_lsn(void)
|
|||
printf("test_safe_commit_reflects_granting_voter_lsn: PASS\n");
|
||||
}
|
||||
|
||||
/* ── bug-1060: a new leader must NEVER truncate its own committed entries ─────
|
||||
*
|
||||
* Leader Completeness (Raft §5.4): a server that wins an election must already
|
||||
* hold every entry committed by a prior leader, and it must never truncate its
|
||||
* own log to "catch down" to a slower majority.
|
||||
*
|
||||
* Production root cause (bug-1060): raft_cb_become_leader truncated its WAL down
|
||||
* to ha_compute_safe_committed_lsn()'s majority-threshold element. When the new
|
||||
* leader durably holds entries ABOVE that element (it acked them as a standby
|
||||
* under the prior primary — so they were committed), truncating to the majority
|
||||
* element physically destroyed those committed entries and rewound commitIndex.
|
||||
* e.g. granted voters {self=47, peer=44} → median safe_lsn=44, but self holds
|
||||
* committed entries 45..47 → truncate(44) deletes 45..47. The leader then
|
||||
* wrote its leadership NOP at LSN 45 (term+1) while the diverged peers still
|
||||
* held the old job at LSN 45 → permanent prevote deadlock + committed-job loss.
|
||||
*
|
||||
* Invariant under test: the promotion truncation floor is clamped to the new
|
||||
* leader's own durable tip, so persist_truncate_wal_above is a no-op on the
|
||||
* leader's durable log. safe_lsn only governs removal of entries STRICTLY ABOVE
|
||||
* the leader's durable tip (non-durable, divergent tail).
|
||||
*
|
||||
* RED before the fix: ha_promotion_truncate_floor does not exist (undefined
|
||||
* reference).
|
||||
*/
|
||||
static void test_promotion_never_truncates_own_committed_entries(void)
|
||||
{
|
||||
/* The bug-1060 case: majority element (44) is BELOW the leader's own durable
|
||||
* tip (47). The truncation floor must clamp UP to the durable tip so the
|
||||
* leader's committed entries 45..47 are never cut. */
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
||||
47ULL, ha_promotion_truncate_floor(/*safe_lsn=*/44ULL, /*durable_tip=*/47ULL),
|
||||
"truncation floor must never fall below the leader's own durable tip "
|
||||
"(Leader Completeness: a new leader never truncates its own committed log)");
|
||||
|
||||
/* Equal: floor stays at the durable tip (no truncation). */
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
||||
47ULL, ha_promotion_truncate_floor(47ULL, 47ULL),
|
||||
"equal safe_lsn and durable tip → floor at the tip (no-op truncation)");
|
||||
|
||||
/* safe_lsn ABOVE the durable tip: only a divergent tail strictly above the
|
||||
* leader's durable log can be removed — safe_lsn governs there. */
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
||||
50ULL, ha_promotion_truncate_floor(50ULL, 47ULL),
|
||||
"when safe_lsn exceeds the durable tip, the floor follows safe_lsn "
|
||||
"(removes only the non-durable divergent tail above the leader's log)");
|
||||
|
||||
/* Single-node / fresh boot: durable tip 0 → floor follows safe_lsn. */
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
||||
0ULL, ha_promotion_truncate_floor(0ULL, 0ULL),
|
||||
"empty log: floor 0 (nothing to truncate)");
|
||||
|
||||
printf("test_promotion_never_truncates_own_committed_entries: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test suite entry point ───────────────────────────────────────────────── */
|
||||
|
||||
void test_ha_election(void)
|
||||
|
|
@ -1059,4 +1113,7 @@ void test_ha_election(void)
|
|||
|
||||
/* bug-828 — safe-commit-from-votes must reflect the granting voter LSN */
|
||||
RUN_TEST(test_safe_commit_reflects_granting_voter_lsn);
|
||||
|
||||
/* bug-1060 — a new leader must never truncate its own committed entries */
|
||||
RUN_TEST(test_promotion_never_truncates_own_committed_entries);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -309,6 +309,75 @@ static void test_recovered_member_advertises_real_applied_lsn(void)
|
|||
printf("test_recovered_member_advertises_real_applied_lsn: PASS\n");
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* bug-1042: after recovery, the raft LIBRARY's log_len/commit_index */
|
||||
/* must be raised to the recovered position. ha_init_raft runs BEFORE */
|
||||
/* persist_recover on a standby-rejoin boot, leaving the library at */
|
||||
/* log_len=0; without ha_sync_raft_log_from_recovery the library then */
|
||||
/* rejects every AppendEntries (logOk fail for prevLogIndex>=1) and */
|
||||
/* the rejoined member never converges. */
|
||||
/* ------------------------------------------------------------------ */
|
||||
static void test_recovered_member_syncs_raft_log_len(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_rec1042_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
/* Phase 1: write WAL entries, flush, stop. */
|
||||
{
|
||||
persist_t* p = make_fresh_persist(tmp);
|
||||
TEST_ASSERT_NOT_NULL(p);
|
||||
for (int i = 0; i < 5; i++)
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
persist_wal_async(p, WAL_JOB_UPDATE, "z", 1));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
|
||||
teardown_persist(p);
|
||||
}
|
||||
|
||||
/* Phase 2: restart — recover the WAL, then mimic the init-ordering hazard:
|
||||
* the raft library was init'd with log_len=0 BEFORE recovery ran. The sync
|
||||
* must raise the library's log_len to the recovered durable tip. */
|
||||
{
|
||||
persist_t* p2 = make_fresh_persist(tmp);
|
||||
TEST_ASSERT_NOT_NULL(p2);
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_recover(p2));
|
||||
|
||||
uint64_t recovered_floor = persist_durable_floor(p2);
|
||||
TEST_ASSERT_TRUE(recovered_floor > 0);
|
||||
|
||||
ha_state_t ha;
|
||||
memset(&ha, 0, sizeof(ha));
|
||||
/* Library came up empty (ha_init_raft before recovery). */
|
||||
ha.raft.log_len = 0;
|
||||
ha.raft.commit_index = 0;
|
||||
ha.raft.term = 0;
|
||||
atomic_store_explicit(&ha.term, 4ULL, memory_order_release);
|
||||
/* The helper requires a non-NULL raft_c back-ref (production wiring). */
|
||||
ha.raft_c = (cluster_t*)0x1; /* non-NULL sentinel; helper never derefs it */
|
||||
|
||||
ha_sync_raft_log_from_recovery(&ha, p2);
|
||||
|
||||
/* The bug: log_len used to stay 0 → logOk rejects every AppendEntries. */
|
||||
TEST_ASSERT_EQUAL_UINT64(recovered_floor, ha.raft.log_len);
|
||||
/* commit_index advanced to the durable floor, clamped to log_len. */
|
||||
TEST_ASSERT_EQUAL_UINT64(recovered_floor, ha.raft.commit_index);
|
||||
TEST_ASSERT_TRUE(ha.raft.commit_index <= ha.raft.log_len);
|
||||
/* term aligned to the persisted/restored HA term. */
|
||||
TEST_ASSERT_EQUAL_UINT64(4ULL, ha.raft.term);
|
||||
|
||||
/* Monotonic: never lowers an already-higher library position. */
|
||||
ha.raft.log_len = recovered_floor + 100;
|
||||
ha.raft.commit_index = recovered_floor + 50;
|
||||
ha_sync_raft_log_from_recovery(&ha, p2);
|
||||
TEST_ASSERT_EQUAL_UINT64(recovered_floor + 100, ha.raft.log_len);
|
||||
TEST_ASSERT_EQUAL_UINT64(recovered_floor + 50, ha.raft.commit_index);
|
||||
|
||||
teardown_persist(p2);
|
||||
}
|
||||
|
||||
printf("test_recovered_member_syncs_raft_log_len: PASS\n");
|
||||
}
|
||||
|
||||
void test_persist_recovery(void)
|
||||
{
|
||||
RUN_TEST(test_recover_from_empty_state);
|
||||
|
|
@ -317,4 +386,5 @@ void test_persist_recovery(void)
|
|||
RUN_TEST(test_recover_rejects_corrupt_snapshot);
|
||||
RUN_TEST(test_recover_picks_latest_snapshot);
|
||||
RUN_TEST(test_recovered_member_advertises_real_applied_lsn);
|
||||
RUN_TEST(test_recovered_member_syncs_raft_log_len);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1529,6 +1529,141 @@ static void test_catchup_applied_lsn_is_last_held_no_skip(void)
|
|||
printf("test_catchup_applied_lsn_is_last_held_no_skip: PASS\n");
|
||||
}
|
||||
|
||||
/* ================================================================== */
|
||||
/* bug-1067 — cross-term catch-up: the AppendEntries header term is the */
|
||||
/* leader's CURRENT term, NOT the streamed entry's stored term */
|
||||
/* ================================================================== */
|
||||
|
||||
/* Root cause: ha_consume_append_entries used the entry's stored term (entry_term)
|
||||
* as the AppendEntries header term. For LIVE appends that is the leader's current
|
||||
* term (leader appends at its term), but for CATCH-UP streaming of HISTORICAL
|
||||
* entries written by a PRIOR leader, entry_term < the current leader's term. The
|
||||
* library's stale-term check (term < s->term → ae_reject_stale) then permanently
|
||||
* rejected every historical entry, so the lagging standby could never catch up
|
||||
* (standby_reported_lsn never advanced; the streamer retransmitted forever).
|
||||
*
|
||||
* The fix carries the leader's current term out-of-band per entry
|
||||
* (wal_catchup_entry_t.leader_current_term, WalCatchupEntry.leaderCurrentTerm @8),
|
||||
* set by the streamer to ha->term, and uses THAT as the AE header term while the
|
||||
* entry's stored term remains the per-entry term for logOk / on_persist_entry.
|
||||
*
|
||||
* RED before the fix: the leader_current_term field does not exist (compile
|
||||
* error); once added but before the receiver reads it, ae_term == entry_term (4) <
|
||||
* s->term (5) → stale-reject → LSN 2 never persists. */
|
||||
static void test_intake_crossterm_catchup_uses_leader_current_term(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_ts_xterm_catchup_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t* c = make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
/* Standby has already adopted the current leader's term (5) from heartbeats. */
|
||||
make_follower(c, /*term*/5, /*n_peers*/3);
|
||||
|
||||
/* Local log holds a HISTORICAL entry LSN1@term4 (written by the prior leader
|
||||
* at term 4, replicated to this standby). */
|
||||
uint8_t blob[64];
|
||||
size_t n = pack_entry(blob, WAL_JOB_SUBMIT, "a", 1);
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 4, blob, n));
|
||||
c->ha.raft.log_len = 1;
|
||||
TEST_ASSERT_EQUAL_UINT64(4, persist_entry_term(&c->persist, 1));
|
||||
|
||||
/* The CURRENT leader (term 5) streams the historical entry LSN2@term4 as a
|
||||
* catch-up sub-entry: the entry's STORED term is 4, but the AppendEntries
|
||||
* header term is the leader's CURRENT term 5 (leader_current_term). prev tuple
|
||||
* (1,4) matches the local log so logOk holds. */
|
||||
uint8_t lpay[1] = { 'b' };
|
||||
wal_catchup_entry_t sub;
|
||||
memset(&sub, 0, sizeof(sub));
|
||||
sub.lsn = 2;
|
||||
sub.entry_type = WAL_JOB_SUBMIT;
|
||||
sub.term = 4; /* the entry's STORED term (prior leader) */
|
||||
sub.leader_current_term = 5; /* the streaming leader's CURRENT term */
|
||||
sub.prev_log_index = 1;
|
||||
sub.prev_log_term = 4;
|
||||
sub.raw_payload = lpay;
|
||||
sub.raw_payload_len = sizeof(lpay);
|
||||
|
||||
wal_catchup_entry_t* sub_arr[1] = { &sub };
|
||||
wal_catchup_entry_t batch;
|
||||
memset(&batch, 0, sizeof(batch));
|
||||
batch.n_entries = 1;
|
||||
batch.entries = sub_arr;
|
||||
batch.term = 4;
|
||||
batch.leader_current_term = 5;
|
||||
batch.prev_log_index = 1;
|
||||
batch.prev_log_term = 4;
|
||||
|
||||
int rc = ha_apply_wal_catchup_entry(&c->ha, &c->persist, &batch, /*ep*/NULL, c);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
|
||||
/* THE bug-1067 assertion: the historical entry is ACCEPTED (not stale-rejected)
|
||||
* and persisted at its stored term 4. Pre-fix it was rejected (absent). */
|
||||
TEST_ASSERT_EQUAL_UINT64(4, persist_entry_term(&c->persist, 2));
|
||||
/* The agreed prefix (LSN1@term4) is untouched. */
|
||||
TEST_ASSERT_EQUAL_UINT64(4, persist_entry_term(&c->persist, 1));
|
||||
/* The leader's current term (5) equals our term — no spurious term change. */
|
||||
TEST_ASSERT_EQUAL_UINT64(5, c->ha.raft.term);
|
||||
|
||||
teardown_cluster(c);
|
||||
printf("test_intake_crossterm_catchup_uses_leader_current_term: PASS\n");
|
||||
}
|
||||
|
||||
/* Safety guard: a GENUINELY stale leader (leader_current_term < our term) must
|
||||
* STILL be rejected — the fix must not open a hole letting a deposed leader's
|
||||
* divergent entries through. */
|
||||
static void test_intake_stale_leader_term_still_rejected(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_ts_stale_leader_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t* c = make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
make_follower(c, /*term*/5, /*n_peers*/3);
|
||||
|
||||
uint8_t blob[64];
|
||||
size_t n = pack_entry(blob, WAL_JOB_SUBMIT, "a", 1);
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 4, blob, n));
|
||||
c->ha.raft.log_len = 1;
|
||||
|
||||
/* A deposed leader at term 3 (< our term 5) tries to stream. Even though the
|
||||
* entry's stored term and prev tuple look locally plausible, the AE header
|
||||
* term (leader_current_term=3) is stale → reject. */
|
||||
uint8_t lpay[1] = { 'z' };
|
||||
wal_catchup_entry_t sub;
|
||||
memset(&sub, 0, sizeof(sub));
|
||||
sub.lsn = 2;
|
||||
sub.entry_type = WAL_JOB_SUBMIT;
|
||||
sub.term = 3;
|
||||
sub.leader_current_term = 3; /* STALE leader term < our term 5 */
|
||||
sub.prev_log_index = 1;
|
||||
sub.prev_log_term = 4;
|
||||
sub.raw_payload = lpay;
|
||||
sub.raw_payload_len = sizeof(lpay);
|
||||
|
||||
wal_catchup_entry_t* sub_arr[1] = { &sub };
|
||||
wal_catchup_entry_t batch;
|
||||
memset(&batch, 0, sizeof(batch));
|
||||
batch.n_entries = 1;
|
||||
batch.entries = sub_arr;
|
||||
batch.term = 3;
|
||||
batch.leader_current_term = 3;
|
||||
batch.prev_log_index = 1;
|
||||
batch.prev_log_term = 4;
|
||||
|
||||
int rc = ha_apply_wal_catchup_entry(&c->ha, &c->persist, &batch, /*ep*/NULL, c);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
|
||||
/* Stale leader's entry was REJECTED — LSN 2 absent, our term unchanged. */
|
||||
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(&c->persist, 2));
|
||||
TEST_ASSERT_EQUAL_UINT64(5, c->ha.raft.term);
|
||||
|
||||
teardown_cluster(c);
|
||||
printf("test_intake_stale_leader_term_still_rejected: PASS\n");
|
||||
}
|
||||
|
||||
/* ================================================================== */
|
||||
/* Task 3.3 — drive raft_on_append_entries_reply + raft_advance_commit */
|
||||
/* on the replication ACK path */
|
||||
|
|
@ -2065,6 +2200,8 @@ void test_term_safety_adapter(void)
|
|||
RUN_TEST(test_intake_logmismatch_rejected_no_false_convergence);
|
||||
RUN_TEST(test_live_broadcast_receive_persists_leader_term);
|
||||
RUN_TEST(test_catchup_applied_lsn_is_last_held_no_skip);
|
||||
RUN_TEST(test_intake_crossterm_catchup_uses_leader_current_term);
|
||||
RUN_TEST(test_intake_stale_leader_term_still_rejected);
|
||||
RUN_TEST(test_repl_ack_currentterm_quorum_advances_commit);
|
||||
RUN_TEST(test_repl_ack_priorterm_quorum_holds_commit);
|
||||
RUN_TEST(test_repl_ack_staleterm_reply_dropped);
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
|
||||
#include "config.h" /* WAL_HDR_CRC_SIZE, WAL_CRC_NONE, MAX_WAL_PAYLOAD_BYTES */
|
||||
#include "controller.capnp.h" /* Job decode for WAL_JOB_SUBMIT */
|
||||
#include "utils.h" /* fmt_jobid — human-readable JobId formatting */
|
||||
#include "wal_types.h"
|
||||
|
||||
#define WAL_SEG_FMT "%s/seg-%016" PRIx64 ".wal"
|
||||
|
|
@ -303,8 +304,9 @@ static void scan_segment(const char* seg_path, FILE* out,
|
|||
}
|
||||
}
|
||||
|
||||
/* Per-entry line, with jobid/subtime for submits. */
|
||||
char info[96] = "";
|
||||
/* Per-entry line, with jobid/subtime for submits. Sized to hold the
|
||||
* formatted JobId (fmt_jobid, up to ~40 chars) + subtime + state. */
|
||||
char info[192] = "";
|
||||
int is_dup = 0;
|
||||
int dup_identical = 0;
|
||||
if (hdr.type == WAL_JOB_SUBMIT) {
|
||||
|
|
@ -312,9 +314,16 @@ static void scan_segment(const char* seg_path, FILE* out,
|
|||
uint64_t jobid = 0;
|
||||
if (decode_submit_info(payload, hdr.payload_len, &si) == 0) {
|
||||
jobid = si.jobid;
|
||||
/* Render the JobId in the same human-readable form the client
|
||||
* (joblist / jobsub) prints — fmt_jobid (utils.c) — so a waldump
|
||||
* jobid can be eyeballed against an `expect-job <id>` / joblist
|
||||
* entry without converting the raw uint64. */
|
||||
char jidStr[128];
|
||||
if (fmt_jobid(jidStr, sizeof(jidStr), si.jobid) < 0)
|
||||
snprintf(jidStr, sizeof(jidStr), "%" PRIu64, si.jobid);
|
||||
snprintf(info, sizeof(info),
|
||||
"jobid=%" PRIu64 " subtime=%" PRIu64 " state=%u",
|
||||
si.jobid, si.subtime, si.state);
|
||||
"jobid=%s subtime=%" PRIu64 " state=%u",
|
||||
jidStr, si.subtime, si.state);
|
||||
} else {
|
||||
snprintf(info, sizeof(info), "jobid=?");
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue