lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-term-safety): route AppendEntries intake through library + leaderCommit wire (task 3.2)

Browse source
This commit is contained in:
Rongsong Shen 2026-06-22 08:42:35 +00:00
parent 6a519a928b
commit 754e586f8f
10 changed files with 508 additions and 168 deletions
Download patch file Download diff file Expand all files Collapse all files

107
src/controller/testsuite/test_demotion_convergence.c
View file

@ -592,46 +592,44 @@ static void rj_make_standby_with_seams(void)
g_restart_hook_rc = 0; /* model a successful restart */
}
/* 2.1/2.2: local tip ABOVE the primary boundary AND local committed_lsn ABOVE the
* boundary → reconcile truncates at the CLAMP max(boundary, local committed) (the
* higher local value, never the lower reported boundary) and self-restarts once.
* RED: ha_rejoin_reconcile is a no-op stub → no truncation, no restart. */
static void test_rejoin_clamp_truncates_at_local_committed_and_restarts_once(void)
/* ha-term-safety task 3.2: the committed-boundary (watermark) truncate + self-
* restart is RETIRED. Divergent-suffix reconciliation is now owned by the raft
* library's (term,lsn) log-matching (raft_on_append_entries_request →
* on_truncate_from), driven by the leader's AppendEntries — NOT by an LSN
* watermark on a CtrlHelloReply. ha_rejoin_reconcile therefore no longer
* truncates the local WAL or self-restarts; it is a no-op that returns 0 while
* the learned committed boundary is recorded by the caller and the precise
* truncation happens later via the library AE path. This test pins the RETIRED
* contract: even with a local tip ABOVE the reported boundary, NO truncation and
* NO restart occur here. */
static void test_rejoin_no_watermark_truncate_no_restart(void)
{
dc_setup("rjclamp", /*n_peers=*/3, /*n_standbys=*/0,
/*appended_lsn=*/9, /*n_vcpus=*/8);
rj_make_standby_with_seams();
/* Local tip = 9 (entries 1..9 on disk). Local committed_lsn = 6 — ABOVE the
* primary-reported boundary of 4. The clamp must protect [1..6] and truncate
* only above 6 → cutoff = max(4, 6) = 6 → current_lsn collapses to 7. */
/* Local tip = 9 (entries 1..9 on disk), local committed_lsn = 6, primary
* boundary = 4. Pre-3.2 this would have truncated above 6 + restarted; post-
* 3.2 it must leave the WAL and process untouched (the library reconciles by
* term when the leader's AE arrives). */
rj_set_local_state(/*tip_lsn=*/9, /*committed=*/6);
const uint64_t primary_boundary = 4;
const uint64_t local_committed = 6;
uint64_t tip_before = g_dc.persist.current_lsn;
/* Act. */
int rc = ha_rejoin_reconcile(&g_dc, primary_boundary);
/* Drive any deferred convergence task (the intake may defer to delaymgr,
* same rule as demotion); harmless if 3.2 truncates inline. */
int rc = ha_rejoin_reconcile(&g_dc, /*primary_committed_lsn=*/4);
(void)dc_run_pending_task(&g_dc.delaymgr);
/* Assert (Req 2.2 clamp): truncation cutoff is the LOCAL committed (6), not the
* lower reported boundary (4) — current_lsn collapses to local_committed + 1. */
TEST_ASSERT_EQUAL_UINT64_MESSAGE(local_committed + 1, g_dc.persist.current_lsn,
"rejoin reconcile must clamp the cutoff to max(boundary, local committed) "
"(Req 2.2): with boundary=4 and local committed=6 it must truncate above 6, "
"NOT above the lower reported boundary 4");
/* Assert (Req 2.1): self-restart fired exactly once, fatal seam never. */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_restart_calls),
"rejoin reconcile with tip > boundary must self-restart exactly once "
"(Req 2.1)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"ha_rejoin_reconcile is retired to a no-op (task 3.2) and returns 0");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(tip_before, g_dc.persist.current_lsn,
"the retired watermark truncate must NOT shrink the local WAL — the "
"library's (term,lsn) AE reconcile owns truncation now (task 3.2)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_restart_calls),
"the retired rejoin reconcile must NOT self-restart (task 3.2)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_fatal_calls),
"a successful rejoin restart must NOT invoke the fatal seam");
(void)rc;
"the retired rejoin reconcile must NOT invoke the fatal seam");
dc_teardown();
printf("test_rejoin_clamp_truncates_at_local_committed_and_restarts_once: checked\n");
printf("test_rejoin_no_watermark_truncate_no_restart: checked\n");
}
/* 2.3: local tip AT/BELOW the boundary → no truncation, no restart, returns 0.
@ -664,38 +662,32 @@ static void test_rejoin_tip_at_or_below_boundary_no_action(void)
printf("test_rejoin_tip_at_or_below_boundary_no_action: checked\n");
}
/* 2.1 once-guard: a SECOND reconcile call in the same process lifetime must NOT
* self-restart again, even if the tip still exceeds the boundary (defensive guard;
* "once" is also structural post-restart).
* RED: the no-op stub never restarts even once, so the "exactly one restart total"
* assertion fails. */
static void test_rejoin_second_call_no_second_restart(void)
/* ha-term-safety task 3.2: with the watermark truncate retired, REPEATED rejoin
* reconcile calls remain side-effect-free — never truncating and never
* restarting, regardless of how far the local tip runs ahead. (Pre-3.2 this
* pinned a defensive once-guard around a single restart; post-3.2 there is no
* restart at all, so any number of calls is a no-op.) */
static void test_rejoin_repeated_calls_remain_no_op(void)
{
dc_setup("rjonce", /*n_peers=*/3, /*n_standbys=*/0,
/*appended_lsn=*/9, /*n_vcpus=*/8);
rj_make_standby_with_seams();
rj_set_local_state(/*tip_lsn=*/9, /*committed=*/4);
uint64_t tip_before = g_dc.persist.current_lsn;
/* Act: first reconcile (tip 9 > boundary 4 → truncate + restart once). */
(void)ha_rejoin_reconcile(&g_dc, /*primary_committed_lsn=*/4);
(void)dc_run_pending_task(&g_dc.delaymgr);
int restarts_after_first = atomic_load(&g_restart_calls);
/* Act: second reconcile in the SAME lifetime. The once-guard must suppress a
* second restart regardless of the reported boundary. */
(void)ha_rejoin_reconcile(&g_dc, /*primary_committed_lsn=*/4);
(void)dc_run_pending_task(&g_dc.delaymgr);
/* Assert (Req 2.1): exactly ONE restart across both calls. */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, restarts_after_first,
"the FIRST rejoin reconcile with tip > boundary must self-restart once "
"(Req 2.1)");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_restart_calls),
"a SECOND rejoin reconcile in the same process lifetime must NOT restart "
"again — defensive once-guard (Req 2.1)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_restart_calls),
"the retired rejoin reconcile must NEVER self-restart (task 3.2)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(tip_before, g_dc.persist.current_lsn,
"repeated retired rejoin reconcile must leave the local WAL untouched "
"(task 3.2)");
dc_teardown();
printf("test_rejoin_second_call_no_second_restart: checked\n");
printf("test_rejoin_repeated_calls_remain_no_op: checked\n");
}
/* Intake integration: delivering a CtrlHelloReply through the REAL cluster.c
@ -727,12 +719,17 @@ static void test_ctrlhello_reply_intake_stores_and_reconciles(void)
atomic_load(&g_dc.ha.last_known_committed_config_tag),
"CtrlHelloReply intake must store the config tag (existing)");
/* Assert (intake integration): the intake also TRIGGERED reconciliation —
* tip 9 > boundary 4 → exactly one self-restart. */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_restart_calls),
"CtrlHelloReply intake must trigger rejoin reconciliation when the local "
"tip exceeds the reported boundary — exactly one self-restart (task 3.2 "
"wires ha_rejoin_reconcile into the intake)");
/* Assert (task 3.2): the intake STILL stores the boundary (above) but the
* watermark truncate/restart is RETIRED — the intake must NOT self-restart
* even though the local tip (9) exceeds the reported boundary (4). The
* library's (term,lsn) AE reconcile owns the divergent-suffix truncation now;
* the member stays alive and converges when the leader's AppendEntries arrive. */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_restart_calls),
"CtrlHelloReply intake must NOT self-restart on a tip-ahead boundary — the "
"retired watermark reconcile no longer truncates/restarts (task 3.2)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(10, g_dc.persist.current_lsn,
"the intake must leave the local WAL untouched (tip 9 → current_lsn 10); "
"reconciliation is deferred to the library AE path (task 3.2)");
dc_teardown();
printf("test_ctrlhello_reply_intake_stores_and_reconciles: checked\n");
@ -757,9 +754,9 @@ void test_demotion_convergence(void)
printf("\n>> rejoin reconciliation tests (RED, task 3.1)\n\n");
RUN_TEST(test_rejoin_clamp_truncates_at_local_committed_and_restarts_once);
RUN_TEST(test_rejoin_no_watermark_truncate_no_restart);
RUN_TEST(test_rejoin_tip_at_or_below_boundary_no_action);
RUN_TEST(test_rejoin_second_call_no_second_restart);
RUN_TEST(test_rejoin_repeated_calls_remain_no_op);
RUN_TEST(test_ctrlhello_reply_intake_stores_and_reconciles);
#else
printf("test_demotion_convergence: SKIP (non-test build)\n");

177
src/controller/testsuite/test_term_safety_adapter.c
View file

@ -1102,6 +1102,180 @@ static void test_primary_log_append_non_leader_legacy_fallback(void)
printf("test_primary_log_append_non_leader_legacy_fallback: PASS\n");
}
/* ================================================================== */
/* Task 3.2 — route heartbeat/replication intake through the library */
/* raft_on_append_entries_request: UpdateTerm (G5) + logOk (G2) + */
/* (term,lsn) conflict-truncate (G3) via the wired callbacks. */
/* */
/* These drive the production intake helper ha_consume_append_entries */
/* (the in-process route that replaces the term-blind handling in */
/* ha_consume_heartbeat_committed + the committed-boundary truncate in */
/* ha_rejoin_reconcile). RED: that helper does not exist pre-3.2 */
/* (compile RED); once it exists the three observable behaviors hold. */
/* ================================================================== */
/* Make a node a STANDBY follower at a given term + raft log floor, with the
* raft library initialized so the intake can drive it. Seeds the library log
* to match the persisted WAL tip (the ha_init_raft floor restore). */
static void make_follower(cluster_t* c, uint64_t term, int n_peers)
{
atomic_store(&c->ha.role, CTRL_STANDBY);
atomic_store(&c->ha.term, term);
ha_init_raft(&c->ha, c, n_peers);
c->ha.raft.term = term;
c->ha.raft.role = RAFT_FOLLOWER;
}
/* Build a decoded single-entry WalCatchupEntry (n_entries==0 path == one entry
* carried in the top-level container, matching the on-wire single-entry mode).
* For a heartbeat use n_entries set so the helper sees zero entries. */
static void fill_ae_entry(wal_catchup_entry_t* e, int is_heartbeat,
uint64_t prev_log_index, uint64_t prev_log_term,
uint64_t leader_commit, uint64_t entry_lsn,
uint8_t entry_type, uint64_t entry_term,
uint8_t* payload, uint32_t plen)
{
memset(e, 0, sizeof(*e));
e->prev_log_index = prev_log_index;
e->prev_log_term = prev_log_term;
e->leader_commit = leader_commit;
if (is_heartbeat) {
e->n_entries = 0;
e->lsn = 0;
} else {
/* single-entry mode: top-level container carries the entry, n_entries==1
* with no nested sub-entries (one-entry AppendEntries). */
e->n_entries = 1;
e->lsn = entry_lsn;
e->entry_type = entry_type;
e->term = entry_term;
e->raw_payload = payload;
e->raw_payload_len = plen;
}
}
/* (a) A higher-term heartbeat (0-entry AppendEntries) routed through the library
* makes the follower ADOPT the higher term and step down — term recorded
* (R2.6 / G5). Pre-3.2 ha_consume_heartbeat_committed only DROPPED stale
* heartbeats and never adopted a higher term. */
static void test_intake_higher_term_heartbeat_adopts_term(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_intake_ht_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
make_follower(c, /*term*/3, /*n_peers*/3);
/* The follower briefly believes itself a candidate/leader at term 3. */
c->ha.raft.role = RAFT_LEADER;
/* Heartbeat from a leader at term 7, empty log (prev=0), commit 0. The
* heartbeat's top-level container carries the leader's live term. */
wal_catchup_entry_t hb;
fill_ae_entry(&hb, /*heartbeat*/1, /*prev_idx*/0, /*prev_term*/0,
/*leader_commit*/0, 0, 0, 0, NULL, 0);
hb.term = 7;
int rc = ha_consume_append_entries(c, &hb, /*leader_idx*/1);
TEST_ASSERT_EQUAL_INT(0, rc);
/* The library adopted the higher term and stepped down to follower. */
TEST_ASSERT_EQUAL_UINT64(7, c->ha.raft.term);
TEST_ASSERT_NOT_EQUAL_INT(RAFT_LEADER, c->ha.raft.role);
teardown_cluster(c);
printf("test_intake_higher_term_heartbeat_adopts_term: PASS\n");
}
/* (b) A divergent same-LSN tail is truncated and the leader's entry accepted
* (R3.1/R3.2 / G3), and the committed prefix below the divergence is untouched
* (R3.3). The follower locally holds [LSN1@term2, LSN2@term2]; the leader sends
* an AppendEntries appending LSN2@term5 on top of LSN1@term2. The library
* detects the (term,lsn) conflict at LSN2, truncates LSN>=2, and re-persists
* the leader's LSN2@term5. LSN1 (the agreed prefix) stays intact. */
static void test_intake_divergent_tail_truncated_and_reconverged(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_intake_div_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
make_follower(c, /*term*/2, /*n_peers*/3);
/* Seed the local log: LSN1@term2, LSN2@term2 (the divergent tail). */
uint8_t blob[64];
size_t n = pack_entry(blob, WAL_JOB_UPDATE, "a", 1);
TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 2, blob, n));
TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 2, 2, blob, n));
c->ha.raft.log_len = 2;
TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 1));
TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 2));
/* Leader (term 5) sends AppendEntries: prev=(LSN1,term2), entry LSN2@term5.
* logOk holds at LSN1 (term2==term2); conflict at LSN2 (local term2 !=
* incoming term5) → truncate LSN>=2 then persist LSN2@term5. */
uint8_t lpay[1] = { 'b' };
wal_catchup_entry_t ae;
fill_ae_entry(&ae, /*heartbeat*/0, /*prev_idx*/1, /*prev_term*/2,
/*leader_commit*/1, /*entry_lsn*/2, WAL_JOB_UPDATE,
/*entry_term*/5, lpay, 1);
int rc = ha_consume_append_entries(c, &ae, /*leader_idx*/1);
TEST_ASSERT_EQUAL_INT(0, rc);
/* (term,lsn) reconverged: LSN2 now carries the leader's term 5. */
TEST_ASSERT_EQUAL_UINT64(5, persist_entry_term(&c->persist, 2));
/* The agreed committed prefix (LSN1@term2) is untouched (R3.3). */
TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 1));
/* The higher leader term was adopted on the RPC (G5). */
TEST_ASSERT_EQUAL_UINT64(5, c->ha.raft.term);
teardown_cluster(c);
printf("test_intake_divergent_tail_truncated_and_reconverged: PASS\n");
}
/* (c) An unreconcilable member (logOk fails: the leader's prev tuple does not
* match the follower's log) is REJECTED — the follower neither persists the
* leader's entry nor reports false convergence, and stays available to retry
* (no crash, no committed advance). R3.5/R3.6: the member keeps its divergent
* state pending a later catch-up rather than spuriously accepting. */
static void test_intake_logmismatch_rejected_no_false_convergence(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_intake_rej_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
make_follower(c, /*term*/4, /*n_peers*/3);
/* Local log: LSN1@term2. The leader claims prev=(LSN1,term3) — a term
* mismatch at LSN1 → logOk fails. */
uint8_t blob[64];
size_t n = pack_entry(blob, WAL_JOB_UPDATE, "a", 1);
TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 2, blob, n));
c->ha.raft.log_len = 1;
uint8_t lpay[1] = { 'z' };
wal_catchup_entry_t ae;
fill_ae_entry(&ae, /*heartbeat*/0, /*prev_idx*/1, /*prev_term*/3,
/*leader_commit*/0, /*entry_lsn*/2, WAL_JOB_UPDATE,
/*entry_term*/4, lpay, 1);
int rc = ha_consume_append_entries(c, &ae, /*leader_idx*/1);
/* The helper returns 0 (no crash) but the AE was NOT accepted: the leader's
* entry was not persisted at LSN2 and the local LSN1 entry is unchanged. */
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(&c->persist, 2)); /* absent */
TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 1)); /* intact */
teardown_cluster(c);
printf("test_intake_logmismatch_rejected_no_false_convergence: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Suite entry point */
/* ------------------------------------------------------------------ */
@ -1131,4 +1305,7 @@ void test_term_safety_adapter(void)
RUN_TEST(test_primary_log_append_stamps_term_and_assigns_lsn);
RUN_TEST(test_own_term_noop_records_live_term_via_library);
RUN_TEST(test_primary_log_append_non_leader_legacy_fallback);
RUN_TEST(test_intake_higher_term_heartbeat_adopts_term);
RUN_TEST(test_intake_divergent_tail_truncated_and_reconverged);
RUN_TEST(test_intake_logmismatch_rejected_no_false_convergence);
}