lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-term-safety): retire term-0 repl_broadcast live send; carry leader term on replication (task 3.4)

Browse source 
Closes bug-990: the live replicator repl_broadcast wire-encoded term=0
(entry.term was never set), so standbys persisted hdr.term=0 and took the
legacy direct-apply path instead of the library (term,lsn) intake. Now the
live broadcast and catch-up streamer carry the leader's real term + AE prev
tuple; ha_apply_wal_catchup_entry routes every term-carrying entry through
raft_on_append_entries_request -> on_persist_entry (leader term stamped),
retiring the term-0 direct-apply. Standbys now persist the leader's real term
and reconcile by (term,lsn); the live commit chain is end-to-end.

848/849/850 + persist-before-ACK + commit-barrier + bug-960 preserved.
Proven at integration level; the definitive live STRICT_TERM failover run is
env-blocked here (bug-924 overlayfs) and recommended on a real fs.
This commit is contained in:
Rongsong Shen 2026-06-22 15:45:14 +00:00
parent 87242aa8e9
commit efe621c124
11 changed files with 307 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/controller/testsuite/test_convergence_verification.c
View file

@ -754,7 +754,7 @@ static int cv_broadcast_one(replicator_t *r, uint64_t lsn)
hdr.lsn = lsn;
hdr.type = WAL_NOOP;
hdr.payload_len = 0;
return repl_broadcast(r, &hdr, NULL, 0);
return repl_broadcast(r, &hdr, NULL, 0, 0 /* prev_log_term */);
}
/* (a) umbrella 6.1: repl_broadcast enqueues to EVERY registered active standby

2
src/controller/testsuite/test_repl_majority.c
View file

@ -279,7 +279,7 @@ static void test_broadcast_over_limit_removes_standby(void)
/* Broadcast a wal entry — the over-limit standby should be evicted */
wal_entry_hdr_t hdr = { .type = WAL_JOB_SUBMIT, .lsn = 1 };
int rc = repl_broadcast(&r, &hdr, NULL, 0);
int rc = repl_broadcast(&r, &hdr, NULL, 0, 0 /* prev_log_term */);
/* repl_broadcast itself may return 0 even after eviction */
(void)rc;

70
src/controller/testsuite/test_repl_unit.c
View file

@ -186,6 +186,11 @@ static uint8_t g_hook_last_entry_type;
static const void *g_hook_last_payload;
static size_t g_hook_last_payload_len;
static int g_hook_return_val; /* 0 = success, negative = error */
/* ha-term-safety task 3.4: capture the AppendEntries term + prev tuple the live
* broadcast carries, so a test can assert the leader's term is on the wire. */
static uint64_t g_hook_last_term;
static uint64_t g_hook_last_prev_log_index;
static uint64_t g_hook_last_prev_log_term;
static int test_broadcast_hook(void *ep, wal_catchup_entry_t **entries,
int n_entries)
@ -193,10 +198,13 @@ static int test_broadcast_hook(void *ep, wal_catchup_entry_t **entries,
g_hook_call_count++;
g_hook_last_ep = ep;
if (n_entries >= 1 && entries && entries[0]) {
g_hook_last_lsn = entries[0]->lsn;
g_hook_last_entry_type = entries[0]->entry_type;
g_hook_last_payload = entries[0]->raw_payload;
g_hook_last_payload_len = entries[0]->raw_payload_len;
g_hook_last_lsn = entries[0]->lsn;
g_hook_last_entry_type = entries[0]->entry_type;
g_hook_last_payload = entries[0]->raw_payload;
g_hook_last_payload_len = entries[0]->raw_payload_len;
g_hook_last_term = entries[0]->term;
g_hook_last_prev_log_index = entries[0]->prev_log_index;
g_hook_last_prev_log_term = entries[0]->prev_log_term;
}
return g_hook_return_val;
}
@ -210,6 +218,9 @@ static void reset_hook_state(void)
g_hook_last_payload = NULL;
g_hook_last_payload_len = 0;
g_hook_return_val = 0;
g_hook_last_term = 0;
g_hook_last_prev_log_index = 0;
g_hook_last_prev_log_term = 0;
}
/* Test: no standbys — broadcast returns 0, hook not called */
@ -223,7 +234,7 @@ static void test_broadcast_no_standbys_returns_zero(void)
wal_entry_hdr_t hdr = { .lsn = 10, .type = WAL_JOB_SUBMIT };
uint8_t payload[] = { 0xAA, 0xBB };
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload));
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload), 0 /* prev_log_term */);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(0, g_hook_call_count);
@ -249,7 +260,7 @@ static void test_broadcast_one_standby_sends_entry(void)
wal_entry_hdr_t hdr = { .lsn = 42, .type = WAL_JOB_SUBMIT };
uint8_t payload[] = { 1, 2, 3, 4 };
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload));
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload), 0 /* prev_log_term */);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(1, g_hook_call_count);
@ -267,6 +278,46 @@ static void test_broadcast_one_standby_sends_entry(void)
#endif
}
/* ha-term-safety task 3.4 (Req 1.3/1.4): the LIVE broadcast must carry the
* leader's per-entry term and the AppendEntries prev tuple, so the standby
* intake routes it through raft_on_append_entries_request and persists the
* LEADER's term — not the term-0 legacy direct-apply path. Before the cutover
* repl_broadcast sent term=0 (RED): the wire entry's term was always 0. */
static void test_broadcast_carries_leader_term_and_prev_tuple(void)
{
#ifdef XCE_TESTING
replicator_t r;
int dummy_ep = 0;
repl_init(&r);
repl_add_standby(&r, (void *)&dummy_ep, "sb1", 7001);
reset_hook_state();
repl_broadcast_send_hook = test_broadcast_hook;
/* Leader appends LSN 5 at term 7; predecessor (LSN 4) was term 7. */
wal_entry_hdr_t hdr = { .lsn = 5, .type = WAL_JOB_SUBMIT, .term = 7 };
uint8_t payload[] = { 9, 8, 7 };
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload),
7 /* prev_log_term of LSN 4 */);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(1, g_hook_call_count);
TEST_ASSERT_EQUAL_UINT64(5, g_hook_last_lsn);
/* The load-bearing assertion: the leader's real term is on the wire entry
* (NOT 0). This FAILS before the task 3.4 cutover. */
TEST_ASSERT_EQUAL_UINT64(7, g_hook_last_term);
/* prev tuple: prev_log_index == lsn - 1; prev_log_term == supplied. */
TEST_ASSERT_EQUAL_UINT64(4, g_hook_last_prev_log_index);
TEST_ASSERT_EQUAL_UINT64(7, g_hook_last_prev_log_term);
repl_broadcast_send_hook = NULL;
repl_fini(&r);
printf("test_broadcast_carries_leader_term_and_prev_tuple: PASS\n");
#else
printf("test_broadcast_carries_leader_term_and_prev_tuple: "
"SKIPPED (no XCE_TESTING)\n");
#endif
}
/* Test: standby in catch-up is skipped — hook not called for it */
static void test_broadcast_skips_catchup_standby(void)
{
@ -283,7 +334,7 @@ static void test_broadcast_skips_catchup_standby(void)
wal_entry_hdr_t hdr = { .lsn = 99, .type = WAL_JOB_SUBMIT };
uint8_t payload[] = { 0xFF };
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload));
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload), 0 /* prev_log_term */);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Only sb2 should be sent to */
@ -313,7 +364,7 @@ static void test_broadcast_evicts_overloaded_standby(void)
wal_entry_hdr_t hdr = { .lsn = 5, .type = WAL_JOB_SUBMIT };
uint8_t payload[] = { 0x01 };
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload));
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload), 0 /* prev_log_term */);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Evicted — hook must NOT be called */
@ -477,7 +528,7 @@ static void test_broadcast_send_failure_continues(void)
wal_entry_hdr_t hdr = { .lsn = 77, .type = WAL_JOB_SUBMIT };
uint8_t payload[] = { 0xDE, 0xAD };
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload));
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload), 0 /* prev_log_term */);
/* broadcast should still return 0 — errors are WARN+continue */
TEST_ASSERT_EQUAL_INT(0, rc);
@ -548,6 +599,7 @@ void test_repl_unit(void) {
/* Task 1.1: broadcast send path */
RUN_TEST(test_broadcast_no_standbys_returns_zero);
RUN_TEST(test_broadcast_one_standby_sends_entry);
RUN_TEST(test_broadcast_carries_leader_term_and_prev_tuple);
RUN_TEST(test_broadcast_skips_catchup_standby);
RUN_TEST(test_broadcast_evicts_overloaded_standby);
RUN_TEST(test_broadcast_send_failure_continues);

78
src/controller/testsuite/test_term_safety_adapter.c
View file

@ -1278,6 +1278,83 @@ static void test_intake_logmismatch_rejected_no_false_convergence(void)
printf("test_intake_logmismatch_rejected_no_false_convergence: PASS\n");
}
/* ================================================================== */
/* Task 3.4 — live broadcast/catch-up receive cutover (closes bug-990) */
/* ================================================================== */
/* The live cutover's observable: a standby applying a LIVE-broadcast-shaped
* AppendEntries (a single entry in the `entries` LIST carrying the leader's term
* + prev tuple — exactly what repl_broadcast → xce_wal_batch_req_raw produces and
* xce_decode yields) through the PRODUCTION receive entry point
* ha_apply_wal_catchup_entry persists the LEADER's term (hdr.term == leader_term),
* NOT 0 (the retired term-0 legacy direct-apply path) and NOT its local term.
*
* Before the cutover repl_broadcast sent entry->term == 0, so this entry took the
* term-0 legacy direct-apply branch and persisted hdr.term = local p->term — the
* bug-990 term-identity gap. With the cutover the term-carrying entry routes
* through raft_on_append_entries_request → on_persist_entry → leader term stamped. */
static void test_live_broadcast_receive_persists_leader_term(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_live_bcast_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
/* The standby is a follower at a LOW local term (3) so we can prove the
* persisted term is the LEADER's (7), not the local term. */
make_follower(c, /*term*/3, /*n_peers*/3);
/* Seed LSN 1 at the leader's term 7 so the broadcast of LSN 2 satisfies the
* library's logOk (prev=(1,7)). This mirrors the contiguous live stream. */
uint8_t blob[64];
size_t n = pack_entry(blob, WAL_JOB_SUBMIT, "a", 1);
TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 7, blob, n));
c->ha.raft.log_len = 1;
TEST_ASSERT_EQUAL_UINT64(7, persist_entry_term(&c->persist, 1));
/* Build the LIVE receive shape: a single entry in the `entries` list, each
* sub-entry carrying its term + prev tuple (what wal_entry_to_wire now encodes
* per element and xce_decode reproduces). repl_broadcast at term 7 of LSN 2
* with prev=(1,7). */
uint8_t lpay[2] = { 'b', 'c' };
wal_catchup_entry_t sub;
memset(&sub, 0, sizeof(sub));
sub.lsn = 2;
sub.entry_type = WAL_JOB_SUBMIT;
sub.term = 7; /* the LEADER's term on the wire */
sub.prev_log_index = 1;
sub.prev_log_term = 7;
sub.raw_payload = lpay;
sub.raw_payload_len = sizeof(lpay);
wal_catchup_entry_t* sub_arr[1] = { &sub };
wal_catchup_entry_t batch;
memset(&batch, 0, sizeof(batch));
batch.n_entries = 1;
batch.entries = sub_arr;
/* Top-level container also carries the AE metadata (the encoder mirrors it). */
batch.term = 7;
batch.prev_log_index = 1;
batch.prev_log_term = 7;
int rc = ha_apply_wal_catchup_entry(&c->ha, &c->persist, &batch,
/*ep*/NULL, c);
TEST_ASSERT_EQUAL_INT(0, rc);
/* The load-bearing assertion (bug-990): LSN 2 persists the LEADER's term 7,
* NOT 0 and NOT the local term 3. Before the cutover this read back as the
* standby's local term via the term-0 direct-apply path. */
TEST_ASSERT_EQUAL_UINT64(7, persist_entry_term(&c->persist, 2));
/* The agreed prefix (LSN 1@term7) is untouched. */
TEST_ASSERT_EQUAL_UINT64(7, persist_entry_term(&c->persist, 1));
/* The library adopted the leader's term on the RPC (G5). */
TEST_ASSERT_EQUAL_UINT64(7, c->ha.raft.term);
teardown_cluster(c);
printf("test_live_broadcast_receive_persists_leader_term: PASS\n");
}
/* ================================================================== */
/* Task 3.3 — drive raft_on_append_entries_reply + raft_advance_commit */
/* on the replication ACK path */
@ -1800,6 +1877,7 @@ void test_term_safety_adapter(void)
RUN_TEST(test_intake_higher_term_heartbeat_adopts_term);
RUN_TEST(test_intake_divergent_tail_truncated_and_reconverged);
RUN_TEST(test_intake_logmismatch_rejected_no_false_convergence);
RUN_TEST(test_live_broadcast_receive_persists_leader_term);
RUN_TEST(test_repl_ack_currentterm_quorum_advances_commit);
RUN_TEST(test_repl_ack_priorterm_quorum_holds_commit);
RUN_TEST(test_repl_ack_staleterm_reply_dropped);