lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(raft-ha-core-safety): streamer fail-loud bounded-retry, never ship empty (task 2.4)

Browse source 
Root-cause source fix: ha_stream_wal_to_standby reads the entry body via
stream_read_entry_body (bounded retry max 4, no sleep); on persistent short
pread/malloc-fail it fails loud + ABORTS the stream (flush valid prefix,
drop unreadable entry, -EIO) instead of shipping PLEN=0. Incomplete stream
re-driven by ACK-driven catch-up. With the 2.2 guard this closes the live
cross-term payload-loss divergence. 2.3 CORE RED -> GREEN (fail-loud: no
empty persisted; + retry-recover test ships full 64). Term-blind structural
retirement deferred to 6.2 (empty-over-full safety enforced by 2.2 guard +
pinned). 12/12 core suite; siblings green; no regression introduced.
Phase 2 complete.
This commit is contained in:
Rongsong Shen 2026-06-27 15:42:37 +00:00
parent ae8291ed56
commit 63b9c5a81a
3 changed files with 210 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

99
src/controller/testsuite/test_ha_core_safety.c
View file

@ -41,6 +41,7 @@
* Requirements: 7.1 (force divergence RED on baseline), 7.4 (unit harness).
*/
#define _GNU_SOURCE
#include <errno.h>
#include <inttypes.h>
#include <stdatomic.h>
#include <stdint.h>
@ -614,6 +615,90 @@ static int hcs_2_3_target_shipped_empty(void)
return 0;
}
/* The send_len captured for the target LSN (0 if it was never shipped). */
static uint32_t hcs_2_3_target_send_len(void)
{
for (int i = 0; i < g_sends_n; i++)
if (g_sends[i].lsn == g_fault_lsn)
return g_sends[i].send_len;
return 0;
}
/* task 2.4: a TRANSIENT body-read fault — fails the first g_fault_transient_left
* attempts, then lets the read succeed. Proves the bounded retry RECOVERS the
* full payload (the streamer ships SEND_len==source_len), the counterpart to the
* permanent-fault fail-loud path above. */
static int g_fault_transient_left;
static int hcs_2_4_transient_fault_hook(uint64_t lsn, uint64_t stored_term,
uint64_t leader_term, uint32_t payload_len)
{
(void)payload_len;
if (lsn != g_fault_lsn)
return 0;
if (stored_term < leader_term)
g_fault_saw_crossterm = 1;
if (g_fault_transient_left > 0) {
g_fault_transient_left--;
return 1; /* force a short read on this attempt */
}
return 0; /* recovered — let the real read complete */
}
/* GREEN (task 2.4) — a transient short read is recovered by the bounded retry:
* the entry is shipped FULL (SEND_len == source_len), never empty. */
static void test_2_4_streamer_short_read_retry_recovers(void)
{
hcs_member_t src, rcv;
TEST_ASSERT_EQUAL_INT(0, hcs_member_up(&src, "s24src"));
TEST_ASSERT_EQUAL_INT(0, hcs_member_up(&rcv, "s24rcv"));
enum { TGT = 1, STORED_TERM = 3, LEADER_TERM = 4 };
uint8_t full[HCS_FULL_LEN];
hcs_fill_body(full);
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
hcs_wal_put(&src, TGT, STORED_TERM, WAL_JOB_SUBMIT, full, HCS_FULL_LEN),
"source full write must persist (real fs required — bug-924)");
atomic_store_explicit(&src.p->committed_lsn, (uint64_t)TGT,
memory_order_release);
/* Fail the first 2 read attempts (< the bounded budget) then recover. */
hcs_2_3_reset(&rcv, /*fault_lsn=*/(uint64_t)TGT);
g_fault_transient_left = 2;
ha_stream_body_read_fault_hook = hcs_2_4_transient_fault_hook;
xce_wal_stream_request_hook = hcs_2_3_capture_hook;
int rc = ha_stream_wal_to_standby((endpoint_t*)NULL, src.wal_dir, src.p,
/*leader_term=*/(uint64_t)LEADER_TERM,
/*standby_lsn=*/0,
/*committed_lsn=*/(uint64_t)TGT);
ha_stream_body_read_fault_hook = NULL;
xce_wal_stream_request_hook = NULL;
uint32_t send_len = hcs_2_3_target_send_len();
hcs_probe_t rcv_pr;
int rcv_rc = hcs_wal_probe(&rcv, (uint64_t)TGT, &rcv_pr);
printf("test_2_4 retry-recover LSN %d: source_len=%u SEND_len=%u RECV_len=%u "
"(rc=%d, crossterm_seen=%d)\n", (int)TGT, HCS_FULL_LEN, send_len,
rcv_pr.present ? rcv_pr.len : 0u, rc, g_fault_saw_crossterm);
TEST_ASSERT_TRUE_MESSAGE(g_fault_saw_crossterm,
"the transient fault must fire on the cross-term entry");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"a transient short read recovered within the retry budget — the stream "
"completes cleanly (rc==0), not aborted");
/* The recovered entry is shipped FULL — SEND_len equals the source length. */
TEST_ASSERT_EQUAL_UINT32_MESSAGE(HCS_FULL_LEN, send_len,
"GREEN: the bounded retry recovered the body; the entry is shipped with "
"its FULL payload (SEND_len == source_len), never empty");
TEST_ASSERT_EQUAL_INT(0, rcv_rc);
TEST_ASSERT_TRUE_MESSAGE(rcv_pr.present && rcv_pr.len == HCS_FULL_LEN,
"GREEN: the receiver persisted the FULL recovered body at the target LSN");
hcs_member_down(&src);
hcs_member_down(&rcv);
printf("test_2_4_streamer_short_read_retry_recovers: done\n");
}
/* CORE RED — the streamer must never ship a non-empty cross-term entry empty. */
static void test_2_3_streamer_short_read_must_not_ship_empty(void)
{
@ -660,7 +745,16 @@ static void test_2_3_streamer_short_read_must_not_ship_empty(void)
if (g_sends[i].lsn == (uint64_t)TGT)
send_len = g_sends[i].send_len;
hcs_probe_t rcv_pr;
TEST_ASSERT_EQUAL_INT(0, hcs_wal_probe(&rcv, (uint64_t)TGT, &rcv_pr));
/* task 2.4: when the fix correctly ABORTS the stream rather than ship empty,
* the receiver gets NOTHING, so its WAL dir has no segment and the probe
* returns -ENOENT (absent) — which equally satisfies the contract "the
* receiver must never hold an empty committed entry". A clean scan (rc==0,
* not-present, e.g. the retry-recovers path persisting the FULL body) is also
* valid; only a present-with-empty body below is a failure. */
int rcv_rc = hcs_wal_probe(&rcv, (uint64_t)TGT, &rcv_pr);
TEST_ASSERT_TRUE_MESSAGE(rcv_rc == 0 || rcv_rc == -ENOENT,
"receiver WAL probe must scan cleanly (rc==0) or find no segment at all "
"(rc==-ENOENT, the aborted-stream outcome) — never a probe error");
printf("test_2_3 cross-term LSN %d: stored_term=%d leader_term=%d "
"source_len=%u SEND_len=%u RECV_len=%u (rc=%d, crossterm_seen=%d)\n",
(int)TGT, (int)STORED_TERM, (int)LEADER_TERM, src_pr.len, send_len,
@ -949,6 +1043,9 @@ void test_ha_core_safety(void)
* PIN — an unrepairable below-tip catch-up divergence raises the
* non-promotable fence (task-2.2 mechanism via the route). */
RUN_TEST(test_2_3_streamer_short_read_must_not_ship_empty);
/* task 2.4 GREEN: the counterpart recover path — a transient short read is
* recovered by the bounded retry and the entry is shipped FULL. */
RUN_TEST(test_2_4_streamer_short_read_retry_recovers);
RUN_TEST(test_2_3_termblind_route_payload_guarded);
RUN_TEST(test_2_3_unrepairable_catchup_raises_nonpromotable_fence);
/* Scaffolding: each shared seam is usable by 2.3 / 4.1 / 5.1. */