lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(raft-drive-async): resolve snapshot chunk-drop (bug-1386) and chunk misordering (bug-1389)

Browse source 
ha_send_snapshot_to_standby ignored the network send's return value in its
chunk loop, silently truncating a transfer on a mid-stream error while still
logging "complete" (bug-1386). Fixing the silent drop exposed a second,
independent bug in the same pipeline: ha_handle_snapshot_chunk appended
received chunks in arrival order instead of at their wire-carried
chunk_offset, so out-of-order delivery across the vtask pool corrupted the
reassembled file even though every chunk's bytes were individually correct
(bug-1389). Fixed by checking every fallible call in the send loop and
switching the receiver to pwrite-at-offset with a count-based completion
tracker.

Adds snapverify/snapdiff xce_test commands (offline checkpoint-file
structural validator and byte-level comparator) used to diagnose bug-1389 and
kept as standing diagnostic tools for future snapshot-transfer triage.

g_drive_async_configpush_offload flipped back ON after both fixes, validated
via 6-round chaotic + 3/3 force-snapshot soak (raft-drive-async task 3.3).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SDWVNQLKxqTVHh42cH7rV3
This commit is contained in:
Rongsong Shen 2026-07-03 02:40:12 +00:00
parent 02e46f6ba1
commit 5cb36794d2
14 changed files with 1553 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

305
src/controller/testsuite/test_config_repl.c
View file

@ -1652,6 +1652,135 @@ static void test_wait_all_with_timeout_fast_threads(void)
elapsed_ms, acked);
}
/* ── push_config_transport_only endpoint lifetime (bug-1381) ───── */
/* push_thread_send_hook: test-only seam declared in config_repl.c (same
* pattern as xce_wal_stream_request_hook) letting this test intercept and
* park push_thread's send so it can probe endpoint_ref/unref bracketing
* around a concurrent teardown, without a real transport connection. */
extern int (*push_thread_send_hook)(endpoint_t *ep, xce_peerinfo_t *from,
uint64_t new_tag, const uint8_t *sha256,
const uint8_t *bytes, size_t blen);
static atomic_int g_cpto_parked;
static atomic_int g_cpto_release;
static int cpto_parking_send_hook(endpoint_t *ep, xce_peerinfo_t *from,
uint64_t new_tag, const uint8_t *sha256,
const uint8_t *bytes, size_t blen)
{
(void)ep; (void)from; (void)new_tag; (void)sha256; (void)bytes; (void)blen;
atomic_store_explicit(&g_cpto_parked, 1, memory_order_release);
while (!atomic_load_explicit(&g_cpto_release, memory_order_acquire))
usleep(1000);
return 0;
}
static int cpto_wait_flag(atomic_int *flag, int target, int timeout_ms)
{
int waited = 0;
while (atomic_load_explicit(flag, memory_order_acquire) != target) {
if (waited >= timeout_ms)
return -1;
usleep(2000);
waited += 2;
}
return 0;
}
typedef struct {
cluster_t *c;
int acked;
int n;
} cpto_driver_arg_t;
static void *cpto_driver_fn(void *arg)
{
cpto_driver_arg_t *a = (cpto_driver_arg_t *)arg;
uint8_t sha[32];
uint8_t body[4] = { 1, 2, 3, 4 };
memset(sha, 0, sizeof(sha));
a->acked = push_config_transport_only(a->c, body, sizeof(body), sha, 99,
&a->n);
return NULL;
}
/*
* bug-1381: push_config_transport_only spawns a push_thread per registered
* standby that reads ctx->ep for the full duration of the transport send.
* Before the fix, no ref was taken on ep, so a concurrent RDHUP teardown
* (endpoint_killer -> endpoint_unref, the bug-1298 refcount system) could
* free ep out from under the still-running push_thread -- a UAF. This test
* parks a push_thread mid-send via push_thread_send_hook, simulates the
* owner's teardown, and verifies ep survives until the send completes.
*/
static void test_configpush_transport_holds_endpoint_ref_across_send(void)
{
cluster_t c;
memset(&c, 0, sizeof(c));
xce_peerinfo_t myhost_info;
memset(&myhost_info, 0, sizeof(myhost_info));
c.myhost = &myhost_info;
ha_config_t cfg;
memset(&cfg, 0, sizeof(cfg));
cfg.heartbeat_timeout_ms = 10000;
cfg.election_timeout_ms = 1500;
ha_init(&c.ha, &cfg, 0);
c.ha.standby_endpoints = calloc(1, sizeof(zarray_t));
TEST_ASSERT_NOT_NULL(c.ha.standby_endpoints);
zarray_init(c.ha.standby_endpoints, 4, NULL);
/* Heap-allocated endpoint standing in for a real standby connection:
* refcnt starts at 1, representing the owner (the standby zarray)
* holding the creator reference. */
endpoint_t *ep = endpoint_get();
TEST_ASSERT_NOT_NULL(ep);
zarray_store(c.ha.standby_endpoints, ep);
atomic_store(&g_cpto_parked, 0);
atomic_store(&g_cpto_release, 0);
push_thread_send_hook = cpto_parking_send_hook;
cpto_driver_arg_t driver = { .c = &c, .acked = -1, .n = -1 };
pthread_t driver_thread;
TEST_ASSERT_EQUAL_INT(0,
pthread_create(&driver_thread, NULL, cpto_driver_fn, &driver));
TEST_ASSERT_EQUAL_INT_MESSAGE(0, cpto_wait_flag(&g_cpto_parked, 1, 2000),
"push_thread must park in the send hook before we probe refcnt");
TEST_ASSERT_EQUAL_INT_MESSAGE(2,
atomic_load_explicit(&ep->refcnt, memory_order_acquire),
"push_config_transport_only must hold its own ref (owner's 1 + "
"push_thread's 1) while the send is in flight");
/* Simulate the owner's teardown (RDHUP -> zarray_del -> endpoint_killer
* -> endpoint_unref, bug-1298) racing the in-flight push. */
endpoint_unref(ep);
TEST_ASSERT_EQUAL_INT_MESSAGE(1,
atomic_load_explicit(&ep->refcnt, memory_order_acquire),
"ep must survive the owner's unref while push_thread is still in "
"flight -- bug-1381");
atomic_store_explicit(&g_cpto_release, 1, memory_order_release);
pthread_join(driver_thread, NULL);
TEST_ASSERT_EQUAL_INT(1, driver.n);
TEST_ASSERT_EQUAL_INT(1, driver.acked);
/* ep is now freed (push_thread's completion unref dropped the last
* reference) -- do not touch it again. */
push_thread_send_hook = NULL;
zarray_fini(c.ha.standby_endpoints);
free(c.ha.standby_endpoints);
c.ha.standby_endpoints = NULL;
ha_fini(&c.ha);
printf("test_configpush_transport_holds_endpoint_ref_across_send: PASS\n");
}
#endif /* XCE_TESTING */
/* ── raw capnp shims (Task A) ───────────────────── */
@ -3034,6 +3163,71 @@ static void test_snapshot_transfer_small_less_than_one_chunk(void)
printf("test_snapshot_transfer_small_less_than_one_chunk: PASS\n");
}
/* bug-1386: a chunk send that fails mid-transfer (e.g. -ECONNRESET while the
* standby endpoint is mid-teardown/reconnect) must stop the loop and be
* propagated, not be silently swallowed while the function still reports
* "transfer complete". */
static int g_snaptest_fail_after_calls = -1; /* -1 = never fail */
static int snaptest_fail_after_hook(endpoint_t *ep, xce_request_t *req)
{
(void)ep;
if (g_snaptest_chunk_count >= 16) return -1;
memcpy(&g_snaptest_chunks[g_snaptest_chunk_count],
req->body.snapshottransfermsg, sizeof(snapshot_transfer_t));
g_snaptest_chunk_count++;
if (g_snaptest_fail_after_calls >= 0 &&
g_snaptest_chunk_count > g_snaptest_fail_after_calls)
return -ECONNRESET;
return 0;
}
static void test_snapshot_transfer_stops_and_propagates_send_failure(void)
{
char tmpdir[] = "/tmp/xcectest_snaptrans_XXXXXX";
TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir));
/* 3 chunks: 2 full 64KB chunks + a partial third, so a mid-stream
* failure on chunk 1 leaves chunk 2 (the would-be is_last chunk)
* unsent. */
char snap_path[512];
snprintf(snap_path, sizeof(snap_path), "%s/snap-00000000000000c8.capnp", tmpdir);
FILE *f = fopen(snap_path, "wb");
TEST_ASSERT_NOT_NULL(f);
char header[16] = "XCESNAP\0";
uint64_t lsn = 200;
memcpy(header + 8, &lsn, 8);
fwrite(header, 1, 16, f);
char zero[4096];
memset(zero, 0, sizeof(zero));
size_t total = 65536 + 65536 + 4096;
for (size_t i = 16; i < total; i += 4096)
fwrite(zero, 1, (i + 4096 > total) ? (total - i) : 4096, f);
fclose(f);
extern int (*xce_snapshot_chunk_hook)(endpoint_t *, xce_request_t *);
xce_snapshot_chunk_hook = snaptest_fail_after_hook;
snaptest_reset();
g_snaptest_fail_after_calls = 1; /* chunk_index 0 succeeds, chunk_index 1 fails */
extern int ha_send_snapshot_to_standby(endpoint_t *, const char *,
uint64_t, uint64_t);
int rc = ha_send_snapshot_to_standby(NULL, tmpdir, lsn, 0);
TEST_ASSERT_EQUAL_INT(-ECONNRESET, rc);
/* Loop must stop right after the failing chunk — chunk_index 2 (the
* is_last chunk) must never be attempted. */
TEST_ASSERT_EQUAL_INT(2, g_snaptest_chunk_count);
TEST_ASSERT_EQUAL_INT(0, g_snaptest_chunks[1].is_last);
xce_snapshot_chunk_hook = NULL;
g_snaptest_fail_after_calls = -1;
char cmd[512];
snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir);
system(cmd);
printf("test_snapshot_transfer_stops_and_propagates_send_failure: PASS\n");
}
/* ── Task 3.5c: Standby snapshot receive handler ──────────────────────────── */
/* Helper: initialise a minimal ha_state_t for snapshot receive tests */
@ -3446,6 +3640,113 @@ static void test_snapshot_recv_128kb_two_chunks(void)
printf("test_snapshot_recv_128kb_two_chunks: PASS\n");
}
/* bug-1389: a live --force-snapshot soak observed the vtask pool dispatching
* a 12-chunk transfer's chunks out of chunk_index order — chunk 2 was
* processed AFTER chunks 3 and 4 (arrival order 0,1,3,4,2,5,6,7,8,9,10,11).
* Live crc32 instrumentation proved every chunk's CONTENT survived the wire
* intact; the bug was that ha_handle_snapshot_chunk appended bytes in
* ARRIVAL order (fopen "ab") instead of writing each chunk at its own
* chunk_offset, so chunks 2/3/4 landed at the wrong file positions —
* same total size, wrong content, exactly matching the corruption the soak
* saw. This test replays that exact arrival order and asserts the
* assembled file is byte-identical to the correctly-ordered source,
* regardless of the order ha_handle_snapshot_chunk was called in. */
static void test_snapshot_recv_out_of_order_chunks_assemble_correctly(void)
{
char tmpdir[] = "/tmp/xcectest_snaprecv3_XXXXXX";
TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir));
char snap_dir[512];
snprintf(snap_dir, sizeof(snap_dir), "%s/snap", tmpdir);
TEST_ASSERT_EQUAL_INT(0, mkdir(snap_dir, 0700));
/* 4 chunks of 64 bytes each = 256-byte body. Must be a WELL-FORMED
* entity-bearing checkpoint (same reason as
* test_snapshot_recv_two_chunks_assembles_and_renames above): legacy
* 57-byte header (cfgref_present=0) + format_version=1 + job_count=0 +
* queue_count=0, then filler out to 256 bytes. The filler is
* position-dependent (distinct per source byte, not per chunk) so ANY
* chunk landing at the wrong file offset is immediately detectable via
* a whole-buffer memcmp, regardless of which physical chunk a given
* structural field happens to fall in. */
uint64_t snap_lsn = 400;
size_t total_size = 256;
uint8_t body[256];
memset(body, 0, sizeof(body));
memcpy(body, "XCESNAP\0", 8);
memcpy(body + 8, &snap_lsn, 8);
for (int i = 16; i < 256; i++)
body[i] = (uint8_t)(i * 3 + 7); /* position-dependent, non-repeating */
uint32_t fake_fmt = 1u;
uint32_t fake_job_count = 0u;
uint32_t fake_queue_count = 0u;
memcpy(body + 57, &fake_fmt, sizeof(fake_fmt));
memcpy(body + 61, &fake_job_count, sizeof(fake_job_count));
memcpy(body + 65, &fake_queue_count, sizeof(fake_queue_count));
persist_t *p = calloc(1, sizeof(persist_t));
TEST_ASSERT_NOT_NULL(p);
persist_config_t pcfg = persist_config_defaults();
persist_set_workdir(p, tmpdir, "ctrl-snaprecv3");
persist_init(p, &pcfg, NULL);
persist_start(p);
ha_state_t ha;
snap_recv_ha_init(&ha);
/* Deliver out of chunk_index order: 0, 2, 1, 3 (isLast). */
int order[4] = { 0, 2, 1, 3 };
for (int oi = 0; oi < 4; oi++) {
int idx = order[oi];
snapshot_transfer_t c;
memset(&c, 0, sizeof(c));
c.snapshot_lsn = snap_lsn;
c.chunk_index = (uint32_t)idx;
c.chunk_offset = (uint64_t)(idx * 64);
c.total_size = (uint64_t)total_size;
c.is_last = (idx == 3);
int rc = ha_handle_snapshot_chunk(&ha, p, snap_dir, &c, body + idx * 64, 64);
TEST_ASSERT_EQUAL_INT(0, rc);
}
/* Final file must exist (finalize triggers once all 4 chunks are in,
* regardless of the delivery order above) and be byte-identical to the
* correctly-ordered source body — proves each chunk landed at its own
* chunk_offset, not at the position it happened to arrive in. */
char final_path[600];
snprintf(final_path, sizeof(final_path),
"%s/snap-%016" PRIx64 ".bin", snap_dir, snap_lsn);
struct stat st;
TEST_ASSERT_EQUAL_INT_MESSAGE(0, stat(final_path, &st),
"snap-{lsn}.bin should exist once all "
"chunks are in, regardless of arrival order");
TEST_ASSERT_EQUAL_UINT64((uint64_t)total_size, (uint64_t)st.st_size);
FILE *f = fopen(final_path, "rb");
TEST_ASSERT_NOT_NULL(f);
uint8_t got[256];
TEST_ASSERT_EQUAL_size_t(256, fread(got, 1, 256, f));
fclose(f);
TEST_ASSERT_EQUAL_MEMORY_MESSAGE(body, got, 256,
"assembled file must be byte-identical to the source despite "
"out-of-chunk_index-order delivery (bug-1389)");
uint64_t got_lsn = (uint64_t)atomic_load_explicit(&ha.applied_lsn,
memory_order_acquire);
TEST_ASSERT_EQUAL_UINT64(snap_lsn, got_lsn);
ha_fini(&ha);
persist_stop(p);
persist_fini(p);
free(p);
char cmd[512];
snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir);
system(cmd);
printf("test_snapshot_recv_out_of_order_chunks_assemble_correctly: PASS\n");
}
/* Test 3.5d: reconnected standby discards snap.part and restarts from chunk 0.
*
* Design §Snapshot Transfer: "Mid-transfer reconnect always restarts snapshot
@ -3709,6 +4010,8 @@ void test_config_repl(void)
RUN_TEST(test_load_primary_config_missing_returns_error);
RUN_TEST(test_wait_all_with_timeout_slow_threads);
RUN_TEST(test_wait_all_with_timeout_fast_threads);
/* bug-1381: push_config_transport_only endpoint-lifetime regression */
RUN_TEST(test_configpush_transport_holds_endpoint_ref_across_send);
/* Task 2.1: OPCode_walCatchupEntry handler */
RUN_TEST(test_wal_catchup_entry_advances_applied_lsn);
RUN_TEST(test_wal_catchup_entry_error_skipped_no_crash);
@ -3733,6 +4036,7 @@ void test_config_repl(void)
/* Task 3.5: snapshot chunk transfer (sender side) */
RUN_TEST(test_snapshot_transfer_two_chunks);
RUN_TEST(test_snapshot_transfer_small_less_than_one_chunk);
RUN_TEST(test_snapshot_transfer_stops_and_propagates_send_failure);
/* Task 3.5c: standby snapshot receive handler */
RUN_TEST(test_snapshot_recv_two_chunks_assembles_and_renames);
/* Task 6.1: legacy position-only file consumed end-to-end via the
@ -3742,6 +4046,7 @@ void test_config_repl(void)
RUN_TEST(test_snapshot_recv_null_args_fail);
/* Task 6.4-A: 128KB snapshot in 2 chunks — full-size receive path */
RUN_TEST(test_snapshot_recv_128kb_two_chunks);
RUN_TEST(test_snapshot_recv_out_of_order_chunks_assemble_correctly);
/* Task 3.5d: reconnect discards snap.part */
RUN_TEST(test_snapshot_recv_reconnect_discards_snappart);
/* Task 5.1: handle_ctrlhello connection setup only — no repl messages */