wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity

fix(raft-drive-async): resolve snapshot chunk-drop (bug-1386) and chunk misordering (bug-1389)

Browse source 
ha_send_snapshot_to_standby ignored the network send's return value in its
chunk loop, silently truncating a transfer on a mid-stream error while still
logging "complete" (bug-1386). Fixing the silent drop exposed a second,
independent bug in the same pipeline: ha_handle_snapshot_chunk appended
received chunks in arrival order instead of at their wire-carried
chunk_offset, so out-of-order delivery across the vtask pool corrupted the
reassembled file even though every chunk's bytes were individually correct
(bug-1389). Fixed by checking every fallible call in the send loop and
switching the receiver to pwrite-at-offset with a count-based completion
tracker.

Adds snapverify/snapdiff xce_test commands (offline checkpoint-file
structural validator and byte-level comparator) used to diagnose bug-1389 and
kept as standing diagnostic tools for future snapshot-transfer triage.

g_drive_async_configpush_offload flipped back ON after both fixes, validated
via 6-round chaotic + 3/3 force-snapshot soak (raft-drive-async task 3.3).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SDWVNQLKxqTVHh42cH7rV3
This commit is contained in:
Rongsong Shen 2026-07-03 02:40:12 +00:00
parent 02e46f6ba1
commit 5cb36794d2
14 changed files with 1553 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

109
src/lib/testsuite/test_wal_wire.c
View file

@ -248,6 +248,113 @@ static void test_snapshot_chunk_roundtrip(void)
printf("test_snapshot_chunk_roundtrip: PASS\n");
}
/* bug-1389: a force-snapshot soak observed the RECEIVED chunk's content
* diverging from the SENT content starting at an exact SNAPSHOT_CHUNK_SIZE
* (65536-byte) boundary, byte-shifted by a nondeterministic amount, while
* the total size stayed correct. test_snapshot_chunk_roundtrip above only
* exercises a 300-byte payload — far short of the 65536-byte production
* chunk size that actually forces xce_snapshot_chunk_encode_raw's
* capn_new_list8 call to overflow the root segment and cross into a second
* segment via a far pointer (the exact mechanism bug-847 already proved can
* corrupt a capnp Data field in this codebase, for a different message
* shape). This reproduces the REAL production chunk size in a single
* deterministic encode->decode round-trip, no network involved — isolating
* whether the corruption is in this library's capnp handling at all, or
* only manifests under real transport/timing conditions. */
static void test_snapshot_chunk_roundtrip_full_size_payload(void)
{
/* Arrange: exactly SNAPSHOT_CHUNK_SIZE (config_repl.c) bytes, with a
* position-dependent, non-repeating-within-256-bytes pattern so ANY
* byte-shift (not just a zeroed/dropped payload) is immediately
* detectable by TEST_ASSERT_EQUAL_MEMORY instead of accidentally
* matching shifted-but-still-plausible content. */
enum { CHUNK_SZ = 65536 };
uint8_t *payload = malloc(CHUNK_SZ);
TEST_ASSERT_NOT_NULL(payload);
for (size_t i = 0; i < CHUNK_SZ; i++)
payload[i] = (uint8_t)((i * 2654435761u) >> 24); /* Knuth multiplicative hash */
snapshot_transfer_t chunk;
memset(&chunk, 0, sizeof(chunk));
chunk.snapshot_lsn = 117;
chunk.chunk_index = 0;
chunk.chunk_offset = 0;
chunk.total_size = 2 * CHUNK_SZ;
chunk.is_last = 0;
chunk.checkpoint_term = 4;
io_packet_t *pkt = (io_packet_t *)calloc(1, sizeof(*pkt));
TEST_ASSERT_NOT_NULL(pkt);
TEST_ASSERT_EQUAL_INT(0,
xce_snapshot_chunk_encode_raw(&chunk, payload, CHUNK_SZ, &test_peer,
11, pkt));
/* Act */
xce_request_t req;
memset(&req, 0, sizeof(req));
TEST_ASSERT_EQUAL_INT(0, xce_decode(pkt, &req));
/* Assert: envelope + every byte of a full-size chunk survives intact */
TEST_ASSERT_EQUAL_INT(OPCode_snapshotTransfer, req.opcode);
TEST_ASSERT_EQUAL_INT(Request_body_snapshottransfermsg, req.body_which);
snapshot_transfer_t *rx = req.body.snapshottransfermsg;
TEST_ASSERT_NOT_NULL(rx);
TEST_ASSERT_EQUAL_UINT64(117, rx->snapshot_lsn);
TEST_ASSERT_EQUAL_UINT64(4, rx->checkpoint_term);
TEST_ASSERT_EQUAL_size_t(CHUNK_SZ, rx->raw_payload_len);
TEST_ASSERT_NOT_NULL(rx->raw_payload);
TEST_ASSERT_EQUAL_MEMORY(payload, rx->raw_payload, CHUNK_SZ);
xce_free_request(&req);
free(payload);
printf("test_snapshot_chunk_roundtrip_full_size_payload: PASS\n");
}
/* bug-1389: same as above but for chunk_index 1 (the SECOND chunk of a
* multi-chunk transfer) — the observed force-snapshot corruption started at
* a chunk-boundary OTHER than the very first chunk in one of the two
* repro runs (offset 131072 == chunk index 2's start), so this covers a
* non-zero chunk_index/chunk_offset combination too, not just chunk 0. */
static void test_snapshot_chunk_roundtrip_full_size_payload_nonzero_index(void)
{
enum { CHUNK_SZ = 65536 };
uint8_t *payload = malloc(CHUNK_SZ);
TEST_ASSERT_NOT_NULL(payload);
for (size_t i = 0; i < CHUNK_SZ; i++)
payload[i] = (uint8_t)((i * 2654435761u) >> 24);
snapshot_transfer_t chunk;
memset(&chunk, 0, sizeof(chunk));
chunk.snapshot_lsn = 118;
chunk.chunk_index = 2;
chunk.chunk_offset = 2 * CHUNK_SZ;
chunk.total_size = 4 * CHUNK_SZ;
chunk.is_last = 0;
chunk.checkpoint_term = 4;
io_packet_t *pkt = (io_packet_t *)calloc(1, sizeof(*pkt));
TEST_ASSERT_NOT_NULL(pkt);
TEST_ASSERT_EQUAL_INT(0,
xce_snapshot_chunk_encode_raw(&chunk, payload, CHUNK_SZ, &test_peer,
12, pkt));
xce_request_t req;
memset(&req, 0, sizeof(req));
TEST_ASSERT_EQUAL_INT(0, xce_decode(pkt, &req));
snapshot_transfer_t *rx = req.body.snapshottransfermsg;
TEST_ASSERT_NOT_NULL(rx);
TEST_ASSERT_EQUAL_UINT32(2, rx->chunk_index);
TEST_ASSERT_EQUAL_UINT64(2 * CHUNK_SZ, rx->chunk_offset);
TEST_ASSERT_EQUAL_size_t(CHUNK_SZ, rx->raw_payload_len);
TEST_ASSERT_NOT_NULL(rx->raw_payload);
TEST_ASSERT_EQUAL_MEMORY(payload, rx->raw_payload, CHUNK_SZ);
xce_free_request(&req);
free(payload);
printf("test_snapshot_chunk_roundtrip_full_size_payload_nonzero_index: PASS\n");
}
static void test_snapshot_chunk_empty_payload(void)
{
/* Zero-length payload (defensive): envelope round-trips, raw NULL/0 */
@ -286,5 +393,7 @@ void test_wal_wire(void)
RUN_TEST(test_wal_batch_entry_without_payload);
RUN_TEST(test_wal_batch_encode_rejects_invalid_args);
RUN_TEST(test_snapshot_chunk_roundtrip);
RUN_TEST(test_snapshot_chunk_roundtrip_full_size_payload);
RUN_TEST(test_snapshot_chunk_roundtrip_full_size_payload_nonzero_index);
RUN_TEST(test_snapshot_chunk_empty_payload);
}