lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(raft-drive-async): resolve snapshot chunk-drop (bug-1386) and chunk misordering (bug-1389)

Browse source 
ha_send_snapshot_to_standby ignored the network send's return value in its
chunk loop, silently truncating a transfer on a mid-stream error while still
logging "complete" (bug-1386). Fixing the silent drop exposed a second,
independent bug in the same pipeline: ha_handle_snapshot_chunk appended
received chunks in arrival order instead of at their wire-carried
chunk_offset, so out-of-order delivery across the vtask pool corrupted the
reassembled file even though every chunk's bytes were individually correct
(bug-1389). Fixed by checking every fallible call in the send loop and
switching the receiver to pwrite-at-offset with a count-based completion
tracker.

Adds snapverify/snapdiff xce_test commands (offline checkpoint-file
structural validator and byte-level comparator) used to diagnose bug-1389 and
kept as standing diagnostic tools for future snapshot-transfer triage.

g_drive_async_configpush_offload flipped back ON after both fixes, validated
via 6-round chaotic + 3/3 force-snapshot soak (raft-drive-async task 3.3).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SDWVNQLKxqTVHh42cH7rV3
This commit is contained in:
Rongsong Shen 2026-07-03 02:40:12 +00:00
parent 02e46f6ba1
commit 5cb36794d2
14 changed files with 1553 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

54
src/controller/testsuite/test.c
View file

@ -25,6 +25,8 @@
#include "preason.h"
#include "walcompare.h" /* cross-member committed-prefix divergence (cmd_walcompare) */
#include "waldump.h" /* offline WAL inspector (cmd_waldump) */
#include "snapdiff.h" /* byte-level checkpoint-file comparator (cmd_snapdiff) */
#include "snapverify.h" /* offline checkpoint-file structural validator (cmd_snapverify) */
#include "xce/cluster.h" /* xce_parse_peers, xce_cluster_config_t */
#include "xce/types.h"
@ -127,6 +129,8 @@ extern int cmd_wait_primary(int, char*[]);
extern int cmd_ctrlstatus(int, char*[]);
extern int cmd_waldump(int, char*[]);
extern int cmd_walcompare(int, char*[]);
extern int cmd_snapverify(int, char*[]);
extern int cmd_snapdiff(int, char*[]);
static int worker_house_keeping(void*);
/* Session verdict / loop state (xce-test-e2e-primitives task 1.1).
@ -186,6 +190,8 @@ static command_t top_commands[] = {
{ "echo", cmd_echo, "Print a message" },
{ "waldump", cmd_waldump, "Inspect WAL segments" },
{ "walcompare", cmd_walcompare, "Diff member committed WAL prefixes" },
{ "snapverify", cmd_snapverify, "Verify checkpoint file structure" },
{ "snapdiff", cmd_snapdiff, "Byte-diff two checkpoint files" },
{ "quit", cmd_quit, "Quit xce test" },
{ (char*)NULL, (cmd_func_t)NULL, (char*)NULL }
};
@ -225,6 +231,8 @@ static command_t client_commands[] = {
{ "ctrlstatus", cmd_ctrlstatus, "show controller/cluster status" },
{ "waldump", cmd_waldump, "Inspect WAL segments" },
{ "walcompare", cmd_walcompare, "Diff member committed WAL prefixes" },
{ "snapverify", cmd_snapverify, "Verify checkpoint file structure" },
{ "snapdiff", cmd_snapdiff, "Byte-diff two checkpoint files" },
{ "shutdown", cmd_shutdown, "shutdown xce controller" },
{ "jobrun", cmd_jobrun, "run a job" },
{ "jobsub", cmd_jobsub, "submit a job" },
@ -798,6 +806,52 @@ int cmd_walcompare(int argc, char* argv[])
return -1;
}
/* snapverify <snap_file> [<snap_file> ...] — offline checkpoint-file
* structural validator for force-snapshot triage (bug-1389: OPCode_
* snapshotTransfer chunk-content corruption). Each path is a snap-*.capnp
* (locally generated) or snap-*.bin (received via ha_handle_snapshot_chunk)
* file. Read-only: walks the same layout persist_load_snapshot() parses
* without restoring anything, and on the first structural defect reports the
* exact byte offset and how many job/queue frames parsed cleanly before it —
* the diagnostic a bare "persist_load_snapshot failed rc=-5" log line
* doesn't give you. Returns nonzero if any path failed to verify. */
int cmd_snapverify(int argc, char* argv[])
{
if (argc < 2) {
fprintf(stderr, "usage: snapverify <snap_file> [more...]\n");
return -1;
}
int rc = 0;
for (int i = 1; i < argc; i++) {
snapverify_report_t rep;
int r = snapverify_path(argv[i], stdout, &rep);
if (r != 0)
rc = -1;
}
return rc;
}
/* snapdiff <source_file> <received_file> — byte-level checkpoint-file
* comparator for force-snapshot triage (bug-1389). `source_file` is the
* primary's own locally generated checkpoint (snap-*.capnp); `received_file`
* is the copy a standby assembled from the wire (snap-*.bin). Reports the
* first differing byte offset and whether it lands exactly on an
* OPCode_snapshotTransfer chunk boundary — the decisive signal for "the
* transport corrupted one chunk" vs "these were never the same content".
* Returns 0 if identical, nonzero otherwise. */
int cmd_snapdiff(int argc, char* argv[])
{
if (argc != 3) {
fprintf(stderr, "usage: snapdiff <source_file> <received_file>\n");
return -1;
}
snapdiff_report_t rep;
int rc = snapdiff_paths(argv[1], argv[2], stdout, &rep);
return (rc == 0) ? 0 : -1;
}
int cmd_up(int argc, char* argv[])
{
if (active_ctx->parent != NULL) {