forked from lei.chen/myxce
fix(raft-drive-async): resolve snapshot chunk-drop (bug-1386) and chunk misordering (bug-1389)
ha_send_snapshot_to_standby ignored the network send's return value in its chunk loop, silently truncating a transfer on a mid-stream error while still logging "complete" (bug-1386). Fixing the silent drop exposed a second, independent bug in the same pipeline: ha_handle_snapshot_chunk appended received chunks in arrival order instead of at their wire-carried chunk_offset, so out-of-order delivery across the vtask pool corrupted the reassembled file even though every chunk's bytes were individually correct (bug-1389). Fixed by checking every fallible call in the send loop and switching the receiver to pwrite-at-offset with a count-based completion tracker. Adds snapverify/snapdiff xce_test commands (offline checkpoint-file structural validator and byte-level comparator) used to diagnose bug-1389 and kept as standing diagnostic tools for future snapshot-transfer triage. g_drive_async_configpush_offload flipped back ON after both fixes, validated via 6-round chaotic + 3/3 force-snapshot soak (raft-drive-async task 3.3). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SDWVNQLKxqTVHh42cH7rV3
This commit is contained in:
parent
02e46f6ba1
commit
5cb36794d2
14 changed files with 1553 additions and 43 deletions
284
src/controller/testsuite/snapverify.c
Normal file
284
src/controller/testsuite/snapverify.c
Normal file
|
|
@ -0,0 +1,284 @@
|
|||
#define _GNU_SOURCE
|
||||
#include "snapverify.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <inttypes.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/* Mirrors persist.c's SNAPSHOT_FORMAT_VERSION / header layout exactly — see
|
||||
* persist.c write_snapshot() (~line 1758) and load_snapshot() (~line 2325). */
|
||||
#define SNAP_FORMAT_VERSION 1u
|
||||
#define SNAP_HEADER_MAX (16 + 1 + 8 + 32) /* magic+lsn + optional cfgref tail */
|
||||
|
||||
static void fail(snapverify_report_t* rep, int rc, off_t off, const char* why)
|
||||
{
|
||||
rep->ok = 0;
|
||||
rep->fail_offset = off;
|
||||
snprintf(rep->fail_reason, sizeof(rep->fail_reason), "%s", why);
|
||||
(void)rc;
|
||||
}
|
||||
|
||||
int snapverify_path(const char* path, FILE* out, snapverify_report_t* rep)
|
||||
{
|
||||
if (!path)
|
||||
return -EINVAL;
|
||||
|
||||
snapverify_report_t local;
|
||||
memset(&local, 0, sizeof(local));
|
||||
local.ok = 1;
|
||||
local.fail_offset = -1;
|
||||
|
||||
int fd = open(path, O_RDONLY);
|
||||
if (fd < 0) {
|
||||
int saved = errno;
|
||||
fail(&local, -saved, -1, strerror(saved));
|
||||
if (rep)
|
||||
*rep = local;
|
||||
if (out)
|
||||
fprintf(out, "snapverify: %s: %s\n", path, strerror(saved));
|
||||
return -saved;
|
||||
}
|
||||
|
||||
struct stat st;
|
||||
if (fstat(fd, &st) != 0) {
|
||||
int saved = errno;
|
||||
close(fd);
|
||||
fail(&local, -saved, -1, strerror(saved));
|
||||
if (rep)
|
||||
*rep = local;
|
||||
if (out)
|
||||
fprintf(out, "snapverify: %s: fstat: %s\n", path, strerror(saved));
|
||||
return -saved;
|
||||
}
|
||||
local.file_size = st.st_size;
|
||||
|
||||
char header[SNAP_HEADER_MAX];
|
||||
ssize_t n = read(fd, header, sizeof(header));
|
||||
if (n < 16) {
|
||||
close(fd);
|
||||
fail(&local, -EIO, 0, "short read on header (need >= 16 bytes)");
|
||||
if (rep)
|
||||
*rep = local;
|
||||
if (out)
|
||||
fprintf(out, "snapverify: %s: %s\n", path, local.fail_reason);
|
||||
return -EIO;
|
||||
}
|
||||
|
||||
if (memcmp(header, "XCESNAP\0", 8) != 0) {
|
||||
close(fd);
|
||||
fail(&local, -EINVAL, 0, "bad magic (not XCESNAP)");
|
||||
if (rep)
|
||||
*rep = local;
|
||||
if (out)
|
||||
fprintf(out, "snapverify: %s: %s\n", path, local.fail_reason);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
memcpy(&local.checkpoint_lsn, header + 8, 8);
|
||||
if (n >= SNAP_HEADER_MAX && header[16] != 0)
|
||||
local.has_cfgref = 1;
|
||||
|
||||
off_t consumed = (off_t)n;
|
||||
|
||||
if (st.st_size <= consumed) {
|
||||
/* Legacy, position-only checkpoint — not a defect (design.md). */
|
||||
close(fd);
|
||||
local.consumed = consumed;
|
||||
if (rep)
|
||||
*rep = local;
|
||||
if (out)
|
||||
fprintf(out,
|
||||
"snapverify: %s: OK (legacy position-only, lsn=%" PRIu64
|
||||
", size=%jd)\n",
|
||||
path, local.checkpoint_lsn, (intmax_t)local.file_size);
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t fmt = 0;
|
||||
if (read(fd, &fmt, sizeof(fmt)) != (ssize_t)sizeof(fmt)) {
|
||||
close(fd);
|
||||
fail(&local, -EIO, consumed, "short read on format_version");
|
||||
if (rep)
|
||||
*rep = local;
|
||||
if (out)
|
||||
fprintf(out, "snapverify: %s: %s (offset %jd)\n", path,
|
||||
local.fail_reason, (intmax_t)consumed);
|
||||
return -EIO;
|
||||
}
|
||||
consumed += (off_t)sizeof(fmt);
|
||||
local.has_entity_section = 1;
|
||||
local.format_version = fmt;
|
||||
|
||||
if (fmt != SNAP_FORMAT_VERSION) {
|
||||
close(fd);
|
||||
fail(&local, -EINVAL, consumed - (off_t)sizeof(fmt),
|
||||
"unrecognized format_version");
|
||||
if (rep)
|
||||
*rep = local;
|
||||
if (out)
|
||||
fprintf(out, "snapverify: %s: %s (got %u, expected %u, offset %jd)\n",
|
||||
path, local.fail_reason, fmt, SNAP_FORMAT_VERSION,
|
||||
(intmax_t)local.fail_offset);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
uint32_t job_count = 0;
|
||||
if (read(fd, &job_count, sizeof(job_count)) != (ssize_t)sizeof(job_count)) {
|
||||
close(fd);
|
||||
fail(&local, -EIO, consumed, "short read on job_count");
|
||||
if (rep)
|
||||
*rep = local;
|
||||
if (out)
|
||||
fprintf(out, "snapverify: %s: %s (offset %jd)\n", path,
|
||||
local.fail_reason, (intmax_t)consumed);
|
||||
return -EIO;
|
||||
}
|
||||
consumed += (off_t)sizeof(job_count);
|
||||
local.job_count = job_count;
|
||||
|
||||
for (uint32_t i = 0; i < job_count; i++) {
|
||||
off_t len_off = consumed;
|
||||
uint32_t plen = 0;
|
||||
if (read(fd, &plen, sizeof(plen)) != (ssize_t)sizeof(plen)) {
|
||||
close(fd);
|
||||
fail(&local, -EIO, len_off, "short read on job payload_len");
|
||||
if (rep)
|
||||
*rep = local;
|
||||
if (out)
|
||||
fprintf(out, "snapverify: %s: job %u: %s (offset %jd)\n", path,
|
||||
i, local.fail_reason, (intmax_t)len_off);
|
||||
return -EIO;
|
||||
}
|
||||
consumed += (off_t)sizeof(plen);
|
||||
|
||||
if (plen == 0 || (off_t)plen > st.st_size - consumed) {
|
||||
close(fd);
|
||||
char why[160];
|
||||
snprintf(why, sizeof(why),
|
||||
"bad payload_len=%u (remaining=%jd)", plen,
|
||||
(intmax_t)(st.st_size - consumed));
|
||||
fail(&local, -EIO, len_off, why);
|
||||
if (rep)
|
||||
*rep = local;
|
||||
if (out)
|
||||
fprintf(out, "snapverify: %s: job %u: %s (length field at "
|
||||
"offset %jd)\n",
|
||||
path, i, local.fail_reason, (intmax_t)len_off);
|
||||
return -EIO;
|
||||
}
|
||||
|
||||
if (lseek(fd, plen, SEEK_CUR) < 0) {
|
||||
close(fd);
|
||||
fail(&local, -EIO, consumed, "seek past job payload failed");
|
||||
if (rep)
|
||||
*rep = local;
|
||||
if (out)
|
||||
fprintf(out, "snapverify: %s: job %u: %s\n", path, i,
|
||||
local.fail_reason);
|
||||
return -EIO;
|
||||
}
|
||||
consumed += (off_t)plen;
|
||||
local.jobs_ok++;
|
||||
}
|
||||
|
||||
uint32_t queue_count = 0;
|
||||
if (read(fd, &queue_count, sizeof(queue_count)) !=
|
||||
(ssize_t)sizeof(queue_count)) {
|
||||
close(fd);
|
||||
fail(&local, -EIO, consumed, "short read on queue_count");
|
||||
if (rep)
|
||||
*rep = local;
|
||||
if (out)
|
||||
fprintf(out, "snapverify: %s: %s (offset %jd)\n", path,
|
||||
local.fail_reason, (intmax_t)consumed);
|
||||
return -EIO;
|
||||
}
|
||||
consumed += (off_t)sizeof(queue_count);
|
||||
local.queue_count = queue_count;
|
||||
|
||||
for (uint32_t i = 0; i < queue_count; i++) {
|
||||
off_t len_off = consumed;
|
||||
uint32_t name_len = 0;
|
||||
if (read(fd, &name_len, sizeof(name_len)) != (ssize_t)sizeof(name_len)) {
|
||||
close(fd);
|
||||
fail(&local, -EIO, len_off, "short read on queue name_len");
|
||||
if (rep)
|
||||
*rep = local;
|
||||
if (out)
|
||||
fprintf(out, "snapverify: %s: queue %u: %s (offset %jd)\n",
|
||||
path, i, local.fail_reason, (intmax_t)len_off);
|
||||
return -EIO;
|
||||
}
|
||||
consumed += (off_t)sizeof(name_len);
|
||||
|
||||
if (name_len == 0 || (off_t)name_len > st.st_size - consumed) {
|
||||
close(fd);
|
||||
char why[160];
|
||||
snprintf(why, sizeof(why),
|
||||
"bad name_len=%u (remaining=%jd)", name_len,
|
||||
(intmax_t)(st.st_size - consumed));
|
||||
fail(&local, -EIO, len_off, why);
|
||||
if (rep)
|
||||
*rep = local;
|
||||
if (out)
|
||||
fprintf(out, "snapverify: %s: queue %u: %s (length field at "
|
||||
"offset %jd)\n",
|
||||
path, i, local.fail_reason, (intmax_t)len_off);
|
||||
return -EIO;
|
||||
}
|
||||
|
||||
if (lseek(fd, name_len, SEEK_CUR) < 0) {
|
||||
close(fd);
|
||||
fail(&local, -EIO, consumed, "seek past queue name failed");
|
||||
if (rep)
|
||||
*rep = local;
|
||||
if (out)
|
||||
fprintf(out, "snapverify: %s: queue %u: %s\n", path, i,
|
||||
local.fail_reason);
|
||||
return -EIO;
|
||||
}
|
||||
consumed += (off_t)name_len;
|
||||
|
||||
uint32_t status = 0;
|
||||
if (read(fd, &status, sizeof(status)) != (ssize_t)sizeof(status)) {
|
||||
close(fd);
|
||||
fail(&local, -EIO, consumed, "short read on queue status");
|
||||
if (rep)
|
||||
*rep = local;
|
||||
if (out)
|
||||
fprintf(out, "snapverify: %s: queue %u: %s (offset %jd)\n",
|
||||
path, i, local.fail_reason, (intmax_t)consumed);
|
||||
return -EIO;
|
||||
}
|
||||
consumed += (off_t)sizeof(status);
|
||||
local.queues_ok++;
|
||||
}
|
||||
|
||||
close(fd);
|
||||
local.consumed = consumed;
|
||||
if (consumed != st.st_size) {
|
||||
char why[160];
|
||||
snprintf(why, sizeof(why),
|
||||
"trailing garbage: consumed=%jd but file_size=%jd",
|
||||
(intmax_t)consumed, (intmax_t)st.st_size);
|
||||
fail(&local, -EIO, consumed, why);
|
||||
if (rep)
|
||||
*rep = local;
|
||||
if (out)
|
||||
fprintf(out, "snapverify: %s: %s\n", path, local.fail_reason);
|
||||
return -EIO;
|
||||
}
|
||||
|
||||
if (rep)
|
||||
*rep = local;
|
||||
if (out)
|
||||
fprintf(out,
|
||||
"snapverify: %s: OK (lsn=%" PRIu64 " fmt=%u jobs=%u queues=%u "
|
||||
"size=%jd)\n",
|
||||
path, local.checkpoint_lsn, local.format_version,
|
||||
local.job_count, local.queue_count, (intmax_t)local.file_size);
|
||||
return 0;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue