forked from lei.chen/myxce
Rongsong Shen
5cb36794d2
ha_send_snapshot_to_standby ignored the network send's return value in its chunk loop, silently truncating a transfer on a mid-stream error while still logging "complete" (bug-1386). Fixing the silent drop exposed a second, independent bug in the same pipeline: ha_handle_snapshot_chunk appended received chunks in arrival order instead of at their wire-carried chunk_offset, so out-of-order delivery across the vtask pool corrupted the reassembled file even though every chunk's bytes were individually correct (bug-1389). Fixed by checking every fallible call in the send loop and switching the receiver to pwrite-at-offset with a count-based completion tracker. Adds snapverify/snapdiff xce_test commands (offline checkpoint-file structural validator and byte-level comparator) used to diagnose bug-1389 and kept as standing diagnostic tools for future snapshot-transfer triage. g_drive_async_configpush_offload flipped back ON after both fixes, validated via 6-round chaotic + 3/3 force-snapshot soak (raft-drive-async task 3.3). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SDWVNQLKxqTVHh42cH7rV3
284 lines
9.2 KiB
C
284 lines
9.2 KiB
C
#define _GNU_SOURCE
|
|
#include "snapverify.h"
|
|
|
|
#include <errno.h>
|
|
#include <fcntl.h>
|
|
#include <inttypes.h>
|
|
#include <string.h>
|
|
#include <sys/stat.h>
|
|
#include <unistd.h>
|
|
|
|
/* Mirrors persist.c's SNAPSHOT_FORMAT_VERSION / header layout exactly — see
|
|
* persist.c write_snapshot() (~line 1758) and load_snapshot() (~line 2325). */
|
|
#define SNAP_FORMAT_VERSION 1u
|
|
#define SNAP_HEADER_MAX (16 + 1 + 8 + 32) /* magic+lsn + optional cfgref tail */
|
|
|
|
static void fail(snapverify_report_t* rep, int rc, off_t off, const char* why)
|
|
{
|
|
rep->ok = 0;
|
|
rep->fail_offset = off;
|
|
snprintf(rep->fail_reason, sizeof(rep->fail_reason), "%s", why);
|
|
(void)rc;
|
|
}
|
|
|
|
int snapverify_path(const char* path, FILE* out, snapverify_report_t* rep)
|
|
{
|
|
if (!path)
|
|
return -EINVAL;
|
|
|
|
snapverify_report_t local;
|
|
memset(&local, 0, sizeof(local));
|
|
local.ok = 1;
|
|
local.fail_offset = -1;
|
|
|
|
int fd = open(path, O_RDONLY);
|
|
if (fd < 0) {
|
|
int saved = errno;
|
|
fail(&local, -saved, -1, strerror(saved));
|
|
if (rep)
|
|
*rep = local;
|
|
if (out)
|
|
fprintf(out, "snapverify: %s: %s\n", path, strerror(saved));
|
|
return -saved;
|
|
}
|
|
|
|
struct stat st;
|
|
if (fstat(fd, &st) != 0) {
|
|
int saved = errno;
|
|
close(fd);
|
|
fail(&local, -saved, -1, strerror(saved));
|
|
if (rep)
|
|
*rep = local;
|
|
if (out)
|
|
fprintf(out, "snapverify: %s: fstat: %s\n", path, strerror(saved));
|
|
return -saved;
|
|
}
|
|
local.file_size = st.st_size;
|
|
|
|
char header[SNAP_HEADER_MAX];
|
|
ssize_t n = read(fd, header, sizeof(header));
|
|
if (n < 16) {
|
|
close(fd);
|
|
fail(&local, -EIO, 0, "short read on header (need >= 16 bytes)");
|
|
if (rep)
|
|
*rep = local;
|
|
if (out)
|
|
fprintf(out, "snapverify: %s: %s\n", path, local.fail_reason);
|
|
return -EIO;
|
|
}
|
|
|
|
if (memcmp(header, "XCESNAP\0", 8) != 0) {
|
|
close(fd);
|
|
fail(&local, -EINVAL, 0, "bad magic (not XCESNAP)");
|
|
if (rep)
|
|
*rep = local;
|
|
if (out)
|
|
fprintf(out, "snapverify: %s: %s\n", path, local.fail_reason);
|
|
return -EINVAL;
|
|
}
|
|
|
|
memcpy(&local.checkpoint_lsn, header + 8, 8);
|
|
if (n >= SNAP_HEADER_MAX && header[16] != 0)
|
|
local.has_cfgref = 1;
|
|
|
|
off_t consumed = (off_t)n;
|
|
|
|
if (st.st_size <= consumed) {
|
|
/* Legacy, position-only checkpoint — not a defect (design.md). */
|
|
close(fd);
|
|
local.consumed = consumed;
|
|
if (rep)
|
|
*rep = local;
|
|
if (out)
|
|
fprintf(out,
|
|
"snapverify: %s: OK (legacy position-only, lsn=%" PRIu64
|
|
", size=%jd)\n",
|
|
path, local.checkpoint_lsn, (intmax_t)local.file_size);
|
|
return 0;
|
|
}
|
|
|
|
uint32_t fmt = 0;
|
|
if (read(fd, &fmt, sizeof(fmt)) != (ssize_t)sizeof(fmt)) {
|
|
close(fd);
|
|
fail(&local, -EIO, consumed, "short read on format_version");
|
|
if (rep)
|
|
*rep = local;
|
|
if (out)
|
|
fprintf(out, "snapverify: %s: %s (offset %jd)\n", path,
|
|
local.fail_reason, (intmax_t)consumed);
|
|
return -EIO;
|
|
}
|
|
consumed += (off_t)sizeof(fmt);
|
|
local.has_entity_section = 1;
|
|
local.format_version = fmt;
|
|
|
|
if (fmt != SNAP_FORMAT_VERSION) {
|
|
close(fd);
|
|
fail(&local, -EINVAL, consumed - (off_t)sizeof(fmt),
|
|
"unrecognized format_version");
|
|
if (rep)
|
|
*rep = local;
|
|
if (out)
|
|
fprintf(out, "snapverify: %s: %s (got %u, expected %u, offset %jd)\n",
|
|
path, local.fail_reason, fmt, SNAP_FORMAT_VERSION,
|
|
(intmax_t)local.fail_offset);
|
|
return -EINVAL;
|
|
}
|
|
|
|
uint32_t job_count = 0;
|
|
if (read(fd, &job_count, sizeof(job_count)) != (ssize_t)sizeof(job_count)) {
|
|
close(fd);
|
|
fail(&local, -EIO, consumed, "short read on job_count");
|
|
if (rep)
|
|
*rep = local;
|
|
if (out)
|
|
fprintf(out, "snapverify: %s: %s (offset %jd)\n", path,
|
|
local.fail_reason, (intmax_t)consumed);
|
|
return -EIO;
|
|
}
|
|
consumed += (off_t)sizeof(job_count);
|
|
local.job_count = job_count;
|
|
|
|
for (uint32_t i = 0; i < job_count; i++) {
|
|
off_t len_off = consumed;
|
|
uint32_t plen = 0;
|
|
if (read(fd, &plen, sizeof(plen)) != (ssize_t)sizeof(plen)) {
|
|
close(fd);
|
|
fail(&local, -EIO, len_off, "short read on job payload_len");
|
|
if (rep)
|
|
*rep = local;
|
|
if (out)
|
|
fprintf(out, "snapverify: %s: job %u: %s (offset %jd)\n", path,
|
|
i, local.fail_reason, (intmax_t)len_off);
|
|
return -EIO;
|
|
}
|
|
consumed += (off_t)sizeof(plen);
|
|
|
|
if (plen == 0 || (off_t)plen > st.st_size - consumed) {
|
|
close(fd);
|
|
char why[160];
|
|
snprintf(why, sizeof(why),
|
|
"bad payload_len=%u (remaining=%jd)", plen,
|
|
(intmax_t)(st.st_size - consumed));
|
|
fail(&local, -EIO, len_off, why);
|
|
if (rep)
|
|
*rep = local;
|
|
if (out)
|
|
fprintf(out, "snapverify: %s: job %u: %s (length field at "
|
|
"offset %jd)\n",
|
|
path, i, local.fail_reason, (intmax_t)len_off);
|
|
return -EIO;
|
|
}
|
|
|
|
if (lseek(fd, plen, SEEK_CUR) < 0) {
|
|
close(fd);
|
|
fail(&local, -EIO, consumed, "seek past job payload failed");
|
|
if (rep)
|
|
*rep = local;
|
|
if (out)
|
|
fprintf(out, "snapverify: %s: job %u: %s\n", path, i,
|
|
local.fail_reason);
|
|
return -EIO;
|
|
}
|
|
consumed += (off_t)plen;
|
|
local.jobs_ok++;
|
|
}
|
|
|
|
uint32_t queue_count = 0;
|
|
if (read(fd, &queue_count, sizeof(queue_count)) !=
|
|
(ssize_t)sizeof(queue_count)) {
|
|
close(fd);
|
|
fail(&local, -EIO, consumed, "short read on queue_count");
|
|
if (rep)
|
|
*rep = local;
|
|
if (out)
|
|
fprintf(out, "snapverify: %s: %s (offset %jd)\n", path,
|
|
local.fail_reason, (intmax_t)consumed);
|
|
return -EIO;
|
|
}
|
|
consumed += (off_t)sizeof(queue_count);
|
|
local.queue_count = queue_count;
|
|
|
|
for (uint32_t i = 0; i < queue_count; i++) {
|
|
off_t len_off = consumed;
|
|
uint32_t name_len = 0;
|
|
if (read(fd, &name_len, sizeof(name_len)) != (ssize_t)sizeof(name_len)) {
|
|
close(fd);
|
|
fail(&local, -EIO, len_off, "short read on queue name_len");
|
|
if (rep)
|
|
*rep = local;
|
|
if (out)
|
|
fprintf(out, "snapverify: %s: queue %u: %s (offset %jd)\n",
|
|
path, i, local.fail_reason, (intmax_t)len_off);
|
|
return -EIO;
|
|
}
|
|
consumed += (off_t)sizeof(name_len);
|
|
|
|
if (name_len == 0 || (off_t)name_len > st.st_size - consumed) {
|
|
close(fd);
|
|
char why[160];
|
|
snprintf(why, sizeof(why),
|
|
"bad name_len=%u (remaining=%jd)", name_len,
|
|
(intmax_t)(st.st_size - consumed));
|
|
fail(&local, -EIO, len_off, why);
|
|
if (rep)
|
|
*rep = local;
|
|
if (out)
|
|
fprintf(out, "snapverify: %s: queue %u: %s (length field at "
|
|
"offset %jd)\n",
|
|
path, i, local.fail_reason, (intmax_t)len_off);
|
|
return -EIO;
|
|
}
|
|
|
|
if (lseek(fd, name_len, SEEK_CUR) < 0) {
|
|
close(fd);
|
|
fail(&local, -EIO, consumed, "seek past queue name failed");
|
|
if (rep)
|
|
*rep = local;
|
|
if (out)
|
|
fprintf(out, "snapverify: %s: queue %u: %s\n", path, i,
|
|
local.fail_reason);
|
|
return -EIO;
|
|
}
|
|
consumed += (off_t)name_len;
|
|
|
|
uint32_t status = 0;
|
|
if (read(fd, &status, sizeof(status)) != (ssize_t)sizeof(status)) {
|
|
close(fd);
|
|
fail(&local, -EIO, consumed, "short read on queue status");
|
|
if (rep)
|
|
*rep = local;
|
|
if (out)
|
|
fprintf(out, "snapverify: %s: queue %u: %s (offset %jd)\n",
|
|
path, i, local.fail_reason, (intmax_t)consumed);
|
|
return -EIO;
|
|
}
|
|
consumed += (off_t)sizeof(status);
|
|
local.queues_ok++;
|
|
}
|
|
|
|
close(fd);
|
|
local.consumed = consumed;
|
|
if (consumed != st.st_size) {
|
|
char why[160];
|
|
snprintf(why, sizeof(why),
|
|
"trailing garbage: consumed=%jd but file_size=%jd",
|
|
(intmax_t)consumed, (intmax_t)st.st_size);
|
|
fail(&local, -EIO, consumed, why);
|
|
if (rep)
|
|
*rep = local;
|
|
if (out)
|
|
fprintf(out, "snapverify: %s: %s\n", path, local.fail_reason);
|
|
return -EIO;
|
|
}
|
|
|
|
if (rep)
|
|
*rep = local;
|
|
if (out)
|
|
fprintf(out,
|
|
"snapverify: %s: OK (lsn=%" PRIu64 " fmt=%u jobs=%u queues=%u "
|
|
"size=%jd)\n",
|
|
path, local.checkpoint_lsn, local.format_version,
|
|
local.job_count, local.queue_count, (intmax_t)local.file_size);
|
|
return 0;
|
|
}
|