myxce/src/controller/testsuite/test_snapverify.c
Rongsong Shen 5cb36794d2 fix(raft-drive-async): resolve snapshot chunk-drop (bug-1386) and chunk misordering (bug-1389)
ha_send_snapshot_to_standby ignored the network send's return value in its
chunk loop, silently truncating a transfer on a mid-stream error while still
logging "complete" (bug-1386). Fixing the silent drop exposed a second,
independent bug in the same pipeline: ha_handle_snapshot_chunk appended
received chunks in arrival order instead of at their wire-carried
chunk_offset, so out-of-order delivery across the vtask pool corrupted the
reassembled file even though every chunk's bytes were individually correct
(bug-1389). Fixed by checking every fallible call in the send loop and
switching the receiver to pwrite-at-offset with a count-based completion
tracker.

Adds snapverify/snapdiff xce_test commands (offline checkpoint-file
structural validator and byte-level comparator) used to diagnose bug-1389 and
kept as standing diagnostic tools for future snapshot-transfer triage.

g_drive_async_configpush_offload flipped back ON after both fixes, validated
via 6-round chaotic + 3/3 force-snapshot soak (raft-drive-async task 3.3).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SDWVNQLKxqTVHh42cH7rV3
2026-07-03 02:40:12 +00:00

229 lines
7.4 KiB
C

#define _GNU_SOURCE
#include <errno.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <unity/unity.h>
#include "snapverify.h"
static char g_dir[256];
static void fresh_dir(void)
{
strcpy(g_dir, "/tmp/xce_snapverify_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(g_dir));
}
static void cleanup_dir(void)
{
char cmd[512];
snprintf(cmd, sizeof(cmd), "rm -rf %s", g_dir);
system(cmd);
}
/* Writes a checkpoint file matching persist.c write_snapshot()'s exact
* on-disk layout (magic+lsn+cfgref(41, all-zero/absent) + fmt + job_count +
* per-job[len+buf] + queue_count + per-queue[namelen+name+status]).
* job_lens/job_count and queue_names/queue_count are caller-controlled so
* tests can inject a specific defect. `truncate_last_bytes` drops that many
* bytes off the END of the fully-written file (0 = write it whole). */
static void write_snap(const char* path, uint64_t lsn, uint32_t fmt,
const uint32_t* job_lens, uint32_t job_count,
const char** queue_names, const uint32_t* queue_status,
uint32_t queue_count, int truncate_last_bytes)
{
FILE* fp = fopen(path, "wb");
TEST_ASSERT_NOT_NULL(fp);
char header[16 + 1 + 8 + 32];
memset(header, 0, sizeof(header));
memcpy(header, "XCESNAP\0", 8);
memcpy(header + 8, &lsn, 8);
fwrite(header, 1, sizeof(header), fp);
fwrite(&fmt, sizeof(fmt), 1, fp);
fwrite(&job_count, sizeof(job_count), 1, fp);
for (uint32_t i = 0; i < job_count; i++) {
fwrite(&job_lens[i], sizeof(job_lens[i]), 1, fp);
uint8_t* buf = malloc(job_lens[i]);
memset(buf, 0xAB, job_lens[i]);
fwrite(buf, 1, job_lens[i], fp);
free(buf);
}
fwrite(&queue_count, sizeof(queue_count), 1, fp);
for (uint32_t i = 0; i < queue_count; i++) {
uint32_t nlen = (uint32_t)strlen(queue_names[i]);
fwrite(&nlen, sizeof(nlen), 1, fp);
fwrite(queue_names[i], 1, nlen, fp);
fwrite(&queue_status[i], sizeof(queue_status[i]), 1, fp);
}
fclose(fp);
if (truncate_last_bytes > 0) {
long sz = 0;
FILE* f = fopen(path, "rb");
fseek(f, 0, SEEK_END);
sz = ftell(f);
fclose(f);
TEST_ASSERT_TRUE(truncate(path, sz - truncate_last_bytes) == 0);
}
}
static void test_legacy_position_only_file_is_ok(void)
{
fresh_dir();
char path[512];
snprintf(path, sizeof(path), "%s/snap-000000000000000a.capnp", g_dir);
char header[16];
memcpy(header, "XCESNAP\0", 8);
uint64_t lsn = 10;
memcpy(header + 8, &lsn, 8);
FILE* fp = fopen(path, "wb");
TEST_ASSERT_NOT_NULL(fp);
fwrite(header, 1, sizeof(header), fp);
fclose(fp);
snapverify_report_t rep;
int rc = snapverify_path(path, stdout, &rep);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(1, rep.ok);
TEST_ASSERT_EQUAL_INT(0, rep.has_entity_section);
TEST_ASSERT_EQUAL_UINT64(10, rep.checkpoint_lsn);
cleanup_dir();
}
static void test_entity_bearing_well_formed_file_is_ok(void)
{
fresh_dir();
char path[512];
snprintf(path, sizeof(path), "%s/snap-0000000000000064.capnp", g_dir);
uint32_t job_lens[3] = { 100, 200, 50 };
const char* qnames[2] = { "gpu", "cpu" };
uint32_t qstatus[2] = { 1, 0 };
write_snap(path, 100, 1, job_lens, 3, qnames, qstatus, 2, 0);
snapverify_report_t rep;
int rc = snapverify_path(path, stdout, &rep);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(1, rep.ok);
TEST_ASSERT_EQUAL_INT(1, rep.has_entity_section);
TEST_ASSERT_EQUAL_UINT32(1, rep.format_version);
TEST_ASSERT_EQUAL_UINT32(3, rep.job_count);
TEST_ASSERT_EQUAL_UINT32(3, rep.jobs_ok);
TEST_ASSERT_EQUAL_UINT32(2, rep.queue_count);
TEST_ASSERT_EQUAL_UINT32(2, rep.queues_ok);
TEST_ASSERT_EQUAL_INT(-1, (int)rep.fail_offset);
cleanup_dir();
}
static void test_bad_magic_is_einval(void)
{
fresh_dir();
char path[512];
snprintf(path, sizeof(path), "%s/snap-bad.capnp", g_dir);
FILE* fp = fopen(path, "wb");
TEST_ASSERT_NOT_NULL(fp);
fwrite("NOTASNAP", 1, 8, fp);
char zero[8] = {0};
fwrite(zero, 1, 8, fp);
fclose(fp);
snapverify_report_t rep;
int rc = snapverify_path(path, stdout, &rep);
TEST_ASSERT_EQUAL_INT(-EINVAL, rc);
TEST_ASSERT_EQUAL_INT(0, rep.ok);
cleanup_dir();
}
static void test_truncated_header_is_eio(void)
{
fresh_dir();
char path[512];
snprintf(path, sizeof(path), "%s/snap-short.capnp", g_dir);
FILE* fp = fopen(path, "wb");
TEST_ASSERT_NOT_NULL(fp);
fwrite("XCE", 1, 3, fp); /* far short of the 16-byte minimum */
fclose(fp);
snapverify_report_t rep;
int rc = snapverify_path(path, stdout, &rep);
TEST_ASSERT_EQUAL_INT(-EIO, rc);
TEST_ASSERT_EQUAL_INT(0, rep.ok);
cleanup_dir();
}
static void test_unrecognized_format_version_is_einval(void)
{
fresh_dir();
char path[512];
snprintf(path, sizeof(path), "%s/snap-0000000000000001.capnp", g_dir);
uint32_t job_lens[1] = { 10 };
write_snap(path, 1, 99 /* bogus version */, job_lens, 1, NULL, NULL, 0, 0);
snapverify_report_t rep;
int rc = snapverify_path(path, stdout, &rep);
TEST_ASSERT_EQUAL_INT(-EINVAL, rc);
TEST_ASSERT_EQUAL_INT(0, rep.ok);
TEST_ASSERT_EQUAL_UINT32(99, rep.format_version);
cleanup_dir();
}
/* bug-1389 regression shape: a job frame's length field decodes to garbage
* (e.g. transport corrupted a chunk) mid-stream. snapverify must stop at
* exactly the bad job, report how many parsed cleanly before it, and pinpoint
* the byte offset of the bad length field — the diagnostic this tool exists
* to produce instead of a bare "persist_load_snapshot failed rc=-5". */
static void test_corrupt_job_payload_len_reports_exact_failure_point(void)
{
fresh_dir();
char path[512];
snprintf(path, sizeof(path), "%s/snap-0000000000000002.capnp", g_dir);
uint32_t job_lens[4] = { 50, 60, 70, 80 };
write_snap(path, 2, 1, job_lens, 4, NULL, NULL, 0, 0);
/* Corrupt job 2's length field (3rd job, after header(57)+fmt(4)+
* job_count(4)+job0(4+50)+job1(4+60) = 57+4+4+54+64 = 183) to a value
* that cannot possibly fit in the remaining file. */
FILE* fp = fopen(path, "r+b");
TEST_ASSERT_NOT_NULL(fp);
long job2_len_off = 57 + 4 + 4 + (4 + 50) + (4 + 60);
fseek(fp, job2_len_off, SEEK_SET);
uint32_t garbage = 0x7FFFFFFF;
fwrite(&garbage, sizeof(garbage), 1, fp);
fclose(fp);
snapverify_report_t rep;
int rc = snapverify_path(path, stdout, &rep);
TEST_ASSERT_EQUAL_INT(-EIO, rc);
TEST_ASSERT_EQUAL_INT(0, rep.ok);
TEST_ASSERT_EQUAL_UINT32(2, rep.jobs_ok); /* job 0, job 1 parsed fine */
TEST_ASSERT_EQUAL_INT(job2_len_off, (int)rep.fail_offset);
cleanup_dir();
}
static void test_missing_path_errors(void)
{
snapverify_report_t rep;
int rc = snapverify_path("/tmp/xce_snapverify_does_not_exist_zzz", stdout,
&rep);
TEST_ASSERT_LESS_THAN_INT(0, rc);
}
void test_snapverify(void)
{
RUN_TEST(test_legacy_position_only_file_is_ok);
RUN_TEST(test_entity_bearing_well_formed_file_is_ok);
RUN_TEST(test_bad_magic_is_einval);
RUN_TEST(test_truncated_header_is_eio);
RUN_TEST(test_unrecognized_format_version_is_einval);
RUN_TEST(test_corrupt_job_payload_len_reports_exact_failure_point);
RUN_TEST(test_missing_path_errors);
}