wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity
myxce/src/controller/testsuite/test_e2e_capnp.c
Rongsong Shen e53c086ec5 fix(schema): rename struct Data -> XceData to unshadow capnp builtin blob type 
controller.capnp declared `struct Data $C.mapname("xce_data_t")` (an
OPCode_get query-result tagged union) at file scope, which shadows
Cap'n Proto's builtin Data blob type for every later `: Data` field in
the same file. This silently miscompiled 7 fields across 6 structs as
a struct-pointer type instead of a raw byte blob: UpgradeConfigRequest
.sha256/.bytes, ReplConfigReply.sha256/.bytes, ReplConfigPushRequest
.sha256/.bytes, ReplEnvelope.payload, SnapshotTransfer's chunk
payload, WalCatchupEntry.payload/.batchPayloads, and CtrlHelloRequest
.key. Production C code had already routed around it for years via
hand-rolled raw capn_ptr manipulation (see request.c's own comments);
nothing before now needed the generated codec/accessors to actually
work for these fields.

Renamed the local struct to XceData (capnp-level name only; the
$C.mapname("xce_data_t") C typedef is unchanged) and explicitly
repointed getdata/pushmsg at XceData by name, since those two fields
correctly want the local tagged union. Updated every derived
generated-identifier reference across 19 hand-written files spanning
controller, lib, scheduler, and the Go apiserver/xcecli CGo bindings.
Fixed 5 call sites in lib/request.c that had been typed against the
old workaround alias but now legitimately receive the correct
capn_data type (a pure type-name swap -- Data_ptr/XceData_ptr and
capn_data share the identical {capn_ptr p} layout).

Verified via full rebuild+test: lib (1/1), controller (3/3, incl. the
job-array push regression tests), scheduler (15/15), Go apiserver +
xcecli (all packages green), and a direct pycapnp round-trip proving
UpgradeConfigRequest.sha256/.bytes now encode/decode correctly.

Bug: bug-1461 (.wolf/buglog.json). Unblocks admin-config-tool task 3.3.
2026-07-22 09:22:04 +00:00

349 lines
14 KiB
C
Raw Blame History

/* P0-15e: Cap'n Proto partial merge semantics
*
* Verifies that the three persistence Cap'n Proto mapped types
* (capnp_persist_cfg_t, capnp_snap_hdr_t, capnp_repl_envelope_t)
* and the generated struct PersistConfig / SnapshotHeader / ReplEnvelope
* round-trip through their C struct representations correctly.
*
* These are structural/compile-time + value round-trip tests that span
* the Cap'n Proto schema generation (P0-C) and the typemap.h definitions.
*/
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
#include <capnp_c.h>
#include "config.h"
#include "controller.capnp.h"
#include "persist.h"
/* ------------------------------------------------------------------ */
/* Helpers: persist_config_t → capnp_persist_cfg_t round-trip */
/* ------------------------------------------------------------------ */
/* Manually map persist_config_t fields to capnp_persist_cfg_t, matching the
* field mapping in the persist.c checkpoint serialisation path. */
static capnp_persist_cfg_t cfg_to_capnp(const persist_config_t* c)
{
capnp_persist_cfg_t out;
memset(&out, 0, sizeof(out));
out.walSyncMs = c->wal_sync_ms;
out.snapIntervalS = c->snap_interval_s;
out.snapWalSizeMb = c->snap_wal_size_mb;
out.snapKeepN = c->snap_keep_n;
out.transientJobTimeoutS = c->transient_job_timeout_s;
out.finishedRetainS = c->finished_retain_s;
out.workerRetainS = c->worker_retain_s;
out.syncTimeoutMs = c->sync_timeout_ms;
return out;
}
/* ------------------------------------------------------------------ */
/* Test 1: persist_config_t → capnp_persist_cfg_t round-trip */
/* ------------------------------------------------------------------ */
static void test_persist_config_capnp_round_trip(void)
{
persist_config_t src = persist_config_defaults();
/* Override to known values for deterministic checking */
src.wal_sync_ms = 42u;
src.snap_interval_s = 120u;
src.snap_wal_size_mb = 128u;
src.snap_keep_n = 3u;
src.transient_job_timeout_s = 600u;
src.finished_retain_s = 7200u;
src.worker_retain_s = 172800u;
src.sync_timeout_ms = 100u;
capnp_persist_cfg_t out = cfg_to_capnp(&src);
TEST_ASSERT_EQUAL_UINT32(src.wal_sync_ms, out.walSyncMs);
TEST_ASSERT_EQUAL_UINT32(src.snap_interval_s, out.snapIntervalS);
TEST_ASSERT_EQUAL_UINT32(src.snap_wal_size_mb, out.snapWalSizeMb);
TEST_ASSERT_EQUAL_UINT32(src.snap_keep_n, out.snapKeepN);
TEST_ASSERT_EQUAL_UINT32(src.transient_job_timeout_s, out.transientJobTimeoutS);
TEST_ASSERT_EQUAL_UINT32(src.finished_retain_s, out.finishedRetainS);
TEST_ASSERT_EQUAL_UINT32(src.worker_retain_s, out.workerRetainS);
TEST_ASSERT_EQUAL_UINT32(src.sync_timeout_ms, out.syncTimeoutMs);
printf("test_persist_config_capnp_round_trip: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Test 2: SnapshotHeader struct fields and version default */
/* ------------------------------------------------------------------ */
static void test_snapshot_header_capnp_fields(void)
{
/* Verify struct size constant matches the documented schema */
TEST_ASSERT_EQUAL_INT(48, (int)SnapshotHeader_struct_bytes_count);
/* Populate a capnp_snap_hdr_t with known values and check them */
capnp_snap_hdr_t hdr;
memset(&hdr, 0, sizeof(hdr));
hdr.lsn = 12345ULL;
hdr.term = 2ULL;
hdr.timestamp = 999999ULL;
hdr.version = 1u;
hdr.nJobs = 10u;
hdr.nQueues = 3u;
hdr.nWorkers = 5u;
hdr.checksum = 0xDEADBEEFu;
hdr.headerChecksum = 0xCAFEBABEu;
TEST_ASSERT_EQUAL_UINT64(12345ULL, hdr.lsn);
TEST_ASSERT_EQUAL_UINT64(2ULL, hdr.term);
TEST_ASSERT_EQUAL_UINT64(999999ULL, hdr.timestamp);
TEST_ASSERT_EQUAL_UINT32(1u, hdr.version);
TEST_ASSERT_EQUAL_UINT32(10u, hdr.nJobs);
TEST_ASSERT_EQUAL_UINT32(3u, hdr.nQueues);
TEST_ASSERT_EQUAL_UINT32(5u, hdr.nWorkers);
TEST_ASSERT_EQUAL_UINT32(0xDEADBEEFu, hdr.checksum);
TEST_ASSERT_EQUAL_UINT32(0xCAFEBABEu, hdr.headerChecksum);
printf("test_snapshot_header_capnp_fields: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Test 3: ReplEnvelope struct fields accessible */
/* ------------------------------------------------------------------ */
static void test_repl_envelope_capnp_fields(void)
{
/* Verify struct size constant */
TEST_ASSERT_EQUAL_INT(40, (int)ReplEnvelope_struct_bytes_count);
capnp_repl_envelope_t env;
memset(&env, 0, sizeof(env));
env.term = 7ULL;
env.lsn = 100ULL;
env.entryType = (uint32_t)WAL_JOB_SUBMIT;
env.payload = NULL; /* optional data pointer */
env.crc = 0x12345678u;
env.prevCrc = WAL_CRC_NONE;
TEST_ASSERT_EQUAL_UINT64(7ULL, env.term);
TEST_ASSERT_EQUAL_UINT64(100ULL, env.lsn);
TEST_ASSERT_EQUAL_UINT32(WAL_JOB_SUBMIT, env.entryType);
TEST_ASSERT_EQUAL_UINT32(0x12345678u, env.crc);
TEST_ASSERT_EQUAL_UINT32(WAL_CRC_NONE, env.prevCrc);
printf("test_repl_envelope_capnp_fields: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Test 4: RequestVoteRequest carries the Raft (lastLogTerm, */
/* lastLogIndex) tuple through a real encode → wire → decode round-trip */
/* (ha-term-safety task 1.2, Req 2.3/2.4/2.6). */
/* ------------------------------------------------------------------ */
static void test_request_vote_last_log_tuple_round_trip(void)
{
request_vote_req_t src;
memset(&src, 0, sizeof(src));
src.term = 5ULL;
src.candidate_id = "node-1";
src.config_tag = 9ULL;
src.last_log_lsn = 100ULL;
src.last_log_term = 4ULL; /* NEW: candidate's last-log term */
src.last_log_index = 100ULL; /* NEW: candidate's last-log index */
/* Encode the typemap struct into a capnp message, serialize to a flat
* buffer, then decode it back — exercises BOTH the schema field and the
* typemap.h mirror mapping (encode_/decode_ go through the C struct). */
struct capn c;
capn_init_malloc(&c);
struct capn_segment *cs = capn_root(&c).seg;
/* encode_*_ptr allocates a struct in the segment and writes the C struct
* fields into it (via the $C.mapname mapping). */
RequestVoteRequest_ptr p;
encode_RequestVoteRequest_ptr(cs, &p, &src);
capn_setp(capn_root(&c), 0, p.p);
int64_t sz = capn_size(&c);
TEST_ASSERT_TRUE(sz > 0);
uint8_t *buf = (uint8_t *)malloc((size_t)sz);
TEST_ASSERT_NOT_NULL(buf);
capn_write_mem(&c, buf, (size_t)sz, 0);
capn_free(&c);
struct capn d;
TEST_ASSERT_EQUAL_INT(0, capn_init_mem(&d, buf, (size_t)sz, 0));
RequestVoteRequest_ptr dp;
dp.p = capn_getp(capn_root(&d), 0, 1);
struct RequestVoteRequest wire;
read_RequestVoteRequest(&wire, dp);
request_vote_req_t dst;
memset(&dst, 0, sizeof(dst));
decode_RequestVoteRequest(&dst, &wire);
TEST_ASSERT_EQUAL_UINT64(5ULL, dst.term);
TEST_ASSERT_EQUAL_UINT64(9ULL, dst.config_tag);
TEST_ASSERT_EQUAL_UINT64(100ULL, dst.last_log_lsn);
TEST_ASSERT_EQUAL_UINT64(4ULL, dst.last_log_term); /* survives the wire */
TEST_ASSERT_EQUAL_UINT64(100ULL, dst.last_log_index); /* survives the wire */
capn_free(&d);
free(buf);
printf("test_request_vote_last_log_tuple_round_trip: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Test 5: WalCatchupEntry (replication envelope) round-trips the */
/* AppendEntries term fields (prev_log_index, prev_log_term, term) */
/* (ha-term-safety task 1.1, Req 1.3/1.4). */
/* ------------------------------------------------------------------ */
static void test_wal_catchup_append_term_fields_round_trip(void)
{
wal_catchup_entry_t src;
memset(&src, 0, sizeof(src));
src.lsn = 42ULL;
src.entry_type = (uint8_t)WAL_JOB_SUBMIT;
src.term = 7ULL; /* NEW: this entry's term stamp */
src.prev_log_index = 41ULL; /* NEW: lsn - 1 */
src.prev_log_term = 6ULL; /* NEW: term at prev_log_index */
struct capn c;
capn_init_malloc(&c);
struct capn_segment *cs = capn_root(&c).seg;
WalCatchupEntry_ptr p;
encode_WalCatchupEntry_ptr(cs, &p, &src);
capn_setp(capn_root(&c), 0, p.p);
int64_t sz = capn_size(&c);
TEST_ASSERT_TRUE(sz > 0);
uint8_t *buf = (uint8_t *)malloc((size_t)sz);
TEST_ASSERT_NOT_NULL(buf);
capn_write_mem(&c, buf, (size_t)sz, 0);
capn_free(&c);
struct capn d;
TEST_ASSERT_EQUAL_INT(0, capn_init_mem(&d, buf, (size_t)sz, 0));
WalCatchupEntry_ptr dp;
dp.p = capn_getp(capn_root(&d), 0, 1);
struct WalCatchupEntry wire;
read_WalCatchupEntry(&wire, dp);
wal_catchup_entry_t dst;
memset(&dst, 0, sizeof(dst));
decode_WalCatchupEntry(&dst, &wire);
TEST_ASSERT_EQUAL_UINT64(42ULL, dst.lsn);
TEST_ASSERT_EQUAL_UINT64(7ULL, dst.term); /* survives the wire */
TEST_ASSERT_EQUAL_UINT64(41ULL, dst.prev_log_index); /* survives the wire */
TEST_ASSERT_EQUAL_UINT64(6ULL, dst.prev_log_term); /* survives the wire */
capn_free(&d);
free(buf);
printf("test_wal_catchup_append_term_fields_round_trip: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Test 6: Data union's new jobArray variant (job-array-single-wal-record */
/* task 1.2, Req 6.1/6.2) round-trips the compact (base jobid, array size, */
/* one shared spec) push descriptor through the wire -- reusing the client */
/* submit request's existing JobSubmit shape verbatim (no new struct). */
/* This only proves the additive schema variant exists and round-trips; */
/* nothing calls it yet (wiring into sched/apiserver push is task 5.1/5.2). */
/* ------------------------------------------------------------------ */
static void test_data_job_array_variant_round_trip(void)
{
job_submit_data_t submit;
memset(&submit, 0, sizeof(submit));
submit.queue = "batch";
submit.user = "rshen";
submit.mintasks = 1u;
submit.maxtasks = 4u;
submit.start_info = NULL;
submit.job_name = "array-job";
submit.project_name = "proj";
submit.user_group = "grp";
submit.job_description = "job-array-single-wal-record round-trip test";
submit.resreq = "select=1";
xce_job_submit_t src;
memset(&src, 0, sizeof(src));
src.jobid = 5000ULL; /* base jobid */
src.array_size = 250ULL;
src.submit = &submit;
xce_data_t data;
memset(&data, 0, sizeof(data));
data.table = XceData_data_jobArray;
data.data.job_array = &src;
/* Encode -> flat wire buffer -> decode, exercising both the schema
* field (jobArray @6) and the typemap.h mirror mapping (xce_data_t's
* new job_array union member), the same pattern Test 4/5 use. */
struct capn c;
capn_init_malloc(&c);
struct capn_segment *cs = capn_root(&c).seg;
XceData_ptr p;
encode_XceData_ptr(cs, &p, &data);
capn_setp(capn_root(&c), 0, p.p);
int64_t sz = capn_size(&c);
TEST_ASSERT_TRUE(sz > 0);
uint8_t *buf = (uint8_t *)malloc((size_t)sz);
TEST_ASSERT_NOT_NULL(buf);
capn_write_mem(&c, buf, (size_t)sz, 0);
capn_free(&c);
struct capn d;
TEST_ASSERT_EQUAL_INT(0, capn_init_mem(&d, buf, (size_t)sz, 0));
XceData_ptr dp;
dp.p = capn_getp(capn_root(&d), 0, 1);
struct XceData wire;
read_XceData(&wire, dp);
xce_data_t dst;
memset(&dst, 0, sizeof(dst));
decode_XceData(&dst, &wire);
TEST_ASSERT_EQUAL_INT(XceData_data_jobArray, dst.table);
TEST_ASSERT_NOT_NULL(dst.data.job_array);
TEST_ASSERT_EQUAL_UINT64(5000ULL, dst.data.job_array->jobid);
TEST_ASSERT_EQUAL_UINT64(250ULL, dst.data.job_array->array_size);
TEST_ASSERT_NOT_NULL(dst.data.job_array->submit);
TEST_ASSERT_EQUAL_STRING("batch", dst.data.job_array->submit->queue);
TEST_ASSERT_EQUAL_STRING("rshen", dst.data.job_array->submit->user);
TEST_ASSERT_EQUAL_UINT32(1u, dst.data.job_array->submit->mintasks);
TEST_ASSERT_EQUAL_UINT32(4u, dst.data.job_array->submit->maxtasks);
TEST_ASSERT_EQUAL_STRING("array-job", dst.data.job_array->submit->job_name);
/* Untouched: the existing schedjobs/jobs/etc. wire layout is unaffected --
* spot-check the enum values these variants relied on before this change
* did not shift underneath them. */
TEST_ASSERT_EQUAL_INT(0, (int)XceData_data_schedjobs);
TEST_ASSERT_EQUAL_INT(2, (int)XceData_data_jobs);
TEST_ASSERT_EQUAL_INT(5, (int)XceData_data_resvals);
TEST_ASSERT_EQUAL_INT(6, (int)XceData_data_jobArray);
free_XceData(&dst);
capn_free(&d);
free(buf);
printf("test_data_job_array_variant_round_trip: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Suite entry point */
/* ------------------------------------------------------------------ */
void test_e2e_capnp(void)
{
UnitySetTestFile(__FILE__);
printf("\n>> P0-15e: Cap'n Proto partial merge semantics\n\n");
RUN_TEST(test_persist_config_capnp_round_trip);
RUN_TEST(test_snapshot_header_capnp_fields);
RUN_TEST(test_repl_envelope_capnp_fields);
RUN_TEST(test_request_vote_last_log_tuple_round_trip);
RUN_TEST(test_wal_catchup_append_term_fields_round_trip);
RUN_TEST(test_data_job_array_variant_round_trip);
}
View git blame Copy permalink