lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(schema): rename struct Data -> XceData to unshadow capnp builtin blob type

Browse source 
controller.capnp declared `struct Data $C.mapname("xce_data_t")` (an
OPCode_get query-result tagged union) at file scope, which shadows
Cap'n Proto's builtin Data blob type for every later `: Data` field in
the same file. This silently miscompiled 7 fields across 6 structs as
a struct-pointer type instead of a raw byte blob: UpgradeConfigRequest
.sha256/.bytes, ReplConfigReply.sha256/.bytes, ReplConfigPushRequest
.sha256/.bytes, ReplEnvelope.payload, SnapshotTransfer's chunk
payload, WalCatchupEntry.payload/.batchPayloads, and CtrlHelloRequest
.key. Production C code had already routed around it for years via
hand-rolled raw capn_ptr manipulation (see request.c's own comments);
nothing before now needed the generated codec/accessors to actually
work for these fields.

Renamed the local struct to XceData (capnp-level name only; the
$C.mapname("xce_data_t") C typedef is unchanged) and explicitly
repointed getdata/pushmsg at XceData by name, since those two fields
correctly want the local tagged union. Updated every derived
generated-identifier reference across 19 hand-written files spanning
controller, lib, scheduler, and the Go apiserver/xcecli CGo bindings.
Fixed 5 call sites in lib/request.c that had been typed against the
old workaround alias but now legitimately receive the correct
capn_data type (a pure type-name swap -- Data_ptr/XceData_ptr and
capn_data share the identical {capn_ptr p} layout).

Verified via full rebuild+test: lib (1/1), controller (3/3, incl. the
job-array push regression tests), scheduler (15/15), Go apiserver +
xcecli (all packages green), and a direct pycapnp round-trip proving
UpgradeConfigRequest.sha256/.bytes now encode/decode correctly.

Bug: bug-1461 (.wolf/buglog.json). Unblocks admin-config-tool task 3.3.
This commit is contained in:
Rongsong Shen 2026-07-22 09:22:04 +00:00
parent 7ed3a3941f
commit e53c086ec5
21 changed files with 144 additions and 144 deletions
Download patch file Download diff file Expand all files Collapse all files

10
src/controller/apiserver.c
View file

@ -32,7 +32,7 @@ static int xce_apiserver_push_workers(cluster_t* c, endpoint_t* ep)
memset(&data, 0x0, sizeof(data));
xlist_init(&wlist, 8192);
data.table = Data_data_workers;
data.table = XceData_data_workers;
data.data.workers = &workers;
collect_workers(c, (collect_check_fn_t)check_worker_, &wlist);
@ -77,7 +77,7 @@ static int xce_apiserver_push_jobs(cluster_t* c, endpoint_t* ep)
memset(&data, 0x0, sizeof(data));
xlist_init(&jlist, 1024 * 1024);
data.table = Data_data_jobs;
data.table = XceData_data_jobs;
data.data.jobs = &jobs;
collect_jobs(c, (collect_check_fn_t)check_job_, &jlist);
@ -108,7 +108,7 @@ static int apiserver_push_joblist(cluster_t* c, xlist_t* l)
.jobs = (xce_job_t**)l->data };
memset(&data, 0x0, sizeof(data));
data.table = Data_data_jobs;
data.table = XceData_data_jobs;
data.data.jobs = &jobs;
rc = xce_push_data_req(c->apiservers[i].ep, c->myhost, &data,
@ -177,7 +177,7 @@ static int apiserver_push_jobarray(cluster_t* c, xlist_t* l)
int rc;
memset(&data, 0x0, sizeof(data));
data.table = Data_data_jobArray;
data.table = XceData_data_jobArray;
data.data.job_array = &job_array;
memset(&request, 0x0, sizeof(request));
@ -430,7 +430,7 @@ static int apiserver_push_workerlist(cluster_t* c, endpoint_t* ep, xlist_t* l)
memset(&data, 0x0, sizeof(data));
data.table = Data_data_workers;
data.table = XceData_data_workers;
data.data.workers = &workers;
rc = xce_push_data_req(ep, c->myhost, &data, check_controller, c, 100);

12
src/controller/opget.c
View file

@ -199,18 +199,18 @@ int do_get_op(cluster_t* c, endpoint_t* ep, xce_request_t* request)
init_reply_by_req(&reply, request);
switch (request->body.getmsg->table) {
case Data_data_resources:
case XceData_data_resources:
rc = do_reply_reslist(c, ep, request);
break;
case Data_data_workers:
case Data_data_schedworkers:
case XceData_data_workers:
case XceData_data_schedworkers:
rc = do_reply_workerlist(c, ep, request);
break;
case Data_data_schedjobs:
case Data_data_jobs:
case XceData_data_schedjobs:
case XceData_data_jobs:
rc = do_reply_joblist(c, ep, request);
break;
case Data_data_resvals:
case XceData_data_resvals:
rc = do_reply_resvals(c, ep, request);
break;
default:

10
src/controller/sched.c
View file

@ -56,7 +56,7 @@ static int xcesched_push_jobarray(cluster_t* c, xlist_t* l)
job_array.submit = first->submit;
memset(&data, 0x0, sizeof(data));
data.table = Data_data_jobArray;
data.table = XceData_data_jobArray;
data.data.job_array = &job_array;
memset(&request, 0x0, sizeof(request));
@ -103,7 +103,7 @@ static int xcesched_push_joblist(cluster_t* c, xlist_t* l)
memset(&data, 0x0, sizeof(data));
data.table = Data_data_schedjobs;
data.table = XceData_data_schedjobs;
data.data.jobs = &jobs;
/* No committed jobs to push → nothing to send (avoids an empty payload). */
@ -127,7 +127,7 @@ static int xcesched_push_workerlist(cluster_t* c, xlist_t* l)
memset(&data, 0x0, sizeof(data));
data.table = Data_data_schedworkers;
data.table = XceData_data_schedworkers;
data.data.workers = &workers;
rc = xce_push_data_req(c->sched.ep, c->myhost, &data, check_controller, c,
@ -263,7 +263,7 @@ static int xcesched_push_jobs(cluster_t* c)
xlist_init(&jlist, 1024 * 1024);
data.table = Data_data_schedjobs;
data.table = XceData_data_schedjobs;
data.data.jobs = &jobs;
/* ha-convergence-recovery (3.1, 3.2): scheduler-bound full push gates on the
@ -297,7 +297,7 @@ static int xcesched_push_workers(cluster_t* c)
memset(&data, 0x0, sizeof(data));
xlist_init(&wlist, 8192);
data.table = Data_data_schedworkers;
data.table = XceData_data_schedworkers;
data.data.workers = &workers;
collect_workers(c, (collect_check_fn_t)check_worker_, &wlist);

8
src/controller/testsuite/test.c
View file

@ -3412,12 +3412,12 @@ static int myreq_handler(xce_request_t* request, void* args)
dataset_t* datamsg = request->body.datamsg;
switch (datamsg->table) {
case Data_data_schedjobs:
case Data_data_jobs:
case XceData_data_schedjobs:
case XceData_data_jobs:
sched_on_jobs(datamsg->nrows, (xce_job_t**)datamsg->rows);
break;
case Data_data_schedworkers:
case Data_data_workers:
case XceData_data_schedworkers:
case XceData_data_workers:
sched_on_workers(datamsg->nrows, (xce_worker_t**)datamsg->rows);
break;
default:

26
src/controller/testsuite/test_e2e_capnp.c
View file

@ -275,7 +275,7 @@ static void test_data_job_array_variant_round_trip(void)
xce_data_t data;
memset(&data, 0, sizeof(data));
data.table = Data_data_jobArray;
data.table = XceData_data_jobArray;
data.data.job_array = &src;
/* Encode -> flat wire buffer -> decode, exercising both the schema
@ -285,8 +285,8 @@ static void test_data_job_array_variant_round_trip(void)
capn_init_malloc(&c);
struct capn_segment *cs = capn_root(&c).seg;
Data_ptr p;
encode_Data_ptr(cs, &p, &data);
XceData_ptr p;
encode_XceData_ptr(cs, &p, &data);
capn_setp(capn_root(&c), 0, p.p);
int64_t sz = capn_size(&c);
@ -298,15 +298,15 @@ static void test_data_job_array_variant_round_trip(void)
struct capn d;
TEST_ASSERT_EQUAL_INT(0, capn_init_mem(&d, buf, (size_t)sz, 0));
Data_ptr dp;
XceData_ptr dp;
dp.p = capn_getp(capn_root(&d), 0, 1);
struct Data wire;
read_Data(&wire, dp);
struct XceData wire;
read_XceData(&wire, dp);
xce_data_t dst;
memset(&dst, 0, sizeof(dst));
decode_Data(&dst, &wire);
decode_XceData(&dst, &wire);
TEST_ASSERT_EQUAL_INT(Data_data_jobArray, dst.table);
TEST_ASSERT_EQUAL_INT(XceData_data_jobArray, dst.table);
TEST_ASSERT_NOT_NULL(dst.data.job_array);
TEST_ASSERT_EQUAL_UINT64(5000ULL, dst.data.job_array->jobid);
TEST_ASSERT_EQUAL_UINT64(250ULL, dst.data.job_array->array_size);
@ -320,12 +320,12 @@ static void test_data_job_array_variant_round_trip(void)
/* Untouched: the existing schedjobs/jobs/etc. wire layout is unaffected --
* spot-check the enum values these variants relied on before this change
* did not shift underneath them. */
TEST_ASSERT_EQUAL_INT(0, (int)Data_data_schedjobs);
TEST_ASSERT_EQUAL_INT(2, (int)Data_data_jobs);
TEST_ASSERT_EQUAL_INT(5, (int)Data_data_resvals);
TEST_ASSERT_EQUAL_INT(6, (int)Data_data_jobArray);
TEST_ASSERT_EQUAL_INT(0, (int)XceData_data_schedjobs);
TEST_ASSERT_EQUAL_INT(2, (int)XceData_data_jobs);
TEST_ASSERT_EQUAL_INT(5, (int)XceData_data_resvals);
TEST_ASSERT_EQUAL_INT(6, (int)XceData_data_jobArray);
free_Data(&dst);
free_XceData(&dst);
capn_free(&d);
free(buf);
printf("test_data_job_array_variant_round_trip: PASS\n");

10
src/controller/testsuite/test_job_array_apiserver_push.c
View file

@ -201,12 +201,12 @@ static void ap_scan_pushes(endpoint_t* ep, ap_push_obs_t* obs)
rp.p = capn_getp(capn_root(&c), 0, 1);
read_Request(&rq, rp);
if (rq.body_which == Request_body_pushmsg) {
struct Data d;
struct XceData d;
read_Data(&d, rq.body.pushmsg);
read_XceData(&d, rq.body.pushmsg);
obs->n_pushes += 1;
obs->last_data_which = (int)d.data_which;
if (d.data_which == Data_data_jobArray) {
if (d.data_which == XceData_data_jobArray) {
struct JobSubmit js;
read_JobSubmit(&js, d.data.jobArray);
@ -255,7 +255,7 @@ test_multi_element_job_new_produces_one_compact_jobarray_push_per_server(void)
"job structs and not fewer than one per connected server (Req "
"6.2, 6.4)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
Data_data_jobArray, obs.last_data_which,
XceData_data_jobArray, obs.last_data_which,
"the push must encode the compact Data.jobArray descriptor, not "
"Data.jobs (N individually-encoded job structs)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
@ -302,7 +302,7 @@ static void test_single_element_job_new_is_unaffected(void)
"an ordinary single-job notification must still produce exactly "
"one push packet per connected API server, unchanged");
TEST_ASSERT_EQUAL_INT_MESSAGE(
Data_data_jobs, obs.last_data_which,
XceData_data_jobs, obs.last_data_which,
"a single-element notification must go through the EXISTING, "
"UNMODIFIED apiserver_push_joblist() path (Data.jobs), never "
"the new compact jobArray path (Req 8.1)");

10
src/controller/testsuite/test_job_array_sched_push.c
View file

@ -161,12 +161,12 @@ static void jsp_scan_pushes(endpoint_t* ep, jsp_push_obs_t* obs)
rp.p = capn_getp(capn_root(&c), 0, 1);
read_Request(&rq, rp);
if (rq.body_which == Request_body_pushmsg) {
struct Data d;
struct XceData d;
read_Data(&d, rq.body.pushmsg);
read_XceData(&d, rq.body.pushmsg);
obs->n_pushes += 1;
obs->last_data_which = (int)d.data_which;
if (d.data_which == Data_data_jobArray) {
if (d.data_which == XceData_data_jobArray) {
struct JobSubmit js;
read_JobSubmit(&js, d.data.jobArray);
@ -209,7 +209,7 @@ static void test_multi_element_job_new_produces_one_compact_jobarray_push(void)
"a 5-element array notification must produce exactly ONE push packet, "
"not one per element (Req 6.1, 6.4)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
Data_data_jobArray, obs.last_data_which,
XceData_data_jobArray, obs.last_data_which,
"the push must encode the compact Data.jobArray descriptor, not "
"Data.schedjobs/Data.jobs (N individually-encoded job structs)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
@ -251,7 +251,7 @@ static void test_single_element_job_new_is_unaffected(void)
"an ordinary single-job notification must still produce exactly one "
"push packet, unchanged");
TEST_ASSERT_EQUAL_INT_MESSAGE(
Data_data_schedjobs, obs.last_data_which,
XceData_data_schedjobs, obs.last_data_which,
"a single-element notification must go through the EXISTING, "
"UNMODIFIED xcesched_push_joblist() path (Data.schedjobs), never the "
"new compact jobArray path (Req 8.1)");