lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
myxce/src/controller/testsuite/test_worker_join.c
Rongsong Shen 70360ccfe8
fix(controller): resource hooks only carry values the controller accepted (bug-1583) 
do_update_resvals() fired the AFTER_EVENT HOOK_DETAIL_RESOURCE_UPDATE with
the raw client xce_resval_list_t. update_resval_list() rejects unknown
resource names (-ENOENT) but only logged it, so both listeners -- the
scheduler push and the apiserver store (apiserver_resource_hook_fun ->
handler.go) -- received names the controller had rejected, and the
apiserver stored them.

- update_resval_list() takes an optional `applied` view (caller-sized,
  borrowed pointers) and fills it with just the accepted items.
- do_update_resvals() hands that view to the AFTER_EVENT hook, and skips
  it when nothing was applied. The request still owns every item.

New test_update_resval_list_reports_only_applied_items (RED before: both
items reported). Controller suite 1237/0. Live /xce-test-resources with
mem=5 totally-fake-resource-xyz=1: mem MATCH at controller, scheduler and
apiserver; the fake name is rejected at injection and appears 0 times in
scheduler.log / apiserver.log (verdict FAIL by design, for the rejected
name).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-27 03:19:17 +00:00

1713 lines
61 KiB
C
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/*
* test_worker_join.c — TDD tests for worker join refactor
*
* Task 1.1: Failing tests for apply_join_state (shared status/resource update)
* Task 1.2: Failing tests for init_worker (first-time join path)
* Task 1.3: Failing tests for rejoin_worker (reconnect update with admin-bit preservation)
*
* RED phase: apply_join_state / init_worker / rejoin_worker are stubs that
* return 0 without modifying state, so all assertions below fail until
* Tasks 2–3 provide real implementations.
*/
/* rtti.h must come first — it defines 'var' and 'typeinfo_t' used by kvmgr.h */
#include <rtti.h>
#include <errno.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <unity/unity.h>
#include "cluster.h"
#include "controller.capnp.h" /* worker-rejoin-resync task 6.1: OPCode_push/
* OPCode_jobStateQuery, ConfigData_data_*,
* Request_body_* constants */
#include "jobj.h" /* worker-rejoin-resync task 6.1: makeJob, for a real
* non-terminal job the arm-round wiring tests need */
#include "hook.h" /* bug-1851: xce_hook_init/xce_add_hook */
#include "kvmgr.h"
#include "netio.h" /* worker-rejoin-resync task 6.1: endpoint_get/
* endpoint_deq/ITPTR_TO_PACKET, for real wire capture */
#include "request.h" /* worker-rejoin-resync task 6.1: xce_decode,
* xce_free_request */
#include "resobj.h"
#include "resync.h" /* worker-rejoin-resync task 6.1: worker_t.resync,
* XCE_WORKER_RECONCILING */
#include "wobj.h"
#include "worker.h"
#include "xce/cluster_def.h"
#include "xce/types.h"
#include "xcealloc.h" /* worker-rejoin-resync task 6.1: make_job_alloc and
* friends, for a job assigned to a test worker's
* host */
#include "xstring.h"
#include "zarray.h"
/* ── Stubs for controller functions not under test ───────────────────── */
/*
* collect_confvars is called by on_worker_join → worker_push_config.
* Stubbed here so worker.c links without pulling in the full dataex.c.
*/
void collect_confvars(cluster_t* c, collect_check_fn_t fn, xlist_t* l)
{
(void)c;
(void)fn;
(void)l;
}
/* ── Forward declarations for functions under test ──────────────────── */
/*
* apply_join_state is non-static in worker.c when XCE_TESTING is defined.
*
* RED: the current stub body does nothing and returns 0, so assertions
* that check the modified state will fail.
*/
int apply_join_state(cluster_t* c, worker_t* w, endpoint_t* ep,
xce_hostconfig_t* hconfig);
/*
* worker-rejoin-resync task 6.1: worker_push_config and worker_send_resync_
* query are non-static in worker.c when XCE_TESTING is defined, mirroring
* apply_join_state's own convention above -- lets these tests drive the
* exact config-push-then-query wire sequence on_worker_join produces,
* without needing a live join flow or get_ep_worker()'s endpoint->worker
* resolution (a separate, pre-existing concern -- see
* test_on_worker_join_sends_config_push_even_when_worker_unresolvable
* below for why).
*/
int worker_push_config(cluster_t* c, endpoint_t* ep);
int worker_send_resync_query(cluster_t* c, worker_t* w, endpoint_t* ep);
/*
* init_worker and rejoin_worker are exported from worker.c.
*
* RED: stubs return 0 without registering or modifying any entry.
*/
int init_worker(cluster_t* c, endpoint_t* ep, const char* host,
hostattrs_t* attrs, xce_hostconfig_t* hconfig);
int rejoin_worker(cluster_t* c, worker_t* w, endpoint_t* ep,
hostattrs_t* attrs, xce_hostconfig_t* hconfig);
/* ── Test fixture ────────────────────────────────────────────────────── */
/* Resource names and value types matching add_hostattrs_resvals in worker.c */
#define N_RES 9
static const char* g_res_names[N_RES] = {
"type", "model", "cpuf", "maxcpus", "maxmem",
"maxswap", "maxtmp", "ndisks", "slots"
};
static const int g_res_valtypes[N_RES] = {
XCE_VAL_STRING, XCE_VAL_STRING,
XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC,
XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC,
XCE_VAL_NUMERIC
};
typedef struct {
cluster_t c;
var workermgr;
var resmgr;
var jobmgr; /* worker-rejoin-resync task 6.1: real jobmgr so the
* apply_join_state arm-round wiring tests below can
* seed a genuine non-terminal job assigned to a test
* worker's host (mirrors test_worker_resync.c's own
* g_twr.jobmgr fixture) */
} test_cluster_t;
static test_cluster_t g_tc;
/* worker-rejoin-resync task 6.1: jobj.c's job type table is a shared,
* process-wide resource across every test file in the controller_unit_test
* binary -- job_obj_init() is explicitly documented (jobj.c) as safe to
* call more than once per process without an intervening job_obj_fini()
* (Req 10.2, the exact "every controller unit-test file that constructs
* jobs calls job_obj_init() from its own setup" scenario this file is now
* one more instance of), so no cross-file coordination is needed -- guard
* only avoids redundant re-init within THIS file's own repeated setup/
* teardown cycles, mirroring test_worker_resync.c's g_twr_jobobj_ready. */
static int g_tc_jobobj_ready;
static void setup_test_cluster(void)
{
kvmgr_params_t str_params = {
.key_type = KVMGR_KEY_STRING,
.nslots = 64,
.delta = 16,
.datakiller = NULL,
};
kvmgr_params_t int_params = {
.key_type = KVMGR_KEY_INT,
.nslots = 32,
.delta = 16,
};
memset(&g_tc, 0, sizeof(g_tc));
if (!g_tc_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
g_tc_jobobj_ready = 1;
}
g_tc.workermgr = makeInstance(&xkvmgr_type, &str_params);
TEST_ASSERT_NOT_NULL(g_tc.workermgr);
g_tc.resmgr = makeInstance(&xkvmgr_type, &str_params);
TEST_ASSERT_NOT_NULL(g_tc.resmgr);
g_tc.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(g_tc.jobmgr);
for (int i = 0; i < N_RES; i++) {
xce_resource_t rdef;
memset(&rdef, 0, sizeof(rdef));
rdef.name = (char*)g_res_names[i];
rdef.valtype = g_res_valtypes[i];
rdef.flags = 0;
rdef.idx = -1;
xce_resource_t* r = makeResource(g_tc.resmgr, &rdef);
TEST_ASSERT_NOT_NULL_MESSAGE(r, g_res_names[i]);
}
g_tc.c.workermgr = g_tc.workermgr;
g_tc.c.resmgr = g_tc.resmgr;
g_tc.c.jobmgr = g_tc.jobmgr;
/* worker-rejoin-resync task 6.1: c->myhost is the `from` peer identity
* worker_push_config()/worker_send_resync_query() pass to xce_
* pushconfig_req()/xce_job_state_query_req() -- both reject a NULL
* `from` outright, so any test exercising the real wire send needs
* this populated (mirrors production cluster.c's own myhost_info
* storage-then-point-at-it pattern, xce/cluster_def.h:101-102). */
g_tc.c.myhost_info.name = (char*)"test-controller";
g_tc.c.myhost_info.key = (char*)"test-key";
g_tc.c.myhost = &g_tc.c.myhost_info;
zarray_init(&g_tc.c.sharedinst, 64, NULL);
}
static void teardown_test_cluster(void)
{
zarray_fini(&g_tc.c.sharedinst);
if (g_tc.resmgr) dereference(g_tc.resmgr);
if (g_tc.workermgr) dereference(g_tc.workermgr);
if (g_tc.jobmgr) dereference(g_tc.jobmgr);
memset(&g_tc, 0, sizeof(g_tc));
}
/* Seed a job directly in the fixture's jobmgr, bypassing dispatch/replay --
* mirrors test_worker_resync.c's twr_seed_job. */
static job_submit_data_t g_tc_seed_submit;
static job_t* seed_test_job(uint64_t jobid, uint32_t state)
{
xce_job_t jdef;
memset(&g_tc_seed_submit, 0, sizeof(g_tc_seed_submit));
g_tc_seed_submit.queue = (char*)"q";
g_tc_seed_submit.user = (char*)"tester";
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = jobid;
jdef.state = state;
jdef.submit = &g_tc_seed_submit;
return makeJob(g_tc.jobmgr, &jdef);
}
/* Build a single-host job allocation assigning `jobid` to `host` -- mirrors
* test_worker_resync.c's twr_make_test_alloc. Caller hands ownership to
* job->job.allocs. */
static xce_job_alloc_t* make_test_job_alloc(uint64_t jobid, const char* host)
{
return make_job_alloc(jobid, 1,
make_host_allocs(1,
make_host_alloc((char*)host, 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
}
/* Create a worker in the test workermgr; attrs strings must be makeString'd. */
static worker_t* make_test_worker(const char* name, hostattrs_t* attrs)
{
xce_worker_t host;
memset(&host, 0, sizeof(host));
host.name = (char*)name;
host.attrs = attrs;
return makeWorker(g_tc.workermgr, &host);
}
/* Retrieve a numeric resource value from a worker's resvals by resource name */
static float get_worker_res_fval(worker_t* w, const char* name)
{
xce_resource_t* r = NULL;
lock(g_tc.c.resmgr, RTTI_LOCK_READ);
key_find(g_tc.c.resmgr, (void*)name, (void**)&r);
unlock(g_tc.c.resmgr);
if (r == NULL || r->idx < 0 || r->idx >= w->host.resvals.caps)
return -1.0f;
xce_val_t* v = (xce_val_t*)w->host.resvals.data[r->idx];
return (v != NULL) ? v->u.fvalue : -1.0f;
}
/*
* Capture whatever update_resvals()/update_resval_list() print during a
* failing update_resval() call, as raw runtime output rather than a
* source-text scan. This test binary never calls mylog_init()/tlog_init()
* (only the real daemon main()s do -- see test_apply_correctness_
* verification.c's own header comment, and the identical established
* precedent in scheduler/testsuite/test_scheduler_connect.c's
* capture_stdout_during_sched_init()), so tlog's internal root log handle
* stays NULL and every ERROR()/WARN() call falls through tlog's early-
* print path, which writes the fully formatted line directly to
* STDOUT_FILENO via a raw write() -- genuine runtime log capture, not a
* guess about which stream PERROR/ERROR happens to use.
*/
static char* capture_stdout_during(void (*fn)(void* ctx), void* ctx)
{
char tmp[] = "/tmp/xce_worker_join_logcap_XXXXXX";
int fd = mkstemp(tmp);
TEST_ASSERT_TRUE_MESSAGE(fd >= 0,
"fixture: mkstemp for stdout capture must succeed");
fflush(stdout);
int saved_stdout = dup(STDOUT_FILENO);
TEST_ASSERT_TRUE_MESSAGE(saved_stdout >= 0,
"fixture: dup(STDOUT_FILENO) must succeed");
TEST_ASSERT_TRUE_MESSAGE(dup2(fd, STDOUT_FILENO) >= 0,
"fixture: dup2 redirect of STDOUT_FILENO must "
"succeed");
fn(ctx);
fflush(stdout);
dup2(saved_stdout, STDOUT_FILENO);
close(saved_stdout);
off_t len = lseek(fd, 0, SEEK_CUR);
TEST_ASSERT_TRUE_MESSAGE(len >= 0,
"fixture: lseek on capture file must succeed");
char* buf = (char*)calloc(1, (size_t)len + 1);
TEST_ASSERT_NOT_NULL(buf);
if (len > 0) {
TEST_ASSERT_TRUE_MESSAGE(lseek(fd, 0, SEEK_SET) == 0,
"fixture: rewind of capture file must "
"succeed");
ssize_t nread = read(fd, buf, (size_t)len);
TEST_ASSERT_TRUE_MESSAGE(nread == len,
"fixture: reading back captured buffer "
"must succeed");
}
close(fd);
unlink(tmp);
return buf;
}
typedef struct {
cluster_t* c;
worker_t* w;
xce_resval_t* val;
} update_resvals_call_ctx_t;
static void call_update_resvals_1(void* ctx_)
{
update_resvals_call_ctx_t* ctx = (update_resvals_call_ctx_t*)ctx_;
update_resvals(ctx->c, ctx->w, 1, ctx->val);
}
/*
* update_resval() returns its failure reason as a raw negative code
* directly (key_find/zarray_put convention -- never via errno), so
* update_resvals()/update_resval_list() logging it via PERROR (which reads
* the process-global errno) reported whatever errno happened to be left
* over from an unrelated earlier call: "Success" if nothing had touched
* it, or a stray EAGAIN/etc from some other syscall in the same event-loop
* iteration -- never the real reason a resource update failed. This test
* reproduces that exact symptom (ambient errno=EAGAIN, unrelated to the
* genuine failure) and proves the fix decodes the actual rc instead.
*/
static void test_update_resvals_logs_actual_rc_not_stale_errno(void)
{
setup_test_cluster();
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
worker_t* w = make_test_worker("host-resval-log", &attrs);
TEST_ASSERT_NOT_NULL(w);
xce_val_t fval;
memset(&fval, 0, sizeof(fval));
fval.which = Value_u_fvalue;
fval.u.fvalue = 1.0f;
xce_resval_t val;
val.name = "does-not-exist-in-resmgr";
val.val = &fval;
/* Simulate the exact reported symptom: ambient errno unrelated to the
* real failure, left over from some other unrelated call. */
errno = EAGAIN;
update_resvals_call_ctx_t ctx = { .c = &g_tc.c, .w = w, .val = &val };
char* captured = capture_stdout_during(call_update_resvals_1, &ctx);
TEST_ASSERT_NOT_NULL(captured);
TEST_ASSERT_NULL_MESSAGE(
strstr(captured, "Resource temporarily unavailable"),
"must not log the ambient errno's string (EAGAIN) -- that is not "
"why update_resval failed");
TEST_ASSERT_NULL_MESSAGE(
strstr(captured, "Success"),
"must not log \"Success\" for a call that returned rc<0");
TEST_ASSERT_NOT_NULL_MESSAGE(
strstr(captured, "does-not-exist-in-resmgr"),
"must name which resource failed");
TEST_ASSERT_NOT_NULL_MESSAGE(
strstr(captured, "No such file or directory"),
"must log the real reason (-ENOENT, from key_find on an "
"unregistered resource name)");
free(captured);
teardown_test_cluster();
printf("test_update_resvals_logs_actual_rc_not_stale_errno: PASS\n");
}
/* bug-1583: update_resval_list() must report which items it actually
* applied, so do_update_resvals() can hand ONLY those to the AFTER_EVENT
* resource hook (scheduler + apiserver). It used to pass the raw client
* list, so the apiserver stored resource names the controller rejected. */
static void test_update_resval_list_reports_only_applied_items(void)
{
setup_test_cluster();
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
worker_t* w = make_test_worker("host-resval-applied", &attrs);
TEST_ASSERT_NOT_NULL(w);
xce_val_t good_val, bad_val;
memset(&good_val, 0, sizeof(good_val));
good_val.which = Value_u_fvalue;
good_val.u.fvalue = 4.0f;
bad_val = good_val;
xce_resval_t good = { .name = "slots", .val = &good_val };
xce_resval_t bad = { .name = "totally-fake-resource-xyz",
.val = &bad_val };
xce_resval_t* items[2] = { &bad, &good };
xce_resval_list_t in = { .count = 2, .vals = items };
xce_resval_t* out_items[2] = { NULL, NULL };
xce_resval_list_t applied = { .count = -1, .vals = out_items };
TEST_ASSERT_EQUAL_INT(0, update_resval_list(&g_tc.c, w, &in, &applied));
TEST_ASSERT_EQUAL_INT_MESSAGE(1, applied.count,
"bug-1583: only the resource the controller accepted may be "
"reported as applied");
TEST_ASSERT_TRUE(applied.vals[0] == &good);
TEST_ASSERT_EQUAL_FLOAT(4.0f, get_worker_res_fval(w, "slots"));
teardown_test_cluster();
}
/* ── Task 1.1: Tests for apply_join_state ──────────────────────────── */
/*
* After apply_join_state, XCE_WORKER_UNAVAIL must be cleared. Req 4.1
*/
static void test_apply_join_state_clears_unavail(void)
{
setup_test_cluster();
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
worker_t* w = make_test_worker("host-ajs-1", &attrs);
TEST_ASSERT_NOT_NULL(w);
lock(w, RTTI_LOCK_WRITE);
w->host.status |= XCE_WORKER_UNAVAIL;
unlock(w);
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!"; /* no type override */
hconfig.hostmodel = (char*)"!"; /* no model override */
hconfig.mxj = 2;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
lock(w, RTTI_LOCK_WRITE);
int rc = apply_join_state(&g_tc.c, w, &ep, &hconfig);
unlock(w);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_FALSE_MESSAGE(w->host.status & XCE_WORKER_UNAVAIL,
"UNAVAIL must be cleared after apply_join_state");
teardown_test_cluster();
printf("test_apply_join_state_clears_unavail: PASS\n");
}
/*
* When hconfig provides non-"!" overrides, hosttype and hostmodel must
* match those values after apply_join_state. Req 4.2
*/
static void test_apply_join_state_applies_hosttype_override(void)
{
setup_test_cluster();
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("OLD_TYPE");
attrs.hostmodel = makeString("OLD_MODEL");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 2;
attrs.maxmem = 8;
attrs.maxswap = 16;
attrs.maxtmp = 32;
attrs.ndisks = 1;
worker_t* w = make_test_worker("host-ajs-2", &attrs);
TEST_ASSERT_NOT_NULL(w);
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"LINUX64";
hconfig.hostmodel = (char*)"x86_64";
hconfig.mxj = 4;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
lock(w, RTTI_LOCK_WRITE);
int rc = apply_join_state(&g_tc.c, w, &ep, &hconfig);
unlock(w);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_STRING_MESSAGE("LINUX64", w->host.attrs->hosttype,
"hosttype must be overridden from hconfig");
TEST_ASSERT_EQUAL_STRING_MESSAGE("x86_64", w->host.attrs->hostmodel,
"hostmodel must be overridden from hconfig");
teardown_test_cluster();
printf("test_apply_join_state_applies_hosttype_override: PASS\n");
}
/*
* After apply_join_state, 'slots' equals hconfig.mxj and numeric host-attr
* resources match the worker's attrs. Req 4.2, 4.3
*/
static void test_apply_join_state_updates_resource_values(void)
{
setup_test_cluster();
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 3.0f;
attrs.maxcpus = 8;
attrs.maxmem = 64;
attrs.maxswap = 128;
attrs.maxtmp = 256;
attrs.ndisks = 4;
worker_t* w = make_test_worker("host-ajs-3", &attrs);
TEST_ASSERT_NOT_NULL(w);
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 6;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
lock(w, RTTI_LOCK_WRITE);
int rc = apply_join_state(&g_tc.c, w, &ep, &hconfig);
unlock(w);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_FLOAT_MESSAGE(6.0f, get_worker_res_fval(w, "slots"),
"slots must equal hconfig.mxj");
TEST_ASSERT_EQUAL_FLOAT(3.0f, get_worker_res_fval(w, "cpuf"));
TEST_ASSERT_EQUAL_FLOAT(8.0f, get_worker_res_fval(w, "maxcpus"));
TEST_ASSERT_EQUAL_FLOAT(64.0f, get_worker_res_fval(w, "maxmem"));
TEST_ASSERT_EQUAL_FLOAT(128.0f, get_worker_res_fval(w, "maxswap"));
TEST_ASSERT_EQUAL_FLOAT(256.0f, get_worker_res_fval(w, "maxtmp"));
TEST_ASSERT_EQUAL_FLOAT(4.0f, get_worker_res_fval(w, "ndisks"));
teardown_test_cluster();
printf("test_apply_join_state_updates_resource_values: PASS\n");
}
/*
* hconfig.mxj == -1 is the schema's documented "auto-detect" sentinel
* (hosts.schema.json: "max_slot": {"minimum": -1, "description": "-1 means
* auto-detect"}) -- apply_join_state must fall back to the worker's own
* reported maxcpus, NOT persist -1 (or, before the xceadmin mapping fix,
* silently ship 0 and permanently disable scheduling on that host: mxj is
* UInt32 on the wire, so an unset/absent max_slot used to decode as 0,
* which is not < 0 and so never reached this fallback -- see
* xcecommon.mapping.HOSTCONFIG_MXJ_AUTO_DETECT). This is the ONLY test in
* this file that exercises the nslots<0 branch; every sibling test above
* uses a positive hconfig.mxj and so cannot catch a regression here.
*/
static void test_apply_join_state_mxj_negative_one_falls_back_to_maxcpus(void)
{
setup_test_cluster();
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 12;
attrs.maxmem = 64;
attrs.maxswap = 128;
attrs.maxtmp = 256;
attrs.ndisks = 2;
worker_t* w = make_test_worker("host-ajs-autodetect", &attrs);
TEST_ASSERT_NOT_NULL(w);
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = -1;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
lock(w, RTTI_LOCK_WRITE);
int rc = apply_join_state(&g_tc.c, w, &ep, &hconfig);
unlock(w);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_FLOAT_MESSAGE(
12.0f, get_worker_res_fval(w, "slots"),
"mxj=-1 must auto-detect slots from the worker's reported maxcpus, "
"not persist -1 or silently leave slots at 0");
teardown_test_cluster();
printf("test_apply_join_state_mxj_negative_one_falls_back_to_maxcpus: "
"PASS\n");
}
/* ── Task 1.2: Tests for init_worker ────────────────────────────────── */
/*
* After init_worker for a new hostname, key_find must find the entry and
* return a non-negative index. Req 2.1, 2.3
*/
static void test_init_worker_creates_new_entry(void)
{
setup_test_cluster();
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
int idx = init_worker(&g_tc.c, &ep, "new-host-1", &attrs, &hconfig);
TEST_ASSERT_GREATER_OR_EQUAL_MESSAGE(0, idx,
"init_worker must return non-negative index");
worker_t* w = NULL;
lock(g_tc.c.workermgr, RTTI_LOCK_READ);
int rc = key_find(g_tc.c.workermgr, (void*)"new-host-1", (void**)&w);
unlock(g_tc.c.workermgr);
TEST_ASSERT_GREATER_THAN_MESSAGE(-1, rc,
"worker must be registered in workermgr after init");
TEST_ASSERT_NOT_NULL_MESSAGE(w, "key_find must return a valid worker pointer");
teardown_test_cluster();
printf("test_init_worker_creates_new_entry: PASS\n");
}
/*
* The index returned by init_worker must match w->idx. Req 2.3
*/
static void test_init_worker_returns_correct_idx(void)
{
setup_test_cluster();
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 2;
attrs.maxmem = 8;
attrs.maxswap = 16;
attrs.maxtmp = 32;
attrs.ndisks = 1;
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 2;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
int idx = init_worker(&g_tc.c, &ep, "new-host-2", &attrs, &hconfig);
TEST_ASSERT_GREATER_OR_EQUAL(0, idx);
worker_t* w = NULL;
lock(g_tc.c.workermgr, RTTI_LOCK_READ);
key_find(g_tc.c.workermgr, (void*)"new-host-2", (void**)&w);
unlock(g_tc.c.workermgr);
TEST_ASSERT_NOT_NULL(w);
TEST_ASSERT_EQUAL_INT_MESSAGE(idx, w->idx,
"returned idx must match the worker's idx field");
teardown_test_cluster();
printf("test_init_worker_returns_correct_idx: PASS\n");
}
/* ── Task 1.3: Tests for rejoin_worker ──────────────────────────────── */
/*
* rejoin_worker must update hostattrs and ep on the existing entry. Req 3.1, 3.2
*/
static void test_rejoin_worker_updates_attrs_and_ep(void)
{
setup_test_cluster();
hostattrs_t old_attrs;
memset(&old_attrs, 0, sizeof(old_attrs));
old_attrs.hosttype = makeString("OLD_TYPE");
old_attrs.hostmodel = makeString("OLD_MODEL");
old_attrs.cpufactor = 1.0f;
old_attrs.maxcpus = 2;
old_attrs.maxmem = 8;
old_attrs.maxswap = 16;
old_attrs.maxtmp = 32;
old_attrs.ndisks = 1;
worker_t* w = make_test_worker("rejoin-host-1", &old_attrs);
TEST_ASSERT_NOT_NULL(w);
hostattrs_t new_attrs;
memset(&new_attrs, 0, sizeof(new_attrs));
new_attrs.hosttype = makeString("NEW_TYPE");
new_attrs.hostmodel = makeString("NEW_MODEL");
new_attrs.cpufactor = 2.0f;
new_attrs.maxcpus = 8;
new_attrs.maxmem = 64;
new_attrs.maxswap = 128;
new_attrs.maxtmp = 256;
new_attrs.ndisks = 2;
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
int idx = rejoin_worker(&g_tc.c, w, &ep, &new_attrs, &hconfig);
TEST_ASSERT_GREATER_OR_EQUAL_MESSAGE(0, idx,
"rejoin_worker must return non-negative index");
TEST_ASSERT_EQUAL_STRING_MESSAGE("NEW_TYPE", w->host.attrs->hosttype,
"hosttype must be updated to new attrs value");
TEST_ASSERT_EQUAL_STRING_MESSAGE("NEW_MODEL", w->host.attrs->hostmodel,
"hostmodel must be updated to new attrs value");
TEST_ASSERT_EQUAL_FLOAT_MESSAGE(2.0f, w->host.attrs->cpufactor,
"cpufactor must be updated to new value");
TEST_ASSERT_EQUAL_PTR_MESSAGE(&ep, w->ep,
"ep must be updated to the new endpoint");
/* Release the test's own references to the new strings */
dereference(new_attrs.hosttype);
dereference(new_attrs.hostmodel);
teardown_test_cluster();
printf("test_rejoin_worker_updates_attrs_and_ep: PASS\n");
}
/*
* Admin-set status bits must survive a rejoin_worker call. Req 3.3
*/
static void test_rejoin_worker_preserves_admin_bits(void)
{
setup_test_cluster();
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
worker_t* w = make_test_worker("rejoin-host-2", &attrs);
TEST_ASSERT_NOT_NULL(w);
/* Set all admin bits before the rejoin */
lock(w, RTTI_LOCK_WRITE);
w->host.status |= (XCE_WORKER_LOCKEDU | XCE_WORKER_CLOSED
| XCE_WORKER_CLOSED_BY_ADMIN);
unlock(w);
hostattrs_t new_attrs;
memset(&new_attrs, 0, sizeof(new_attrs));
new_attrs.hosttype = makeString("LINUX");
new_attrs.hostmodel = makeString("x86");
new_attrs.cpufactor = 2.0f;
new_attrs.maxcpus = 8;
new_attrs.maxmem = 32;
new_attrs.maxswap = 64;
new_attrs.maxtmp = 128;
new_attrs.ndisks = 2;
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
rejoin_worker(&g_tc.c, w, &ep, &new_attrs, &hconfig);
TEST_ASSERT_TRUE_MESSAGE(w->host.status & XCE_WORKER_LOCKEDU,
"LOCKEDU must be preserved after rejoin");
TEST_ASSERT_TRUE_MESSAGE(w->host.status & XCE_WORKER_CLOSED,
"CLOSED must be preserved after rejoin");
TEST_ASSERT_TRUE_MESSAGE(w->host.status & XCE_WORKER_CLOSED_BY_ADMIN,
"CLOSED_BY_ADMIN must be preserved after rejoin");
dereference(new_attrs.hosttype);
dereference(new_attrs.hostmodel);
teardown_test_cluster();
printf("test_rejoin_worker_preserves_admin_bits: PASS\n");
}
/* ── Task 6.1: apply_join_state arms a reconciliation round ──────────── */
/*
* Real (heap) endpoint_t via endpoint_get() -- NOT a hand-rolled/zeroed
* stack stand-in (bug-378: an under-sized/wrong-typed stand-in for
* endpoint_t caused a global-buffer-overflow when passed where a real
* endpoint_t* was expected). Needed only by the tests below that actually
* send something over the wire (worker_push_config/worker_send_resync_
* query); the pre-existing apply_join_state/init_worker/rejoin_worker
* tests above never send anything and so are unaffected, still using a
* plain zeroed stack endpoint_t. Mirrors src/lib/testsuite/
* test_job_state_wire.c's own established pattern for this exact
* situation.
*/
static endpoint_t* new_test_endpoint(void)
{
endpoint_t* ep = endpoint_get();
TEST_ASSERT_NOT_NULL(ep);
return ep;
}
/* Pop one queued page off ep->outq and decode it. Fails the test if the
* queue is empty. Caller must xce_free_request() the result. */
static void pop_and_decode(endpoint_t* ep, xce_request_t* out)
{
data_item_t* it = endpoint_deq(ep);
TEST_ASSERT_NOT_NULL(it);
io_packet_t* pkt = ITPTR_TO_PACKET(it);
memset(out, 0, sizeof(*out));
TEST_ASSERT_EQUAL_INT(0, xce_decode(pkt, out)); /* consumes/frees pkt */
}
static void assert_queue_empty(endpoint_t* ep)
{
TEST_ASSERT_NULL(endpoint_deq(ep));
}
/*
* rejoin_worker (via apply_join_state) must arm a reconciliation round --
* w->resync populated, XCE_WORKER_RECONCILING set, the query set matching
* exactly the host's non-terminal jobs -- when the controller currently
* holds one or more non-terminal jobs for that host. Req 1.1, 2.1, 2.2.
*/
static void test_rejoin_worker_arms_reconciliation_round_when_jobs_exist(void)
{
setup_test_cluster();
job_t* j = seed_test_job(96001, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs = make_test_job_alloc(96001, "arm-host-1");
TEST_ASSERT_NOT_NULL(j->job.allocs);
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
worker_t* w = make_test_worker("arm-host-1", &attrs);
TEST_ASSERT_NOT_NULL(w);
hostattrs_t new_attrs;
memset(&new_attrs, 0, sizeof(new_attrs));
new_attrs.hosttype = makeString("LINUX");
new_attrs.hostmodel = makeString("x86");
new_attrs.cpufactor = 1.0f;
new_attrs.maxcpus = 4;
new_attrs.maxmem = 16;
new_attrs.maxswap = 32;
new_attrs.maxtmp = 64;
new_attrs.ndisks = 1;
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
int idx = rejoin_worker(&g_tc.c, w, &ep, &new_attrs, &hconfig);
TEST_ASSERT_GREATER_OR_EQUAL(0, idx);
TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
"a worker rejoining with a known non-terminal job must have a "
"reconciliation round armed (Req 1.1)");
TEST_ASSERT_TRUE_MESSAGE(w->host.status & XCE_WORKER_RECONCILING,
"XCE_WORKER_RECONCILING must be set once a round is armed");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, w->resync->n_queried_jobids,
"the round's query set must match exactly the host's non-terminal "
"jobs (Req 2.1)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(96001, w->resync->queried_jobids[0],
"the queried job must be identified by its unique jobid (Req 2.2)");
dereference(new_attrs.hosttype);
dereference(new_attrs.hostmodel);
teardown_test_cluster();
printf("test_rejoin_worker_arms_reconciliation_round_when_jobs_exist: "
"PASS\n");
}
/*
* A worker rejoining with NO non-terminal jobs currently assigned to it
* must NOT have a round armed. Req 1.2.
*/
static void test_rejoin_worker_no_round_when_no_jobs(void)
{
setup_test_cluster();
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
worker_t* w = make_test_worker("arm-host-2", &attrs);
TEST_ASSERT_NOT_NULL(w);
hostattrs_t new_attrs = attrs;
new_attrs.hosttype = makeString("LINUX");
new_attrs.hostmodel = makeString("x86");
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
rejoin_worker(&g_tc.c, w, &ep, &new_attrs, &hconfig);
TEST_ASSERT_NULL_MESSAGE(w->resync,
"a worker rejoining with no non-terminal jobs must not have a "
"reconciliation round armed (Req 1.2)");
TEST_ASSERT_FALSE_MESSAGE(w->host.status & XCE_WORKER_RECONCILING,
"XCE_WORKER_RECONCILING must not be set");
dereference(new_attrs.hosttype);
dereference(new_attrs.hostmodel);
teardown_test_cluster();
printf("test_rejoin_worker_no_round_when_no_jobs: PASS\n");
}
/*
* A terminal job (DONE) assigned to the host must not be queried -- only
* non-terminal jobs count toward the trigger (Req 1.1's own "non-terminal"
* qualifier).
*/
static void test_rejoin_worker_no_round_when_only_terminal_jobs(void)
{
setup_test_cluster();
job_t* j = seed_test_job(96002, JOB_STAT_DONE);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs = make_test_job_alloc(96002, "arm-host-3");
TEST_ASSERT_NOT_NULL(j->job.allocs);
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
worker_t* w = make_test_worker("arm-host-3", &attrs);
TEST_ASSERT_NOT_NULL(w);
hostattrs_t new_attrs = attrs;
new_attrs.hosttype = makeString("LINUX");
new_attrs.hostmodel = makeString("x86");
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
rejoin_worker(&g_tc.c, w, &ep, &new_attrs, &hconfig);
TEST_ASSERT_NULL_MESSAGE(w->resync,
"a terminal (DONE) job must not count toward the reconciliation "
"trigger");
dereference(new_attrs.hosttype);
dereference(new_attrs.hostmodel);
teardown_test_cluster();
printf("test_rejoin_worker_no_round_when_only_terminal_jobs: PASS\n");
}
/*
* bug-1851: a string resval ("type"/"model") must own its bytes rather
* than alias the interned xstring in w->host.attrs. rejoin_worker drops
* the old attrs strings, then apply_join_state -> resync_arm_round
* resolves the PREVIOUS join's still-outstanding round, which fires the
* worker hooks (resync_resolve_round -> persist_notify_worker_status_
* updated) BEFORE add_hostattrs_resvals re-seeds resvals. Live, the
* apiserver hook's collect_resval strdup()ed the freed hostmodel and the
* newly-promoted primary died (ASAN heap-use-after-free). This hook stands
* in for it: it reads both string resvals at exactly that point.
*/
static char g_1851_seen_type[64];
static char g_1851_seen_model[64];
static int g_1851_hook_calls;
static void copy_string_resval(worker_t* w, const char* name, char* out,
size_t out_sz)
{
xce_resource_t* r = NULL;
xce_val_t* v;
lock(g_tc.c.resmgr, RTTI_LOCK_READ);
key_find(g_tc.c.resmgr, (void*)name, (void**)&r);
unlock(g_tc.c.resmgr);
if (r == NULL || r->idx < 0 || r->idx >= w->host.resvals.caps)
return;
v = (xce_val_t*)w->host.resvals.data[r->idx];
if (v != NULL && v->which == Value_u_svalue && v->u.svalue != NULL)
snprintf(out, out_sz, "%s", v->u.svalue);
}
static int bug1851_worker_hook(int type, int situation, int detail,
void* cls, va_list ap)
{
(void)type;
(void)situation;
(void)detail;
(void)cls;
(void)va_arg(ap, cluster_t*);
worker_t* w = va_arg(ap, worker_t*);
g_1851_hook_calls++;
copy_string_resval(w, "type", g_1851_seen_type, sizeof(g_1851_seen_type));
copy_string_resval(w, "model", g_1851_seen_model,
sizeof(g_1851_seen_model));
return 0;
}
static void test_rejoin_worker_string_resvals_survive_outstanding_round(void)
{
setup_test_cluster();
TEST_ASSERT_EQUAL_INT(
0, xce_hook_init(&g_tc.c.hooksys, XCE_CONTROLLER_HOOK_MAX));
TEST_ASSERT_EQUAL_INT(0, xce_add_hook(&g_tc.c.hooksys,
XCE_CONTROLLER_HOOK_WORKER,
bug1851_worker_hook, NULL));
g_1851_hook_calls = 0;
g_1851_seen_type[0] = '\0';
g_1851_seen_model[0] = '\0';
/* A non-terminal job on the host, so each rejoin arms a round and the
* second rejoin finds the first one still outstanding. */
job_t* j = seed_test_job(96851, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs = make_test_job_alloc(96851, "uaf-host-1851");
TEST_ASSERT_NOT_NULL(j->job.allocs);
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("TYPE-1851-A");
attrs.hostmodel = makeString("MODEL-1851-A");
attrs.maxcpus = 4;
worker_t* w = make_test_worker("uaf-host-1851", &attrs);
TEST_ASSERT_NOT_NULL(w);
/* makeWorker took its own references; drop the test's so w->host.attrs
* holds the ONLY reference to the "A" strings. */
dereference(attrs.hosttype);
dereference(attrs.hostmodel);
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
/* First join: seeds resvals from the "A" strings, arms round #1. */
TEST_ASSERT_GREATER_OR_EQUAL(0, apply_join_state(&g_tc.c, w, &ep, &hconfig));
TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
"fixture: the first join must leave a round outstanding");
/* Second join with different strings: must resolve round #1 (firing
* the hook) while "A" is being released. */
hostattrs_t new_attrs;
memset(&new_attrs, 0, sizeof(new_attrs));
new_attrs.hosttype = makeString("TYPE-1851-B");
new_attrs.hostmodel = makeString("MODEL-1851-B");
new_attrs.maxcpus = 4;
g_1851_hook_calls = 0;
TEST_ASSERT_GREATER_OR_EQUAL(
0, rejoin_worker(&g_tc.c, w, &ep, &new_attrs, &hconfig));
TEST_ASSERT_GREATER_THAN_INT_MESSAGE(0, g_1851_hook_calls,
"fixture: resolving the outstanding round must fire the worker hook");
TEST_ASSERT_EQUAL_STRING_MESSAGE("TYPE-1851-A", g_1851_seen_type,
"the hook must read the previous (still-valid) type resval, not "
"freed memory");
TEST_ASSERT_EQUAL_STRING_MESSAGE("MODEL-1851-A", g_1851_seen_model,
"the hook must read the previous (still-valid) model resval, not "
"freed memory");
/* Ownership: the resval is a private copy, never the attrs xstring. */
xce_val_t val;
memset(&val, 0, sizeof(val));
TEST_ASSERT_EQUAL_INT(1, get_resval(&g_tc.c, (char*)"model",
(char*)"uaf-host-1851", &val));
TEST_ASSERT_EQUAL_STRING("MODEL-1851-B", val.u.svalue);
TEST_ASSERT_TRUE_MESSAGE(val.u.svalue != w->host.attrs->hostmodel,
"a string resval must own its bytes, not alias w->host.attrs");
dereference(new_attrs.hosttype);
dereference(new_attrs.hostmodel);
xce_hook_fini(&g_tc.c.hooksys);
teardown_test_cluster();
printf("test_rejoin_worker_string_resvals_survive_outstanding_round: "
"PASS\n");
}
/*
* Req 1.3: the SAME trigger condition -- no special-casing needed -- must
* apply to a host's first-ever join (init_worker), not just a rejoin.
*/
static void test_init_worker_arms_reconciliation_round_when_jobs_exist(void)
{
setup_test_cluster();
job_t* j = seed_test_job(96003, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs = make_test_job_alloc(96003, "arm-host-4-first-join");
TEST_ASSERT_NOT_NULL(j->job.allocs);
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
int idx = init_worker(&g_tc.c, &ep, "arm-host-4-first-join", &attrs,
&hconfig);
TEST_ASSERT_GREATER_OR_EQUAL(0, idx);
worker_t* w = NULL;
lock(g_tc.c.workermgr, RTTI_LOCK_READ);
key_find(g_tc.c.workermgr, (void*)"arm-host-4-first-join", (void**)&w);
unlock(g_tc.c.workermgr);
TEST_ASSERT_NOT_NULL(w);
TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
"a host's first-ever join must arm a reconciliation round exactly "
"like a rejoin, with no special-casing needed (Req 1.3)");
TEST_ASSERT_EQUAL_INT(1, w->resync->n_queried_jobids);
TEST_ASSERT_EQUAL_UINT64(96003, w->resync->queried_jobids[0]);
teardown_test_cluster();
printf("test_init_worker_arms_reconciliation_round_when_jobs_exist: "
"PASS\n");
}
/* ── Task 6.1: worker_send_resync_query ───────────────────────────────── */
/*
* When a round is armed for `w`, worker_send_resync_query() must send
* exactly that round's job IDs as an OPCode_jobStateQuery. Req 2.1, 2.2.
*/
static void test_worker_send_resync_query_sends_armed_round(void)
{
setup_test_cluster();
job_t* j1 = seed_test_job(96010, JOB_STAT_RUN);
j1->job.allocs = make_test_job_alloc(96010, "query-host-1");
job_t* j2 = seed_test_job(96011, JOB_STAT_RUN);
j2->job.allocs = make_test_job_alloc(96011, "query-host-1");
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
worker_t* w = make_test_worker("query-host-1", &attrs);
TEST_ASSERT_NOT_NULL(w);
hostattrs_t new_attrs = attrs;
new_attrs.hosttype = makeString("LINUX");
new_attrs.hostmodel = makeString("x86");
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
endpoint_t stack_ep;
memset(&stack_ep, 0, sizeof(stack_ep));
rejoin_worker(&g_tc.c, w, &stack_ep, &new_attrs, &hconfig);
TEST_ASSERT_NOT_NULL(w->resync);
TEST_ASSERT_EQUAL_INT(2, w->resync->n_queried_jobids);
endpoint_t* ep = new_test_endpoint();
TEST_ASSERT_EQUAL_INT(0, worker_send_resync_query(&g_tc.c, w, ep));
xce_request_t decoded;
pop_and_decode(ep, &decoded);
TEST_ASSERT_EQUAL_INT(OPCode_jobStateQuery, decoded.opcode);
TEST_ASSERT_EQUAL_INT(Request_body_jobstatequerymsg, decoded.body_which);
TEST_ASSERT_NOT_NULL(decoded.body.jobstatequerymsg);
TEST_ASSERT_EQUAL_INT(2, decoded.body.jobstatequerymsg->n_total);
int saw_96010 = 0, saw_96011 = 0;
for (int i = 0; i < decoded.body.jobstatequerymsg->count; i++) {
uint64_t id = decoded.body.jobstatequerymsg->jobids[i];
if (id == 96010) saw_96010 = 1;
if (id == 96011) saw_96011 = 1;
}
TEST_ASSERT_TRUE_MESSAGE(saw_96010, "queried set must include 96010");
TEST_ASSERT_TRUE_MESSAGE(saw_96011, "queried set must include 96011");
xce_free_request(&decoded);
assert_queue_empty(ep);
dereference(new_attrs.hosttype);
dereference(new_attrs.hostmodel);
endpoint_fini(ep);
free(ep);
teardown_test_cluster();
printf("test_worker_send_resync_query_sends_armed_round: PASS\n");
}
/*
* When no round is armed (w->resync == NULL), worker_send_resync_query()
* must send nothing at all -- the Req 1.2 no-round case's downstream
* completion criterion ("no round, no query").
*/
static void test_worker_send_resync_query_sends_nothing_when_no_round(void)
{
setup_test_cluster();
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
worker_t* w = make_test_worker("query-host-2", &attrs);
TEST_ASSERT_NOT_NULL(w);
TEST_ASSERT_NULL(w->resync);
endpoint_t* ep = new_test_endpoint();
TEST_ASSERT_EQUAL_INT(0, worker_send_resync_query(&g_tc.c, w, ep));
assert_queue_empty(ep);
endpoint_fini(ep);
free(ep);
teardown_test_cluster();
printf("test_worker_send_resync_query_sends_nothing_when_no_round: "
"PASS\n");
}
/* ── Task 7.5: worker_send_resync_query genuinely paginates (bug-1755) ──
*
* Prior to task 7.5, worker_send_resync_query() hardcoded pktlimit=0 in its
* xce_job_state_query_req() call -- xce_job_state_query_req()'s own
* pktlimit<=0 convention (request.c) means "pktlimit = nrows", i.e. always
* exactly ONE wire page, however many jobs are queried. Task 7.4's own test
* (test_worker_resync.c, test_large_job_count_pagination_correctness) only
* proved the underlying xce_job_state_query_req() pagination MECHANISM by
* calling it directly with an explicit page-size override, bypassing this
* production call site entirely (documented live as bug-1755: "effectively
* dead code in production today because its only call site never supplies
* a bound").
*
* This test drives the REAL production call site -- worker_send_resync_
* query() itself, not a direct xce_job_state_query_req() call -- with a
* non-terminal job count on the host that genuinely exceeds the new
* XCE_RESYNC_QUERY_PKTLIMIT page cap (worker.c), and asserts the query is
* observably split across multiple wire pages: more than one packet is
* enqueued, each page's `pos` picks up exactly where the previous page's
* `count` left off, every jobid appears in exactly one page, and every
* page (including the final, deliberately partial one) carries the same
* full n_total. Req 8.1-8.3.
*/
#define TWJ_RESYNC_QUERY_PKTLIMIT 100 /* mirrors production
* XCE_RESYNC_QUERY_PKTLIMIT
* (worker.c) -- not a real
* production symbol reference (that
* #define has file-local linkage),
* kept in sync by this test's own
* page-count assertion below, which
* would fail loudly if the two ever
* drifted apart */
#define TWJ_RESYNC_QUERY_N_JOBS 250 /* > TWJ_RESYNC_QUERY_PKTLIMIT, and not
* a multiple of it, so the final wire
* page is genuinely partial (50
* entries), not just N full pages */
#define TWJ_RESYNC_QUERY_JOBID_BASE 970000ULL
#define TWJ_RESYNC_QUERY_N_PAGES \
((TWJ_RESYNC_QUERY_N_JOBS + TWJ_RESYNC_QUERY_PKTLIMIT - 1) \
/ TWJ_RESYNC_QUERY_PKTLIMIT)
static void
test_worker_send_resync_query_paginates_across_multiple_wire_pages(void)
{
setup_test_cluster();
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
worker_t* w = make_test_worker("query-host-paginate", &attrs);
TEST_ASSERT_NOT_NULL(w);
for (int i = 0; i < TWJ_RESYNC_QUERY_N_JOBS; i++) {
uint64_t jobid = TWJ_RESYNC_QUERY_JOBID_BASE + (uint64_t)i;
job_t* j = seed_test_job(jobid, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs = make_test_job_alloc(jobid, "query-host-paginate");
TEST_ASSERT_NOT_NULL(j->job.allocs);
}
hostattrs_t new_attrs = attrs;
new_attrs.hosttype = makeString("LINUX");
new_attrs.hostmodel = makeString("x86");
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
endpoint_t stack_ep;
memset(&stack_ep, 0, sizeof(stack_ep));
rejoin_worker(&g_tc.c, w, &stack_ep, &new_attrs, &hconfig);
TEST_ASSERT_NOT_NULL(w->resync);
TEST_ASSERT_EQUAL_INT(TWJ_RESYNC_QUERY_N_JOBS, w->resync->n_queried_jobids);
endpoint_t* ep = new_test_endpoint();
/* The REAL production call site -- no xce_job_state_query_req()
* bypass. */
TEST_ASSERT_EQUAL_INT(0, worker_send_resync_query(&g_tc.c, w, ep));
{
bool seen[TWJ_RESYNC_QUERY_N_JOBS];
int running_total = 0;
int pages = 0;
memset(seen, 0, sizeof(seen));
while (running_total < TWJ_RESYNC_QUERY_N_JOBS) {
xce_request_t decoded;
int j;
pop_and_decode(ep, &decoded);
TEST_ASSERT_EQUAL_INT(OPCode_jobStateQuery, decoded.opcode);
TEST_ASSERT_EQUAL_INT(Request_body_jobstatequerymsg,
decoded.body_which);
TEST_ASSERT_NOT_NULL(decoded.body.jobstatequerymsg);
TEST_ASSERT_EQUAL_INT_MESSAGE(TWJ_RESYNC_QUERY_N_JOBS,
decoded.body.jobstatequerymsg->n_total,
"every page must carry the same full n_total, not a "
"per-page total (Req 2.1)");
TEST_ASSERT_EQUAL_INT_MESSAGE(running_total,
decoded.body.jobstatequerymsg->pos,
"each page's pos must pick up exactly where the previous "
"page's count left off -- no gap, no overlap (Req 8.3)");
TEST_ASSERT_TRUE_MESSAGE(
decoded.body.jobstatequerymsg->count
<= TWJ_RESYNC_QUERY_PKTLIMIT,
"no single page may exceed the configured page cap "
"(Req 8.3)");
for (j = 0; j < decoded.body.jobstatequerymsg->count; j++) {
uint64_t jobid = decoded.body.jobstatequerymsg->jobids[j];
uint64_t idx = jobid - TWJ_RESYNC_QUERY_JOBID_BASE;
TEST_ASSERT_TRUE_MESSAGE(
idx < (uint64_t)TWJ_RESYNC_QUERY_N_JOBS,
"a jobid appearing on the wire must be one of the "
"armed round's own jobids");
TEST_ASSERT_FALSE_MESSAGE(seen[idx],
"no jobid may be sent twice across pages (Req 8.3)");
seen[idx] = true;
}
running_total += decoded.body.jobstatequerymsg->count;
pages++;
xce_free_request(&decoded);
}
TEST_ASSERT_EQUAL_INT_MESSAGE(TWJ_RESYNC_QUERY_N_PAGES, pages,
"Req 8.1-8.3: a job count exceeding the real production page "
"cap must genuinely span multiple wire pages -- observably, "
"via the REAL worker_send_resync_query() call site, not just "
"theoretically supported by xce_job_state_query_req()'s own "
"pagination loop (bug-1755, closed by task 7.5)");
TEST_ASSERT_GREATER_THAN_INT_MESSAGE(1, pages,
"a single collapsed page here would mean the hardcoded-0 "
"regression (bug-1755) is back");
for (int i = 0; i < TWJ_RESYNC_QUERY_N_JOBS; i++) {
TEST_ASSERT_TRUE_MESSAGE(seen[i],
"every armed jobid must appear exactly once across all "
"pages -- none silently dropped past a page boundary "
"(Req 8.3)");
}
}
assert_queue_empty(ep);
dereference(new_attrs.hosttype);
dereference(new_attrs.hostmodel);
endpoint_fini(ep);
free(ep);
teardown_test_cluster();
printf("test_worker_send_resync_query_paginates_across_multiple_wire_"
"pages: PASS\n");
}
/* ── Task 6.1: config push, then query, in that fixed order ──────────── */
/*
* design.md's File Structure Plan for on_worker_join is explicit that this
* ordering is load-bearing ("config push, then query", "not as an
* extension of the join call itself"), and this spec's own Implementation
* Notes (task 4.3) warn that a placement-sensitive guarantee needs a
* STRUCTURAL assertion, not just an end-state one -- a behavioral test
* alone can pass even with the wrong order if it only checks that both
* packets eventually arrive. This test drives the exact two calls on_
* worker_join makes, in the exact order it makes them, and asserts the
* WIRE ORDER of the resulting packets directly.
*/
static void test_config_push_then_resync_query_wire_order(void)
{
setup_test_cluster();
cluster_config_t cc;
memset(&cc, 0, sizeof(cc));
cc.clustername = (char*)"test-cluster";
myconf_t myconf;
memset(&myconf, 0, sizeof(myconf));
myconf.cluster_config = &cc;
g_tc.c.myconf = &myconf;
job_t* j = seed_test_job(96020, JOB_STAT_RUN);
j->job.allocs = make_test_job_alloc(96020, "order-host-1");
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
worker_t* w = make_test_worker("order-host-1", &attrs);
TEST_ASSERT_NOT_NULL(w);
hostattrs_t new_attrs = attrs;
new_attrs.hosttype = makeString("LINUX");
new_attrs.hostmodel = makeString("x86");
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
endpoint_t stack_ep;
memset(&stack_ep, 0, sizeof(stack_ep));
rejoin_worker(&g_tc.c, w, &stack_ep, &new_attrs, &hconfig);
TEST_ASSERT_NOT_NULL(w->resync);
endpoint_t* ep = new_test_endpoint();
/* The exact two calls on_worker_join() makes, in the exact order. */
worker_push_config(&g_tc.c, ep);
worker_send_resync_query(&g_tc.c, w, ep);
xce_request_t first;
pop_and_decode(ep, &first);
TEST_ASSERT_EQUAL_INT_MESSAGE(OPCode_push, first.opcode,
"the config push must be the FIRST packet on the wire");
xce_free_request(&first);
xce_request_t second;
pop_and_decode(ep, &second);
TEST_ASSERT_EQUAL_INT_MESSAGE(OPCode_jobStateQuery, second.opcode,
"the reconciliation query must be the SECOND packet on the wire, "
"strictly after the config push");
xce_free_request(&second);
assert_queue_empty(ep);
dereference(new_attrs.hosttype);
dereference(new_attrs.hostmodel);
endpoint_fini(ep);
free(ep);
teardown_test_cluster();
printf("test_config_push_then_resync_query_wire_order: PASS\n");
}
/*
* Req 3.7 (config-delivery gap): worker_push_config() must carry the
* REAL, currently-configured workerResyncWindowMs value -- not a
* hardcoded constant. Two distinct configured values must decode back to
* two distinct wire values.
*/
static void test_worker_push_config_carries_configured_resync_window(void)
{
setup_test_cluster();
cluster_config_t cc;
memset(&cc, 0, sizeof(cc));
cc.clustername = (char*)"test-cluster";
cc.worker_resync_window_ms = 424242;
myconf_t myconf;
memset(&myconf, 0, sizeof(myconf));
myconf.cluster_config = &cc;
g_tc.c.myconf = &myconf;
endpoint_t* ep = new_test_endpoint();
TEST_ASSERT_EQUAL_INT(0, worker_push_config(&g_tc.c, ep));
xce_request_t decoded;
pop_and_decode(ep, &decoded);
TEST_ASSERT_EQUAL_INT(OPCode_push, decoded.opcode);
TEST_ASSERT_EQUAL_INT(Request_body_configpushmsg, decoded.body_which);
TEST_ASSERT_NOT_NULL(decoded.body.configpushmsg);
TEST_ASSERT_EQUAL_INT(ConfigData_data_workerconfig,
decoded.body.configpushmsg->data_which);
TEST_ASSERT_NOT_NULL(decoded.body.configpushmsg->data.conf4worker);
TEST_ASSERT_EQUAL_INT_MESSAGE(424242,
decoded.body.configpushmsg->data.conf4worker->worker_resync_window_ms,
"the pushed value must be the real configured ClusterConfig."
"workerResyncWindowMs, not a hardcoded constant");
xce_free_request(&decoded);
assert_queue_empty(ep);
endpoint_fini(ep);
free(ep);
/* A second, different configured value must decode back differently --
* confirms the field is genuinely read from config each call, not
* memoized/hardcoded from the first call above. */
cc.worker_resync_window_ms = 7;
ep = new_test_endpoint();
TEST_ASSERT_EQUAL_INT(0, worker_push_config(&g_tc.c, ep));
pop_and_decode(ep, &decoded);
TEST_ASSERT_EQUAL_INT_MESSAGE(7,
decoded.body.configpushmsg->data.conf4worker->worker_resync_window_ms,
"a second, different configured value must be reflected on the "
"wire, proving the field is read live, not cached");
xce_free_request(&decoded);
endpoint_fini(ep);
free(ep);
teardown_test_cluster();
printf("test_worker_push_config_carries_configured_resync_window: "
"PASS\n");
}
/* ── Dispatch ───────────────────────────────────────────────────────── */
void test_worker_join(void)
{
UnitySetTestFile(__FILE__);
printf("\n>> worker join refactor tests\n\n");
/* xstring_table_init must be called once before any makeString usage */
xstring_table_init(139);
RUN_TEST(test_update_resvals_logs_actual_rc_not_stale_errno);
RUN_TEST(test_update_resval_list_reports_only_applied_items);
/* Task 1.1: apply_join_state (shared status/resource update) */
RUN_TEST(test_apply_join_state_clears_unavail);
RUN_TEST(test_apply_join_state_applies_hosttype_override);
RUN_TEST(test_apply_join_state_updates_resource_values);
RUN_TEST(test_apply_join_state_mxj_negative_one_falls_back_to_maxcpus);
/* Task 1.2: init_worker (first-time join) */
RUN_TEST(test_init_worker_creates_new_entry);
RUN_TEST(test_init_worker_returns_correct_idx);
/* Task 1.3: rejoin_worker (reconnect with admin-bit preservation) */
RUN_TEST(test_rejoin_worker_updates_attrs_and_ep);
RUN_TEST(test_rejoin_worker_preserves_admin_bits);
/* worker-rejoin-resync task 6.1: apply_join_state arms a
* reconciliation round; worker_send_resync_query sends it; config
* push then query in that fixed order; the workerResyncWindowMs
* config-delivery gap. */
RUN_TEST(test_rejoin_worker_arms_reconciliation_round_when_jobs_exist);
RUN_TEST(test_rejoin_worker_no_round_when_no_jobs);
RUN_TEST(test_rejoin_worker_no_round_when_only_terminal_jobs);
RUN_TEST(test_rejoin_worker_string_resvals_survive_outstanding_round);
RUN_TEST(test_init_worker_arms_reconciliation_round_when_jobs_exist);
RUN_TEST(test_worker_send_resync_query_sends_armed_round);
RUN_TEST(test_worker_send_resync_query_sends_nothing_when_no_round);
RUN_TEST(
test_worker_send_resync_query_paginates_across_multiple_wire_pages);
RUN_TEST(test_config_push_then_resync_query_wire_order);
RUN_TEST(test_worker_push_config_carries_configured_resync_window);
}
Reference in a new issue View git blame Copy permalink