/* * test_worker_join.c — TDD tests for worker join refactor * * Task 1.1: Failing tests for apply_join_state (shared status/resource update) * Task 1.2: Failing tests for init_worker (first-time join path) * Task 1.3: Failing tests for rejoin_worker (reconnect update with admin-bit preservation) * * RED phase: apply_join_state / init_worker / rejoin_worker are stubs that * return 0 without modifying state, so all assertions below fail until * Tasks 2–3 provide real implementations. */ /* rtti.h must come first — it defines 'var' and 'typeinfo_t' used by kvmgr.h */ #include #include #include #include #include #include #include #include #include #include "cluster.h" #include "controller.capnp.h" /* worker-rejoin-resync task 6.1: OPCode_push/ * OPCode_jobStateQuery, ConfigData_data_*, * Request_body_* constants */ #include "jobj.h" /* worker-rejoin-resync task 6.1: makeJob, for a real * non-terminal job the arm-round wiring tests need */ #include "hook.h" /* bug-1851: xce_hook_init/xce_add_hook */ #include "kvmgr.h" #include "netio.h" /* worker-rejoin-resync task 6.1: endpoint_get/ * endpoint_deq/ITPTR_TO_PACKET, for real wire capture */ #include "request.h" /* worker-rejoin-resync task 6.1: xce_decode, * xce_free_request */ #include "resobj.h" #include "resync.h" /* worker-rejoin-resync task 6.1: worker_t.resync, * XCE_WORKER_RECONCILING */ #include "wobj.h" #include "worker.h" #include "xce/cluster_def.h" #include "xce/types.h" #include "xcealloc.h" /* worker-rejoin-resync task 6.1: make_job_alloc and * friends, for a job assigned to a test worker's * host */ #include "xstring.h" #include "zarray.h" /* ── Stubs for controller functions not under test ───────────────────── */ /* * collect_confvars is called by on_worker_join → worker_push_config. * Stubbed here so worker.c links without pulling in the full dataex.c. */ void collect_confvars(cluster_t* c, collect_check_fn_t fn, xlist_t* l) { (void)c; (void)fn; (void)l; } /* ── Forward declarations for functions under test ──────────────────── */ /* * apply_join_state is non-static in worker.c when XCE_TESTING is defined. * * RED: the current stub body does nothing and returns 0, so assertions * that check the modified state will fail. */ int apply_join_state(cluster_t* c, worker_t* w, endpoint_t* ep, xce_hostconfig_t* hconfig); /* * worker-rejoin-resync task 6.1: worker_push_config and worker_send_resync_ * query are non-static in worker.c when XCE_TESTING is defined, mirroring * apply_join_state's own convention above -- lets these tests drive the * exact config-push-then-query wire sequence on_worker_join produces, * without needing a live join flow or get_ep_worker()'s endpoint->worker * resolution (a separate, pre-existing concern -- see * test_on_worker_join_sends_config_push_even_when_worker_unresolvable * below for why). */ int worker_push_config(cluster_t* c, endpoint_t* ep); int worker_send_resync_query(cluster_t* c, worker_t* w, endpoint_t* ep); /* * init_worker and rejoin_worker are exported from worker.c. * * RED: stubs return 0 without registering or modifying any entry. */ int init_worker(cluster_t* c, endpoint_t* ep, const char* host, hostattrs_t* attrs, xce_hostconfig_t* hconfig); int rejoin_worker(cluster_t* c, worker_t* w, endpoint_t* ep, hostattrs_t* attrs, xce_hostconfig_t* hconfig); /* ── Test fixture ────────────────────────────────────────────────────── */ /* Resource names and value types matching add_hostattrs_resvals in worker.c */ #define N_RES 9 static const char* g_res_names[N_RES] = { "type", "model", "cpuf", "maxcpus", "maxmem", "maxswap", "maxtmp", "ndisks", "slots" }; static const int g_res_valtypes[N_RES] = { XCE_VAL_STRING, XCE_VAL_STRING, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC }; typedef struct { cluster_t c; var workermgr; var resmgr; var jobmgr; /* worker-rejoin-resync task 6.1: real jobmgr so the * apply_join_state arm-round wiring tests below can * seed a genuine non-terminal job assigned to a test * worker's host (mirrors test_worker_resync.c's own * g_twr.jobmgr fixture) */ } test_cluster_t; static test_cluster_t g_tc; /* worker-rejoin-resync task 6.1: jobj.c's job type table is a shared, * process-wide resource across every test file in the controller_unit_test * binary -- job_obj_init() is explicitly documented (jobj.c) as safe to * call more than once per process without an intervening job_obj_fini() * (Req 10.2, the exact "every controller unit-test file that constructs * jobs calls job_obj_init() from its own setup" scenario this file is now * one more instance of), so no cross-file coordination is needed -- guard * only avoids redundant re-init within THIS file's own repeated setup/ * teardown cycles, mirroring test_worker_resync.c's g_twr_jobobj_ready. */ static int g_tc_jobobj_ready; static void setup_test_cluster(void) { kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING, .nslots = 64, .delta = 16, .datakiller = NULL, }; kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16, }; memset(&g_tc, 0, sizeof(g_tc)); if (!g_tc_jobobj_ready) { TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32)); g_tc_jobobj_ready = 1; } g_tc.workermgr = makeInstance(&xkvmgr_type, &str_params); TEST_ASSERT_NOT_NULL(g_tc.workermgr); g_tc.resmgr = makeInstance(&xkvmgr_type, &str_params); TEST_ASSERT_NOT_NULL(g_tc.resmgr); g_tc.jobmgr = makeInstance(&xkvmgr_type, &int_params); TEST_ASSERT_NOT_NULL(g_tc.jobmgr); for (int i = 0; i < N_RES; i++) { xce_resource_t rdef; memset(&rdef, 0, sizeof(rdef)); rdef.name = (char*)g_res_names[i]; rdef.valtype = g_res_valtypes[i]; rdef.flags = 0; rdef.idx = -1; xce_resource_t* r = makeResource(g_tc.resmgr, &rdef); TEST_ASSERT_NOT_NULL_MESSAGE(r, g_res_names[i]); } g_tc.c.workermgr = g_tc.workermgr; g_tc.c.resmgr = g_tc.resmgr; g_tc.c.jobmgr = g_tc.jobmgr; /* worker-rejoin-resync task 6.1: c->myhost is the `from` peer identity * worker_push_config()/worker_send_resync_query() pass to xce_ * pushconfig_req()/xce_job_state_query_req() -- both reject a NULL * `from` outright, so any test exercising the real wire send needs * this populated (mirrors production cluster.c's own myhost_info * storage-then-point-at-it pattern, xce/cluster_def.h:101-102). */ g_tc.c.myhost_info.name = (char*)"test-controller"; g_tc.c.myhost_info.key = (char*)"test-key"; g_tc.c.myhost = &g_tc.c.myhost_info; zarray_init(&g_tc.c.sharedinst, 64, NULL); } static void teardown_test_cluster(void) { zarray_fini(&g_tc.c.sharedinst); if (g_tc.resmgr) dereference(g_tc.resmgr); if (g_tc.workermgr) dereference(g_tc.workermgr); if (g_tc.jobmgr) dereference(g_tc.jobmgr); memset(&g_tc, 0, sizeof(g_tc)); } /* Seed a job directly in the fixture's jobmgr, bypassing dispatch/replay -- * mirrors test_worker_resync.c's twr_seed_job. */ static job_submit_data_t g_tc_seed_submit; static job_t* seed_test_job(uint64_t jobid, uint32_t state) { xce_job_t jdef; memset(&g_tc_seed_submit, 0, sizeof(g_tc_seed_submit)); g_tc_seed_submit.queue = (char*)"q"; g_tc_seed_submit.user = (char*)"tester"; memset(&jdef, 0, sizeof(jdef)); jdef.jobid = jobid; jdef.state = state; jdef.submit = &g_tc_seed_submit; return makeJob(g_tc.jobmgr, &jdef); } /* Build a single-host job allocation assigning `jobid` to `host` -- mirrors * test_worker_resync.c's twr_make_test_alloc. Caller hands ownership to * job->job.allocs. */ static xce_job_alloc_t* make_test_job_alloc(uint64_t jobid, const char* host) { return make_job_alloc(jobid, 1, make_host_allocs(1, make_host_alloc((char*)host, 1, make_allocs(1, make_alloc((char*)"slots", NULL, 0, 1.0f, 0))))); } /* Create a worker in the test workermgr; attrs strings must be makeString'd. */ static worker_t* make_test_worker(const char* name, hostattrs_t* attrs) { xce_worker_t host; memset(&host, 0, sizeof(host)); host.name = (char*)name; host.attrs = attrs; return makeWorker(g_tc.workermgr, &host); } /* Retrieve a numeric resource value from a worker's resvals by resource name */ static float get_worker_res_fval(worker_t* w, const char* name) { xce_resource_t* r = NULL; lock(g_tc.c.resmgr, RTTI_LOCK_READ); key_find(g_tc.c.resmgr, (void*)name, (void**)&r); unlock(g_tc.c.resmgr); if (r == NULL || r->idx < 0 || r->idx >= w->host.resvals.caps) return -1.0f; xce_val_t* v = (xce_val_t*)w->host.resvals.data[r->idx]; return (v != NULL) ? v->u.fvalue : -1.0f; } /* * Capture whatever update_resvals()/update_resval_list() print during a * failing update_resval() call, as raw runtime output rather than a * source-text scan. This test binary never calls mylog_init()/tlog_init() * (only the real daemon main()s do -- see test_apply_correctness_ * verification.c's own header comment, and the identical established * precedent in scheduler/testsuite/test_scheduler_connect.c's * capture_stdout_during_sched_init()), so tlog's internal root log handle * stays NULL and every ERROR()/WARN() call falls through tlog's early- * print path, which writes the fully formatted line directly to * STDOUT_FILENO via a raw write() -- genuine runtime log capture, not a * guess about which stream PERROR/ERROR happens to use. */ static char* capture_stdout_during(void (*fn)(void* ctx), void* ctx) { char tmp[] = "/tmp/xce_worker_join_logcap_XXXXXX"; int fd = mkstemp(tmp); TEST_ASSERT_TRUE_MESSAGE(fd >= 0, "fixture: mkstemp for stdout capture must succeed"); fflush(stdout); int saved_stdout = dup(STDOUT_FILENO); TEST_ASSERT_TRUE_MESSAGE(saved_stdout >= 0, "fixture: dup(STDOUT_FILENO) must succeed"); TEST_ASSERT_TRUE_MESSAGE(dup2(fd, STDOUT_FILENO) >= 0, "fixture: dup2 redirect of STDOUT_FILENO must " "succeed"); fn(ctx); fflush(stdout); dup2(saved_stdout, STDOUT_FILENO); close(saved_stdout); off_t len = lseek(fd, 0, SEEK_CUR); TEST_ASSERT_TRUE_MESSAGE(len >= 0, "fixture: lseek on capture file must succeed"); char* buf = (char*)calloc(1, (size_t)len + 1); TEST_ASSERT_NOT_NULL(buf); if (len > 0) { TEST_ASSERT_TRUE_MESSAGE(lseek(fd, 0, SEEK_SET) == 0, "fixture: rewind of capture file must " "succeed"); ssize_t nread = read(fd, buf, (size_t)len); TEST_ASSERT_TRUE_MESSAGE(nread == len, "fixture: reading back captured buffer " "must succeed"); } close(fd); unlink(tmp); return buf; } typedef struct { cluster_t* c; worker_t* w; xce_resval_t* val; } update_resvals_call_ctx_t; static void call_update_resvals_1(void* ctx_) { update_resvals_call_ctx_t* ctx = (update_resvals_call_ctx_t*)ctx_; update_resvals(ctx->c, ctx->w, 1, ctx->val); } /* * update_resval() returns its failure reason as a raw negative code * directly (key_find/zarray_put convention -- never via errno), so * update_resvals()/update_resval_list() logging it via PERROR (which reads * the process-global errno) reported whatever errno happened to be left * over from an unrelated earlier call: "Success" if nothing had touched * it, or a stray EAGAIN/etc from some other syscall in the same event-loop * iteration -- never the real reason a resource update failed. This test * reproduces that exact symptom (ambient errno=EAGAIN, unrelated to the * genuine failure) and proves the fix decodes the actual rc instead. */ static void test_update_resvals_logs_actual_rc_not_stale_errno(void) { setup_test_cluster(); hostattrs_t attrs; memset(&attrs, 0, sizeof(attrs)); attrs.hosttype = makeString("LINUX"); attrs.hostmodel = makeString("x86"); worker_t* w = make_test_worker("host-resval-log", &attrs); TEST_ASSERT_NOT_NULL(w); xce_val_t fval; memset(&fval, 0, sizeof(fval)); fval.which = Value_u_fvalue; fval.u.fvalue = 1.0f; xce_resval_t val; val.name = "does-not-exist-in-resmgr"; val.val = &fval; /* Simulate the exact reported symptom: ambient errno unrelated to the * real failure, left over from some other unrelated call. */ errno = EAGAIN; update_resvals_call_ctx_t ctx = { .c = &g_tc.c, .w = w, .val = &val }; char* captured = capture_stdout_during(call_update_resvals_1, &ctx); TEST_ASSERT_NOT_NULL(captured); TEST_ASSERT_NULL_MESSAGE( strstr(captured, "Resource temporarily unavailable"), "must not log the ambient errno's string (EAGAIN) -- that is not " "why update_resval failed"); TEST_ASSERT_NULL_MESSAGE( strstr(captured, "Success"), "must not log \"Success\" for a call that returned rc<0"); TEST_ASSERT_NOT_NULL_MESSAGE( strstr(captured, "does-not-exist-in-resmgr"), "must name which resource failed"); TEST_ASSERT_NOT_NULL_MESSAGE( strstr(captured, "No such file or directory"), "must log the real reason (-ENOENT, from key_find on an " "unregistered resource name)"); free(captured); teardown_test_cluster(); printf("test_update_resvals_logs_actual_rc_not_stale_errno: PASS\n"); } /* bug-1583: update_resval_list() must report which items it actually * applied, so do_update_resvals() can hand ONLY those to the AFTER_EVENT * resource hook (scheduler + apiserver). It used to pass the raw client * list, so the apiserver stored resource names the controller rejected. */ static void test_update_resval_list_reports_only_applied_items(void) { setup_test_cluster(); hostattrs_t attrs; memset(&attrs, 0, sizeof(attrs)); attrs.hosttype = makeString("LINUX"); attrs.hostmodel = makeString("x86"); worker_t* w = make_test_worker("host-resval-applied", &attrs); TEST_ASSERT_NOT_NULL(w); xce_val_t good_val, bad_val; memset(&good_val, 0, sizeof(good_val)); good_val.which = Value_u_fvalue; good_val.u.fvalue = 4.0f; bad_val = good_val; xce_resval_t good = { .name = "slots", .val = &good_val }; xce_resval_t bad = { .name = "totally-fake-resource-xyz", .val = &bad_val }; xce_resval_t* items[2] = { &bad, &good }; xce_resval_list_t in = { .count = 2, .vals = items }; xce_resval_t* out_items[2] = { NULL, NULL }; xce_resval_list_t applied = { .count = -1, .vals = out_items }; TEST_ASSERT_EQUAL_INT(0, update_resval_list(&g_tc.c, w, &in, &applied)); TEST_ASSERT_EQUAL_INT_MESSAGE(1, applied.count, "bug-1583: only the resource the controller accepted may be " "reported as applied"); TEST_ASSERT_TRUE(applied.vals[0] == &good); TEST_ASSERT_EQUAL_FLOAT(4.0f, get_worker_res_fval(w, "slots")); teardown_test_cluster(); } /* ── Task 1.1: Tests for apply_join_state ──────────────────────────── */ /* * After apply_join_state, XCE_WORKER_UNAVAIL must be cleared. Req 4.1 */ static void test_apply_join_state_clears_unavail(void) { setup_test_cluster(); hostattrs_t attrs; memset(&attrs, 0, sizeof(attrs)); attrs.hosttype = makeString("LINUX"); attrs.hostmodel = makeString("x86"); attrs.cpufactor = 1.0f; attrs.maxcpus = 4; attrs.maxmem = 16; attrs.maxswap = 32; attrs.maxtmp = 64; attrs.ndisks = 1; worker_t* w = make_test_worker("host-ajs-1", &attrs); TEST_ASSERT_NOT_NULL(w); lock(w, RTTI_LOCK_WRITE); w->host.status |= XCE_WORKER_UNAVAIL; unlock(w); xce_hostconfig_t hconfig; memset(&hconfig, 0, sizeof(hconfig)); hconfig.hosttype = (char*)"!"; /* no type override */ hconfig.hostmodel = (char*)"!"; /* no model override */ hconfig.mxj = 2; endpoint_t ep; memset(&ep, 0, sizeof(ep)); lock(w, RTTI_LOCK_WRITE); int rc = apply_join_state(&g_tc.c, w, &ep, &hconfig); unlock(w); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_FALSE_MESSAGE(w->host.status & XCE_WORKER_UNAVAIL, "UNAVAIL must be cleared after apply_join_state"); teardown_test_cluster(); printf("test_apply_join_state_clears_unavail: PASS\n"); } /* * When hconfig provides non-"!" overrides, hosttype and hostmodel must * match those values after apply_join_state. Req 4.2 */ static void test_apply_join_state_applies_hosttype_override(void) { setup_test_cluster(); hostattrs_t attrs; memset(&attrs, 0, sizeof(attrs)); attrs.hosttype = makeString("OLD_TYPE"); attrs.hostmodel = makeString("OLD_MODEL"); attrs.cpufactor = 1.0f; attrs.maxcpus = 2; attrs.maxmem = 8; attrs.maxswap = 16; attrs.maxtmp = 32; attrs.ndisks = 1; worker_t* w = make_test_worker("host-ajs-2", &attrs); TEST_ASSERT_NOT_NULL(w); xce_hostconfig_t hconfig; memset(&hconfig, 0, sizeof(hconfig)); hconfig.hosttype = (char*)"LINUX64"; hconfig.hostmodel = (char*)"x86_64"; hconfig.mxj = 4; endpoint_t ep; memset(&ep, 0, sizeof(ep)); lock(w, RTTI_LOCK_WRITE); int rc = apply_join_state(&g_tc.c, w, &ep, &hconfig); unlock(w); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_STRING_MESSAGE("LINUX64", w->host.attrs->hosttype, "hosttype must be overridden from hconfig"); TEST_ASSERT_EQUAL_STRING_MESSAGE("x86_64", w->host.attrs->hostmodel, "hostmodel must be overridden from hconfig"); teardown_test_cluster(); printf("test_apply_join_state_applies_hosttype_override: PASS\n"); } /* * After apply_join_state, 'slots' equals hconfig.mxj and numeric host-attr * resources match the worker's attrs. Req 4.2, 4.3 */ static void test_apply_join_state_updates_resource_values(void) { setup_test_cluster(); hostattrs_t attrs; memset(&attrs, 0, sizeof(attrs)); attrs.hosttype = makeString("LINUX"); attrs.hostmodel = makeString("x86"); attrs.cpufactor = 3.0f; attrs.maxcpus = 8; attrs.maxmem = 64; attrs.maxswap = 128; attrs.maxtmp = 256; attrs.ndisks = 4; worker_t* w = make_test_worker("host-ajs-3", &attrs); TEST_ASSERT_NOT_NULL(w); xce_hostconfig_t hconfig; memset(&hconfig, 0, sizeof(hconfig)); hconfig.hosttype = (char*)"!"; hconfig.hostmodel = (char*)"!"; hconfig.mxj = 6; endpoint_t ep; memset(&ep, 0, sizeof(ep)); lock(w, RTTI_LOCK_WRITE); int rc = apply_join_state(&g_tc.c, w, &ep, &hconfig); unlock(w); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_FLOAT_MESSAGE(6.0f, get_worker_res_fval(w, "slots"), "slots must equal hconfig.mxj"); TEST_ASSERT_EQUAL_FLOAT(3.0f, get_worker_res_fval(w, "cpuf")); TEST_ASSERT_EQUAL_FLOAT(8.0f, get_worker_res_fval(w, "maxcpus")); TEST_ASSERT_EQUAL_FLOAT(64.0f, get_worker_res_fval(w, "maxmem")); TEST_ASSERT_EQUAL_FLOAT(128.0f, get_worker_res_fval(w, "maxswap")); TEST_ASSERT_EQUAL_FLOAT(256.0f, get_worker_res_fval(w, "maxtmp")); TEST_ASSERT_EQUAL_FLOAT(4.0f, get_worker_res_fval(w, "ndisks")); teardown_test_cluster(); printf("test_apply_join_state_updates_resource_values: PASS\n"); } /* * hconfig.mxj == -1 is the schema's documented "auto-detect" sentinel * (hosts.schema.json: "max_slot": {"minimum": -1, "description": "-1 means * auto-detect"}) -- apply_join_state must fall back to the worker's own * reported maxcpus, NOT persist -1 (or, before the xceadmin mapping fix, * silently ship 0 and permanently disable scheduling on that host: mxj is * UInt32 on the wire, so an unset/absent max_slot used to decode as 0, * which is not < 0 and so never reached this fallback -- see * xcecommon.mapping.HOSTCONFIG_MXJ_AUTO_DETECT). This is the ONLY test in * this file that exercises the nslots<0 branch; every sibling test above * uses a positive hconfig.mxj and so cannot catch a regression here. */ static void test_apply_join_state_mxj_negative_one_falls_back_to_maxcpus(void) { setup_test_cluster(); hostattrs_t attrs; memset(&attrs, 0, sizeof(attrs)); attrs.hosttype = makeString("LINUX"); attrs.hostmodel = makeString("x86"); attrs.cpufactor = 1.0f; attrs.maxcpus = 12; attrs.maxmem = 64; attrs.maxswap = 128; attrs.maxtmp = 256; attrs.ndisks = 2; worker_t* w = make_test_worker("host-ajs-autodetect", &attrs); TEST_ASSERT_NOT_NULL(w); xce_hostconfig_t hconfig; memset(&hconfig, 0, sizeof(hconfig)); hconfig.hosttype = (char*)"!"; hconfig.hostmodel = (char*)"!"; hconfig.mxj = -1; endpoint_t ep; memset(&ep, 0, sizeof(ep)); lock(w, RTTI_LOCK_WRITE); int rc = apply_join_state(&g_tc.c, w, &ep, &hconfig); unlock(w); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_FLOAT_MESSAGE( 12.0f, get_worker_res_fval(w, "slots"), "mxj=-1 must auto-detect slots from the worker's reported maxcpus, " "not persist -1 or silently leave slots at 0"); teardown_test_cluster(); printf("test_apply_join_state_mxj_negative_one_falls_back_to_maxcpus: " "PASS\n"); } /* ── Task 1.2: Tests for init_worker ────────────────────────────────── */ /* * After init_worker for a new hostname, key_find must find the entry and * return a non-negative index. Req 2.1, 2.3 */ static void test_init_worker_creates_new_entry(void) { setup_test_cluster(); hostattrs_t attrs; memset(&attrs, 0, sizeof(attrs)); attrs.hosttype = makeString("LINUX"); attrs.hostmodel = makeString("x86"); attrs.cpufactor = 1.0f; attrs.maxcpus = 4; attrs.maxmem = 16; attrs.maxswap = 32; attrs.maxtmp = 64; attrs.ndisks = 1; xce_hostconfig_t hconfig; memset(&hconfig, 0, sizeof(hconfig)); hconfig.hosttype = (char*)"!"; hconfig.hostmodel = (char*)"!"; hconfig.mxj = 4; endpoint_t ep; memset(&ep, 0, sizeof(ep)); int idx = init_worker(&g_tc.c, &ep, "new-host-1", &attrs, &hconfig); TEST_ASSERT_GREATER_OR_EQUAL_MESSAGE(0, idx, "init_worker must return non-negative index"); worker_t* w = NULL; lock(g_tc.c.workermgr, RTTI_LOCK_READ); int rc = key_find(g_tc.c.workermgr, (void*)"new-host-1", (void**)&w); unlock(g_tc.c.workermgr); TEST_ASSERT_GREATER_THAN_MESSAGE(-1, rc, "worker must be registered in workermgr after init"); TEST_ASSERT_NOT_NULL_MESSAGE(w, "key_find must return a valid worker pointer"); teardown_test_cluster(); printf("test_init_worker_creates_new_entry: PASS\n"); } /* * The index returned by init_worker must match w->idx. Req 2.3 */ static void test_init_worker_returns_correct_idx(void) { setup_test_cluster(); hostattrs_t attrs; memset(&attrs, 0, sizeof(attrs)); attrs.hosttype = makeString("LINUX"); attrs.hostmodel = makeString("x86"); attrs.cpufactor = 1.0f; attrs.maxcpus = 2; attrs.maxmem = 8; attrs.maxswap = 16; attrs.maxtmp = 32; attrs.ndisks = 1; xce_hostconfig_t hconfig; memset(&hconfig, 0, sizeof(hconfig)); hconfig.hosttype = (char*)"!"; hconfig.hostmodel = (char*)"!"; hconfig.mxj = 2; endpoint_t ep; memset(&ep, 0, sizeof(ep)); int idx = init_worker(&g_tc.c, &ep, "new-host-2", &attrs, &hconfig); TEST_ASSERT_GREATER_OR_EQUAL(0, idx); worker_t* w = NULL; lock(g_tc.c.workermgr, RTTI_LOCK_READ); key_find(g_tc.c.workermgr, (void*)"new-host-2", (void**)&w); unlock(g_tc.c.workermgr); TEST_ASSERT_NOT_NULL(w); TEST_ASSERT_EQUAL_INT_MESSAGE(idx, w->idx, "returned idx must match the worker's idx field"); teardown_test_cluster(); printf("test_init_worker_returns_correct_idx: PASS\n"); } /* ── Task 1.3: Tests for rejoin_worker ──────────────────────────────── */ /* * rejoin_worker must update hostattrs and ep on the existing entry. Req 3.1, 3.2 */ static void test_rejoin_worker_updates_attrs_and_ep(void) { setup_test_cluster(); hostattrs_t old_attrs; memset(&old_attrs, 0, sizeof(old_attrs)); old_attrs.hosttype = makeString("OLD_TYPE"); old_attrs.hostmodel = makeString("OLD_MODEL"); old_attrs.cpufactor = 1.0f; old_attrs.maxcpus = 2; old_attrs.maxmem = 8; old_attrs.maxswap = 16; old_attrs.maxtmp = 32; old_attrs.ndisks = 1; worker_t* w = make_test_worker("rejoin-host-1", &old_attrs); TEST_ASSERT_NOT_NULL(w); hostattrs_t new_attrs; memset(&new_attrs, 0, sizeof(new_attrs)); new_attrs.hosttype = makeString("NEW_TYPE"); new_attrs.hostmodel = makeString("NEW_MODEL"); new_attrs.cpufactor = 2.0f; new_attrs.maxcpus = 8; new_attrs.maxmem = 64; new_attrs.maxswap = 128; new_attrs.maxtmp = 256; new_attrs.ndisks = 2; xce_hostconfig_t hconfig; memset(&hconfig, 0, sizeof(hconfig)); hconfig.hosttype = (char*)"!"; hconfig.hostmodel = (char*)"!"; hconfig.mxj = 4; endpoint_t ep; memset(&ep, 0, sizeof(ep)); int idx = rejoin_worker(&g_tc.c, w, &ep, &new_attrs, &hconfig); TEST_ASSERT_GREATER_OR_EQUAL_MESSAGE(0, idx, "rejoin_worker must return non-negative index"); TEST_ASSERT_EQUAL_STRING_MESSAGE("NEW_TYPE", w->host.attrs->hosttype, "hosttype must be updated to new attrs value"); TEST_ASSERT_EQUAL_STRING_MESSAGE("NEW_MODEL", w->host.attrs->hostmodel, "hostmodel must be updated to new attrs value"); TEST_ASSERT_EQUAL_FLOAT_MESSAGE(2.0f, w->host.attrs->cpufactor, "cpufactor must be updated to new value"); TEST_ASSERT_EQUAL_PTR_MESSAGE(&ep, w->ep, "ep must be updated to the new endpoint"); /* Release the test's own references to the new strings */ dereference(new_attrs.hosttype); dereference(new_attrs.hostmodel); teardown_test_cluster(); printf("test_rejoin_worker_updates_attrs_and_ep: PASS\n"); } /* * Admin-set status bits must survive a rejoin_worker call. Req 3.3 */ static void test_rejoin_worker_preserves_admin_bits(void) { setup_test_cluster(); hostattrs_t attrs; memset(&attrs, 0, sizeof(attrs)); attrs.hosttype = makeString("LINUX"); attrs.hostmodel = makeString("x86"); attrs.cpufactor = 1.0f; attrs.maxcpus = 4; attrs.maxmem = 16; attrs.maxswap = 32; attrs.maxtmp = 64; attrs.ndisks = 1; worker_t* w = make_test_worker("rejoin-host-2", &attrs); TEST_ASSERT_NOT_NULL(w); /* Set all admin bits before the rejoin */ lock(w, RTTI_LOCK_WRITE); w->host.status |= (XCE_WORKER_LOCKEDU | XCE_WORKER_CLOSED | XCE_WORKER_CLOSED_BY_ADMIN); unlock(w); hostattrs_t new_attrs; memset(&new_attrs, 0, sizeof(new_attrs)); new_attrs.hosttype = makeString("LINUX"); new_attrs.hostmodel = makeString("x86"); new_attrs.cpufactor = 2.0f; new_attrs.maxcpus = 8; new_attrs.maxmem = 32; new_attrs.maxswap = 64; new_attrs.maxtmp = 128; new_attrs.ndisks = 2; xce_hostconfig_t hconfig; memset(&hconfig, 0, sizeof(hconfig)); hconfig.hosttype = (char*)"!"; hconfig.hostmodel = (char*)"!"; hconfig.mxj = 4; endpoint_t ep; memset(&ep, 0, sizeof(ep)); rejoin_worker(&g_tc.c, w, &ep, &new_attrs, &hconfig); TEST_ASSERT_TRUE_MESSAGE(w->host.status & XCE_WORKER_LOCKEDU, "LOCKEDU must be preserved after rejoin"); TEST_ASSERT_TRUE_MESSAGE(w->host.status & XCE_WORKER_CLOSED, "CLOSED must be preserved after rejoin"); TEST_ASSERT_TRUE_MESSAGE(w->host.status & XCE_WORKER_CLOSED_BY_ADMIN, "CLOSED_BY_ADMIN must be preserved after rejoin"); dereference(new_attrs.hosttype); dereference(new_attrs.hostmodel); teardown_test_cluster(); printf("test_rejoin_worker_preserves_admin_bits: PASS\n"); } /* ── Task 6.1: apply_join_state arms a reconciliation round ──────────── */ /* * Real (heap) endpoint_t via endpoint_get() -- NOT a hand-rolled/zeroed * stack stand-in (bug-378: an under-sized/wrong-typed stand-in for * endpoint_t caused a global-buffer-overflow when passed where a real * endpoint_t* was expected). Needed only by the tests below that actually * send something over the wire (worker_push_config/worker_send_resync_ * query); the pre-existing apply_join_state/init_worker/rejoin_worker * tests above never send anything and so are unaffected, still using a * plain zeroed stack endpoint_t. Mirrors src/lib/testsuite/ * test_job_state_wire.c's own established pattern for this exact * situation. */ static endpoint_t* new_test_endpoint(void) { endpoint_t* ep = endpoint_get(); TEST_ASSERT_NOT_NULL(ep); return ep; } /* Pop one queued page off ep->outq and decode it. Fails the test if the * queue is empty. Caller must xce_free_request() the result. */ static void pop_and_decode(endpoint_t* ep, xce_request_t* out) { data_item_t* it = endpoint_deq(ep); TEST_ASSERT_NOT_NULL(it); io_packet_t* pkt = ITPTR_TO_PACKET(it); memset(out, 0, sizeof(*out)); TEST_ASSERT_EQUAL_INT(0, xce_decode(pkt, out)); /* consumes/frees pkt */ } static void assert_queue_empty(endpoint_t* ep) { TEST_ASSERT_NULL(endpoint_deq(ep)); } /* * rejoin_worker (via apply_join_state) must arm a reconciliation round -- * w->resync populated, XCE_WORKER_RECONCILING set, the query set matching * exactly the host's non-terminal jobs -- when the controller currently * holds one or more non-terminal jobs for that host. Req 1.1, 2.1, 2.2. */ static void test_rejoin_worker_arms_reconciliation_round_when_jobs_exist(void) { setup_test_cluster(); job_t* j = seed_test_job(96001, JOB_STAT_RUN); TEST_ASSERT_NOT_NULL(j); j->job.allocs = make_test_job_alloc(96001, "arm-host-1"); TEST_ASSERT_NOT_NULL(j->job.allocs); hostattrs_t attrs; memset(&attrs, 0, sizeof(attrs)); attrs.hosttype = makeString("LINUX"); attrs.hostmodel = makeString("x86"); attrs.cpufactor = 1.0f; attrs.maxcpus = 4; attrs.maxmem = 16; attrs.maxswap = 32; attrs.maxtmp = 64; attrs.ndisks = 1; worker_t* w = make_test_worker("arm-host-1", &attrs); TEST_ASSERT_NOT_NULL(w); hostattrs_t new_attrs; memset(&new_attrs, 0, sizeof(new_attrs)); new_attrs.hosttype = makeString("LINUX"); new_attrs.hostmodel = makeString("x86"); new_attrs.cpufactor = 1.0f; new_attrs.maxcpus = 4; new_attrs.maxmem = 16; new_attrs.maxswap = 32; new_attrs.maxtmp = 64; new_attrs.ndisks = 1; xce_hostconfig_t hconfig; memset(&hconfig, 0, sizeof(hconfig)); hconfig.hosttype = (char*)"!"; hconfig.hostmodel = (char*)"!"; hconfig.mxj = 4; endpoint_t ep; memset(&ep, 0, sizeof(ep)); int idx = rejoin_worker(&g_tc.c, w, &ep, &new_attrs, &hconfig); TEST_ASSERT_GREATER_OR_EQUAL(0, idx); TEST_ASSERT_NOT_NULL_MESSAGE(w->resync, "a worker rejoining with a known non-terminal job must have a " "reconciliation round armed (Req 1.1)"); TEST_ASSERT_TRUE_MESSAGE(w->host.status & XCE_WORKER_RECONCILING, "XCE_WORKER_RECONCILING must be set once a round is armed"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, w->resync->n_queried_jobids, "the round's query set must match exactly the host's non-terminal " "jobs (Req 2.1)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(96001, w->resync->queried_jobids[0], "the queried job must be identified by its unique jobid (Req 2.2)"); dereference(new_attrs.hosttype); dereference(new_attrs.hostmodel); teardown_test_cluster(); printf("test_rejoin_worker_arms_reconciliation_round_when_jobs_exist: " "PASS\n"); } /* * A worker rejoining with NO non-terminal jobs currently assigned to it * must NOT have a round armed. Req 1.2. */ static void test_rejoin_worker_no_round_when_no_jobs(void) { setup_test_cluster(); hostattrs_t attrs; memset(&attrs, 0, sizeof(attrs)); attrs.hosttype = makeString("LINUX"); attrs.hostmodel = makeString("x86"); attrs.cpufactor = 1.0f; attrs.maxcpus = 4; attrs.maxmem = 16; attrs.maxswap = 32; attrs.maxtmp = 64; attrs.ndisks = 1; worker_t* w = make_test_worker("arm-host-2", &attrs); TEST_ASSERT_NOT_NULL(w); hostattrs_t new_attrs = attrs; new_attrs.hosttype = makeString("LINUX"); new_attrs.hostmodel = makeString("x86"); xce_hostconfig_t hconfig; memset(&hconfig, 0, sizeof(hconfig)); hconfig.hosttype = (char*)"!"; hconfig.hostmodel = (char*)"!"; hconfig.mxj = 4; endpoint_t ep; memset(&ep, 0, sizeof(ep)); rejoin_worker(&g_tc.c, w, &ep, &new_attrs, &hconfig); TEST_ASSERT_NULL_MESSAGE(w->resync, "a worker rejoining with no non-terminal jobs must not have a " "reconciliation round armed (Req 1.2)"); TEST_ASSERT_FALSE_MESSAGE(w->host.status & XCE_WORKER_RECONCILING, "XCE_WORKER_RECONCILING must not be set"); dereference(new_attrs.hosttype); dereference(new_attrs.hostmodel); teardown_test_cluster(); printf("test_rejoin_worker_no_round_when_no_jobs: PASS\n"); } /* * A terminal job (DONE) assigned to the host must not be queried -- only * non-terminal jobs count toward the trigger (Req 1.1's own "non-terminal" * qualifier). */ static void test_rejoin_worker_no_round_when_only_terminal_jobs(void) { setup_test_cluster(); job_t* j = seed_test_job(96002, JOB_STAT_DONE); TEST_ASSERT_NOT_NULL(j); j->job.allocs = make_test_job_alloc(96002, "arm-host-3"); TEST_ASSERT_NOT_NULL(j->job.allocs); hostattrs_t attrs; memset(&attrs, 0, sizeof(attrs)); attrs.hosttype = makeString("LINUX"); attrs.hostmodel = makeString("x86"); attrs.cpufactor = 1.0f; attrs.maxcpus = 4; attrs.maxmem = 16; attrs.maxswap = 32; attrs.maxtmp = 64; attrs.ndisks = 1; worker_t* w = make_test_worker("arm-host-3", &attrs); TEST_ASSERT_NOT_NULL(w); hostattrs_t new_attrs = attrs; new_attrs.hosttype = makeString("LINUX"); new_attrs.hostmodel = makeString("x86"); xce_hostconfig_t hconfig; memset(&hconfig, 0, sizeof(hconfig)); hconfig.hosttype = (char*)"!"; hconfig.hostmodel = (char*)"!"; hconfig.mxj = 4; endpoint_t ep; memset(&ep, 0, sizeof(ep)); rejoin_worker(&g_tc.c, w, &ep, &new_attrs, &hconfig); TEST_ASSERT_NULL_MESSAGE(w->resync, "a terminal (DONE) job must not count toward the reconciliation " "trigger"); dereference(new_attrs.hosttype); dereference(new_attrs.hostmodel); teardown_test_cluster(); printf("test_rejoin_worker_no_round_when_only_terminal_jobs: PASS\n"); } /* * bug-1851: a string resval ("type"/"model") must own its bytes rather * than alias the interned xstring in w->host.attrs. rejoin_worker drops * the old attrs strings, then apply_join_state -> resync_arm_round * resolves the PREVIOUS join's still-outstanding round, which fires the * worker hooks (resync_resolve_round -> persist_notify_worker_status_ * updated) BEFORE add_hostattrs_resvals re-seeds resvals. Live, the * apiserver hook's collect_resval strdup()ed the freed hostmodel and the * newly-promoted primary died (ASAN heap-use-after-free). This hook stands * in for it: it reads both string resvals at exactly that point. */ static char g_1851_seen_type[64]; static char g_1851_seen_model[64]; static int g_1851_hook_calls; static void copy_string_resval(worker_t* w, const char* name, char* out, size_t out_sz) { xce_resource_t* r = NULL; xce_val_t* v; lock(g_tc.c.resmgr, RTTI_LOCK_READ); key_find(g_tc.c.resmgr, (void*)name, (void**)&r); unlock(g_tc.c.resmgr); if (r == NULL || r->idx < 0 || r->idx >= w->host.resvals.caps) return; v = (xce_val_t*)w->host.resvals.data[r->idx]; if (v != NULL && v->which == Value_u_svalue && v->u.svalue != NULL) snprintf(out, out_sz, "%s", v->u.svalue); } static int bug1851_worker_hook(int type, int situation, int detail, void* cls, va_list ap) { (void)type; (void)situation; (void)detail; (void)cls; (void)va_arg(ap, cluster_t*); worker_t* w = va_arg(ap, worker_t*); g_1851_hook_calls++; copy_string_resval(w, "type", g_1851_seen_type, sizeof(g_1851_seen_type)); copy_string_resval(w, "model", g_1851_seen_model, sizeof(g_1851_seen_model)); return 0; } static void test_rejoin_worker_string_resvals_survive_outstanding_round(void) { setup_test_cluster(); TEST_ASSERT_EQUAL_INT( 0, xce_hook_init(&g_tc.c.hooksys, XCE_CONTROLLER_HOOK_MAX)); TEST_ASSERT_EQUAL_INT(0, xce_add_hook(&g_tc.c.hooksys, XCE_CONTROLLER_HOOK_WORKER, bug1851_worker_hook, NULL)); g_1851_hook_calls = 0; g_1851_seen_type[0] = '\0'; g_1851_seen_model[0] = '\0'; /* A non-terminal job on the host, so each rejoin arms a round and the * second rejoin finds the first one still outstanding. */ job_t* j = seed_test_job(96851, JOB_STAT_RUN); TEST_ASSERT_NOT_NULL(j); j->job.allocs = make_test_job_alloc(96851, "uaf-host-1851"); TEST_ASSERT_NOT_NULL(j->job.allocs); hostattrs_t attrs; memset(&attrs, 0, sizeof(attrs)); attrs.hosttype = makeString("TYPE-1851-A"); attrs.hostmodel = makeString("MODEL-1851-A"); attrs.maxcpus = 4; worker_t* w = make_test_worker("uaf-host-1851", &attrs); TEST_ASSERT_NOT_NULL(w); /* makeWorker took its own references; drop the test's so w->host.attrs * holds the ONLY reference to the "A" strings. */ dereference(attrs.hosttype); dereference(attrs.hostmodel); xce_hostconfig_t hconfig; memset(&hconfig, 0, sizeof(hconfig)); hconfig.hosttype = (char*)"!"; hconfig.hostmodel = (char*)"!"; hconfig.mxj = 4; endpoint_t ep; memset(&ep, 0, sizeof(ep)); /* First join: seeds resvals from the "A" strings, arms round #1. */ TEST_ASSERT_GREATER_OR_EQUAL(0, apply_join_state(&g_tc.c, w, &ep, &hconfig)); TEST_ASSERT_NOT_NULL_MESSAGE(w->resync, "fixture: the first join must leave a round outstanding"); /* Second join with different strings: must resolve round #1 (firing * the hook) while "A" is being released. */ hostattrs_t new_attrs; memset(&new_attrs, 0, sizeof(new_attrs)); new_attrs.hosttype = makeString("TYPE-1851-B"); new_attrs.hostmodel = makeString("MODEL-1851-B"); new_attrs.maxcpus = 4; g_1851_hook_calls = 0; TEST_ASSERT_GREATER_OR_EQUAL( 0, rejoin_worker(&g_tc.c, w, &ep, &new_attrs, &hconfig)); TEST_ASSERT_GREATER_THAN_INT_MESSAGE(0, g_1851_hook_calls, "fixture: resolving the outstanding round must fire the worker hook"); TEST_ASSERT_EQUAL_STRING_MESSAGE("TYPE-1851-A", g_1851_seen_type, "the hook must read the previous (still-valid) type resval, not " "freed memory"); TEST_ASSERT_EQUAL_STRING_MESSAGE("MODEL-1851-A", g_1851_seen_model, "the hook must read the previous (still-valid) model resval, not " "freed memory"); /* Ownership: the resval is a private copy, never the attrs xstring. */ xce_val_t val; memset(&val, 0, sizeof(val)); TEST_ASSERT_EQUAL_INT(1, get_resval(&g_tc.c, (char*)"model", (char*)"uaf-host-1851", &val)); TEST_ASSERT_EQUAL_STRING("MODEL-1851-B", val.u.svalue); TEST_ASSERT_TRUE_MESSAGE(val.u.svalue != w->host.attrs->hostmodel, "a string resval must own its bytes, not alias w->host.attrs"); dereference(new_attrs.hosttype); dereference(new_attrs.hostmodel); xce_hook_fini(&g_tc.c.hooksys); teardown_test_cluster(); printf("test_rejoin_worker_string_resvals_survive_outstanding_round: " "PASS\n"); } /* * Req 1.3: the SAME trigger condition -- no special-casing needed -- must * apply to a host's first-ever join (init_worker), not just a rejoin. */ static void test_init_worker_arms_reconciliation_round_when_jobs_exist(void) { setup_test_cluster(); job_t* j = seed_test_job(96003, JOB_STAT_RUN); TEST_ASSERT_NOT_NULL(j); j->job.allocs = make_test_job_alloc(96003, "arm-host-4-first-join"); TEST_ASSERT_NOT_NULL(j->job.allocs); hostattrs_t attrs; memset(&attrs, 0, sizeof(attrs)); attrs.hosttype = makeString("LINUX"); attrs.hostmodel = makeString("x86"); attrs.cpufactor = 1.0f; attrs.maxcpus = 4; attrs.maxmem = 16; attrs.maxswap = 32; attrs.maxtmp = 64; attrs.ndisks = 1; xce_hostconfig_t hconfig; memset(&hconfig, 0, sizeof(hconfig)); hconfig.hosttype = (char*)"!"; hconfig.hostmodel = (char*)"!"; hconfig.mxj = 4; endpoint_t ep; memset(&ep, 0, sizeof(ep)); int idx = init_worker(&g_tc.c, &ep, "arm-host-4-first-join", &attrs, &hconfig); TEST_ASSERT_GREATER_OR_EQUAL(0, idx); worker_t* w = NULL; lock(g_tc.c.workermgr, RTTI_LOCK_READ); key_find(g_tc.c.workermgr, (void*)"arm-host-4-first-join", (void**)&w); unlock(g_tc.c.workermgr); TEST_ASSERT_NOT_NULL(w); TEST_ASSERT_NOT_NULL_MESSAGE(w->resync, "a host's first-ever join must arm a reconciliation round exactly " "like a rejoin, with no special-casing needed (Req 1.3)"); TEST_ASSERT_EQUAL_INT(1, w->resync->n_queried_jobids); TEST_ASSERT_EQUAL_UINT64(96003, w->resync->queried_jobids[0]); teardown_test_cluster(); printf("test_init_worker_arms_reconciliation_round_when_jobs_exist: " "PASS\n"); } /* ── Task 6.1: worker_send_resync_query ───────────────────────────────── */ /* * When a round is armed for `w`, worker_send_resync_query() must send * exactly that round's job IDs as an OPCode_jobStateQuery. Req 2.1, 2.2. */ static void test_worker_send_resync_query_sends_armed_round(void) { setup_test_cluster(); job_t* j1 = seed_test_job(96010, JOB_STAT_RUN); j1->job.allocs = make_test_job_alloc(96010, "query-host-1"); job_t* j2 = seed_test_job(96011, JOB_STAT_RUN); j2->job.allocs = make_test_job_alloc(96011, "query-host-1"); hostattrs_t attrs; memset(&attrs, 0, sizeof(attrs)); attrs.hosttype = makeString("LINUX"); attrs.hostmodel = makeString("x86"); attrs.cpufactor = 1.0f; attrs.maxcpus = 4; attrs.maxmem = 16; attrs.maxswap = 32; attrs.maxtmp = 64; attrs.ndisks = 1; worker_t* w = make_test_worker("query-host-1", &attrs); TEST_ASSERT_NOT_NULL(w); hostattrs_t new_attrs = attrs; new_attrs.hosttype = makeString("LINUX"); new_attrs.hostmodel = makeString("x86"); xce_hostconfig_t hconfig; memset(&hconfig, 0, sizeof(hconfig)); hconfig.hosttype = (char*)"!"; hconfig.hostmodel = (char*)"!"; hconfig.mxj = 4; endpoint_t stack_ep; memset(&stack_ep, 0, sizeof(stack_ep)); rejoin_worker(&g_tc.c, w, &stack_ep, &new_attrs, &hconfig); TEST_ASSERT_NOT_NULL(w->resync); TEST_ASSERT_EQUAL_INT(2, w->resync->n_queried_jobids); endpoint_t* ep = new_test_endpoint(); TEST_ASSERT_EQUAL_INT(0, worker_send_resync_query(&g_tc.c, w, ep)); xce_request_t decoded; pop_and_decode(ep, &decoded); TEST_ASSERT_EQUAL_INT(OPCode_jobStateQuery, decoded.opcode); TEST_ASSERT_EQUAL_INT(Request_body_jobstatequerymsg, decoded.body_which); TEST_ASSERT_NOT_NULL(decoded.body.jobstatequerymsg); TEST_ASSERT_EQUAL_INT(2, decoded.body.jobstatequerymsg->n_total); int saw_96010 = 0, saw_96011 = 0; for (int i = 0; i < decoded.body.jobstatequerymsg->count; i++) { uint64_t id = decoded.body.jobstatequerymsg->jobids[i]; if (id == 96010) saw_96010 = 1; if (id == 96011) saw_96011 = 1; } TEST_ASSERT_TRUE_MESSAGE(saw_96010, "queried set must include 96010"); TEST_ASSERT_TRUE_MESSAGE(saw_96011, "queried set must include 96011"); xce_free_request(&decoded); assert_queue_empty(ep); dereference(new_attrs.hosttype); dereference(new_attrs.hostmodel); endpoint_fini(ep); free(ep); teardown_test_cluster(); printf("test_worker_send_resync_query_sends_armed_round: PASS\n"); } /* * When no round is armed (w->resync == NULL), worker_send_resync_query() * must send nothing at all -- the Req 1.2 no-round case's downstream * completion criterion ("no round, no query"). */ static void test_worker_send_resync_query_sends_nothing_when_no_round(void) { setup_test_cluster(); hostattrs_t attrs; memset(&attrs, 0, sizeof(attrs)); attrs.hosttype = makeString("LINUX"); attrs.hostmodel = makeString("x86"); attrs.cpufactor = 1.0f; attrs.maxcpus = 4; attrs.maxmem = 16; attrs.maxswap = 32; attrs.maxtmp = 64; attrs.ndisks = 1; worker_t* w = make_test_worker("query-host-2", &attrs); TEST_ASSERT_NOT_NULL(w); TEST_ASSERT_NULL(w->resync); endpoint_t* ep = new_test_endpoint(); TEST_ASSERT_EQUAL_INT(0, worker_send_resync_query(&g_tc.c, w, ep)); assert_queue_empty(ep); endpoint_fini(ep); free(ep); teardown_test_cluster(); printf("test_worker_send_resync_query_sends_nothing_when_no_round: " "PASS\n"); } /* ── Task 7.5: worker_send_resync_query genuinely paginates (bug-1755) ── * * Prior to task 7.5, worker_send_resync_query() hardcoded pktlimit=0 in its * xce_job_state_query_req() call -- xce_job_state_query_req()'s own * pktlimit<=0 convention (request.c) means "pktlimit = nrows", i.e. always * exactly ONE wire page, however many jobs are queried. Task 7.4's own test * (test_worker_resync.c, test_large_job_count_pagination_correctness) only * proved the underlying xce_job_state_query_req() pagination MECHANISM by * calling it directly with an explicit page-size override, bypassing this * production call site entirely (documented live as bug-1755: "effectively * dead code in production today because its only call site never supplies * a bound"). * * This test drives the REAL production call site -- worker_send_resync_ * query() itself, not a direct xce_job_state_query_req() call -- with a * non-terminal job count on the host that genuinely exceeds the new * XCE_RESYNC_QUERY_PKTLIMIT page cap (worker.c), and asserts the query is * observably split across multiple wire pages: more than one packet is * enqueued, each page's `pos` picks up exactly where the previous page's * `count` left off, every jobid appears in exactly one page, and every * page (including the final, deliberately partial one) carries the same * full n_total. Req 8.1-8.3. */ #define TWJ_RESYNC_QUERY_PKTLIMIT 100 /* mirrors production * XCE_RESYNC_QUERY_PKTLIMIT * (worker.c) -- not a real * production symbol reference (that * #define has file-local linkage), * kept in sync by this test's own * page-count assertion below, which * would fail loudly if the two ever * drifted apart */ #define TWJ_RESYNC_QUERY_N_JOBS 250 /* > TWJ_RESYNC_QUERY_PKTLIMIT, and not * a multiple of it, so the final wire * page is genuinely partial (50 * entries), not just N full pages */ #define TWJ_RESYNC_QUERY_JOBID_BASE 970000ULL #define TWJ_RESYNC_QUERY_N_PAGES \ ((TWJ_RESYNC_QUERY_N_JOBS + TWJ_RESYNC_QUERY_PKTLIMIT - 1) \ / TWJ_RESYNC_QUERY_PKTLIMIT) static void test_worker_send_resync_query_paginates_across_multiple_wire_pages(void) { setup_test_cluster(); hostattrs_t attrs; memset(&attrs, 0, sizeof(attrs)); attrs.hosttype = makeString("LINUX"); attrs.hostmodel = makeString("x86"); attrs.cpufactor = 1.0f; attrs.maxcpus = 4; attrs.maxmem = 16; attrs.maxswap = 32; attrs.maxtmp = 64; attrs.ndisks = 1; worker_t* w = make_test_worker("query-host-paginate", &attrs); TEST_ASSERT_NOT_NULL(w); for (int i = 0; i < TWJ_RESYNC_QUERY_N_JOBS; i++) { uint64_t jobid = TWJ_RESYNC_QUERY_JOBID_BASE + (uint64_t)i; job_t* j = seed_test_job(jobid, JOB_STAT_RUN); TEST_ASSERT_NOT_NULL(j); j->job.allocs = make_test_job_alloc(jobid, "query-host-paginate"); TEST_ASSERT_NOT_NULL(j->job.allocs); } hostattrs_t new_attrs = attrs; new_attrs.hosttype = makeString("LINUX"); new_attrs.hostmodel = makeString("x86"); xce_hostconfig_t hconfig; memset(&hconfig, 0, sizeof(hconfig)); hconfig.hosttype = (char*)"!"; hconfig.hostmodel = (char*)"!"; hconfig.mxj = 4; endpoint_t stack_ep; memset(&stack_ep, 0, sizeof(stack_ep)); rejoin_worker(&g_tc.c, w, &stack_ep, &new_attrs, &hconfig); TEST_ASSERT_NOT_NULL(w->resync); TEST_ASSERT_EQUAL_INT(TWJ_RESYNC_QUERY_N_JOBS, w->resync->n_queried_jobids); endpoint_t* ep = new_test_endpoint(); /* The REAL production call site -- no xce_job_state_query_req() * bypass. */ TEST_ASSERT_EQUAL_INT(0, worker_send_resync_query(&g_tc.c, w, ep)); { bool seen[TWJ_RESYNC_QUERY_N_JOBS]; int running_total = 0; int pages = 0; memset(seen, 0, sizeof(seen)); while (running_total < TWJ_RESYNC_QUERY_N_JOBS) { xce_request_t decoded; int j; pop_and_decode(ep, &decoded); TEST_ASSERT_EQUAL_INT(OPCode_jobStateQuery, decoded.opcode); TEST_ASSERT_EQUAL_INT(Request_body_jobstatequerymsg, decoded.body_which); TEST_ASSERT_NOT_NULL(decoded.body.jobstatequerymsg); TEST_ASSERT_EQUAL_INT_MESSAGE(TWJ_RESYNC_QUERY_N_JOBS, decoded.body.jobstatequerymsg->n_total, "every page must carry the same full n_total, not a " "per-page total (Req 2.1)"); TEST_ASSERT_EQUAL_INT_MESSAGE(running_total, decoded.body.jobstatequerymsg->pos, "each page's pos must pick up exactly where the previous " "page's count left off -- no gap, no overlap (Req 8.3)"); TEST_ASSERT_TRUE_MESSAGE( decoded.body.jobstatequerymsg->count <= TWJ_RESYNC_QUERY_PKTLIMIT, "no single page may exceed the configured page cap " "(Req 8.3)"); for (j = 0; j < decoded.body.jobstatequerymsg->count; j++) { uint64_t jobid = decoded.body.jobstatequerymsg->jobids[j]; uint64_t idx = jobid - TWJ_RESYNC_QUERY_JOBID_BASE; TEST_ASSERT_TRUE_MESSAGE( idx < (uint64_t)TWJ_RESYNC_QUERY_N_JOBS, "a jobid appearing on the wire must be one of the " "armed round's own jobids"); TEST_ASSERT_FALSE_MESSAGE(seen[idx], "no jobid may be sent twice across pages (Req 8.3)"); seen[idx] = true; } running_total += decoded.body.jobstatequerymsg->count; pages++; xce_free_request(&decoded); } TEST_ASSERT_EQUAL_INT_MESSAGE(TWJ_RESYNC_QUERY_N_PAGES, pages, "Req 8.1-8.3: a job count exceeding the real production page " "cap must genuinely span multiple wire pages -- observably, " "via the REAL worker_send_resync_query() call site, not just " "theoretically supported by xce_job_state_query_req()'s own " "pagination loop (bug-1755, closed by task 7.5)"); TEST_ASSERT_GREATER_THAN_INT_MESSAGE(1, pages, "a single collapsed page here would mean the hardcoded-0 " "regression (bug-1755) is back"); for (int i = 0; i < TWJ_RESYNC_QUERY_N_JOBS; i++) { TEST_ASSERT_TRUE_MESSAGE(seen[i], "every armed jobid must appear exactly once across all " "pages -- none silently dropped past a page boundary " "(Req 8.3)"); } } assert_queue_empty(ep); dereference(new_attrs.hosttype); dereference(new_attrs.hostmodel); endpoint_fini(ep); free(ep); teardown_test_cluster(); printf("test_worker_send_resync_query_paginates_across_multiple_wire_" "pages: PASS\n"); } /* ── Task 6.1: config push, then query, in that fixed order ──────────── */ /* * design.md's File Structure Plan for on_worker_join is explicit that this * ordering is load-bearing ("config push, then query", "not as an * extension of the join call itself"), and this spec's own Implementation * Notes (task 4.3) warn that a placement-sensitive guarantee needs a * STRUCTURAL assertion, not just an end-state one -- a behavioral test * alone can pass even with the wrong order if it only checks that both * packets eventually arrive. This test drives the exact two calls on_ * worker_join makes, in the exact order it makes them, and asserts the * WIRE ORDER of the resulting packets directly. */ static void test_config_push_then_resync_query_wire_order(void) { setup_test_cluster(); cluster_config_t cc; memset(&cc, 0, sizeof(cc)); cc.clustername = (char*)"test-cluster"; myconf_t myconf; memset(&myconf, 0, sizeof(myconf)); myconf.cluster_config = &cc; g_tc.c.myconf = &myconf; job_t* j = seed_test_job(96020, JOB_STAT_RUN); j->job.allocs = make_test_job_alloc(96020, "order-host-1"); hostattrs_t attrs; memset(&attrs, 0, sizeof(attrs)); attrs.hosttype = makeString("LINUX"); attrs.hostmodel = makeString("x86"); attrs.cpufactor = 1.0f; attrs.maxcpus = 4; attrs.maxmem = 16; attrs.maxswap = 32; attrs.maxtmp = 64; attrs.ndisks = 1; worker_t* w = make_test_worker("order-host-1", &attrs); TEST_ASSERT_NOT_NULL(w); hostattrs_t new_attrs = attrs; new_attrs.hosttype = makeString("LINUX"); new_attrs.hostmodel = makeString("x86"); xce_hostconfig_t hconfig; memset(&hconfig, 0, sizeof(hconfig)); hconfig.hosttype = (char*)"!"; hconfig.hostmodel = (char*)"!"; hconfig.mxj = 4; endpoint_t stack_ep; memset(&stack_ep, 0, sizeof(stack_ep)); rejoin_worker(&g_tc.c, w, &stack_ep, &new_attrs, &hconfig); TEST_ASSERT_NOT_NULL(w->resync); endpoint_t* ep = new_test_endpoint(); /* The exact two calls on_worker_join() makes, in the exact order. */ worker_push_config(&g_tc.c, ep); worker_send_resync_query(&g_tc.c, w, ep); xce_request_t first; pop_and_decode(ep, &first); TEST_ASSERT_EQUAL_INT_MESSAGE(OPCode_push, first.opcode, "the config push must be the FIRST packet on the wire"); xce_free_request(&first); xce_request_t second; pop_and_decode(ep, &second); TEST_ASSERT_EQUAL_INT_MESSAGE(OPCode_jobStateQuery, second.opcode, "the reconciliation query must be the SECOND packet on the wire, " "strictly after the config push"); xce_free_request(&second); assert_queue_empty(ep); dereference(new_attrs.hosttype); dereference(new_attrs.hostmodel); endpoint_fini(ep); free(ep); teardown_test_cluster(); printf("test_config_push_then_resync_query_wire_order: PASS\n"); } /* * Req 3.7 (config-delivery gap): worker_push_config() must carry the * REAL, currently-configured workerResyncWindowMs value -- not a * hardcoded constant. Two distinct configured values must decode back to * two distinct wire values. */ static void test_worker_push_config_carries_configured_resync_window(void) { setup_test_cluster(); cluster_config_t cc; memset(&cc, 0, sizeof(cc)); cc.clustername = (char*)"test-cluster"; cc.worker_resync_window_ms = 424242; myconf_t myconf; memset(&myconf, 0, sizeof(myconf)); myconf.cluster_config = &cc; g_tc.c.myconf = &myconf; endpoint_t* ep = new_test_endpoint(); TEST_ASSERT_EQUAL_INT(0, worker_push_config(&g_tc.c, ep)); xce_request_t decoded; pop_and_decode(ep, &decoded); TEST_ASSERT_EQUAL_INT(OPCode_push, decoded.opcode); TEST_ASSERT_EQUAL_INT(Request_body_configpushmsg, decoded.body_which); TEST_ASSERT_NOT_NULL(decoded.body.configpushmsg); TEST_ASSERT_EQUAL_INT(ConfigData_data_workerconfig, decoded.body.configpushmsg->data_which); TEST_ASSERT_NOT_NULL(decoded.body.configpushmsg->data.conf4worker); TEST_ASSERT_EQUAL_INT_MESSAGE(424242, decoded.body.configpushmsg->data.conf4worker->worker_resync_window_ms, "the pushed value must be the real configured ClusterConfig." "workerResyncWindowMs, not a hardcoded constant"); xce_free_request(&decoded); assert_queue_empty(ep); endpoint_fini(ep); free(ep); /* A second, different configured value must decode back differently -- * confirms the field is genuinely read from config each call, not * memoized/hardcoded from the first call above. */ cc.worker_resync_window_ms = 7; ep = new_test_endpoint(); TEST_ASSERT_EQUAL_INT(0, worker_push_config(&g_tc.c, ep)); pop_and_decode(ep, &decoded); TEST_ASSERT_EQUAL_INT_MESSAGE(7, decoded.body.configpushmsg->data.conf4worker->worker_resync_window_ms, "a second, different configured value must be reflected on the " "wire, proving the field is read live, not cached"); xce_free_request(&decoded); endpoint_fini(ep); free(ep); teardown_test_cluster(); printf("test_worker_push_config_carries_configured_resync_window: " "PASS\n"); } /* ── Dispatch ───────────────────────────────────────────────────────── */ void test_worker_join(void) { UnitySetTestFile(__FILE__); printf("\n>> worker join refactor tests\n\n"); /* xstring_table_init must be called once before any makeString usage */ xstring_table_init(139); RUN_TEST(test_update_resvals_logs_actual_rc_not_stale_errno); RUN_TEST(test_update_resval_list_reports_only_applied_items); /* Task 1.1: apply_join_state (shared status/resource update) */ RUN_TEST(test_apply_join_state_clears_unavail); RUN_TEST(test_apply_join_state_applies_hosttype_override); RUN_TEST(test_apply_join_state_updates_resource_values); RUN_TEST(test_apply_join_state_mxj_negative_one_falls_back_to_maxcpus); /* Task 1.2: init_worker (first-time join) */ RUN_TEST(test_init_worker_creates_new_entry); RUN_TEST(test_init_worker_returns_correct_idx); /* Task 1.3: rejoin_worker (reconnect with admin-bit preservation) */ RUN_TEST(test_rejoin_worker_updates_attrs_and_ep); RUN_TEST(test_rejoin_worker_preserves_admin_bits); /* worker-rejoin-resync task 6.1: apply_join_state arms a * reconciliation round; worker_send_resync_query sends it; config * push then query in that fixed order; the workerResyncWindowMs * config-delivery gap. */ RUN_TEST(test_rejoin_worker_arms_reconciliation_round_when_jobs_exist); RUN_TEST(test_rejoin_worker_no_round_when_no_jobs); RUN_TEST(test_rejoin_worker_no_round_when_only_terminal_jobs); RUN_TEST(test_rejoin_worker_string_resvals_survive_outstanding_round); RUN_TEST(test_init_worker_arms_reconciliation_round_when_jobs_exist); RUN_TEST(test_worker_send_resync_query_sends_armed_round); RUN_TEST(test_worker_send_resync_query_sends_nothing_when_no_round); RUN_TEST( test_worker_send_resync_query_paginates_across_multiple_wire_pages); RUN_TEST(test_config_push_then_resync_query_wire_order); RUN_TEST(test_worker_push_config_carries_configured_resync_window); }