lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

merge: sync Rongsong's scheduler-framework (88 commits, f01ae353..c5b172c7)

Browse source 
Two-sided divergence from f01ae353 (2026-08-30): theirs +88 (worker
rejoin-resync reconciliation, terminal-state bug-1737/1738/1739/1764,
scheduler-sim verbosity, hermetic offline build, go mod vendor of Go deps
in apiserver -- 3375 pure-adds), ours +89 (job-finish robustness bug-1509
B1/B3, PEND-reason, worker reconnect resend, apiserver job control).

Conflict resolutions (kept both sides' fixes):
- controller_connector.c checking_connection(): combine theirs
  xcecd_resync_is_pending() rejoin deferral gate with ours
  resend_job_status() actual-state re-report on rejoin success.
- job.c: keep both unknown-finish registry (ours, bug-1509) and
  record_job_finish_fact_at (theirs, bug-1747); keep both
  mark_jobs_unknown_core family (ours) and record_job_reconcile_*
  (theirs). No double-notify: on_job_finish unchanged, theirs is
  resync-path-only.
- scheduler handler.c/test_handlers.c/meson: theirs schedExitJob
  exit_code branch (bug-1737) + ours jobDispatchFail success-ack no-op
  (bug-1512) + configpush enq_pend_update (bug-1513) both present; both
  test suites and REPL/sim targets wired.
- apiserver jobs.go: keep ours normalizeJobInt64/jobArrayIdx superset +
  JobIDHuman; drop theirs strict IntOrInt64 early-reject (superseded --
  ours tolerates float64/json.Number, no panic path).
- jobs_test.go: keep both regression suites (ours bug-1490, theirs
  bug-1742).
- .kiro/specs scheduler-sim-*/worker-rejoin-resync + .wolf/*: keep our
  deletions (established convention -- drop Rongsong bookkeeping on merge).

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
lei.chen 2026-09-03 14:11:43 +00:00
commit 5e2985542f
3469 changed files with 1466599 additions and 382 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/controller/testsuite/controller_test.c
View file

@ -103,12 +103,14 @@ extern void test_upgrade_snapshot_restore(void);
extern void test_upgrade_snapshot_apply(void);
extern void test_bug857_regression(void);
extern void test_sched_admin_dispatch(void);
extern void test_jobstatereport_dispatch(void);
extern void test_fetchfile_disconnect(void);
extern void test_job_dispatch(void);
extern void test_dispatch_commit(void);
extern void test_dispatch_fail_terminal(void);
extern void test_dispatch_fail_retry(void);
extern void test_jobdispatch_no_double_notify(void);
extern void test_worker_resync(void);
void setUp(void) { return; }
@ -281,12 +283,14 @@ int main(int argc, char* argv[])
TEST(test_upgrade_snapshot_apply);
TEST(test_bug857_regression);
TEST(test_sched_admin_dispatch);
TEST(test_jobstatereport_dispatch);
TEST(test_fetchfile_disconnect);
TEST(test_job_dispatch);
TEST(test_dispatch_commit);
TEST(test_dispatch_fail_terminal);
TEST(test_dispatch_fail_retry);
TEST(test_jobdispatch_no_double_notify);
TEST(test_worker_resync);
return UNITY_END();
}

53
src/controller/testsuite/test_commit_barrier_rollout.c
View file

@ -982,6 +982,37 @@ static void test_gate_no_quorum_admits_clrpreason_fact(void)
printf("test_gate_no_quorum_admits_clrpreason_fact: PASS\n");
}
/* ── OPCode_jobStateReport admitted without write quorum ───────────────────
* (worker-rejoin-resync task 6.2, found live while auditing this exact gate
* for task 6.2's own standby-redirect investigation -- design.md only
* anticipated the STANDBY case, per bug-1700's precedent, and did not
* separately consider a PRIMARY that has temporarily lost write quorum.
* OPCode_jobStateReport is a worker's already-happened job-state fact, sent
* fire-and-forget (xce_job_state_report_req(), request.c) with no
* redelivery on rejection -- exactly the same shape as OPCode_notify, which
* this same allowlist already exists to protect from bug-396's silent-loss
* failure mode. Rejecting a reconciliation report outright during a
* transient quorum-loss window would reproduce that exact loss mode for
* the specific facts this whole feature exists to recover (bug-1741). */
static void test_gate_no_quorum_admits_jobstatereport_fact(void)
{
int rc;
/* Arrange: quorum absent */
fact_gate_setup(3, 0);
/* Act */
rc = ha_dispatch_gate(&g_fg_ha, OPCode_jobStateReport);
/* Assert */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"OPCode_jobStateReport (worker reconciliation fact) must be "
"admitted without write quorum, exactly like OPCode_notify");
fact_gate_teardown();
printf("test_gate_no_quorum_admits_jobstatereport_fact: PASS\n");
}
/* ── PIN: request-shaped opcodes stay rejected without quorum (Req 3.2) ───
* The allowlist is FACTS ONLY: every request-shaped opcode (a requester
* awaits confirmation, served by the task-3 handler barrier) must keep
@ -1018,16 +1049,17 @@ static void test_gate_no_quorum_still_rejects_request_opcodes(void)
static void test_gate_with_quorum_admits_all_opcodes(void)
{
static const struct { int opcode; const char *name; } ops[] = {
{ OPCode_notify, "OPCode_notify" },
{ OPCode_update, "OPCode_update" },
{ OPCode_setpreason, "OPCode_setpreason" },
{ OPCode_clrpreason, "OPCode_clrpreason" },
{ OPCode_jobsub, "OPCode_jobsub" },
{ OPCode_jobcontrol, "OPCode_jobcontrol" },
{ OPCode_queuecontrol, "OPCode_queuecontrol" },
{ OPCode_hostcontrol, "OPCode_hostcontrol" },
{ OPCode_jobdispatch, "OPCode_jobdispatch" },
{ OPCode_join, "OPCode_join" },
{ OPCode_notify, "OPCode_notify" },
{ OPCode_update, "OPCode_update" },
{ OPCode_setpreason, "OPCode_setpreason" },
{ OPCode_clrpreason, "OPCode_clrpreason" },
{ OPCode_jobStateReport, "OPCode_jobStateReport" },
{ OPCode_jobsub, "OPCode_jobsub" },
{ OPCode_jobcontrol, "OPCode_jobcontrol" },
{ OPCode_queuecontrol, "OPCode_queuecontrol" },
{ OPCode_hostcontrol, "OPCode_hostcontrol" },
{ OPCode_jobdispatch, "OPCode_jobdispatch" },
{ OPCode_join, "OPCode_join" },
};
size_t i;
@ -2459,6 +2491,7 @@ void test_commit_barrier_rollout(void)
RUN_TEST(test_gate_no_quorum_admits_update_fact);
RUN_TEST(test_gate_no_quorum_admits_setpreason_fact);
RUN_TEST(test_gate_no_quorum_admits_clrpreason_fact);
RUN_TEST(test_gate_no_quorum_admits_jobstatereport_fact);
RUN_TEST(test_gate_no_quorum_still_rejects_request_opcodes);
RUN_TEST(test_gate_with_quorum_admits_all_opcodes);
/* Task 5.1 — config barrier: configAck arithmetic + upgrade failure

333
src/controller/testsuite/test_dispatch_fail_retry.c
View file

@ -1,4 +1,5 @@
/* test_dispatch_fail_retry.c -- scheduler-dispatch-commit task 3.3
/* test_dispatch_fail_retry.c -- scheduler-dispatch-commit task 3.3;
* extended by worker-rejoin-resync task 2.3
*
* Proves dispatch_a_job_() (job.c) now RESOLVES a retryable (transient/
* environmental) dispatch failure -- incrementing the consecutive-failure
@ -19,6 +20,25 @@
* - XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY (E_XCE_BAD_WORKER)
* - XCE_DISPATCH_FAIL_SITE_RPC_FAILURE (xce_startjob_req() rc<0)
*
* worker-rejoin-resync task 2.3 (Req 5.1, 5.4) split dispatch_a_job_()'s
* combined "(w->ep == NULL) || (w->host.status != 0)" check into two
* branches: a disconnected worker (w->ep == NULL) is now resolved
* separately and DIFFERENTLY from XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY
* above -- it bypasses xce_classify_dispatch_fail()/mark_job_fail()
* entirely and returns the job to PEND immediately via
* record_job_reconcile_pending_fact() (task 2.1), with no
* dispatch_fail_count increment and no retry. This is NOT one of the four
* RETRYABLE sites this file otherwise covers -- it is added here anyway
* (rather than to test_worker_resync.c) because it is fundamentally a
* dispatch_a_job_() behavior/regression test exercising the same
* combined-check split site SITE 3 below already covers, sharing this
* file's fixture/canary/notify-capture machinery. See the new tests below
* SITE 3. SITE 3 itself (XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY,
* w->host.status != 0) is updated to use a CONNECTED worker (w->ep != NULL)
* with a nonzero status bit -- it previously used an ep-left-NULL worker to
* reach the combined check, which now takes the new, separate branch
* instead.
*
* test_worker_notfound_resolves_retryable (below) seeds an unregistered
* worker key, matching test_dispatch_commit.c's own established fixture
* for the "worker not found" scenario -- that file's own header notes this
@ -253,6 +273,28 @@ static void dfr_assert_ref_not_over_released(job_t* j, uint64_t expect_jobid)
"heap-use-after-free");
}
/* worker-rejoin-resync task 2.3: same canary technique as above, applied to
* `w` -- the new disconnected-worker branch dereference(w)s the reference
* dispatch_a_job_() itself took at its key_find() lookup, and this is the
* exact class of reference bug bug-1627 fixed (leak site #3, the WORKER_
* NOT_READY/ep-NULL combined check). */
static void dfr_arm_worker_canary(worker_t* w)
{
reference(w);
}
static void dfr_assert_worker_ref_not_over_released(worker_t* w,
const char* expect_name)
{
dereference(w);
TEST_ASSERT_EQUAL_STRING_MESSAGE(expect_name, w->host.name,
"w must still be safely readable after releasing the canary "
"reference -- an over-release inside dispatch_a_job_test() would "
"have freed w while the canary was still held, making this a "
"heap-use-after-free");
}
/* A "broken" endpoint: real socket fd, real iomgr, deliberately NOT
* registered in the iomgr's epoll -- EPOLL_CTL_MOD on it fails with ENOENT
* (the exact post-failed-reconnect condition endpoint_update_epoll()
@ -368,13 +410,25 @@ static void test_worker_notfound_resolves_retryable(void)
}
/* ── SITE 3: XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY ───────────────────
* A registered worker whose ep is left NULL (calloc default) hits
* "(w->ep == NULL) || (w->host.status != 0)". */
* A registered, CONNECTED worker (w->ep != NULL) with a nonzero
* w->host.status bit (XCE_WORKER_BUSY, picked arbitrarily -- any bit takes
* the same unchanged path) hits "w->host.status != 0" -- the still-
* unmodified branch of dispatch_a_job_()'s split combined check
* (worker-rejoin-resync task 2.3). Before that split, this test used an
* ep-left-NULL worker to reach the combined check; that condition now
* takes the NEW, separate disconnected-worker branch instead (see the new
* tests below), which resolves differently (immediate PEND, no retry, no
* dispatch_fail_count increment) -- so this test is updated to use a
* connected-but-busy worker, keeping it a genuine, unchanged regression
* proof for XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY's own retry-then-
* terminal-fail behavior (Req 5.1's design.md backstop). */
#define DFR_JOBID_WORKER_NOTREADY 91202ULL
static void test_worker_notready_resolves_retryable(void)
{
job_t* j;
worker_t* w = NULL;
endpoint_t connected_ep;
xce_job_alloc_t* alloc;
int rc;
int before_count;
@ -382,7 +436,18 @@ static void test_worker_notready_resolves_retryable(void)
dfr_setup();
j = dfr_seed_job(DFR_JOBID_WORKER_NOTREADY, "dfr-q2");
dfr_seed_worker("dfr-wkr-notready", NULL); /* ep left NULL: not ready */
/* A non-NULL ep (its contents are never touched -- this branch fails
* before any RPC is attempted) plus a nonzero status bit: connected,
* but not ready. */
memset(&connected_ep, 0, sizeof(connected_ep));
dfr_seed_worker("dfr-wkr-notready", &connected_ep);
lock(g_dfr.workermgr, RTTI_LOCK_READ);
key_find(g_dfr.workermgr, (void*)"dfr-wkr-notready", (void**)&w);
unlock(g_dfr.workermgr);
TEST_ASSERT_NOT_NULL(w);
w->host.status = XCE_WORKER_BUSY;
alloc = make_job_alloc(DFR_JOBID_WORKER_NOTREADY, 1,
make_host_allocs(1,
@ -391,6 +456,13 @@ static void test_worker_notready_resolves_retryable(void)
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(alloc);
/* worker-rejoin-resync task 2.3: explicit proof this test still reaches
* XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY's branch of the split check --
* a connected (non-NULL ep), nonzero-status worker, not the new
* disconnected-worker branch below. */
TEST_ASSERT_NOT_NULL(w->ep);
TEST_ASSERT_TRUE(w->host.status != 0);
cbrp_notify_capture_reset_ext();
before_count = cbrp_notify_capture_count_ext();
@ -407,7 +479,8 @@ static void test_worker_notready_resolves_retryable(void)
"Req 4.4: no partial allocation data may be left behind");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(1, j->dispatch_fail_count,
"Req 4.3: the consecutive-failure counter must increment by "
"exactly 1");
"exactly 1 -- unchanged existing WORKER_NOT_READY retry behavior "
"(worker-rejoin-resync task 2.3 must not affect this branch)");
dfr_assert_ref_not_over_released(j, DFR_JOBID_WORKER_NOTREADY);
@ -426,6 +499,250 @@ static void test_worker_notready_resolves_retryable(void)
printf("test_worker_notready_resolves_retryable: PASS\n");
}
/* ── worker-rejoin-resync task 2.3: disconnected-worker branch ─────────
* (Req 5.1, 5.4) -- NOT one of the four RETRYABLE sites above: a dispatch
* attempt to a worker the controller is not currently connected to
* (w->ep == NULL) now fails immediately and returns the job to PEND via
* record_job_reconcile_pending_fact(), bypassing xce_classify_dispatch_
* fail()/mark_job_fail() entirely -- no dispatch_fail_count increment, no
* retry, no terminal resolution.
*
* Realistic pre-dispatch state: dispatch_a_job_()'s only real caller
* dispatches a job the scheduler picked from its PEND pool, so this test
* seeds an ordinary PEND job (dfr_seed_job's default) -- record_job_
* reconcile_pending_fact()'s apply_job_update_transition(job, JOB_STAT_
* PEND) call therefore hits JOB_TRANSITION_NOOP_DUPLICATE (job already
* PEND), not APPLIED: no mutation, no allocs touch (there is none to
* release), no durable fact, no notify -- exactly as design.md's own
* Components section describes for this outcome, and as this task's own
* prompt anticipates ("handle this correctly: still don't call mark_job_
* fail, still don't retry, but there's nothing to release/notify either
* since nothing changed"). */
#define DFR_JOBID_DISCONNECTED 91206ULL
static int g_dfr_update_hook_fires;
static int dfr_job_update_spy_hook(int type, int situation, int detail,
void* data, va_list ap)
{
(void)type; (void)data; (void)ap;
if (situation == HOOK_AFTER_EVENT && detail == HOOK_DETAIL_JOB_UPDATE)
g_dfr_update_hook_fires++;
return 0;
}
static void test_disconnected_worker_fails_immediately_no_retry(void)
{
job_t* j;
worker_t* w = NULL;
xce_job_alloc_t* alloc;
int rc;
int before_count;
uint32_t fail_count_before;
dfr_setup();
g_dfr_update_hook_fires = 0;
TEST_ASSERT_EQUAL_INT(0,
xce_add_hook(&g_dfr.hooksys, XCE_CONTROLLER_HOOK_JOB,
dfr_job_update_spy_hook, NULL));
j = dfr_seed_job(DFR_JOBID_DISCONNECTED, "dfr-q-disc");
fail_count_before = j->dispatch_fail_count;
dfr_seed_worker("dfr-wkr-disc1", NULL); /* ep left NULL: disconnected */
lock(g_dfr.workermgr, RTTI_LOCK_READ);
key_find(g_dfr.workermgr, (void*)"dfr-wkr-disc1", (void**)&w);
unlock(g_dfr.workermgr);
TEST_ASSERT_NOT_NULL(w);
TEST_ASSERT_NULL_MESSAGE(w->ep, "the disconnected branch requires "
"w->ep == NULL -- confirm the fixture actually constructs that");
alloc = make_job_alloc(DFR_JOBID_DISCONNECTED, 1,
make_host_allocs(1,
make_host_alloc((char*)"dfr-wkr-disc1", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(alloc);
cbrp_notify_capture_reset_ext();
before_count = cbrp_notify_capture_count_ext();
/* No reference/leak regression (task 2.3's own Tests-to-write clause):
* canary both j (bug-1627's original leak class) and w (this branch's
* own dereference(w), the same site bug-1627 fixed for the WORKER_NOT_
* READY branch). */
dfr_arm_ref_canary(j);
dfr_arm_worker_canary(w);
rc = dispatch_a_job_test(&g_dfr, NULL, alloc);
TEST_ASSERT_TRUE_MESSAGE(rc < 0,
"a dispatch to a disconnected worker must still fail");
/* Observable completion (Req 5.4): job ends up PEND (unchanged -- it
* started PEND, a realistic pre-dispatch state), single attempt, no
* retry loop. */
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state,
"Req 5.4: a dispatch to a disconnected worker must return the job "
"to pending (here: leave it PEND, since it started PEND)");
TEST_ASSERT_NOT_EQUAL_MESSAGE(JOB_STAT_EXIT, j->job.state,
"Req 5.4: this must NOT become a permanent/terminal failure -- "
"mark_job_fail() must never be invoked on this path");
TEST_ASSERT_NULL_MESSAGE(j->job.allocs,
"no partial allocation data may be left behind");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(fail_count_before, j->dispatch_fail_count,
"Req 5.4: dispatch_fail_count must be left UNCHANGED -- this "
"branch must never reach xce_classify_dispatch_fail()'s retry-"
"counting machinery");
/* "single attempt, no retry loop": notify_dispatch() is dispatch_a_
* job_()'s sole retry-signaling mechanism (see the four RETRYABLE-site
* tests above) -- it must never fire on this path. */
TEST_ASSERT_EQUAL_INT_MESSAGE(before_count, cbrp_notify_capture_count_ext(),
"Req 5.4: a disconnected-worker dispatch failure must never call "
"notify_dispatch() -- there is no retry to signal");
/* NOOP_DUPLICATE outcome (job was already PEND): no mutation occurred,
* so no notify is needed or expected either -- omitting the notify
* here is correct, not bug-1679's omission-is-a-bug case (that applies
* only to an actual APPLIED transition; see the APPLIED-path test
* below for that proof). */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_dfr_update_hook_fires,
"a no-op reconciliation (job already PEND) must not fire the "
"scheduler notify path -- nothing changed");
dfr_assert_ref_not_over_released(j, DFR_JOBID_DISCONNECTED);
dfr_assert_worker_ref_not_over_released(w, "dfr-wkr-disc1");
xce_disable_hook(&g_dfr.hooksys, XCE_CONTROLLER_HOOK_JOB,
dfr_job_update_spy_hook);
free_JobAlloc_ptr(&alloc);
dfr_teardown();
printf("test_disconnected_worker_fails_immediately_no_retry: PASS\n");
}
/* ── worker-rejoin-resync task 2.3: JOB_TRANSITION_APPLIED wiring proof
* ────────────────────────────────────────────────────────────────────
* The NOOP_DUPLICATE test above exercises the only precondition dispatch_
* a_job_()'s real caller can currently construct (a PEND job -- the
* scheduler only ever dispatches from its PEND pool, so dispatch_a_job_()
* is never actually invoked on a RUN job today; the same "not reachable
* via any real caller in the currently-implemented system" caveat
* test_rpc_failure_bound_resets_after_intervening_success's header
* documents for the four RETRYABLE sites applies here too -- see bug-1631).
* This test instead directly seeds a RUN job with a real, committed
* job->job.allocs (mirroring test_worker_resync.c's own technique) to
* exercise record_job_reconcile_pending_fact()'s OTHER outcome
* (JOB_TRANSITION_APPLIED, not NOOP_DUPLICATE) THROUGH this exact new
* dispatch_a_job_() call site -- proving the branch's result-handling
* (unlock -> notify-only-on-APPLIED -> dereference) is wired correctly,
* complementing task 2.1's test_worker_resync.c (which already proves
* record_job_reconcile_pending_fact() itself is correct in isolation, not
* through this caller). Not a claim that THIS exact precondition (RUN job
* hitting the disconnected branch) is reachable in production today --
* design.md's own bug-1631 narrative is that record_job_reconcile_pending_
* fact() releases allocs at the point of RECONCILIATION (a separate,
* later-task call site, resync.c), so by the time any job reaches a fresh
* dispatch_a_job_() call it is already PEND with allocs already NULL. */
#define DFR_JOBID_DISCONNECTED_RUN 91207ULL
static uint64_t dfr_current_lsn(void)
{
uint64_t lsn;
pthread_mutex_lock(&g_dfr.persist.lock);
lsn = g_dfr.persist.current_lsn;
pthread_mutex_unlock(&g_dfr.persist.lock);
return lsn;
}
static void test_disconnected_worker_reconciles_run_job_releases_allocs(void)
{
job_t* j;
worker_t* w = NULL;
xce_job_alloc_t* prior_alloc;
xce_job_alloc_t* redispatch_alloc;
int rc;
int before_count;
uint64_t lsn_before, lsn_after;
dfr_setup();
g_dfr_update_hook_fires = 0;
TEST_ASSERT_EQUAL_INT(0,
xce_add_hook(&g_dfr.hooksys, XCE_CONTROLLER_HOOK_JOB,
dfr_job_update_spy_hook, NULL));
j = dfr_seed_job(DFR_JOBID_DISCONNECTED_RUN, "dfr-q-disc2");
/* Simulate a prior successful dispatch (mirrors test_worker_resync.c's
* own technique): real allocs committed, state RUN, ownership
* transferred into job->job.allocs. */
prior_alloc = make_job_alloc(DFR_JOBID_DISCONNECTED_RUN, 1,
make_host_allocs(1,
make_host_alloc((char*)"dfr-wkr-disc2-prior", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(prior_alloc);
j->job.state = JOB_STAT_RUN;
j->job.allocs = prior_alloc;
dfr_seed_worker("dfr-wkr-disc2", NULL); /* ep left NULL: disconnected */
lock(g_dfr.workermgr, RTTI_LOCK_READ);
key_find(g_dfr.workermgr, (void*)"dfr-wkr-disc2", (void**)&w);
unlock(g_dfr.workermgr);
TEST_ASSERT_NOT_NULL(w);
redispatch_alloc = make_job_alloc(DFR_JOBID_DISCONNECTED_RUN, 1,
make_host_allocs(1,
make_host_alloc((char*)"dfr-wkr-disc2", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(redispatch_alloc);
cbrp_notify_capture_reset_ext();
before_count = cbrp_notify_capture_count_ext();
lsn_before = dfr_current_lsn();
dfr_arm_ref_canary(j);
dfr_arm_worker_canary(w);
rc = dispatch_a_job_test(&g_dfr, NULL, redispatch_alloc);
TEST_ASSERT_TRUE_MESSAGE(rc < 0,
"a dispatch to a disconnected worker must still fail");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state,
"Req 4.6/5.4: a RUN job redispatched to a disconnected worker must "
"be returned to PEND");
TEST_ASSERT_NULL_MESSAGE(j->job.allocs,
"bug-1631: the job's prior resource allocation must be released, "
"so a later redispatch never inherits a stale allocation");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, j->dispatch_fail_count,
"Req 5.4: dispatch_fail_count must stay untouched on this path");
TEST_ASSERT_EQUAL_INT_MESSAGE(before_count, cbrp_notify_capture_count_ext(),
"notify_dispatch() (the retry-signal path) must never fire here");
lsn_after = dfr_current_lsn();
TEST_ASSERT_TRUE_MESSAGE(lsn_after > lsn_before,
"an APPLIED reconciliation must durably append a WAL_JOB_UPDATE "
"fact");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_dfr_update_hook_fires,
"bug-1679: an APPLIED reconciliation must notify the scheduler "
"exactly once, via this exact dispatch_a_job_() call site");
dfr_assert_ref_not_over_released(j, DFR_JOBID_DISCONNECTED_RUN);
dfr_assert_worker_ref_not_over_released(w, "dfr-wkr-disc2");
xce_disable_hook(&g_dfr.hooksys, XCE_CONTROLLER_HOOK_JOB,
dfr_job_update_spy_hook);
free_JobAlloc_ptr(&redispatch_alloc);
dfr_teardown();
printf("test_disconnected_worker_reconciles_run_job_releases_allocs: "
"PASS\n");
}
/* ── SITE 4: XCE_DISPATCH_FAIL_SITE_RPC_FAILURE ────────────────────────
* A ready worker whose ep is deliberately "broken" (see dfr_setup_broken_ep
* above) makes xce_startjob_req() itself fail. */
@ -814,4 +1131,10 @@ void test_dispatch_fail_retry(void)
RUN_TEST(test_rpc_failure_resolves_retryable);
RUN_TEST(test_rpc_failure_bounded_escalation);
RUN_TEST(test_rpc_failure_bound_resets_after_intervening_success);
printf("\n>> dispatch_a_job_() disconnected-worker fail-fast tests "
"(worker-rejoin-resync task 2.3, Req 5.1, 5.4)\n\n");
RUN_TEST(test_disconnected_worker_fails_immediately_no_retry);
RUN_TEST(test_disconnected_worker_reconciles_run_job_releases_allocs);
}

322
src/controller/testsuite/test_jobstatereport_dispatch.c Normal file
View file

@ -0,0 +1,322 @@
/*
* test_jobstatereport_dispatch.c — Unity tests for the OPCode_jobStateReport
* controller dispatch case (worker-rejoin-resync spec, task 6.2, Req 8.1,
* 8.3).
*
* Exercises myreq_handler's OPCode_jobStateReport case through the real
* decode+dispatch entry point exposed for tests, myreq_handler_for_test
* (cluster.c, #ifdef XCE_TESTING) — same precedent test_sched_admin_
* dispatch.c already established for a comparable primary-only opcode.
*
* Covers task 6.2's own explicit standby-redirect investigation: a report
* (or, by the same generic mechanism, a query) sent to a non-primary
* controller must receive the SAME redirect behavior an equivalent
* existing primary-only message already receives, not a bare unrecoverable
* error — proven here exactly as test_sched_admin_dispatch.c's own
* test_schedadmin_standby_with_known_primary_redirects/test_schedadmin_
* standby_no_known_primary_no_redirect prove it for OPCode_schedAdmin, the
* "real existing primary-only opcode" ground truth this task's brief
* calls for. OPCode_jobStateReport is deliberately absent from ha_
* dispatch_gate()'s CTRL_STANDBY allow-list (ha.c) — exactly like OPCode_
* schedAdmin/OPCode_upgradeConfig — so this redirect behavior requires NO
* opcode-specific code in cluster.c's own dispatch case: it is entirely a
* property of the GENERIC gate-rejection block that already runs ahead of
* every opcode's switch case (cluster.c, the bug-1700 fix). These tests
* prove that generic mechanism actually covers this opcode too, rather
* than assuming it does.
*
* The OTHER half of task 6.2's own brief — multi-page reassembly, applied
* exactly once, equivalent to a single-page report — is proven against a
* REAL worker_t and REAL job table in test_worker_resync.c (resync_handle_
* report_page()'s own direct tests), not re-proven here through the wire.
* See this file's own comment further down (immediately before the RUN_
* TEST list) for exactly why a "real wire dispatch resolves a real
* worker_t" version of that proof cannot be built in THIS test binary, and
* why that is a pre-existing, unrelated architectural property, not a gap
* in this task's own coverage.
*/
#define _GNU_SOURCE
#include <errno.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
#include "cluster.h"
#include "controller.capnp.h"
#include "ha.h"
#include "netio.h"
#include "request.h"
#include "typemap.h"
#include "xce/errnox.h"
#include "xce/types.h"
/* myreq_handler exposed for tests (cluster.c) — same seam test_sched_admin_
* dispatch.c uses. `pkt` ownership: consumed (freed) by the call. */
extern int myreq_handler_for_test(endpoint_t* ep, io_packet_t* pkt);
/* ── Fixture ────────────────────────────────────────────────────────────
*
* A bare calloc'd cluster_t, exactly like test_sched_admin_dispatch.c's
* sa_make_cluster() — sufficient for every test below, none of which
* reaches past ha_dispatch_gate() or handle_jobstatereport()'s own first
* line (get_ep_worker()).
*/
static cluster_t* jsr_make_gate_cluster(void)
{
cluster_t* c = calloc(1, sizeof(cluster_t));
TEST_ASSERT_NOT_NULL(c);
mylock_init(&(c->sched.lock));
return c;
}
static void jsr_gate_teardown(cluster_t* c)
{
if (c == NULL)
return;
mylock_fini(&(c->sched.lock));
free(c);
}
/* Build a wire-encoded OPCode_jobStateReport request packet carrying a
* minimal single-entry, single-page report — the exact content is
* immaterial for the standby-redirect tests below (the request never
* reaches the dispatch case at all). */
static io_packet_t* jsr_build_request_pkt(uint64_t jobid)
{
xce_job_state_entry_t entry;
xce_job_state_entry_t* entries[1];
xce_job_state_report_t body;
memset(&entry, 0, sizeof(entry));
entry.jobid = jobid;
entry.status = JobStateStatus_running;
entry.queried = 1;
entries[0] = &entry;
memset(&body, 0, sizeof(body));
body.n_total = 1;
body.pos = 0;
body.count = 1;
body.entries = entries;
xce_request_t req;
memset(&req, 0, sizeof(req));
req.opcode = OPCode_jobStateReport;
req.body_which = Request_body_jobstatereportmsg;
req.body.jobstatereportmsg = &body;
io_packet_t* pkt = calloc(1, sizeof(*pkt));
TEST_ASSERT_NOT_NULL(pkt);
TEST_ASSERT_TRUE_MESSAGE(xce_encode(&req, pkt) >= 0,
"failed to encode JobStateReport packet");
return pkt;
}
/* Pop the single reply packet myreq_handler queued on ep->outq (if any) and
* decode it into *out. Mirrors test_sched_admin_dispatch.c's sa_decode_
* queued_reply exactly. */
static void jsr_decode_queued_reply(endpoint_t* ep, xce_request_t* out)
{
data_item_t* it = endpoint_deq(ep);
TEST_ASSERT_NOT_NULL_MESSAGE(it, "expected a reply packet on ep->outq");
io_packet_t* pkt = ITPTR_TO_PACKET(it);
memset(out, 0, sizeof(*out));
int rc = xce_decode(pkt, out);
TEST_ASSERT_EQUAL_INT(0, rc);
}
/* ── Standby redirect (task 6.2's own explicit investigation) ─────────────
*
* Ground truth: OPCode_schedAdmin's identical fixture/assertions
* (test_sched_admin_dispatch.c), the "real existing primary-only opcode"
* this task's brief calls for. */
static void test_jobstatereport_rejects_when_not_primary(void)
{
cluster_t* c = jsr_make_gate_cluster();
atomic_store(&c->ha.role, CTRL_STANDBY);
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
ep.owner = c;
io_packet_t* pkt = jsr_build_request_pkt(1001);
int rc = myreq_handler_for_test(&ep, pkt);
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rc);
xce_request_t decoded;
jsr_decode_queued_reply(&ep, &decoded);
TEST_ASSERT_EQUAL_INT(Request_body_replymsg, decoded.body_which);
TEST_ASSERT_NOT_NULL(decoded.body.replymsg);
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, decoded.body.replymsg->status);
TEST_ASSERT_EQUAL_INT(RequestReply_data_nodata,
decoded.body.replymsg->data_which);
xce_free_request(&decoded);
jsr_gate_teardown(c);
printf("test_jobstatereport_rejects_when_not_primary: PASS\n");
}
/* bug-1700 precedent: a standby dispatch-gate rejection with a KNOWN
* primary must carry a redirect hint — proven here with NO opcode-specific
* code added anywhere for OPCode_jobStateReport: it works purely because
* this opcode is absent from ha_dispatch_gate's CTRL_STANDBY allow-list,
* so the GENERIC post-gate rejection block (cluster.c) fills the redirect
* for it exactly as it already does for OPCode_schedAdmin/OPCode_join. */
static void test_jobstatereport_standby_with_known_primary_redirects(void)
{
cluster_t* c = jsr_make_gate_cluster();
atomic_store(&c->ha.role, CTRL_STANDBY);
strncpy(c->ha.primary_host, "10.0.1.9", sizeof(c->ha.primary_host) - 1);
c->ha.primary_port = 7003;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
ep.owner = c;
io_packet_t* pkt = jsr_build_request_pkt(1002);
int rc = myreq_handler_for_test(&ep, pkt);
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rc);
xce_request_t decoded;
jsr_decode_queued_reply(&ep, &decoded);
TEST_ASSERT_EQUAL_INT(Request_body_replymsg, decoded.body_which);
xce_reply_t* rr = decoded.body.replymsg;
TEST_ASSERT_NOT_NULL(rr);
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rr->status);
TEST_ASSERT_NOT_NULL_MESSAGE(
rr->redirect_host,
"a standby dispatch-gate rejection with a known primary must carry "
"that primary's address, exactly like every other primary-only "
"opcode already does (bug-1700)");
TEST_ASSERT_EQUAL_STRING("10.0.1.9", rr->redirect_host);
TEST_ASSERT_EQUAL_UINT16(7003, rr->redirect_port);
xce_free_request(&decoded);
jsr_gate_teardown(c);
printf("test_jobstatereport_standby_with_known_primary_redirects:"
" PASS\n");
}
/* Regression companion: no known primary yet must not crash and must
* decode as a non-redirecting reply (empty host / zero port). */
static void test_jobstatereport_standby_no_known_primary_no_redirect(void)
{
cluster_t* c = jsr_make_gate_cluster();
atomic_store(&c->ha.role, CTRL_STANDBY);
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
ep.owner = c;
io_packet_t* pkt = jsr_build_request_pkt(1003);
int rc = myreq_handler_for_test(&ep, pkt);
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rc);
xce_request_t decoded;
jsr_decode_queued_reply(&ep, &decoded);
TEST_ASSERT_EQUAL_INT(Request_body_replymsg, decoded.body_which);
xce_reply_t* rr = decoded.body.replymsg;
TEST_ASSERT_NOT_NULL(rr);
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rr->status);
TEST_ASSERT_TRUE_MESSAGE(
rr->redirect_host == NULL || rr->redirect_host[0] == '\0',
"no known primary: redirect_host must decode as empty/absent");
TEST_ASSERT_EQUAL_UINT16(0, rr->redirect_port);
xce_free_request(&decoded);
jsr_gate_teardown(c);
printf("test_jobstatereport_standby_no_known_primary_no_redirect:"
" PASS\n");
}
/* A PRIMARY with write quorum handles the request normally — dispatched
* past the gate into handle_jobstatereport(), which returns -ENOENT for an
* endpoint that does not resolve to a known worker (this bare fixture's
* `ep` was never registered as one) rather than any standby-style
* rejection. Proves the gate's happy path is unaffected by this task's
* change and that the new dispatch case is genuinely reached. */
static void test_jobstatereport_primary_reaches_dispatch_case(void)
{
cluster_t* c = jsr_make_gate_cluster();
atomic_store(&c->ha.role, CTRL_PRIMARY);
/* n_peers defaults to 0 (calloc) -- single-node cluster, trivially has
* write quorum (ha_has_write_quorum: n_peers <= 1 -> true). */
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
ep.owner = c;
/* ep.data left NULL -> get_ep_role()/get_ep_worker() treat this as "not
* a worker endpoint" and return NULL/-EINVAL, exercising handle_
* jobstatereport's own not-a-worker branch. */
io_packet_t* pkt = jsr_build_request_pkt(1004);
int rc = myreq_handler_for_test(&ep, pkt);
TEST_ASSERT_EQUAL_INT_MESSAGE(-ENOENT, rc,
"a primary with quorum must reach handle_jobstatereport() itself "
"(not a gate rejection) -- which reports -ENOENT for an endpoint "
"with no resolvable worker_t");
jsr_gate_teardown(c);
printf("test_jobstatereport_primary_reaches_dispatch_case: PASS\n");
}
/* ── Why no "real wire dispatch resolves a real worker_t" test lives here ──
*
* A genuine end-to-end test (two real wire-encoded OPCode_jobStateReport
* packets through myreq_handler_for_test, resolving a REAL worker_t via
* get_ep_worker() and converging a REAL job table) was attempted here and
* found structurally impossible in THIS binary: get_ep_worker() (misc.c)
* is NOT one of controller_unit_test_src's linked sources (meson.build) --
* test_commit_barrier_reply.c instead provides controller_unit_test's ONLY
* definition of it, a fixed test double (`return g_stub_ep_worker;`,
* file-static, defaulting to NULL and settable only by that file's own
* tests) that no other test file can point at a real worker_t. Confirmed
* live (not assumed) by direct instrumentation: a page built with a
* correct, verified-by-hand ep_private_t{role=HostRole_worker, idx=w->idx}
* -- independently proven resolvable via key_find()/vget() called directly
* against the same workermgr -- still made get_ep_worker() return NULL,
* because the symbol actually linked into this binary is the stub, not
* misc.c's real implementation.
*
* This is a pre-existing architectural property of controller_unit_test
* (several other cluster.c-adjacent functions -- notify(), update_host_
* runusage(), on_scheduler_join(), add_apiserver(), on_apiserver_join() --
* are stubbed the same way in the same file, for the same reason), not
* something this task introduced or can fix without either modifying
* shared test infrastructure many other tests depend on (out of this
* task's boundary) or standing up a new dedicated test binary that links
* the real misc.c (the pattern host_runusage_unit_test/resval_typeresolve_
* unit_test already establish for exactly this class of problem, bug-1735)
* -- a real option, but a disproportionate scope expansion for what the
* unit tests below plus test_worker_resync.c's own tests already prove
* between them: handle_jobstatereport() (cluster.c) is a two-line function
* (get_ep_worker() then resync_handle_report_page()) whose own correctness
* is not in question -- the two things worth proving are (a) cluster.c's
* switch genuinely routes OPCode_jobStateReport to it, proven above by
* test_jobstatereport_primary_reaches_dispatch_case's distinctive -ENOENT
* return (get_ep_worker's stub-NULL path, reached only via this exact
* call chain), and (b) multi-page reassembly + exactly-once apply +
* equivalence-to-single-page, proven exhaustively against a REAL worker_t
* and REAL job table by resync_handle_report_page()'s own direct tests in
* test_worker_resync.c (test_handle_report_page_multi_page_reassembles_
* and_applies_once and its siblings) -- the same function handle_
* jobstatereport() calls, unmodified.
*/
void test_jobstatereport_dispatch(void)
{
UnitySetTestFile(__FILE__);
RUN_TEST(test_jobstatereport_rejects_when_not_primary);
RUN_TEST(test_jobstatereport_standby_with_known_primary_redirects);
RUN_TEST(test_jobstatereport_standby_no_known_primary_no_redirect);
RUN_TEST(test_jobstatereport_primary_reaches_dispatch_case);
}

817
src/controller/testsuite/test_worker_join.c
View file

@ -14,6 +14,7 @@
#include <rtti.h>
#include <errno.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@ -22,12 +23,26 @@
#include <unity/unity.h>
#include "cluster.h"
#include "controller.capnp.h" /* worker-rejoin-resync task 6.1: OPCode_push/
* OPCode_jobStateQuery, ConfigData_data_*,
* Request_body_* constants */
#include "jobj.h" /* worker-rejoin-resync task 6.1: makeJob, for a real
* non-terminal job the arm-round wiring tests need */
#include "kvmgr.h"
#include "netio.h" /* worker-rejoin-resync task 6.1: endpoint_get/
* endpoint_deq/ITPTR_TO_PACKET, for real wire capture */
#include "request.h" /* worker-rejoin-resync task 6.1: xce_decode,
* xce_free_request */
#include "resobj.h"
#include "resync.h" /* worker-rejoin-resync task 6.1: worker_t.resync,
* XCE_WORKER_RECONCILING */
#include "wobj.h"
#include "worker.h"
#include "xce/cluster_def.h"
#include "xce/types.h"
#include "xcealloc.h" /* worker-rejoin-resync task 6.1: make_job_alloc and
* friends, for a job assigned to a test worker's
* host */
#include "xstring.h"
#include "zarray.h"
@ -55,6 +70,19 @@ void collect_confvars(cluster_t* c, collect_check_fn_t fn, xlist_t* l)
int apply_join_state(cluster_t* c, worker_t* w, endpoint_t* ep,
xce_hostconfig_t* hconfig);
/*
* worker-rejoin-resync task 6.1: worker_push_config and worker_send_resync_
* query are non-static in worker.c when XCE_TESTING is defined, mirroring
* apply_join_state's own convention above -- lets these tests drive the
* exact config-push-then-query wire sequence on_worker_join produces,
* without needing a live join flow or get_ep_worker()'s endpoint->worker
* resolution (a separate, pre-existing concern -- see
* test_on_worker_join_sends_config_push_even_when_worker_unresolvable
* below for why).
*/
int worker_push_config(cluster_t* c, endpoint_t* ep);
int worker_send_resync_query(cluster_t* c, worker_t* w, endpoint_t* ep);
/*
* init_worker and rejoin_worker are exported from worker.c.
*
@ -84,10 +112,26 @@ typedef struct {
cluster_t c;
var workermgr;
var resmgr;
var jobmgr; /* worker-rejoin-resync task 6.1: real jobmgr so the
* apply_join_state arm-round wiring tests below can
* seed a genuine non-terminal job assigned to a test
* worker's host (mirrors test_worker_resync.c's own
* g_twr.jobmgr fixture) */
} test_cluster_t;
static test_cluster_t g_tc;
/* worker-rejoin-resync task 6.1: jobj.c's job type table is a shared,
* process-wide resource across every test file in the controller_unit_test
* binary -- job_obj_init() is explicitly documented (jobj.c) as safe to
* call more than once per process without an intervening job_obj_fini()
* (Req 10.2, the exact "every controller unit-test file that constructs
* jobs calls job_obj_init() from its own setup" scenario this file is now
* one more instance of), so no cross-file coordination is needed -- guard
* only avoids redundant re-init within THIS file's own repeated setup/
* teardown cycles, mirroring test_worker_resync.c's g_twr_jobobj_ready. */
static int g_tc_jobobj_ready;
static void setup_test_cluster(void)
{
kvmgr_params_t str_params = {
@ -96,15 +140,28 @@ static void setup_test_cluster(void)
.delta = 16,
.datakiller = NULL,
};
kvmgr_params_t int_params = {
.key_type = KVMGR_KEY_INT,
.nslots = 32,
.delta = 16,
};
memset(&g_tc, 0, sizeof(g_tc));
if (!g_tc_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
g_tc_jobobj_ready = 1;
}
g_tc.workermgr = makeInstance(&xkvmgr_type, &str_params);
TEST_ASSERT_NOT_NULL(g_tc.workermgr);
g_tc.resmgr = makeInstance(&xkvmgr_type, &str_params);
TEST_ASSERT_NOT_NULL(g_tc.resmgr);
g_tc.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(g_tc.jobmgr);
for (int i = 0; i < N_RES; i++) {
xce_resource_t rdef;
memset(&rdef, 0, sizeof(rdef));
@ -119,6 +176,18 @@ static void setup_test_cluster(void)
g_tc.c.workermgr = g_tc.workermgr;
g_tc.c.resmgr = g_tc.resmgr;
g_tc.c.jobmgr = g_tc.jobmgr;
/* worker-rejoin-resync task 6.1: c->myhost is the `from` peer identity
* worker_push_config()/worker_send_resync_query() pass to xce_
* pushconfig_req()/xce_job_state_query_req() -- both reject a NULL
* `from` outright, so any test exercising the real wire send needs
* this populated (mirrors production cluster.c's own myhost_info
* storage-then-point-at-it pattern, xce/cluster_def.h:101-102). */
g_tc.c.myhost_info.name = (char*)"test-controller";
g_tc.c.myhost_info.key = (char*)"test-key";
g_tc.c.myhost = &g_tc.c.myhost_info;
zarray_init(&g_tc.c.sharedinst, 64, NULL);
}
@ -127,9 +196,42 @@ static void teardown_test_cluster(void)
zarray_fini(&g_tc.c.sharedinst);
if (g_tc.resmgr) dereference(g_tc.resmgr);
if (g_tc.workermgr) dereference(g_tc.workermgr);
if (g_tc.jobmgr) dereference(g_tc.jobmgr);
memset(&g_tc, 0, sizeof(g_tc));
}
/* Seed a job directly in the fixture's jobmgr, bypassing dispatch/replay --
* mirrors test_worker_resync.c's twr_seed_job. */
static job_submit_data_t g_tc_seed_submit;
static job_t* seed_test_job(uint64_t jobid, uint32_t state)
{
xce_job_t jdef;
memset(&g_tc_seed_submit, 0, sizeof(g_tc_seed_submit));
g_tc_seed_submit.queue = (char*)"q";
g_tc_seed_submit.user = (char*)"tester";
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = jobid;
jdef.state = state;
jdef.submit = &g_tc_seed_submit;
return makeJob(g_tc.jobmgr, &jdef);
}
/* Build a single-host job allocation assigning `jobid` to `host` -- mirrors
* test_worker_resync.c's twr_make_test_alloc. Caller hands ownership to
* job->job.allocs. */
static xce_job_alloc_t* make_test_job_alloc(uint64_t jobid, const char* host)
{
return make_job_alloc(jobid, 1,
make_host_allocs(1,
make_host_alloc((char*)host, 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
}
/* Create a worker in the test workermgr; attrs strings must be makeString'd. */
static worker_t* make_test_worker(const char* name, hostattrs_t* attrs)
{
@ -691,6 +793,706 @@ static void test_rejoin_worker_preserves_admin_bits(void)
printf("test_rejoin_worker_preserves_admin_bits: PASS\n");
}
/* ── Task 6.1: apply_join_state arms a reconciliation round ──────────── */
/*
* Real (heap) endpoint_t via endpoint_get() -- NOT a hand-rolled/zeroed
* stack stand-in (bug-378: an under-sized/wrong-typed stand-in for
* endpoint_t caused a global-buffer-overflow when passed where a real
* endpoint_t* was expected). Needed only by the tests below that actually
* send something over the wire (worker_push_config/worker_send_resync_
* query); the pre-existing apply_join_state/init_worker/rejoin_worker
* tests above never send anything and so are unaffected, still using a
* plain zeroed stack endpoint_t. Mirrors src/lib/testsuite/
* test_job_state_wire.c's own established pattern for this exact
* situation.
*/
static endpoint_t* new_test_endpoint(void)
{
endpoint_t* ep = endpoint_get();
TEST_ASSERT_NOT_NULL(ep);
return ep;
}
/* Pop one queued page off ep->outq and decode it. Fails the test if the
* queue is empty. Caller must xce_free_request() the result. */
static void pop_and_decode(endpoint_t* ep, xce_request_t* out)
{
data_item_t* it = endpoint_deq(ep);
TEST_ASSERT_NOT_NULL(it);
io_packet_t* pkt = ITPTR_TO_PACKET(it);
memset(out, 0, sizeof(*out));
TEST_ASSERT_EQUAL_INT(0, xce_decode(pkt, out)); /* consumes/frees pkt */
}
static void assert_queue_empty(endpoint_t* ep)
{
TEST_ASSERT_NULL(endpoint_deq(ep));
}
/*
* rejoin_worker (via apply_join_state) must arm a reconciliation round --
* w->resync populated, XCE_WORKER_RECONCILING set, the query set matching
* exactly the host's non-terminal jobs -- when the controller currently
* holds one or more non-terminal jobs for that host. Req 1.1, 2.1, 2.2.
*/
static void test_rejoin_worker_arms_reconciliation_round_when_jobs_exist(void)
{
setup_test_cluster();
job_t* j = seed_test_job(96001, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs = make_test_job_alloc(96001, "arm-host-1");
TEST_ASSERT_NOT_NULL(j->job.allocs);
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
worker_t* w = make_test_worker("arm-host-1", &attrs);
TEST_ASSERT_NOT_NULL(w);
hostattrs_t new_attrs;
memset(&new_attrs, 0, sizeof(new_attrs));
new_attrs.hosttype = makeString("LINUX");
new_attrs.hostmodel = makeString("x86");
new_attrs.cpufactor = 1.0f;
new_attrs.maxcpus = 4;
new_attrs.maxmem = 16;
new_attrs.maxswap = 32;
new_attrs.maxtmp = 64;
new_attrs.ndisks = 1;
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
int idx = rejoin_worker(&g_tc.c, w, &ep, &new_attrs, &hconfig);
TEST_ASSERT_GREATER_OR_EQUAL(0, idx);
TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
"a worker rejoining with a known non-terminal job must have a "
"reconciliation round armed (Req 1.1)");
TEST_ASSERT_TRUE_MESSAGE(w->host.status & XCE_WORKER_RECONCILING,
"XCE_WORKER_RECONCILING must be set once a round is armed");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, w->resync->n_queried_jobids,
"the round's query set must match exactly the host's non-terminal "
"jobs (Req 2.1)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(96001, w->resync->queried_jobids[0],
"the queried job must be identified by its unique jobid (Req 2.2)");
dereference(new_attrs.hosttype);
dereference(new_attrs.hostmodel);
teardown_test_cluster();
printf("test_rejoin_worker_arms_reconciliation_round_when_jobs_exist: "
"PASS\n");
}
/*
* A worker rejoining with NO non-terminal jobs currently assigned to it
* must NOT have a round armed. Req 1.2.
*/
static void test_rejoin_worker_no_round_when_no_jobs(void)
{
setup_test_cluster();
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
worker_t* w = make_test_worker("arm-host-2", &attrs);
TEST_ASSERT_NOT_NULL(w);
hostattrs_t new_attrs = attrs;
new_attrs.hosttype = makeString("LINUX");
new_attrs.hostmodel = makeString("x86");
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
rejoin_worker(&g_tc.c, w, &ep, &new_attrs, &hconfig);
TEST_ASSERT_NULL_MESSAGE(w->resync,
"a worker rejoining with no non-terminal jobs must not have a "
"reconciliation round armed (Req 1.2)");
TEST_ASSERT_FALSE_MESSAGE(w->host.status & XCE_WORKER_RECONCILING,
"XCE_WORKER_RECONCILING must not be set");
dereference(new_attrs.hosttype);
dereference(new_attrs.hostmodel);
teardown_test_cluster();
printf("test_rejoin_worker_no_round_when_no_jobs: PASS\n");
}
/*
* A terminal job (DONE) assigned to the host must not be queried -- only
* non-terminal jobs count toward the trigger (Req 1.1's own "non-terminal"
* qualifier).
*/
static void test_rejoin_worker_no_round_when_only_terminal_jobs(void)
{
setup_test_cluster();
job_t* j = seed_test_job(96002, JOB_STAT_DONE);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs = make_test_job_alloc(96002, "arm-host-3");
TEST_ASSERT_NOT_NULL(j->job.allocs);
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
worker_t* w = make_test_worker("arm-host-3", &attrs);
TEST_ASSERT_NOT_NULL(w);
hostattrs_t new_attrs = attrs;
new_attrs.hosttype = makeString("LINUX");
new_attrs.hostmodel = makeString("x86");
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
rejoin_worker(&g_tc.c, w, &ep, &new_attrs, &hconfig);
TEST_ASSERT_NULL_MESSAGE(w->resync,
"a terminal (DONE) job must not count toward the reconciliation "
"trigger");
dereference(new_attrs.hosttype);
dereference(new_attrs.hostmodel);
teardown_test_cluster();
printf("test_rejoin_worker_no_round_when_only_terminal_jobs: PASS\n");
}
/*
* Req 1.3: the SAME trigger condition -- no special-casing needed -- must
* apply to a host's first-ever join (init_worker), not just a rejoin.
*/
static void test_init_worker_arms_reconciliation_round_when_jobs_exist(void)
{
setup_test_cluster();
job_t* j = seed_test_job(96003, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs = make_test_job_alloc(96003, "arm-host-4-first-join");
TEST_ASSERT_NOT_NULL(j->job.allocs);
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
int idx = init_worker(&g_tc.c, &ep, "arm-host-4-first-join", &attrs,
&hconfig);
TEST_ASSERT_GREATER_OR_EQUAL(0, idx);
worker_t* w = NULL;
lock(g_tc.c.workermgr, RTTI_LOCK_READ);
key_find(g_tc.c.workermgr, (void*)"arm-host-4-first-join", (void**)&w);
unlock(g_tc.c.workermgr);
TEST_ASSERT_NOT_NULL(w);
TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
"a host's first-ever join must arm a reconciliation round exactly "
"like a rejoin, with no special-casing needed (Req 1.3)");
TEST_ASSERT_EQUAL_INT(1, w->resync->n_queried_jobids);
TEST_ASSERT_EQUAL_UINT64(96003, w->resync->queried_jobids[0]);
teardown_test_cluster();
printf("test_init_worker_arms_reconciliation_round_when_jobs_exist: "
"PASS\n");
}
/* ── Task 6.1: worker_send_resync_query ───────────────────────────────── */
/*
* When a round is armed for `w`, worker_send_resync_query() must send
* exactly that round's job IDs as an OPCode_jobStateQuery. Req 2.1, 2.2.
*/
static void test_worker_send_resync_query_sends_armed_round(void)
{
setup_test_cluster();
job_t* j1 = seed_test_job(96010, JOB_STAT_RUN);
j1->job.allocs = make_test_job_alloc(96010, "query-host-1");
job_t* j2 = seed_test_job(96011, JOB_STAT_RUN);
j2->job.allocs = make_test_job_alloc(96011, "query-host-1");
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
worker_t* w = make_test_worker("query-host-1", &attrs);
TEST_ASSERT_NOT_NULL(w);
hostattrs_t new_attrs = attrs;
new_attrs.hosttype = makeString("LINUX");
new_attrs.hostmodel = makeString("x86");
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
endpoint_t stack_ep;
memset(&stack_ep, 0, sizeof(stack_ep));
rejoin_worker(&g_tc.c, w, &stack_ep, &new_attrs, &hconfig);
TEST_ASSERT_NOT_NULL(w->resync);
TEST_ASSERT_EQUAL_INT(2, w->resync->n_queried_jobids);
endpoint_t* ep = new_test_endpoint();
TEST_ASSERT_EQUAL_INT(0, worker_send_resync_query(&g_tc.c, w, ep));
xce_request_t decoded;
pop_and_decode(ep, &decoded);
TEST_ASSERT_EQUAL_INT(OPCode_jobStateQuery, decoded.opcode);
TEST_ASSERT_EQUAL_INT(Request_body_jobstatequerymsg, decoded.body_which);
TEST_ASSERT_NOT_NULL(decoded.body.jobstatequerymsg);
TEST_ASSERT_EQUAL_INT(2, decoded.body.jobstatequerymsg->n_total);
int saw_96010 = 0, saw_96011 = 0;
for (int i = 0; i < decoded.body.jobstatequerymsg->count; i++) {
uint64_t id = decoded.body.jobstatequerymsg->jobids[i];
if (id == 96010) saw_96010 = 1;
if (id == 96011) saw_96011 = 1;
}
TEST_ASSERT_TRUE_MESSAGE(saw_96010, "queried set must include 96010");
TEST_ASSERT_TRUE_MESSAGE(saw_96011, "queried set must include 96011");
xce_free_request(&decoded);
assert_queue_empty(ep);
dereference(new_attrs.hosttype);
dereference(new_attrs.hostmodel);
endpoint_fini(ep);
free(ep);
teardown_test_cluster();
printf("test_worker_send_resync_query_sends_armed_round: PASS\n");
}
/*
* When no round is armed (w->resync == NULL), worker_send_resync_query()
* must send nothing at all -- the Req 1.2 no-round case's downstream
* completion criterion ("no round, no query").
*/
static void test_worker_send_resync_query_sends_nothing_when_no_round(void)
{
setup_test_cluster();
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
worker_t* w = make_test_worker("query-host-2", &attrs);
TEST_ASSERT_NOT_NULL(w);
TEST_ASSERT_NULL(w->resync);
endpoint_t* ep = new_test_endpoint();
TEST_ASSERT_EQUAL_INT(0, worker_send_resync_query(&g_tc.c, w, ep));
assert_queue_empty(ep);
endpoint_fini(ep);
free(ep);
teardown_test_cluster();
printf("test_worker_send_resync_query_sends_nothing_when_no_round: "
"PASS\n");
}
/* ── Task 7.5: worker_send_resync_query genuinely paginates (bug-1755) ──
*
* Prior to task 7.5, worker_send_resync_query() hardcoded pktlimit=0 in its
* xce_job_state_query_req() call -- xce_job_state_query_req()'s own
* pktlimit<=0 convention (request.c) means "pktlimit = nrows", i.e. always
* exactly ONE wire page, however many jobs are queried. Task 7.4's own test
* (test_worker_resync.c, test_large_job_count_pagination_correctness) only
* proved the underlying xce_job_state_query_req() pagination MECHANISM by
* calling it directly with an explicit page-size override, bypassing this
* production call site entirely (documented live as bug-1755: "effectively
* dead code in production today because its only call site never supplies
* a bound").
*
* This test drives the REAL production call site -- worker_send_resync_
* query() itself, not a direct xce_job_state_query_req() call -- with a
* non-terminal job count on the host that genuinely exceeds the new
* XCE_RESYNC_QUERY_PKTLIMIT page cap (worker.c), and asserts the query is
* observably split across multiple wire pages: more than one packet is
* enqueued, each page's `pos` picks up exactly where the previous page's
* `count` left off, every jobid appears in exactly one page, and every
* page (including the final, deliberately partial one) carries the same
* full n_total. Req 8.1-8.3.
*/
#define TWJ_RESYNC_QUERY_PKTLIMIT 100 /* mirrors production
* XCE_RESYNC_QUERY_PKTLIMIT
* (worker.c) -- not a real
* production symbol reference (that
* #define has file-local linkage),
* kept in sync by this test's own
* page-count assertion below, which
* would fail loudly if the two ever
* drifted apart */
#define TWJ_RESYNC_QUERY_N_JOBS 250 /* > TWJ_RESYNC_QUERY_PKTLIMIT, and not
* a multiple of it, so the final wire
* page is genuinely partial (50
* entries), not just N full pages */
#define TWJ_RESYNC_QUERY_JOBID_BASE 970000ULL
#define TWJ_RESYNC_QUERY_N_PAGES \
((TWJ_RESYNC_QUERY_N_JOBS + TWJ_RESYNC_QUERY_PKTLIMIT - 1) \
/ TWJ_RESYNC_QUERY_PKTLIMIT)
static void
test_worker_send_resync_query_paginates_across_multiple_wire_pages(void)
{
setup_test_cluster();
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
worker_t* w = make_test_worker("query-host-paginate", &attrs);
TEST_ASSERT_NOT_NULL(w);
for (int i = 0; i < TWJ_RESYNC_QUERY_N_JOBS; i++) {
uint64_t jobid = TWJ_RESYNC_QUERY_JOBID_BASE + (uint64_t)i;
job_t* j = seed_test_job(jobid, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs = make_test_job_alloc(jobid, "query-host-paginate");
TEST_ASSERT_NOT_NULL(j->job.allocs);
}
hostattrs_t new_attrs = attrs;
new_attrs.hosttype = makeString("LINUX");
new_attrs.hostmodel = makeString("x86");
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
endpoint_t stack_ep;
memset(&stack_ep, 0, sizeof(stack_ep));
rejoin_worker(&g_tc.c, w, &stack_ep, &new_attrs, &hconfig);
TEST_ASSERT_NOT_NULL(w->resync);
TEST_ASSERT_EQUAL_INT(TWJ_RESYNC_QUERY_N_JOBS, w->resync->n_queried_jobids);
endpoint_t* ep = new_test_endpoint();
/* The REAL production call site -- no xce_job_state_query_req()
* bypass. */
TEST_ASSERT_EQUAL_INT(0, worker_send_resync_query(&g_tc.c, w, ep));
{
bool seen[TWJ_RESYNC_QUERY_N_JOBS];
int running_total = 0;
int pages = 0;
memset(seen, 0, sizeof(seen));
while (running_total < TWJ_RESYNC_QUERY_N_JOBS) {
xce_request_t decoded;
int j;
pop_and_decode(ep, &decoded);
TEST_ASSERT_EQUAL_INT(OPCode_jobStateQuery, decoded.opcode);
TEST_ASSERT_EQUAL_INT(Request_body_jobstatequerymsg,
decoded.body_which);
TEST_ASSERT_NOT_NULL(decoded.body.jobstatequerymsg);
TEST_ASSERT_EQUAL_INT_MESSAGE(TWJ_RESYNC_QUERY_N_JOBS,
decoded.body.jobstatequerymsg->n_total,
"every page must carry the same full n_total, not a "
"per-page total (Req 2.1)");
TEST_ASSERT_EQUAL_INT_MESSAGE(running_total,
decoded.body.jobstatequerymsg->pos,
"each page's pos must pick up exactly where the previous "
"page's count left off -- no gap, no overlap (Req 8.3)");
TEST_ASSERT_TRUE_MESSAGE(
decoded.body.jobstatequerymsg->count
<= TWJ_RESYNC_QUERY_PKTLIMIT,
"no single page may exceed the configured page cap "
"(Req 8.3)");
for (j = 0; j < decoded.body.jobstatequerymsg->count; j++) {
uint64_t jobid = decoded.body.jobstatequerymsg->jobids[j];
uint64_t idx = jobid - TWJ_RESYNC_QUERY_JOBID_BASE;
TEST_ASSERT_TRUE_MESSAGE(
idx < (uint64_t)TWJ_RESYNC_QUERY_N_JOBS,
"a jobid appearing on the wire must be one of the "
"armed round's own jobids");
TEST_ASSERT_FALSE_MESSAGE(seen[idx],
"no jobid may be sent twice across pages (Req 8.3)");
seen[idx] = true;
}
running_total += decoded.body.jobstatequerymsg->count;
pages++;
xce_free_request(&decoded);
}
TEST_ASSERT_EQUAL_INT_MESSAGE(TWJ_RESYNC_QUERY_N_PAGES, pages,
"Req 8.1-8.3: a job count exceeding the real production page "
"cap must genuinely span multiple wire pages -- observably, "
"via the REAL worker_send_resync_query() call site, not just "
"theoretically supported by xce_job_state_query_req()'s own "
"pagination loop (bug-1755, closed by task 7.5)");
TEST_ASSERT_GREATER_THAN_INT_MESSAGE(1, pages,
"a single collapsed page here would mean the hardcoded-0 "
"regression (bug-1755) is back");
for (int i = 0; i < TWJ_RESYNC_QUERY_N_JOBS; i++) {
TEST_ASSERT_TRUE_MESSAGE(seen[i],
"every armed jobid must appear exactly once across all "
"pages -- none silently dropped past a page boundary "
"(Req 8.3)");
}
}
assert_queue_empty(ep);
dereference(new_attrs.hosttype);
dereference(new_attrs.hostmodel);
endpoint_fini(ep);
free(ep);
teardown_test_cluster();
printf("test_worker_send_resync_query_paginates_across_multiple_wire_"
"pages: PASS\n");
}
/* ── Task 6.1: config push, then query, in that fixed order ──────────── */
/*
* design.md's File Structure Plan for on_worker_join is explicit that this
* ordering is load-bearing ("config push, then query", "not as an
* extension of the join call itself"), and this spec's own Implementation
* Notes (task 4.3) warn that a placement-sensitive guarantee needs a
* STRUCTURAL assertion, not just an end-state one -- a behavioral test
* alone can pass even with the wrong order if it only checks that both
* packets eventually arrive. This test drives the exact two calls on_
* worker_join makes, in the exact order it makes them, and asserts the
* WIRE ORDER of the resulting packets directly.
*/
static void test_config_push_then_resync_query_wire_order(void)
{
setup_test_cluster();
cluster_config_t cc;
memset(&cc, 0, sizeof(cc));
cc.clustername = (char*)"test-cluster";
myconf_t myconf;
memset(&myconf, 0, sizeof(myconf));
myconf.cluster_config = &cc;
g_tc.c.myconf = &myconf;
job_t* j = seed_test_job(96020, JOB_STAT_RUN);
j->job.allocs = make_test_job_alloc(96020, "order-host-1");
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
worker_t* w = make_test_worker("order-host-1", &attrs);
TEST_ASSERT_NOT_NULL(w);
hostattrs_t new_attrs = attrs;
new_attrs.hosttype = makeString("LINUX");
new_attrs.hostmodel = makeString("x86");
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
endpoint_t stack_ep;
memset(&stack_ep, 0, sizeof(stack_ep));
rejoin_worker(&g_tc.c, w, &stack_ep, &new_attrs, &hconfig);
TEST_ASSERT_NOT_NULL(w->resync);
endpoint_t* ep = new_test_endpoint();
/* The exact two calls on_worker_join() makes, in the exact order. */
worker_push_config(&g_tc.c, ep);
worker_send_resync_query(&g_tc.c, w, ep);
xce_request_t first;
pop_and_decode(ep, &first);
TEST_ASSERT_EQUAL_INT_MESSAGE(OPCode_push, first.opcode,
"the config push must be the FIRST packet on the wire");
xce_free_request(&first);
xce_request_t second;
pop_and_decode(ep, &second);
TEST_ASSERT_EQUAL_INT_MESSAGE(OPCode_jobStateQuery, second.opcode,
"the reconciliation query must be the SECOND packet on the wire, "
"strictly after the config push");
xce_free_request(&second);
assert_queue_empty(ep);
dereference(new_attrs.hosttype);
dereference(new_attrs.hostmodel);
endpoint_fini(ep);
free(ep);
teardown_test_cluster();
printf("test_config_push_then_resync_query_wire_order: PASS\n");
}
/*
* Req 3.7 (config-delivery gap): worker_push_config() must carry the
* REAL, currently-configured workerResyncWindowMs value -- not a
* hardcoded constant. Two distinct configured values must decode back to
* two distinct wire values.
*/
static void test_worker_push_config_carries_configured_resync_window(void)
{
setup_test_cluster();
cluster_config_t cc;
memset(&cc, 0, sizeof(cc));
cc.clustername = (char*)"test-cluster";
cc.worker_resync_window_ms = 424242;
myconf_t myconf;
memset(&myconf, 0, sizeof(myconf));
myconf.cluster_config = &cc;
g_tc.c.myconf = &myconf;
endpoint_t* ep = new_test_endpoint();
TEST_ASSERT_EQUAL_INT(0, worker_push_config(&g_tc.c, ep));
xce_request_t decoded;
pop_and_decode(ep, &decoded);
TEST_ASSERT_EQUAL_INT(OPCode_push, decoded.opcode);
TEST_ASSERT_EQUAL_INT(Request_body_configpushmsg, decoded.body_which);
TEST_ASSERT_NOT_NULL(decoded.body.configpushmsg);
TEST_ASSERT_EQUAL_INT(ConfigData_data_workerconfig,
decoded.body.configpushmsg->data_which);
TEST_ASSERT_NOT_NULL(decoded.body.configpushmsg->data.conf4worker);
TEST_ASSERT_EQUAL_INT_MESSAGE(424242,
decoded.body.configpushmsg->data.conf4worker->worker_resync_window_ms,
"the pushed value must be the real configured ClusterConfig."
"workerResyncWindowMs, not a hardcoded constant");
xce_free_request(&decoded);
assert_queue_empty(ep);
endpoint_fini(ep);
free(ep);
/* A second, different configured value must decode back differently --
* confirms the field is genuinely read from config each call, not
* memoized/hardcoded from the first call above. */
cc.worker_resync_window_ms = 7;
ep = new_test_endpoint();
TEST_ASSERT_EQUAL_INT(0, worker_push_config(&g_tc.c, ep));
pop_and_decode(ep, &decoded);
TEST_ASSERT_EQUAL_INT_MESSAGE(7,
decoded.body.configpushmsg->data.conf4worker->worker_resync_window_ms,
"a second, different configured value must be reflected on the "
"wire, proving the field is read live, not cached");
xce_free_request(&decoded);
endpoint_fini(ep);
free(ep);
teardown_test_cluster();
printf("test_worker_push_config_carries_configured_resync_window: "
"PASS\n");
}
/* ── Dispatch ───────────────────────────────────────────────────────── */
void test_worker_join(void)
@ -716,4 +1518,19 @@ void test_worker_join(void)
/* Task 1.3: rejoin_worker (reconnect with admin-bit preservation) */
RUN_TEST(test_rejoin_worker_updates_attrs_and_ep);
RUN_TEST(test_rejoin_worker_preserves_admin_bits);
/* worker-rejoin-resync task 6.1: apply_join_state arms a
* reconciliation round; worker_send_resync_query sends it; config
* push then query in that fixed order; the workerResyncWindowMs
* config-delivery gap. */
RUN_TEST(test_rejoin_worker_arms_reconciliation_round_when_jobs_exist);
RUN_TEST(test_rejoin_worker_no_round_when_no_jobs);
RUN_TEST(test_rejoin_worker_no_round_when_only_terminal_jobs);
RUN_TEST(test_init_worker_arms_reconciliation_round_when_jobs_exist);
RUN_TEST(test_worker_send_resync_query_sends_armed_round);
RUN_TEST(test_worker_send_resync_query_sends_nothing_when_no_round);
RUN_TEST(
test_worker_send_resync_query_paginates_across_multiple_wire_pages);
RUN_TEST(test_config_push_then_resync_query_wire_order);
RUN_TEST(test_worker_push_config_carries_configured_resync_window);
}

4780
src/controller/testsuite/test_worker_resync.c Normal file
View file

File diff suppressed because it is too large Load diff