diff --git a/.kiro/specs/worker-rejoin-resync/tasks.md b/.kiro/specs/worker-rejoin-resync/tasks.md index dc005606..52ee8378 100644 --- a/.kiro/specs/worker-rejoin-resync/tasks.md +++ b/.kiro/specs/worker-rejoin-resync/tasks.md @@ -36,7 +36,7 @@ - _Boundary: xceadmin config-build_ - _Depends: 1.1_ -- [ ] 2. Core: controller job-table reconciliation outcomes +- [x] 2. Core: controller job-table reconciliation outcomes - [x] 2.1 Return an unrecognized job to pending state, releasing its resource allocation - When a worker reports a queried job as unrecognized, transition that job back to pending using the existing state-transition guard so a terminal or already-more-final job is never overwritten - Release any resource allocation the job was carrying, so a job returned to pending never carries a stale allocation from a prior dispatch into its next dispatch attempt @@ -53,7 +53,7 @@ - _Requirements: 4.1, 4.3, 4.4, 4.7, 4.8, 7.1, 7.2, 7.3_ - _Boundary: job.c additions_ -- [ ] 2.3 Fail a dispatch to a disconnected worker immediately, without retry or job failure +- [x] 2.3 Fail a dispatch to a disconnected worker immediately, without retry or job failure - When the controller attempts to dispatch a job to a worker it is not currently connected to, fail that specific dispatch attempt right away and return the job to pending, reusing the pending-and-release behavior from task 2.1, rather than retrying the attempt or exhausting a retry count toward permanent failure - Leave the existing behavior for every other reason a worker might not be ready (locked, busy, closed, or mid-reconciliation) completely unchanged — confirm those cases still retry and eventually fail exactly as before - Observable completion: dispatching to a disconnected worker returns the job to pending on the first attempt with the job's failure-retry count untouched and no permanent failure recorded, while dispatching to a worker that is connected but otherwise not ready still follows the pre-existing retry-then-fail behavior unchanged diff --git a/src/controller/job.c b/src/controller/job.c index aa6b2e1f..891f3cbe 100644 --- a/src/controller/job.c +++ b/src/controller/job.c @@ -1510,7 +1510,66 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc) } lock(w, RTTI_LOCK_READ); - if ((w->ep == NULL) || (w->host.status != 0)) { + if (w->ep == NULL) { + /* worker-rejoin-resync task 2.3 (Req 5.1, 5.4): a dispatch attempt + * to a worker the controller is not currently connected to (w->ep + * == NULL) is resolved here, BEFORE the w->host.status != 0 check + * below ever reaches xce_classify_dispatch_fail() at all -- + * deliberately a bypass of that retry-then-terminal-fail machinery, + * not a third value added to xce_dispatch_fail_class_t (design.md's + * "Controller: job.c" Components section: that enum's whole + * contract is a two-way TERMINAL/RETRYABLE choice every existing + * call site's switch already exhaustively handles). Retrying the + * dispatch itself doesn't fit "the worker isn't even connected," + * and this case must stay recoverable-via-reschedule rather than + * ever resolve to mark_job_fail()'s terminal outcome -- so this + * branch calls neither xce_classify_dispatch_fail() nor + * mark_job_fail(), and never touches j->dispatch_fail_count. */ + int dfrc; + int result; + + ERROR("worker %s is not connected (ep NULL), status %x", + w->host.name, w->host.status); + free_launchinfo(&launchinfo); + + SET_ERROR(E_XCE_BAD_WORKER); + /* Capture errno immediately after SET_ERROR(), before any further + * call -- errno is not guaranteed stable across unlock(w)/ + * dereference(w)/lock(j)/etc., mirroring the WORKER_NOT_READY + * branch's own discipline below. */ + dfrc = -errno; + + unlock(w); + + dereference(w); + + /* Req 5.4: resolves via record_job_reconcile_pending_fact() (task + * 2.1) directly -- eager mutation + allocs release (bug-1631) under + * this write lock, durable append inside that call, then (only on + * an actual APPLIED transition) this caller's own post-unlock + * persist_notify_job_updated_already_persisted() call, exactly as + * that function's own contract requires (cluster.h doc comment). + * Closes the same bug-1627 reference-leak class as the WORKER_NOT_ + * READY branch below -- j's reference (held since the jobmgr lookup + * at the top of this function) is released via dereference(j) on + * every exit from this branch. */ + lock(j, RTTI_LOCK_WRITE); + + result = record_job_reconcile_pending_fact(c, j); + + unlock(j); + + if (result == (int)JOB_TRANSITION_APPLIED) { + persist_notify_job_updated_already_persisted(c, j); + } + /* JOB_TRANSITION_REJECTED_BACKWARD/NOOP_DUPLICATE: nothing changed + * (job already terminal, or already PEND) -- no notify needed, + * matching record_job_reconcile_pending_fact's own contract. */ + + dereference(j); + + return dfrc; + } else if (w->host.status != 0) { int dfrc; xce_dispatch_fail_class_t dfclass; job_failinfo_t failinfo = { .jobid = alloc->jobid }; diff --git a/src/controller/testsuite/test_dispatch_fail_retry.c b/src/controller/testsuite/test_dispatch_fail_retry.c index 37ceec17..810d4924 100644 --- a/src/controller/testsuite/test_dispatch_fail_retry.c +++ b/src/controller/testsuite/test_dispatch_fail_retry.c @@ -1,4 +1,5 @@ -/* test_dispatch_fail_retry.c -- scheduler-dispatch-commit task 3.3 +/* test_dispatch_fail_retry.c -- scheduler-dispatch-commit task 3.3; + * extended by worker-rejoin-resync task 2.3 * * Proves dispatch_a_job_() (job.c) now RESOLVES a retryable (transient/ * environmental) dispatch failure -- incrementing the consecutive-failure @@ -19,6 +20,25 @@ * - XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY (E_XCE_BAD_WORKER) * - XCE_DISPATCH_FAIL_SITE_RPC_FAILURE (xce_startjob_req() rc<0) * + * worker-rejoin-resync task 2.3 (Req 5.1, 5.4) split dispatch_a_job_()'s + * combined "(w->ep == NULL) || (w->host.status != 0)" check into two + * branches: a disconnected worker (w->ep == NULL) is now resolved + * separately and DIFFERENTLY from XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY + * above -- it bypasses xce_classify_dispatch_fail()/mark_job_fail() + * entirely and returns the job to PEND immediately via + * record_job_reconcile_pending_fact() (task 2.1), with no + * dispatch_fail_count increment and no retry. This is NOT one of the four + * RETRYABLE sites this file otherwise covers -- it is added here anyway + * (rather than to test_worker_resync.c) because it is fundamentally a + * dispatch_a_job_() behavior/regression test exercising the same + * combined-check split site SITE 3 below already covers, sharing this + * file's fixture/canary/notify-capture machinery. See the new tests below + * SITE 3. SITE 3 itself (XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY, + * w->host.status != 0) is updated to use a CONNECTED worker (w->ep != NULL) + * with a nonzero status bit -- it previously used an ep-left-NULL worker to + * reach the combined check, which now takes the new, separate branch + * instead. + * * test_worker_notfound_resolves_retryable (below) seeds an unregistered * worker key, matching test_dispatch_commit.c's own established fixture * for the "worker not found" scenario -- that file's own header notes this @@ -253,6 +273,28 @@ static void dfr_assert_ref_not_over_released(job_t* j, uint64_t expect_jobid) "heap-use-after-free"); } +/* worker-rejoin-resync task 2.3: same canary technique as above, applied to + * `w` -- the new disconnected-worker branch dereference(w)s the reference + * dispatch_a_job_() itself took at its key_find() lookup, and this is the + * exact class of reference bug bug-1627 fixed (leak site #3, the WORKER_ + * NOT_READY/ep-NULL combined check). */ +static void dfr_arm_worker_canary(worker_t* w) +{ + reference(w); +} + +static void dfr_assert_worker_ref_not_over_released(worker_t* w, + const char* expect_name) +{ + dereference(w); + + TEST_ASSERT_EQUAL_STRING_MESSAGE(expect_name, w->host.name, + "w must still be safely readable after releasing the canary " + "reference -- an over-release inside dispatch_a_job_test() would " + "have freed w while the canary was still held, making this a " + "heap-use-after-free"); +} + /* A "broken" endpoint: real socket fd, real iomgr, deliberately NOT * registered in the iomgr's epoll -- EPOLL_CTL_MOD on it fails with ENOENT * (the exact post-failed-reconnect condition endpoint_update_epoll() @@ -368,13 +410,25 @@ static void test_worker_notfound_resolves_retryable(void) } /* ── SITE 3: XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY ─────────────────── - * A registered worker whose ep is left NULL (calloc default) hits - * "(w->ep == NULL) || (w->host.status != 0)". */ + * A registered, CONNECTED worker (w->ep != NULL) with a nonzero + * w->host.status bit (XCE_WORKER_BUSY, picked arbitrarily -- any bit takes + * the same unchanged path) hits "w->host.status != 0" -- the still- + * unmodified branch of dispatch_a_job_()'s split combined check + * (worker-rejoin-resync task 2.3). Before that split, this test used an + * ep-left-NULL worker to reach the combined check; that condition now + * takes the NEW, separate disconnected-worker branch instead (see the new + * tests below), which resolves differently (immediate PEND, no retry, no + * dispatch_fail_count increment) -- so this test is updated to use a + * connected-but-busy worker, keeping it a genuine, unchanged regression + * proof for XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY's own retry-then- + * terminal-fail behavior (Req 5.1's design.md backstop). */ #define DFR_JOBID_WORKER_NOTREADY 91202ULL static void test_worker_notready_resolves_retryable(void) { job_t* j; + worker_t* w = NULL; + endpoint_t connected_ep; xce_job_alloc_t* alloc; int rc; int before_count; @@ -382,7 +436,18 @@ static void test_worker_notready_resolves_retryable(void) dfr_setup(); j = dfr_seed_job(DFR_JOBID_WORKER_NOTREADY, "dfr-q2"); - dfr_seed_worker("dfr-wkr-notready", NULL); /* ep left NULL: not ready */ + + /* A non-NULL ep (its contents are never touched -- this branch fails + * before any RPC is attempted) plus a nonzero status bit: connected, + * but not ready. */ + memset(&connected_ep, 0, sizeof(connected_ep)); + dfr_seed_worker("dfr-wkr-notready", &connected_ep); + + lock(g_dfr.workermgr, RTTI_LOCK_READ); + key_find(g_dfr.workermgr, (void*)"dfr-wkr-notready", (void**)&w); + unlock(g_dfr.workermgr); + TEST_ASSERT_NOT_NULL(w); + w->host.status = XCE_WORKER_BUSY; alloc = make_job_alloc(DFR_JOBID_WORKER_NOTREADY, 1, make_host_allocs(1, @@ -391,6 +456,13 @@ static void test_worker_notready_resolves_retryable(void) make_alloc((char*)"slots", NULL, 0, 1.0f, 0))))); TEST_ASSERT_NOT_NULL(alloc); + /* worker-rejoin-resync task 2.3: explicit proof this test still reaches + * XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY's branch of the split check -- + * a connected (non-NULL ep), nonzero-status worker, not the new + * disconnected-worker branch below. */ + TEST_ASSERT_NOT_NULL(w->ep); + TEST_ASSERT_TRUE(w->host.status != 0); + cbrp_notify_capture_reset_ext(); before_count = cbrp_notify_capture_count_ext(); @@ -407,7 +479,8 @@ static void test_worker_notready_resolves_retryable(void) "Req 4.4: no partial allocation data may be left behind"); TEST_ASSERT_EQUAL_UINT32_MESSAGE(1, j->dispatch_fail_count, "Req 4.3: the consecutive-failure counter must increment by " - "exactly 1"); + "exactly 1 -- unchanged existing WORKER_NOT_READY retry behavior " + "(worker-rejoin-resync task 2.3 must not affect this branch)"); dfr_assert_ref_not_over_released(j, DFR_JOBID_WORKER_NOTREADY); @@ -426,6 +499,250 @@ static void test_worker_notready_resolves_retryable(void) printf("test_worker_notready_resolves_retryable: PASS\n"); } +/* ── worker-rejoin-resync task 2.3: disconnected-worker branch ───────── + * (Req 5.1, 5.4) -- NOT one of the four RETRYABLE sites above: a dispatch + * attempt to a worker the controller is not currently connected to + * (w->ep == NULL) now fails immediately and returns the job to PEND via + * record_job_reconcile_pending_fact(), bypassing xce_classify_dispatch_ + * fail()/mark_job_fail() entirely -- no dispatch_fail_count increment, no + * retry, no terminal resolution. + * + * Realistic pre-dispatch state: dispatch_a_job_()'s only real caller + * dispatches a job the scheduler picked from its PEND pool, so this test + * seeds an ordinary PEND job (dfr_seed_job's default) -- record_job_ + * reconcile_pending_fact()'s apply_job_update_transition(job, JOB_STAT_ + * PEND) call therefore hits JOB_TRANSITION_NOOP_DUPLICATE (job already + * PEND), not APPLIED: no mutation, no allocs touch (there is none to + * release), no durable fact, no notify -- exactly as design.md's own + * Components section describes for this outcome, and as this task's own + * prompt anticipates ("handle this correctly: still don't call mark_job_ + * fail, still don't retry, but there's nothing to release/notify either + * since nothing changed"). */ +#define DFR_JOBID_DISCONNECTED 91206ULL + +static int g_dfr_update_hook_fires; + +static int dfr_job_update_spy_hook(int type, int situation, int detail, + void* data, va_list ap) +{ + (void)type; (void)data; (void)ap; + if (situation == HOOK_AFTER_EVENT && detail == HOOK_DETAIL_JOB_UPDATE) + g_dfr_update_hook_fires++; + return 0; +} + +static void test_disconnected_worker_fails_immediately_no_retry(void) +{ + job_t* j; + worker_t* w = NULL; + xce_job_alloc_t* alloc; + int rc; + int before_count; + uint32_t fail_count_before; + + dfr_setup(); + g_dfr_update_hook_fires = 0; + TEST_ASSERT_EQUAL_INT(0, + xce_add_hook(&g_dfr.hooksys, XCE_CONTROLLER_HOOK_JOB, + dfr_job_update_spy_hook, NULL)); + + j = dfr_seed_job(DFR_JOBID_DISCONNECTED, "dfr-q-disc"); + fail_count_before = j->dispatch_fail_count; + + dfr_seed_worker("dfr-wkr-disc1", NULL); /* ep left NULL: disconnected */ + + lock(g_dfr.workermgr, RTTI_LOCK_READ); + key_find(g_dfr.workermgr, (void*)"dfr-wkr-disc1", (void**)&w); + unlock(g_dfr.workermgr); + TEST_ASSERT_NOT_NULL(w); + TEST_ASSERT_NULL_MESSAGE(w->ep, "the disconnected branch requires " + "w->ep == NULL -- confirm the fixture actually constructs that"); + + alloc = make_job_alloc(DFR_JOBID_DISCONNECTED, 1, + make_host_allocs(1, + make_host_alloc((char*)"dfr-wkr-disc1", 1, + make_allocs(1, + make_alloc((char*)"slots", NULL, 0, 1.0f, 0))))); + TEST_ASSERT_NOT_NULL(alloc); + + cbrp_notify_capture_reset_ext(); + before_count = cbrp_notify_capture_count_ext(); + + /* No reference/leak regression (task 2.3's own Tests-to-write clause): + * canary both j (bug-1627's original leak class) and w (this branch's + * own dereference(w), the same site bug-1627 fixed for the WORKER_NOT_ + * READY branch). */ + dfr_arm_ref_canary(j); + dfr_arm_worker_canary(w); + + rc = dispatch_a_job_test(&g_dfr, NULL, alloc); + TEST_ASSERT_TRUE_MESSAGE(rc < 0, + "a dispatch to a disconnected worker must still fail"); + + /* Observable completion (Req 5.4): job ends up PEND (unchanged -- it + * started PEND, a realistic pre-dispatch state), single attempt, no + * retry loop. */ + TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state, + "Req 5.4: a dispatch to a disconnected worker must return the job " + "to pending (here: leave it PEND, since it started PEND)"); + TEST_ASSERT_NOT_EQUAL_MESSAGE(JOB_STAT_EXIT, j->job.state, + "Req 5.4: this must NOT become a permanent/terminal failure -- " + "mark_job_fail() must never be invoked on this path"); + TEST_ASSERT_NULL_MESSAGE(j->job.allocs, + "no partial allocation data may be left behind"); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(fail_count_before, j->dispatch_fail_count, + "Req 5.4: dispatch_fail_count must be left UNCHANGED -- this " + "branch must never reach xce_classify_dispatch_fail()'s retry-" + "counting machinery"); + + /* "single attempt, no retry loop": notify_dispatch() is dispatch_a_ + * job_()'s sole retry-signaling mechanism (see the four RETRYABLE-site + * tests above) -- it must never fire on this path. */ + TEST_ASSERT_EQUAL_INT_MESSAGE(before_count, cbrp_notify_capture_count_ext(), + "Req 5.4: a disconnected-worker dispatch failure must never call " + "notify_dispatch() -- there is no retry to signal"); + + /* NOOP_DUPLICATE outcome (job was already PEND): no mutation occurred, + * so no notify is needed or expected either -- omitting the notify + * here is correct, not bug-1679's omission-is-a-bug case (that applies + * only to an actual APPLIED transition; see the APPLIED-path test + * below for that proof). */ + TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_dfr_update_hook_fires, + "a no-op reconciliation (job already PEND) must not fire the " + "scheduler notify path -- nothing changed"); + + dfr_assert_ref_not_over_released(j, DFR_JOBID_DISCONNECTED); + dfr_assert_worker_ref_not_over_released(w, "dfr-wkr-disc1"); + + xce_disable_hook(&g_dfr.hooksys, XCE_CONTROLLER_HOOK_JOB, + dfr_job_update_spy_hook); + free_JobAlloc_ptr(&alloc); + dfr_teardown(); + printf("test_disconnected_worker_fails_immediately_no_retry: PASS\n"); +} + +/* ── worker-rejoin-resync task 2.3: JOB_TRANSITION_APPLIED wiring proof + * ──────────────────────────────────────────────────────────────────── + * The NOOP_DUPLICATE test above exercises the only precondition dispatch_ + * a_job_()'s real caller can currently construct (a PEND job -- the + * scheduler only ever dispatches from its PEND pool, so dispatch_a_job_() + * is never actually invoked on a RUN job today; the same "not reachable + * via any real caller in the currently-implemented system" caveat + * test_rpc_failure_bound_resets_after_intervening_success's header + * documents for the four RETRYABLE sites applies here too -- see bug-1631). + * This test instead directly seeds a RUN job with a real, committed + * job->job.allocs (mirroring test_worker_resync.c's own technique) to + * exercise record_job_reconcile_pending_fact()'s OTHER outcome + * (JOB_TRANSITION_APPLIED, not NOOP_DUPLICATE) THROUGH this exact new + * dispatch_a_job_() call site -- proving the branch's result-handling + * (unlock -> notify-only-on-APPLIED -> dereference) is wired correctly, + * complementing task 2.1's test_worker_resync.c (which already proves + * record_job_reconcile_pending_fact() itself is correct in isolation, not + * through this caller). Not a claim that THIS exact precondition (RUN job + * hitting the disconnected branch) is reachable in production today -- + * design.md's own bug-1631 narrative is that record_job_reconcile_pending_ + * fact() releases allocs at the point of RECONCILIATION (a separate, + * later-task call site, resync.c), so by the time any job reaches a fresh + * dispatch_a_job_() call it is already PEND with allocs already NULL. */ +#define DFR_JOBID_DISCONNECTED_RUN 91207ULL + +static uint64_t dfr_current_lsn(void) +{ + uint64_t lsn; + + pthread_mutex_lock(&g_dfr.persist.lock); + lsn = g_dfr.persist.current_lsn; + pthread_mutex_unlock(&g_dfr.persist.lock); + return lsn; +} + +static void test_disconnected_worker_reconciles_run_job_releases_allocs(void) +{ + job_t* j; + worker_t* w = NULL; + xce_job_alloc_t* prior_alloc; + xce_job_alloc_t* redispatch_alloc; + int rc; + int before_count; + uint64_t lsn_before, lsn_after; + + dfr_setup(); + g_dfr_update_hook_fires = 0; + TEST_ASSERT_EQUAL_INT(0, + xce_add_hook(&g_dfr.hooksys, XCE_CONTROLLER_HOOK_JOB, + dfr_job_update_spy_hook, NULL)); + + j = dfr_seed_job(DFR_JOBID_DISCONNECTED_RUN, "dfr-q-disc2"); + + /* Simulate a prior successful dispatch (mirrors test_worker_resync.c's + * own technique): real allocs committed, state RUN, ownership + * transferred into job->job.allocs. */ + prior_alloc = make_job_alloc(DFR_JOBID_DISCONNECTED_RUN, 1, + make_host_allocs(1, + make_host_alloc((char*)"dfr-wkr-disc2-prior", 1, + make_allocs(1, + make_alloc((char*)"slots", NULL, 0, 1.0f, 0))))); + TEST_ASSERT_NOT_NULL(prior_alloc); + j->job.state = JOB_STAT_RUN; + j->job.allocs = prior_alloc; + + dfr_seed_worker("dfr-wkr-disc2", NULL); /* ep left NULL: disconnected */ + + lock(g_dfr.workermgr, RTTI_LOCK_READ); + key_find(g_dfr.workermgr, (void*)"dfr-wkr-disc2", (void**)&w); + unlock(g_dfr.workermgr); + TEST_ASSERT_NOT_NULL(w); + + redispatch_alloc = make_job_alloc(DFR_JOBID_DISCONNECTED_RUN, 1, + make_host_allocs(1, + make_host_alloc((char*)"dfr-wkr-disc2", 1, + make_allocs(1, + make_alloc((char*)"slots", NULL, 0, 1.0f, 0))))); + TEST_ASSERT_NOT_NULL(redispatch_alloc); + + cbrp_notify_capture_reset_ext(); + before_count = cbrp_notify_capture_count_ext(); + lsn_before = dfr_current_lsn(); + + dfr_arm_ref_canary(j); + dfr_arm_worker_canary(w); + + rc = dispatch_a_job_test(&g_dfr, NULL, redispatch_alloc); + TEST_ASSERT_TRUE_MESSAGE(rc < 0, + "a dispatch to a disconnected worker must still fail"); + + TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state, + "Req 4.6/5.4: a RUN job redispatched to a disconnected worker must " + "be returned to PEND"); + TEST_ASSERT_NULL_MESSAGE(j->job.allocs, + "bug-1631: the job's prior resource allocation must be released, " + "so a later redispatch never inherits a stale allocation"); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, j->dispatch_fail_count, + "Req 5.4: dispatch_fail_count must stay untouched on this path"); + + TEST_ASSERT_EQUAL_INT_MESSAGE(before_count, cbrp_notify_capture_count_ext(), + "notify_dispatch() (the retry-signal path) must never fire here"); + + lsn_after = dfr_current_lsn(); + TEST_ASSERT_TRUE_MESSAGE(lsn_after > lsn_before, + "an APPLIED reconciliation must durably append a WAL_JOB_UPDATE " + "fact"); + + TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_dfr_update_hook_fires, + "bug-1679: an APPLIED reconciliation must notify the scheduler " + "exactly once, via this exact dispatch_a_job_() call site"); + + dfr_assert_ref_not_over_released(j, DFR_JOBID_DISCONNECTED_RUN); + dfr_assert_worker_ref_not_over_released(w, "dfr-wkr-disc2"); + + xce_disable_hook(&g_dfr.hooksys, XCE_CONTROLLER_HOOK_JOB, + dfr_job_update_spy_hook); + free_JobAlloc_ptr(&redispatch_alloc); + dfr_teardown(); + printf("test_disconnected_worker_reconciles_run_job_releases_allocs: " + "PASS\n"); +} + /* ── SITE 4: XCE_DISPATCH_FAIL_SITE_RPC_FAILURE ──────────────────────── * A ready worker whose ep is deliberately "broken" (see dfr_setup_broken_ep * above) makes xce_startjob_req() itself fail. */ @@ -814,4 +1131,10 @@ void test_dispatch_fail_retry(void) RUN_TEST(test_rpc_failure_resolves_retryable); RUN_TEST(test_rpc_failure_bounded_escalation); RUN_TEST(test_rpc_failure_bound_resets_after_intervening_success); + + printf("\n>> dispatch_a_job_() disconnected-worker fail-fast tests " + "(worker-rejoin-resync task 2.3, Req 5.1, 5.4)\n\n"); + + RUN_TEST(test_disconnected_worker_fails_immediately_no_retry); + RUN_TEST(test_disconnected_worker_reconciles_run_job_releases_allocs); }