/* test_dispatch_fail_retry.c -- scheduler-dispatch-commit task 3.3; * extended by worker-rejoin-resync task 2.3 * * Proves dispatch_a_job_() (job.c) now RESOLVES a retryable (transient/ * environmental) dispatch failure -- incrementing the consecutive-failure * counter (job_t.dispatch_fail_count, task 1.3) under the job's own write * lock, reporting the failure via notify_dispatch() so the job can be * retried when the incremented count is still within the bound, or * resolving the job to permanently failed (mirroring task 3.2's terminal * resolution exactly) once the bound is exceeded -- instead of leaking the * job reference it holds, as it did before this task (bug-1627). * * Covers the four failure sites design.md's "Controller Dispatch-Failure * Classification" component always classifies RETRYABLE today (confirmed * by task 1.1's own xce_classify_dispatch_fail() switch, unmodified/reused * here, not re-tested): * * - XCE_DISPATCH_FAIL_SITE_WORKER_LOOKUP (workermgr key_find() rc<0) * - XCE_DISPATCH_FAIL_SITE_WORKER_NULL (worker not found, w==NULL) * - XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY (E_XCE_BAD_WORKER) * - XCE_DISPATCH_FAIL_SITE_RPC_FAILURE (xce_startjob_req() rc<0) * * worker-rejoin-resync task 2.3 (Req 5.1, 5.4) split dispatch_a_job_()'s * combined "(w->ep == NULL) || (w->host.status != 0)" check into two * branches: a disconnected worker (w->ep == NULL) is now resolved * separately and DIFFERENTLY from XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY * above -- it bypasses xce_classify_dispatch_fail()/mark_job_fail() * entirely and returns the job to PEND immediately via * record_job_reconcile_pending_fact() (task 2.1), with no * dispatch_fail_count increment and no retry. This is NOT one of the four * RETRYABLE sites this file otherwise covers -- it is added here anyway * (rather than to test_worker_resync.c) because it is fundamentally a * dispatch_a_job_() behavior/regression test exercising the same * combined-check split site SITE 3 below already covers, sharing this * file's fixture/canary/notify-capture machinery. See the new tests below * SITE 3. SITE 3 itself (XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY, * w->host.status != 0) is updated to use a CONNECTED worker (w->ep != NULL) * with a nonzero status bit -- it previously used an ep-left-NULL worker to * reach the combined check, which now takes the new, separate branch * instead. * * test_worker_notfound_resolves_retryable (below) seeds an unregistered * worker key, matching test_dispatch_commit.c's own established fixture * for the "worker not found" scenario -- that file's own header notes this * takes dispatch_a_job_()'s WORKER_NULL branch (w stays NULL, rc>=0 from * key_find()) rather than WORKER_LOOKUP (a genuine key_find() rc<0, which * this test suite has no black-box way to force without reaching into * librtti's own internals, an external/opaque dependency). Both sites * share the exact same classify/resolve procedure in job.c (this task's * whole point is that the four retryable sites are structurally * identical), so this one test's assertions equally prove either site's * behavior regardless of which the runtime actually takes -- it is * deliberately written not to depend on which. * * Req 4.1: a retryable dispatch failure below the bound reports the * failure via notify_dispatch() so the job can be retried -- job.state * stays PEND, dispatch_fail_count increments by exactly 1. * Req 4.3: consecutive retryable failures with no intervening success are * evaluated against a bounded limit (design.md's stated default: 3) -- * the 4th consecutive failure resolves the job to terminal instead of * retrying again. * Req 4.4: a job returned to pending following a retryable failure has no * stale resource-allocation/commitment data -- job.allocs stays NULL * (task 3.1 gates its only commit on a successful xce_startjob_req(), * never reached on any of these four failure paths, so this is satisfied * vacuously by construction -- asserted here explicitly anyway). * Req 4.5: same vacuous-by-construction guarantee for the bound-exceeded * terminal resolution. * Also (bug-1627's actual fix): each of the four sites now * dereference(j)s the reference taken at the top of dispatch_a_job_() -- * proven via the same manual "canary" reference technique task 3.2's * test_dispatch_fail_terminal.c established (see that file's header for * the full rationale; job_t's real refcount lives in jobj.c's file-local * job_private_t with no accessor exposed to a test). * * notify_dispatch() observability: this suite's controller_unit_test * binary excludes dataex.c (the real notify()'s home) -- the sole notify() * definition linked into this binary is test_commit_barrier_reply.c's own * capture stub. That file exposes a handful of minimal, read-only * accessor functions (cbrp_notify_capture_*_ext, added by this task) so * this file can observe a notify_dispatch() call without duplicating any * mocking machinery. * * dispatch_a_job_() is static/file-local; reached here through * dispatch_a_job_test(), task 3.1's existing XCE_TESTING-only wrapper -- * reused directly, not duplicated. * * Fixture mirrors test_dispatch_fail_terminal.c's dft_setup/dft_teardown * pattern (real jobmgr/workermgr/qmgr/hooksys + a real disk-backed * persist_t with cluster == NULL) trimmed to what these four failure sites * need. No hook functions are ever registered against g_dfr.hooksys in * this fixture -- mark_job_fail()'s two xce_run_hooks() calls therefore * iterate zero registered handlers and are safe no-ops here. */ #include #include #include #include #include #include #include #include #include #include "cluster.h" #include "controller.capnp.h" #include "hook.h" #include "jobj.h" #include "kvmgr.h" #include "netio.h" #include "persist.h" #include "qobj.h" #include "wobj.h" #include "xce/cluster_def.h" #include "xce/types.h" #include "xcealloc.h" #include "xstring.h" /* dispatch_a_job_test: task 3.1's XCE_TESTING-only wrapper (job.c), thinly * exposing the static dispatch_a_job_() for direct testing. */ extern int dispatch_a_job_test(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc); /* test_commit_barrier_reply.c's minimal notify()-capture accessors (added * by this task) -- see this file's header. */ extern int cbrp_notify_capture_count_ext(void); extern int cbrp_notify_capture_event_ext(int idx); extern uint64_t cbrp_notify_capture_jobid_ext(int idx); extern int cbrp_notify_capture_code_ext(int idx); extern void cbrp_notify_capture_reset_ext(void); /* jobj.c's job type table and the process-global xstring table are shared * across every test file in the controller_unit_test binary -- guard * idempotently, mirroring every sibling fixture in this suite. */ static int g_dfr_jobobj_ready; static cluster_t g_dfr; static char g_dfr_tmpdir[256]; static void dfr_setup(void) { kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16 }; kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING, .nslots = 32, .delta = 16 }; persist_config_t cfg; if (makeString((char*)"dfr-probe") == NULL) xstring_table_init(64); if (!g_dfr_jobobj_ready) { TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32)); g_dfr_jobobj_ready = 1; } memset(&g_dfr, 0, sizeof(g_dfr)); TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&g_dfr.hooksys, XCE_CONTROLLER_HOOK_MAX)); g_dfr.jobmgr = makeInstance(&xkvmgr_type, &int_params); g_dfr.workermgr = makeInstance(&xkvmgr_type, &str_params); g_dfr.qmgr = makeInstance(&xkvmgr_type, &str_params); TEST_ASSERT_NOT_NULL(g_dfr.jobmgr); TEST_ASSERT_NOT_NULL(g_dfr.workermgr); TEST_ASSERT_NOT_NULL(g_dfr.qmgr); cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; strcpy(g_dfr_tmpdir, "/tmp/xce_dfr33_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(g_dfr_tmpdir)); persist_set_workdir(&g_dfr.persist, g_dfr_tmpdir, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(&g_dfr.persist, &cfg, NULL)); TEST_ASSERT_EQUAL_INT(0, persist_start(&g_dfr.persist)); } static void dfr_teardown(void) { char cmd[512]; persist_stop(&g_dfr.persist); persist_fini(&g_dfr.persist); if (g_dfr.workermgr) dereference(g_dfr.workermgr); if (g_dfr.qmgr) dereference(g_dfr.qmgr); if (g_dfr.jobmgr) dereference(g_dfr.jobmgr); xce_hook_fini(&g_dfr.hooksys); snprintf(cmd, sizeof(cmd), "rm -rf %s", g_dfr_tmpdir); if (system(cmd) != 0) fprintf(stderr, "cleanup warning: rm -rf %s failed\n", g_dfr_tmpdir); } /* Seeds a queue and a PEND job with a real submit/start_info against * g_dfr.jobmgr/qmgr (dispatch_a_job_() reads submit->queue and * submit->start_info). Returns the seeded job_t* (borrowed -- g_dfr.jobmgr * owns the creation reference). */ static job_t* dfr_seed_job(uint64_t jobid, const char* qname) { xce_queue_t qdef; job_startinfo_t startinfo; job_submit_data_t submit; xce_job_t jdef; job_t* j; memset(&qdef, 0, sizeof(qdef)); qdef.queue = (char*)qname; TEST_ASSERT_NOT_NULL(makeQueue(g_dfr.qmgr, &qdef)); memset(&startinfo, 0, sizeof(startinfo)); startinfo.cmdline = (char*)"/bin/true"; memset(&submit, 0, sizeof(submit)); submit.queue = (char*)qname; submit.user = (char*)"tester"; submit.start_info = &startinfo; memset(&jdef, 0, sizeof(jdef)); jdef.jobid = jobid; jdef.state = JOB_STAT_PEND; jdef.submit = &submit; j = makeJob(g_dfr.jobmgr, &jdef); TEST_ASSERT_NOT_NULL(j); return j; } /* Registers a worker in g_dfr.workermgr with ep pointed at the given * endpoint (host.status defaults to 0/ready via calloc). */ static void dfr_seed_worker(const char* wname, endpoint_t* ep) { xce_worker_t wdef; hostattrs_t* attrs; worker_t* w = NULL; attrs = calloc(1, sizeof(*attrs)); TEST_ASSERT_NOT_NULL(attrs); attrs->hosttype = makeString("LINUX"); attrs->hostmodel = makeString("x86"); attrs->cpufactor = 1.0f; attrs->maxcpus = 4; memset(&wdef, 0, sizeof(wdef)); wdef.name = (char*)wname; wdef.attrs = attrs; TEST_ASSERT_NOT_NULL(makeWorker(g_dfr.workermgr, &wdef)); lock(g_dfr.workermgr, RTTI_LOCK_READ); key_find(g_dfr.workermgr, (void*)wname, (void**)&w); unlock(g_dfr.workermgr); TEST_ASSERT_NOT_NULL(w); w->ep = ep; } /* Canary-reference technique (see test_dispatch_fail_terminal.c's header * for the full rationale): arm BEFORE the call under test, release AFTER * and re-touch j -- an over-release inside dispatch_a_job_test() would * have freed j while the canary is still held, a heap-use-after-free ASAN * catches on the post-call touch. */ static void dfr_arm_ref_canary(job_t* j) { reference(j); } static void dfr_assert_ref_not_over_released(job_t* j, uint64_t expect_jobid) { dereference(j); TEST_ASSERT_EQUAL_UINT64_MESSAGE(expect_jobid, j->job.jobid, "j must still be safely readable after releasing the canary " "reference -- an over-release inside dispatch_a_job_test() would " "have freed j while the canary was still held, making this a " "heap-use-after-free"); } /* worker-rejoin-resync task 2.3: same canary technique as above, applied to * `w` -- the new disconnected-worker branch dereference(w)s the reference * dispatch_a_job_() itself took at its key_find() lookup, and this is the * exact class of reference bug bug-1627 fixed (leak site #3, the WORKER_ * NOT_READY/ep-NULL combined check). */ static void dfr_arm_worker_canary(worker_t* w) { reference(w); } static void dfr_assert_worker_ref_not_over_released(worker_t* w, const char* expect_name) { dereference(w); TEST_ASSERT_EQUAL_STRING_MESSAGE(expect_name, w->host.name, "w must still be safely readable after releasing the canary " "reference -- an over-release inside dispatch_a_job_test() would " "have freed w while the canary was still held, making this a " "heap-use-after-free"); } /* A "broken" endpoint: real socket fd, real iomgr, deliberately NOT * registered in the iomgr's epoll -- EPOLL_CTL_MOD on it fails with ENOENT * (the exact post-failed-reconnect condition endpoint_update_epoll() * surfaces, bug-646), making xce_startjob_req() -> xce_request() -> * endpoint_enq() fail deterministically and repeatably. This is the same * technique test_netio_arm_failure.c/test_request_send_propagation.c * (src/lib/testsuite/) already establish and prove live for exactly this * failure mode; dataq_enq() itself still always succeeds first (the * request bytes get queued), only the poll-arm step fails, so this is * safe to call repeatedly against the same endpoint -- ep->flags never * reaches the target value, so the same failure recurs every call. */ static iomgr_t g_dfr_broken_iomgr; static endpoint_t* g_dfr_broken_ep; static void dfr_setup_broken_ep(void) { memset(&g_dfr_broken_iomgr, 0, sizeof(g_dfr_broken_iomgr)); TEST_ASSERT_EQUAL_INT(0, iomgr_init(&g_dfr_broken_iomgr)); g_dfr_broken_ep = endpoint_get(); TEST_ASSERT_NOT_NULL(g_dfr_broken_ep); g_dfr_broken_ep->fd = socket(AF_INET, SOCK_STREAM, 0); TEST_ASSERT_TRUE(g_dfr_broken_ep->fd >= 0); g_dfr_broken_ep->iomgr = &g_dfr_broken_iomgr; g_dfr_broken_ep->ssl_handshake_done = 1; g_dfr_broken_ep->flags = 0; } static void dfr_teardown_broken_ep(void) { endpoint_fini(g_dfr_broken_ep); /* closes fd, drains/frees queued pkts */ free(g_dfr_broken_ep); g_dfr_broken_ep = NULL; close(g_dfr_broken_iomgr.epoll_fd); } /* Frees every packet left queued on an endpoint's outq (the startjob request * a SUCCESSFUL dispatch_a_job_test() call enqueues) -- mirrors * test_dispatch_commit.c's dc_drain_ep exactly; this file otherwise never * needs a live dispatch to succeed, so it has no drain helper of its own * until the counter-reset test below. */ static void dfr_drain_ep(endpoint_t* ep) { data_item_t* it; while ((it = endpoint_deq(ep)) != NULL) { io_packet_t* pkt = ITPTR_TO_PACKET(it); free_packet(pkt); free(pkt); } } /* ── SITE 1/2: worker lookup finds nothing ───────────────────────────── * See file header: covers whichever of XCE_DISPATCH_FAIL_SITE_WORKER_LOOKUP * / XCE_DISPATCH_FAIL_SITE_WORKER_NULL the runtime actually takes -- both * resolve via the identical classify/resolve procedure, so this test's * assertions hold either way. */ #define DFR_JOBID_WORKER_NOTFOUND 91201ULL static void test_worker_notfound_resolves_retryable(void) { job_t* j; xce_job_alloc_t* alloc; int rc; int before_count; dfr_setup(); j = dfr_seed_job(DFR_JOBID_WORKER_NOTFOUND, "dfr-q1"); /* Deliberately no matching worker registered in g_dfr.workermgr. */ alloc = make_job_alloc(DFR_JOBID_WORKER_NOTFOUND, 1, make_host_allocs(1, make_host_alloc((char*)"no-such-worker-dfr", 1, make_allocs(1, make_alloc((char*)"slots", NULL, 0, 1.0f, 0))))); TEST_ASSERT_NOT_NULL(alloc); cbrp_notify_capture_reset_ext(); before_count = cbrp_notify_capture_count_ext(); dfr_arm_ref_canary(j); rc = dispatch_a_job_test(&g_dfr, NULL, alloc); TEST_ASSERT_TRUE_MESSAGE(rc < 0, "a dispatch to a nonexistent worker must still fail, unchanged " "return contract from before this task"); TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state, "Req 4.1: a retryable dispatch failure below the bound must leave " "the job PEND, eligible for another attempt"); TEST_ASSERT_NULL_MESSAGE(j->job.allocs, "Req 4.4: no partial allocation data may be left behind"); TEST_ASSERT_EQUAL_UINT32_MESSAGE(1, j->dispatch_fail_count, "Req 4.3: the consecutive-failure counter must increment by " "exactly 1"); dfr_assert_ref_not_over_released(j, DFR_JOBID_WORKER_NOTFOUND); TEST_ASSERT_EQUAL_INT_MESSAGE(before_count + 1, cbrp_notify_capture_count_ext(), "Req 4.1: a below-bound retryable failure must call " "notify_dispatch() exactly once"); TEST_ASSERT_EQUAL_INT(NotifyData_data_jobDispatchFail, cbrp_notify_capture_event_ext(before_count)); TEST_ASSERT_EQUAL_UINT64(DFR_JOBID_WORKER_NOTFOUND, cbrp_notify_capture_jobid_ext(before_count)); TEST_ASSERT_EQUAL_INT(rc, cbrp_notify_capture_code_ext(before_count)); free_JobAlloc_ptr(&alloc); dfr_teardown(); printf("test_worker_notfound_resolves_retryable: PASS\n"); } /* ── SITE 3: XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY ─────────────────── * A registered, CONNECTED worker (w->ep != NULL) with a nonzero * w->host.status bit (XCE_WORKER_BUSY, picked arbitrarily -- any bit takes * the same unchanged path) hits "w->host.status != 0" -- the still- * unmodified branch of dispatch_a_job_()'s split combined check * (worker-rejoin-resync task 2.3). Before that split, this test used an * ep-left-NULL worker to reach the combined check; that condition now * takes the NEW, separate disconnected-worker branch instead (see the new * tests below), which resolves differently (immediate PEND, no retry, no * dispatch_fail_count increment) -- so this test is updated to use a * connected-but-busy worker, keeping it a genuine, unchanged regression * proof for XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY's own retry-then- * terminal-fail behavior (Req 5.1's design.md backstop). */ #define DFR_JOBID_WORKER_NOTREADY 91202ULL static void test_worker_notready_resolves_retryable(void) { job_t* j; worker_t* w = NULL; endpoint_t connected_ep; xce_job_alloc_t* alloc; int rc; int before_count; dfr_setup(); j = dfr_seed_job(DFR_JOBID_WORKER_NOTREADY, "dfr-q2"); /* A non-NULL ep (its contents are never touched -- this branch fails * before any RPC is attempted) plus a nonzero status bit: connected, * but not ready. */ memset(&connected_ep, 0, sizeof(connected_ep)); dfr_seed_worker("dfr-wkr-notready", &connected_ep); lock(g_dfr.workermgr, RTTI_LOCK_READ); key_find(g_dfr.workermgr, (void*)"dfr-wkr-notready", (void**)&w); unlock(g_dfr.workermgr); TEST_ASSERT_NOT_NULL(w); w->host.status = XCE_WORKER_BUSY; alloc = make_job_alloc(DFR_JOBID_WORKER_NOTREADY, 1, make_host_allocs(1, make_host_alloc((char*)"dfr-wkr-notready", 1, make_allocs(1, make_alloc((char*)"slots", NULL, 0, 1.0f, 0))))); TEST_ASSERT_NOT_NULL(alloc); /* worker-rejoin-resync task 2.3: explicit proof this test still reaches * XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY's branch of the split check -- * a connected (non-NULL ep), nonzero-status worker, not the new * disconnected-worker branch below. */ TEST_ASSERT_NOT_NULL(w->ep); TEST_ASSERT_TRUE(w->host.status != 0); cbrp_notify_capture_reset_ext(); before_count = cbrp_notify_capture_count_ext(); dfr_arm_ref_canary(j); rc = dispatch_a_job_test(&g_dfr, NULL, alloc); TEST_ASSERT_TRUE_MESSAGE(rc < 0, "a dispatch to a not-ready worker must still fail, unchanged " "return contract from before this task"); TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state, "Req 4.1: a retryable dispatch failure below the bound must leave " "the job PEND, eligible for another attempt"); TEST_ASSERT_NULL_MESSAGE(j->job.allocs, "Req 4.4: no partial allocation data may be left behind"); TEST_ASSERT_EQUAL_UINT32_MESSAGE(1, j->dispatch_fail_count, "Req 4.3: the consecutive-failure counter must increment by " "exactly 1 -- unchanged existing WORKER_NOT_READY retry behavior " "(worker-rejoin-resync task 2.3 must not affect this branch)"); dfr_assert_ref_not_over_released(j, DFR_JOBID_WORKER_NOTREADY); TEST_ASSERT_EQUAL_INT_MESSAGE(before_count + 1, cbrp_notify_capture_count_ext(), "Req 4.1: a below-bound retryable failure must call " "notify_dispatch() exactly once"); TEST_ASSERT_EQUAL_INT(NotifyData_data_jobDispatchFail, cbrp_notify_capture_event_ext(before_count)); TEST_ASSERT_EQUAL_UINT64(DFR_JOBID_WORKER_NOTREADY, cbrp_notify_capture_jobid_ext(before_count)); TEST_ASSERT_EQUAL_INT(rc, cbrp_notify_capture_code_ext(before_count)); free_JobAlloc_ptr(&alloc); dfr_teardown(); printf("test_worker_notready_resolves_retryable: PASS\n"); } /* ── worker-rejoin-resync task 2.3: disconnected-worker branch ───────── * (Req 5.1, 5.4) -- NOT one of the four RETRYABLE sites above: a dispatch * attempt to a worker the controller is not currently connected to * (w->ep == NULL) now fails immediately and returns the job to PEND via * record_job_reconcile_pending_fact(), bypassing xce_classify_dispatch_ * fail()/mark_job_fail() entirely -- no dispatch_fail_count increment, no * retry, no terminal resolution. * * Realistic pre-dispatch state: dispatch_a_job_()'s only real caller * dispatches a job the scheduler picked from its PEND pool, so this test * seeds an ordinary PEND job (dfr_seed_job's default) -- record_job_ * reconcile_pending_fact()'s apply_job_update_transition(job, JOB_STAT_ * PEND) call therefore hits JOB_TRANSITION_NOOP_DUPLICATE (job already * PEND), not APPLIED: no mutation, no allocs touch (there is none to * release), no durable fact, no notify -- exactly as design.md's own * Components section describes for this outcome, and as this task's own * prompt anticipates ("handle this correctly: still don't call mark_job_ * fail, still don't retry, but there's nothing to release/notify either * since nothing changed"). */ #define DFR_JOBID_DISCONNECTED 91206ULL static int g_dfr_update_hook_fires; static int dfr_job_update_spy_hook(int type, int situation, int detail, void* data, va_list ap) { (void)type; (void)data; (void)ap; if (situation == HOOK_AFTER_EVENT && detail == HOOK_DETAIL_JOB_UPDATE) g_dfr_update_hook_fires++; return 0; } static void test_disconnected_worker_fails_immediately_no_retry(void) { job_t* j; worker_t* w = NULL; xce_job_alloc_t* alloc; int rc; int before_count; uint32_t fail_count_before; dfr_setup(); g_dfr_update_hook_fires = 0; TEST_ASSERT_EQUAL_INT(0, xce_add_hook(&g_dfr.hooksys, XCE_CONTROLLER_HOOK_JOB, dfr_job_update_spy_hook, NULL)); j = dfr_seed_job(DFR_JOBID_DISCONNECTED, "dfr-q-disc"); fail_count_before = j->dispatch_fail_count; dfr_seed_worker("dfr-wkr-disc1", NULL); /* ep left NULL: disconnected */ lock(g_dfr.workermgr, RTTI_LOCK_READ); key_find(g_dfr.workermgr, (void*)"dfr-wkr-disc1", (void**)&w); unlock(g_dfr.workermgr); TEST_ASSERT_NOT_NULL(w); TEST_ASSERT_NULL_MESSAGE(w->ep, "the disconnected branch requires " "w->ep == NULL -- confirm the fixture actually constructs that"); alloc = make_job_alloc(DFR_JOBID_DISCONNECTED, 1, make_host_allocs(1, make_host_alloc((char*)"dfr-wkr-disc1", 1, make_allocs(1, make_alloc((char*)"slots", NULL, 0, 1.0f, 0))))); TEST_ASSERT_NOT_NULL(alloc); cbrp_notify_capture_reset_ext(); before_count = cbrp_notify_capture_count_ext(); /* No reference/leak regression (task 2.3's own Tests-to-write clause): * canary both j (bug-1627's original leak class) and w (this branch's * own dereference(w), the same site bug-1627 fixed for the WORKER_NOT_ * READY branch). */ dfr_arm_ref_canary(j); dfr_arm_worker_canary(w); rc = dispatch_a_job_test(&g_dfr, NULL, alloc); TEST_ASSERT_TRUE_MESSAGE(rc < 0, "a dispatch to a disconnected worker must still fail"); /* Observable completion (Req 5.4): job ends up PEND (unchanged -- it * started PEND, a realistic pre-dispatch state), single attempt, no * retry loop. */ TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state, "Req 5.4: a dispatch to a disconnected worker must return the job " "to pending (here: leave it PEND, since it started PEND)"); TEST_ASSERT_NOT_EQUAL_MESSAGE(JOB_STAT_EXIT, j->job.state, "Req 5.4: this must NOT become a permanent/terminal failure -- " "mark_job_fail() must never be invoked on this path"); TEST_ASSERT_NULL_MESSAGE(j->job.allocs, "no partial allocation data may be left behind"); TEST_ASSERT_EQUAL_UINT32_MESSAGE(fail_count_before, j->dispatch_fail_count, "Req 5.4: dispatch_fail_count must be left UNCHANGED -- this " "branch must never reach xce_classify_dispatch_fail()'s retry-" "counting machinery"); /* "single attempt, no retry loop": notify_dispatch() is dispatch_a_ * job_()'s sole retry-signaling mechanism (see the four RETRYABLE-site * tests above) -- it must never fire on this path. */ TEST_ASSERT_EQUAL_INT_MESSAGE(before_count, cbrp_notify_capture_count_ext(), "Req 5.4: a disconnected-worker dispatch failure must never call " "notify_dispatch() -- there is no retry to signal"); /* NOOP_DUPLICATE outcome (job was already PEND): no mutation occurred, * so no notify is needed or expected either -- omitting the notify * here is correct, not bug-1679's omission-is-a-bug case (that applies * only to an actual APPLIED transition; see the APPLIED-path test * below for that proof). */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_dfr_update_hook_fires, "a no-op reconciliation (job already PEND) must not fire the " "scheduler notify path -- nothing changed"); dfr_assert_ref_not_over_released(j, DFR_JOBID_DISCONNECTED); dfr_assert_worker_ref_not_over_released(w, "dfr-wkr-disc1"); xce_disable_hook(&g_dfr.hooksys, XCE_CONTROLLER_HOOK_JOB, dfr_job_update_spy_hook); free_JobAlloc_ptr(&alloc); dfr_teardown(); printf("test_disconnected_worker_fails_immediately_no_retry: PASS\n"); } /* ── worker-rejoin-resync task 2.3: JOB_TRANSITION_APPLIED wiring proof * ──────────────────────────────────────────────────────────────────── * The NOOP_DUPLICATE test above exercises the only precondition dispatch_ * a_job_()'s real caller can currently construct (a PEND job -- the * scheduler only ever dispatches from its PEND pool, so dispatch_a_job_() * is never actually invoked on a RUN job today; the same "not reachable * via any real caller in the currently-implemented system" caveat * test_rpc_failure_bound_resets_after_intervening_success's header * documents for the four RETRYABLE sites applies here too -- see bug-1631). * This test instead directly seeds a RUN job with a real, committed * job->job.allocs (mirroring test_worker_resync.c's own technique) to * exercise record_job_reconcile_pending_fact()'s OTHER outcome * (JOB_TRANSITION_APPLIED, not NOOP_DUPLICATE) THROUGH this exact new * dispatch_a_job_() call site -- proving the branch's result-handling * (unlock -> notify-only-on-APPLIED -> dereference) is wired correctly, * complementing task 2.1's test_worker_resync.c (which already proves * record_job_reconcile_pending_fact() itself is correct in isolation, not * through this caller). Not a claim that THIS exact precondition (RUN job * hitting the disconnected branch) is reachable in production today -- * design.md's own bug-1631 narrative is that record_job_reconcile_pending_ * fact() releases allocs at the point of RECONCILIATION (a separate, * later-task call site, resync.c), so by the time any job reaches a fresh * dispatch_a_job_() call it is already PEND with allocs already NULL. */ #define DFR_JOBID_DISCONNECTED_RUN 91207ULL static uint64_t dfr_current_lsn(void) { uint64_t lsn; pthread_mutex_lock(&g_dfr.persist.lock); lsn = g_dfr.persist.current_lsn; pthread_mutex_unlock(&g_dfr.persist.lock); return lsn; } static void test_disconnected_worker_reconciles_run_job_releases_allocs(void) { job_t* j; worker_t* w = NULL; xce_job_alloc_t* prior_alloc; xce_job_alloc_t* redispatch_alloc; int rc; int before_count; uint64_t lsn_before, lsn_after; dfr_setup(); g_dfr_update_hook_fires = 0; TEST_ASSERT_EQUAL_INT(0, xce_add_hook(&g_dfr.hooksys, XCE_CONTROLLER_HOOK_JOB, dfr_job_update_spy_hook, NULL)); j = dfr_seed_job(DFR_JOBID_DISCONNECTED_RUN, "dfr-q-disc2"); /* Simulate a prior successful dispatch (mirrors test_worker_resync.c's * own technique): real allocs committed, state RUN, ownership * transferred into job->job.allocs. */ prior_alloc = make_job_alloc(DFR_JOBID_DISCONNECTED_RUN, 1, make_host_allocs(1, make_host_alloc((char*)"dfr-wkr-disc2-prior", 1, make_allocs(1, make_alloc((char*)"slots", NULL, 0, 1.0f, 0))))); TEST_ASSERT_NOT_NULL(prior_alloc); j->job.state = JOB_STAT_RUN; j->job.allocs = prior_alloc; dfr_seed_worker("dfr-wkr-disc2", NULL); /* ep left NULL: disconnected */ lock(g_dfr.workermgr, RTTI_LOCK_READ); key_find(g_dfr.workermgr, (void*)"dfr-wkr-disc2", (void**)&w); unlock(g_dfr.workermgr); TEST_ASSERT_NOT_NULL(w); redispatch_alloc = make_job_alloc(DFR_JOBID_DISCONNECTED_RUN, 1, make_host_allocs(1, make_host_alloc((char*)"dfr-wkr-disc2", 1, make_allocs(1, make_alloc((char*)"slots", NULL, 0, 1.0f, 0))))); TEST_ASSERT_NOT_NULL(redispatch_alloc); cbrp_notify_capture_reset_ext(); before_count = cbrp_notify_capture_count_ext(); lsn_before = dfr_current_lsn(); dfr_arm_ref_canary(j); dfr_arm_worker_canary(w); rc = dispatch_a_job_test(&g_dfr, NULL, redispatch_alloc); TEST_ASSERT_TRUE_MESSAGE(rc < 0, "a dispatch to a disconnected worker must still fail"); TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state, "Req 4.6/5.4: a RUN job redispatched to a disconnected worker must " "be returned to PEND"); TEST_ASSERT_NULL_MESSAGE(j->job.allocs, "bug-1631: the job's prior resource allocation must be released, " "so a later redispatch never inherits a stale allocation"); TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, j->dispatch_fail_count, "Req 5.4: dispatch_fail_count must stay untouched on this path"); TEST_ASSERT_EQUAL_INT_MESSAGE(before_count, cbrp_notify_capture_count_ext(), "notify_dispatch() (the retry-signal path) must never fire here"); lsn_after = dfr_current_lsn(); TEST_ASSERT_TRUE_MESSAGE(lsn_after > lsn_before, "an APPLIED reconciliation must durably append a WAL_JOB_UPDATE " "fact"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_dfr_update_hook_fires, "bug-1679: an APPLIED reconciliation must notify the scheduler " "exactly once, via this exact dispatch_a_job_() call site"); dfr_assert_ref_not_over_released(j, DFR_JOBID_DISCONNECTED_RUN); dfr_assert_worker_ref_not_over_released(w, "dfr-wkr-disc2"); xce_disable_hook(&g_dfr.hooksys, XCE_CONTROLLER_HOOK_JOB, dfr_job_update_spy_hook); free_JobAlloc_ptr(&redispatch_alloc); dfr_teardown(); printf("test_disconnected_worker_reconciles_run_job_releases_allocs: " "PASS\n"); } /* ── SITE 4: XCE_DISPATCH_FAIL_SITE_RPC_FAILURE ──────────────────────── * A ready worker whose ep is deliberately "broken" (see dfr_setup_broken_ep * above) makes xce_startjob_req() itself fail. */ #define DFR_JOBID_RPC_FAILURE 91203ULL static void test_rpc_failure_resolves_retryable(void) { job_t* j; xce_job_alloc_t* alloc; int rc; int before_count; dfr_setup(); dfr_setup_broken_ep(); j = dfr_seed_job(DFR_JOBID_RPC_FAILURE, "dfr-q3"); dfr_seed_worker("dfr-wkr-rpcfail", g_dfr_broken_ep); alloc = make_job_alloc(DFR_JOBID_RPC_FAILURE, 1, make_host_allocs(1, make_host_alloc((char*)"dfr-wkr-rpcfail", 1, make_allocs(1, make_alloc((char*)"slots", NULL, 0, 1.0f, 0))))); TEST_ASSERT_NOT_NULL(alloc); cbrp_notify_capture_reset_ext(); before_count = cbrp_notify_capture_count_ext(); dfr_arm_ref_canary(j); rc = dispatch_a_job_test(&g_dfr, NULL, alloc); TEST_ASSERT_TRUE_MESSAGE(rc < 0, "xce_startjob_req() against a broken (unarmed-poll) endpoint must " "fail, unchanged return contract from before this task"); TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state, "Req 4.1: a retryable dispatch failure below the bound must leave " "the job PEND, eligible for another attempt"); TEST_ASSERT_NULL_MESSAGE(j->job.allocs, "Req 4.4: no partial allocation data may be left behind"); TEST_ASSERT_EQUAL_UINT32_MESSAGE(1, j->dispatch_fail_count, "Req 4.3: the consecutive-failure counter must increment by " "exactly 1"); dfr_assert_ref_not_over_released(j, DFR_JOBID_RPC_FAILURE); TEST_ASSERT_EQUAL_INT_MESSAGE(before_count + 1, cbrp_notify_capture_count_ext(), "Req 4.1: a below-bound retryable failure must call " "notify_dispatch() exactly once"); TEST_ASSERT_EQUAL_INT(NotifyData_data_jobDispatchFail, cbrp_notify_capture_event_ext(before_count)); TEST_ASSERT_EQUAL_UINT64(DFR_JOBID_RPC_FAILURE, cbrp_notify_capture_jobid_ext(before_count)); TEST_ASSERT_EQUAL_INT(rc, cbrp_notify_capture_code_ext(before_count)); free_JobAlloc_ptr(&alloc); dfr_teardown_broken_ep(); dfr_teardown(); printf("test_rpc_failure_resolves_retryable: PASS\n"); } /* ── Bounded escalation (Req 4.3) ─────────────────────────────────────── * Drives the SAME job through XCE_DISPATCH_FAIL_SITE_RPC_FAILURE * repeatedly (easiest of the four to trigger deterministically and * repeatably via the broken-endpoint technique). Confirms the 3rd * consecutive failure (== design.md's stated default bound) still * resolves retryable, and the 4th resolves TERMINAL instead. */ #define DFR_JOBID_BOUND 91204ULL static void test_rpc_failure_bounded_escalation(void) { job_t* j; int i; dfr_setup(); dfr_setup_broken_ep(); j = dfr_seed_job(DFR_JOBID_BOUND, "dfr-q4"); dfr_seed_worker("dfr-wkr-bound", g_dfr_broken_ep); /* Attempts 1-3: within the bound (XCE_DISPATCH_FAIL_RETRY_BOUND == 3, * job.c) -- each must resolve retryable. */ for (i = 1; i <= 3; i++) { xce_job_alloc_t* alloc; int rc; alloc = make_job_alloc(DFR_JOBID_BOUND, 1, make_host_allocs(1, make_host_alloc((char*)"dfr-wkr-bound", 1, make_allocs(1, make_alloc((char*)"slots", NULL, 0, 1.0f, 0))))); TEST_ASSERT_NOT_NULL(alloc); rc = dispatch_a_job_test(&g_dfr, NULL, alloc); TEST_ASSERT_TRUE(rc < 0); TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state, "an attempt within the bound must leave the job PEND, still " "eligible for retry"); TEST_ASSERT_EQUAL_UINT32((uint32_t)i, j->dispatch_fail_count); TEST_ASSERT_NULL(j->job.allocs); free_JobAlloc_ptr(&alloc); } /* Attempt 4: exceeds the bound -- must resolve TERMINAL instead of * retrying again. */ { xce_job_alloc_t* alloc; int rc; alloc = make_job_alloc(DFR_JOBID_BOUND, 1, make_host_allocs(1, make_host_alloc((char*)"dfr-wkr-bound", 1, make_allocs(1, make_alloc((char*)"slots", NULL, 0, 1.0f, 0))))); TEST_ASSERT_NOT_NULL(alloc); rc = dispatch_a_job_test(&g_dfr, NULL, alloc); TEST_ASSERT_TRUE(rc < 0); TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_EXIT, j->job.state, "Req 4.3: the failure that pushes dispatch_fail_count past the " "bound must resolve the job to terminal instead of retrying " "again"); TEST_ASSERT_EQUAL_UINT32_MESSAGE(4, j->dispatch_fail_count, "the counter itself keeps counting past the bound -- only the " "resolution changes"); TEST_ASSERT_NULL_MESSAGE(j->job.allocs, "Req 4.5: no partial allocation data may be left behind on the " "terminal resolution either"); free_JobAlloc_ptr(&alloc); } dfr_teardown_broken_ep(); dfr_teardown(); printf("test_rpc_failure_bounded_escalation: PASS\n"); } /* ── task 5.2, scenario 3: bound resets across an intervening success ── * * Req 4.3's literal text: "consecutive... dispatch failures with no * intervening successful dispatch". test_rpc_failure_bounded_escalation * (above) already proves the bound itself (3 retryable, 4th terminal) for * an UNINTERRUPTED failure streak; test_dc_success_commits_alloc_state_ * and_resets_counter (test_dispatch_commit.c) already proves a success * resets the field to 0 in isolation, but seeds the pre-success count by * directly poking j->dispatch_fail_count rather than by driving a real * prior failure through dispatch_a_job_test() -- so neither test proves * that a SUBSEQUENT real failure, after a real intervening success, is * actually evaluated as if only one failure had ever occurred. This test * closes that gap: fail -> succeed -> fail, all three driven through * dispatch_a_job_test() itself, no manual field pokes anywhere. * * Technique: dispatch_a_job_() never gates on job.state (confirmed by * reading job.c directly -- it operates purely from alloc/jobid/worker * lookups) and re-resolves the worker via a fresh key_find() on every * call, so toggling the SAME worker's w->ep between the "broken" endpoint * (dfr_setup_broken_ep's RPC-failure technique, XCE_DISPATCH_FAIL_SITE_ * RPC_FAILURE) and a "working" one (test_dispatch_commit.c's zeroed, * no-iomgr g_dc_worker_ep technique -- xce_startjob_req()'s enqueue * succeeds unconditionally against it) between calls is sufficient to * drive fail -> succeed -> fail against the identical job/worker pair, * with no need for a second fixture. * * The genuinely load-bearing proof (this is the part most likely to catch * a subtly-wrong fix, e.g. the reset being skipped or landing on the * wrong value): after the intervening success, the counter must count * PURELY from the post-reset baseline -- checked at each of three more * consecutive failures (i=1,2,3, the loop below). This assertion is * GENUINELY PROVEN, both by direct runtime observation (a temporary debug * probe confirmed dispatch_fail_count reads exactly 1 after the loop's * first call, immediately following a confirmed-0 post-success read -- if * the reset (task 3.1's `j->dispatch_fail_count = 0;`) had been skipped or * wrong, this would read 2, not 1) and by a real revert/rebuild/rerun of * that exact line (see the note below the loop's count assertion). * * IMPORTANT FINDING (bug-1631, logged this task): this same scenario also * exposed that dispatch_a_job_()'s four retryable-failure branches never * reset j->job.state/j->job.allocs, which is correct for a job's very * FIRST dispatch attempt (never left PEND, nothing to release) but would * NOT be correct once a job has had a prior REAL successful dispatch, as * this scenario's step 2 constructs -- Requirement 4.1 ("return the job * to a pending state") and Requirement 4.4/4.5 ("release the resource- * allocation and commitment data") would not be satisfied in that case. * A subsequent kiro-debug investigation (HIGH confidence) determined this * "job is RUN, then re-dispatched, then fails" scenario is NOT reachable * via any real caller in the currently-implemented system -- see the * loop's own comment for the full reachability analysis, and bug-1631 for * the complete write-up (a legitimate, deliberately future-scoped finding, * not an in-scope defect). Accordingly, this test does NOT assert * job.state/job.allocs on the post-success retryable failures below or in * step 4 -- only the counter-reset behavior (Req 4.3, genuinely reachable * and required) and the bound-exceeded terminal state transition (step 4, * reachable via a normal PEND-origin dispatch) are asserted. */ #define DFR_JOBID_RESET 91205ULL static void test_rpc_failure_bound_resets_after_intervening_success(void) { job_t* j; worker_t* w = NULL; endpoint_t working_ep; int i; dfr_setup(); dfr_setup_broken_ep(); j = dfr_seed_job(DFR_JOBID_RESET, "dfr-q5"); dfr_seed_worker("dfr-wkr-reset", g_dfr_broken_ep); memset(&working_ep, 0, sizeof(working_ep)); working_ep.fd = -1; /* zeroed, no iomgr -- xce_startjob_req()'s enqueue * succeeds unconditionally against it (see file * header / test_dispatch_commit.c's g_dc_worker_ep) */ /* Step 1: one real retryable failure via the broken endpoint -- * dispatch_fail_count -> 1, job stays PEND. Identical technique to * test_rpc_failure_resolves_retryable above, driven inline here so the * whole fail -> succeed -> fail sequence stays in one test. */ { xce_job_alloc_t* alloc; int rc; alloc = make_job_alloc(DFR_JOBID_RESET, 1, make_host_allocs(1, make_host_alloc((char*)"dfr-wkr-reset", 1, make_allocs(1, make_alloc((char*)"slots", NULL, 0, 1.0f, 0))))); TEST_ASSERT_NOT_NULL(alloc); rc = dispatch_a_job_test(&g_dfr, NULL, alloc); TEST_ASSERT_TRUE(rc < 0); TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state, "step 1: the first real failure must leave the job PEND"); TEST_ASSERT_EQUAL_UINT32(1, j->dispatch_fail_count); TEST_ASSERT_NULL(j->job.allocs); free_JobAlloc_ptr(&alloc); } /* Step 2: swap the SAME worker's ep to a working one and dispatch * again -- a genuine successful dispatch, driven through the real * production success path (job.c), not a manual field poke. Must * commit job.allocs/state=RUN AND reset dispatch_fail_count to 0. */ lock(g_dfr.workermgr, RTTI_LOCK_READ); key_find(g_dfr.workermgr, (void*)"dfr-wkr-reset", (void**)&w); unlock(g_dfr.workermgr); TEST_ASSERT_NOT_NULL(w); w->ep = &working_ep; { xce_job_alloc_t* alloc; int rc; alloc = make_job_alloc(DFR_JOBID_RESET, 1, make_host_allocs(1, make_host_alloc((char*)"dfr-wkr-reset", 1, make_allocs(1, make_alloc((char*)"slots", NULL, 0, 1.0f, 0))))); TEST_ASSERT_NOT_NULL(alloc); rc = dispatch_a_job_test(&g_dfr, NULL, alloc); TEST_ASSERT_TRUE_MESSAGE(rc >= 0, "step 2: a dispatch to a live, no-iomgr worker endpoint must " "succeed -- xce_startjob_req() is a fire-and-forget enqueue, " "not a blocking round trip"); TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j->job.state, "step 2: the intervening successful dispatch must commit RUN"); TEST_ASSERT_NOT_NULL(j->job.allocs); TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, j->dispatch_fail_count, "Req 4.3: a successful dispatch must reset the consecutive " "retryable-failure counter to zero"); free_JobAlloc_ptr(&alloc); /* allocs already MOVE_PTR'd out -- top- * level struct only */ } dfr_drain_ep(&working_ep); /* Step 3: swap back to the broken endpoint and fail 3 more times -- the * genuinely load-bearing proof (see this test's header). Each of these * three must still resolve retryable/PEND; if the reset above had not * genuinely happened, the job would already be TERMINAL by the second * iteration of this loop (count 1 carried over + 2 new = 3... and the * THIRD would push it to 4, terminal one failure early -- see header * for the exact divergence this loop is designed to catch). */ w->ep = g_dfr_broken_ep; for (i = 1; i <= 3; i++) { xce_job_alloc_t* alloc; int rc; alloc = make_job_alloc(DFR_JOBID_RESET, 1, make_host_allocs(1, make_host_alloc((char*)"dfr-wkr-reset", 1, make_allocs(1, make_alloc((char*)"slots", NULL, 0, 1.0f, 0))))); TEST_ASSERT_NOT_NULL(alloc); rc = dispatch_a_job_test(&g_dfr, NULL, alloc); TEST_ASSERT_TRUE(rc < 0); /* THE primary proof this scenario exists for (task 5.2's own text: * "confirm the bound is evaluated AS IF ONLY ONE FAILURE HAD * OCCURRED"). Genuinely proven correct: if the counter reset * (task 3.1's `j->dispatch_fail_count = 0;`) were skipped or * wrong, this would already read (uint32_t)i + 1 (the pre-reset 1 * carried over) -- confirmed via a real revert/rebuild/rerun (see * this test's header). */ TEST_ASSERT_EQUAL_UINT32_MESSAGE((uint32_t)i, j->dispatch_fail_count, "Req 4.3: the counter must count purely from the post-success " "reset, not from the job's total pre-success failure history"); /* NOT asserted here: job.state == JOB_STAT_PEND / job.allocs == * NULL after this retryable failure. A kiro-debug investigation * (HIGH confidence; bug-1631) found this exact sub-scenario -- a * retryable dispatch failure occurring while the job's * controller-side state is already RUN, from step 2's real * success -- is not reachable via any real caller in the * currently-implemented system: dispatch_a_job_()'s only real * caller, handle_jobdispatch(), fires only on the scheduler's own * fresh DISPATCH decision, and the scheduler's picker permanently * removes a job from pendJobs the moment it dispatches it (task * 2.1) -- nothing implemented today ever re-adds a successfully- * dispatched (RUN) job back to pendJobs for redispatch. The only * mechanism design.md anticipates for that is scheduler- * preemption, still unimplemented (requirements-generated phase, * no approvals). Revisit once that (or an equivalent) mechanism * lands -- see bug-1631 for the full analysis. */ free_JobAlloc_ptr(&alloc); } /* Step 4: a 4th consecutive failure since the reset -- now exceeds the * bound, resolves TERMINAL, pinning the exact boundary the reset * establishes (not one failure earlier, not one failure later). */ { xce_job_alloc_t* alloc; int rc; alloc = make_job_alloc(DFR_JOBID_RESET, 1, make_host_allocs(1, make_host_alloc((char*)"dfr-wkr-reset", 1, make_allocs(1, make_alloc((char*)"slots", NULL, 0, 1.0f, 0))))); TEST_ASSERT_NOT_NULL(alloc); rc = dispatch_a_job_test(&g_dfr, NULL, alloc); TEST_ASSERT_TRUE(rc < 0); /* mark_job_fail()/apply_job_fail_transition() sets job.state = * JOB_STAT_EXIT UNCONDITIONALLY (confirmed by reading its full * body, job.c ~2303-2325 -- an absolute assignment, not gated on * the prior value), so this assertion is NOT affected by bug-1631 * and is expected to genuinely pass. */ TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_EXIT, j->job.state, "Req 4.3: the 4th consecutive failure since the reset must " "resolve the job terminal instead of retrying again"); TEST_ASSERT_EQUAL_UINT32(4, j->dispatch_fail_count); /* NOT asserted here: job.allocs == NULL on this terminal * resolution. Same bug-1631 unreachable-today scenario as the * loop above (apply_job_fail_transition() never touches * job.allocs, but only matters once a RUN job with committed * allocs can reach a retryable/terminal failure at all) -- see * that comment for the full analysis. */ free_JobAlloc_ptr(&alloc); } dfr_teardown_broken_ep(); dfr_teardown(); printf("test_rpc_failure_bound_resets_after_intervening_success: " "PASS\n"); } void test_dispatch_fail_retry(void) { UnitySetTestFile(__FILE__); printf("\n>> dispatch_a_job_() retryable dispatch-failure resolution " "tests (Req 4.1, 4.3, 4.4, 4.5)\n\n"); RUN_TEST(test_worker_notfound_resolves_retryable); RUN_TEST(test_worker_notready_resolves_retryable); RUN_TEST(test_rpc_failure_resolves_retryable); RUN_TEST(test_rpc_failure_bounded_escalation); RUN_TEST(test_rpc_failure_bound_resets_after_intervening_success); printf("\n>> dispatch_a_job_() disconnected-worker fail-fast tests " "(worker-rejoin-resync task 2.3, Req 5.1, 5.4)\n\n"); RUN_TEST(test_disconnected_worker_fails_immediately_no_retry); RUN_TEST(test_disconnected_worker_reconciles_run_job_releases_allocs); }