merge: sync Rongsong's scheduler-framework into chatbot
Full merge of origin/scheduler-framework (scheduler-core restructure,
failover-adoption, WAL_JOB_DISPATCH, apiserver backend health/heartbeat,
build-time versioning, bug-1673/1676/1678/1679/1680).
Conflict resolution:
- scheduler: adopt Rongsong's scheduler-core architecture (driver loop,
queue project/picker fields, resource bug-1569 fix)
- worker/controller_connector.c: keep Rongsong's reconnect/rejoin
structure, PORT our bug-1509 resend_job_status onto the rejoin success
path (his reconnect didn't re-report job state)
- apiserver backend: take Rongsong's health/heartbeat (supersedes our
e505066 circuit breaker)
- group.c: keep our bug-1513 comment
- misc: take theirs for additive fixes (go_build versioning,
test_group.c, persist.c RUSAGE_UPDATE, FilterRows bug-1680)
NOTE: our scheduler fixes (bug-1494 family dispatch-fail requeue,
PEND-reason classification) are NOT in Rongsong's branch — needs
verification/porting onto scheduler-core (separate follow-up).
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
lei.chen
commit
4e894d2184
395 changed files with 88517 additions and 5742 deletions
|
|
@ -1,3 +1,4 @@
|
|||
#include <assert.h>
|
||||
#include <errno.h>
|
||||
#include <pthread.h>
|
||||
#include <signal.h>
|
||||
|
|
@ -22,6 +23,19 @@ static int record_job_finish_fact(cluster_t*, job_t*, xce_jobfinish_t*);
|
|||
static int record_job_fail_fact(cluster_t*, job_t*, job_failinfo_t*);
|
||||
static int mark_job_fail(cluster_t*, job_t*, job_failinfo_t*);
|
||||
static int mark_job_update(cluster_t*, job_t*, int);
|
||||
/* scheduler-dispatch-commit task 3.3: dispatch_a_job_()'s four retryable
|
||||
* failure sites (job.c, below) need to notify the scheduler of a retryable
|
||||
* failure -- notify_dispatch() is defined AFTER dispatch_a_job_() in this
|
||||
* file with no forward declaration until now. Mirrors mark_job_fail's own
|
||||
* forward declaration immediately above; body/signature unmodified. */
|
||||
static void notify_dispatch(cluster_t*, endpoint_t*, uint64_t, int);
|
||||
|
||||
/* scheduler-dispatch-commit task 3.3 (Req 4.3): default bounded-retry limit
|
||||
* for a job's consecutive retryable dispatch failures with no intervening
|
||||
* successful dispatch -- design.md's own stated default (3 consecutive
|
||||
* failures). The failure that pushes job.dispatch_fail_count past this
|
||||
* bound resolves the job to terminal instead of retrying again. */
|
||||
#define XCE_DISPATCH_FAIL_RETRY_BOUND 3
|
||||
|
||||
/* A job is terminal once it carries a DONE or EXIT bit. Terminal jobs reject
|
||||
* backward (terminal -> non-terminal) transitions and treat re-applied
|
||||
|
|
@ -771,6 +785,36 @@ int restore_job_state(cluster_t* c, uint64_t jobid, uint32_t state)
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* bug-1678: WAL_JOB_DISPATCH's own sibling of restore_job_state() above, for
|
||||
* apply_wal_entry()'s standby catch-up / replicated-apply seam -- takes
|
||||
* ownership of an optional decoded resource allocation (NULL for a
|
||||
* state-only WAL_JOB_DISPATCH entry, e.g. one written before this fix) so a
|
||||
* promoted standby recovers EXEC_HOST the same way apply_committed_entry's
|
||||
* WAL_JOB_DISPATCH case already does for the primary's own boot replay. */
|
||||
int restore_job_dispatch_state(cluster_t* c, uint64_t jobid, uint32_t state,
|
||||
xce_job_alloc_t* alloc)
|
||||
{
|
||||
job_t* j = NULL;
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&j);
|
||||
unlock(c->jobmgr);
|
||||
if (!j) {
|
||||
if (alloc != NULL)
|
||||
free_JobAlloc_ptr(&alloc);
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
j->job.state = (int)state;
|
||||
if (alloc != NULL) {
|
||||
if (j->job.allocs != NULL)
|
||||
free_JobAlloc_ptr(&(j->job.allocs));
|
||||
j->job.allocs = alloc;
|
||||
}
|
||||
unlock(j);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int get_nslots_in_alloc(xce_host_alloc_t* alloc)
|
||||
{
|
||||
int i;
|
||||
|
|
@ -828,6 +872,96 @@ static int mk_launchinfo(xce_launch_t* launch, xce_job_alloc_t* alloc)
|
|||
return rc;
|
||||
}
|
||||
|
||||
/*
|
||||
* scheduler-dispatch-commit task 1.1 (Req 4.1, 4.2): classifies one of
|
||||
* dispatch_a_job_()'s six post-lookup failure points as retryable
|
||||
* (transient/environmental -- return the job to PEND for a later retry) or
|
||||
* terminal (a locally invalid decision that retrying will not fix -- resolve
|
||||
* the job to a terminal failed state). Declared static/file-local: nothing
|
||||
* outside job.c consumes this classification (design.md's own placement-
|
||||
* correction note). dispatch_a_job_() itself is wired to call this at each
|
||||
* of its six failure points by a later task; this task only adds the
|
||||
* disconnected, independently-testable classification step --
|
||||
* dispatch_a_job_() below is unmodified by this task.
|
||||
*/
|
||||
typedef enum {
|
||||
XCE_DISPATCH_FAIL_SITE_WORKER_LOOKUP = 0, /* workermgr key_find() rc<0 */
|
||||
XCE_DISPATCH_FAIL_SITE_WORKER_NULL = 1, /* worker not found (w==NULL) */
|
||||
XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY = 2, /* E_XCE_BAD_WORKER */
|
||||
XCE_DISPATCH_FAIL_SITE_INVALID_ALLOC = 3, /* nc <= 0 */
|
||||
XCE_DISPATCH_FAIL_SITE_QUEUE_EXEC = 4, /* get_queue_exec() failure */
|
||||
XCE_DISPATCH_FAIL_SITE_RPC_FAILURE = 5, /* xce_startjob_req() rc<0 */
|
||||
} xce_dispatch_fail_site_t;
|
||||
|
||||
typedef enum {
|
||||
XCE_DISPATCH_FAIL_RETRYABLE = 0,
|
||||
XCE_DISPATCH_FAIL_TERMINAL = 1,
|
||||
} xce_dispatch_fail_class_t;
|
||||
|
||||
/* Keyed on WHICH failure point was hit (site identity), not on the raw rc
|
||||
* that point returned: workermgr's key_find() and get_queue_exec() are two
|
||||
* independently-implemented failure sources not guaranteed to return
|
||||
* disjoint error codes (e.g. both could plausibly return -ENOENT for
|
||||
* unrelated reasons), so classifying by rc value alone would be ambiguous
|
||||
* whenever two sites happen to collide.
|
||||
*
|
||||
* Every named site is handled explicitly by its own case. The `default:`
|
||||
* arm below exists for the impossible-by-construction case of a `site` with
|
||||
* no matching enumerator (every current/planned call site passes a literal
|
||||
* enumerator, so this can only be reached by a genuine programming-invariant
|
||||
* violation, not by any operational/environmental condition) -- a plain
|
||||
* default-less switch does NOT get compiler-enforced exhaustiveness in C
|
||||
* (verified directly: GCC emits only a non-fatal `-Wreturn-type` "control
|
||||
* reaches end of non-void function" warning here, and this project sets no
|
||||
* -Werror anywhere), so falling off the end would silently return an
|
||||
* indeterminate value instead. This mirrors persist.c's wal_classify()
|
||||
* default arm exactly: ERROR-log and return a safe-side classification
|
||||
* (TERMINAL -- never risk an unbounded retry loop on an unrecognized site)
|
||||
* unconditionally, and additionally assert() only under XCE_TESTING, so
|
||||
* xce_controller itself (built without -DXCE_TESTING) can never self-
|
||||
* terminate on this path (CLAUDE.md's controller fail-loud/never-self-
|
||||
* terminate discipline: assert()/abort is reserved for exactly this kind of
|
||||
* impossible-state bug, but production must still never exit on it), while
|
||||
* controller_unit_test (built WITH -DXCE_TESTING) crashes loud and
|
||||
* immediately if it's ever hit. */
|
||||
static xce_dispatch_fail_class_t
|
||||
xce_classify_dispatch_fail(xce_dispatch_fail_site_t site)
|
||||
{
|
||||
switch (site) {
|
||||
case XCE_DISPATCH_FAIL_SITE_WORKER_LOOKUP:
|
||||
case XCE_DISPATCH_FAIL_SITE_WORKER_NULL:
|
||||
case XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY:
|
||||
case XCE_DISPATCH_FAIL_SITE_RPC_FAILURE:
|
||||
return XCE_DISPATCH_FAIL_RETRYABLE;
|
||||
case XCE_DISPATCH_FAIL_SITE_INVALID_ALLOC:
|
||||
case XCE_DISPATCH_FAIL_SITE_QUEUE_EXEC:
|
||||
return XCE_DISPATCH_FAIL_TERMINAL;
|
||||
default:
|
||||
ERROR("xce_classify_dispatch_fail: unrecognized failure site %d -- "
|
||||
"TERMINAL fallback; add it to the classification table",
|
||||
(int)site);
|
||||
#ifdef XCE_TESTING
|
||||
assert(0
|
||||
&& "unrecognized dispatch-fail site -- add it to "
|
||||
"xce_classify_dispatch_fail");
|
||||
#endif
|
||||
return XCE_DISPATCH_FAIL_TERMINAL;
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
/* Test-only accessor: xce_classify_dispatch_fail() and its enum types are
|
||||
* intentionally undeclared in any header (nothing outside job.c consumes
|
||||
* them), so the unit test reaches it through this thin, int-typed wrapper
|
||||
* instead -- mirroring job_array_apply_test_set_force_fail_at()'s existing
|
||||
* XCE_TESTING-only-export pattern earlier in this file (xce_controller
|
||||
* itself is built without -DXCE_TESTING, so this never ships). */
|
||||
int xce_classify_dispatch_fail_test(int site)
|
||||
{
|
||||
return (int)xce_classify_dispatch_fail((xce_dispatch_fail_site_t)site);
|
||||
}
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
||||
{
|
||||
int rc = 0;
|
||||
|
|
@ -837,8 +971,16 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
int nc;
|
||||
xce_job_exec_t einfo;
|
||||
xce_launch_t launchinfo;
|
||||
char jobidStr[64];
|
||||
|
||||
TRACE("%s jobid %ld", __func__, alloc->jobid);
|
||||
/* bug-1588: fmt_jobid() renders the raw idgen()-packed jobid in the
|
||||
* same human-readable "cluster-apiserver-timestamp-sequence" form
|
||||
* already used everywhere else this codebase logs/displays a jobid
|
||||
* (scheduler.c/handler.c/worker's process_manager.c, xcecli's job
|
||||
* listing, waldump) -- job.c's own TRACE lines were the one place
|
||||
* still printing the raw 64-bit packed integer. */
|
||||
fmt_jobid(jobidStr, sizeof(jobidStr), alloc->jobid);
|
||||
TRACE("%s jobid %s", __func__, jobidStr);
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
rc = key_find(c->jobmgr, (void*)alloc->jobid, (void**)&j);
|
||||
|
|
@ -859,6 +1001,54 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
|
||||
nc = alloc->nhosts;
|
||||
if (nc <= 0) {
|
||||
/* scheduler-dispatch-commit task 3.2 (Req 4.2, 4.5): resolves this
|
||||
* terminal (locally-invalid) dispatch failure -- j's own reference
|
||||
* held since the lookup above is released via the dereference(j)
|
||||
* call below exactly as it already was before this task (not
|
||||
* duplicated). Nothing was ever committed to j->job.allocs on this
|
||||
* path (task 3.1 gates that commit on a successful
|
||||
* xce_startjob_req(), never reached here), so Req 4.5's "release
|
||||
* any tentative allocation data" is satisfied vacuously by
|
||||
* construction -- there is nothing to release. */
|
||||
xce_dispatch_fail_class_t dfclass;
|
||||
job_failinfo_t failinfo = { .jobid = alloc->jobid };
|
||||
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
|
||||
dfclass
|
||||
= xce_classify_dispatch_fail(XCE_DISPATCH_FAIL_SITE_INVALID_ALLOC);
|
||||
switch (dfclass) {
|
||||
case XCE_DISPATCH_FAIL_TERMINAL:
|
||||
break;
|
||||
case XCE_DISPATCH_FAIL_RETRYABLE:
|
||||
default:
|
||||
/* Impossible per today's classification table (confirmed:
|
||||
* xce_classify_dispatch_fail() returns TERMINAL for this site
|
||||
* unconditionally) -- task 3.3 is what wires real retryable
|
||||
* handling at the other four sites, not this one. Fail loud
|
||||
* and fall through to the same terminal resolution as a safe
|
||||
* fallback, mirroring xce_classify_dispatch_fail()'s own
|
||||
* default: convention. */
|
||||
ERROR("dispatch_a_job_: XCE_DISPATCH_FAIL_SITE_INVALID_ALLOC "
|
||||
"classified non-terminal (%d) -- impossible per the "
|
||||
"current classification table; resolving terminal as a "
|
||||
"safe fallback", (int)dfclass);
|
||||
#ifdef XCE_TESTING
|
||||
assert(0
|
||||
&& "XCE_DISPATCH_FAIL_SITE_INVALID_ALLOC classified "
|
||||
"non-terminal -- update the classification table or "
|
||||
"this call site");
|
||||
#endif
|
||||
break;
|
||||
}
|
||||
|
||||
failinfo.code = -EINVAL;
|
||||
failinfo.detail = "dispatch failed: invalid allocation";
|
||||
|
||||
mark_job_fail(c, j, &failinfo);
|
||||
|
||||
unlock(j);
|
||||
|
||||
dereference(j);
|
||||
|
||||
return -EINVAL;
|
||||
|
|
@ -878,27 +1068,202 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
}
|
||||
unlock(c->workermgr);
|
||||
if (rc < 0) {
|
||||
/* scheduler-dispatch-commit task 3.3 (Req 4.1, 4.3, 4.4, 4.5):
|
||||
* resolves this retryable (transient/environmental) dispatch
|
||||
* failure -- closes bug-1627's leak site #1 (no dereference(j)
|
||||
* existed on this path before this task). rc already holds the
|
||||
* stable natural error code from key_find() above; no further call
|
||||
* touches it before it is returned below. Nothing was ever
|
||||
* committed to j->job.allocs on this path (task 3.1 gates that
|
||||
* commit on a successful xce_startjob_req(), never reached here),
|
||||
* so Req 4.4/4.5's "release any tentative allocation data" is
|
||||
* satisfied vacuously by construction. */
|
||||
xce_dispatch_fail_class_t dfclass;
|
||||
job_failinfo_t failinfo = { .jobid = alloc->jobid };
|
||||
bool resolved_retryable = false;
|
||||
uint32_t failcount;
|
||||
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
|
||||
dfclass = xce_classify_dispatch_fail(
|
||||
XCE_DISPATCH_FAIL_SITE_WORKER_LOOKUP);
|
||||
switch (dfclass) {
|
||||
case XCE_DISPATCH_FAIL_RETRYABLE:
|
||||
failcount = ++j->dispatch_fail_count;
|
||||
if (failcount <= XCE_DISPATCH_FAIL_RETRY_BOUND) {
|
||||
resolved_retryable = true;
|
||||
} else {
|
||||
failinfo.code = rc;
|
||||
failinfo.detail = "dispatch failed: worker lookup failed "
|
||||
"repeatedly, retry bound exceeded";
|
||||
mark_job_fail(c, j, &failinfo);
|
||||
}
|
||||
break;
|
||||
case XCE_DISPATCH_FAIL_TERMINAL:
|
||||
default:
|
||||
/* Impossible per today's classification table (confirmed:
|
||||
* xce_classify_dispatch_fail() returns RETRYABLE for this site
|
||||
* unconditionally) -- fail loud and fall through to the same
|
||||
* terminal resolution as a safe fallback, mirroring
|
||||
* xce_classify_dispatch_fail()'s own default: convention. */
|
||||
ERROR("dispatch_a_job_: XCE_DISPATCH_FAIL_SITE_WORKER_LOOKUP "
|
||||
"classified TERMINAL (%d) -- impossible per the current "
|
||||
"classification table; resolving terminal as a safe "
|
||||
"fallback", (int)dfclass);
|
||||
#ifdef XCE_TESTING
|
||||
assert(0
|
||||
&& "XCE_DISPATCH_FAIL_SITE_WORKER_LOOKUP classified "
|
||||
"terminal -- update the classification table or "
|
||||
"this call site");
|
||||
#endif
|
||||
failinfo.code = rc;
|
||||
failinfo.detail = "dispatch failed: worker lookup failed";
|
||||
mark_job_fail(c, j, &failinfo);
|
||||
break;
|
||||
}
|
||||
|
||||
unlock(j);
|
||||
|
||||
dereference(j);
|
||||
|
||||
if (resolved_retryable) {
|
||||
notify_dispatch(c, ep, alloc->jobid, rc);
|
||||
}
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
if (w == NULL) {
|
||||
int dfrc;
|
||||
xce_dispatch_fail_class_t dfclass;
|
||||
job_failinfo_t failinfo = { .jobid = alloc->jobid };
|
||||
bool resolved_retryable = false;
|
||||
uint32_t failcount;
|
||||
|
||||
/* scheduler-dispatch-commit task 3.3: capture errno FIRST, before
|
||||
* any other call (free_launchinfo() included) -- errno is not
|
||||
* guaranteed stable across subsequent calls. */
|
||||
dfrc = -errno;
|
||||
|
||||
free_launchinfo(&launchinfo);
|
||||
return -errno;
|
||||
|
||||
/* Req 4.1, 4.3, 4.4, 4.5: resolves this retryable (transient/
|
||||
* environmental) dispatch failure -- closes bug-1627's leak site
|
||||
* #2 (no dereference(j) existed on this path before this task). */
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
|
||||
dfclass = xce_classify_dispatch_fail(
|
||||
XCE_DISPATCH_FAIL_SITE_WORKER_NULL);
|
||||
switch (dfclass) {
|
||||
case XCE_DISPATCH_FAIL_RETRYABLE:
|
||||
failcount = ++j->dispatch_fail_count;
|
||||
if (failcount <= XCE_DISPATCH_FAIL_RETRY_BOUND) {
|
||||
resolved_retryable = true;
|
||||
} else {
|
||||
failinfo.code = dfrc;
|
||||
failinfo.detail = "dispatch failed: worker not found "
|
||||
"repeatedly, retry bound exceeded";
|
||||
mark_job_fail(c, j, &failinfo);
|
||||
}
|
||||
break;
|
||||
case XCE_DISPATCH_FAIL_TERMINAL:
|
||||
default:
|
||||
ERROR("dispatch_a_job_: XCE_DISPATCH_FAIL_SITE_WORKER_NULL "
|
||||
"classified TERMINAL (%d) -- impossible per the current "
|
||||
"classification table; resolving terminal as a safe "
|
||||
"fallback", (int)dfclass);
|
||||
#ifdef XCE_TESTING
|
||||
assert(0
|
||||
&& "XCE_DISPATCH_FAIL_SITE_WORKER_NULL classified "
|
||||
"terminal -- update the classification table or "
|
||||
"this call site");
|
||||
#endif
|
||||
failinfo.code = dfrc;
|
||||
failinfo.detail = "dispatch failed: worker not found";
|
||||
mark_job_fail(c, j, &failinfo);
|
||||
break;
|
||||
}
|
||||
|
||||
unlock(j);
|
||||
|
||||
dereference(j);
|
||||
|
||||
if (resolved_retryable) {
|
||||
notify_dispatch(c, ep, alloc->jobid, dfrc);
|
||||
}
|
||||
|
||||
return dfrc;
|
||||
}
|
||||
|
||||
lock(w, RTTI_LOCK_READ);
|
||||
if ((w->ep == NULL) || (w->host.status != 0)) {
|
||||
int dfrc;
|
||||
xce_dispatch_fail_class_t dfclass;
|
||||
job_failinfo_t failinfo = { .jobid = alloc->jobid };
|
||||
bool resolved_retryable = false;
|
||||
uint32_t failcount;
|
||||
|
||||
ERROR("worker %s is not in correct state, status %x ep %p",
|
||||
w->host.name, w->host.status, w->ep);
|
||||
free_launchinfo(&launchinfo);
|
||||
|
||||
SET_ERROR(E_XCE_BAD_WORKER);
|
||||
/* scheduler-dispatch-commit task 3.3: capture errno immediately
|
||||
* after SET_ERROR(), before any further call -- errno is not
|
||||
* guaranteed stable across unlock(w)/dereference(w)/lock(j)/etc. */
|
||||
dfrc = -errno;
|
||||
|
||||
unlock(w);
|
||||
|
||||
dereference(w);
|
||||
|
||||
return -errno;
|
||||
/* Req 4.1, 4.3, 4.4, 4.5: resolves this retryable (transient/
|
||||
* environmental) dispatch failure -- closes bug-1627's leak site
|
||||
* #3 (no dereference(j) existed on this path before this task). */
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
|
||||
dfclass = xce_classify_dispatch_fail(
|
||||
XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY);
|
||||
switch (dfclass) {
|
||||
case XCE_DISPATCH_FAIL_RETRYABLE:
|
||||
failcount = ++j->dispatch_fail_count;
|
||||
if (failcount <= XCE_DISPATCH_FAIL_RETRY_BOUND) {
|
||||
resolved_retryable = true;
|
||||
} else {
|
||||
failinfo.code = dfrc;
|
||||
failinfo.detail = "dispatch failed: worker not ready "
|
||||
"repeatedly, retry bound exceeded";
|
||||
mark_job_fail(c, j, &failinfo);
|
||||
}
|
||||
break;
|
||||
case XCE_DISPATCH_FAIL_TERMINAL:
|
||||
default:
|
||||
ERROR("dispatch_a_job_: "
|
||||
"XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY classified "
|
||||
"TERMINAL (%d) -- impossible per the current "
|
||||
"classification table; resolving terminal as a safe "
|
||||
"fallback", (int)dfclass);
|
||||
#ifdef XCE_TESTING
|
||||
assert(0
|
||||
&& "XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY classified "
|
||||
"terminal -- update the classification table or "
|
||||
"this call site");
|
||||
#endif
|
||||
failinfo.code = dfrc;
|
||||
failinfo.detail = "dispatch failed: worker not ready";
|
||||
mark_job_fail(c, j, &failinfo);
|
||||
break;
|
||||
}
|
||||
|
||||
unlock(j);
|
||||
|
||||
dereference(j);
|
||||
|
||||
if (resolved_retryable) {
|
||||
notify_dispatch(c, ep, alloc->jobid, dfrc);
|
||||
}
|
||||
|
||||
return dfrc;
|
||||
}
|
||||
|
||||
/* FIXME: we may consider to optimize when a job has enter a queue? */
|
||||
|
|
@ -908,8 +1273,62 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
|
||||
rc = get_queue_exec(c, j->job.submit->queue, &q_pre_exec, &q_post_exec);
|
||||
if (rc < 0) {
|
||||
/* scheduler-dispatch-commit task 3.2 (Req 4.2, 4.5): resolves
|
||||
* this terminal (locally-invalid) dispatch failure -- j's own
|
||||
* reference held since the lookup above is released via the
|
||||
* dereference(j) call below exactly as it already was before
|
||||
* this task (not duplicated). Nothing was ever committed to
|
||||
* j->job.allocs on this path (task 3.1 gates that commit on a
|
||||
* successful xce_startjob_req(), never reached here), so Req
|
||||
* 4.5's "release any tentative allocation data" is satisfied
|
||||
* vacuously by construction. unlock(w)/dereference(w)/
|
||||
* free_launchinfo(&launchinfo) stay in their existing
|
||||
* positions -- only this new lock/classify/mark_job_fail/
|
||||
* unlock sequence is inserted around the existing
|
||||
* dereference(j) call. */
|
||||
xce_dispatch_fail_class_t dfclass;
|
||||
job_failinfo_t failinfo = { .jobid = alloc->jobid };
|
||||
|
||||
unlock(w);
|
||||
dereference(w);
|
||||
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
|
||||
dfclass = xce_classify_dispatch_fail(
|
||||
XCE_DISPATCH_FAIL_SITE_QUEUE_EXEC);
|
||||
switch (dfclass) {
|
||||
case XCE_DISPATCH_FAIL_TERMINAL:
|
||||
break;
|
||||
case XCE_DISPATCH_FAIL_RETRYABLE:
|
||||
default:
|
||||
/* Impossible per today's classification table (confirmed:
|
||||
* xce_classify_dispatch_fail() returns TERMINAL for this
|
||||
* site unconditionally) -- task 3.3 is what wires real
|
||||
* retryable handling at the other four sites, not this
|
||||
* one. Fail loud and fall through to the same terminal
|
||||
* resolution as a safe fallback, mirroring
|
||||
* xce_classify_dispatch_fail()'s own default: convention. */
|
||||
ERROR("dispatch_a_job_: XCE_DISPATCH_FAIL_SITE_QUEUE_EXEC "
|
||||
"classified non-terminal (%d) -- impossible per the "
|
||||
"current classification table; resolving terminal "
|
||||
"as a safe fallback", (int)dfclass);
|
||||
#ifdef XCE_TESTING
|
||||
assert(0
|
||||
&& "XCE_DISPATCH_FAIL_SITE_QUEUE_EXEC classified "
|
||||
"non-terminal -- update the classification table "
|
||||
"or this call site");
|
||||
#endif
|
||||
break;
|
||||
}
|
||||
|
||||
failinfo.code = rc;
|
||||
failinfo.detail = "dispatch failed: queue exec info lookup "
|
||||
"failed";
|
||||
|
||||
mark_job_fail(c, j, &failinfo);
|
||||
|
||||
unlock(j);
|
||||
|
||||
dereference(j);
|
||||
free_launchinfo(&launchinfo);
|
||||
|
||||
|
|
@ -936,24 +1355,146 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
|
||||
free_launchinfo(&launchinfo);
|
||||
|
||||
if (j->job.allocs != NULL) {
|
||||
free_JobAlloc_ptr(&(j->job.allocs));
|
||||
}
|
||||
j->job.allocs = (xce_job_alloc_t*)calloc(1, sizeof(xce_job_alloc_t));
|
||||
j->job.allocs->jobid = alloc->jobid;
|
||||
j->job.allocs->nhosts = nc;
|
||||
MOVE_PTR(j->job.allocs->allocs, allocs);
|
||||
alloc->allocs = NULL;
|
||||
/* scheduler-dispatch-commit task 3.1 (Req 2.1-2.4, 5.1): the allocation
|
||||
* commit -- previously unconditional regardless of xce_startjob_req()'s
|
||||
* outcome -- is now gated on the dispatch to the worker actually
|
||||
* succeeding, and the job's transition to a running state happens in
|
||||
* that same successful outcome. Protected under this component's own
|
||||
* lock(j, RTTI_LOCK_WRITE): dispatch_a_job_() previously took no lock on
|
||||
* `j` itself here (only on c->jobmgr/c->workermgr/w), a real
|
||||
* pre-existing gap this restructuring corrects for this call site
|
||||
* (design.md's Concurrency Strategy note -- not fixed anywhere else in
|
||||
* this file). On failure, state is left untouched (still PEND) and
|
||||
* nothing is committed, so there is nothing to release (Req 4.4/4.5
|
||||
* satisfied vacuously by construction for this path). The worker's own
|
||||
* later confirmation that the job actually started
|
||||
* (record_job_start_fact()/WAL_JOB_START) continues to fire exactly as
|
||||
* it does today, unaffected by this change. */
|
||||
if (rc >= 0) {
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
|
||||
if (j->job.allocs != NULL) {
|
||||
free_JobAlloc_ptr(&(j->job.allocs));
|
||||
}
|
||||
j->job.allocs = (xce_job_alloc_t*)calloc(1, sizeof(xce_job_alloc_t));
|
||||
j->job.allocs->jobid = alloc->jobid;
|
||||
j->job.allocs->nhosts = nc;
|
||||
MOVE_PTR(j->job.allocs->allocs, allocs);
|
||||
alloc->allocs = NULL;
|
||||
|
||||
j->job.state = JOB_STAT_RUN;
|
||||
|
||||
/* Reset the consecutive-retryable-failure counter (task 1.3): a
|
||||
* fresh successful dispatch clears any prior failure history, so a
|
||||
* job that later fails again is evaluated against the bound as if
|
||||
* it had only failed once (Req 4.3; the bound itself is exercised
|
||||
* end-to-end by task 5.2, not here). */
|
||||
j->dispatch_fail_count = 0;
|
||||
|
||||
/* Durable record of the dispatch-time RUN transition (task 1.2),
|
||||
* now including the resource allocation itself (bug-1678): the
|
||||
* plain {jobid, state} shape persist_submit_job_fact() writes was
|
||||
* not enough to reconstruct job.allocs on a restart replay, leaving
|
||||
* EXEC_HOST permanently blank for a job that was RUNNING before
|
||||
* shutdown. j->job.allocs is read here, still under this same
|
||||
* write lock, as the freshly-constructed canonical struct just
|
||||
* stored above -- not the (now partially moved-from) local
|
||||
* `alloc`/`allocs` variables. */
|
||||
(void)persist_submit_job_dispatch_fact(&c->persist, alloc->jobid,
|
||||
JOB_STAT_RUN, j->job.allocs);
|
||||
|
||||
unlock(j);
|
||||
|
||||
/* bug-1679: the mutation above is live and synchronous -- unlike
|
||||
* record_job_start_fact()'s durable-append-only sibling, this
|
||||
* function never defers to apply_committed_entry() for its own
|
||||
* notification, and by the time that WAL_JOB_DISPATCH entry does
|
||||
* apply, state already matches (a correct, harmless no-op) so its
|
||||
* own hook-fire is silently skipped. Without this explicit call,
|
||||
* the apiserver is never told this job reached RUN. persist.h's
|
||||
* doc comment on persist_notify_job_updated_already_persisted has
|
||||
* the full rationale and the double-WAL-append hazard it guards
|
||||
* against. */
|
||||
persist_notify_job_updated_already_persisted(c, j);
|
||||
} else {
|
||||
xce_dispatch_fail_class_t dfclass;
|
||||
job_failinfo_t failinfo = { .jobid = alloc->jobid };
|
||||
bool resolved_retryable = false;
|
||||
uint32_t failcount;
|
||||
|
||||
if (rc < 0) {
|
||||
SET_ERROR(-rc);
|
||||
PERROR("xce_startjob_req");
|
||||
return rc;
|
||||
|
||||
/* scheduler-dispatch-commit task 3.3 (Req 4.1, 4.3, 4.4, 4.5):
|
||||
* resolves this retryable (transient/environmental) dispatch
|
||||
* failure -- closes bug-1627's leak site #4 (no dereference(j)
|
||||
* existed on this path before this task). rc already holds the
|
||||
* stable, captured return of xce_startjob_req() -- no call between
|
||||
* that assignment and here has touched it. */
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
|
||||
dfclass = xce_classify_dispatch_fail(
|
||||
XCE_DISPATCH_FAIL_SITE_RPC_FAILURE);
|
||||
switch (dfclass) {
|
||||
case XCE_DISPATCH_FAIL_RETRYABLE:
|
||||
failcount = ++j->dispatch_fail_count;
|
||||
if (failcount <= XCE_DISPATCH_FAIL_RETRY_BOUND) {
|
||||
resolved_retryable = true;
|
||||
} else {
|
||||
failinfo.code = rc;
|
||||
failinfo.detail = "dispatch failed: xce_startjob_req "
|
||||
"failed repeatedly, retry bound "
|
||||
"exceeded";
|
||||
mark_job_fail(c, j, &failinfo);
|
||||
}
|
||||
break;
|
||||
case XCE_DISPATCH_FAIL_TERMINAL:
|
||||
default:
|
||||
/* Impossible per today's classification table (confirmed:
|
||||
* xce_classify_dispatch_fail() returns RETRYABLE for this site
|
||||
* unconditionally) -- fail loud and fall through to the same
|
||||
* terminal resolution as a safe fallback, mirroring
|
||||
* xce_classify_dispatch_fail()'s own default: convention. */
|
||||
ERROR("dispatch_a_job_: XCE_DISPATCH_FAIL_SITE_RPC_FAILURE "
|
||||
"classified TERMINAL (%d) -- impossible per the current "
|
||||
"classification table; resolving terminal as a safe "
|
||||
"fallback", (int)dfclass);
|
||||
#ifdef XCE_TESTING
|
||||
assert(0
|
||||
&& "XCE_DISPATCH_FAIL_SITE_RPC_FAILURE classified "
|
||||
"terminal -- update the classification table or "
|
||||
"this call site");
|
||||
#endif
|
||||
failinfo.code = rc;
|
||||
failinfo.detail = "dispatch failed: xce_startjob_req failed";
|
||||
mark_job_fail(c, j, &failinfo);
|
||||
break;
|
||||
}
|
||||
|
||||
unlock(j);
|
||||
|
||||
dereference(j);
|
||||
|
||||
if (resolved_retryable) {
|
||||
notify_dispatch(c, ep, alloc->jobid, rc);
|
||||
}
|
||||
}
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
/* Test-only accessor: dispatch_a_job_() is static/file-local (its only
|
||||
* caller, handle_jobdispatch(), is unmodified by this task) -- mirroring
|
||||
* xce_classify_dispatch_fail_test()'s existing XCE_TESTING-only-export
|
||||
* pattern earlier in this file (xce_controller itself is built without
|
||||
* -DXCE_TESTING, so this never ships). */
|
||||
int dispatch_a_job_test(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
||||
{
|
||||
return dispatch_a_job_(c, ep, alloc);
|
||||
}
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
int job_ctrl(cluster_t* c, xce_jobctrl_t* ctrinfo)
|
||||
{
|
||||
uint64_t jobid = ctrinfo->jobid;
|
||||
|
|
@ -1131,19 +1672,21 @@ int handle_jobdispatch(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
xce_reply(ep, c->myhost, &reply);
|
||||
|
||||
for (i = 0; i < dispatchmsg->njobs; i++) {
|
||||
int rc1;
|
||||
xce_job_alloc_t* alloc = NULL;
|
||||
|
||||
MOVE_PTR(alloc, dispatchmsg->allocs[i]);
|
||||
|
||||
rc1 = dispatch_a_job_(c, ep, alloc);
|
||||
if (rc1 < 0) {
|
||||
/* notify dispatch fail */
|
||||
notify_dispatch(c, ep, alloc->jobid, rc1);
|
||||
} else {
|
||||
/* notify dispatch ok */
|
||||
notify_dispatch(c, ep, alloc->jobid, 0);
|
||||
}
|
||||
/* scheduler-dispatch-commit task 4.1 (Req 3.1, 4.1, 4.2): reporting
|
||||
* the dispatch outcome is now entirely owned by dispatch_a_job_()
|
||||
* itself (tasks 3.1-3.3) -- it already calls notify_dispatch() on
|
||||
* the retryable-failure path and relies on mark_job_fail()'s own
|
||||
* hook chain for the terminal-failure path, never on success. This
|
||||
* caller used to unconditionally call notify_dispatch() again for
|
||||
* every job regardless of that outcome, producing a conflicting
|
||||
* double report (e.g. both jobDispatchFail and schedFailJob for the
|
||||
* same terminal failure). The return value has nothing left for
|
||||
* this caller to act on. */
|
||||
(void)dispatch_a_job_(c, ep, alloc);
|
||||
|
||||
free_JobAlloc_ptr(&(alloc));
|
||||
}
|
||||
|
|
@ -1658,8 +2201,11 @@ int on_job_start(cluster_t* c, job_pidinfo_t* info)
|
|||
{
|
||||
job_t* job = NULL;
|
||||
int rc;
|
||||
char jobidStr[64];
|
||||
|
||||
TRACE("%s jobid %ld", __func__, info->jobid);
|
||||
/* bug-1588: see dispatch_a_job_()'s own comment on fmt_jobid(). */
|
||||
fmt_jobid(jobidStr, sizeof(jobidStr), info->jobid);
|
||||
TRACE("%s jobid %s", __func__, jobidStr);
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
rc = key_find(c->jobmgr, (void*)info->jobid, (void**)&job);
|
||||
|
|
@ -1896,8 +2442,11 @@ int test_unknown_finish_take(uint64_t jobid, xce_jobfinish_t* out)
|
|||
int on_job_finish(cluster_t* c, xce_jobfinish_t* finish)
|
||||
{
|
||||
job_t* job = NULL;
|
||||
char jobidStr[64];
|
||||
|
||||
TRACE("%s job finish jobid %ld exit code %d", __func__, finish->jobid,
|
||||
/* bug-1588: see dispatch_a_job_()'s own comment on fmt_jobid(). */
|
||||
fmt_jobid(jobidStr, sizeof(jobidStr), finish->jobid);
|
||||
TRACE("%s job finish jobid %s exit code %d", __func__, jobidStr,
|
||||
finish->exit_code);
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
|
|
@ -2315,8 +2864,11 @@ static int record_job_postdone_fact(cluster_t* c, job_t* job)
|
|||
int on_job_fail(cluster_t* c, job_failinfo_t* failinfo)
|
||||
{
|
||||
job_t* job = NULL;
|
||||
char jobidStr[64];
|
||||
|
||||
TRACE("%s job fail jobid %ld code %d", __func__, failinfo->jobid,
|
||||
/* bug-1588: see dispatch_a_job_()'s own comment on fmt_jobid(). */
|
||||
fmt_jobid(jobidStr, sizeof(jobidStr), failinfo->jobid);
|
||||
TRACE("%s job fail jobid %s code %d", __func__, jobidStr,
|
||||
failinfo->code);
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
|
|
@ -2344,8 +2896,11 @@ int on_job_stop(cluster_t* c, uint64_t jobid)
|
|||
{
|
||||
job_t* job = NULL;
|
||||
int state;
|
||||
char jobidStr[64];
|
||||
|
||||
TRACE("%s job stop jobid %ld", __func__, jobid);
|
||||
/* bug-1588: see dispatch_a_job_()'s own comment on fmt_jobid(). */
|
||||
fmt_jobid(jobidStr, sizeof(jobidStr), jobid);
|
||||
TRACE("%s job stop jobid %s", __func__, jobidStr);
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&job);
|
||||
|
|
@ -2379,8 +2934,11 @@ int on_job_resume(cluster_t* c, uint64_t jobid)
|
|||
{
|
||||
job_t* job = NULL;
|
||||
int state;
|
||||
char jobidStr[64];
|
||||
|
||||
TRACE("%s job resume jobid %ld", __func__, jobid);
|
||||
/* bug-1588: see dispatch_a_job_()'s own comment on fmt_jobid(). */
|
||||
fmt_jobid(jobidStr, sizeof(jobidStr), jobid);
|
||||
TRACE("%s job resume jobid %s", __func__, jobidStr);
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&job);
|
||||
|
|
@ -2412,8 +2970,11 @@ int on_job_resume(cluster_t* c, uint64_t jobid)
|
|||
int on_job_postdone(cluster_t* c, endpoint_t* ep, uint64_t jobid)
|
||||
{
|
||||
job_t* job = NULL;
|
||||
char jobidStr[64];
|
||||
|
||||
TRACE("%s job post done jobid %ld", __func__, jobid);
|
||||
/* bug-1588: see dispatch_a_job_()'s own comment on fmt_jobid(). */
|
||||
fmt_jobid(jobidStr, sizeof(jobidStr), jobid);
|
||||
TRACE("%s job post done jobid %s", __func__, jobidStr);
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&job);
|
||||
|
|
@ -2586,8 +3147,11 @@ static int set_preason(cluster_t* c, xce_pendreason_t* reasons)
|
|||
{
|
||||
job_t* j = NULL;
|
||||
xce_job_t* job = NULL;
|
||||
char jobidStr[64];
|
||||
|
||||
TRACE("%s jobid %ld", __func__, reasons->jobid);
|
||||
/* bug-1588: see dispatch_a_job_()'s own comment on fmt_jobid(). */
|
||||
fmt_jobid(jobidStr, sizeof(jobidStr), reasons->jobid);
|
||||
TRACE("%s jobid %s", __func__, jobidStr);
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)reasons->jobid, (void**)&j);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue