merge: sync Rongsong's scheduler-framework (122 commits, 070c7869..f01ae353)
Introduces WAL replay field-completeness fixes (bug-1687..1697, 1731), scheduler snapshot job-allocation restore, heap UAF fixes (vtask_enq, xcealloc, builtin_fcfs, limit), HA fixes (bug-1694/1695/1696), xceadmin retry, worker signal-exit severity, and their test suites. Conflicts resolved keeping both sides (drop-audit passed): - worker.c: mark_worker_jobs_exit (bug-1509 B1) + fetchfile_signal_worker_disconnect (bug-1693) - handler.go: DefaultBroker.Publish (EventJobFinished) + bug-1731 logging - config_repl.c / test_config_repl.c: our ns-resolution comment / unique-tmpdir - test_dispatch_commit.c: both B3-UNKWN and bug-1697 start_time assertions Excluded Rongsong's bookkeeping per policy: .wolf/, .kiro/, CHANGES.md, reports/; .claude/ + CLAUDE.md kept ours (his settings.json has machine-absolute paths).
This commit is contained in:
lei.chen
commit
3b6f53bb65
168 changed files with 20744 additions and 568495 deletions
|
|
@ -790,9 +790,16 @@ int restore_job_state(cluster_t* c, uint64_t jobid, uint32_t state)
|
|||
* ownership of an optional decoded resource allocation (NULL for a
|
||||
* state-only WAL_JOB_DISPATCH entry, e.g. one written before this fix) so a
|
||||
* promoted standby recovers EXEC_HOST the same way apply_committed_entry's
|
||||
* WAL_JOB_DISPATCH case already does for the primary's own boot replay. */
|
||||
* WAL_JOB_DISPATCH case already does for the primary's own boot replay.
|
||||
*
|
||||
* bug-1697: also takes a wire-carried `start_time` (0 meaning "not carried
|
||||
* -- leave job.start_time untouched", matching every sibling restore_job_*
|
||||
* function's own "0 == not carried" convention). job.start_time is now
|
||||
* DISPATCH-owned (see wal_types.h's WAL_JOB_START doc comment for the full
|
||||
* rationale), so this is the seam that must restore it on a promoted
|
||||
* standby, not WAL_JOB_START's own restore_job_start_state() below. */
|
||||
int restore_job_dispatch_state(cluster_t* c, uint64_t jobid, uint32_t state,
|
||||
xce_job_alloc_t* alloc)
|
||||
uint64_t start_time, xce_job_alloc_t* alloc)
|
||||
{
|
||||
job_t* j = NULL;
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
|
|
@ -806,6 +813,8 @@ int restore_job_dispatch_state(cluster_t* c, uint64_t jobid, uint32_t state,
|
|||
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
j->job.state = (int)state;
|
||||
if (start_time != 0)
|
||||
j->job.start_time = (time_t)start_time;
|
||||
if (alloc != NULL) {
|
||||
if (j->job.allocs != NULL)
|
||||
free_JobAlloc_ptr(&(j->job.allocs));
|
||||
|
|
@ -815,6 +824,310 @@ int restore_job_dispatch_state(cluster_t* c, uint64_t jobid, uint32_t state,
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* bug-1687: WAL_JOB_START's own sibling of restore_job_dispatch_state()
|
||||
* above (bug-1678), for apply_wal_entry()'s standby catch-up / replicated-
|
||||
* apply seam -- takes ownership of an optional decoded job_pidinfo_t (NULL
|
||||
* for a state-only WAL_JOB_START entry, e.g. one written before this fix)
|
||||
* so a promoted standby recovers the worker-confirmed pids/pgids/exdata the
|
||||
* same way apply_committed_entry's WAL_JOB_START case already does for the
|
||||
* primary's own boot replay.
|
||||
*
|
||||
* bug-1697: no longer takes/touches start_time at all -- job.start_time is
|
||||
* now exclusively DISPATCH-owned (restore_job_dispatch_state() above), so
|
||||
* this function must leave it alone entirely rather than risk clobbering
|
||||
* the correct dispatch-time value with a stale worker-ack-time one (see
|
||||
* wal_types.h's WAL_JOB_START doc comment for the full rationale). */
|
||||
int restore_job_start_state(cluster_t* c, uint64_t jobid, uint32_t state,
|
||||
job_pidinfo_t* pidinfo)
|
||||
{
|
||||
job_t* j = NULL;
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&j);
|
||||
unlock(c->jobmgr);
|
||||
if (!j) {
|
||||
if (pidinfo != NULL)
|
||||
free_JobPIDInfo_ptr(&pidinfo);
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
j->job.state = (int)state;
|
||||
if (pidinfo != NULL) {
|
||||
if (j->job.pidinfo != NULL)
|
||||
free_JobPIDInfo_ptr(&(j->job.pidinfo));
|
||||
j->job.pidinfo = pidinfo;
|
||||
}
|
||||
unlock(j);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* bug-1689: WAL_JOB_FINISH's own sibling of restore_job_start_state()/
|
||||
* restore_job_dispatch_state() above (bug-1687/bug-1678), for
|
||||
* apply_wal_entry()'s standby catch-up / replicated-apply seam -- takes
|
||||
* ownership of an optional decoded job_exitinfo_t (NULL for a state-only
|
||||
* WAL_JOB_FINISH entry, e.g. one written before this fix) and a
|
||||
* wire-carried finish_time (0 meaning "not carried -- leave
|
||||
* job.finish_time/run_time untouched", matching this seam's own pre-fix
|
||||
* behavior of never touching them for WAL_JOB_FINISH at all) so a promoted
|
||||
* standby recovers the worker-confirmed exit_code and the true finish_time
|
||||
* the same way apply_committed_entry's WAL_JOB_FINISH case already does for
|
||||
* the primary's own boot replay. job_exitinfo_t has no nested pointers (a
|
||||
* single int), so it is released with plain FREEUP, not a capnp
|
||||
* free_*_ptr helper. */
|
||||
int restore_job_finish_state(cluster_t* c, uint64_t jobid, uint32_t state,
|
||||
uint64_t finish_time, job_exitinfo_t* exit_info)
|
||||
{
|
||||
job_t* j = NULL;
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&j);
|
||||
unlock(c->jobmgr);
|
||||
if (!j) {
|
||||
FREEUP(exit_info);
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
j->job.state = (int)state;
|
||||
if (finish_time != 0)
|
||||
j->job.finish_time = (time_t)finish_time;
|
||||
if (exit_info != NULL) {
|
||||
FREEUP(j->job.exit_info);
|
||||
j->job.exit_info = exit_info;
|
||||
}
|
||||
if (finish_time != 0)
|
||||
j->job.run_time = (int)(j->job.finish_time - j->job.start_time);
|
||||
unlock(j);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* bug-1690: merge (never wholesale free-and-replace) a freshly-decoded
|
||||
* xce_job_runusage_t into *dst. Takes ownership of `src` -- always
|
||||
* consumed/freed by this call (either adopted whole, partially cannibalized
|
||||
* for its new-host entries, or fully released), regardless of outcome.
|
||||
*
|
||||
* WHY A MERGE, NOT A REPLACE: *dst can be the job's LIVE run_usage --
|
||||
* update_job_rusages() (job.c) mutates it synchronously and in place, via
|
||||
* update_host_runusage() (misc.c), BEFORE the corresponding WAL fact is even
|
||||
* appended. On the primary, persist_apply_pump_run() (persist.c) later
|
||||
* re-applies that SAME fact via apply_committed_entry() moments afterward,
|
||||
* on the SAME live job object -- so *dst can genuinely be that live,
|
||||
* already-current structure at the moment this runs, not just a stale
|
||||
* replay artifact. Critically, update_host_runusage() sources each
|
||||
* xce_host_runusage_t.host_name from makeString() (src/lib/xstring.c) -- an
|
||||
* RTTI-managed, refcounted intern-table pointer, NOT a plain malloc()
|
||||
* allocation. RTTI's own header-before-data layout (librtti's
|
||||
* RTTI_DATA_TO_OBJ: the object header sits BEFORE the pointer callers hold)
|
||||
* means a plain free() on that pointer -- which the capnp-generated
|
||||
* free_JobRunusage_ptr()/free_HostRunusage_ptr() would do, being written for
|
||||
* genuinely capnp-decoded, plain-malloc'd Text fields -- corrupts the heap.
|
||||
* (job_destroy()'s own manual FREEUP-based walk over run_usage, jobj.c, has
|
||||
* this exact same latent mismatch already, independent of this fix -- flagged
|
||||
* separately, not fixed here, since jobj.c/misc.c are outside this fix's
|
||||
* scope.)
|
||||
*
|
||||
* The merge below NEVER frees an existing host_name: an already-present
|
||||
* host (matched by name) only has its scalar cpu_time/mem_usage overwritten
|
||||
* in place; only a genuinely NEW host's own freshly-decoded (always
|
||||
* plain-malloc'd) host_name is adopted, by ownership transfer, never a
|
||||
* live/xstring one. `src`'s own shell plus every entry NOT transferred is
|
||||
* released via the ordinary capnp free path at the end -- safe, since `src`
|
||||
* is always capnp-decoded, end to end, regardless of what *dst was. */
|
||||
void merge_job_runusage(xce_job_runusage_t** dst, xce_job_runusage_t* src)
|
||||
{
|
||||
xce_job_runusage_t* d;
|
||||
int i, k;
|
||||
|
||||
if (*dst == NULL) {
|
||||
/* Nothing live to preserve — adopt the decoded copy outright,
|
||||
* exactly like every other restore_job_*_state() sibling. */
|
||||
*dst = src;
|
||||
return;
|
||||
}
|
||||
|
||||
d = *dst;
|
||||
/* Scalar job-level fields: no pointers, always safe to overwrite in
|
||||
* place regardless of *dst's provenance. */
|
||||
d->max_mem = src->max_mem;
|
||||
d->avg_mem = src->avg_mem;
|
||||
d->used_mem = src->used_mem;
|
||||
d->cpu_time = src->cpu_time;
|
||||
d->peak_cpu_time = src->peak_cpu_time;
|
||||
d->peak_cpu_duration = src->peak_cpu_duration;
|
||||
d->mem_efficiency = src->mem_efficiency;
|
||||
d->avg_cpu_efficiency = src->avg_cpu_efficiency;
|
||||
d->peak_cpu_efficiency = src->peak_cpu_efficiency;
|
||||
|
||||
for (i = 0; i < src->num_hosts; i++) {
|
||||
xce_host_runusage_t* sh = src->host_runusage[i];
|
||||
int matched = -1;
|
||||
|
||||
if (sh == NULL)
|
||||
continue;
|
||||
|
||||
for (k = 0; k < d->num_hosts; k++) {
|
||||
if (d->host_runusage[k] != NULL &&
|
||||
d->host_runusage[k]->host_name != NULL &&
|
||||
sh->host_name != NULL &&
|
||||
strcmp(sh->host_name, d->host_runusage[k]->host_name) == 0) {
|
||||
matched = k;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (matched >= 0) {
|
||||
/* Existing host: overwrite scalars only. d's host_name pointer
|
||||
* (possibly xstring-backed) is never touched or freed here. */
|
||||
d->host_runusage[matched]->cpu_time = sh->cpu_time;
|
||||
d->host_runusage[matched]->mem_usage = sh->mem_usage;
|
||||
d->host_runusage[matched]->mem_swap = sh->mem_swap;
|
||||
d->host_runusage[matched]->tasks_current = sh->tasks_current;
|
||||
} else {
|
||||
/* Genuinely new host: append, transferring ownership of sh's
|
||||
* own plain-malloc'd host_name directly -- never xstring. */
|
||||
xce_host_runusage_t** grown = (xce_host_runusage_t**)realloc(
|
||||
d->host_runusage,
|
||||
(size_t)(d->num_hosts + 1) * sizeof(xce_host_runusage_t*));
|
||||
if (grown == NULL)
|
||||
continue; /* OOM -- skip this host, keep the rest merged */
|
||||
d->host_runusage = grown;
|
||||
d->host_runusage[d->num_hosts] = sh;
|
||||
d->num_hosts++;
|
||||
src->host_runusage[i] = NULL; /* ownership transferred -- the
|
||||
* free_JobRunusage_ptr() call
|
||||
* below must not also free it */
|
||||
}
|
||||
}
|
||||
|
||||
/* Release src's shell + every entry whose ownership was NOT transferred
|
||||
* above -- always safe, src is capnp-decoded end to end. */
|
||||
free_JobRunusage_ptr(&src);
|
||||
}
|
||||
|
||||
/* bug-1690: WAL_JOB_RUSAGE's own sibling of restore_job_state() above, for
|
||||
* apply_wal_entry()'s standby catch-up / replicated-apply seam -- takes
|
||||
* ownership of an optional decoded xce_job_runusage_t (NULL for a
|
||||
* state-only WAL_JOB_RUSAGE entry -- either one written before this fix, or
|
||||
* one fired while job.run_usage was still NULL at write time) so a promoted
|
||||
* standby recovers the client-reported per-host CPU/mem usage the same way
|
||||
* apply_committed_entry's WAL_JOB_RUSAGE case already does for the primary's
|
||||
* own boot replay. Unconditional absolute state overwrite, matching every
|
||||
* other restore_job_*_state() sibling on this seam -- no transition-result
|
||||
* guarding happens on standby catch-up. Merges (never frees-and-replaces)
|
||||
* run_usage -- see merge_job_runusage()'s doc comment above for why. */
|
||||
int restore_job_rusage_state(cluster_t* c, uint64_t jobid, uint32_t state,
|
||||
xce_job_runusage_t* runusage)
|
||||
{
|
||||
job_t* j = NULL;
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&j);
|
||||
unlock(c->jobmgr);
|
||||
if (!j) {
|
||||
if (runusage != NULL)
|
||||
free_JobRunusage_ptr(&runusage);
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
j->job.state = (int)state;
|
||||
if (runusage != NULL)
|
||||
merge_job_runusage(&(j->job.run_usage), runusage);
|
||||
unlock(j);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* bug-1691: WAL_JOB_CKPT's own sibling of restore_job_state() above, for
|
||||
* apply_wal_entry()'s standby catch-up / replicated-apply seam -- takes
|
||||
* ownership of an optional decoded ckptinfo string (NULL for a state-only
|
||||
* WAL_JOB_CKPT entry, e.g. one written before this fix, or one fired while
|
||||
* job.ckptinfo was still NULL at write time) so a promoted standby recovers
|
||||
* the checkpoint info the same way apply_committed_entry's WAL_JOB_CKPT
|
||||
* branch already restores it on the primary's own boot replay. Absolute
|
||||
* state overwrite, matching every other restore_job_*_state() sibling on
|
||||
* this seam -- no transition-result guarding happens on standby catch-up.
|
||||
*
|
||||
* `ckptinfo` is a TRANSIENT, caller-owned decode buffer (persist.c's
|
||||
* decode_ckptinfo()), NOT itself an xstring -- j->job.ckptinfo is
|
||||
* xstring-backed (makeString(), xstring.c), so adoption always goes through
|
||||
* makeString() (which copies its input into its own refcounted allocation)
|
||||
* and the existing value is released via dereference() (the RTTI verb),
|
||||
* never free()/FREEUP, mirroring on_job_ckpt()'s own
|
||||
* dereference(j->job.ckptinfo) call. `ckptinfo` itself is always freed here
|
||||
* before returning, regardless of whether adoption happened -- the value
|
||||
* actually stored in j->job.ckptinfo is always a freshly-made makeString()
|
||||
* copy, never the input pointer itself. */
|
||||
int restore_job_ckpt_state(cluster_t* c, uint64_t jobid, uint32_t state,
|
||||
char* ckptinfo)
|
||||
{
|
||||
job_t* j = NULL;
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&j);
|
||||
unlock(c->jobmgr);
|
||||
if (!j) {
|
||||
FREEUP(ckptinfo);
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
j->job.state = (int)state;
|
||||
if (ckptinfo != NULL) {
|
||||
if (j->job.ckptinfo != NULL)
|
||||
dereference(j->job.ckptinfo);
|
||||
j->job.ckptinfo = makeString(ckptinfo);
|
||||
}
|
||||
unlock(j);
|
||||
FREEUP(ckptinfo);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* bug-1692: WAL_JOB_PREASON's own sibling of restore_job_ckpt_state() above,
|
||||
* for apply_wal_entry()'s standby catch-up / replicated-apply seam.
|
||||
*
|
||||
* has_field distinguishes a pre-fix bare {jobid,state} payload (has_field=0,
|
||||
* pend_reasons left untouched -- full backward compatibility) from a
|
||||
* post-fix payload that explicitly carries pend_reasons' new value
|
||||
* (has_field=1): preason non-NULL adopts a freshly-decoded value, preason
|
||||
* NULL means the field was genuinely cleared (do_clr_preason) and
|
||||
* pend_reasons should be reset to NULL too -- pend_reasons is the one
|
||||
* field-parity fact type in this family (bug-1678/1687/1689/1690/1691) with
|
||||
* a real CLEAR operation, so unlike restore_job_ckpt_state()/
|
||||
* restore_job_rusage_state() a NULL decoded value is NOT automatically
|
||||
* "nothing to adopt".
|
||||
*
|
||||
* preason (when non-NULL) is always freshly heap-allocated by
|
||||
* decode_jobpreason() (persist.c) -- plain calloc()/strdup() throughout
|
||||
* (verified via erase_job_preason()'s FREEUP-only release and every
|
||||
* add_*_preason() constructor in src/lib/preason.c, plus the
|
||||
* capnp-generated decode_HostBasedReason/decode_ResourceBasedReason using
|
||||
* STRING_DUP+calloc), NOT xstring/RTTI-backed like ckptinfo or rusage's
|
||||
* host_name -- so a straightforward free-old-then-adopt-new via
|
||||
* free_JobPendReason_ptr() is safe here, no merge-not-replace treatment
|
||||
* needed (unlike bug-1690's WAL_JOB_RUSAGE). Ownership of preason always
|
||||
* transfers to job->job.pend_reasons on success; released here if the job
|
||||
* is gone. */
|
||||
int restore_job_preason_state(cluster_t* c, uint64_t jobid, uint32_t state,
|
||||
int has_field, xce_job_preason_t* preason)
|
||||
{
|
||||
job_t* j = NULL;
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&j);
|
||||
unlock(c->jobmgr);
|
||||
if (!j) {
|
||||
if (preason != NULL)
|
||||
free_JobPendReason_ptr(&preason);
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
j->job.state = (int)state;
|
||||
if (has_field) {
|
||||
if (j->job.pend_reasons != NULL)
|
||||
free_JobPendReason_ptr(&(j->job.pend_reasons));
|
||||
j->job.pend_reasons = preason; /* NULL (cleared) or the decoded value */
|
||||
}
|
||||
unlock(j);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int get_nslots_in_alloc(xce_host_alloc_t* alloc)
|
||||
{
|
||||
int i;
|
||||
|
|
@ -1384,6 +1697,18 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
|
||||
j->job.state = JOB_STAT_RUN;
|
||||
|
||||
/* bug-1697: start_time is now stamped HERE, synchronously alongside
|
||||
* the RUN transition, rather than left to the worker's later
|
||||
* WAL_JOB_START report. dispatch_a_job_() setting job.state =
|
||||
* JOB_STAT_RUN synchronously means any WAL_JOB_START fact for this
|
||||
* job always finds state already == RUN by the time it applies
|
||||
* (apply_job_update_transition() reports JOB_TRANSITION_NOOP_
|
||||
* DUPLICATE), so it can never be the genuine first RUN transition
|
||||
* and start_time adoption gated on that transition would be
|
||||
* permanently dead code -- see wal_types.h's WAL_JOB_START doc
|
||||
* comment for the full rationale. */
|
||||
j->job.start_time = time(NULL);
|
||||
|
||||
/* Reset the consecutive-retryable-failure counter (task 1.3): a
|
||||
* fresh successful dispatch clears any prior failure history, so a
|
||||
* job that later fails again is evaluated against the bound as if
|
||||
|
|
@ -1399,9 +1724,14 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
* shutdown. j->job.allocs is read here, still under this same
|
||||
* write lock, as the freshly-constructed canonical struct just
|
||||
* stored above -- not the (now partially moved-from) local
|
||||
* `alloc`/`allocs` variables. */
|
||||
* `alloc`/`allocs` variables. bug-1697: also carries j->job.
|
||||
* start_time, the same snapshot just stamped above under this same
|
||||
* write lock, so a restart replay recovers the true dispatch-time
|
||||
* clock instead of leaving it at 0 forever. */
|
||||
(void)persist_submit_job_dispatch_fact(&c->persist, alloc->jobid,
|
||||
JOB_STAT_RUN, j->job.allocs);
|
||||
JOB_STAT_RUN,
|
||||
(uint64_t)j->job.start_time,
|
||||
j->job.allocs);
|
||||
|
||||
unlock(j);
|
||||
|
||||
|
|
@ -2190,9 +2520,25 @@ static int record_job_start_fact(cluster_t* c, job_t* job, job_pidinfo_t* info)
|
|||
* 5.1, 5.3). JOB_STAT_RUN is the resultant state a START always
|
||||
* produces once past the guard above; apply_committed_entry() will
|
||||
* re-validate the same guard against whatever state is current AT
|
||||
* APPLY TIME before setting it. */
|
||||
(void)persist_submit_job_fact(&c->persist, WAL_JOB_START, job->job.jobid,
|
||||
JOB_STAT_RUN);
|
||||
* APPLY TIME before setting it.
|
||||
*
|
||||
* bug-1687: persist_submit_job_start_fact() carries `info` (the real
|
||||
* worker-reported pids/pgids/exdata) and a receipt-time start_time
|
||||
* alongside {jobid, state=RUN} -- see that function's doc comment and
|
||||
* wal_types.h's WAL_JOB_START payload note for the full rationale
|
||||
* (mirrors bug-1678's WAL_JOB_DISPATCH fix). Before this fix,
|
||||
* persist_submit_job_fact()'s bare {jobid, state} payload silently
|
||||
* dropped `info` entirely, so job.pidinfo replayed permanently NULL and
|
||||
* job.start_time was re-fabricated (time(NULL)) fresh at every replay
|
||||
* instead of carrying the true worker-reported time. Capturing
|
||||
* start_time HERE, once, at receipt time (rather than re-deriving
|
||||
* time(NULL) independently at each of apply_committed_entry's/
|
||||
* apply_wal_entry's OWN apply-time moments) makes it the single
|
||||
* authoritative value both apply seams converge on. */
|
||||
time_t start_time = time(NULL);
|
||||
(void)persist_submit_job_start_fact(&c->persist, job->job.jobid,
|
||||
JOB_STAT_RUN, (uint64_t)start_time,
|
||||
info);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -2297,10 +2643,28 @@ static int record_job_finish_fact(cluster_t* c, job_t* job,
|
|||
|
||||
/* Durable append only — no mutation, no AFTER_EVENT firing here (Req
|
||||
* 5.1, 5.3). The resultant state is deterministic from finish->exit_code
|
||||
* alone, independent of current job state. */
|
||||
(void)persist_submit_job_fact(&c->persist, WAL_JOB_FINISH, job->job.jobid,
|
||||
(finish->exit_code == 0) ? JOB_STAT_DONE
|
||||
: JOB_STAT_EXIT);
|
||||
* alone, independent of current job state.
|
||||
*
|
||||
* bug-1689: persist_submit_job_finish_fact() carries a receipt-time
|
||||
* finish_time and the worker-reported exit_code alongside {jobid,
|
||||
* state} -- see that function's doc comment and wal_types.h's
|
||||
* WAL_JOB_FINISH payload note for the full rationale (mirrors
|
||||
* bug-1687's WAL_JOB_START fix). Before this fix, persist_submit_job_
|
||||
* fact()'s bare {jobid, state} payload silently dropped both: job.
|
||||
* exit_info replayed permanently NULL and job.finish_time (and its
|
||||
* derived job.run_time) were re-fabricated (time(NULL)) fresh at every
|
||||
* apply instead of carrying the true receipt-time value. Capturing
|
||||
* finish_time HERE, once, at receipt time (rather than re-deriving
|
||||
* time(NULL) independently at each of apply_committed_entry's/
|
||||
* apply_wal_entry's OWN apply-time moments) makes it the single
|
||||
* authoritative value both apply seams converge on. */
|
||||
time_t finish_time = time(NULL);
|
||||
(void)persist_submit_job_finish_fact(&c->persist, job->job.jobid,
|
||||
(finish->exit_code == 0)
|
||||
? JOB_STAT_DONE
|
||||
: JOB_STAT_EXIT,
|
||||
(uint64_t)finish_time,
|
||||
finish->exit_code);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -3124,13 +3488,21 @@ int update_job_rusages(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
update_host_runusage(&(j->job.run_usage), host, rusages->count,
|
||||
rusages->usages);
|
||||
|
||||
/* bug-1735: derive job-level max_mem/avg_mem/used_mem/cpu_time from the
|
||||
* per-host entries update_host_runusage() just updated -- replaces the
|
||||
* standing "update other fields of job rusage later" FIXME. */
|
||||
aggregate_job_runusage(j->job.run_usage);
|
||||
|
||||
update_pids(&(j->job.pidinfo), rusages->npids, rusages->pids);
|
||||
|
||||
update_pgids(&(j->job.pidinfo), rusages->npgids, rusages->pgids);
|
||||
|
||||
/* FIXME
|
||||
* update other fields of job rusage later
|
||||
*/
|
||||
/* TEMP diagnostic (bug-1731 job-finish-notify-loss investigation).
|
||||
* DEBUG3-gated -- see bug-1731's own "re-add before next occurrence"
|
||||
* recommendation; the DEBUG3 log-retention bump (bug-1731 support,
|
||||
* main.c) exists so this survives a live burst without rotating out. */
|
||||
TRACE("DIAG update_job_rusages: jobid=%ld state=0x%x (about to fire "
|
||||
"live JOB_UPDATE hook)",
|
||||
(long)j->job.jobid, (unsigned)j->job.state);
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||||
HOOK_DETAIL_JOB_UPDATE, c, j);
|
||||
|
||||
|
|
@ -3265,6 +3637,11 @@ int do_clr_preason(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
|
||||
typedef struct fetchfile_pending {
|
||||
uint64_t jobid;
|
||||
worker_t* worker; /* bug-1693: which worker this request was
|
||||
* proxied to -- lets a disconnect be
|
||||
* matched to the pending waiter(s) it
|
||||
* affects; see
|
||||
* fetchfile_signal_worker_disconnect(). */
|
||||
pthread_mutex_t mu;
|
||||
pthread_cond_t cv;
|
||||
int done;
|
||||
|
|
@ -3322,6 +3699,78 @@ void fetchfile_signal(uint64_t jobid, int status, xce_fetchfile_reply_t* reply)
|
|||
pthread_mutex_unlock(&p->mu);
|
||||
}
|
||||
|
||||
/* bug-1693 (Decision Log: "No client reply-wait timeout as a fix for a hung
|
||||
* request -- fail loud at the real transport/source instead of masking with
|
||||
* a deadline backstop"). Before this fix, handle_worker_leave() (worker.c)
|
||||
* only ever nulled w->ep on a worker disconnect -- it never woke a pending
|
||||
* on_fetchfile() waiter proxied to that worker, so a mid-wait disconnect
|
||||
* could only ever be observed at on_fetchfile's own 10s deadline below.
|
||||
* Called from handle_worker_leave() immediately on disconnect. Unlike
|
||||
* fetchfile_signal() (keyed by jobid, at most one match), more than one
|
||||
* fetchfile request can legitimately be in flight against the SAME worker
|
||||
* at once, so this walks the whole list and signals every match -- a no-op
|
||||
* if none are pending against `w`. Deliberately takes only g_fetchfile_mu +
|
||||
* each matched waiter's own mu, never any worker_t lock, so callers do not
|
||||
* need to hold (or avoid holding) `w`'s lock around this call. */
|
||||
void fetchfile_signal_worker_disconnect(worker_t* w)
|
||||
{
|
||||
fetchfile_pending_t* p;
|
||||
|
||||
pthread_mutex_lock(&g_fetchfile_mu);
|
||||
for (p = g_fetchfile_list; p != NULL; p = p->next) {
|
||||
if (p->worker != w) continue;
|
||||
pthread_mutex_lock(&p->mu);
|
||||
p->status = -ECONNRESET;
|
||||
p->reply = NULL;
|
||||
p->done = 1;
|
||||
pthread_cond_signal(&p->cv);
|
||||
pthread_mutex_unlock(&p->mu);
|
||||
}
|
||||
pthread_mutex_unlock(&g_fetchfile_mu);
|
||||
}
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
/* Test seam (bug-1693): exercises fetchfile_signal_worker_disconnect()'s
|
||||
* demux against a real fetchfile_pending_t wait, without needing a full
|
||||
* on_fetchfile()/real c->jobmgr+c->workermgr fixture (job/worker lookup is
|
||||
* out of this fix's scope -- only the disconnect-to-waiter signal path is
|
||||
* under test). Registers a bare pending entry keyed to `w`, blocks with a
|
||||
* deadline exactly like on_fetchfile's own wait loop below, and reports the
|
||||
* outcome via *out_status. Returns 1 if woken by a signal, 0 if the
|
||||
* deadline lapsed first (mirrors pending.done). */
|
||||
int fetchfile_wait_for_test(worker_t* w, long timeout_ms, int* out_status)
|
||||
{
|
||||
fetchfile_pending_t pending = {
|
||||
.jobid = 0, .worker = w,
|
||||
.mu = PTHREAD_MUTEX_INITIALIZER, .cv = PTHREAD_COND_INITIALIZER,
|
||||
.done = 0, .status = -ETIMEDOUT, .reply = NULL, .next = NULL
|
||||
};
|
||||
struct timespec deadline;
|
||||
|
||||
clock_gettime(CLOCK_REALTIME, &deadline);
|
||||
deadline.tv_sec += timeout_ms / 1000;
|
||||
deadline.tv_nsec += (timeout_ms % 1000) * 1000000L;
|
||||
if (deadline.tv_nsec >= 1000000000L) {
|
||||
deadline.tv_sec += 1;
|
||||
deadline.tv_nsec -= 1000000000L;
|
||||
}
|
||||
|
||||
fetchfile_register(&pending);
|
||||
|
||||
pthread_mutex_lock(&pending.mu);
|
||||
while (!pending.done) {
|
||||
int wrc = pthread_cond_timedwait(&pending.cv, &pending.mu, &deadline);
|
||||
if (wrc == ETIMEDOUT) break;
|
||||
}
|
||||
pthread_mutex_unlock(&pending.mu);
|
||||
|
||||
fetchfile_unregister(&pending);
|
||||
|
||||
*out_status = pending.status;
|
||||
return pending.done;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* ---- on_fetchfile: proxy fetchfile from apiserver to worker ---- */
|
||||
|
||||
int on_fetchfile(cluster_t* c, endpoint_t* caller_ep, xce_request_t* request)
|
||||
|
|
@ -3405,6 +3854,12 @@ int on_fetchfile(cluster_t* c, endpoint_t* caller_ep, xce_request_t* request)
|
|||
return -ENOENT;
|
||||
}
|
||||
|
||||
/* bug-1693: record which worker this request is proxied to (still
|
||||
* holding w's read lock, so w->ep/identity are stable here) so a later
|
||||
* disconnect can find and wake this waiter -- see
|
||||
* fetchfile_signal_worker_disconnect()'s doc comment above. */
|
||||
pending.worker = w;
|
||||
|
||||
/* Register pending before sending to avoid a race with the reply */
|
||||
fetchfile_register(&pending);
|
||||
rc = xce_fetchfile_req(w->ep, msg);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue