feat(worker-rejoin-resync): apply reported outcomes idempotently (task 3.2)
Add resync_apply_report()/resync_apply_running_entry()/ resync_apply_finished_entry()/resync_apply_unrecognized_entry() to resync.c: routes each entry in a worker's job-state report to the correct existing idempotent path (still-running -> clear from outstanding set, no fact; finished-success/failure -> record_job_finish_fact_at/ on_job_fail; unrecognized -> record_job_reconcile_pending_fact; unqueried extra -> record_job_reconcile_assign_fact or no-op), discards reports for superseded rounds (Req 6.1/6.3), and resolves the round once its outstanding set empties. Also fixes bug-1747 (Req 4.2): adds record_job_finish_fact_at(), additive alongside the unmodified record_job_finish_fact, so a finished-success reconciliation entry records the WORKER-reported finish time instead of reconciliation-processing time -- proven via a real WAL-write + persist_recover() restart-replay test. The FAIL-side half of the same gap (WAL_JOB_FAIL carries no fail-time wire slot at all, for any caller) is confirmed pre-existing and systemic, correctly left out of this task's scope. Three review rounds surfaced and fixed two real, distinct bugs in the unqueried-extra-job routing guard (see tasks.md's Implementation Notes for the full account and the general lesson about re-deriving a full truth table after a partial boolean-guard fix): round 1 caught the wrong axis (terminal-vs-non-terminal instead of assignment-status, Req 4.8); round 2 caught that the first fix's short-circuit clobbered an already-terminal job's DONE/EXIT back to RUN (Req 4.4), including confirming the implementer's own test had certified the bug as correct; round 3 independently re-derived the final three-way guard from scratch and found no further defect. bug-1748 logged for the guard fix. Full controller suite: 1239 tests, 0 failures. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Y4M5MJM2zK6nU43da267Ku
This commit is contained in:
parent
442fcf4544
commit
eec2596c5d
8 changed files with 1600 additions and 7 deletions
|
|
@ -241,6 +241,28 @@ extern int record_job_reconcile_pending_fact(cluster_t*, job_t*);
|
|||
extern int record_job_reconcile_assign_fact(cluster_t*, job_t*, worker_t*,
|
||||
xce_job_state_entry_t*);
|
||||
|
||||
/* worker-rejoin-resync task 3.2 (bug-1747 fix, Req 4.2): record_job_finish_
|
||||
* fact()'s worker-reported-finish-time sibling, for the reconciliation
|
||||
* finished-success path only. Same body/guard/hook-fire/deferred-append
|
||||
* pattern as record_job_finish_fact() (job.c, unmodified) -- differs only
|
||||
* in passing `finish_time_ms` (converted ms -> s) to persist_submit_job_
|
||||
* finish_fact() instead of time(NULL). See job.c's own doc comment on this
|
||||
* function for the full rationale, including the persist_submit_job_
|
||||
* finish_fact() unit confirmation.
|
||||
*
|
||||
* Precondition: caller holds `job` under RTTI_LOCK_WRITE on entry and keeps
|
||||
* holding it until this call returns -- this function does not lock/unlock
|
||||
* `job` itself, matching record_job_finish_fact's own convention.
|
||||
*
|
||||
* DEFERRED pattern (unlike record_job_reconcile_pending_fact/assign_fact
|
||||
* above): this function only durably appends the fact; it does NOT mutate
|
||||
* job's in-memory state and the caller must NOT call persist_notify_job_
|
||||
* updated_already_persisted() afterward -- exactly like record_job_finish_
|
||||
* fact(), the mutation and notify happen later, via apply_committed_entry's
|
||||
* replay of the committed WAL_JOB_FINISH entry. */
|
||||
extern int record_job_finish_fact_at(cluster_t*, job_t*, xce_jobfinish_t*,
|
||||
uint64_t);
|
||||
|
||||
extern int on_job_start(cluster_t*, job_pidinfo_t*);
|
||||
extern int on_job_finish(cluster_t*, xce_jobfinish_t*);
|
||||
extern int on_job_fail(cluster_t*, job_failinfo_t*);
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@
|
|||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "typemap.h" /* xce_job_state_entry_t/xce_job_state_report_t --
|
||||
* worker-rejoin-resync task 3.2 */
|
||||
#include "xce/cluster_def.h"
|
||||
#include "xce/types.h"
|
||||
|
||||
|
|
@ -75,6 +77,103 @@ extern void resync_resolve_round(cluster_t* c, worker_t* w);
|
|||
* Precondition: caller holds at least `w`'s read lock. */
|
||||
extern int resync_round_is_current(worker_t* w, uint64_t token);
|
||||
|
||||
/* worker-rejoin-resync task 3.2 (Req 4.1-4.5, 6.1-6.3): applies a worker's
|
||||
* already-decoded job-state report to `c`'s job table -- the glue that maps
|
||||
* each `xce_job_state_entry_t` outcome in `report` to the correct existing
|
||||
* (record_job_fail_fact via on_job_fail, job.c) or new (record_job_finish_
|
||||
* fact_at, task 3.2/bug-1747; record_job_reconcile_pending_fact/record_job_
|
||||
* reconcile_assign_fact, task 2.1/2.2, job.c) idempotent fact-recording
|
||||
* call, per design.md's "System Flows" sequence diagram (`loop each
|
||||
* reported entry`).
|
||||
*
|
||||
* Routing per `xce_job_state_entry_t.status`:
|
||||
* - JobStateStatus_running, entry->queried true: cleared from the round's
|
||||
* outstanding set; no fact recorded.
|
||||
* - JobStateStatus_running, entry->queried false (Req 3.6 extra): a
|
||||
* genuine THREE-WAY split (review round 2 fix -- a two-way boolean
|
||||
* guard is WRONG and introduces a Req 4.4 violation, see resync.c's
|
||||
* own doc comment on resync_apply_running_entry for the full incident):
|
||||
* (a) TERMINAL -> ALWAYS a no-op, regardless of host (Req 4.4 -- a
|
||||
* terminal job is never assign-eligible; record_job_reconcile_assign_
|
||||
* fact() has no backward-transition guard of its own and would
|
||||
* unconditionally clobber DONE/EXIT back to RUN if called here).
|
||||
* (b) non-terminal AND already has SOME live non-terminal host
|
||||
* assignment (`job->job.allocs != NULL` -- to THIS reporting worker's
|
||||
* own host, or to a DIFFERENT one, it does not matter which) -> a
|
||||
* no-op (already validly assigned; calling assign here would still
|
||||
* wrongly reset start_time and re-append a WAL_JOB_DISPATCH fact for a
|
||||
* job that was never actually lost).
|
||||
* (c) non-terminal AND genuinely UNASSIGNED (`allocs == NULL`) -> Req
|
||||
* 4.8's real trigger ("no existing non-terminal record... assigned
|
||||
* elsewhere" -- there IS a non-terminal record, but it carries no
|
||||
* assignment at all, e.g. still PEND because its WAL_JOB_DISPATCH fact
|
||||
* never reached durable majority commit before a controller failover,
|
||||
* even though the worker really is running it) -> reconciles via
|
||||
* record_job_reconcile_assign_fact() (Req 4.8), confirmed against that
|
||||
* function's own doc comment/assert (job.c), which documents itself as
|
||||
* the running-only case. A jobid with NO record at all in the job
|
||||
* table cannot be reconciled this way (that function mutates an
|
||||
* existing job_t, it does not construct one) and is logged and skipped
|
||||
* -- out of this feature's scope; every job a worker can legitimately
|
||||
* run was submitted through this controller at some point, so this is
|
||||
* expected to be unreachable in practice.
|
||||
* - JobStateStatus_finishedOk: routed through record_job_finish_fact_at()
|
||||
* (task 3.2/bug-1747 fix, job.c) -- record_job_finish_fact()'s own
|
||||
* worker-reported-finish-time sibling, so the recorded finish_time is
|
||||
* entry->finish_time_ms (Req 3.5), not reconciliation-processing time
|
||||
* (Req 4.2). record_job_finish_fact() itself is unmodified; its own
|
||||
* caller (on_job_finish, an ordinary worker report) still uses
|
||||
* receipt-time, correctly, for that near-instantaneous case.
|
||||
* - JobStateStatus_finishedFail: routed through on_job_fail() (job.c),
|
||||
* unchanged -- Requirement 4.2 remains unsatisfiable for this outcome
|
||||
* via any existing path in this codebase (WAL_JOB_FAIL carries no
|
||||
* finish-time slot at any layer, for any caller -- a pre-existing,
|
||||
* systemic gap that predates this feature; see buglog.json bug-1747).
|
||||
* - Both finished outcomes apply identically whether entry->queried is
|
||||
* true or false (Req 4.7: an unqueried finished extra reconciles
|
||||
* through the same path as a queried one; there is no "assign" case for
|
||||
* a finished outcome). Cleared from the outstanding set when
|
||||
* entry->queried is true.
|
||||
* - JobStateStatus_unrecognized: routed through record_job_reconcile_
|
||||
* pending_fact() (Req 4.6, task 2.1); cleared from the outstanding set.
|
||||
*
|
||||
* Req 6.1/6.3: if `round_token` is not `w`'s CURRENTLY outstanding round
|
||||
* (resync_round_is_current), the entire report is discarded before
|
||||
* anything else is touched -- no fact is applied, no outstanding-set
|
||||
* mutation occurs. This also makes a duplicate re-application of an
|
||||
* already-fully-resolved report (Req 6.2) a pure no-op by the same
|
||||
* mechanism: once every queried job has a reconciled outcome,
|
||||
* resync_resolve_round() clears w->resync, so `round_token` is no longer
|
||||
* current on a later identical call -- no facts are re-applied, no
|
||||
* additional WAL entries appended (a duplicate applied WHILE the round is
|
||||
* still open instead re-applies each fact through job.c's own idempotent
|
||||
* guards, which are themselves safe no-ops on a repeat).
|
||||
*
|
||||
* Req 5.1/5.2: once the round's outstanding set becomes empty (every
|
||||
* queried jobid has been cleared), the round is resolved
|
||||
* (resync_resolve_round()), clearing XCE_WORKER_RECONCILING so `w` becomes
|
||||
* eligible for dispatch again.
|
||||
*
|
||||
* Locking: unlike resync_arm_round/resync_resolve_round/resync_round_is_
|
||||
* current above, this function manages its OWN `w` write-locking
|
||||
* internally and narrowly (never nested with a job's own write lock) --
|
||||
* mirroring dispatch_a_job_'s established discipline (job.c: every branch
|
||||
* releases `w`'s lock BEFORE acquiring a job's) rather than this file's
|
||||
* other three functions' "caller holds w's lock for the whole call"
|
||||
* convention, which fits their simple, single-object, single-call-site
|
||||
* mutations but not this function's fan-out into per-job job.c calls, each
|
||||
* requiring its own job-level lock. The caller must NOT pre-hold `w`'s
|
||||
* lock when calling this function.
|
||||
*
|
||||
* `report` is treated as a single, already-fully-assembled list of
|
||||
* `report->count` entries (`report->entries[0..count)`) -- multi-page wire
|
||||
* reassembly is a later task's concern (6.2), out of this function's scope.
|
||||
*
|
||||
* Returns 0 on success (whether the report was applied or discarded as
|
||||
* stale), or a negative errno for invalid arguments (NULL c/w/report). */
|
||||
extern int resync_apply_report(cluster_t* c, worker_t* w, uint64_t round_token,
|
||||
xce_job_state_report_t* report);
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
/* worker-rejoin-resync task 3.1 test seam: directly invokes the SAME
|
||||
* internal timeout-fire logic resync_arm_round's own delaymgr callback runs
|
||||
|
|
|
|||
|
|
@ -2729,6 +2729,64 @@ static int record_job_finish_fact(cluster_t* c, job_t* job,
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* worker-rejoin-resync task 3.2 (bug-1747 fix, Req 4.2): WAL_JOB_FINISH's
|
||||
* own worker-reported-finish-time sibling of record_job_finish_fact() above
|
||||
* -- identical guard/hook-fire/durable-append body, except it passes a
|
||||
* CALLER-supplied finish time (converted from the worker's reported
|
||||
* finish_time_ms, Req 3.5) to persist_submit_job_finish_fact() instead of
|
||||
* always deriving time(NULL) at receipt time. record_job_finish_fact()
|
||||
* itself is deliberately left UNMODIFIED: its own callers (on_job_finish's
|
||||
* ordinary, near-instantaneous fire-and-forget worker report) are correctly
|
||||
* served by receipt-time ~= true finish-time; only a RECONCILED finish
|
||||
* (received long after the fact, on rejoin) needs the worker's own reported
|
||||
* time instead -- mirroring task 2.1/2.2's own "new function alongside an
|
||||
* unmodified existing one" pattern (record_job_reconcile_pending_fact/
|
||||
* record_job_reconcile_assign_fact), not a new precedent.
|
||||
*
|
||||
* persist_submit_job_finish_fact()'s own `finish_time` parameter is SECONDS
|
||||
* (time_t semantics) -- confirmed by its only other call site
|
||||
* (record_job_finish_fact() above: `time_t finish_time = time(NULL);
|
||||
* ...(uint64_t)finish_time`) and by the WAL_JOB_FINISH decode side
|
||||
* (apply_committed_entry, persist.c: `j->job.finish_time =
|
||||
* (time_t)wire_finish_time;`, a direct, unconverted assignment). The
|
||||
* worker-reported wire field (xce_job_state_entry_t.finish_time_ms,
|
||||
* typemap.h) is MILLISECONDS ("finishTimeMs" on the wire, Req 3.5) -- this
|
||||
* function converts ms -> s (`finish_time_ms / 1000`) before calling
|
||||
* persist_submit_job_finish_fact(), so the caller (resync.c) can pass the
|
||||
* raw worker-reported value straight through without needing to know this
|
||||
* unit detail itself.
|
||||
*
|
||||
* Same locking convention as record_job_finish_fact(): caller holds `job`
|
||||
* under RTTI_LOCK_WRITE on entry and keeps holding it until this call
|
||||
* returns; this function does not lock/unlock `job` itself. Same DEFERRED
|
||||
* pattern too: durable append only here, no mutation -- the in-memory state
|
||||
* change happens later via apply_committed_entry()'s replay, exactly as an
|
||||
* ordinary finish already works (unlike record_job_reconcile_pending_fact/
|
||||
* assign_fact's own EAGER pattern) -- so the caller must NOT call
|
||||
* persist_notify_job_updated_already_persisted() after this returns; that
|
||||
* notify happens later, at apply time, exactly as it already does for
|
||||
* record_job_finish_fact(). */
|
||||
int record_job_finish_fact_at(cluster_t* c, job_t* job,
|
||||
xce_jobfinish_t* finish, uint64_t finish_time_ms)
|
||||
{
|
||||
if (job_state_is_terminal(job->job.state)) {
|
||||
apply_job_finish_transition(job, finish); /* logs + no-op, unchanged */
|
||||
return 0;
|
||||
}
|
||||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_JOB_EXIT, c, finish, job);
|
||||
|
||||
(void)persist_submit_job_finish_fact(&c->persist, job->job.jobid,
|
||||
(finish->exit_code == 0)
|
||||
? JOB_STAT_DONE
|
||||
: JOB_STAT_EXIT,
|
||||
finish_time_ms / 1000,
|
||||
finish->exit_code);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int on_job_finish(cluster_t* c, xce_jobfinish_t* finish)
|
||||
{
|
||||
job_t* job = NULL;
|
||||
|
|
|
|||
|
|
@ -23,7 +23,11 @@
|
|||
#include <time.h>
|
||||
|
||||
#include "cluster.h"
|
||||
#include "controller.capnp.h" /* JobStateStatus_* -- worker-rejoin-resync
|
||||
* task 3.2 */
|
||||
#include "delay.h"
|
||||
#include "persist.h" /* persist_notify_job_updated_already_persisted --
|
||||
* worker-rejoin-resync task 3.2 */
|
||||
#include "resync.h"
|
||||
#include "utils.h"
|
||||
#include "xce/types.h"
|
||||
|
|
@ -412,6 +416,343 @@ int resync_round_is_current(worker_t* w, uint64_t token)
|
|||
return w->resync->round_token == token;
|
||||
}
|
||||
|
||||
/* ── Report application (task 3.2, Req 4.1-4.5, 6.1-6.3) ─────────────────
|
||||
*
|
||||
* Maps each JobStateEntry in a worker's job-state report to the correct
|
||||
* idempotent job.c fact-recording call, then resolves the round once every
|
||||
* queried job has a reconciled outcome. See resync.h's doc comment on
|
||||
* resync_apply_report() for the full per-outcome routing contract, and
|
||||
* design.md's "System Flows" sequence diagram (the `loop each reported
|
||||
* entry` block) for the design this implements.
|
||||
*/
|
||||
|
||||
/* Req 6.2: removes `jobid` from w->resync's outstanding query set, if
|
||||
* present -- swap-with-last-and-shrink (the array is small, bounded by one
|
||||
* host's non-terminal job count per resync_collect_query_jobids' own
|
||||
* reasoning, so O(n) removal is not a concern). A no-op when w->resync is
|
||||
* NULL or `jobid` is not present -- safe to call unconditionally and
|
||||
* repeatedly (an already-cleared id, or one that was never queried, e.g. an
|
||||
* unqueried extra). Caller holds w's write lock. */
|
||||
static void resync_clear_outstanding(worker_t* w, uint64_t jobid)
|
||||
{
|
||||
resync_state_t* rs = w->resync;
|
||||
int i;
|
||||
|
||||
if (rs == NULL)
|
||||
return;
|
||||
|
||||
for (i = 0; i < rs->n_queried_jobids; i++) {
|
||||
if (rs->queried_jobids[i] == jobid) {
|
||||
rs->queried_jobids[i]
|
||||
= rs->queried_jobids[rs->n_queried_jobids - 1];
|
||||
rs->n_queried_jobids--;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Req 5.1/5.2: resolves w's round if its outstanding set has just become
|
||||
* empty. Caller holds w's write lock and must have already confirmed the
|
||||
* round is still current for the token in question (see
|
||||
* resync_settle_entry below) -- this function itself only re-checks
|
||||
* w->resync != NULL (resync_resolve_round's own precondition), not token
|
||||
* identity, since the caller's own check under the SAME uninterrupted lock
|
||||
* hold already establishes that. */
|
||||
static void resync_maybe_resolve(cluster_t* c, worker_t* w)
|
||||
{
|
||||
if (w->resync != NULL && w->resync->n_queried_jobids == 0)
|
||||
resync_resolve_round(c, w);
|
||||
}
|
||||
|
||||
/* Req 4.1, 5.1, 5.2, 6.1-6.3: clears `jobid` from w's outstanding set for
|
||||
* round `token` and, if the set has just become empty, resolves the round
|
||||
* -- both performed under a SINGLE w write-lock acquisition, both gated on
|
||||
* `token` still being w's CURRENT round. The gate matters: a concurrent Req
|
||||
* 5.3 timeout fire, or a fresh resync_arm_round superseding this round on
|
||||
* another thread, can happen between this report-apply loop's per-entry
|
||||
* fact application (which holds no lock on `w`) and this call -- without
|
||||
* re-validating here, a stale round's late bookkeeping could otherwise
|
||||
* mutate a DIFFERENT (newer) round's resync_state_t. This is also the exact
|
||||
* mechanism that makes re-applying an already-fully-resolved report a safe
|
||||
* no-op (Req 6.2): once resync_resolve_round() has run, w->resync is NULL,
|
||||
* so `token` no longer reports current and this becomes a pure no-op. */
|
||||
static void resync_settle_entry(cluster_t* c, worker_t* w, uint64_t token,
|
||||
uint64_t jobid)
|
||||
{
|
||||
lock(w, RTTI_LOCK_WRITE);
|
||||
if (resync_round_is_current(w, token)) {
|
||||
resync_clear_outstanding(w, jobid);
|
||||
resync_maybe_resolve(c, w);
|
||||
}
|
||||
unlock(w);
|
||||
}
|
||||
|
||||
/* Looks up `jobid` in c->jobmgr, reference()-ing it on success -- the same
|
||||
* key_find+reference idiom on_job_start/on_job_finish/on_job_fail/
|
||||
* dispatch_a_job_ (job.c) already use throughout this codebase. Returns
|
||||
* NULL if not found -- never treated as an error by this file's own callers
|
||||
* (see resync_apply_running_entry's doc comment for why a genuinely-
|
||||
* untracked jobid is a legitimate, if narrow, possibility). Caller must
|
||||
* dereference() the result when done, if non-NULL. */
|
||||
static job_t* resync_lookup_job(cluster_t* c, uint64_t jobid)
|
||||
{
|
||||
job_t* j = NULL;
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&j);
|
||||
if (j != NULL)
|
||||
reference(j);
|
||||
unlock(c->jobmgr);
|
||||
|
||||
return j;
|
||||
}
|
||||
|
||||
/* Req 4.1, 4.4, 4.7, 4.8: routes a "running" entry. A QUERIED running job's
|
||||
* own state already correctly reflects "running" -- no fact is recorded
|
||||
* (design.md's sequence diagram: "else still running -> clear this jobid
|
||||
* from outstanding set (no-op fact)"); the caller clears the outstanding
|
||||
* set for every queried entry uniformly (resync_apply_report below), so
|
||||
* this function has nothing further to do for that case.
|
||||
*
|
||||
* An UNQUERIED extra (Req 3.6) running job is a genuine THREE-WAY split, not
|
||||
* a two-way boolean (review round 2 regression fix, following round 1's own
|
||||
* review-caught fix -- the round-1 guard introduced a NEW Req 4.4 violation:
|
||||
* `!terminal && assigned-to-w` short-circuits to `false` whenever the job IS
|
||||
* terminal, falling through to the assign branch and unconditionally
|
||||
* clobbering a DONE/EXIT job back to RUN -- confirmed live: round 1's own
|
||||
* test seeded a TERMINAL j_extra and asserted it became RUN, certifying the
|
||||
* bug as "correct" instead of catching it):
|
||||
*
|
||||
* (a) TERMINAL (resync_job_is_terminal()) -> ALWAYS a no-op, regardless of
|
||||
* host. Requirement 4.4 ("shall not overwrite that state with a less
|
||||
* final or older report") is unconditional -- a terminal job is never
|
||||
* assign-eligible, full stop. record_job_reconcile_assign_fact() has NO
|
||||
* backward-transition guard of its own (its own doc comment: "no
|
||||
* reject/no-op case" is a documented CALLER precondition, never
|
||||
* re-checked internally) and unconditionally sets job->job.state =
|
||||
* JOB_STAT_RUN -- calling it here would clobber DONE/EXIT back to RUN.
|
||||
* (b) Non-terminal AND job->job.allocs != NULL (already has SOME live
|
||||
* non-terminal host assignment -- to THIS reporting worker's own host,
|
||||
* or to a DIFFERENT one, it does not matter which) -> a no-op. It is
|
||||
* already validly assigned; record_job_reconcile_assign_fact() would
|
||||
* still be wrong to call even though its net STATE effect looks
|
||||
* harmless (RUN -> RUN) -- it unconditionally resets job->job.start_time
|
||||
* and re-appends a fresh WAL_JOB_DISPATCH fact, neither of which is
|
||||
* warranted for a job that was never actually lost/unassigned.
|
||||
* (c) Non-terminal AND job->job.allocs == NULL (genuinely UNASSIGNED) ->
|
||||
* Req 4.8's real trigger condition ("no existing non-terminal record...
|
||||
* assigned elsewhere" -- there IS a non-terminal record, but it carries
|
||||
* no assignment at all) -- reconciled via record_job_reconcile_assign_
|
||||
* fact() (task 2.2), documented (and hard-asserted, job.c) as the
|
||||
* running-only case.
|
||||
*
|
||||
* Concrete motivating scenario for (c) (the exact case design.md's own
|
||||
* "Documented scope decision" paragraph describes as this function's
|
||||
* target, and record_job_reconcile_assign_fact()'s core justification for
|
||||
* existing at all): a job the controller currently shows PEND (non-
|
||||
* terminal, allocs == NULL, unassigned to any host) because its
|
||||
* WAL_JOB_DISPATCH fact never reached durable majority commit before a
|
||||
* controller failover -- even though dispatch_a_job_() already sent
|
||||
* xce_startjob_req() to the worker and the worker really is running it (the
|
||||
* RPC fires BEFORE dispatch_a_job_()'s own eager mutation + WAL append, so
|
||||
* the two can diverge exactly at a failover boundary). Such a job correctly
|
||||
* reconciles via record_job_reconcile_assign_fact() (host now matches the
|
||||
* reporting worker, allocs stays NULL per Req 4.8's own documented scope
|
||||
* decision).
|
||||
*
|
||||
* A jobid with NO record AT ALL in the job table cannot be reconciled via
|
||||
* record_job_reconcile_assign_fact() either -- that function mutates an
|
||||
* EXISTING job_t, it does not construct one -- so this is logged and
|
||||
* skipped rather than fabricating a full job record from a bare entry;
|
||||
* every job a worker can legitimately run was, by this system's own
|
||||
* architecture, submitted through this controller at some point, so this
|
||||
* branch is expected to be unreachable in practice, not a routine path. */
|
||||
static void resync_apply_running_entry(cluster_t* c, worker_t* w,
|
||||
xce_job_state_entry_t* entry)
|
||||
{
|
||||
job_t* job;
|
||||
|
||||
if (entry->queried)
|
||||
return;
|
||||
|
||||
job = resync_lookup_job(c, entry->jobid);
|
||||
if (job == NULL) {
|
||||
WARN("resync: unqueried extra job %lu reported running by worker "
|
||||
"%s has no record at all in the job table -- cannot reconcile "
|
||||
"(no existing job_t to assign onto); skipped",
|
||||
(unsigned long)entry->jobid,
|
||||
(w->host.name != NULL) ? w->host.name : "?");
|
||||
return;
|
||||
}
|
||||
|
||||
lock(job, RTTI_LOCK_WRITE);
|
||||
if (resync_job_is_terminal(job->job.state)) {
|
||||
/* (a) Terminal is always final -- Req 4.4. Never assign-eligible,
|
||||
* regardless of host. */
|
||||
unlock(job);
|
||||
} else if (job->job.allocs != NULL) {
|
||||
/* (b) Non-terminal AND already has SOME live non-terminal host
|
||||
* assignment -- a genuine no-op; do not reset start_time or
|
||||
* re-append a WAL_JOB_DISPATCH fact for a job that was never
|
||||
* actually lost/unassigned. */
|
||||
unlock(job);
|
||||
} else {
|
||||
/* (c) Non-terminal AND genuinely UNASSIGNED -- Req 4.8 fires. */
|
||||
int rc = record_job_reconcile_assign_fact(c, job, w, entry);
|
||||
unlock(job);
|
||||
if (rc == (int)JOB_TRANSITION_APPLIED)
|
||||
persist_notify_job_updated_already_persisted(c, job);
|
||||
}
|
||||
|
||||
dereference(job);
|
||||
}
|
||||
|
||||
/* Req 4.1-4.5, 4.7: routes a "finished" entry (success or failure) through
|
||||
* the SAME existing, deferred-pattern fact-recording used for an ordinary
|
||||
* worker finish/fail report. Applies identically whether entry->queried is
|
||||
* true or false (Req 4.7: an unqueried finished extra reconciles through
|
||||
* "the same idempotent commit-then-apply path... used for queried jobs" --
|
||||
* there is no separate "assign" case for a finished outcome; record_job_
|
||||
* reconcile_assign_fact() is documented as running-only, see job.c's own
|
||||
* doc comment on it).
|
||||
*
|
||||
* finishedOk: bug-1747 fix (Req 4.2) -- routed through the NEW record_job_
|
||||
* finish_fact_at() (job.c), not on_job_finish()/record_job_finish_fact(),
|
||||
* so the job's recorded finish_time is the WORKER-reported entry->
|
||||
* finish_time_ms (Req 3.5), not reconciliation-processing wall-clock time.
|
||||
* record_job_finish_fact_at() expects the caller to already hold job's
|
||||
* write lock (like record_job_reconcile_pending_fact/assign_fact above),
|
||||
* unlike on_job_finish() which locks internally -- this function performs
|
||||
* that lookup/lock/unlock/dereference sequence itself, mirroring on_job_
|
||||
* finish()'s own internal sequence exactly (job.c). A jobid not found in
|
||||
* the job table is a silent no-op, mirroring on_job_finish()'s own
|
||||
* "can not find job" behavior for an unknown jobid.
|
||||
*
|
||||
* finishedFail: still routed through the existing, UNMODIFIED on_job_fail()
|
||||
* /record_job_fail_fact() -- Requirement 4.2 remains unsatisfiable for this
|
||||
* outcome via any existing path in this codebase, not a reconciliation-
|
||||
* specific gap: WAL_JOB_FAIL's payload (persist_submit_job_fact_ex(...,
|
||||
* WAL_JOB_FAIL, jobid, 0, WAL_JOB_FAIL_STAGE_FLAG|stage)) carries no
|
||||
* finish-time slot at any layer, and apply_job_fail_transition() (job.c)
|
||||
* unconditionally sets job->job.finish_time = time(NULL) at APPLY time
|
||||
* (not even receipt time) for EVERY fail-reporting caller in this system,
|
||||
* confirmed by direct inspection -- a pre-existing, systemic gap that
|
||||
* predates this feature and is out of this task's boundary to fix (see
|
||||
* buglog.json bug-1747, re-scoped). The reconciled job's STATE (EXIT) is
|
||||
* always correct either way; only its recorded finish_time reflects
|
||||
* apply-processing time rather than the worker's true report. */
|
||||
static void resync_apply_finished_entry(cluster_t* c,
|
||||
xce_job_state_entry_t* entry)
|
||||
{
|
||||
if (entry->status == JobStateStatus_finishedOk) {
|
||||
job_t* job = resync_lookup_job(c, entry->jobid);
|
||||
|
||||
if (job != NULL) {
|
||||
xce_jobfinish_t finish
|
||||
= { .jobid = entry->jobid, .exit_code = 0 };
|
||||
|
||||
lock(job, RTTI_LOCK_WRITE);
|
||||
record_job_finish_fact_at(c, job, &finish,
|
||||
entry->finish_time_ms);
|
||||
unlock(job);
|
||||
|
||||
dereference(job);
|
||||
}
|
||||
} else {
|
||||
job_failinfo_t fail
|
||||
= { .jobid = entry->jobid,
|
||||
.stage = JOB_EXEC_STAGE_MAIN,
|
||||
.code = entry->exit_code,
|
||||
.detail = (char*)"worker-rejoin-resync reconciliation" };
|
||||
on_job_fail(c, &fail);
|
||||
}
|
||||
}
|
||||
|
||||
/* Req 4.1, 4.5, 4.6: routes an "unrecognized" entry through record_job_
|
||||
* reconcile_pending_fact() (task 2.1). Per Req 3.1/3.3, a worker only ever
|
||||
* reports "unrecognized" for a jobid it was explicitly asked about (an
|
||||
* unqueried extra is, by construction of Req 3.6's own "currently running
|
||||
* or recently finished" trigger, never reported unrecognized) -- so this
|
||||
* path is queried-only in practice; the caller (resync_apply_report below)
|
||||
* still clears the outstanding set uniformly for every queried entry
|
||||
* regardless of which branch handled it. */
|
||||
static void resync_apply_unrecognized_entry(cluster_t* c,
|
||||
xce_job_state_entry_t* entry)
|
||||
{
|
||||
job_t* job = resync_lookup_job(c, entry->jobid);
|
||||
int rc;
|
||||
|
||||
if (job == NULL)
|
||||
return;
|
||||
|
||||
lock(job, RTTI_LOCK_WRITE);
|
||||
rc = record_job_reconcile_pending_fact(c, job);
|
||||
unlock(job);
|
||||
|
||||
if (rc == (int)JOB_TRANSITION_APPLIED)
|
||||
persist_notify_job_updated_already_persisted(c, job);
|
||||
|
||||
dereference(job);
|
||||
}
|
||||
|
||||
int resync_apply_report(cluster_t* c, worker_t* w, uint64_t round_token,
|
||||
xce_job_state_report_t* report)
|
||||
{
|
||||
int i;
|
||||
|
||||
if (c == NULL || w == NULL || report == NULL)
|
||||
return -EINVAL;
|
||||
if (report->count > 0 && report->entries == NULL)
|
||||
return -EINVAL;
|
||||
|
||||
/* Req 6.1, 6.3: a report for a round already superseded/discarded is
|
||||
* dropped in its entirety, before anything else is touched -- checked
|
||||
* under w's own write lock (matching every other w->resync mutation
|
||||
* this function goes on to make, via resync_settle_entry, per entry
|
||||
* below). */
|
||||
lock(w, RTTI_LOCK_WRITE);
|
||||
if (!resync_round_is_current(w, round_token)) {
|
||||
unlock(w);
|
||||
return 0;
|
||||
}
|
||||
unlock(w);
|
||||
|
||||
/* Per-entry routing never holds w's write lock while also locking a
|
||||
* job's own write lock -- mirrors dispatch_a_job_'s established
|
||||
* discipline (job.c: every branch releases w's lock BEFORE acquiring a
|
||||
* job's) rather than this file's other three functions' "caller holds
|
||||
* w's lock for the whole call" convention, which fits their simple,
|
||||
* single-object, single-call-site mutations but not this function's
|
||||
* fan-out into per-job job.c calls, each requiring its own job-level
|
||||
* lock. w's lock is only (re)acquired, narrowly, inside
|
||||
* resync_settle_entry's own outstanding-set/maybe-resolve mutation. */
|
||||
for (i = 0; i < report->count; i++) {
|
||||
xce_job_state_entry_t* entry = report->entries[i];
|
||||
|
||||
if (entry == NULL)
|
||||
continue;
|
||||
|
||||
switch (entry->status) {
|
||||
case JobStateStatus_running:
|
||||
resync_apply_running_entry(c, w, entry);
|
||||
break;
|
||||
case JobStateStatus_finishedOk:
|
||||
case JobStateStatus_finishedFail:
|
||||
resync_apply_finished_entry(c, entry);
|
||||
break;
|
||||
case JobStateStatus_unrecognized:
|
||||
default:
|
||||
resync_apply_unrecognized_entry(c, entry);
|
||||
break;
|
||||
}
|
||||
|
||||
if (entry->queried)
|
||||
resync_settle_entry(c, w, round_token, entry->jobid);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
/* worker-rejoin-resync task 3.1 test seam (resync.h): builds a
|
||||
* resync_timeout_ctx_t exactly as resync_arm_round's own delay_task_ms call
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
Loading…
Add table
Add a link
Reference in a new issue