feat(worker-rejoin-resync): apply reported outcomes idempotently (task 3.2)
Add resync_apply_report()/resync_apply_running_entry()/ resync_apply_finished_entry()/resync_apply_unrecognized_entry() to resync.c: routes each entry in a worker's job-state report to the correct existing idempotent path (still-running -> clear from outstanding set, no fact; finished-success/failure -> record_job_finish_fact_at/ on_job_fail; unrecognized -> record_job_reconcile_pending_fact; unqueried extra -> record_job_reconcile_assign_fact or no-op), discards reports for superseded rounds (Req 6.1/6.3), and resolves the round once its outstanding set empties. Also fixes bug-1747 (Req 4.2): adds record_job_finish_fact_at(), additive alongside the unmodified record_job_finish_fact, so a finished-success reconciliation entry records the WORKER-reported finish time instead of reconciliation-processing time -- proven via a real WAL-write + persist_recover() restart-replay test. The FAIL-side half of the same gap (WAL_JOB_FAIL carries no fail-time wire slot at all, for any caller) is confirmed pre-existing and systemic, correctly left out of this task's scope. Three review rounds surfaced and fixed two real, distinct bugs in the unqueried-extra-job routing guard (see tasks.md's Implementation Notes for the full account and the general lesson about re-deriving a full truth table after a partial boolean-guard fix): round 1 caught the wrong axis (terminal-vs-non-terminal instead of assignment-status, Req 4.8); round 2 caught that the first fix's short-circuit clobbered an already-terminal job's DONE/EXIT back to RUN (Req 4.4), including confirming the implementer's own test had certified the bug as correct; round 3 independently re-derived the final three-way guard from scratch and found no further defect. bug-1748 logged for the guard fix. Full controller suite: 1239 tests, 0 failures. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Y4M5MJM2zK6nU43da267Ku
This commit is contained in:
parent
442fcf4544
commit
eec2596c5d
8 changed files with 1600 additions and 7 deletions
|
|
@ -23,7 +23,11 @@
|
|||
#include <time.h>
|
||||
|
||||
#include "cluster.h"
|
||||
#include "controller.capnp.h" /* JobStateStatus_* -- worker-rejoin-resync
|
||||
* task 3.2 */
|
||||
#include "delay.h"
|
||||
#include "persist.h" /* persist_notify_job_updated_already_persisted --
|
||||
* worker-rejoin-resync task 3.2 */
|
||||
#include "resync.h"
|
||||
#include "utils.h"
|
||||
#include "xce/types.h"
|
||||
|
|
@ -412,6 +416,343 @@ int resync_round_is_current(worker_t* w, uint64_t token)
|
|||
return w->resync->round_token == token;
|
||||
}
|
||||
|
||||
/* ── Report application (task 3.2, Req 4.1-4.5, 6.1-6.3) ─────────────────
|
||||
*
|
||||
* Maps each JobStateEntry in a worker's job-state report to the correct
|
||||
* idempotent job.c fact-recording call, then resolves the round once every
|
||||
* queried job has a reconciled outcome. See resync.h's doc comment on
|
||||
* resync_apply_report() for the full per-outcome routing contract, and
|
||||
* design.md's "System Flows" sequence diagram (the `loop each reported
|
||||
* entry` block) for the design this implements.
|
||||
*/
|
||||
|
||||
/* Req 6.2: removes `jobid` from w->resync's outstanding query set, if
|
||||
* present -- swap-with-last-and-shrink (the array is small, bounded by one
|
||||
* host's non-terminal job count per resync_collect_query_jobids' own
|
||||
* reasoning, so O(n) removal is not a concern). A no-op when w->resync is
|
||||
* NULL or `jobid` is not present -- safe to call unconditionally and
|
||||
* repeatedly (an already-cleared id, or one that was never queried, e.g. an
|
||||
* unqueried extra). Caller holds w's write lock. */
|
||||
static void resync_clear_outstanding(worker_t* w, uint64_t jobid)
|
||||
{
|
||||
resync_state_t* rs = w->resync;
|
||||
int i;
|
||||
|
||||
if (rs == NULL)
|
||||
return;
|
||||
|
||||
for (i = 0; i < rs->n_queried_jobids; i++) {
|
||||
if (rs->queried_jobids[i] == jobid) {
|
||||
rs->queried_jobids[i]
|
||||
= rs->queried_jobids[rs->n_queried_jobids - 1];
|
||||
rs->n_queried_jobids--;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Req 5.1/5.2: resolves w's round if its outstanding set has just become
|
||||
* empty. Caller holds w's write lock and must have already confirmed the
|
||||
* round is still current for the token in question (see
|
||||
* resync_settle_entry below) -- this function itself only re-checks
|
||||
* w->resync != NULL (resync_resolve_round's own precondition), not token
|
||||
* identity, since the caller's own check under the SAME uninterrupted lock
|
||||
* hold already establishes that. */
|
||||
static void resync_maybe_resolve(cluster_t* c, worker_t* w)
|
||||
{
|
||||
if (w->resync != NULL && w->resync->n_queried_jobids == 0)
|
||||
resync_resolve_round(c, w);
|
||||
}
|
||||
|
||||
/* Req 4.1, 5.1, 5.2, 6.1-6.3: clears `jobid` from w's outstanding set for
|
||||
* round `token` and, if the set has just become empty, resolves the round
|
||||
* -- both performed under a SINGLE w write-lock acquisition, both gated on
|
||||
* `token` still being w's CURRENT round. The gate matters: a concurrent Req
|
||||
* 5.3 timeout fire, or a fresh resync_arm_round superseding this round on
|
||||
* another thread, can happen between this report-apply loop's per-entry
|
||||
* fact application (which holds no lock on `w`) and this call -- without
|
||||
* re-validating here, a stale round's late bookkeeping could otherwise
|
||||
* mutate a DIFFERENT (newer) round's resync_state_t. This is also the exact
|
||||
* mechanism that makes re-applying an already-fully-resolved report a safe
|
||||
* no-op (Req 6.2): once resync_resolve_round() has run, w->resync is NULL,
|
||||
* so `token` no longer reports current and this becomes a pure no-op. */
|
||||
static void resync_settle_entry(cluster_t* c, worker_t* w, uint64_t token,
|
||||
uint64_t jobid)
|
||||
{
|
||||
lock(w, RTTI_LOCK_WRITE);
|
||||
if (resync_round_is_current(w, token)) {
|
||||
resync_clear_outstanding(w, jobid);
|
||||
resync_maybe_resolve(c, w);
|
||||
}
|
||||
unlock(w);
|
||||
}
|
||||
|
||||
/* Looks up `jobid` in c->jobmgr, reference()-ing it on success -- the same
|
||||
* key_find+reference idiom on_job_start/on_job_finish/on_job_fail/
|
||||
* dispatch_a_job_ (job.c) already use throughout this codebase. Returns
|
||||
* NULL if not found -- never treated as an error by this file's own callers
|
||||
* (see resync_apply_running_entry's doc comment for why a genuinely-
|
||||
* untracked jobid is a legitimate, if narrow, possibility). Caller must
|
||||
* dereference() the result when done, if non-NULL. */
|
||||
static job_t* resync_lookup_job(cluster_t* c, uint64_t jobid)
|
||||
{
|
||||
job_t* j = NULL;
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&j);
|
||||
if (j != NULL)
|
||||
reference(j);
|
||||
unlock(c->jobmgr);
|
||||
|
||||
return j;
|
||||
}
|
||||
|
||||
/* Req 4.1, 4.4, 4.7, 4.8: routes a "running" entry. A QUERIED running job's
|
||||
* own state already correctly reflects "running" -- no fact is recorded
|
||||
* (design.md's sequence diagram: "else still running -> clear this jobid
|
||||
* from outstanding set (no-op fact)"); the caller clears the outstanding
|
||||
* set for every queried entry uniformly (resync_apply_report below), so
|
||||
* this function has nothing further to do for that case.
|
||||
*
|
||||
* An UNQUERIED extra (Req 3.6) running job is a genuine THREE-WAY split, not
|
||||
* a two-way boolean (review round 2 regression fix, following round 1's own
|
||||
* review-caught fix -- the round-1 guard introduced a NEW Req 4.4 violation:
|
||||
* `!terminal && assigned-to-w` short-circuits to `false` whenever the job IS
|
||||
* terminal, falling through to the assign branch and unconditionally
|
||||
* clobbering a DONE/EXIT job back to RUN -- confirmed live: round 1's own
|
||||
* test seeded a TERMINAL j_extra and asserted it became RUN, certifying the
|
||||
* bug as "correct" instead of catching it):
|
||||
*
|
||||
* (a) TERMINAL (resync_job_is_terminal()) -> ALWAYS a no-op, regardless of
|
||||
* host. Requirement 4.4 ("shall not overwrite that state with a less
|
||||
* final or older report") is unconditional -- a terminal job is never
|
||||
* assign-eligible, full stop. record_job_reconcile_assign_fact() has NO
|
||||
* backward-transition guard of its own (its own doc comment: "no
|
||||
* reject/no-op case" is a documented CALLER precondition, never
|
||||
* re-checked internally) and unconditionally sets job->job.state =
|
||||
* JOB_STAT_RUN -- calling it here would clobber DONE/EXIT back to RUN.
|
||||
* (b) Non-terminal AND job->job.allocs != NULL (already has SOME live
|
||||
* non-terminal host assignment -- to THIS reporting worker's own host,
|
||||
* or to a DIFFERENT one, it does not matter which) -> a no-op. It is
|
||||
* already validly assigned; record_job_reconcile_assign_fact() would
|
||||
* still be wrong to call even though its net STATE effect looks
|
||||
* harmless (RUN -> RUN) -- it unconditionally resets job->job.start_time
|
||||
* and re-appends a fresh WAL_JOB_DISPATCH fact, neither of which is
|
||||
* warranted for a job that was never actually lost/unassigned.
|
||||
* (c) Non-terminal AND job->job.allocs == NULL (genuinely UNASSIGNED) ->
|
||||
* Req 4.8's real trigger condition ("no existing non-terminal record...
|
||||
* assigned elsewhere" -- there IS a non-terminal record, but it carries
|
||||
* no assignment at all) -- reconciled via record_job_reconcile_assign_
|
||||
* fact() (task 2.2), documented (and hard-asserted, job.c) as the
|
||||
* running-only case.
|
||||
*
|
||||
* Concrete motivating scenario for (c) (the exact case design.md's own
|
||||
* "Documented scope decision" paragraph describes as this function's
|
||||
* target, and record_job_reconcile_assign_fact()'s core justification for
|
||||
* existing at all): a job the controller currently shows PEND (non-
|
||||
* terminal, allocs == NULL, unassigned to any host) because its
|
||||
* WAL_JOB_DISPATCH fact never reached durable majority commit before a
|
||||
* controller failover -- even though dispatch_a_job_() already sent
|
||||
* xce_startjob_req() to the worker and the worker really is running it (the
|
||||
* RPC fires BEFORE dispatch_a_job_()'s own eager mutation + WAL append, so
|
||||
* the two can diverge exactly at a failover boundary). Such a job correctly
|
||||
* reconciles via record_job_reconcile_assign_fact() (host now matches the
|
||||
* reporting worker, allocs stays NULL per Req 4.8's own documented scope
|
||||
* decision).
|
||||
*
|
||||
* A jobid with NO record AT ALL in the job table cannot be reconciled via
|
||||
* record_job_reconcile_assign_fact() either -- that function mutates an
|
||||
* EXISTING job_t, it does not construct one -- so this is logged and
|
||||
* skipped rather than fabricating a full job record from a bare entry;
|
||||
* every job a worker can legitimately run was, by this system's own
|
||||
* architecture, submitted through this controller at some point, so this
|
||||
* branch is expected to be unreachable in practice, not a routine path. */
|
||||
static void resync_apply_running_entry(cluster_t* c, worker_t* w,
|
||||
xce_job_state_entry_t* entry)
|
||||
{
|
||||
job_t* job;
|
||||
|
||||
if (entry->queried)
|
||||
return;
|
||||
|
||||
job = resync_lookup_job(c, entry->jobid);
|
||||
if (job == NULL) {
|
||||
WARN("resync: unqueried extra job %lu reported running by worker "
|
||||
"%s has no record at all in the job table -- cannot reconcile "
|
||||
"(no existing job_t to assign onto); skipped",
|
||||
(unsigned long)entry->jobid,
|
||||
(w->host.name != NULL) ? w->host.name : "?");
|
||||
return;
|
||||
}
|
||||
|
||||
lock(job, RTTI_LOCK_WRITE);
|
||||
if (resync_job_is_terminal(job->job.state)) {
|
||||
/* (a) Terminal is always final -- Req 4.4. Never assign-eligible,
|
||||
* regardless of host. */
|
||||
unlock(job);
|
||||
} else if (job->job.allocs != NULL) {
|
||||
/* (b) Non-terminal AND already has SOME live non-terminal host
|
||||
* assignment -- a genuine no-op; do not reset start_time or
|
||||
* re-append a WAL_JOB_DISPATCH fact for a job that was never
|
||||
* actually lost/unassigned. */
|
||||
unlock(job);
|
||||
} else {
|
||||
/* (c) Non-terminal AND genuinely UNASSIGNED -- Req 4.8 fires. */
|
||||
int rc = record_job_reconcile_assign_fact(c, job, w, entry);
|
||||
unlock(job);
|
||||
if (rc == (int)JOB_TRANSITION_APPLIED)
|
||||
persist_notify_job_updated_already_persisted(c, job);
|
||||
}
|
||||
|
||||
dereference(job);
|
||||
}
|
||||
|
||||
/* Req 4.1-4.5, 4.7: routes a "finished" entry (success or failure) through
|
||||
* the SAME existing, deferred-pattern fact-recording used for an ordinary
|
||||
* worker finish/fail report. Applies identically whether entry->queried is
|
||||
* true or false (Req 4.7: an unqueried finished extra reconciles through
|
||||
* "the same idempotent commit-then-apply path... used for queried jobs" --
|
||||
* there is no separate "assign" case for a finished outcome; record_job_
|
||||
* reconcile_assign_fact() is documented as running-only, see job.c's own
|
||||
* doc comment on it).
|
||||
*
|
||||
* finishedOk: bug-1747 fix (Req 4.2) -- routed through the NEW record_job_
|
||||
* finish_fact_at() (job.c), not on_job_finish()/record_job_finish_fact(),
|
||||
* so the job's recorded finish_time is the WORKER-reported entry->
|
||||
* finish_time_ms (Req 3.5), not reconciliation-processing wall-clock time.
|
||||
* record_job_finish_fact_at() expects the caller to already hold job's
|
||||
* write lock (like record_job_reconcile_pending_fact/assign_fact above),
|
||||
* unlike on_job_finish() which locks internally -- this function performs
|
||||
* that lookup/lock/unlock/dereference sequence itself, mirroring on_job_
|
||||
* finish()'s own internal sequence exactly (job.c). A jobid not found in
|
||||
* the job table is a silent no-op, mirroring on_job_finish()'s own
|
||||
* "can not find job" behavior for an unknown jobid.
|
||||
*
|
||||
* finishedFail: still routed through the existing, UNMODIFIED on_job_fail()
|
||||
* /record_job_fail_fact() -- Requirement 4.2 remains unsatisfiable for this
|
||||
* outcome via any existing path in this codebase, not a reconciliation-
|
||||
* specific gap: WAL_JOB_FAIL's payload (persist_submit_job_fact_ex(...,
|
||||
* WAL_JOB_FAIL, jobid, 0, WAL_JOB_FAIL_STAGE_FLAG|stage)) carries no
|
||||
* finish-time slot at any layer, and apply_job_fail_transition() (job.c)
|
||||
* unconditionally sets job->job.finish_time = time(NULL) at APPLY time
|
||||
* (not even receipt time) for EVERY fail-reporting caller in this system,
|
||||
* confirmed by direct inspection -- a pre-existing, systemic gap that
|
||||
* predates this feature and is out of this task's boundary to fix (see
|
||||
* buglog.json bug-1747, re-scoped). The reconciled job's STATE (EXIT) is
|
||||
* always correct either way; only its recorded finish_time reflects
|
||||
* apply-processing time rather than the worker's true report. */
|
||||
static void resync_apply_finished_entry(cluster_t* c,
|
||||
xce_job_state_entry_t* entry)
|
||||
{
|
||||
if (entry->status == JobStateStatus_finishedOk) {
|
||||
job_t* job = resync_lookup_job(c, entry->jobid);
|
||||
|
||||
if (job != NULL) {
|
||||
xce_jobfinish_t finish
|
||||
= { .jobid = entry->jobid, .exit_code = 0 };
|
||||
|
||||
lock(job, RTTI_LOCK_WRITE);
|
||||
record_job_finish_fact_at(c, job, &finish,
|
||||
entry->finish_time_ms);
|
||||
unlock(job);
|
||||
|
||||
dereference(job);
|
||||
}
|
||||
} else {
|
||||
job_failinfo_t fail
|
||||
= { .jobid = entry->jobid,
|
||||
.stage = JOB_EXEC_STAGE_MAIN,
|
||||
.code = entry->exit_code,
|
||||
.detail = (char*)"worker-rejoin-resync reconciliation" };
|
||||
on_job_fail(c, &fail);
|
||||
}
|
||||
}
|
||||
|
||||
/* Req 4.1, 4.5, 4.6: routes an "unrecognized" entry through record_job_
|
||||
* reconcile_pending_fact() (task 2.1). Per Req 3.1/3.3, a worker only ever
|
||||
* reports "unrecognized" for a jobid it was explicitly asked about (an
|
||||
* unqueried extra is, by construction of Req 3.6's own "currently running
|
||||
* or recently finished" trigger, never reported unrecognized) -- so this
|
||||
* path is queried-only in practice; the caller (resync_apply_report below)
|
||||
* still clears the outstanding set uniformly for every queried entry
|
||||
* regardless of which branch handled it. */
|
||||
static void resync_apply_unrecognized_entry(cluster_t* c,
|
||||
xce_job_state_entry_t* entry)
|
||||
{
|
||||
job_t* job = resync_lookup_job(c, entry->jobid);
|
||||
int rc;
|
||||
|
||||
if (job == NULL)
|
||||
return;
|
||||
|
||||
lock(job, RTTI_LOCK_WRITE);
|
||||
rc = record_job_reconcile_pending_fact(c, job);
|
||||
unlock(job);
|
||||
|
||||
if (rc == (int)JOB_TRANSITION_APPLIED)
|
||||
persist_notify_job_updated_already_persisted(c, job);
|
||||
|
||||
dereference(job);
|
||||
}
|
||||
|
||||
int resync_apply_report(cluster_t* c, worker_t* w, uint64_t round_token,
|
||||
xce_job_state_report_t* report)
|
||||
{
|
||||
int i;
|
||||
|
||||
if (c == NULL || w == NULL || report == NULL)
|
||||
return -EINVAL;
|
||||
if (report->count > 0 && report->entries == NULL)
|
||||
return -EINVAL;
|
||||
|
||||
/* Req 6.1, 6.3: a report for a round already superseded/discarded is
|
||||
* dropped in its entirety, before anything else is touched -- checked
|
||||
* under w's own write lock (matching every other w->resync mutation
|
||||
* this function goes on to make, via resync_settle_entry, per entry
|
||||
* below). */
|
||||
lock(w, RTTI_LOCK_WRITE);
|
||||
if (!resync_round_is_current(w, round_token)) {
|
||||
unlock(w);
|
||||
return 0;
|
||||
}
|
||||
unlock(w);
|
||||
|
||||
/* Per-entry routing never holds w's write lock while also locking a
|
||||
* job's own write lock -- mirrors dispatch_a_job_'s established
|
||||
* discipline (job.c: every branch releases w's lock BEFORE acquiring a
|
||||
* job's) rather than this file's other three functions' "caller holds
|
||||
* w's lock for the whole call" convention, which fits their simple,
|
||||
* single-object, single-call-site mutations but not this function's
|
||||
* fan-out into per-job job.c calls, each requiring its own job-level
|
||||
* lock. w's lock is only (re)acquired, narrowly, inside
|
||||
* resync_settle_entry's own outstanding-set/maybe-resolve mutation. */
|
||||
for (i = 0; i < report->count; i++) {
|
||||
xce_job_state_entry_t* entry = report->entries[i];
|
||||
|
||||
if (entry == NULL)
|
||||
continue;
|
||||
|
||||
switch (entry->status) {
|
||||
case JobStateStatus_running:
|
||||
resync_apply_running_entry(c, w, entry);
|
||||
break;
|
||||
case JobStateStatus_finishedOk:
|
||||
case JobStateStatus_finishedFail:
|
||||
resync_apply_finished_entry(c, entry);
|
||||
break;
|
||||
case JobStateStatus_unrecognized:
|
||||
default:
|
||||
resync_apply_unrecognized_entry(c, entry);
|
||||
break;
|
||||
}
|
||||
|
||||
if (entry->queried)
|
||||
resync_settle_entry(c, w, round_token, entry->jobid);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
/* worker-rejoin-resync task 3.1 test seam (resync.h): builds a
|
||||
* resync_timeout_ctx_t exactly as resync_arm_round's own delay_task_ms call
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue