merge: sync Rongsong's scheduler-framework (88 commits, f01ae353..c5b172c7)
Two-sided divergence from f01ae353 (2026-08-30): theirs +88 (worker
rejoin-resync reconciliation, terminal-state bug-1737/1738/1739/1764,
scheduler-sim verbosity, hermetic offline build, go mod vendor of Go deps
in apiserver -- 3375 pure-adds), ours +89 (job-finish robustness bug-1509
B1/B3, PEND-reason, worker reconnect resend, apiserver job control).
Conflict resolutions (kept both sides' fixes):
- controller_connector.c checking_connection(): combine theirs
xcecd_resync_is_pending() rejoin deferral gate with ours
resend_job_status() actual-state re-report on rejoin success.
- job.c: keep both unknown-finish registry (ours, bug-1509) and
record_job_finish_fact_at (theirs, bug-1747); keep both
mark_jobs_unknown_core family (ours) and record_job_reconcile_*
(theirs). No double-notify: on_job_finish unchanged, theirs is
resync-path-only.
- scheduler handler.c/test_handlers.c/meson: theirs schedExitJob
exit_code branch (bug-1737) + ours jobDispatchFail success-ack no-op
(bug-1512) + configpush enq_pend_update (bug-1513) both present; both
test suites and REPL/sim targets wired.
- apiserver jobs.go: keep ours normalizeJobInt64/jobArrayIdx superset +
JobIDHuman; drop theirs strict IntOrInt64 early-reject (superseded --
ours tolerates float64/json.Number, no panic path).
- jobs_test.go: keep both regression suites (ours bug-1490, theirs
bug-1742).
- .kiro/specs scheduler-sim-*/worker-rejoin-resync + .wolf/*: keep our
deletions (established convention -- drop Rongsong bookkeeping on merge).
Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
lei.chen
commit
5e2985542f
3469 changed files with 1466599 additions and 382 deletions
|
|
@ -25,6 +25,7 @@
|
|||
|
||||
#include "jobj.h"
|
||||
#include "resobj.h"
|
||||
#include "resync.h" /* worker-rejoin-resync task 6.2: resync_handle_report_page */
|
||||
#include "utils.h"
|
||||
#include "worker.h"
|
||||
#include "xce/errnox.h"
|
||||
|
|
@ -237,6 +238,58 @@ static int handle_snapshot_transfer(cluster_t *c, xce_request_t *req)
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* handle_jobstatereport: dispatch handler for OPCode_jobStateReport
|
||||
* (worker-rejoin-resync task 6.2, Req 8.1, 8.3) -- the worker's answer to a
|
||||
* job-state reconciliation query the controller sent it (worker.c's
|
||||
* worker_send_resync_query(), task 6.1). Resolves the reporting worker from
|
||||
* the inbound connection via get_ep_worker() (misc.c) -- the same ep-
|
||||
* >worker_t resolution on_worker_join() already uses to SEND that query
|
||||
* (worker.c) -- then delegates wire-page reassembly + apply to resync_
|
||||
* handle_report_page() (resync.c, task 6.2), which reassembles a report
|
||||
* that arrives across multiple pages and hands the complete result to
|
||||
* resync_apply_report() (task 3.2) exactly once, never once per page.
|
||||
*
|
||||
* Fire-and-forget from the worker's side (xce_job_state_report_req(),
|
||||
* request.c -- the same fire-and-forget shape xce_job_state_query_req()
|
||||
* already uses to SEND the query) -- no reply is expected or sent back,
|
||||
* matching OPCode_walCatchupEntry/OPCode_snapshotTransfer's existing
|
||||
* no-reply convention immediately above for this class of one-way,
|
||||
* already-happened-fact daemon-to-daemon message.
|
||||
*
|
||||
* A page from a connection that does not resolve to a known worker
|
||||
* (get_ep_worker() returns NULL -- e.g. a stale/torn-down connection racing
|
||||
* teardown) is logged and dropped: there is no worker_t to reassemble or
|
||||
* apply this report against, and nothing to reply to either way. */
|
||||
static int handle_jobstatereport(cluster_t* c, endpoint_t* ep,
|
||||
xce_request_t* req)
|
||||
{
|
||||
xce_job_state_report_t* page = req->body.jobstatereportmsg;
|
||||
worker_t* w;
|
||||
int rc;
|
||||
|
||||
if (page == NULL)
|
||||
return 0;
|
||||
|
||||
w = get_ep_worker(c, ep);
|
||||
if (w == NULL) {
|
||||
WARN("handle_jobstatereport: endpoint %p does not resolve to a "
|
||||
"known worker -- dropping report page (pos=%d count=%d "
|
||||
"ntotal=%d)", (void*)ep, page->pos, page->count, page->n_total);
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
rc = resync_handle_report_page(c, w, page);
|
||||
if (rc < 0)
|
||||
ERROR("handle_jobstatereport: worker %s: failed to reassemble/apply "
|
||||
"report page (pos=%d count=%d ntotal=%d) rc=%d",
|
||||
(w->host.name != NULL) ? w->host.name : "?", page->pos,
|
||||
page->count, page->n_total, rc);
|
||||
|
||||
dereference(w);
|
||||
|
||||
return (rc < 0) ? rc : 0;
|
||||
}
|
||||
|
||||
/* ---- OPCode_schedAdmin: forward-and-block-for-reply support (task 10.1) --
|
||||
*
|
||||
* The controller is a stateless pass-through for OPCode_schedAdmin: it
|
||||
|
|
@ -522,6 +575,29 @@ static int myreq_handler(vtask_t* task)
|
|||
case OPCode_jobdispatch:
|
||||
rc = handle_jobdispatch(c, ep, &request);
|
||||
break;
|
||||
case OPCode_jobStateReport:
|
||||
/* worker-rejoin-resync task 6.2 (Req 8.1, 8.3): the controller only
|
||||
* ever RECEIVES OPCode_jobStateReport (worker -> controller) -- the
|
||||
* sibling opcode, OPCode_jobStateQuery, flows controller -> worker
|
||||
* only (sent by worker_send_resync_query(), worker.c) and is never
|
||||
* inbound to this switch; the worker's own myreq_handler
|
||||
* (controller_connector.c, task 4.3) is what dispatches THAT one.
|
||||
* No standby-redirect handling is added here: ha_dispatch_gate()
|
||||
* above already rejects this opcode on every non-primary role with
|
||||
* -XCE_ERR_STANDBY (it is deliberately absent from the CTRL_STANDBY
|
||||
* allow-list, ha.c) BEFORE this switch is ever reached, and the
|
||||
* generic post-gate rejection block a few lines up already
|
||||
* populates the redirect hint (bug-1700) for exactly that status --
|
||||
* so a standby-received report is redirected the same way every
|
||||
* other primary-only opcode already is, with zero opcode-specific
|
||||
* code needed here. See ha.c's CTRL_PRIMARY-without-quorum branch
|
||||
* for the ADJACENT, genuinely-needed fix this task found: admitting
|
||||
* this opcode as a fact-bearing op there (alongside OPCode_notify)
|
||||
* so a report is never rejected outright during a transient quorum
|
||||
* loss (bug-396's exact failure mode, which this whole feature
|
||||
* exists to correct). */
|
||||
rc = handle_jobstatereport(c, ep, &request);
|
||||
break;
|
||||
case OPCode_jobsub:
|
||||
rc = handle_jobsubmit(c, ep, &request);
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -4688,11 +4688,22 @@ int ha_dispatch_gate(ha_state_t *ha, int opcode)
|
|||
* Rejecting them loses the fact forever: the reporter never
|
||||
* retries (bug-396). Admit so the RETAIN path can append and
|
||||
* hold them until quorum returns (Req 3.2; umbrella 2.8 —
|
||||
* facts are retained, never dropped). */
|
||||
* facts are retained, never dropped).
|
||||
*
|
||||
* worker-rejoin-resync task 6.2: OPCode_jobStateReport is the
|
||||
* exact same shape — a worker's already-happened job-state
|
||||
* facts, sent fire-and-forget (xce_job_state_report_req(),
|
||||
* request.c) with no redelivery on rejection, exactly like
|
||||
* OPCode_notify. Rejecting it outright here during a transient
|
||||
* quorum-loss window would reproduce bug-396's own loss mode
|
||||
* for the specific facts this whole feature exists to recover
|
||||
* (bug-1741) — found live while auditing this gate for task
|
||||
* 6.2's own standby-redirect investigation, not assumed. */
|
||||
case OPCode_notify:
|
||||
case OPCode_update:
|
||||
case OPCode_setpreason:
|
||||
case OPCode_clrpreason:
|
||||
case OPCode_jobStateReport:
|
||||
return 0;
|
||||
default:
|
||||
return -XCE_ERR_NOT_READY;
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@
|
|||
#include "myconf.h"
|
||||
#include "mylock.h"
|
||||
#include "netio.h"
|
||||
#include "typemap.h" /* xce_job_state_entry_t -- worker-rejoin-resync task 2.2 */
|
||||
#include "vtaskmgr.h"
|
||||
#include "worker.h"
|
||||
#include "xce/cluster_def.h"
|
||||
|
|
@ -205,6 +206,85 @@ extern job_transition_result_t apply_job_update_transition(job_t*, int);
|
|||
* RTTI_LOCK_WRITE. */
|
||||
extern job_transition_result_t apply_job_postdone_transition(job_t*);
|
||||
|
||||
/* worker-rejoin-resync task 2.1 (Req 4.6, 5.4, 7.1-7.3): returns a job the
|
||||
* worker reports as unrecognized back to PEND, releasing any resource
|
||||
* allocation it was carrying (bug-1631) so a later redispatch never
|
||||
* inherits a stale allocation from the prior dispatch. Follows
|
||||
* dispatch_a_job_()'s own eager-mutation-plus-explicit-notify pattern, NOT
|
||||
* record_job_finish_fact's deferred one -- see design.md's "Controller:
|
||||
* job.c" Components section for the full rationale.
|
||||
*
|
||||
* Precondition: caller holds `job` under RTTI_LOCK_WRITE on entry and does
|
||||
* not release it until this call returns -- matching apply_job_update_
|
||||
* transition's own calling convention (e.g. mark_job_update's callers).
|
||||
* This function does not lock/unlock `job` itself.
|
||||
*
|
||||
* Returns the job_transition_result_t of the underlying apply_job_update_
|
||||
* transition(job, JOB_STAT_PEND) call, cast to int, so the caller can tell
|
||||
* JOB_TRANSITION_APPLIED apart from JOB_TRANSITION_REJECTED_BACKWARD/
|
||||
* JOB_TRANSITION_NOOP_DUPLICATE exactly as that guard's other callers
|
||||
* already do. Only on JOB_TRANSITION_APPLIED does this function release
|
||||
* job->job.allocs (if non-NULL) and durably append the fact -- a rejected
|
||||
* or no-op outcome touches neither (Req 4.3/4.4).
|
||||
*
|
||||
* Unlike record_job_finish_fact/record_job_fail_fact, the caller (not this
|
||||
* function) must call persist_notify_job_updated_already_persisted(c, job)
|
||||
* itself, AFTER releasing the write lock, exactly once, whenever this
|
||||
* returns JOB_TRANSITION_APPLIED -- omitting it reproduces bug-1679 at this
|
||||
* call site. */
|
||||
extern int record_job_reconcile_pending_fact(cluster_t*, job_t*);
|
||||
|
||||
/* worker-rejoin-resync task 2.2 (Req 4.1, 4.3, 4.4, 4.7, 4.8, 7.1-7.3):
|
||||
* records a job the worker reports running that the controller has no
|
||||
* existing non-terminal record of ("unqueried extra", Req 3.6) as assigned
|
||||
* to the reporting worker `w`, matching `report`'s reported status --
|
||||
* dispatch_a_job_()'s own eager-mutation-plus-explicit-notify pattern, NOT
|
||||
* record_job_finish_fact's deferred one -- see design.md's "Controller:
|
||||
* job.c" Components section, and job.c's own doc comment on this function
|
||||
* for the full rationale (including why job->job.allocs is deliberately
|
||||
* left NULL, and why there is no separate "host assignment" field for this
|
||||
* function to set beyond job->job.state).
|
||||
*
|
||||
* Precondition: caller holds `job` under RTTI_LOCK_WRITE on entry and does
|
||||
* not release it until this call returns -- this function does not
|
||||
* lock/unlock `job` itself. Unlike record_job_reconcile_pending_fact above,
|
||||
* this function performs no backward-transition guard of its own: the
|
||||
* caller must only invoke it when the controller has no existing
|
||||
* non-terminal record of this job (Req 4.8's own trigger condition), so
|
||||
* there is nothing more-final to clobber.
|
||||
*
|
||||
* Always returns (int)JOB_TRANSITION_APPLIED -- this outcome has no
|
||||
* reject/no-op case (unlike record_job_reconcile_pending_fact's
|
||||
* REJECTED_BACKWARD/NOOP_DUPLICATE). Matching that sibling's contract, the
|
||||
* caller (not this function) must call
|
||||
* persist_notify_job_updated_already_persisted(c, job) itself, AFTER
|
||||
* releasing the write lock, exactly once -- omitting it reproduces bug-1679
|
||||
* at this call site. */
|
||||
extern int record_job_reconcile_assign_fact(cluster_t*, job_t*, worker_t*,
|
||||
xce_job_state_entry_t*);
|
||||
|
||||
/* worker-rejoin-resync task 3.2 (bug-1747 fix, Req 4.2): record_job_finish_
|
||||
* fact()'s worker-reported-finish-time sibling, for the reconciliation
|
||||
* finished-success path only. Same body/guard/hook-fire/deferred-append
|
||||
* pattern as record_job_finish_fact() (job.c, unmodified) -- differs only
|
||||
* in passing `finish_time_ms` (converted ms -> s) to persist_submit_job_
|
||||
* finish_fact() instead of time(NULL). See job.c's own doc comment on this
|
||||
* function for the full rationale, including the persist_submit_job_
|
||||
* finish_fact() unit confirmation.
|
||||
*
|
||||
* Precondition: caller holds `job` under RTTI_LOCK_WRITE on entry and keeps
|
||||
* holding it until this call returns -- this function does not lock/unlock
|
||||
* `job` itself, matching record_job_finish_fact's own convention.
|
||||
*
|
||||
* DEFERRED pattern (unlike record_job_reconcile_pending_fact/assign_fact
|
||||
* above): this function only durably appends the fact; it does NOT mutate
|
||||
* job's in-memory state and the caller must NOT call persist_notify_job_
|
||||
* updated_already_persisted() afterward -- exactly like record_job_finish_
|
||||
* fact(), the mutation and notify happen later, via apply_committed_entry's
|
||||
* replay of the committed WAL_JOB_FINISH entry. */
|
||||
extern int record_job_finish_fact_at(cluster_t*, job_t*, xce_jobfinish_t*,
|
||||
uint64_t);
|
||||
|
||||
extern int on_job_start(cluster_t*, job_pidinfo_t*);
|
||||
extern int on_job_finish(cluster_t*, xce_jobfinish_t*);
|
||||
extern int on_job_fail(cluster_t*, job_failinfo_t*);
|
||||
|
|
|
|||
283
src/controller/include/resync.h
Normal file
283
src/controller/include/resync.h
Normal file
|
|
@ -0,0 +1,283 @@
|
|||
#if !defined(_RESYNC_H_)
|
||||
#define _RESYNC_H_ 1
|
||||
|
||||
#include "config.h"
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "typemap.h" /* xce_job_state_entry_t/xce_job_state_report_t --
|
||||
* worker-rejoin-resync task 3.2 */
|
||||
#include "xce/cluster_def.h"
|
||||
#include "xce/types.h"
|
||||
|
||||
/* worker-rejoin-resync task 3.1 (Req 1.1-1.3, 2.1-2.2, 5.3, 6.1, 6.3):
|
||||
* per-worker job-state reconciliation round lifecycle -- arm on trigger,
|
||||
* track queried job IDs + a round token, and resolve on either the
|
||||
* Requirement 5.3 delaymgr timeout or (later, task 3.2) normal report
|
||||
* completion. `resync_state_t` itself lives in xce/types.h (task 1.2),
|
||||
* attached via `worker_t.resync`; this file owns building/populating it
|
||||
* and its lifecycle, not its shape. See design.md "Controller: resync.c".
|
||||
*
|
||||
* Every function below follows the SAME locking convention job.c's
|
||||
* record_job_reconcile_pending_fact/record_job_reconcile_assign_fact
|
||||
* already established (design.md Components): the caller holds the
|
||||
* relevant per-worker write lock (matching rejoin_worker's/apply_join_
|
||||
* state's own convention, worker.c) BEFORE calling and keeps holding it
|
||||
* until the call returns -- these functions never lock/unlock `w`
|
||||
* themselves.
|
||||
*/
|
||||
|
||||
/* Arms a fresh reconciliation round for `w`: enumerates every non-terminal
|
||||
* job in `c`'s job table currently assigned to w's host (head/primary host,
|
||||
* matching job.c's own existing "job's assigned host" representation --
|
||||
* job->job.allocs->allocs[0]->host, e.g. job.c's exechost lookup and
|
||||
* job-control routing), builds the round's query-job-ID set, assigns a
|
||||
* fresh monotonic round token (Req 6.1/6.3), sets deadline = now +
|
||||
* ClusterConfig.workerResyncTimeoutMs (Req 5.3), attaches the resulting
|
||||
* resync_state_t to w->resync, sets XCE_WORKER_RECONCILING on
|
||||
* w->host.status, and schedules a non-blocking delaymgr one-shot timeout
|
||||
* that resolves the round if nothing else does first.
|
||||
*
|
||||
* A worker with zero matching non-terminal jobs is a documented no-op (Req
|
||||
* 1.2): w->resync stays NULL/untouched, XCE_WORKER_RECONCILING is not set,
|
||||
* and no timeout is scheduled.
|
||||
*
|
||||
* If `w` already carries an outstanding round on entry (w->resync != NULL
|
||||
* -- e.g. this rejoin interrupts one that never resolved), that prior round
|
||||
* is discarded first (Req 6.1, via resync_resolve_round) before the fresh
|
||||
* one is armed, so the new round's token always supersedes the old one and
|
||||
* a later report carrying the old token is provably stale (Req 6.3).
|
||||
*
|
||||
* Precondition: caller already holds `w`'s write lock.
|
||||
*
|
||||
* Returns the number of jobs armed into the round (>=0; 0 for the
|
||||
* documented Req 1.2 no-op case) on success, or a negative errno (no round
|
||||
* is armed/left partially armed on failure). */
|
||||
extern int resync_arm_round(cluster_t* c, worker_t* w);
|
||||
|
||||
/* Resolves (clears) whatever reconciliation round is currently outstanding
|
||||
* for `w`, if any: clears XCE_WORKER_RECONCILING on w->host.status, frees
|
||||
* and detaches w->resync. Called both by the Req 5.3 timeout firing
|
||||
* (internally, via resync_arm_round's own delaymgr callback) and, later
|
||||
* (task 3.2), once every queried job in the round has a reconciled
|
||||
* outcome. Idempotent: a no-op when w->resync is already NULL -- safe to
|
||||
* call speculatively.
|
||||
*
|
||||
* Precondition: caller holds `w`'s write lock. */
|
||||
extern void resync_resolve_round(cluster_t* c, worker_t* w);
|
||||
|
||||
/* Req 6.3: true (nonzero) iff `token` is still the round token CURRENTLY
|
||||
* outstanding for `w` -- i.e. w->resync != NULL && w->resync->round_token
|
||||
* == token. False for a stale/superseded round (a newer resync_arm_round
|
||||
* call replaced it) or an already-resolved one (w->resync == NULL). This is
|
||||
* the discard-on-stale-report mechanism Req 6.1/6.3 require: a later
|
||||
* report-apply caller (task 3.2) checks a report's carried round token
|
||||
* against this before applying anything from it.
|
||||
*
|
||||
* Precondition: caller holds at least `w`'s read lock. */
|
||||
extern int resync_round_is_current(worker_t* w, uint64_t token);
|
||||
|
||||
/* worker-rejoin-resync task 3.2 (Req 4.1-4.5, 6.1-6.3): applies a worker's
|
||||
* already-decoded job-state report to `c`'s job table -- the glue that maps
|
||||
* each `xce_job_state_entry_t` outcome in `report` to the correct existing
|
||||
* (record_job_fail_fact via on_job_fail, job.c) or new (record_job_finish_
|
||||
* fact_at, task 3.2/bug-1747; record_job_reconcile_pending_fact/record_job_
|
||||
* reconcile_assign_fact, task 2.1/2.2, job.c) idempotent fact-recording
|
||||
* call, per design.md's "System Flows" sequence diagram (`loop each
|
||||
* reported entry`).
|
||||
*
|
||||
* Routing per `xce_job_state_entry_t.status`:
|
||||
* - JobStateStatus_running, entry->queried true: cleared from the round's
|
||||
* outstanding set; no fact recorded.
|
||||
* - JobStateStatus_running, entry->queried false (Req 3.6 extra): a
|
||||
* genuine THREE-WAY split (review round 2 fix -- a two-way boolean
|
||||
* guard is WRONG and introduces a Req 4.4 violation, see resync.c's
|
||||
* own doc comment on resync_apply_running_entry for the full incident):
|
||||
* (a) TERMINAL -> ALWAYS a no-op, regardless of host (Req 4.4 -- a
|
||||
* terminal job is never assign-eligible; record_job_reconcile_assign_
|
||||
* fact() has no backward-transition guard of its own and would
|
||||
* unconditionally clobber DONE/EXIT back to RUN if called here).
|
||||
* (b) non-terminal AND already has SOME live non-terminal host
|
||||
* assignment (`job->job.allocs != NULL` -- to THIS reporting worker's
|
||||
* own host, or to a DIFFERENT one, it does not matter which) -> a
|
||||
* no-op (already validly assigned; calling assign here would still
|
||||
* wrongly reset start_time and re-append a WAL_JOB_DISPATCH fact for a
|
||||
* job that was never actually lost).
|
||||
* (c) non-terminal AND genuinely UNASSIGNED (`allocs == NULL`) -> Req
|
||||
* 4.8's real trigger ("no existing non-terminal record... assigned
|
||||
* elsewhere" -- there IS a non-terminal record, but it carries no
|
||||
* assignment at all, e.g. still PEND because its WAL_JOB_DISPATCH fact
|
||||
* never reached durable majority commit before a controller failover,
|
||||
* even though the worker really is running it) -> reconciles via
|
||||
* record_job_reconcile_assign_fact() (Req 4.8), confirmed against that
|
||||
* function's own doc comment/assert (job.c), which documents itself as
|
||||
* the running-only case. A jobid with NO record at all in the job
|
||||
* table cannot be reconciled this way (that function mutates an
|
||||
* existing job_t, it does not construct one) and is logged and skipped
|
||||
* -- out of this feature's scope; every job a worker can legitimately
|
||||
* run was submitted through this controller at some point, so this is
|
||||
* expected to be unreachable in practice.
|
||||
* - JobStateStatus_finishedOk: routed through record_job_finish_fact_at()
|
||||
* (task 3.2/bug-1747 fix, job.c) -- record_job_finish_fact()'s own
|
||||
* worker-reported-finish-time sibling, so the recorded finish_time is
|
||||
* entry->finish_time_ms (Req 3.5), not reconciliation-processing time
|
||||
* (Req 4.2). record_job_finish_fact() itself is unmodified; its own
|
||||
* caller (on_job_finish, an ordinary worker report) still uses
|
||||
* receipt-time, correctly, for that near-instantaneous case.
|
||||
* - JobStateStatus_finishedFail: routed through on_job_fail() (job.c),
|
||||
* unchanged -- Requirement 4.2 remains unsatisfiable for this outcome
|
||||
* via any existing path in this codebase (WAL_JOB_FAIL carries no
|
||||
* finish-time slot at any layer, for any caller -- a pre-existing,
|
||||
* systemic gap that predates this feature; see buglog.json bug-1747).
|
||||
* - Both finished outcomes apply identically whether entry->queried is
|
||||
* true or false (Req 4.7: an unqueried finished extra reconciles
|
||||
* through the same path as a queried one; there is no "assign" case for
|
||||
* a finished outcome). Cleared from the outstanding set when
|
||||
* entry->queried is true.
|
||||
* - JobStateStatus_unrecognized: routed through record_job_reconcile_
|
||||
* pending_fact() (Req 4.6, task 2.1); cleared from the outstanding set.
|
||||
*
|
||||
* Req 6.1/6.3: if `round_token` is not `w`'s CURRENTLY outstanding round
|
||||
* (resync_round_is_current), the entire report is discarded before
|
||||
* anything else is touched -- no fact is applied, no outstanding-set
|
||||
* mutation occurs. This also makes a duplicate re-application of an
|
||||
* already-fully-resolved report (Req 6.2) a pure no-op by the same
|
||||
* mechanism: once every queried job has a reconciled outcome,
|
||||
* resync_resolve_round() clears w->resync, so `round_token` is no longer
|
||||
* current on a later identical call -- no facts are re-applied, no
|
||||
* additional WAL entries appended (a duplicate applied WHILE the round is
|
||||
* still open instead re-applies each fact through job.c's own idempotent
|
||||
* guards, which are themselves safe no-ops on a repeat).
|
||||
*
|
||||
* Req 5.1/5.2: once the round's outstanding set becomes empty (every
|
||||
* queried jobid has been cleared), the round is resolved
|
||||
* (resync_resolve_round()), clearing XCE_WORKER_RECONCILING so `w` becomes
|
||||
* eligible for dispatch again.
|
||||
*
|
||||
* Locking: unlike resync_arm_round/resync_resolve_round/resync_round_is_
|
||||
* current above, this function manages its OWN `w` write-locking
|
||||
* internally and narrowly (never nested with a job's own write lock) --
|
||||
* mirroring dispatch_a_job_'s established discipline (job.c: every branch
|
||||
* releases `w`'s lock BEFORE acquiring a job's) rather than this file's
|
||||
* other three functions' "caller holds w's lock for the whole call"
|
||||
* convention, which fits their simple, single-object, single-call-site
|
||||
* mutations but not this function's fan-out into per-job job.c calls, each
|
||||
* requiring its own job-level lock. The caller must NOT pre-hold `w`'s
|
||||
* lock when calling this function.
|
||||
*
|
||||
* `report` is treated as a single, already-fully-assembled list of
|
||||
* `report->count` entries (`report->entries[0..count)`) -- multi-page wire
|
||||
* reassembly is a later task's concern (6.2), out of this function's scope.
|
||||
*
|
||||
* Returns 0 on success (whether the report was applied or discarded as
|
||||
* stale), or a negative errno for invalid arguments (NULL c/w/report). */
|
||||
extern int resync_apply_report(cluster_t* c, worker_t* w, uint64_t round_token,
|
||||
xce_job_state_report_t* report);
|
||||
|
||||
/* worker-rejoin-resync task 6.2 (Req 8.1, 8.3): the real type behind
|
||||
* worker_t.report_accum (xce/types.h -- stored there as an opaque void* to
|
||||
* avoid a types.h/typemap.h include cycle, since xce_job_state_entry_t is a
|
||||
* typemap.h type). Reassembles an OPCode_jobStateReport that arrives split
|
||||
* across multiple wire pages, before resync_apply_report() ever sees it --
|
||||
* mirrors xcecd_resync.c's worker-side query_accum/query_accum_ntotal/
|
||||
* query_accum_received statics (task 4.3), moved from file-static (a
|
||||
* worker daemon has only one controller connection) to a per-worker_t
|
||||
* field (a controller serves many workers concurrently, so the
|
||||
* accumulator must be keyed per reporting worker). Lives separately from
|
||||
* resync_state_t (w->resync) rather than as a field on it: resync_apply_
|
||||
* report() may resolve (free) w->resync partway through settling a fully-
|
||||
* assembled report's entries, and this accumulator's own memory must not
|
||||
* be tied to that lifetime -- resync_handle_report_page() below always
|
||||
* detaches (NULLs w->report_accum) and takes sole ownership of this struct
|
||||
* BEFORE calling resync_apply_report(), so there is nothing for that call
|
||||
* to free out from under it either way. */
|
||||
typedef struct {
|
||||
/* Captured from w->resync->round_token the instant this report's first
|
||||
* page (pos == 0) arrives (0, the reserved "no token" sentinel, if no
|
||||
* round is outstanding yet) -- NOT re-read at apply time. This is what
|
||||
* lets a report that started under one round but finishes assembling
|
||||
* after that round was superseded/discarded (Req 6.1, 6.3) still
|
||||
* discard correctly via resync_apply_report()'s own resync_round_is_
|
||||
* current() check, rather than being wrongly matched against whatever
|
||||
* round happens to be current once the LAST page arrives. */
|
||||
uint64_t round_token;
|
||||
int n_total;
|
||||
int n_received;
|
||||
/* Deep copies (plain struct-value copies -- xce_job_state_entry_t has
|
||||
* no nested heap pointers), one malloc'd xce_job_state_entry_t per
|
||||
* entry, indexed positionally by wire pos, mirroring xce_job_state_
|
||||
* report_t.entries' own array-of-pointers shape (typemap.h) so the
|
||||
* assembled report can be hand off to resync_apply_report() directly,
|
||||
* with no reshaping. A deep copy is required, not merely a borrowed
|
||||
* pointer into the incoming page: each page's xce_job_state_entry_t*
|
||||
* values are owned by that page's own xce_request_t, freed by xce_
|
||||
* free_request() the moment the dispatch handler that decoded it
|
||||
* returns -- long before a later page (let alone the final one) has
|
||||
* even arrived. */
|
||||
xce_job_state_entry_t** entries;
|
||||
} resync_report_accum_t;
|
||||
|
||||
/* Handles one incoming OPCode_jobStateReport wire page for worker `w`:
|
||||
* accumulates it into w->report_accum (allocating/resetting on pos == 0,
|
||||
* exactly like xcecd_resync_handle_query_page()'s query_accum reset-on-
|
||||
* pos-0 convention -- a stale/incomplete accumulation left over from an
|
||||
* aborted prior report, e.g. this worker disconnected mid-report on a
|
||||
* prior round, is always discarded by the next report's own first page
|
||||
* rather than corrupting it). Once every entry has arrived (n_received ==
|
||||
* n_total), hands the fully-assembled report to resync_apply_report()
|
||||
* EXACTLY ONCE -- never once per page -- using the round_token captured at
|
||||
* this report's first page, then frees the accumulator itself.
|
||||
*
|
||||
* Precondition: caller does NOT hold w's lock -- this function acquires it
|
||||
* itself, narrowly, for each page's accumulator mutation, and always
|
||||
* releases it again BEFORE calling resync_apply_report() (which manages
|
||||
* its own w locking internally -- resync.h's own doc comment on that
|
||||
* function). Mirrors resync_apply_report's own "never nested" locking
|
||||
* discipline.
|
||||
*
|
||||
* Returns 0 if this page was accepted but the report is not yet complete
|
||||
* (more pages still expected), 1 if this page completed the report and it
|
||||
* was handed to resync_apply_report() (regardless of whether resync_apply_
|
||||
* report applied it or discarded it as stale -- see above), or a negative
|
||||
* errno for a malformed page (NULL c/w/page, or a page whose pos/count
|
||||
* fall outside [0, n_total) -- the in-progress accumulation, if any, is
|
||||
* discarded in that case too, so a later page cannot be mismatched against
|
||||
* a corrupt partial buffer). */
|
||||
extern int resync_handle_report_page(cluster_t* c, worker_t* w,
|
||||
const xce_job_state_report_t* page);
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
/* worker-rejoin-resync task 6.2 test seam: counts how many times resync_
|
||||
* handle_report_page() has actually invoked resync_apply_report() (process-
|
||||
* wide, across every worker) -- a direct, structural proof that a report
|
||||
* spanning N wire pages results in exactly ONE resync_apply_report() call,
|
||||
* never N, distinct from (and stronger than) merely asserting the resulting
|
||||
* job-table end state looks correct (job.c's own idempotent fact guards
|
||||
* would make even a buggy once-per-page implementation LOOK correct on a
|
||||
* happy-path report with no interleaved round changes -- see this function's
|
||||
* own doc comment above and testing.md's "structural, not just behavioral"
|
||||
* guidance for placement/count-sensitive logic). */
|
||||
extern int resync_apply_report_call_count_for_test(void);
|
||||
extern void resync_apply_report_call_count_reset_for_test(void);
|
||||
#endif
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
/* worker-rejoin-resync task 3.1 test seam: directly invokes the SAME
|
||||
* internal timeout-fire logic resync_arm_round's own delaymgr callback runs
|
||||
* (resync_on_timeout, resync.c -- file-static, not otherwise reachable from
|
||||
* a test) for the given `token`, without needing a real delaymgr timer
|
||||
* thread or any real wall-clock wait -- treats "the timer fired" as a
|
||||
* directly-testable event, exactly like any other function call. Exercises
|
||||
* the round-token staleness check exactly as a genuine late fire would: a
|
||||
* no-op if `token` is no longer `w`'s current round. Unlike resync_arm_
|
||||
* round/resync_resolve_round above, this ACQUIRES `w`'s own write lock
|
||||
* itself (mirroring the real delaymgr callback, which runs with no lock
|
||||
* already held by any caller) -- so `w` must be a real RTTI-managed
|
||||
* worker_t (e.g. via makeWorker), not a bare stack struct, or the lock call
|
||||
* is undefined behavior. Compiled out of production (mirrors cluster.h's
|
||||
* job_array_apply_test_* seams). */
|
||||
extern void resync_test_fire_timeout(cluster_t* c, worker_t* w,
|
||||
uint64_t token);
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
|
@ -13,6 +13,7 @@
|
|||
#include "persist.h"
|
||||
#include "preason.h"
|
||||
#include "request.h"
|
||||
#include "typemap.h" /* xce_job_state_entry_t -- worker-rejoin-resync task 2.2 */
|
||||
#include "xce/cluster_def.h"
|
||||
#include "xce/errnox.h"
|
||||
#include "xce/types.h"
|
||||
|
|
@ -1509,7 +1510,66 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
}
|
||||
|
||||
lock(w, RTTI_LOCK_READ);
|
||||
if ((w->ep == NULL) || (w->host.status != 0)) {
|
||||
if (w->ep == NULL) {
|
||||
/* worker-rejoin-resync task 2.3 (Req 5.1, 5.4): a dispatch attempt
|
||||
* to a worker the controller is not currently connected to (w->ep
|
||||
* == NULL) is resolved here, BEFORE the w->host.status != 0 check
|
||||
* below ever reaches xce_classify_dispatch_fail() at all --
|
||||
* deliberately a bypass of that retry-then-terminal-fail machinery,
|
||||
* not a third value added to xce_dispatch_fail_class_t (design.md's
|
||||
* "Controller: job.c" Components section: that enum's whole
|
||||
* contract is a two-way TERMINAL/RETRYABLE choice every existing
|
||||
* call site's switch already exhaustively handles). Retrying the
|
||||
* dispatch itself doesn't fit "the worker isn't even connected,"
|
||||
* and this case must stay recoverable-via-reschedule rather than
|
||||
* ever resolve to mark_job_fail()'s terminal outcome -- so this
|
||||
* branch calls neither xce_classify_dispatch_fail() nor
|
||||
* mark_job_fail(), and never touches j->dispatch_fail_count. */
|
||||
int dfrc;
|
||||
int result;
|
||||
|
||||
ERROR("worker %s is not connected (ep NULL), status %x",
|
||||
w->host.name, w->host.status);
|
||||
free_launchinfo(&launchinfo);
|
||||
|
||||
SET_ERROR(E_XCE_BAD_WORKER);
|
||||
/* Capture errno immediately after SET_ERROR(), before any further
|
||||
* call -- errno is not guaranteed stable across unlock(w)/
|
||||
* dereference(w)/lock(j)/etc., mirroring the WORKER_NOT_READY
|
||||
* branch's own discipline below. */
|
||||
dfrc = -errno;
|
||||
|
||||
unlock(w);
|
||||
|
||||
dereference(w);
|
||||
|
||||
/* Req 5.4: resolves via record_job_reconcile_pending_fact() (task
|
||||
* 2.1) directly -- eager mutation + allocs release (bug-1631) under
|
||||
* this write lock, durable append inside that call, then (only on
|
||||
* an actual APPLIED transition) this caller's own post-unlock
|
||||
* persist_notify_job_updated_already_persisted() call, exactly as
|
||||
* that function's own contract requires (cluster.h doc comment).
|
||||
* Closes the same bug-1627 reference-leak class as the WORKER_NOT_
|
||||
* READY branch below -- j's reference (held since the jobmgr lookup
|
||||
* at the top of this function) is released via dereference(j) on
|
||||
* every exit from this branch. */
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
|
||||
result = record_job_reconcile_pending_fact(c, j);
|
||||
|
||||
unlock(j);
|
||||
|
||||
if (result == (int)JOB_TRANSITION_APPLIED) {
|
||||
persist_notify_job_updated_already_persisted(c, j);
|
||||
}
|
||||
/* JOB_TRANSITION_REJECTED_BACKWARD/NOOP_DUPLICATE: nothing changed
|
||||
* (job already terminal, or already PEND) -- no notify needed,
|
||||
* matching record_job_reconcile_pending_fact's own contract. */
|
||||
|
||||
dereference(j);
|
||||
|
||||
return dfrc;
|
||||
} else if (w->host.status != 0) {
|
||||
int dfrc;
|
||||
xce_dispatch_fail_class_t dfclass;
|
||||
job_failinfo_t failinfo = { .jobid = alloc->jobid };
|
||||
|
|
@ -2803,6 +2863,64 @@ int test_unknown_finish_take(uint64_t jobid, xce_jobfinish_t* out)
|
|||
}
|
||||
#endif
|
||||
|
||||
/* worker-rejoin-resync task 3.2 (bug-1747 fix, Req 4.2): WAL_JOB_FINISH's
|
||||
* own worker-reported-finish-time sibling of record_job_finish_fact() above
|
||||
* -- identical guard/hook-fire/durable-append body, except it passes a
|
||||
* CALLER-supplied finish time (converted from the worker's reported
|
||||
* finish_time_ms, Req 3.5) to persist_submit_job_finish_fact() instead of
|
||||
* always deriving time(NULL) at receipt time. record_job_finish_fact()
|
||||
* itself is deliberately left UNMODIFIED: its own callers (on_job_finish's
|
||||
* ordinary, near-instantaneous fire-and-forget worker report) are correctly
|
||||
* served by receipt-time ~= true finish-time; only a RECONCILED finish
|
||||
* (received long after the fact, on rejoin) needs the worker's own reported
|
||||
* time instead -- mirroring task 2.1/2.2's own "new function alongside an
|
||||
* unmodified existing one" pattern (record_job_reconcile_pending_fact/
|
||||
* record_job_reconcile_assign_fact), not a new precedent.
|
||||
*
|
||||
* persist_submit_job_finish_fact()'s own `finish_time` parameter is SECONDS
|
||||
* (time_t semantics) -- confirmed by its only other call site
|
||||
* (record_job_finish_fact() above: `time_t finish_time = time(NULL);
|
||||
* ...(uint64_t)finish_time`) and by the WAL_JOB_FINISH decode side
|
||||
* (apply_committed_entry, persist.c: `j->job.finish_time =
|
||||
* (time_t)wire_finish_time;`, a direct, unconverted assignment). The
|
||||
* worker-reported wire field (xce_job_state_entry_t.finish_time_ms,
|
||||
* typemap.h) is MILLISECONDS ("finishTimeMs" on the wire, Req 3.5) -- this
|
||||
* function converts ms -> s (`finish_time_ms / 1000`) before calling
|
||||
* persist_submit_job_finish_fact(), so the caller (resync.c) can pass the
|
||||
* raw worker-reported value straight through without needing to know this
|
||||
* unit detail itself.
|
||||
*
|
||||
* Same locking convention as record_job_finish_fact(): caller holds `job`
|
||||
* under RTTI_LOCK_WRITE on entry and keeps holding it until this call
|
||||
* returns; this function does not lock/unlock `job` itself. Same DEFERRED
|
||||
* pattern too: durable append only here, no mutation -- the in-memory state
|
||||
* change happens later via apply_committed_entry()'s replay, exactly as an
|
||||
* ordinary finish already works (unlike record_job_reconcile_pending_fact/
|
||||
* assign_fact's own EAGER pattern) -- so the caller must NOT call
|
||||
* persist_notify_job_updated_already_persisted() after this returns; that
|
||||
* notify happens later, at apply time, exactly as it already does for
|
||||
* record_job_finish_fact(). */
|
||||
int record_job_finish_fact_at(cluster_t* c, job_t* job,
|
||||
xce_jobfinish_t* finish, uint64_t finish_time_ms)
|
||||
{
|
||||
if (job_state_is_terminal(job->job.state)) {
|
||||
apply_job_finish_transition(job, finish); /* logs + no-op, unchanged */
|
||||
return 0;
|
||||
}
|
||||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_JOB_EXIT, c, finish, job);
|
||||
|
||||
(void)persist_submit_job_finish_fact(&c->persist, job->job.jobid,
|
||||
(finish->exit_code == 0)
|
||||
? JOB_STAT_DONE
|
||||
: JOB_STAT_EXIT,
|
||||
finish_time_ms / 1000,
|
||||
finish->exit_code);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int on_job_finish(cluster_t* c, xce_jobfinish_t* finish)
|
||||
{
|
||||
job_t* job = NULL;
|
||||
|
|
@ -3157,6 +3275,188 @@ void mark_worker_jobs_exit(cluster_t* c, const char* hostname)
|
|||
}
|
||||
iter_destroy(&it);
|
||||
unlock(c->jobmgr);
|
||||
|
||||
/* worker-rejoin-resync task 2.1 (Req 4.1-4.6, 5.4, 7.1-7.3): returns a job
|
||||
* the worker reports as unrecognized back to PEND, releasing any resource
|
||||
* allocation it was carrying so a subsequent redispatch attempt never
|
||||
* inherits a stale allocation from a prior dispatch (bug-1631, confirmed
|
||||
* live and deliberately left unfixed until this feature provided the first
|
||||
* real re-admission path -- see design.md's Error Handling section).
|
||||
*
|
||||
* Follows dispatch_a_job_()'s own established pattern (job.c, its `if (rc
|
||||
* >= 0) { ... }` success block): eager, synchronous mutation under the
|
||||
* write lock the caller already holds, immediately followed by a durable
|
||||
* WAL append, immediately followed (by the CALLER, after unlocking) by an
|
||||
* explicit persist_notify_job_updated_already_persisted() call -- NOT
|
||||
* record_job_finish_fact's/record_job_fail_fact's purely-deferred pattern
|
||||
* (mutation withheld until apply_committed_entry replays the committed
|
||||
* fact). The fact recorded here is something the worker already confirmed
|
||||
* (or, in this case, failed to confirm) really happened -- ground truth
|
||||
* already changed outside this controller -- which is exactly dispatch_a_
|
||||
* job_'s own justification for going eager rather than deferred (design.md,
|
||||
* "Controller: job.c" Components section).
|
||||
*
|
||||
* Precondition: caller holds `job` under RTTI_LOCK_WRITE on entry and keeps
|
||||
* holding it until this call returns -- this function does not lock/unlock
|
||||
* `job` itself, matching apply_job_update_transition's own calling
|
||||
* convention (its other caller, mark_job_update above, is called under the
|
||||
* exact same already-held write lock).
|
||||
*
|
||||
* apply_job_update_transition()'s existing job_state_is_backward guard is
|
||||
* the ONLY commit/idempotency guard exercised here -- reused unchanged,
|
||||
* never duplicated: a terminal job's transition is rejected
|
||||
* (JOB_TRANSITION_REJECTED_BACKWARD, Req 4.4) and an already-PEND job's
|
||||
* transition is a no-op (JOB_TRANSITION_NOOP_DUPLICATE, Req 4.3) -- neither
|
||||
* outcome touches job->job.allocs or appends a WAL fact. Only on
|
||||
* JOB_TRANSITION_APPLIED does this release job->job.allocs (if non-NULL)
|
||||
* and durably append the fact via the existing generic persist_submit_job_
|
||||
* fact() -- the plain {jobid, state} shape is sufficient here, same as
|
||||
* every other WAL_JOB_UPDATE caller (no extra fields, unlike DISPATCH/
|
||||
* START/FINISH's richer siblings).
|
||||
*
|
||||
* Returns the job_transition_result_t of the underlying transition, cast to
|
||||
* int, so the caller can tell APPLIED apart from REJECTED_BACKWARD/
|
||||
* NOOP_DUPLICATE. Unlike record_job_finish_fact/record_job_fail_fact, this
|
||||
* function does NOT call persist_notify_job_updated_already_persisted --
|
||||
* the caller must do so itself, after releasing the write lock, exactly
|
||||
* once, for every JOB_TRANSITION_APPLIED result (cluster.h's doc comment on
|
||||
* the prototype carries this same contract). Omitting it reproduces
|
||||
* bug-1679 at this call site. */
|
||||
int record_job_reconcile_pending_fact(cluster_t* c, job_t* job)
|
||||
{
|
||||
job_transition_result_t result;
|
||||
|
||||
result = apply_job_update_transition(job, JOB_STAT_PEND);
|
||||
if (result != JOB_TRANSITION_APPLIED)
|
||||
return (int)result;
|
||||
|
||||
if (job->job.allocs != NULL) {
|
||||
free_JobAlloc_ptr(&(job->job.allocs));
|
||||
job->job.allocs = NULL;
|
||||
}
|
||||
|
||||
(void)persist_submit_job_fact(&c->persist, WAL_JOB_UPDATE, job->job.jobid,
|
||||
JOB_STAT_PEND);
|
||||
|
||||
return (int)result;
|
||||
}
|
||||
|
||||
/* worker-rejoin-resync task 2.2 (Req 4.1, 4.3, 4.4, 4.7, 4.8, 7.1-7.3):
|
||||
* records a job the worker reports as running that the controller has no
|
||||
* existing non-terminal record of -- an "unqueried extra" (Req 3.6) the
|
||||
* controller never had a chance to ask about -- as assigned to the
|
||||
* reporting worker, through the same idempotent commit-then-apply path an
|
||||
* ordinary dispatch uses.
|
||||
*
|
||||
* Follows dispatch_a_job_()'s own established pattern (job.c, its `if (rc
|
||||
* >= 0) { ... }` success block, job.c:1686-1748) exactly as record_job_
|
||||
* reconcile_pending_fact above does: eager, synchronous mutation under the
|
||||
* write lock the caller already holds, immediately followed by a durable
|
||||
* WAL append, immediately followed (by the CALLER, after unlocking) by an
|
||||
* explicit persist_notify_job_updated_already_persisted() call -- NOT
|
||||
* record_job_finish_fact's/record_job_fail_fact's purely-deferred pattern.
|
||||
* The fact recorded here is something the worker already confirmed really
|
||||
* happened (it is running there right now) -- ground truth already changed
|
||||
* outside this controller -- exactly dispatch_a_job_'s own justification
|
||||
* for going eager rather than deferred (design.md, "Controller: job.c"
|
||||
* Components section).
|
||||
*
|
||||
* Precondition: caller holds `job` under RTTI_LOCK_WRITE on entry and keeps
|
||||
* holding it until this call returns -- this function does not lock/unlock
|
||||
* `job` itself, matching record_job_reconcile_pending_fact's own calling
|
||||
* convention above. Unlike that sibling, this function performs no
|
||||
* backward-transition guard check of its own: design.md's Service Interface
|
||||
* Invariants note that both new functions are "never called for a job the
|
||||
* controller already knows is terminal or more-final" as a PRECONDITION the
|
||||
* CALLER enforces, not something either function re-checks with its own
|
||||
* guard -- and for this specific outcome the precondition holds by
|
||||
* construction: Req 4.8's own trigger condition is precisely "the
|
||||
* Controller has no existing non-terminal record of that job assigned
|
||||
* elsewhere", so there is no terminal/more-final in-memory state this call
|
||||
* could possibly clobber. This mirrors dispatch_a_job_'s own inline
|
||||
* success-path mutation (job.c:1686-1717), which likewise sets
|
||||
* job->job.state directly, with no apply_job_update_transition/
|
||||
* job_state_is_backward call.
|
||||
*
|
||||
* "Host assignment": dispatch_a_job_'s own inline mutation (job.c:1686-
|
||||
* 1717) carries "which host" ENTIRELY via job->job.allocs -- the host name
|
||||
* lives inside xce_host_alloc_t.host, nested under xce_job_alloc_t.allocs.
|
||||
* Neither job_t nor xce_job_t (xce/types.h) has any OTHER field
|
||||
* representing a job's assigned worker/host independent of allocs
|
||||
* (confirmed by direct inspection of both struct definitions -- not a
|
||||
* guess). Since this function deliberately leaves job->job.allocs NULL (the
|
||||
* controller genuinely does not know what the scheduler originally computed
|
||||
* for this job -- design.md's Data Models "Documented scope decision"),
|
||||
* there is structurally no separate field left for this function to set to
|
||||
* represent "host assignment" beyond job->job.state itself. `w` is accepted
|
||||
* only to match design.md's own cited service-interface signature -- it is
|
||||
* otherwise inert here (see the (void)w; below): the still-running-outcome
|
||||
* check just below operates on `report->status`, not on `w`, and nothing on
|
||||
* job_t/xce_job_t is mutated from `w` either.
|
||||
*
|
||||
* Returns job_transition_result_t (cast to int), matching record_job_
|
||||
* reconcile_pending_fact's return-type convention above -- always
|
||||
* JOB_TRANSITION_APPLIED (this outcome has no reject/no-op case; see the
|
||||
* Precondition note). Unlike record_job_finish_fact/record_job_fail_fact,
|
||||
* this function does NOT call persist_notify_job_updated_already_persisted
|
||||
* -- the caller must do so itself, after releasing the write lock, exactly
|
||||
* once -- omitting it reproduces bug-1679 at this call site. */
|
||||
int record_job_reconcile_assign_fact(cluster_t* c, job_t* job, worker_t* w,
|
||||
xce_job_state_entry_t* report)
|
||||
{
|
||||
(void)w;
|
||||
|
||||
switch (report->status) {
|
||||
case JobStateStatus_running:
|
||||
break;
|
||||
default:
|
||||
/* This function's only defined caller usage (design.md's sequence
|
||||
* flow -- task 3.2, not yet implemented) is the "unqueried extra
|
||||
* job, still running" outcome: a finished extra routes through the
|
||||
* existing record_job_finish_fact/record_job_fail_fact instead
|
||||
* (Req 4.7), same as a queried finished job. Fail loud rather than
|
||||
* silently mis-recording a finished job as RUN, mirroring
|
||||
* dispatch_a_job_'s own "impossible per today's call convention"
|
||||
* ERROR+assert(0)-in-XCE_TESTING-then-safe-fallback idiom used
|
||||
* throughout this file. */
|
||||
ERROR("record_job_reconcile_assign_fact: called for job %lu with "
|
||||
"unexpected report status %d (expected JobStateStatus_running "
|
||||
"-- a finished/unrecognized outcome must route through a "
|
||||
"different fact function)",
|
||||
(unsigned long)job->job.jobid, report->status);
|
||||
#ifdef XCE_TESTING
|
||||
assert(0
|
||||
&& "record_job_reconcile_assign_fact called with a "
|
||||
"non-running report status");
|
||||
#endif
|
||||
break;
|
||||
}
|
||||
|
||||
job->job.state = JOB_STAT_RUN;
|
||||
|
||||
/* bug-1697: start_time is stamped HERE, synchronously alongside the RUN
|
||||
* transition, mirroring dispatch_a_job_'s own identical stamp
|
||||
* (job.c:1710) -- so that if this job's worker later also sends an
|
||||
* ordinary WAL_JOB_START report for it, that report always finds
|
||||
* state/start_time already set and applies as a harmless no-op, never
|
||||
* the genuine first RUN transition. There is no better time source: the
|
||||
* controller never observed this job's real dispatch time. */
|
||||
job->job.start_time = time(NULL);
|
||||
|
||||
/* Req 4.8 / design.md "Documented scope decision": job->job.allocs is
|
||||
* deliberately left untouched (NULL for a freshly-constructed job_t,
|
||||
* per this file's own makeJob()/create-job convention) -- the
|
||||
* controller has no way to learn what resource allocation the
|
||||
* scheduler originally computed for a job it never held any record of.
|
||||
* bug-1678's persist_submit_job_dispatch_fact() NULL-alloc case is
|
||||
* reused verbatim below, exactly as design.md specifies, rather than
|
||||
* fabricating one. */
|
||||
(void)persist_submit_job_dispatch_fact(&c->persist, job->job.jobid,
|
||||
JOB_STAT_RUN,
|
||||
(uint64_t)job->job.start_time,
|
||||
NULL);
|
||||
|
||||
return (int)JOB_TRANSITION_APPLIED;
|
||||
}
|
||||
|
||||
/* Guarded state-transition core for a postdone fact, extracted from
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ controller_src = [
|
|||
'repl.c',
|
||||
'ctrl_metrics.c',
|
||||
'slowlane.c',
|
||||
'resync.c',
|
||||
]
|
||||
|
||||
incdir = include_directories('.', '../include', '../lib',
|
||||
|
|
@ -191,10 +192,12 @@ controller_unit_test_src = [
|
|||
'testsuite/test_dispatch_fail_terminal.c',
|
||||
'testsuite/test_dispatch_fail_retry.c',
|
||||
'testsuite/test_jobdispatch_no_double_notify.c',
|
||||
'testsuite/test_worker_resync.c',
|
||||
'testsuite/test_upgrade_snapshot_restore.c',
|
||||
'testsuite/test_upgrade_snapshot_apply.c',
|
||||
'testsuite/test_bug857_regression.c',
|
||||
'testsuite/test_sched_admin_dispatch.c',
|
||||
'testsuite/test_jobstatereport_dispatch.c',
|
||||
'testsuite/test_fetchfile_disconnect.c',
|
||||
'ctrlstatus.c',
|
||||
'commit.c',
|
||||
|
|
@ -213,6 +216,7 @@ controller_unit_test_src = [
|
|||
'config_repl.c',
|
||||
'slowlane.c',
|
||||
'cluster.c',
|
||||
'resync.c',
|
||||
]
|
||||
|
||||
ssl_dep = dependency('openssl')
|
||||
|
|
|
|||
|
|
@ -230,6 +230,26 @@ static int capnp_encode_joballoc(const xce_job_alloc_t* a, uint8_t** out,
|
|||
struct JobAlloc cja;
|
||||
JobAlloc_ptr jap;
|
||||
|
||||
/* worker-rejoin-resync task 2.2 (bug-1746): a NULL `a` means "no
|
||||
* allocation to carry" (record_job_reconcile_assign_fact()'s Req 4.8
|
||||
* case -- the controller genuinely never learned one). Produce a
|
||||
* genuinely zero-length buffer, NOT a capnp-encoded null-pointer JobAlloc
|
||||
* (what encode_JobAlloc_ptr()'s existing NULL branch would produce) --
|
||||
* persist_submit_job_dispatch_fact()'s caller (both apply seams,
|
||||
* apply_committed_entry()/apply_wal_entry()) discriminates "is there an
|
||||
* allocation to decode" purely by `alloc_len > 0`, so any non-zero
|
||||
* trailing byte count here would be misread as a real allocation to
|
||||
* decode. encode_JobAlloc() below has no NULL guard of its own (it
|
||||
* unconditionally dereferences `a` on its first line) -- this early
|
||||
* return is the one and only guard needed, matching the sole documented
|
||||
* call convention this function must now support (dispatch_a_job_()'s
|
||||
* own call site continues to always pass non-NULL, unaffected). */
|
||||
if (a == NULL) {
|
||||
*out = NULL;
|
||||
*out_len = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
capn_init_malloc(&c);
|
||||
cs = capn_root(&c).seg;
|
||||
encode_JobAlloc(cs, &cja, (xce_job_alloc_t*)a);
|
||||
|
|
|
|||
982
src/controller/resync.c
Normal file
982
src/controller/resync.c
Normal file
|
|
@ -0,0 +1,982 @@
|
|||
/* resync.c -- worker-rejoin-resync task 3.1 (Req 1.1-1.3, 2.1, 2.2, 5.3,
|
||||
* 6.1, 6.3): per-worker job-state reconciliation round lifecycle.
|
||||
*
|
||||
* Owns exactly what design.md's "Controller: resync.c" section assigns it:
|
||||
* arm a round (query-set construction + round token + deadline + a
|
||||
* non-blocking delaymgr one-shot timeout), resolve a round (on timeout or,
|
||||
* later, on normal completion -- task 3.2), and a round-token staleness
|
||||
* check (Req 6.3) later callers use to discard a report that arrives for an
|
||||
* already-discarded/superseded round.
|
||||
*
|
||||
* Explicitly NOT this file's job (later tasks): sending the query/report
|
||||
* over the wire (task 6.1's worker.c/on_worker_join wiring), and mapping a
|
||||
* reported job-state entry to the correct job.c fact-recording call (task
|
||||
* 3.2's report-apply glue). Every function here is written to be called by
|
||||
* that later wiring, not to include it.
|
||||
*/
|
||||
#include <errno.h>
|
||||
#include <rtti.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
|
||||
#include "cluster.h"
|
||||
#include "controller.capnp.h" /* JobStateStatus_* -- worker-rejoin-resync
|
||||
* task 3.2 */
|
||||
#include "delay.h"
|
||||
#include "persist.h" /* persist_notify_job_updated_already_persisted --
|
||||
* worker-rejoin-resync task 3.2 */
|
||||
#include "resync.h"
|
||||
#include "utils.h"
|
||||
#include "xce/types.h"
|
||||
|
||||
/* ── Round token ─────────────────────────────────────────────────────────
|
||||
*
|
||||
* Req 6.1/6.3: a fresh round must be distinguishable from any prior round
|
||||
* for the SAME worker (a disconnect-then-rejoin re-arming the round, or a
|
||||
* timeout racing a late report). A single process-wide monotonic counter,
|
||||
* shared across every worker's rounds, trivially satisfies this: every arm
|
||||
* anywhere gets a value strictly greater than every arm before it, so in
|
||||
* particular greater than any prior round for the specific worker being
|
||||
* armed right now. This is simpler and collision-free versus a wall-clock
|
||||
* timestamp (which can repeat within the same millisecond under load, or
|
||||
* even go backward across an NTP step) or a per-worker counter (extra state
|
||||
* to carry and initialize per worker_t). Starts at 1 so 0 stays available
|
||||
* as an unambiguous "no token" sentinel if ever needed. */
|
||||
static _Atomic uint64_t g_resync_next_round_token = 1;
|
||||
|
||||
static uint64_t resync_next_round_token(void)
|
||||
{
|
||||
return atomic_fetch_add_explicit(&g_resync_next_round_token, 1,
|
||||
memory_order_relaxed);
|
||||
}
|
||||
|
||||
/* ── Timeout / deadline ──────────────────────────────────────────────────
|
||||
*
|
||||
* Req 5.3's timeout value is ClusterConfig.workerResyncTimeoutMs
|
||||
* (worker_resync_timeout_ms on the wire-decoded cluster_config_t,
|
||||
* clusterconfig.h), reached via c->myconf->cluster_config -- the same
|
||||
* accessor chain already used for the sibling shared-config field
|
||||
* heartbeat_interval (cluster.c:1347/2270's
|
||||
* c->myconf->cluster_config->heartbeat_interval). No xceadmin/schema
|
||||
* default is defined for this field (src/configs/schema/xce.schema.json
|
||||
* declares it with no "default" key); the fixture value used throughout
|
||||
* xceadmin's own tests (src/xceadmin/tests/test_mapping.py,
|
||||
* fixtures/full/configs/xce.yaml) is 15000ms, adopted here as the fallback
|
||||
* for when a live cluster_config isn't available at all (myconf/
|
||||
* cluster_config unset or the field left at its zero-value) -- e.g. a
|
||||
* minimal unit-test cluster_t fixture that never populates myconf, mirroring
|
||||
* myconf.c's own SET_DEFAULT_IF_UNSET convention for heartbeat_interval. */
|
||||
#define RESYNC_DEFAULT_TIMEOUT_MS 15000
|
||||
|
||||
static uint64_t resync_timeout_ms(cluster_t* c)
|
||||
{
|
||||
if (c != NULL && c->myconf != NULL && c->myconf->cluster_config != NULL
|
||||
&& c->myconf->cluster_config->worker_resync_timeout_ms > 0)
|
||||
return (uint64_t)c->myconf->cluster_config->worker_resync_timeout_ms;
|
||||
|
||||
return RESYNC_DEFAULT_TIMEOUT_MS;
|
||||
}
|
||||
|
||||
/* Same CLOCK_MONOTONIC millisecond representation resync_state_t.deadline
|
||||
* documents (xce/types.h) and delay.h's own delay_task_t.nextrun_ms uses
|
||||
* (delay.c's monotonic_ms(), duplicated here rather than exposed by
|
||||
* delay.h/delay.c -- it is a two-line static helper, not worth a new public
|
||||
* API for). */
|
||||
static uint64_t resync_monotonic_ms(void)
|
||||
{
|
||||
struct timespec now;
|
||||
|
||||
clock_gettime(CLOCK_MONOTONIC, &now);
|
||||
return (uint64_t)now.tv_sec * 1000ULL + (uint64_t)now.tv_nsec / 1000000ULL;
|
||||
}
|
||||
|
||||
/* ── Query-set construction (Req 1.1, 2.1, 2.2) ─────────────────────────── */
|
||||
|
||||
/* Mirrors job.c's own private JOB_STATE_TERMINAL_BITS/job_state_is_terminal
|
||||
* (job.c:41-50): a job is terminal once it carries a DONE or EXIT bit.
|
||||
* job.c's predicate is `static inline` and not exported via cluster.h, and
|
||||
* this task's boundary is explicitly "read job.c for reference, don't
|
||||
* modify it" -- so this is an independent definition built from the SAME
|
||||
* public bit constants (xce/types.h's JOB_STAT_DONE/JOB_STAT_EXIT), not a
|
||||
* duplicate of a private symbol and not a change to job.c to export one. */
|
||||
static inline int resync_job_is_terminal(unsigned int state)
|
||||
{
|
||||
return (state & (JOB_STAT_DONE | JOB_STAT_EXIT)) != 0;
|
||||
}
|
||||
|
||||
/* True iff `j`'s head/primary assigned host matches `hostname`. A job's
|
||||
* host assignment lives ENTIRELY in job->job.allocs->allocs[0]->host --
|
||||
* confirmed by reading job.c's own existing readers of this same
|
||||
* information (job.c's mark_job_ctrl-family job-control routing at
|
||||
* job.c:1922-1936; the exechost lookup at job.c:3766-3771) and by
|
||||
* record_job_reconcile_assign_fact's own doc comment (job.c:3031-3045),
|
||||
* which independently confirms neither job_t nor xce_job_t carries any
|
||||
* OTHER field representing a job's assigned host. allocs[0] specifically
|
||||
* is the job's HEAD host, matching requirements.md's Boundary Context
|
||||
* ("Multi-host jobs... the Controller dispatches and tracks a multi-host
|
||||
* job as a single record owned by that job's head/primary host"). Caller
|
||||
* holds `j`'s read lock. */
|
||||
static int resync_job_assigned_to_host(job_t* j, const char* hostname)
|
||||
{
|
||||
xce_job_alloc_t* alloc = j->job.allocs;
|
||||
|
||||
if (alloc == NULL || alloc->nhosts <= 0 || alloc->allocs == NULL
|
||||
|| alloc->allocs[0] == NULL || alloc->allocs[0]->host == NULL)
|
||||
return 0;
|
||||
|
||||
return strcmp(alloc->allocs[0]->host, hostname) == 0;
|
||||
}
|
||||
|
||||
/* Req 1.1, 2.1, 2.2: builds the round's query-job-ID set -- every
|
||||
* non-terminal job in c->jobmgr currently assigned to w's host. Locking
|
||||
* mirrors persist.c's collect_job_frames() nested discipline exactly: the
|
||||
* jobmgr table's own RTTI_LOCK_READ is held for the entire walk (excludes
|
||||
* concurrent insert/delete), and each candidate job's OWN RTTI_LOCK_READ is
|
||||
* additionally, nestedly, taken just around reading its state/allocs --
|
||||
* already the proven-safe pattern in this codebase for a point-in-time-
|
||||
* consistent jobmgr walk. Independent of `w`'s own lock (a different lock
|
||||
* object entirely), so this never conflicts with the write lock the caller
|
||||
* of resync_arm_round already holds on `w`.
|
||||
*
|
||||
* On success (return 0), *out_ids is a malloc'd uint64_t[*out_n] the caller
|
||||
* owns (free() when done); *out_n may be 0 with *out_ids left NULL -- Req
|
||||
* 1.2's "no non-terminal jobs for this host" case is not an error. On
|
||||
* failure (negative errno), *out_ids and *out_n are left NULL/0 and nothing
|
||||
* remains allocated. */
|
||||
static int resync_collect_query_jobids(cluster_t* c, worker_t* w,
|
||||
uint64_t** out_ids, int* out_n)
|
||||
{
|
||||
iterator_t it;
|
||||
int rc;
|
||||
uint64_t* ids = NULL;
|
||||
int n = 0;
|
||||
int cap = 0;
|
||||
|
||||
*out_ids = NULL;
|
||||
*out_n = 0;
|
||||
|
||||
if (c == NULL || c->jobmgr == NULL || w == NULL)
|
||||
return -EINVAL;
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
|
||||
rc = iter_make(&it, c->jobmgr);
|
||||
if (rc < 0) {
|
||||
unlock(c->jobmgr);
|
||||
return 0; /* empty table -- zero matches, not an error (mirrors
|
||||
* collect_job_frames'/collect_jobs' own empty-table
|
||||
* handling) */
|
||||
}
|
||||
|
||||
for (;;) {
|
||||
job_t* j = NULL;
|
||||
int nrc;
|
||||
int match;
|
||||
uint64_t jobid;
|
||||
|
||||
nrc = iter_next(&it, (void**)&j);
|
||||
if (nrc < 0 || nrc == RTTI_ITER_NO_DATA)
|
||||
break;
|
||||
if (j == NULL)
|
||||
continue;
|
||||
|
||||
lock(j, RTTI_LOCK_READ);
|
||||
match = !resync_job_is_terminal(j->job.state)
|
||||
&& resync_job_assigned_to_host(j, w->host.name);
|
||||
jobid = j->job.jobid;
|
||||
unlock(j);
|
||||
|
||||
if (!match)
|
||||
continue;
|
||||
|
||||
if (n == cap) {
|
||||
int newcap = (cap == 0) ? 8 : (cap * 2);
|
||||
uint64_t* grown
|
||||
= (uint64_t*)realloc(ids, (size_t)newcap * sizeof(uint64_t));
|
||||
if (grown == NULL) {
|
||||
free(ids);
|
||||
iter_destroy(&it);
|
||||
unlock(c->jobmgr);
|
||||
*out_ids = NULL;
|
||||
*out_n = 0;
|
||||
return -ENOMEM;
|
||||
}
|
||||
ids = grown;
|
||||
cap = newcap;
|
||||
}
|
||||
ids[n++] = jobid;
|
||||
}
|
||||
|
||||
iter_destroy(&it);
|
||||
unlock(c->jobmgr);
|
||||
|
||||
*out_ids = ids;
|
||||
*out_n = n;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ── Timeout trampoline (Req 5.3) ────────────────────────────────────────
|
||||
*
|
||||
* delay.c/delay.h expose no "cancel a queued task" primitive (delay_task_ms
|
||||
* only ever enqueues; delay_worker's own loop only ever fires-then-frees or
|
||||
* re-arms). So a round that resolves normally (task 3.2, later) BEFORE its
|
||||
* timeout fires cannot make the already-queued delaymgr task disappear --
|
||||
* it will still fire, eventually, on schedule. Safety against a stale fire
|
||||
* acting on an already-cleared/reused resync_state_t therefore comes from
|
||||
* the round token, not from dequeuing: the closure below captures the
|
||||
* round token the timeout was armed FOR, and resync_on_timeout only acts if
|
||||
* resync_round_is_current() still reports that exact token as current for
|
||||
* `w` at fire time. A round that already resolved (normally, or via a
|
||||
* newer resync_arm_round superseding it) makes the old token stale, so the
|
||||
* late fire becomes a pure no-op -- never a double-clear, never touching a
|
||||
* `resync_state_t` that was already freed and possibly replaced. */
|
||||
typedef struct {
|
||||
cluster_t* c;
|
||||
worker_t* w;
|
||||
uint64_t round_token;
|
||||
uint64_t timeout_ms;
|
||||
} resync_timeout_ctx_t;
|
||||
|
||||
/* Runs on the delaymgr timer thread (c->delaymgr's runner_fn, cluster.c's
|
||||
* run_delay_tasks, calls this directly and synchronously). Per the
|
||||
* async-I/O thread-role contract (ha-async-io.md's Thread-role contract
|
||||
* DESIGN INVARIANT): delaymgr must fire timers and trigger work, then
|
||||
* return fast -- no connect(), no file I/O, no unbounded wait. Everything
|
||||
* below is bounded, in-memory bookkeeping under `w`'s own mutex (a status
|
||||
* bit clear, freeing one small heap struct) -- never a blocking call, so
|
||||
* this satisfies that contract by construction, not by inspection alone.
|
||||
*
|
||||
* worker_t* is captured directly (not re-looked-up via c->workermgr):
|
||||
* confirmed by reading cluster.c that no code path ever key_del()s an entry
|
||||
* from c->workermgr once created (worker_obj_killer, its kvmgr datakiller,
|
||||
* is a documented no-op) -- a worker_t, once created, lives for the
|
||||
* process's lifetime, the same assumption ha.c's own delaymgr trampolines
|
||||
* (e.g. ha_phase2_retry_delaymgr_trampoline capturing cluster_t* c) already
|
||||
* make about their own captured pointers. */
|
||||
static int resync_on_timeout(void* arg)
|
||||
{
|
||||
resync_timeout_ctx_t* ctx = (resync_timeout_ctx_t*)arg;
|
||||
|
||||
if (ctx == NULL)
|
||||
return -EINVAL;
|
||||
|
||||
lock(ctx->w, RTTI_LOCK_WRITE);
|
||||
if (resync_round_is_current(ctx->w, ctx->round_token)) {
|
||||
/* Req 5.3: "accepted-as-answered, timed out, or left for the next
|
||||
* rejoin to retry" -- this implementation takes the literal
|
||||
* "timed out" branch: resolve the round now so the worker is never
|
||||
* withheld from dispatch indefinitely (Req 5.1/5.2). A report that
|
||||
* later arrives for this now-resolved round is naturally discarded
|
||||
* by resync_round_is_current's own stale-token check (Req 6.3)
|
||||
* wherever a later caller (task 3.2) applies it -- there is no
|
||||
* separate "reattempted on a subsequent rejoin" action to take
|
||||
* here; that is what a fresh resync_arm_round on the next join
|
||||
* already provides. */
|
||||
WARN("resync: worker %s reconciliation round %lu timed out after "
|
||||
"%lums -- resolving so dispatch is not withheld indefinitely "
|
||||
"(Req 5.3)",
|
||||
(ctx->w->host.name != NULL) ? ctx->w->host.name : "?",
|
||||
(unsigned long)ctx->round_token,
|
||||
(unsigned long)ctx->timeout_ms);
|
||||
resync_resolve_round(ctx->c, ctx->w);
|
||||
}
|
||||
/* else: stale fire -- the round this timeout was armed for already
|
||||
* resolved (normally) or was superseded by a newer resync_arm_round
|
||||
* before the timer got here. No-op by construction (see the comment
|
||||
* above resync_timeout_ctx_t). */
|
||||
unlock(ctx->w);
|
||||
|
||||
free(ctx);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ── Public API ──────────────────────────────────────────────────────── */
|
||||
|
||||
int resync_arm_round(cluster_t* c, worker_t* w)
|
||||
{
|
||||
uint64_t* ids = NULL;
|
||||
int n = 0;
|
||||
int rc;
|
||||
resync_state_t* rs;
|
||||
resync_timeout_ctx_t* ctx;
|
||||
uint64_t timeout_ms;
|
||||
|
||||
if (c == NULL || w == NULL)
|
||||
return -EINVAL;
|
||||
|
||||
/* Req 6.1: an already-outstanding round for this worker (this rejoin
|
||||
* interrupts one that never resolved) is discarded first, so the fresh
|
||||
* round's token always supersedes it -- never two live resync_state_t
|
||||
* generations for the same worker_t at once. */
|
||||
if (w->resync != NULL)
|
||||
resync_resolve_round(c, w);
|
||||
|
||||
rc = resync_collect_query_jobids(c, w, &ids, &n);
|
||||
if (rc < 0) {
|
||||
ERROR("resync_arm_round: worker %s: failed to build reconciliation "
|
||||
"query set: %d -- no round armed (the next rejoin will "
|
||||
"attempt again)",
|
||||
(w->host.name != NULL) ? w->host.name : "?", rc);
|
||||
return rc;
|
||||
}
|
||||
|
||||
if (n == 0) {
|
||||
/* Req 1.2: no non-terminal jobs for this host -- documented no-op:
|
||||
* no round armed, no status bit set, no timer scheduled. */
|
||||
free(ids);
|
||||
return 0;
|
||||
}
|
||||
|
||||
rs = (resync_state_t*)calloc(1, sizeof(*rs));
|
||||
if (rs == NULL) {
|
||||
free(ids);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
rs->round_token = resync_next_round_token();
|
||||
rs->n_queried_jobids = n;
|
||||
rs->queried_jobids = ids;
|
||||
|
||||
timeout_ms = resync_timeout_ms(c);
|
||||
rs->deadline = resync_monotonic_ms() + timeout_ms;
|
||||
|
||||
w->resync = rs;
|
||||
w->host.status |= XCE_WORKER_RECONCILING;
|
||||
|
||||
ctx = (resync_timeout_ctx_t*)calloc(1, sizeof(*ctx));
|
||||
if (ctx == NULL) {
|
||||
/* The round itself is still validly armed (query set, token,
|
||||
* deadline, status bit are all already set above) -- only the
|
||||
* BOUNDED-TIMEOUT half of Req 5.3 is unavailable this time. Fail
|
||||
* loud and stay alive (steering: never fabricate partial success
|
||||
* silently) rather than unwind an otherwise-good round over an
|
||||
* allocation failure for a small bookkeeping struct; a later
|
||||
* report (task 3.2) still resolves it normally. */
|
||||
ERROR("resync_arm_round: worker %s round %lu: timeout context "
|
||||
"alloc failed -- round armed WITHOUT a bounded timeout (Req "
|
||||
"5.3 degraded: resolves normally on report, not via timeout "
|
||||
"this time)",
|
||||
(w->host.name != NULL) ? w->host.name : "?",
|
||||
(unsigned long)rs->round_token);
|
||||
return n;
|
||||
}
|
||||
ctx->c = c;
|
||||
ctx->w = w;
|
||||
ctx->round_token = rs->round_token;
|
||||
ctx->timeout_ms = timeout_ms;
|
||||
|
||||
if (!c->delaymgr.running) {
|
||||
/* Not-yet-started/test-fixture delaymgr -- mirrors ha.c's own
|
||||
* `!c->delaymgr.running` guard (e.g. schedule_upgrade_
|
||||
* materialization_retry, ha_schedule_phase2_retry): the round
|
||||
* stays armed; a test drives resync_on_timeout directly instead of
|
||||
* waiting on a real timer thread that was never started. */
|
||||
free(ctx);
|
||||
return n;
|
||||
}
|
||||
|
||||
rc = delay_task_ms(&c->delaymgr, (int)timeout_ms, 1, resync_on_timeout,
|
||||
ctx);
|
||||
if (rc < 0) {
|
||||
WARN("resync_arm_round: worker %s round %lu: delay_task_ms failed: "
|
||||
"%d -- round armed without a bounded timeout",
|
||||
(w->host.name != NULL) ? w->host.name : "?",
|
||||
(unsigned long)rs->round_token, rc);
|
||||
free(ctx);
|
||||
}
|
||||
|
||||
return n;
|
||||
}
|
||||
|
||||
void resync_resolve_round(cluster_t* c, worker_t* w)
|
||||
{
|
||||
(void)c; /* accepted to match this file's other functions' shape and
|
||||
* what a later caller (task 3.2) already has in hand
|
||||
* alongside `w` -- this function's own body has nothing to do
|
||||
* with `c` today. */
|
||||
|
||||
if (w == NULL || w->resync == NULL)
|
||||
return;
|
||||
|
||||
w->host.status &= ~XCE_WORKER_RECONCILING;
|
||||
|
||||
free(w->resync->queried_jobids);
|
||||
free(w->resync);
|
||||
w->resync = NULL;
|
||||
}
|
||||
|
||||
int resync_round_is_current(worker_t* w, uint64_t token)
|
||||
{
|
||||
if (w == NULL || w->resync == NULL)
|
||||
return 0;
|
||||
|
||||
return w->resync->round_token == token;
|
||||
}
|
||||
|
||||
/* ── Report application (task 3.2, Req 4.1-4.5, 6.1-6.3) ─────────────────
|
||||
*
|
||||
* Maps each JobStateEntry in a worker's job-state report to the correct
|
||||
* idempotent job.c fact-recording call, then resolves the round once every
|
||||
* queried job has a reconciled outcome. See resync.h's doc comment on
|
||||
* resync_apply_report() for the full per-outcome routing contract, and
|
||||
* design.md's "System Flows" sequence diagram (the `loop each reported
|
||||
* entry` block) for the design this implements.
|
||||
*/
|
||||
|
||||
/* Req 6.2: removes `jobid` from w->resync's outstanding query set, if
|
||||
* present -- swap-with-last-and-shrink (the array is small, bounded by one
|
||||
* host's non-terminal job count per resync_collect_query_jobids' own
|
||||
* reasoning, so O(n) removal is not a concern). A no-op when w->resync is
|
||||
* NULL or `jobid` is not present -- safe to call unconditionally and
|
||||
* repeatedly (an already-cleared id, or one that was never queried, e.g. an
|
||||
* unqueried extra). Caller holds w's write lock. */
|
||||
static void resync_clear_outstanding(worker_t* w, uint64_t jobid)
|
||||
{
|
||||
resync_state_t* rs = w->resync;
|
||||
int i;
|
||||
|
||||
if (rs == NULL)
|
||||
return;
|
||||
|
||||
for (i = 0; i < rs->n_queried_jobids; i++) {
|
||||
if (rs->queried_jobids[i] == jobid) {
|
||||
rs->queried_jobids[i]
|
||||
= rs->queried_jobids[rs->n_queried_jobids - 1];
|
||||
rs->n_queried_jobids--;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Req 5.1/5.2: resolves w's round if its outstanding set has just become
|
||||
* empty. Caller holds w's write lock and must have already confirmed the
|
||||
* round is still current for the token in question (see
|
||||
* resync_settle_entry below) -- this function itself only re-checks
|
||||
* w->resync != NULL (resync_resolve_round's own precondition), not token
|
||||
* identity, since the caller's own check under the SAME uninterrupted lock
|
||||
* hold already establishes that. */
|
||||
static void resync_maybe_resolve(cluster_t* c, worker_t* w)
|
||||
{
|
||||
if (w->resync != NULL && w->resync->n_queried_jobids == 0)
|
||||
resync_resolve_round(c, w);
|
||||
}
|
||||
|
||||
/* Req 4.1, 5.1, 5.2, 6.1-6.3: clears `jobid` from w's outstanding set for
|
||||
* round `token` and, if the set has just become empty, resolves the round
|
||||
* -- both performed under a SINGLE w write-lock acquisition, both gated on
|
||||
* `token` still being w's CURRENT round. The gate matters: a concurrent Req
|
||||
* 5.3 timeout fire, or a fresh resync_arm_round superseding this round on
|
||||
* another thread, can happen between this report-apply loop's per-entry
|
||||
* fact application (which holds no lock on `w`) and this call -- without
|
||||
* re-validating here, a stale round's late bookkeeping could otherwise
|
||||
* mutate a DIFFERENT (newer) round's resync_state_t. This is also the exact
|
||||
* mechanism that makes re-applying an already-fully-resolved report a safe
|
||||
* no-op (Req 6.2): once resync_resolve_round() has run, w->resync is NULL,
|
||||
* so `token` no longer reports current and this becomes a pure no-op. */
|
||||
static void resync_settle_entry(cluster_t* c, worker_t* w, uint64_t token,
|
||||
uint64_t jobid)
|
||||
{
|
||||
lock(w, RTTI_LOCK_WRITE);
|
||||
if (resync_round_is_current(w, token)) {
|
||||
resync_clear_outstanding(w, jobid);
|
||||
resync_maybe_resolve(c, w);
|
||||
}
|
||||
unlock(w);
|
||||
}
|
||||
|
||||
/* Looks up `jobid` in c->jobmgr, reference()-ing it on success -- the same
|
||||
* key_find+reference idiom on_job_start/on_job_finish/on_job_fail/
|
||||
* dispatch_a_job_ (job.c) already use throughout this codebase. Returns
|
||||
* NULL if not found -- never treated as an error by this file's own callers
|
||||
* (see resync_apply_running_entry's doc comment for why a genuinely-
|
||||
* untracked jobid is a legitimate, if narrow, possibility). Caller must
|
||||
* dereference() the result when done, if non-NULL. */
|
||||
static job_t* resync_lookup_job(cluster_t* c, uint64_t jobid)
|
||||
{
|
||||
job_t* j = NULL;
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&j);
|
||||
if (j != NULL)
|
||||
reference(j);
|
||||
unlock(c->jobmgr);
|
||||
|
||||
return j;
|
||||
}
|
||||
|
||||
/* Req 4.1, 4.4, 4.7, 4.8: routes a "running" entry. A QUERIED running job's
|
||||
* own state already correctly reflects "running" -- no fact is recorded
|
||||
* (design.md's sequence diagram: "else still running -> clear this jobid
|
||||
* from outstanding set (no-op fact)"); the caller clears the outstanding
|
||||
* set for every queried entry uniformly (resync_apply_report below), so
|
||||
* this function has nothing further to do for that case.
|
||||
*
|
||||
* An UNQUERIED extra (Req 3.6) running job is a genuine THREE-WAY split, not
|
||||
* a two-way boolean (review round 2 regression fix, following round 1's own
|
||||
* review-caught fix -- the round-1 guard introduced a NEW Req 4.4 violation:
|
||||
* `!terminal && assigned-to-w` short-circuits to `false` whenever the job IS
|
||||
* terminal, falling through to the assign branch and unconditionally
|
||||
* clobbering a DONE/EXIT job back to RUN -- confirmed live: round 1's own
|
||||
* test seeded a TERMINAL j_extra and asserted it became RUN, certifying the
|
||||
* bug as "correct" instead of catching it):
|
||||
*
|
||||
* (a) TERMINAL (resync_job_is_terminal()) -> ALWAYS a no-op, regardless of
|
||||
* host. Requirement 4.4 ("shall not overwrite that state with a less
|
||||
* final or older report") is unconditional -- a terminal job is never
|
||||
* assign-eligible, full stop. record_job_reconcile_assign_fact() has NO
|
||||
* backward-transition guard of its own (its own doc comment: "no
|
||||
* reject/no-op case" is a documented CALLER precondition, never
|
||||
* re-checked internally) and unconditionally sets job->job.state =
|
||||
* JOB_STAT_RUN -- calling it here would clobber DONE/EXIT back to RUN.
|
||||
* (b) Non-terminal AND job->job.allocs != NULL (already has SOME live
|
||||
* non-terminal host assignment -- to THIS reporting worker's own host,
|
||||
* or to a DIFFERENT one, it does not matter which) -> a no-op. It is
|
||||
* already validly assigned; record_job_reconcile_assign_fact() would
|
||||
* still be wrong to call even though its net STATE effect looks
|
||||
* harmless (RUN -> RUN) -- it unconditionally resets job->job.start_time
|
||||
* and re-appends a fresh WAL_JOB_DISPATCH fact, neither of which is
|
||||
* warranted for a job that was never actually lost/unassigned.
|
||||
* (c) Non-terminal AND job->job.allocs == NULL (genuinely UNASSIGNED) ->
|
||||
* Req 4.8's real trigger condition ("no existing non-terminal record...
|
||||
* assigned elsewhere" -- there IS a non-terminal record, but it carries
|
||||
* no assignment at all) -- reconciled via record_job_reconcile_assign_
|
||||
* fact() (task 2.2), documented (and hard-asserted, job.c) as the
|
||||
* running-only case.
|
||||
*
|
||||
* Concrete motivating scenario for (c) (the exact case design.md's own
|
||||
* "Documented scope decision" paragraph describes as this function's
|
||||
* target, and record_job_reconcile_assign_fact()'s core justification for
|
||||
* existing at all): a job the controller currently shows PEND (non-
|
||||
* terminal, allocs == NULL, unassigned to any host) because its
|
||||
* WAL_JOB_DISPATCH fact never reached durable majority commit before a
|
||||
* controller failover -- even though dispatch_a_job_() already sent
|
||||
* xce_startjob_req() to the worker and the worker really is running it (the
|
||||
* RPC fires BEFORE dispatch_a_job_()'s own eager mutation + WAL append, so
|
||||
* the two can diverge exactly at a failover boundary). Such a job correctly
|
||||
* reconciles via record_job_reconcile_assign_fact() (host now matches the
|
||||
* reporting worker, allocs stays NULL per Req 4.8's own documented scope
|
||||
* decision).
|
||||
*
|
||||
* A jobid with NO record AT ALL in the job table cannot be reconciled via
|
||||
* record_job_reconcile_assign_fact() either -- that function mutates an
|
||||
* EXISTING job_t, it does not construct one -- so this is logged and
|
||||
* skipped rather than fabricating a full job record from a bare entry;
|
||||
* every job a worker can legitimately run was, by this system's own
|
||||
* architecture, submitted through this controller at some point, so this
|
||||
* branch is expected to be unreachable in practice, not a routine path. */
|
||||
static void resync_apply_running_entry(cluster_t* c, worker_t* w,
|
||||
xce_job_state_entry_t* entry)
|
||||
{
|
||||
job_t* job;
|
||||
|
||||
if (entry->queried)
|
||||
return;
|
||||
|
||||
job = resync_lookup_job(c, entry->jobid);
|
||||
if (job == NULL) {
|
||||
WARN("resync: unqueried extra job %lu reported running by worker "
|
||||
"%s has no record at all in the job table -- cannot reconcile "
|
||||
"(no existing job_t to assign onto); skipped",
|
||||
(unsigned long)entry->jobid,
|
||||
(w->host.name != NULL) ? w->host.name : "?");
|
||||
return;
|
||||
}
|
||||
|
||||
lock(job, RTTI_LOCK_WRITE);
|
||||
if (resync_job_is_terminal(job->job.state)) {
|
||||
/* (a) Terminal is always final -- Req 4.4. Never assign-eligible,
|
||||
* regardless of host. */
|
||||
unlock(job);
|
||||
} else if (job->job.allocs != NULL) {
|
||||
/* (b) Non-terminal AND already has SOME live non-terminal host
|
||||
* assignment -- a genuine no-op; do not reset start_time or
|
||||
* re-append a WAL_JOB_DISPATCH fact for a job that was never
|
||||
* actually lost/unassigned. */
|
||||
unlock(job);
|
||||
} else {
|
||||
/* (c) Non-terminal AND genuinely UNASSIGNED -- Req 4.8 fires. */
|
||||
int rc = record_job_reconcile_assign_fact(c, job, w, entry);
|
||||
unlock(job);
|
||||
if (rc == (int)JOB_TRANSITION_APPLIED)
|
||||
persist_notify_job_updated_already_persisted(c, job);
|
||||
}
|
||||
|
||||
dereference(job);
|
||||
}
|
||||
|
||||
/* Req 4.1-4.5, 4.7: routes a "finished" entry (success or failure) through
|
||||
* the SAME existing, deferred-pattern fact-recording used for an ordinary
|
||||
* worker finish/fail report. Applies identically whether entry->queried is
|
||||
* true or false (Req 4.7: an unqueried finished extra reconciles through
|
||||
* "the same idempotent commit-then-apply path... used for queried jobs" --
|
||||
* there is no separate "assign" case for a finished outcome; record_job_
|
||||
* reconcile_assign_fact() is documented as running-only, see job.c's own
|
||||
* doc comment on it).
|
||||
*
|
||||
* finishedOk: bug-1747 fix (Req 4.2) -- routed through the NEW record_job_
|
||||
* finish_fact_at() (job.c), not on_job_finish()/record_job_finish_fact(),
|
||||
* so the job's recorded finish_time is the WORKER-reported entry->
|
||||
* finish_time_ms (Req 3.5), not reconciliation-processing wall-clock time.
|
||||
* record_job_finish_fact_at() expects the caller to already hold job's
|
||||
* write lock (like record_job_reconcile_pending_fact/assign_fact above),
|
||||
* unlike on_job_finish() which locks internally -- this function performs
|
||||
* that lookup/lock/unlock/dereference sequence itself, mirroring on_job_
|
||||
* finish()'s own internal sequence exactly (job.c). A jobid not found in
|
||||
* the job table is a silent no-op, mirroring on_job_finish()'s own
|
||||
* "can not find job" behavior for an unknown jobid.
|
||||
*
|
||||
* finishedFail: still routed through the existing, UNMODIFIED on_job_fail()
|
||||
* /record_job_fail_fact() -- Requirement 4.2 remains unsatisfiable for this
|
||||
* outcome via any existing path in this codebase, not a reconciliation-
|
||||
* specific gap: WAL_JOB_FAIL's payload (persist_submit_job_fact_ex(...,
|
||||
* WAL_JOB_FAIL, jobid, 0, WAL_JOB_FAIL_STAGE_FLAG|stage)) carries no
|
||||
* finish-time slot at any layer, and apply_job_fail_transition() (job.c)
|
||||
* unconditionally sets job->job.finish_time = time(NULL) at APPLY time
|
||||
* (not even receipt time) for EVERY fail-reporting caller in this system,
|
||||
* confirmed by direct inspection -- a pre-existing, systemic gap that
|
||||
* predates this feature and is out of this task's boundary to fix (see
|
||||
* buglog.json bug-1747, re-scoped). The reconciled job's STATE (EXIT) is
|
||||
* always correct either way; only its recorded finish_time reflects
|
||||
* apply-processing time rather than the worker's true report. */
|
||||
static void resync_apply_finished_entry(cluster_t* c,
|
||||
xce_job_state_entry_t* entry)
|
||||
{
|
||||
if (entry->status == JobStateStatus_finishedOk) {
|
||||
job_t* job = resync_lookup_job(c, entry->jobid);
|
||||
|
||||
if (job != NULL) {
|
||||
xce_jobfinish_t finish
|
||||
= { .jobid = entry->jobid, .exit_code = 0 };
|
||||
|
||||
lock(job, RTTI_LOCK_WRITE);
|
||||
record_job_finish_fact_at(c, job, &finish,
|
||||
entry->finish_time_ms);
|
||||
unlock(job);
|
||||
|
||||
dereference(job);
|
||||
}
|
||||
} else {
|
||||
job_failinfo_t fail
|
||||
= { .jobid = entry->jobid,
|
||||
.stage = JOB_EXEC_STAGE_MAIN,
|
||||
.code = entry->exit_code,
|
||||
.detail = (char*)"worker-rejoin-resync reconciliation" };
|
||||
on_job_fail(c, &fail);
|
||||
}
|
||||
}
|
||||
|
||||
/* Req 4.1, 4.5, 4.6: routes an "unrecognized" entry through record_job_
|
||||
* reconcile_pending_fact() (task 2.1). Per Req 3.1/3.3, a worker only ever
|
||||
* reports "unrecognized" for a jobid it was explicitly asked about (an
|
||||
* unqueried extra is, by construction of Req 3.6's own "currently running
|
||||
* or recently finished" trigger, never reported unrecognized) -- so this
|
||||
* path is queried-only in practice; the caller (resync_apply_report below)
|
||||
* still clears the outstanding set uniformly for every queried entry
|
||||
* regardless of which branch handled it. */
|
||||
static void resync_apply_unrecognized_entry(cluster_t* c,
|
||||
xce_job_state_entry_t* entry)
|
||||
{
|
||||
job_t* job = resync_lookup_job(c, entry->jobid);
|
||||
int rc;
|
||||
|
||||
if (job == NULL)
|
||||
return;
|
||||
|
||||
lock(job, RTTI_LOCK_WRITE);
|
||||
rc = record_job_reconcile_pending_fact(c, job);
|
||||
unlock(job);
|
||||
|
||||
if (rc == (int)JOB_TRANSITION_APPLIED)
|
||||
persist_notify_job_updated_already_persisted(c, job);
|
||||
|
||||
dereference(job);
|
||||
}
|
||||
|
||||
int resync_apply_report(cluster_t* c, worker_t* w, uint64_t round_token,
|
||||
xce_job_state_report_t* report)
|
||||
{
|
||||
int i;
|
||||
|
||||
if (c == NULL || w == NULL || report == NULL)
|
||||
return -EINVAL;
|
||||
if (report->count > 0 && report->entries == NULL)
|
||||
return -EINVAL;
|
||||
|
||||
/* Req 6.1, 6.3: a report for a round already superseded/discarded is
|
||||
* dropped in its entirety, before anything else is touched -- checked
|
||||
* under w's own write lock (matching every other w->resync mutation
|
||||
* this function goes on to make, via resync_settle_entry, per entry
|
||||
* below). */
|
||||
lock(w, RTTI_LOCK_WRITE);
|
||||
if (!resync_round_is_current(w, round_token)) {
|
||||
unlock(w);
|
||||
return 0;
|
||||
}
|
||||
unlock(w);
|
||||
|
||||
/* Per-entry routing never holds w's write lock while also locking a
|
||||
* job's own write lock -- mirrors dispatch_a_job_'s established
|
||||
* discipline (job.c: every branch releases w's lock BEFORE acquiring a
|
||||
* job's) rather than this file's other three functions' "caller holds
|
||||
* w's lock for the whole call" convention, which fits their simple,
|
||||
* single-object, single-call-site mutations but not this function's
|
||||
* fan-out into per-job job.c calls, each requiring its own job-level
|
||||
* lock. w's lock is only (re)acquired, narrowly, inside
|
||||
* resync_settle_entry's own outstanding-set/maybe-resolve mutation. */
|
||||
for (i = 0; i < report->count; i++) {
|
||||
xce_job_state_entry_t* entry = report->entries[i];
|
||||
|
||||
if (entry == NULL)
|
||||
continue;
|
||||
|
||||
switch (entry->status) {
|
||||
case JobStateStatus_running:
|
||||
resync_apply_running_entry(c, w, entry);
|
||||
break;
|
||||
case JobStateStatus_finishedOk:
|
||||
case JobStateStatus_finishedFail:
|
||||
resync_apply_finished_entry(c, entry);
|
||||
break;
|
||||
case JobStateStatus_unrecognized:
|
||||
default:
|
||||
resync_apply_unrecognized_entry(c, entry);
|
||||
break;
|
||||
}
|
||||
|
||||
if (entry->queried)
|
||||
resync_settle_entry(c, w, round_token, entry->jobid);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ── Multi-page report reassembly (task 6.2, Req 8.1, 8.3) ─────────────────
|
||||
*
|
||||
* resync_apply_report() above expects an already-fully-assembled report --
|
||||
* its own doc comment (resync.h): "multi-page wire reassembly is a later
|
||||
* task's concern (6.2), out of this function's scope." The functions below
|
||||
* are that later task: accumulate an incoming OPCode_jobStateReport's wire
|
||||
* pages into worker_t.report_accum (xce/types.h), then call resync_apply_
|
||||
* report() exactly once -- never once per page -- only once the last page
|
||||
* has arrived. Mirrors xcecd_resync_handle_query_page()'s (worker-side,
|
||||
* xcecd_resync.c, task 4.3) accumulate-then-apply-once shape and its
|
||||
* reset-on-pos-0 discard-stale-accumulation convention, adapted from a
|
||||
* single file-static accumulator (a worker daemon has exactly one
|
||||
* controller connection) to a per-worker_t one (a controller serves many
|
||||
* workers concurrently, so the accumulator must be keyed per reporting
|
||||
* worker, guarded by that worker's own RTTI lock).
|
||||
*/
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
/* Structural proof seam (task 6.2): counts actual resync_apply_report()
|
||||
* invocations made BY resync_handle_report_page() below, process-wide.
|
||||
* Lets a test assert "a report split across N pages results in exactly ONE
|
||||
* resync_apply_report() call" directly, rather than only inferring it from
|
||||
* the resulting job-table end state -- job.c's own idempotent fact guards
|
||||
* would make even a buggy once-per-page implementation produce a correct-
|
||||
* looking end state on a happy-path report with no round change
|
||||
* interleaved between pages (the exact class of placement/count bug task
|
||||
* 4.3's own round-1 review missed with a purely behavioral test -- see
|
||||
* tasks.md's Implementation Notes for task 4.3). */
|
||||
static _Atomic int g_resync_apply_report_call_count = 0;
|
||||
|
||||
int resync_apply_report_call_count_for_test(void)
|
||||
{
|
||||
return atomic_load_explicit(&g_resync_apply_report_call_count,
|
||||
memory_order_relaxed);
|
||||
}
|
||||
|
||||
void resync_apply_report_call_count_reset_for_test(void)
|
||||
{
|
||||
atomic_store_explicit(&g_resync_apply_report_call_count, 0,
|
||||
memory_order_relaxed);
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Frees w->report_accum (if any, casting back from the opaque void* xce/
|
||||
* types.h stores it as) and NULLs it. Caller holds w's write lock.
|
||||
* Idempotent -- a no-op when already NULL, safe to call unconditionally
|
||||
* (mirrors resync_resolve_round's own idempotence for the sibling
|
||||
* w->resync field). */
|
||||
static void resync_report_accum_reset(worker_t* w)
|
||||
{
|
||||
resync_report_accum_t* accum = (resync_report_accum_t*)w->report_accum;
|
||||
int i;
|
||||
|
||||
if (accum == NULL)
|
||||
return;
|
||||
|
||||
for (i = 0; i < accum->n_total; i++)
|
||||
free(accum->entries[i]);
|
||||
free(accum->entries);
|
||||
free(accum);
|
||||
|
||||
w->report_accum = NULL;
|
||||
}
|
||||
|
||||
int resync_handle_report_page(cluster_t* c, worker_t* w,
|
||||
const xce_job_state_report_t* page)
|
||||
{
|
||||
resync_report_accum_t* accum;
|
||||
int rc;
|
||||
int i;
|
||||
|
||||
if (c == NULL || w == NULL || page == NULL)
|
||||
return -EINVAL;
|
||||
|
||||
if (page->pos < 0 || page->count < 0)
|
||||
return -EINVAL;
|
||||
|
||||
lock(w, RTTI_LOCK_WRITE);
|
||||
|
||||
if (page->pos == 0) {
|
||||
/* Fresh report starting (or restarting over a stale/incomplete
|
||||
* prior accumulation -- e.g. this worker disconnected mid-report
|
||||
* on an earlier round) -- always reset first, mirroring xcecd_
|
||||
* resync_handle_query_page()'s identical pos==0 convention. */
|
||||
resync_report_accum_reset(w);
|
||||
|
||||
accum = (resync_report_accum_t*)calloc(1, sizeof(*accum));
|
||||
if (accum == NULL) {
|
||||
unlock(w);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
/* Req 6.1, 6.3: captured NOW, from whatever round is current the
|
||||
* instant this report starts arriving -- not re-read later at
|
||||
* apply time. See resync_report_accum_t's own doc comment
|
||||
* (resync.h) for why this specific capture point is load-bearing,
|
||||
* not an arbitrary choice. */
|
||||
accum->round_token = (w->resync != NULL) ? w->resync->round_token : 0;
|
||||
accum->n_total = page->n_total;
|
||||
|
||||
if (page->n_total > 0) {
|
||||
accum->entries = (xce_job_state_entry_t**)calloc(
|
||||
(size_t)page->n_total, sizeof(*accum->entries));
|
||||
if (accum->entries == NULL) {
|
||||
free(accum);
|
||||
unlock(w);
|
||||
return -ENOMEM;
|
||||
}
|
||||
}
|
||||
|
||||
w->report_accum = accum;
|
||||
}
|
||||
|
||||
accum = (resync_report_accum_t*)w->report_accum;
|
||||
|
||||
if (page->count > 0) {
|
||||
if (accum == NULL || page->entries == NULL
|
||||
|| (page->pos + page->count) > accum->n_total) {
|
||||
resync_report_accum_reset(w);
|
||||
unlock(w);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
/* Deep-copy each entry (plain struct-value copy -- xce_job_state_
|
||||
* entry_t has no nested heap pointers): page->entries[i] is owned
|
||||
* by THIS page's own decoded xce_request_t and is freed by xce_
|
||||
* free_request() the moment the dispatch handler that decoded it
|
||||
* returns, long before a later page (let alone the final one) has
|
||||
* even arrived -- a borrowed pointer would be a use-after-free the
|
||||
* instant this function returns 0 awaiting more pages. */
|
||||
for (i = 0; i < page->count; i++) {
|
||||
xce_job_state_entry_t* src = page->entries[i];
|
||||
xce_job_state_entry_t* copy;
|
||||
|
||||
if (src == NULL) {
|
||||
resync_report_accum_reset(w);
|
||||
unlock(w);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
copy = (xce_job_state_entry_t*)malloc(sizeof(*copy));
|
||||
if (copy == NULL) {
|
||||
resync_report_accum_reset(w);
|
||||
unlock(w);
|
||||
return -ENOMEM;
|
||||
}
|
||||
*copy = *src;
|
||||
|
||||
accum->entries[page->pos + i] = copy;
|
||||
}
|
||||
|
||||
accum->n_received += page->count;
|
||||
} else if (accum == NULL) {
|
||||
/* An orphaned continuation page (pos != 0) with no accumulation in
|
||||
* progress at all (its own preceding pos==0 page never arrived, or
|
||||
* arrived and was already discarded/completed) -- nothing to
|
||||
* accumulate into and nothing that will ever complete; reject
|
||||
* rather than silently waiting forever. */
|
||||
unlock(w);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (accum->n_received < accum->n_total) {
|
||||
/* More pages still expected -- gate stays set (via w->report_accum
|
||||
* remaining non-NULL), nothing applied yet. */
|
||||
unlock(w);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Last page: the report is fully assembled. Detach ownership from `w`
|
||||
* and release w's lock BEFORE calling resync_apply_report() -- that
|
||||
* function manages its own w write-locking internally (resync.h's own
|
||||
* doc comment) and must never be called with w's lock already held, or
|
||||
* its own lock(w, RTTI_LOCK_WRITE) call would deadlock against this
|
||||
* one. */
|
||||
w->report_accum = NULL;
|
||||
unlock(w);
|
||||
|
||||
{
|
||||
xce_job_state_report_t report;
|
||||
|
||||
memset(&report, 0, sizeof(report));
|
||||
report.n_total = accum->n_total;
|
||||
report.pos = 0;
|
||||
report.count = accum->n_total;
|
||||
report.entries = accum->entries;
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
atomic_fetch_add_explicit(&g_resync_apply_report_call_count, 1,
|
||||
memory_order_relaxed);
|
||||
#endif
|
||||
rc = resync_apply_report(c, w, accum->round_token, &report);
|
||||
}
|
||||
|
||||
for (i = 0; i < accum->n_total; i++)
|
||||
free(accum->entries[i]);
|
||||
free(accum->entries);
|
||||
free(accum);
|
||||
|
||||
return (rc < 0) ? rc : 1;
|
||||
}
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
/* worker-rejoin-resync task 3.1 test seam (resync.h): builds a
|
||||
* resync_timeout_ctx_t exactly as resync_arm_round's own delay_task_ms call
|
||||
* would have (heap-allocated, since resync_on_timeout unconditionally
|
||||
* free()s it -- matching that exact contract here, not a stack copy, or
|
||||
* the free() below would be undefined behavior) and invokes resync_on_
|
||||
* timeout directly, so a test can treat "the timeout fired" as a plain,
|
||||
* synchronous function call -- no real delaymgr thread, no wall-clock
|
||||
* wait. */
|
||||
void resync_test_fire_timeout(cluster_t* c, worker_t* w, uint64_t token)
|
||||
{
|
||||
resync_timeout_ctx_t* ctx
|
||||
= (resync_timeout_ctx_t*)calloc(1, sizeof(*ctx));
|
||||
|
||||
if (ctx == NULL)
|
||||
return;
|
||||
|
||||
ctx->c = c;
|
||||
ctx->w = w;
|
||||
ctx->round_token = token;
|
||||
ctx->timeout_ms = 0;
|
||||
|
||||
resync_on_timeout(ctx);
|
||||
}
|
||||
#endif
|
||||
|
|
@ -378,7 +378,30 @@ static int sched_job_start_fun(int type, int situation, void* data, va_list ap)
|
|||
return rc;
|
||||
}
|
||||
|
||||
static int sched_job_exit_fun(int type, int situation, void* data, va_list ap)
|
||||
/*
|
||||
* sched_job_finish_fun (bug-1737, renamed from sched_job_exit_fun) --
|
||||
* subscribes to HOOK_DETAIL_JOB_EXIT, which despite its name fires for
|
||||
* EVERY job finish, both success and failure (record_job_finish_fact(),
|
||||
* job.c, decides DONE vs EXIT from finish->exit_code BEFORE firing this
|
||||
* same hook detail either way -- mirrors persist.c's own WAL_JOB_FINISH ->
|
||||
* HOOK_DETAIL_JOB_EXIT mapping, `case HOOK_DETAIL_JOB_EXIT: return
|
||||
* WAL_JOB_FINISH;`). This function previously discarded info->exit_code and
|
||||
* sent only the bare jobid, forcing sched_handle_notify() (scheduler/src/
|
||||
* handler.c) to unconditionally map EVERY finish -- including a clean,
|
||||
* successful one -- to JOB_STAT_EXIT. Under normal conditions a subsequent
|
||||
* HOOK_DETAIL_JOB_UPDATE fire (persist.c, same WAL apply) self-corrected
|
||||
* this moments later via schedUpdateJob's full xce_job_t (whose ->state is
|
||||
* already the real terminal value), but if THAT second notify was dropped
|
||||
* (c->sched.ep briefly NULL during a reconnect -- the same window bug-1731/
|
||||
* bug-1734 already documented) the scheduler was left permanently stuck
|
||||
* showing EXIT for a job the controller correctly recorded as DONE. Fix:
|
||||
* carry info (jobid + exit_code) across the wire via the existing
|
||||
* JobFinishInfo wire shape (same struct apiserver_job_exit_fun,
|
||||
* apiserver.c, already reuses for this exact hook) so
|
||||
* sched_handle_notify() can make the identical exit_code==0-vs-not decision
|
||||
* job.c itself makes, instead of guessing EXIT unconditionally.
|
||||
*/
|
||||
static int sched_job_finish_fun(int type, int situation, void* data, va_list ap)
|
||||
{
|
||||
cluster_t* c = va_arg(ap, cluster_t*);
|
||||
xce_jobfinish_t* info = va_arg(ap, xce_jobfinish_t*);
|
||||
|
|
@ -387,7 +410,7 @@ static int sched_job_exit_fun(int type, int situation, void* data, va_list ap)
|
|||
xce_notify_data_t xdata = {
|
||||
.event = NotifyData_data_schedExitJob,
|
||||
.data = {
|
||||
.sched_exit_job = info->jobid
|
||||
.sched_exit_job = info
|
||||
}
|
||||
};
|
||||
/* clang-format on */
|
||||
|
|
@ -408,6 +431,22 @@ static int sched_job_exit_fun(int type, int situation, void* data, va_list ap)
|
|||
return rc;
|
||||
}
|
||||
|
||||
/*
|
||||
* sched_job_fail_fun -- subscribes to HOOK_DETAIL_JOB_FAIL, a narrower event
|
||||
* than sched_job_finish_fun's HOOK_DETAIL_JOB_EXIT above: it fires for a
|
||||
* PIPELINE-stage failure (dispatch/RPC error, or a post-exec script
|
||||
* failing), not for the job's own exit code. apply_job_fail_transition()
|
||||
* (job.c) only forces JOB_STAT_EXIT for the `default:` case (a genuinely
|
||||
* pre-exit failure with no exit info yet); for JOB_EXEC_STAGE_POST_EXEC/
|
||||
* Q_POST_EXEC it MERGES a JOB_STAT_PERR flag onto whatever DONE/EXIT the
|
||||
* job's own finish already decided, never re-deciding DONE vs EXIT. This
|
||||
* function still always sends the same bare-jobid "treat as EXIT" notify
|
||||
* (matching the `default:` case only) -- bug-1738's
|
||||
* job_state_regresses_terminal() guard in core_events.c's
|
||||
* XCE_CORE_EVENT_JOB_STATE_CHANGE is what stops that from clobbering an
|
||||
* already-DONE job for the POST_EXEC case, since the scheduler has no
|
||||
* JOB_STAT_PERR-equivalent concept of its own to merge instead.
|
||||
*/
|
||||
static int sched_job_fail_fun(int type, int situation, void* data, va_list ap)
|
||||
{
|
||||
cluster_t* c = va_arg(ap, cluster_t*);
|
||||
|
|
@ -526,7 +565,7 @@ static int sched_job_hook_fun(int type, int situation, int detail, void* data,
|
|||
case HOOK_DETAIL_JOB_START:
|
||||
return sched_job_start_fun(type, situation, data, ap);
|
||||
case HOOK_DETAIL_JOB_EXIT:
|
||||
return sched_job_exit_fun(type, situation, data, ap);
|
||||
return sched_job_finish_fun(type, situation, data, ap);
|
||||
case HOOK_DETAIL_JOB_UPDATE:
|
||||
return sched_job_update_fun(type, situation, data, ap);
|
||||
case HOOK_DETAIL_JOB_FAIL:
|
||||
|
|
|
|||
|
|
@ -103,12 +103,14 @@ extern void test_upgrade_snapshot_restore(void);
|
|||
extern void test_upgrade_snapshot_apply(void);
|
||||
extern void test_bug857_regression(void);
|
||||
extern void test_sched_admin_dispatch(void);
|
||||
extern void test_jobstatereport_dispatch(void);
|
||||
extern void test_fetchfile_disconnect(void);
|
||||
extern void test_job_dispatch(void);
|
||||
extern void test_dispatch_commit(void);
|
||||
extern void test_dispatch_fail_terminal(void);
|
||||
extern void test_dispatch_fail_retry(void);
|
||||
extern void test_jobdispatch_no_double_notify(void);
|
||||
extern void test_worker_resync(void);
|
||||
|
||||
void setUp(void) { return; }
|
||||
|
||||
|
|
@ -281,12 +283,14 @@ int main(int argc, char* argv[])
|
|||
TEST(test_upgrade_snapshot_apply);
|
||||
TEST(test_bug857_regression);
|
||||
TEST(test_sched_admin_dispatch);
|
||||
TEST(test_jobstatereport_dispatch);
|
||||
TEST(test_fetchfile_disconnect);
|
||||
TEST(test_job_dispatch);
|
||||
TEST(test_dispatch_commit);
|
||||
TEST(test_dispatch_fail_terminal);
|
||||
TEST(test_dispatch_fail_retry);
|
||||
TEST(test_jobdispatch_no_double_notify);
|
||||
TEST(test_worker_resync);
|
||||
|
||||
return UNITY_END();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -982,6 +982,37 @@ static void test_gate_no_quorum_admits_clrpreason_fact(void)
|
|||
printf("test_gate_no_quorum_admits_clrpreason_fact: PASS\n");
|
||||
}
|
||||
|
||||
/* ── OPCode_jobStateReport admitted without write quorum ───────────────────
|
||||
* (worker-rejoin-resync task 6.2, found live while auditing this exact gate
|
||||
* for task 6.2's own standby-redirect investigation -- design.md only
|
||||
* anticipated the STANDBY case, per bug-1700's precedent, and did not
|
||||
* separately consider a PRIMARY that has temporarily lost write quorum.
|
||||
* OPCode_jobStateReport is a worker's already-happened job-state fact, sent
|
||||
* fire-and-forget (xce_job_state_report_req(), request.c) with no
|
||||
* redelivery on rejection -- exactly the same shape as OPCode_notify, which
|
||||
* this same allowlist already exists to protect from bug-396's silent-loss
|
||||
* failure mode. Rejecting a reconciliation report outright during a
|
||||
* transient quorum-loss window would reproduce that exact loss mode for
|
||||
* the specific facts this whole feature exists to recover (bug-1741). */
|
||||
static void test_gate_no_quorum_admits_jobstatereport_fact(void)
|
||||
{
|
||||
int rc;
|
||||
|
||||
/* Arrange: quorum absent */
|
||||
fact_gate_setup(3, 0);
|
||||
|
||||
/* Act */
|
||||
rc = ha_dispatch_gate(&g_fg_ha, OPCode_jobStateReport);
|
||||
|
||||
/* Assert */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
|
||||
"OPCode_jobStateReport (worker reconciliation fact) must be "
|
||||
"admitted without write quorum, exactly like OPCode_notify");
|
||||
|
||||
fact_gate_teardown();
|
||||
printf("test_gate_no_quorum_admits_jobstatereport_fact: PASS\n");
|
||||
}
|
||||
|
||||
/* ── PIN: request-shaped opcodes stay rejected without quorum (Req 3.2) ───
|
||||
* The allowlist is FACTS ONLY: every request-shaped opcode (a requester
|
||||
* awaits confirmation, served by the task-3 handler barrier) must keep
|
||||
|
|
@ -1018,16 +1049,17 @@ static void test_gate_no_quorum_still_rejects_request_opcodes(void)
|
|||
static void test_gate_with_quorum_admits_all_opcodes(void)
|
||||
{
|
||||
static const struct { int opcode; const char *name; } ops[] = {
|
||||
{ OPCode_notify, "OPCode_notify" },
|
||||
{ OPCode_update, "OPCode_update" },
|
||||
{ OPCode_setpreason, "OPCode_setpreason" },
|
||||
{ OPCode_clrpreason, "OPCode_clrpreason" },
|
||||
{ OPCode_jobsub, "OPCode_jobsub" },
|
||||
{ OPCode_jobcontrol, "OPCode_jobcontrol" },
|
||||
{ OPCode_queuecontrol, "OPCode_queuecontrol" },
|
||||
{ OPCode_hostcontrol, "OPCode_hostcontrol" },
|
||||
{ OPCode_jobdispatch, "OPCode_jobdispatch" },
|
||||
{ OPCode_join, "OPCode_join" },
|
||||
{ OPCode_notify, "OPCode_notify" },
|
||||
{ OPCode_update, "OPCode_update" },
|
||||
{ OPCode_setpreason, "OPCode_setpreason" },
|
||||
{ OPCode_clrpreason, "OPCode_clrpreason" },
|
||||
{ OPCode_jobStateReport, "OPCode_jobStateReport" },
|
||||
{ OPCode_jobsub, "OPCode_jobsub" },
|
||||
{ OPCode_jobcontrol, "OPCode_jobcontrol" },
|
||||
{ OPCode_queuecontrol, "OPCode_queuecontrol" },
|
||||
{ OPCode_hostcontrol, "OPCode_hostcontrol" },
|
||||
{ OPCode_jobdispatch, "OPCode_jobdispatch" },
|
||||
{ OPCode_join, "OPCode_join" },
|
||||
};
|
||||
size_t i;
|
||||
|
||||
|
|
@ -2459,6 +2491,7 @@ void test_commit_barrier_rollout(void)
|
|||
RUN_TEST(test_gate_no_quorum_admits_update_fact);
|
||||
RUN_TEST(test_gate_no_quorum_admits_setpreason_fact);
|
||||
RUN_TEST(test_gate_no_quorum_admits_clrpreason_fact);
|
||||
RUN_TEST(test_gate_no_quorum_admits_jobstatereport_fact);
|
||||
RUN_TEST(test_gate_no_quorum_still_rejects_request_opcodes);
|
||||
RUN_TEST(test_gate_with_quorum_admits_all_opcodes);
|
||||
/* Task 5.1 — config barrier: configAck arithmetic + upgrade failure
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
/* test_dispatch_fail_retry.c -- scheduler-dispatch-commit task 3.3
|
||||
/* test_dispatch_fail_retry.c -- scheduler-dispatch-commit task 3.3;
|
||||
* extended by worker-rejoin-resync task 2.3
|
||||
*
|
||||
* Proves dispatch_a_job_() (job.c) now RESOLVES a retryable (transient/
|
||||
* environmental) dispatch failure -- incrementing the consecutive-failure
|
||||
|
|
@ -19,6 +20,25 @@
|
|||
* - XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY (E_XCE_BAD_WORKER)
|
||||
* - XCE_DISPATCH_FAIL_SITE_RPC_FAILURE (xce_startjob_req() rc<0)
|
||||
*
|
||||
* worker-rejoin-resync task 2.3 (Req 5.1, 5.4) split dispatch_a_job_()'s
|
||||
* combined "(w->ep == NULL) || (w->host.status != 0)" check into two
|
||||
* branches: a disconnected worker (w->ep == NULL) is now resolved
|
||||
* separately and DIFFERENTLY from XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY
|
||||
* above -- it bypasses xce_classify_dispatch_fail()/mark_job_fail()
|
||||
* entirely and returns the job to PEND immediately via
|
||||
* record_job_reconcile_pending_fact() (task 2.1), with no
|
||||
* dispatch_fail_count increment and no retry. This is NOT one of the four
|
||||
* RETRYABLE sites this file otherwise covers -- it is added here anyway
|
||||
* (rather than to test_worker_resync.c) because it is fundamentally a
|
||||
* dispatch_a_job_() behavior/regression test exercising the same
|
||||
* combined-check split site SITE 3 below already covers, sharing this
|
||||
* file's fixture/canary/notify-capture machinery. See the new tests below
|
||||
* SITE 3. SITE 3 itself (XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY,
|
||||
* w->host.status != 0) is updated to use a CONNECTED worker (w->ep != NULL)
|
||||
* with a nonzero status bit -- it previously used an ep-left-NULL worker to
|
||||
* reach the combined check, which now takes the new, separate branch
|
||||
* instead.
|
||||
*
|
||||
* test_worker_notfound_resolves_retryable (below) seeds an unregistered
|
||||
* worker key, matching test_dispatch_commit.c's own established fixture
|
||||
* for the "worker not found" scenario -- that file's own header notes this
|
||||
|
|
@ -253,6 +273,28 @@ static void dfr_assert_ref_not_over_released(job_t* j, uint64_t expect_jobid)
|
|||
"heap-use-after-free");
|
||||
}
|
||||
|
||||
/* worker-rejoin-resync task 2.3: same canary technique as above, applied to
|
||||
* `w` -- the new disconnected-worker branch dereference(w)s the reference
|
||||
* dispatch_a_job_() itself took at its key_find() lookup, and this is the
|
||||
* exact class of reference bug bug-1627 fixed (leak site #3, the WORKER_
|
||||
* NOT_READY/ep-NULL combined check). */
|
||||
static void dfr_arm_worker_canary(worker_t* w)
|
||||
{
|
||||
reference(w);
|
||||
}
|
||||
|
||||
static void dfr_assert_worker_ref_not_over_released(worker_t* w,
|
||||
const char* expect_name)
|
||||
{
|
||||
dereference(w);
|
||||
|
||||
TEST_ASSERT_EQUAL_STRING_MESSAGE(expect_name, w->host.name,
|
||||
"w must still be safely readable after releasing the canary "
|
||||
"reference -- an over-release inside dispatch_a_job_test() would "
|
||||
"have freed w while the canary was still held, making this a "
|
||||
"heap-use-after-free");
|
||||
}
|
||||
|
||||
/* A "broken" endpoint: real socket fd, real iomgr, deliberately NOT
|
||||
* registered in the iomgr's epoll -- EPOLL_CTL_MOD on it fails with ENOENT
|
||||
* (the exact post-failed-reconnect condition endpoint_update_epoll()
|
||||
|
|
@ -368,13 +410,25 @@ static void test_worker_notfound_resolves_retryable(void)
|
|||
}
|
||||
|
||||
/* ── SITE 3: XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY ───────────────────
|
||||
* A registered worker whose ep is left NULL (calloc default) hits
|
||||
* "(w->ep == NULL) || (w->host.status != 0)". */
|
||||
* A registered, CONNECTED worker (w->ep != NULL) with a nonzero
|
||||
* w->host.status bit (XCE_WORKER_BUSY, picked arbitrarily -- any bit takes
|
||||
* the same unchanged path) hits "w->host.status != 0" -- the still-
|
||||
* unmodified branch of dispatch_a_job_()'s split combined check
|
||||
* (worker-rejoin-resync task 2.3). Before that split, this test used an
|
||||
* ep-left-NULL worker to reach the combined check; that condition now
|
||||
* takes the NEW, separate disconnected-worker branch instead (see the new
|
||||
* tests below), which resolves differently (immediate PEND, no retry, no
|
||||
* dispatch_fail_count increment) -- so this test is updated to use a
|
||||
* connected-but-busy worker, keeping it a genuine, unchanged regression
|
||||
* proof for XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY's own retry-then-
|
||||
* terminal-fail behavior (Req 5.1's design.md backstop). */
|
||||
#define DFR_JOBID_WORKER_NOTREADY 91202ULL
|
||||
|
||||
static void test_worker_notready_resolves_retryable(void)
|
||||
{
|
||||
job_t* j;
|
||||
worker_t* w = NULL;
|
||||
endpoint_t connected_ep;
|
||||
xce_job_alloc_t* alloc;
|
||||
int rc;
|
||||
int before_count;
|
||||
|
|
@ -382,7 +436,18 @@ static void test_worker_notready_resolves_retryable(void)
|
|||
dfr_setup();
|
||||
|
||||
j = dfr_seed_job(DFR_JOBID_WORKER_NOTREADY, "dfr-q2");
|
||||
dfr_seed_worker("dfr-wkr-notready", NULL); /* ep left NULL: not ready */
|
||||
|
||||
/* A non-NULL ep (its contents are never touched -- this branch fails
|
||||
* before any RPC is attempted) plus a nonzero status bit: connected,
|
||||
* but not ready. */
|
||||
memset(&connected_ep, 0, sizeof(connected_ep));
|
||||
dfr_seed_worker("dfr-wkr-notready", &connected_ep);
|
||||
|
||||
lock(g_dfr.workermgr, RTTI_LOCK_READ);
|
||||
key_find(g_dfr.workermgr, (void*)"dfr-wkr-notready", (void**)&w);
|
||||
unlock(g_dfr.workermgr);
|
||||
TEST_ASSERT_NOT_NULL(w);
|
||||
w->host.status = XCE_WORKER_BUSY;
|
||||
|
||||
alloc = make_job_alloc(DFR_JOBID_WORKER_NOTREADY, 1,
|
||||
make_host_allocs(1,
|
||||
|
|
@ -391,6 +456,13 @@ static void test_worker_notready_resolves_retryable(void)
|
|||
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
|
||||
TEST_ASSERT_NOT_NULL(alloc);
|
||||
|
||||
/* worker-rejoin-resync task 2.3: explicit proof this test still reaches
|
||||
* XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY's branch of the split check --
|
||||
* a connected (non-NULL ep), nonzero-status worker, not the new
|
||||
* disconnected-worker branch below. */
|
||||
TEST_ASSERT_NOT_NULL(w->ep);
|
||||
TEST_ASSERT_TRUE(w->host.status != 0);
|
||||
|
||||
cbrp_notify_capture_reset_ext();
|
||||
before_count = cbrp_notify_capture_count_ext();
|
||||
|
||||
|
|
@ -407,7 +479,8 @@ static void test_worker_notready_resolves_retryable(void)
|
|||
"Req 4.4: no partial allocation data may be left behind");
|
||||
TEST_ASSERT_EQUAL_UINT32_MESSAGE(1, j->dispatch_fail_count,
|
||||
"Req 4.3: the consecutive-failure counter must increment by "
|
||||
"exactly 1");
|
||||
"exactly 1 -- unchanged existing WORKER_NOT_READY retry behavior "
|
||||
"(worker-rejoin-resync task 2.3 must not affect this branch)");
|
||||
|
||||
dfr_assert_ref_not_over_released(j, DFR_JOBID_WORKER_NOTREADY);
|
||||
|
||||
|
|
@ -426,6 +499,250 @@ static void test_worker_notready_resolves_retryable(void)
|
|||
printf("test_worker_notready_resolves_retryable: PASS\n");
|
||||
}
|
||||
|
||||
/* ── worker-rejoin-resync task 2.3: disconnected-worker branch ─────────
|
||||
* (Req 5.1, 5.4) -- NOT one of the four RETRYABLE sites above: a dispatch
|
||||
* attempt to a worker the controller is not currently connected to
|
||||
* (w->ep == NULL) now fails immediately and returns the job to PEND via
|
||||
* record_job_reconcile_pending_fact(), bypassing xce_classify_dispatch_
|
||||
* fail()/mark_job_fail() entirely -- no dispatch_fail_count increment, no
|
||||
* retry, no terminal resolution.
|
||||
*
|
||||
* Realistic pre-dispatch state: dispatch_a_job_()'s only real caller
|
||||
* dispatches a job the scheduler picked from its PEND pool, so this test
|
||||
* seeds an ordinary PEND job (dfr_seed_job's default) -- record_job_
|
||||
* reconcile_pending_fact()'s apply_job_update_transition(job, JOB_STAT_
|
||||
* PEND) call therefore hits JOB_TRANSITION_NOOP_DUPLICATE (job already
|
||||
* PEND), not APPLIED: no mutation, no allocs touch (there is none to
|
||||
* release), no durable fact, no notify -- exactly as design.md's own
|
||||
* Components section describes for this outcome, and as this task's own
|
||||
* prompt anticipates ("handle this correctly: still don't call mark_job_
|
||||
* fail, still don't retry, but there's nothing to release/notify either
|
||||
* since nothing changed"). */
|
||||
#define DFR_JOBID_DISCONNECTED 91206ULL
|
||||
|
||||
static int g_dfr_update_hook_fires;
|
||||
|
||||
static int dfr_job_update_spy_hook(int type, int situation, int detail,
|
||||
void* data, va_list ap)
|
||||
{
|
||||
(void)type; (void)data; (void)ap;
|
||||
if (situation == HOOK_AFTER_EVENT && detail == HOOK_DETAIL_JOB_UPDATE)
|
||||
g_dfr_update_hook_fires++;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void test_disconnected_worker_fails_immediately_no_retry(void)
|
||||
{
|
||||
job_t* j;
|
||||
worker_t* w = NULL;
|
||||
xce_job_alloc_t* alloc;
|
||||
int rc;
|
||||
int before_count;
|
||||
uint32_t fail_count_before;
|
||||
|
||||
dfr_setup();
|
||||
g_dfr_update_hook_fires = 0;
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
xce_add_hook(&g_dfr.hooksys, XCE_CONTROLLER_HOOK_JOB,
|
||||
dfr_job_update_spy_hook, NULL));
|
||||
|
||||
j = dfr_seed_job(DFR_JOBID_DISCONNECTED, "dfr-q-disc");
|
||||
fail_count_before = j->dispatch_fail_count;
|
||||
|
||||
dfr_seed_worker("dfr-wkr-disc1", NULL); /* ep left NULL: disconnected */
|
||||
|
||||
lock(g_dfr.workermgr, RTTI_LOCK_READ);
|
||||
key_find(g_dfr.workermgr, (void*)"dfr-wkr-disc1", (void**)&w);
|
||||
unlock(g_dfr.workermgr);
|
||||
TEST_ASSERT_NOT_NULL(w);
|
||||
TEST_ASSERT_NULL_MESSAGE(w->ep, "the disconnected branch requires "
|
||||
"w->ep == NULL -- confirm the fixture actually constructs that");
|
||||
|
||||
alloc = make_job_alloc(DFR_JOBID_DISCONNECTED, 1,
|
||||
make_host_allocs(1,
|
||||
make_host_alloc((char*)"dfr-wkr-disc1", 1,
|
||||
make_allocs(1,
|
||||
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
|
||||
TEST_ASSERT_NOT_NULL(alloc);
|
||||
|
||||
cbrp_notify_capture_reset_ext();
|
||||
before_count = cbrp_notify_capture_count_ext();
|
||||
|
||||
/* No reference/leak regression (task 2.3's own Tests-to-write clause):
|
||||
* canary both j (bug-1627's original leak class) and w (this branch's
|
||||
* own dereference(w), the same site bug-1627 fixed for the WORKER_NOT_
|
||||
* READY branch). */
|
||||
dfr_arm_ref_canary(j);
|
||||
dfr_arm_worker_canary(w);
|
||||
|
||||
rc = dispatch_a_job_test(&g_dfr, NULL, alloc);
|
||||
TEST_ASSERT_TRUE_MESSAGE(rc < 0,
|
||||
"a dispatch to a disconnected worker must still fail");
|
||||
|
||||
/* Observable completion (Req 5.4): job ends up PEND (unchanged -- it
|
||||
* started PEND, a realistic pre-dispatch state), single attempt, no
|
||||
* retry loop. */
|
||||
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state,
|
||||
"Req 5.4: a dispatch to a disconnected worker must return the job "
|
||||
"to pending (here: leave it PEND, since it started PEND)");
|
||||
TEST_ASSERT_NOT_EQUAL_MESSAGE(JOB_STAT_EXIT, j->job.state,
|
||||
"Req 5.4: this must NOT become a permanent/terminal failure -- "
|
||||
"mark_job_fail() must never be invoked on this path");
|
||||
TEST_ASSERT_NULL_MESSAGE(j->job.allocs,
|
||||
"no partial allocation data may be left behind");
|
||||
TEST_ASSERT_EQUAL_UINT32_MESSAGE(fail_count_before, j->dispatch_fail_count,
|
||||
"Req 5.4: dispatch_fail_count must be left UNCHANGED -- this "
|
||||
"branch must never reach xce_classify_dispatch_fail()'s retry-"
|
||||
"counting machinery");
|
||||
|
||||
/* "single attempt, no retry loop": notify_dispatch() is dispatch_a_
|
||||
* job_()'s sole retry-signaling mechanism (see the four RETRYABLE-site
|
||||
* tests above) -- it must never fire on this path. */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(before_count, cbrp_notify_capture_count_ext(),
|
||||
"Req 5.4: a disconnected-worker dispatch failure must never call "
|
||||
"notify_dispatch() -- there is no retry to signal");
|
||||
|
||||
/* NOOP_DUPLICATE outcome (job was already PEND): no mutation occurred,
|
||||
* so no notify is needed or expected either -- omitting the notify
|
||||
* here is correct, not bug-1679's omission-is-a-bug case (that applies
|
||||
* only to an actual APPLIED transition; see the APPLIED-path test
|
||||
* below for that proof). */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_dfr_update_hook_fires,
|
||||
"a no-op reconciliation (job already PEND) must not fire the "
|
||||
"scheduler notify path -- nothing changed");
|
||||
|
||||
dfr_assert_ref_not_over_released(j, DFR_JOBID_DISCONNECTED);
|
||||
dfr_assert_worker_ref_not_over_released(w, "dfr-wkr-disc1");
|
||||
|
||||
xce_disable_hook(&g_dfr.hooksys, XCE_CONTROLLER_HOOK_JOB,
|
||||
dfr_job_update_spy_hook);
|
||||
free_JobAlloc_ptr(&alloc);
|
||||
dfr_teardown();
|
||||
printf("test_disconnected_worker_fails_immediately_no_retry: PASS\n");
|
||||
}
|
||||
|
||||
/* ── worker-rejoin-resync task 2.3: JOB_TRANSITION_APPLIED wiring proof
|
||||
* ────────────────────────────────────────────────────────────────────
|
||||
* The NOOP_DUPLICATE test above exercises the only precondition dispatch_
|
||||
* a_job_()'s real caller can currently construct (a PEND job -- the
|
||||
* scheduler only ever dispatches from its PEND pool, so dispatch_a_job_()
|
||||
* is never actually invoked on a RUN job today; the same "not reachable
|
||||
* via any real caller in the currently-implemented system" caveat
|
||||
* test_rpc_failure_bound_resets_after_intervening_success's header
|
||||
* documents for the four RETRYABLE sites applies here too -- see bug-1631).
|
||||
* This test instead directly seeds a RUN job with a real, committed
|
||||
* job->job.allocs (mirroring test_worker_resync.c's own technique) to
|
||||
* exercise record_job_reconcile_pending_fact()'s OTHER outcome
|
||||
* (JOB_TRANSITION_APPLIED, not NOOP_DUPLICATE) THROUGH this exact new
|
||||
* dispatch_a_job_() call site -- proving the branch's result-handling
|
||||
* (unlock -> notify-only-on-APPLIED -> dereference) is wired correctly,
|
||||
* complementing task 2.1's test_worker_resync.c (which already proves
|
||||
* record_job_reconcile_pending_fact() itself is correct in isolation, not
|
||||
* through this caller). Not a claim that THIS exact precondition (RUN job
|
||||
* hitting the disconnected branch) is reachable in production today --
|
||||
* design.md's own bug-1631 narrative is that record_job_reconcile_pending_
|
||||
* fact() releases allocs at the point of RECONCILIATION (a separate,
|
||||
* later-task call site, resync.c), so by the time any job reaches a fresh
|
||||
* dispatch_a_job_() call it is already PEND with allocs already NULL. */
|
||||
#define DFR_JOBID_DISCONNECTED_RUN 91207ULL
|
||||
|
||||
static uint64_t dfr_current_lsn(void)
|
||||
{
|
||||
uint64_t lsn;
|
||||
|
||||
pthread_mutex_lock(&g_dfr.persist.lock);
|
||||
lsn = g_dfr.persist.current_lsn;
|
||||
pthread_mutex_unlock(&g_dfr.persist.lock);
|
||||
return lsn;
|
||||
}
|
||||
|
||||
static void test_disconnected_worker_reconciles_run_job_releases_allocs(void)
|
||||
{
|
||||
job_t* j;
|
||||
worker_t* w = NULL;
|
||||
xce_job_alloc_t* prior_alloc;
|
||||
xce_job_alloc_t* redispatch_alloc;
|
||||
int rc;
|
||||
int before_count;
|
||||
uint64_t lsn_before, lsn_after;
|
||||
|
||||
dfr_setup();
|
||||
g_dfr_update_hook_fires = 0;
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
xce_add_hook(&g_dfr.hooksys, XCE_CONTROLLER_HOOK_JOB,
|
||||
dfr_job_update_spy_hook, NULL));
|
||||
|
||||
j = dfr_seed_job(DFR_JOBID_DISCONNECTED_RUN, "dfr-q-disc2");
|
||||
|
||||
/* Simulate a prior successful dispatch (mirrors test_worker_resync.c's
|
||||
* own technique): real allocs committed, state RUN, ownership
|
||||
* transferred into job->job.allocs. */
|
||||
prior_alloc = make_job_alloc(DFR_JOBID_DISCONNECTED_RUN, 1,
|
||||
make_host_allocs(1,
|
||||
make_host_alloc((char*)"dfr-wkr-disc2-prior", 1,
|
||||
make_allocs(1,
|
||||
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
|
||||
TEST_ASSERT_NOT_NULL(prior_alloc);
|
||||
j->job.state = JOB_STAT_RUN;
|
||||
j->job.allocs = prior_alloc;
|
||||
|
||||
dfr_seed_worker("dfr-wkr-disc2", NULL); /* ep left NULL: disconnected */
|
||||
|
||||
lock(g_dfr.workermgr, RTTI_LOCK_READ);
|
||||
key_find(g_dfr.workermgr, (void*)"dfr-wkr-disc2", (void**)&w);
|
||||
unlock(g_dfr.workermgr);
|
||||
TEST_ASSERT_NOT_NULL(w);
|
||||
|
||||
redispatch_alloc = make_job_alloc(DFR_JOBID_DISCONNECTED_RUN, 1,
|
||||
make_host_allocs(1,
|
||||
make_host_alloc((char*)"dfr-wkr-disc2", 1,
|
||||
make_allocs(1,
|
||||
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
|
||||
TEST_ASSERT_NOT_NULL(redispatch_alloc);
|
||||
|
||||
cbrp_notify_capture_reset_ext();
|
||||
before_count = cbrp_notify_capture_count_ext();
|
||||
lsn_before = dfr_current_lsn();
|
||||
|
||||
dfr_arm_ref_canary(j);
|
||||
dfr_arm_worker_canary(w);
|
||||
|
||||
rc = dispatch_a_job_test(&g_dfr, NULL, redispatch_alloc);
|
||||
TEST_ASSERT_TRUE_MESSAGE(rc < 0,
|
||||
"a dispatch to a disconnected worker must still fail");
|
||||
|
||||
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state,
|
||||
"Req 4.6/5.4: a RUN job redispatched to a disconnected worker must "
|
||||
"be returned to PEND");
|
||||
TEST_ASSERT_NULL_MESSAGE(j->job.allocs,
|
||||
"bug-1631: the job's prior resource allocation must be released, "
|
||||
"so a later redispatch never inherits a stale allocation");
|
||||
TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, j->dispatch_fail_count,
|
||||
"Req 5.4: dispatch_fail_count must stay untouched on this path");
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(before_count, cbrp_notify_capture_count_ext(),
|
||||
"notify_dispatch() (the retry-signal path) must never fire here");
|
||||
|
||||
lsn_after = dfr_current_lsn();
|
||||
TEST_ASSERT_TRUE_MESSAGE(lsn_after > lsn_before,
|
||||
"an APPLIED reconciliation must durably append a WAL_JOB_UPDATE "
|
||||
"fact");
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_dfr_update_hook_fires,
|
||||
"bug-1679: an APPLIED reconciliation must notify the scheduler "
|
||||
"exactly once, via this exact dispatch_a_job_() call site");
|
||||
|
||||
dfr_assert_ref_not_over_released(j, DFR_JOBID_DISCONNECTED_RUN);
|
||||
dfr_assert_worker_ref_not_over_released(w, "dfr-wkr-disc2");
|
||||
|
||||
xce_disable_hook(&g_dfr.hooksys, XCE_CONTROLLER_HOOK_JOB,
|
||||
dfr_job_update_spy_hook);
|
||||
free_JobAlloc_ptr(&redispatch_alloc);
|
||||
dfr_teardown();
|
||||
printf("test_disconnected_worker_reconciles_run_job_releases_allocs: "
|
||||
"PASS\n");
|
||||
}
|
||||
|
||||
/* ── SITE 4: XCE_DISPATCH_FAIL_SITE_RPC_FAILURE ────────────────────────
|
||||
* A ready worker whose ep is deliberately "broken" (see dfr_setup_broken_ep
|
||||
* above) makes xce_startjob_req() itself fail. */
|
||||
|
|
@ -814,4 +1131,10 @@ void test_dispatch_fail_retry(void)
|
|||
RUN_TEST(test_rpc_failure_resolves_retryable);
|
||||
RUN_TEST(test_rpc_failure_bounded_escalation);
|
||||
RUN_TEST(test_rpc_failure_bound_resets_after_intervening_success);
|
||||
|
||||
printf("\n>> dispatch_a_job_() disconnected-worker fail-fast tests "
|
||||
"(worker-rejoin-resync task 2.3, Req 5.1, 5.4)\n\n");
|
||||
|
||||
RUN_TEST(test_disconnected_worker_fails_immediately_no_retry);
|
||||
RUN_TEST(test_disconnected_worker_reconciles_run_job_releases_allocs);
|
||||
}
|
||||
|
|
|
|||
322
src/controller/testsuite/test_jobstatereport_dispatch.c
Normal file
322
src/controller/testsuite/test_jobstatereport_dispatch.c
Normal file
|
|
@ -0,0 +1,322 @@
|
|||
/*
|
||||
* test_jobstatereport_dispatch.c — Unity tests for the OPCode_jobStateReport
|
||||
* controller dispatch case (worker-rejoin-resync spec, task 6.2, Req 8.1,
|
||||
* 8.3).
|
||||
*
|
||||
* Exercises myreq_handler's OPCode_jobStateReport case through the real
|
||||
* decode+dispatch entry point exposed for tests, myreq_handler_for_test
|
||||
* (cluster.c, #ifdef XCE_TESTING) — same precedent test_sched_admin_
|
||||
* dispatch.c already established for a comparable primary-only opcode.
|
||||
*
|
||||
* Covers task 6.2's own explicit standby-redirect investigation: a report
|
||||
* (or, by the same generic mechanism, a query) sent to a non-primary
|
||||
* controller must receive the SAME redirect behavior an equivalent
|
||||
* existing primary-only message already receives, not a bare unrecoverable
|
||||
* error — proven here exactly as test_sched_admin_dispatch.c's own
|
||||
* test_schedadmin_standby_with_known_primary_redirects/test_schedadmin_
|
||||
* standby_no_known_primary_no_redirect prove it for OPCode_schedAdmin, the
|
||||
* "real existing primary-only opcode" ground truth this task's brief
|
||||
* calls for. OPCode_jobStateReport is deliberately absent from ha_
|
||||
* dispatch_gate()'s CTRL_STANDBY allow-list (ha.c) — exactly like OPCode_
|
||||
* schedAdmin/OPCode_upgradeConfig — so this redirect behavior requires NO
|
||||
* opcode-specific code in cluster.c's own dispatch case: it is entirely a
|
||||
* property of the GENERIC gate-rejection block that already runs ahead of
|
||||
* every opcode's switch case (cluster.c, the bug-1700 fix). These tests
|
||||
* prove that generic mechanism actually covers this opcode too, rather
|
||||
* than assuming it does.
|
||||
*
|
||||
* The OTHER half of task 6.2's own brief — multi-page reassembly, applied
|
||||
* exactly once, equivalent to a single-page report — is proven against a
|
||||
* REAL worker_t and REAL job table in test_worker_resync.c (resync_handle_
|
||||
* report_page()'s own direct tests), not re-proven here through the wire.
|
||||
* See this file's own comment further down (immediately before the RUN_
|
||||
* TEST list) for exactly why a "real wire dispatch resolves a real
|
||||
* worker_t" version of that proof cannot be built in THIS test binary, and
|
||||
* why that is a pre-existing, unrelated architectural property, not a gap
|
||||
* in this task's own coverage.
|
||||
*/
|
||||
#define _GNU_SOURCE
|
||||
#include <errno.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include <unity/unity.h>
|
||||
|
||||
#include "cluster.h"
|
||||
#include "controller.capnp.h"
|
||||
#include "ha.h"
|
||||
#include "netio.h"
|
||||
#include "request.h"
|
||||
#include "typemap.h"
|
||||
#include "xce/errnox.h"
|
||||
#include "xce/types.h"
|
||||
|
||||
/* myreq_handler exposed for tests (cluster.c) — same seam test_sched_admin_
|
||||
* dispatch.c uses. `pkt` ownership: consumed (freed) by the call. */
|
||||
extern int myreq_handler_for_test(endpoint_t* ep, io_packet_t* pkt);
|
||||
|
||||
/* ── Fixture ────────────────────────────────────────────────────────────
|
||||
*
|
||||
* A bare calloc'd cluster_t, exactly like test_sched_admin_dispatch.c's
|
||||
* sa_make_cluster() — sufficient for every test below, none of which
|
||||
* reaches past ha_dispatch_gate() or handle_jobstatereport()'s own first
|
||||
* line (get_ep_worker()).
|
||||
*/
|
||||
static cluster_t* jsr_make_gate_cluster(void)
|
||||
{
|
||||
cluster_t* c = calloc(1, sizeof(cluster_t));
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
mylock_init(&(c->sched.lock));
|
||||
return c;
|
||||
}
|
||||
|
||||
static void jsr_gate_teardown(cluster_t* c)
|
||||
{
|
||||
if (c == NULL)
|
||||
return;
|
||||
mylock_fini(&(c->sched.lock));
|
||||
free(c);
|
||||
}
|
||||
|
||||
/* Build a wire-encoded OPCode_jobStateReport request packet carrying a
|
||||
* minimal single-entry, single-page report — the exact content is
|
||||
* immaterial for the standby-redirect tests below (the request never
|
||||
* reaches the dispatch case at all). */
|
||||
static io_packet_t* jsr_build_request_pkt(uint64_t jobid)
|
||||
{
|
||||
xce_job_state_entry_t entry;
|
||||
xce_job_state_entry_t* entries[1];
|
||||
xce_job_state_report_t body;
|
||||
|
||||
memset(&entry, 0, sizeof(entry));
|
||||
entry.jobid = jobid;
|
||||
entry.status = JobStateStatus_running;
|
||||
entry.queried = 1;
|
||||
entries[0] = &entry;
|
||||
|
||||
memset(&body, 0, sizeof(body));
|
||||
body.n_total = 1;
|
||||
body.pos = 0;
|
||||
body.count = 1;
|
||||
body.entries = entries;
|
||||
|
||||
xce_request_t req;
|
||||
memset(&req, 0, sizeof(req));
|
||||
req.opcode = OPCode_jobStateReport;
|
||||
req.body_which = Request_body_jobstatereportmsg;
|
||||
req.body.jobstatereportmsg = &body;
|
||||
|
||||
io_packet_t* pkt = calloc(1, sizeof(*pkt));
|
||||
TEST_ASSERT_NOT_NULL(pkt);
|
||||
TEST_ASSERT_TRUE_MESSAGE(xce_encode(&req, pkt) >= 0,
|
||||
"failed to encode JobStateReport packet");
|
||||
return pkt;
|
||||
}
|
||||
|
||||
/* Pop the single reply packet myreq_handler queued on ep->outq (if any) and
|
||||
* decode it into *out. Mirrors test_sched_admin_dispatch.c's sa_decode_
|
||||
* queued_reply exactly. */
|
||||
static void jsr_decode_queued_reply(endpoint_t* ep, xce_request_t* out)
|
||||
{
|
||||
data_item_t* it = endpoint_deq(ep);
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(it, "expected a reply packet on ep->outq");
|
||||
|
||||
io_packet_t* pkt = ITPTR_TO_PACKET(it);
|
||||
memset(out, 0, sizeof(*out));
|
||||
int rc = xce_decode(pkt, out);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
}
|
||||
|
||||
/* ── Standby redirect (task 6.2's own explicit investigation) ─────────────
|
||||
*
|
||||
* Ground truth: OPCode_schedAdmin's identical fixture/assertions
|
||||
* (test_sched_admin_dispatch.c), the "real existing primary-only opcode"
|
||||
* this task's brief calls for. */
|
||||
|
||||
static void test_jobstatereport_rejects_when_not_primary(void)
|
||||
{
|
||||
cluster_t* c = jsr_make_gate_cluster();
|
||||
atomic_store(&c->ha.role, CTRL_STANDBY);
|
||||
|
||||
endpoint_t ep;
|
||||
memset(&ep, 0, sizeof(ep));
|
||||
ep.owner = c;
|
||||
|
||||
io_packet_t* pkt = jsr_build_request_pkt(1001);
|
||||
int rc = myreq_handler_for_test(&ep, pkt);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rc);
|
||||
|
||||
xce_request_t decoded;
|
||||
jsr_decode_queued_reply(&ep, &decoded);
|
||||
TEST_ASSERT_EQUAL_INT(Request_body_replymsg, decoded.body_which);
|
||||
TEST_ASSERT_NOT_NULL(decoded.body.replymsg);
|
||||
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, decoded.body.replymsg->status);
|
||||
TEST_ASSERT_EQUAL_INT(RequestReply_data_nodata,
|
||||
decoded.body.replymsg->data_which);
|
||||
xce_free_request(&decoded);
|
||||
|
||||
jsr_gate_teardown(c);
|
||||
printf("test_jobstatereport_rejects_when_not_primary: PASS\n");
|
||||
}
|
||||
|
||||
/* bug-1700 precedent: a standby dispatch-gate rejection with a KNOWN
|
||||
* primary must carry a redirect hint — proven here with NO opcode-specific
|
||||
* code added anywhere for OPCode_jobStateReport: it works purely because
|
||||
* this opcode is absent from ha_dispatch_gate's CTRL_STANDBY allow-list,
|
||||
* so the GENERIC post-gate rejection block (cluster.c) fills the redirect
|
||||
* for it exactly as it already does for OPCode_schedAdmin/OPCode_join. */
|
||||
static void test_jobstatereport_standby_with_known_primary_redirects(void)
|
||||
{
|
||||
cluster_t* c = jsr_make_gate_cluster();
|
||||
atomic_store(&c->ha.role, CTRL_STANDBY);
|
||||
strncpy(c->ha.primary_host, "10.0.1.9", sizeof(c->ha.primary_host) - 1);
|
||||
c->ha.primary_port = 7003;
|
||||
|
||||
endpoint_t ep;
|
||||
memset(&ep, 0, sizeof(ep));
|
||||
ep.owner = c;
|
||||
|
||||
io_packet_t* pkt = jsr_build_request_pkt(1002);
|
||||
int rc = myreq_handler_for_test(&ep, pkt);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rc);
|
||||
|
||||
xce_request_t decoded;
|
||||
jsr_decode_queued_reply(&ep, &decoded);
|
||||
TEST_ASSERT_EQUAL_INT(Request_body_replymsg, decoded.body_which);
|
||||
xce_reply_t* rr = decoded.body.replymsg;
|
||||
TEST_ASSERT_NOT_NULL(rr);
|
||||
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rr->status);
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(
|
||||
rr->redirect_host,
|
||||
"a standby dispatch-gate rejection with a known primary must carry "
|
||||
"that primary's address, exactly like every other primary-only "
|
||||
"opcode already does (bug-1700)");
|
||||
TEST_ASSERT_EQUAL_STRING("10.0.1.9", rr->redirect_host);
|
||||
TEST_ASSERT_EQUAL_UINT16(7003, rr->redirect_port);
|
||||
xce_free_request(&decoded);
|
||||
|
||||
jsr_gate_teardown(c);
|
||||
printf("test_jobstatereport_standby_with_known_primary_redirects:"
|
||||
" PASS\n");
|
||||
}
|
||||
|
||||
/* Regression companion: no known primary yet must not crash and must
|
||||
* decode as a non-redirecting reply (empty host / zero port). */
|
||||
static void test_jobstatereport_standby_no_known_primary_no_redirect(void)
|
||||
{
|
||||
cluster_t* c = jsr_make_gate_cluster();
|
||||
atomic_store(&c->ha.role, CTRL_STANDBY);
|
||||
|
||||
endpoint_t ep;
|
||||
memset(&ep, 0, sizeof(ep));
|
||||
ep.owner = c;
|
||||
|
||||
io_packet_t* pkt = jsr_build_request_pkt(1003);
|
||||
int rc = myreq_handler_for_test(&ep, pkt);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rc);
|
||||
|
||||
xce_request_t decoded;
|
||||
jsr_decode_queued_reply(&ep, &decoded);
|
||||
TEST_ASSERT_EQUAL_INT(Request_body_replymsg, decoded.body_which);
|
||||
xce_reply_t* rr = decoded.body.replymsg;
|
||||
TEST_ASSERT_NOT_NULL(rr);
|
||||
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rr->status);
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
rr->redirect_host == NULL || rr->redirect_host[0] == '\0',
|
||||
"no known primary: redirect_host must decode as empty/absent");
|
||||
TEST_ASSERT_EQUAL_UINT16(0, rr->redirect_port);
|
||||
xce_free_request(&decoded);
|
||||
|
||||
jsr_gate_teardown(c);
|
||||
printf("test_jobstatereport_standby_no_known_primary_no_redirect:"
|
||||
" PASS\n");
|
||||
}
|
||||
|
||||
/* A PRIMARY with write quorum handles the request normally — dispatched
|
||||
* past the gate into handle_jobstatereport(), which returns -ENOENT for an
|
||||
* endpoint that does not resolve to a known worker (this bare fixture's
|
||||
* `ep` was never registered as one) rather than any standby-style
|
||||
* rejection. Proves the gate's happy path is unaffected by this task's
|
||||
* change and that the new dispatch case is genuinely reached. */
|
||||
static void test_jobstatereport_primary_reaches_dispatch_case(void)
|
||||
{
|
||||
cluster_t* c = jsr_make_gate_cluster();
|
||||
atomic_store(&c->ha.role, CTRL_PRIMARY);
|
||||
/* n_peers defaults to 0 (calloc) -- single-node cluster, trivially has
|
||||
* write quorum (ha_has_write_quorum: n_peers <= 1 -> true). */
|
||||
|
||||
endpoint_t ep;
|
||||
memset(&ep, 0, sizeof(ep));
|
||||
ep.owner = c;
|
||||
/* ep.data left NULL -> get_ep_role()/get_ep_worker() treat this as "not
|
||||
* a worker endpoint" and return NULL/-EINVAL, exercising handle_
|
||||
* jobstatereport's own not-a-worker branch. */
|
||||
|
||||
io_packet_t* pkt = jsr_build_request_pkt(1004);
|
||||
int rc = myreq_handler_for_test(&ep, pkt);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(-ENOENT, rc,
|
||||
"a primary with quorum must reach handle_jobstatereport() itself "
|
||||
"(not a gate rejection) -- which reports -ENOENT for an endpoint "
|
||||
"with no resolvable worker_t");
|
||||
|
||||
jsr_gate_teardown(c);
|
||||
printf("test_jobstatereport_primary_reaches_dispatch_case: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Why no "real wire dispatch resolves a real worker_t" test lives here ──
|
||||
*
|
||||
* A genuine end-to-end test (two real wire-encoded OPCode_jobStateReport
|
||||
* packets through myreq_handler_for_test, resolving a REAL worker_t via
|
||||
* get_ep_worker() and converging a REAL job table) was attempted here and
|
||||
* found structurally impossible in THIS binary: get_ep_worker() (misc.c)
|
||||
* is NOT one of controller_unit_test_src's linked sources (meson.build) --
|
||||
* test_commit_barrier_reply.c instead provides controller_unit_test's ONLY
|
||||
* definition of it, a fixed test double (`return g_stub_ep_worker;`,
|
||||
* file-static, defaulting to NULL and settable only by that file's own
|
||||
* tests) that no other test file can point at a real worker_t. Confirmed
|
||||
* live (not assumed) by direct instrumentation: a page built with a
|
||||
* correct, verified-by-hand ep_private_t{role=HostRole_worker, idx=w->idx}
|
||||
* -- independently proven resolvable via key_find()/vget() called directly
|
||||
* against the same workermgr -- still made get_ep_worker() return NULL,
|
||||
* because the symbol actually linked into this binary is the stub, not
|
||||
* misc.c's real implementation.
|
||||
*
|
||||
* This is a pre-existing architectural property of controller_unit_test
|
||||
* (several other cluster.c-adjacent functions -- notify(), update_host_
|
||||
* runusage(), on_scheduler_join(), add_apiserver(), on_apiserver_join() --
|
||||
* are stubbed the same way in the same file, for the same reason), not
|
||||
* something this task introduced or can fix without either modifying
|
||||
* shared test infrastructure many other tests depend on (out of this
|
||||
* task's boundary) or standing up a new dedicated test binary that links
|
||||
* the real misc.c (the pattern host_runusage_unit_test/resval_typeresolve_
|
||||
* unit_test already establish for exactly this class of problem, bug-1735)
|
||||
* -- a real option, but a disproportionate scope expansion for what the
|
||||
* unit tests below plus test_worker_resync.c's own tests already prove
|
||||
* between them: handle_jobstatereport() (cluster.c) is a two-line function
|
||||
* (get_ep_worker() then resync_handle_report_page()) whose own correctness
|
||||
* is not in question -- the two things worth proving are (a) cluster.c's
|
||||
* switch genuinely routes OPCode_jobStateReport to it, proven above by
|
||||
* test_jobstatereport_primary_reaches_dispatch_case's distinctive -ENOENT
|
||||
* return (get_ep_worker's stub-NULL path, reached only via this exact
|
||||
* call chain), and (b) multi-page reassembly + exactly-once apply +
|
||||
* equivalence-to-single-page, proven exhaustively against a REAL worker_t
|
||||
* and REAL job table by resync_handle_report_page()'s own direct tests in
|
||||
* test_worker_resync.c (test_handle_report_page_multi_page_reassembles_
|
||||
* and_applies_once and its siblings) -- the same function handle_
|
||||
* jobstatereport() calls, unmodified.
|
||||
*/
|
||||
|
||||
void test_jobstatereport_dispatch(void)
|
||||
{
|
||||
UnitySetTestFile(__FILE__);
|
||||
RUN_TEST(test_jobstatereport_rejects_when_not_primary);
|
||||
RUN_TEST(test_jobstatereport_standby_with_known_primary_redirects);
|
||||
RUN_TEST(test_jobstatereport_standby_no_known_primary_no_redirect);
|
||||
RUN_TEST(test_jobstatereport_primary_reaches_dispatch_case);
|
||||
}
|
||||
|
|
@ -14,6 +14,7 @@
|
|||
#include <rtti.h>
|
||||
|
||||
#include <errno.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
|
@ -22,12 +23,26 @@
|
|||
#include <unity/unity.h>
|
||||
|
||||
#include "cluster.h"
|
||||
#include "controller.capnp.h" /* worker-rejoin-resync task 6.1: OPCode_push/
|
||||
* OPCode_jobStateQuery, ConfigData_data_*,
|
||||
* Request_body_* constants */
|
||||
#include "jobj.h" /* worker-rejoin-resync task 6.1: makeJob, for a real
|
||||
* non-terminal job the arm-round wiring tests need */
|
||||
#include "kvmgr.h"
|
||||
#include "netio.h" /* worker-rejoin-resync task 6.1: endpoint_get/
|
||||
* endpoint_deq/ITPTR_TO_PACKET, for real wire capture */
|
||||
#include "request.h" /* worker-rejoin-resync task 6.1: xce_decode,
|
||||
* xce_free_request */
|
||||
#include "resobj.h"
|
||||
#include "resync.h" /* worker-rejoin-resync task 6.1: worker_t.resync,
|
||||
* XCE_WORKER_RECONCILING */
|
||||
#include "wobj.h"
|
||||
#include "worker.h"
|
||||
#include "xce/cluster_def.h"
|
||||
#include "xce/types.h"
|
||||
#include "xcealloc.h" /* worker-rejoin-resync task 6.1: make_job_alloc and
|
||||
* friends, for a job assigned to a test worker's
|
||||
* host */
|
||||
#include "xstring.h"
|
||||
#include "zarray.h"
|
||||
|
||||
|
|
@ -55,6 +70,19 @@ void collect_confvars(cluster_t* c, collect_check_fn_t fn, xlist_t* l)
|
|||
int apply_join_state(cluster_t* c, worker_t* w, endpoint_t* ep,
|
||||
xce_hostconfig_t* hconfig);
|
||||
|
||||
/*
|
||||
* worker-rejoin-resync task 6.1: worker_push_config and worker_send_resync_
|
||||
* query are non-static in worker.c when XCE_TESTING is defined, mirroring
|
||||
* apply_join_state's own convention above -- lets these tests drive the
|
||||
* exact config-push-then-query wire sequence on_worker_join produces,
|
||||
* without needing a live join flow or get_ep_worker()'s endpoint->worker
|
||||
* resolution (a separate, pre-existing concern -- see
|
||||
* test_on_worker_join_sends_config_push_even_when_worker_unresolvable
|
||||
* below for why).
|
||||
*/
|
||||
int worker_push_config(cluster_t* c, endpoint_t* ep);
|
||||
int worker_send_resync_query(cluster_t* c, worker_t* w, endpoint_t* ep);
|
||||
|
||||
/*
|
||||
* init_worker and rejoin_worker are exported from worker.c.
|
||||
*
|
||||
|
|
@ -84,10 +112,26 @@ typedef struct {
|
|||
cluster_t c;
|
||||
var workermgr;
|
||||
var resmgr;
|
||||
var jobmgr; /* worker-rejoin-resync task 6.1: real jobmgr so the
|
||||
* apply_join_state arm-round wiring tests below can
|
||||
* seed a genuine non-terminal job assigned to a test
|
||||
* worker's host (mirrors test_worker_resync.c's own
|
||||
* g_twr.jobmgr fixture) */
|
||||
} test_cluster_t;
|
||||
|
||||
static test_cluster_t g_tc;
|
||||
|
||||
/* worker-rejoin-resync task 6.1: jobj.c's job type table is a shared,
|
||||
* process-wide resource across every test file in the controller_unit_test
|
||||
* binary -- job_obj_init() is explicitly documented (jobj.c) as safe to
|
||||
* call more than once per process without an intervening job_obj_fini()
|
||||
* (Req 10.2, the exact "every controller unit-test file that constructs
|
||||
* jobs calls job_obj_init() from its own setup" scenario this file is now
|
||||
* one more instance of), so no cross-file coordination is needed -- guard
|
||||
* only avoids redundant re-init within THIS file's own repeated setup/
|
||||
* teardown cycles, mirroring test_worker_resync.c's g_twr_jobobj_ready. */
|
||||
static int g_tc_jobobj_ready;
|
||||
|
||||
static void setup_test_cluster(void)
|
||||
{
|
||||
kvmgr_params_t str_params = {
|
||||
|
|
@ -96,15 +140,28 @@ static void setup_test_cluster(void)
|
|||
.delta = 16,
|
||||
.datakiller = NULL,
|
||||
};
|
||||
kvmgr_params_t int_params = {
|
||||
.key_type = KVMGR_KEY_INT,
|
||||
.nslots = 32,
|
||||
.delta = 16,
|
||||
};
|
||||
|
||||
memset(&g_tc, 0, sizeof(g_tc));
|
||||
|
||||
if (!g_tc_jobobj_ready) {
|
||||
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
|
||||
g_tc_jobobj_ready = 1;
|
||||
}
|
||||
|
||||
g_tc.workermgr = makeInstance(&xkvmgr_type, &str_params);
|
||||
TEST_ASSERT_NOT_NULL(g_tc.workermgr);
|
||||
|
||||
g_tc.resmgr = makeInstance(&xkvmgr_type, &str_params);
|
||||
TEST_ASSERT_NOT_NULL(g_tc.resmgr);
|
||||
|
||||
g_tc.jobmgr = makeInstance(&xkvmgr_type, &int_params);
|
||||
TEST_ASSERT_NOT_NULL(g_tc.jobmgr);
|
||||
|
||||
for (int i = 0; i < N_RES; i++) {
|
||||
xce_resource_t rdef;
|
||||
memset(&rdef, 0, sizeof(rdef));
|
||||
|
|
@ -119,6 +176,18 @@ static void setup_test_cluster(void)
|
|||
|
||||
g_tc.c.workermgr = g_tc.workermgr;
|
||||
g_tc.c.resmgr = g_tc.resmgr;
|
||||
g_tc.c.jobmgr = g_tc.jobmgr;
|
||||
|
||||
/* worker-rejoin-resync task 6.1: c->myhost is the `from` peer identity
|
||||
* worker_push_config()/worker_send_resync_query() pass to xce_
|
||||
* pushconfig_req()/xce_job_state_query_req() -- both reject a NULL
|
||||
* `from` outright, so any test exercising the real wire send needs
|
||||
* this populated (mirrors production cluster.c's own myhost_info
|
||||
* storage-then-point-at-it pattern, xce/cluster_def.h:101-102). */
|
||||
g_tc.c.myhost_info.name = (char*)"test-controller";
|
||||
g_tc.c.myhost_info.key = (char*)"test-key";
|
||||
g_tc.c.myhost = &g_tc.c.myhost_info;
|
||||
|
||||
zarray_init(&g_tc.c.sharedinst, 64, NULL);
|
||||
}
|
||||
|
||||
|
|
@ -127,9 +196,42 @@ static void teardown_test_cluster(void)
|
|||
zarray_fini(&g_tc.c.sharedinst);
|
||||
if (g_tc.resmgr) dereference(g_tc.resmgr);
|
||||
if (g_tc.workermgr) dereference(g_tc.workermgr);
|
||||
if (g_tc.jobmgr) dereference(g_tc.jobmgr);
|
||||
memset(&g_tc, 0, sizeof(g_tc));
|
||||
}
|
||||
|
||||
/* Seed a job directly in the fixture's jobmgr, bypassing dispatch/replay --
|
||||
* mirrors test_worker_resync.c's twr_seed_job. */
|
||||
static job_submit_data_t g_tc_seed_submit;
|
||||
|
||||
static job_t* seed_test_job(uint64_t jobid, uint32_t state)
|
||||
{
|
||||
xce_job_t jdef;
|
||||
|
||||
memset(&g_tc_seed_submit, 0, sizeof(g_tc_seed_submit));
|
||||
g_tc_seed_submit.queue = (char*)"q";
|
||||
g_tc_seed_submit.user = (char*)"tester";
|
||||
|
||||
memset(&jdef, 0, sizeof(jdef));
|
||||
jdef.jobid = jobid;
|
||||
jdef.state = state;
|
||||
jdef.submit = &g_tc_seed_submit;
|
||||
|
||||
return makeJob(g_tc.jobmgr, &jdef);
|
||||
}
|
||||
|
||||
/* Build a single-host job allocation assigning `jobid` to `host` -- mirrors
|
||||
* test_worker_resync.c's twr_make_test_alloc. Caller hands ownership to
|
||||
* job->job.allocs. */
|
||||
static xce_job_alloc_t* make_test_job_alloc(uint64_t jobid, const char* host)
|
||||
{
|
||||
return make_job_alloc(jobid, 1,
|
||||
make_host_allocs(1,
|
||||
make_host_alloc((char*)host, 1,
|
||||
make_allocs(1,
|
||||
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
|
||||
}
|
||||
|
||||
/* Create a worker in the test workermgr; attrs strings must be makeString'd. */
|
||||
static worker_t* make_test_worker(const char* name, hostattrs_t* attrs)
|
||||
{
|
||||
|
|
@ -691,6 +793,706 @@ static void test_rejoin_worker_preserves_admin_bits(void)
|
|||
printf("test_rejoin_worker_preserves_admin_bits: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Task 6.1: apply_join_state arms a reconciliation round ──────────── */
|
||||
|
||||
/*
|
||||
* Real (heap) endpoint_t via endpoint_get() -- NOT a hand-rolled/zeroed
|
||||
* stack stand-in (bug-378: an under-sized/wrong-typed stand-in for
|
||||
* endpoint_t caused a global-buffer-overflow when passed where a real
|
||||
* endpoint_t* was expected). Needed only by the tests below that actually
|
||||
* send something over the wire (worker_push_config/worker_send_resync_
|
||||
* query); the pre-existing apply_join_state/init_worker/rejoin_worker
|
||||
* tests above never send anything and so are unaffected, still using a
|
||||
* plain zeroed stack endpoint_t. Mirrors src/lib/testsuite/
|
||||
* test_job_state_wire.c's own established pattern for this exact
|
||||
* situation.
|
||||
*/
|
||||
static endpoint_t* new_test_endpoint(void)
|
||||
{
|
||||
endpoint_t* ep = endpoint_get();
|
||||
TEST_ASSERT_NOT_NULL(ep);
|
||||
return ep;
|
||||
}
|
||||
|
||||
/* Pop one queued page off ep->outq and decode it. Fails the test if the
|
||||
* queue is empty. Caller must xce_free_request() the result. */
|
||||
static void pop_and_decode(endpoint_t* ep, xce_request_t* out)
|
||||
{
|
||||
data_item_t* it = endpoint_deq(ep);
|
||||
TEST_ASSERT_NOT_NULL(it);
|
||||
|
||||
io_packet_t* pkt = ITPTR_TO_PACKET(it);
|
||||
|
||||
memset(out, 0, sizeof(*out));
|
||||
TEST_ASSERT_EQUAL_INT(0, xce_decode(pkt, out)); /* consumes/frees pkt */
|
||||
}
|
||||
|
||||
static void assert_queue_empty(endpoint_t* ep)
|
||||
{
|
||||
TEST_ASSERT_NULL(endpoint_deq(ep));
|
||||
}
|
||||
|
||||
/*
|
||||
* rejoin_worker (via apply_join_state) must arm a reconciliation round --
|
||||
* w->resync populated, XCE_WORKER_RECONCILING set, the query set matching
|
||||
* exactly the host's non-terminal jobs -- when the controller currently
|
||||
* holds one or more non-terminal jobs for that host. Req 1.1, 2.1, 2.2.
|
||||
*/
|
||||
static void test_rejoin_worker_arms_reconciliation_round_when_jobs_exist(void)
|
||||
{
|
||||
setup_test_cluster();
|
||||
|
||||
job_t* j = seed_test_job(96001, JOB_STAT_RUN);
|
||||
TEST_ASSERT_NOT_NULL(j);
|
||||
j->job.allocs = make_test_job_alloc(96001, "arm-host-1");
|
||||
TEST_ASSERT_NOT_NULL(j->job.allocs);
|
||||
|
||||
hostattrs_t attrs;
|
||||
memset(&attrs, 0, sizeof(attrs));
|
||||
attrs.hosttype = makeString("LINUX");
|
||||
attrs.hostmodel = makeString("x86");
|
||||
attrs.cpufactor = 1.0f;
|
||||
attrs.maxcpus = 4;
|
||||
attrs.maxmem = 16;
|
||||
attrs.maxswap = 32;
|
||||
attrs.maxtmp = 64;
|
||||
attrs.ndisks = 1;
|
||||
|
||||
worker_t* w = make_test_worker("arm-host-1", &attrs);
|
||||
TEST_ASSERT_NOT_NULL(w);
|
||||
|
||||
hostattrs_t new_attrs;
|
||||
memset(&new_attrs, 0, sizeof(new_attrs));
|
||||
new_attrs.hosttype = makeString("LINUX");
|
||||
new_attrs.hostmodel = makeString("x86");
|
||||
new_attrs.cpufactor = 1.0f;
|
||||
new_attrs.maxcpus = 4;
|
||||
new_attrs.maxmem = 16;
|
||||
new_attrs.maxswap = 32;
|
||||
new_attrs.maxtmp = 64;
|
||||
new_attrs.ndisks = 1;
|
||||
|
||||
xce_hostconfig_t hconfig;
|
||||
memset(&hconfig, 0, sizeof(hconfig));
|
||||
hconfig.hosttype = (char*)"!";
|
||||
hconfig.hostmodel = (char*)"!";
|
||||
hconfig.mxj = 4;
|
||||
|
||||
endpoint_t ep;
|
||||
memset(&ep, 0, sizeof(ep));
|
||||
|
||||
int idx = rejoin_worker(&g_tc.c, w, &ep, &new_attrs, &hconfig);
|
||||
TEST_ASSERT_GREATER_OR_EQUAL(0, idx);
|
||||
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
|
||||
"a worker rejoining with a known non-terminal job must have a "
|
||||
"reconciliation round armed (Req 1.1)");
|
||||
TEST_ASSERT_TRUE_MESSAGE(w->host.status & XCE_WORKER_RECONCILING,
|
||||
"XCE_WORKER_RECONCILING must be set once a round is armed");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, w->resync->n_queried_jobids,
|
||||
"the round's query set must match exactly the host's non-terminal "
|
||||
"jobs (Req 2.1)");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(96001, w->resync->queried_jobids[0],
|
||||
"the queried job must be identified by its unique jobid (Req 2.2)");
|
||||
|
||||
dereference(new_attrs.hosttype);
|
||||
dereference(new_attrs.hostmodel);
|
||||
|
||||
teardown_test_cluster();
|
||||
printf("test_rejoin_worker_arms_reconciliation_round_when_jobs_exist: "
|
||||
"PASS\n");
|
||||
}
|
||||
|
||||
/*
|
||||
* A worker rejoining with NO non-terminal jobs currently assigned to it
|
||||
* must NOT have a round armed. Req 1.2.
|
||||
*/
|
||||
static void test_rejoin_worker_no_round_when_no_jobs(void)
|
||||
{
|
||||
setup_test_cluster();
|
||||
|
||||
hostattrs_t attrs;
|
||||
memset(&attrs, 0, sizeof(attrs));
|
||||
attrs.hosttype = makeString("LINUX");
|
||||
attrs.hostmodel = makeString("x86");
|
||||
attrs.cpufactor = 1.0f;
|
||||
attrs.maxcpus = 4;
|
||||
attrs.maxmem = 16;
|
||||
attrs.maxswap = 32;
|
||||
attrs.maxtmp = 64;
|
||||
attrs.ndisks = 1;
|
||||
|
||||
worker_t* w = make_test_worker("arm-host-2", &attrs);
|
||||
TEST_ASSERT_NOT_NULL(w);
|
||||
|
||||
hostattrs_t new_attrs = attrs;
|
||||
new_attrs.hosttype = makeString("LINUX");
|
||||
new_attrs.hostmodel = makeString("x86");
|
||||
|
||||
xce_hostconfig_t hconfig;
|
||||
memset(&hconfig, 0, sizeof(hconfig));
|
||||
hconfig.hosttype = (char*)"!";
|
||||
hconfig.hostmodel = (char*)"!";
|
||||
hconfig.mxj = 4;
|
||||
|
||||
endpoint_t ep;
|
||||
memset(&ep, 0, sizeof(ep));
|
||||
|
||||
rejoin_worker(&g_tc.c, w, &ep, &new_attrs, &hconfig);
|
||||
|
||||
TEST_ASSERT_NULL_MESSAGE(w->resync,
|
||||
"a worker rejoining with no non-terminal jobs must not have a "
|
||||
"reconciliation round armed (Req 1.2)");
|
||||
TEST_ASSERT_FALSE_MESSAGE(w->host.status & XCE_WORKER_RECONCILING,
|
||||
"XCE_WORKER_RECONCILING must not be set");
|
||||
|
||||
dereference(new_attrs.hosttype);
|
||||
dereference(new_attrs.hostmodel);
|
||||
|
||||
teardown_test_cluster();
|
||||
printf("test_rejoin_worker_no_round_when_no_jobs: PASS\n");
|
||||
}
|
||||
|
||||
/*
|
||||
* A terminal job (DONE) assigned to the host must not be queried -- only
|
||||
* non-terminal jobs count toward the trigger (Req 1.1's own "non-terminal"
|
||||
* qualifier).
|
||||
*/
|
||||
static void test_rejoin_worker_no_round_when_only_terminal_jobs(void)
|
||||
{
|
||||
setup_test_cluster();
|
||||
|
||||
job_t* j = seed_test_job(96002, JOB_STAT_DONE);
|
||||
TEST_ASSERT_NOT_NULL(j);
|
||||
j->job.allocs = make_test_job_alloc(96002, "arm-host-3");
|
||||
TEST_ASSERT_NOT_NULL(j->job.allocs);
|
||||
|
||||
hostattrs_t attrs;
|
||||
memset(&attrs, 0, sizeof(attrs));
|
||||
attrs.hosttype = makeString("LINUX");
|
||||
attrs.hostmodel = makeString("x86");
|
||||
attrs.cpufactor = 1.0f;
|
||||
attrs.maxcpus = 4;
|
||||
attrs.maxmem = 16;
|
||||
attrs.maxswap = 32;
|
||||
attrs.maxtmp = 64;
|
||||
attrs.ndisks = 1;
|
||||
|
||||
worker_t* w = make_test_worker("arm-host-3", &attrs);
|
||||
TEST_ASSERT_NOT_NULL(w);
|
||||
|
||||
hostattrs_t new_attrs = attrs;
|
||||
new_attrs.hosttype = makeString("LINUX");
|
||||
new_attrs.hostmodel = makeString("x86");
|
||||
|
||||
xce_hostconfig_t hconfig;
|
||||
memset(&hconfig, 0, sizeof(hconfig));
|
||||
hconfig.hosttype = (char*)"!";
|
||||
hconfig.hostmodel = (char*)"!";
|
||||
hconfig.mxj = 4;
|
||||
|
||||
endpoint_t ep;
|
||||
memset(&ep, 0, sizeof(ep));
|
||||
|
||||
rejoin_worker(&g_tc.c, w, &ep, &new_attrs, &hconfig);
|
||||
|
||||
TEST_ASSERT_NULL_MESSAGE(w->resync,
|
||||
"a terminal (DONE) job must not count toward the reconciliation "
|
||||
"trigger");
|
||||
|
||||
dereference(new_attrs.hosttype);
|
||||
dereference(new_attrs.hostmodel);
|
||||
|
||||
teardown_test_cluster();
|
||||
printf("test_rejoin_worker_no_round_when_only_terminal_jobs: PASS\n");
|
||||
}
|
||||
|
||||
/*
|
||||
* Req 1.3: the SAME trigger condition -- no special-casing needed -- must
|
||||
* apply to a host's first-ever join (init_worker), not just a rejoin.
|
||||
*/
|
||||
static void test_init_worker_arms_reconciliation_round_when_jobs_exist(void)
|
||||
{
|
||||
setup_test_cluster();
|
||||
|
||||
job_t* j = seed_test_job(96003, JOB_STAT_RUN);
|
||||
TEST_ASSERT_NOT_NULL(j);
|
||||
j->job.allocs = make_test_job_alloc(96003, "arm-host-4-first-join");
|
||||
TEST_ASSERT_NOT_NULL(j->job.allocs);
|
||||
|
||||
hostattrs_t attrs;
|
||||
memset(&attrs, 0, sizeof(attrs));
|
||||
attrs.hosttype = makeString("LINUX");
|
||||
attrs.hostmodel = makeString("x86");
|
||||
attrs.cpufactor = 1.0f;
|
||||
attrs.maxcpus = 4;
|
||||
attrs.maxmem = 16;
|
||||
attrs.maxswap = 32;
|
||||
attrs.maxtmp = 64;
|
||||
attrs.ndisks = 1;
|
||||
|
||||
xce_hostconfig_t hconfig;
|
||||
memset(&hconfig, 0, sizeof(hconfig));
|
||||
hconfig.hosttype = (char*)"!";
|
||||
hconfig.hostmodel = (char*)"!";
|
||||
hconfig.mxj = 4;
|
||||
|
||||
endpoint_t ep;
|
||||
memset(&ep, 0, sizeof(ep));
|
||||
|
||||
int idx = init_worker(&g_tc.c, &ep, "arm-host-4-first-join", &attrs,
|
||||
&hconfig);
|
||||
TEST_ASSERT_GREATER_OR_EQUAL(0, idx);
|
||||
|
||||
worker_t* w = NULL;
|
||||
lock(g_tc.c.workermgr, RTTI_LOCK_READ);
|
||||
key_find(g_tc.c.workermgr, (void*)"arm-host-4-first-join", (void**)&w);
|
||||
unlock(g_tc.c.workermgr);
|
||||
TEST_ASSERT_NOT_NULL(w);
|
||||
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
|
||||
"a host's first-ever join must arm a reconciliation round exactly "
|
||||
"like a rejoin, with no special-casing needed (Req 1.3)");
|
||||
TEST_ASSERT_EQUAL_INT(1, w->resync->n_queried_jobids);
|
||||
TEST_ASSERT_EQUAL_UINT64(96003, w->resync->queried_jobids[0]);
|
||||
|
||||
teardown_test_cluster();
|
||||
printf("test_init_worker_arms_reconciliation_round_when_jobs_exist: "
|
||||
"PASS\n");
|
||||
}
|
||||
|
||||
/* ── Task 6.1: worker_send_resync_query ───────────────────────────────── */
|
||||
|
||||
/*
|
||||
* When a round is armed for `w`, worker_send_resync_query() must send
|
||||
* exactly that round's job IDs as an OPCode_jobStateQuery. Req 2.1, 2.2.
|
||||
*/
|
||||
static void test_worker_send_resync_query_sends_armed_round(void)
|
||||
{
|
||||
setup_test_cluster();
|
||||
|
||||
job_t* j1 = seed_test_job(96010, JOB_STAT_RUN);
|
||||
j1->job.allocs = make_test_job_alloc(96010, "query-host-1");
|
||||
job_t* j2 = seed_test_job(96011, JOB_STAT_RUN);
|
||||
j2->job.allocs = make_test_job_alloc(96011, "query-host-1");
|
||||
|
||||
hostattrs_t attrs;
|
||||
memset(&attrs, 0, sizeof(attrs));
|
||||
attrs.hosttype = makeString("LINUX");
|
||||
attrs.hostmodel = makeString("x86");
|
||||
attrs.cpufactor = 1.0f;
|
||||
attrs.maxcpus = 4;
|
||||
attrs.maxmem = 16;
|
||||
attrs.maxswap = 32;
|
||||
attrs.maxtmp = 64;
|
||||
attrs.ndisks = 1;
|
||||
|
||||
worker_t* w = make_test_worker("query-host-1", &attrs);
|
||||
TEST_ASSERT_NOT_NULL(w);
|
||||
|
||||
hostattrs_t new_attrs = attrs;
|
||||
new_attrs.hosttype = makeString("LINUX");
|
||||
new_attrs.hostmodel = makeString("x86");
|
||||
|
||||
xce_hostconfig_t hconfig;
|
||||
memset(&hconfig, 0, sizeof(hconfig));
|
||||
hconfig.hosttype = (char*)"!";
|
||||
hconfig.hostmodel = (char*)"!";
|
||||
hconfig.mxj = 4;
|
||||
|
||||
endpoint_t stack_ep;
|
||||
memset(&stack_ep, 0, sizeof(stack_ep));
|
||||
rejoin_worker(&g_tc.c, w, &stack_ep, &new_attrs, &hconfig);
|
||||
TEST_ASSERT_NOT_NULL(w->resync);
|
||||
TEST_ASSERT_EQUAL_INT(2, w->resync->n_queried_jobids);
|
||||
|
||||
endpoint_t* ep = new_test_endpoint();
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, worker_send_resync_query(&g_tc.c, w, ep));
|
||||
|
||||
xce_request_t decoded;
|
||||
pop_and_decode(ep, &decoded);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(OPCode_jobStateQuery, decoded.opcode);
|
||||
TEST_ASSERT_EQUAL_INT(Request_body_jobstatequerymsg, decoded.body_which);
|
||||
TEST_ASSERT_NOT_NULL(decoded.body.jobstatequerymsg);
|
||||
TEST_ASSERT_EQUAL_INT(2, decoded.body.jobstatequerymsg->n_total);
|
||||
|
||||
int saw_96010 = 0, saw_96011 = 0;
|
||||
for (int i = 0; i < decoded.body.jobstatequerymsg->count; i++) {
|
||||
uint64_t id = decoded.body.jobstatequerymsg->jobids[i];
|
||||
if (id == 96010) saw_96010 = 1;
|
||||
if (id == 96011) saw_96011 = 1;
|
||||
}
|
||||
TEST_ASSERT_TRUE_MESSAGE(saw_96010, "queried set must include 96010");
|
||||
TEST_ASSERT_TRUE_MESSAGE(saw_96011, "queried set must include 96011");
|
||||
|
||||
xce_free_request(&decoded);
|
||||
assert_queue_empty(ep);
|
||||
|
||||
dereference(new_attrs.hosttype);
|
||||
dereference(new_attrs.hostmodel);
|
||||
endpoint_fini(ep);
|
||||
free(ep);
|
||||
|
||||
teardown_test_cluster();
|
||||
printf("test_worker_send_resync_query_sends_armed_round: PASS\n");
|
||||
}
|
||||
|
||||
/*
|
||||
* When no round is armed (w->resync == NULL), worker_send_resync_query()
|
||||
* must send nothing at all -- the Req 1.2 no-round case's downstream
|
||||
* completion criterion ("no round, no query").
|
||||
*/
|
||||
static void test_worker_send_resync_query_sends_nothing_when_no_round(void)
|
||||
{
|
||||
setup_test_cluster();
|
||||
|
||||
hostattrs_t attrs;
|
||||
memset(&attrs, 0, sizeof(attrs));
|
||||
attrs.hosttype = makeString("LINUX");
|
||||
attrs.hostmodel = makeString("x86");
|
||||
attrs.cpufactor = 1.0f;
|
||||
attrs.maxcpus = 4;
|
||||
attrs.maxmem = 16;
|
||||
attrs.maxswap = 32;
|
||||
attrs.maxtmp = 64;
|
||||
attrs.ndisks = 1;
|
||||
|
||||
worker_t* w = make_test_worker("query-host-2", &attrs);
|
||||
TEST_ASSERT_NOT_NULL(w);
|
||||
TEST_ASSERT_NULL(w->resync);
|
||||
|
||||
endpoint_t* ep = new_test_endpoint();
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, worker_send_resync_query(&g_tc.c, w, ep));
|
||||
|
||||
assert_queue_empty(ep);
|
||||
|
||||
endpoint_fini(ep);
|
||||
free(ep);
|
||||
|
||||
teardown_test_cluster();
|
||||
printf("test_worker_send_resync_query_sends_nothing_when_no_round: "
|
||||
"PASS\n");
|
||||
}
|
||||
|
||||
/* ── Task 7.5: worker_send_resync_query genuinely paginates (bug-1755) ──
|
||||
*
|
||||
* Prior to task 7.5, worker_send_resync_query() hardcoded pktlimit=0 in its
|
||||
* xce_job_state_query_req() call -- xce_job_state_query_req()'s own
|
||||
* pktlimit<=0 convention (request.c) means "pktlimit = nrows", i.e. always
|
||||
* exactly ONE wire page, however many jobs are queried. Task 7.4's own test
|
||||
* (test_worker_resync.c, test_large_job_count_pagination_correctness) only
|
||||
* proved the underlying xce_job_state_query_req() pagination MECHANISM by
|
||||
* calling it directly with an explicit page-size override, bypassing this
|
||||
* production call site entirely (documented live as bug-1755: "effectively
|
||||
* dead code in production today because its only call site never supplies
|
||||
* a bound").
|
||||
*
|
||||
* This test drives the REAL production call site -- worker_send_resync_
|
||||
* query() itself, not a direct xce_job_state_query_req() call -- with a
|
||||
* non-terminal job count on the host that genuinely exceeds the new
|
||||
* XCE_RESYNC_QUERY_PKTLIMIT page cap (worker.c), and asserts the query is
|
||||
* observably split across multiple wire pages: more than one packet is
|
||||
* enqueued, each page's `pos` picks up exactly where the previous page's
|
||||
* `count` left off, every jobid appears in exactly one page, and every
|
||||
* page (including the final, deliberately partial one) carries the same
|
||||
* full n_total. Req 8.1-8.3.
|
||||
*/
|
||||
#define TWJ_RESYNC_QUERY_PKTLIMIT 100 /* mirrors production
|
||||
* XCE_RESYNC_QUERY_PKTLIMIT
|
||||
* (worker.c) -- not a real
|
||||
* production symbol reference (that
|
||||
* #define has file-local linkage),
|
||||
* kept in sync by this test's own
|
||||
* page-count assertion below, which
|
||||
* would fail loudly if the two ever
|
||||
* drifted apart */
|
||||
#define TWJ_RESYNC_QUERY_N_JOBS 250 /* > TWJ_RESYNC_QUERY_PKTLIMIT, and not
|
||||
* a multiple of it, so the final wire
|
||||
* page is genuinely partial (50
|
||||
* entries), not just N full pages */
|
||||
#define TWJ_RESYNC_QUERY_JOBID_BASE 970000ULL
|
||||
#define TWJ_RESYNC_QUERY_N_PAGES \
|
||||
((TWJ_RESYNC_QUERY_N_JOBS + TWJ_RESYNC_QUERY_PKTLIMIT - 1) \
|
||||
/ TWJ_RESYNC_QUERY_PKTLIMIT)
|
||||
|
||||
static void
|
||||
test_worker_send_resync_query_paginates_across_multiple_wire_pages(void)
|
||||
{
|
||||
setup_test_cluster();
|
||||
|
||||
hostattrs_t attrs;
|
||||
memset(&attrs, 0, sizeof(attrs));
|
||||
attrs.hosttype = makeString("LINUX");
|
||||
attrs.hostmodel = makeString("x86");
|
||||
attrs.cpufactor = 1.0f;
|
||||
attrs.maxcpus = 4;
|
||||
attrs.maxmem = 16;
|
||||
attrs.maxswap = 32;
|
||||
attrs.maxtmp = 64;
|
||||
attrs.ndisks = 1;
|
||||
|
||||
worker_t* w = make_test_worker("query-host-paginate", &attrs);
|
||||
TEST_ASSERT_NOT_NULL(w);
|
||||
|
||||
for (int i = 0; i < TWJ_RESYNC_QUERY_N_JOBS; i++) {
|
||||
uint64_t jobid = TWJ_RESYNC_QUERY_JOBID_BASE + (uint64_t)i;
|
||||
job_t* j = seed_test_job(jobid, JOB_STAT_RUN);
|
||||
TEST_ASSERT_NOT_NULL(j);
|
||||
j->job.allocs = make_test_job_alloc(jobid, "query-host-paginate");
|
||||
TEST_ASSERT_NOT_NULL(j->job.allocs);
|
||||
}
|
||||
|
||||
hostattrs_t new_attrs = attrs;
|
||||
new_attrs.hosttype = makeString("LINUX");
|
||||
new_attrs.hostmodel = makeString("x86");
|
||||
|
||||
xce_hostconfig_t hconfig;
|
||||
memset(&hconfig, 0, sizeof(hconfig));
|
||||
hconfig.hosttype = (char*)"!";
|
||||
hconfig.hostmodel = (char*)"!";
|
||||
hconfig.mxj = 4;
|
||||
|
||||
endpoint_t stack_ep;
|
||||
memset(&stack_ep, 0, sizeof(stack_ep));
|
||||
rejoin_worker(&g_tc.c, w, &stack_ep, &new_attrs, &hconfig);
|
||||
TEST_ASSERT_NOT_NULL(w->resync);
|
||||
TEST_ASSERT_EQUAL_INT(TWJ_RESYNC_QUERY_N_JOBS, w->resync->n_queried_jobids);
|
||||
|
||||
endpoint_t* ep = new_test_endpoint();
|
||||
|
||||
/* The REAL production call site -- no xce_job_state_query_req()
|
||||
* bypass. */
|
||||
TEST_ASSERT_EQUAL_INT(0, worker_send_resync_query(&g_tc.c, w, ep));
|
||||
|
||||
{
|
||||
bool seen[TWJ_RESYNC_QUERY_N_JOBS];
|
||||
int running_total = 0;
|
||||
int pages = 0;
|
||||
|
||||
memset(seen, 0, sizeof(seen));
|
||||
|
||||
while (running_total < TWJ_RESYNC_QUERY_N_JOBS) {
|
||||
xce_request_t decoded;
|
||||
int j;
|
||||
|
||||
pop_and_decode(ep, &decoded);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(OPCode_jobStateQuery, decoded.opcode);
|
||||
TEST_ASSERT_EQUAL_INT(Request_body_jobstatequerymsg,
|
||||
decoded.body_which);
|
||||
TEST_ASSERT_NOT_NULL(decoded.body.jobstatequerymsg);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(TWJ_RESYNC_QUERY_N_JOBS,
|
||||
decoded.body.jobstatequerymsg->n_total,
|
||||
"every page must carry the same full n_total, not a "
|
||||
"per-page total (Req 2.1)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(running_total,
|
||||
decoded.body.jobstatequerymsg->pos,
|
||||
"each page's pos must pick up exactly where the previous "
|
||||
"page's count left off -- no gap, no overlap (Req 8.3)");
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
decoded.body.jobstatequerymsg->count
|
||||
<= TWJ_RESYNC_QUERY_PKTLIMIT,
|
||||
"no single page may exceed the configured page cap "
|
||||
"(Req 8.3)");
|
||||
|
||||
for (j = 0; j < decoded.body.jobstatequerymsg->count; j++) {
|
||||
uint64_t jobid = decoded.body.jobstatequerymsg->jobids[j];
|
||||
uint64_t idx = jobid - TWJ_RESYNC_QUERY_JOBID_BASE;
|
||||
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
idx < (uint64_t)TWJ_RESYNC_QUERY_N_JOBS,
|
||||
"a jobid appearing on the wire must be one of the "
|
||||
"armed round's own jobids");
|
||||
TEST_ASSERT_FALSE_MESSAGE(seen[idx],
|
||||
"no jobid may be sent twice across pages (Req 8.3)");
|
||||
seen[idx] = true;
|
||||
}
|
||||
|
||||
running_total += decoded.body.jobstatequerymsg->count;
|
||||
pages++;
|
||||
xce_free_request(&decoded);
|
||||
}
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(TWJ_RESYNC_QUERY_N_PAGES, pages,
|
||||
"Req 8.1-8.3: a job count exceeding the real production page "
|
||||
"cap must genuinely span multiple wire pages -- observably, "
|
||||
"via the REAL worker_send_resync_query() call site, not just "
|
||||
"theoretically supported by xce_job_state_query_req()'s own "
|
||||
"pagination loop (bug-1755, closed by task 7.5)");
|
||||
TEST_ASSERT_GREATER_THAN_INT_MESSAGE(1, pages,
|
||||
"a single collapsed page here would mean the hardcoded-0 "
|
||||
"regression (bug-1755) is back");
|
||||
|
||||
for (int i = 0; i < TWJ_RESYNC_QUERY_N_JOBS; i++) {
|
||||
TEST_ASSERT_TRUE_MESSAGE(seen[i],
|
||||
"every armed jobid must appear exactly once across all "
|
||||
"pages -- none silently dropped past a page boundary "
|
||||
"(Req 8.3)");
|
||||
}
|
||||
}
|
||||
|
||||
assert_queue_empty(ep);
|
||||
|
||||
dereference(new_attrs.hosttype);
|
||||
dereference(new_attrs.hostmodel);
|
||||
endpoint_fini(ep);
|
||||
free(ep);
|
||||
|
||||
teardown_test_cluster();
|
||||
printf("test_worker_send_resync_query_paginates_across_multiple_wire_"
|
||||
"pages: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Task 6.1: config push, then query, in that fixed order ──────────── */
|
||||
|
||||
/*
|
||||
* design.md's File Structure Plan for on_worker_join is explicit that this
|
||||
* ordering is load-bearing ("config push, then query", "not as an
|
||||
* extension of the join call itself"), and this spec's own Implementation
|
||||
* Notes (task 4.3) warn that a placement-sensitive guarantee needs a
|
||||
* STRUCTURAL assertion, not just an end-state one -- a behavioral test
|
||||
* alone can pass even with the wrong order if it only checks that both
|
||||
* packets eventually arrive. This test drives the exact two calls on_
|
||||
* worker_join makes, in the exact order it makes them, and asserts the
|
||||
* WIRE ORDER of the resulting packets directly.
|
||||
*/
|
||||
static void test_config_push_then_resync_query_wire_order(void)
|
||||
{
|
||||
setup_test_cluster();
|
||||
|
||||
cluster_config_t cc;
|
||||
memset(&cc, 0, sizeof(cc));
|
||||
cc.clustername = (char*)"test-cluster";
|
||||
myconf_t myconf;
|
||||
memset(&myconf, 0, sizeof(myconf));
|
||||
myconf.cluster_config = &cc;
|
||||
g_tc.c.myconf = &myconf;
|
||||
|
||||
job_t* j = seed_test_job(96020, JOB_STAT_RUN);
|
||||
j->job.allocs = make_test_job_alloc(96020, "order-host-1");
|
||||
|
||||
hostattrs_t attrs;
|
||||
memset(&attrs, 0, sizeof(attrs));
|
||||
attrs.hosttype = makeString("LINUX");
|
||||
attrs.hostmodel = makeString("x86");
|
||||
attrs.cpufactor = 1.0f;
|
||||
attrs.maxcpus = 4;
|
||||
attrs.maxmem = 16;
|
||||
attrs.maxswap = 32;
|
||||
attrs.maxtmp = 64;
|
||||
attrs.ndisks = 1;
|
||||
|
||||
worker_t* w = make_test_worker("order-host-1", &attrs);
|
||||
TEST_ASSERT_NOT_NULL(w);
|
||||
|
||||
hostattrs_t new_attrs = attrs;
|
||||
new_attrs.hosttype = makeString("LINUX");
|
||||
new_attrs.hostmodel = makeString("x86");
|
||||
|
||||
xce_hostconfig_t hconfig;
|
||||
memset(&hconfig, 0, sizeof(hconfig));
|
||||
hconfig.hosttype = (char*)"!";
|
||||
hconfig.hostmodel = (char*)"!";
|
||||
hconfig.mxj = 4;
|
||||
|
||||
endpoint_t stack_ep;
|
||||
memset(&stack_ep, 0, sizeof(stack_ep));
|
||||
rejoin_worker(&g_tc.c, w, &stack_ep, &new_attrs, &hconfig);
|
||||
TEST_ASSERT_NOT_NULL(w->resync);
|
||||
|
||||
endpoint_t* ep = new_test_endpoint();
|
||||
|
||||
/* The exact two calls on_worker_join() makes, in the exact order. */
|
||||
worker_push_config(&g_tc.c, ep);
|
||||
worker_send_resync_query(&g_tc.c, w, ep);
|
||||
|
||||
xce_request_t first;
|
||||
pop_and_decode(ep, &first);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(OPCode_push, first.opcode,
|
||||
"the config push must be the FIRST packet on the wire");
|
||||
xce_free_request(&first);
|
||||
|
||||
xce_request_t second;
|
||||
pop_and_decode(ep, &second);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(OPCode_jobStateQuery, second.opcode,
|
||||
"the reconciliation query must be the SECOND packet on the wire, "
|
||||
"strictly after the config push");
|
||||
xce_free_request(&second);
|
||||
|
||||
assert_queue_empty(ep);
|
||||
|
||||
dereference(new_attrs.hosttype);
|
||||
dereference(new_attrs.hostmodel);
|
||||
endpoint_fini(ep);
|
||||
free(ep);
|
||||
|
||||
teardown_test_cluster();
|
||||
printf("test_config_push_then_resync_query_wire_order: PASS\n");
|
||||
}
|
||||
|
||||
/*
|
||||
* Req 3.7 (config-delivery gap): worker_push_config() must carry the
|
||||
* REAL, currently-configured workerResyncWindowMs value -- not a
|
||||
* hardcoded constant. Two distinct configured values must decode back to
|
||||
* two distinct wire values.
|
||||
*/
|
||||
static void test_worker_push_config_carries_configured_resync_window(void)
|
||||
{
|
||||
setup_test_cluster();
|
||||
|
||||
cluster_config_t cc;
|
||||
memset(&cc, 0, sizeof(cc));
|
||||
cc.clustername = (char*)"test-cluster";
|
||||
cc.worker_resync_window_ms = 424242;
|
||||
myconf_t myconf;
|
||||
memset(&myconf, 0, sizeof(myconf));
|
||||
myconf.cluster_config = &cc;
|
||||
g_tc.c.myconf = &myconf;
|
||||
|
||||
endpoint_t* ep = new_test_endpoint();
|
||||
TEST_ASSERT_EQUAL_INT(0, worker_push_config(&g_tc.c, ep));
|
||||
|
||||
xce_request_t decoded;
|
||||
pop_and_decode(ep, &decoded);
|
||||
TEST_ASSERT_EQUAL_INT(OPCode_push, decoded.opcode);
|
||||
TEST_ASSERT_EQUAL_INT(Request_body_configpushmsg, decoded.body_which);
|
||||
TEST_ASSERT_NOT_NULL(decoded.body.configpushmsg);
|
||||
TEST_ASSERT_EQUAL_INT(ConfigData_data_workerconfig,
|
||||
decoded.body.configpushmsg->data_which);
|
||||
TEST_ASSERT_NOT_NULL(decoded.body.configpushmsg->data.conf4worker);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(424242,
|
||||
decoded.body.configpushmsg->data.conf4worker->worker_resync_window_ms,
|
||||
"the pushed value must be the real configured ClusterConfig."
|
||||
"workerResyncWindowMs, not a hardcoded constant");
|
||||
|
||||
xce_free_request(&decoded);
|
||||
assert_queue_empty(ep);
|
||||
endpoint_fini(ep);
|
||||
free(ep);
|
||||
|
||||
/* A second, different configured value must decode back differently --
|
||||
* confirms the field is genuinely read from config each call, not
|
||||
* memoized/hardcoded from the first call above. */
|
||||
cc.worker_resync_window_ms = 7;
|
||||
ep = new_test_endpoint();
|
||||
TEST_ASSERT_EQUAL_INT(0, worker_push_config(&g_tc.c, ep));
|
||||
pop_and_decode(ep, &decoded);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(7,
|
||||
decoded.body.configpushmsg->data.conf4worker->worker_resync_window_ms,
|
||||
"a second, different configured value must be reflected on the "
|
||||
"wire, proving the field is read live, not cached");
|
||||
xce_free_request(&decoded);
|
||||
endpoint_fini(ep);
|
||||
free(ep);
|
||||
|
||||
teardown_test_cluster();
|
||||
printf("test_worker_push_config_carries_configured_resync_window: "
|
||||
"PASS\n");
|
||||
}
|
||||
|
||||
/* ── Dispatch ───────────────────────────────────────────────────────── */
|
||||
|
||||
void test_worker_join(void)
|
||||
|
|
@ -716,4 +1518,19 @@ void test_worker_join(void)
|
|||
/* Task 1.3: rejoin_worker (reconnect with admin-bit preservation) */
|
||||
RUN_TEST(test_rejoin_worker_updates_attrs_and_ep);
|
||||
RUN_TEST(test_rejoin_worker_preserves_admin_bits);
|
||||
|
||||
/* worker-rejoin-resync task 6.1: apply_join_state arms a
|
||||
* reconciliation round; worker_send_resync_query sends it; config
|
||||
* push then query in that fixed order; the workerResyncWindowMs
|
||||
* config-delivery gap. */
|
||||
RUN_TEST(test_rejoin_worker_arms_reconciliation_round_when_jobs_exist);
|
||||
RUN_TEST(test_rejoin_worker_no_round_when_no_jobs);
|
||||
RUN_TEST(test_rejoin_worker_no_round_when_only_terminal_jobs);
|
||||
RUN_TEST(test_init_worker_arms_reconciliation_round_when_jobs_exist);
|
||||
RUN_TEST(test_worker_send_resync_query_sends_armed_round);
|
||||
RUN_TEST(test_worker_send_resync_query_sends_nothing_when_no_round);
|
||||
RUN_TEST(
|
||||
test_worker_send_resync_query_paginates_across_multiple_wire_pages);
|
||||
RUN_TEST(test_config_push_then_resync_query_wire_order);
|
||||
RUN_TEST(test_worker_push_config_carries_configured_resync_window);
|
||||
}
|
||||
|
|
|
|||
4780
src/controller/testsuite/test_worker_resync.c
Normal file
4780
src/controller/testsuite/test_worker_resync.c
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -7,6 +7,7 @@
|
|||
#include "cluster.h"
|
||||
#include "controller.capnp.h"
|
||||
#include "persist.h"
|
||||
#include "resync.h" /* worker-rejoin-resync task 6.1 */
|
||||
#include "wobj.h"
|
||||
#include "worker.h"
|
||||
#include "xce/cluster_def.h"
|
||||
|
|
@ -331,6 +332,24 @@ static int apply_join_state(cluster_t* c, worker_t* w, endpoint_t* ep,
|
|||
w->host.status &= ~XCE_WORKER_UNAVAIL;
|
||||
w->ep = ep;
|
||||
|
||||
/* worker-rejoin-resync task 6.1 (Req 1.1-1.3): the exact moment this
|
||||
* host becomes "connected" again -- UNAVAIL cleared, ep set -- is the
|
||||
* natural trigger point for job-state reconciliation (design.md
|
||||
* Architecture: "rejoin_worker/apply_join_state ... already run under
|
||||
* the per-worker write lock at the exact moment w->ep is set... this is
|
||||
* the moment the host becomes connected again"). apply_join_state is
|
||||
* shared by BOTH init_worker (first-ever join) and rejoin_worker
|
||||
* (reconnect, whether a plain network reconnect or a rejoin to a new
|
||||
* primary after controller failover) under the SAME already-held write
|
||||
* lock, so this single call site satisfies Req 1.3 without needing to
|
||||
* separately determine which of the three a given join is.
|
||||
* resync_arm_round() is itself a documented no-op (Req 1.2) when the
|
||||
* host has no non-terminal jobs currently assigned to it -- only ARMS
|
||||
* the round here (w->resync + XCE_WORKER_RECONCILING); the actual query
|
||||
* send happens later, from on_worker_join, strictly after the join
|
||||
* reply and the config push (see worker_send_resync_query() below). */
|
||||
resync_arm_round(c, w);
|
||||
|
||||
if (strcmp(hconfig->hosttype, "!") != 0) {
|
||||
dereference(w->host.attrs->hosttype);
|
||||
w->host.attrs->hosttype = makeString(hconfig->hosttype);
|
||||
|
|
@ -586,7 +605,18 @@ static int check_conf4worker_(xce_config_t* c)
|
|||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* worker_push_config is non-static in worker.c when XCE_TESTING is defined
|
||||
* -- worker-rejoin-resync task 6.1, mirroring apply_join_state's own
|
||||
* XCE_TESTING exposure convention above, so tests can drive the exact
|
||||
* config-push-then-query wire ordering directly (see worker_send_resync_
|
||||
* query() below) without needing a live join flow.
|
||||
*/
|
||||
#ifdef XCE_TESTING
|
||||
int worker_push_config(cluster_t* c, endpoint_t* ep)
|
||||
#else
|
||||
static int worker_push_config(cluster_t* c, endpoint_t* ep)
|
||||
#endif
|
||||
{
|
||||
xce_configdata_t configdata;
|
||||
int rc = 0;
|
||||
|
|
@ -610,6 +640,20 @@ static int worker_push_config(cluster_t* c, endpoint_t* ep)
|
|||
configdata.data.conf4worker->clustername
|
||||
= c->myconf->cluster_config->clustername;
|
||||
|
||||
/* worker-rejoin-resync task 6.1 (Req 3.7): mirrors the clustername
|
||||
* field precedent immediately above exactly -- pulled directly from
|
||||
* ClusterConfig rather than the generic confvars key/value list, no new
|
||||
* opcode. Closes the gap task 4.1 discovered and documented in
|
||||
* xcecd_finished_job_retention_set_window_ms()'s own doc comment
|
||||
* (xcecd_worker_utils.h): "the worker daemon has no runtime delivery
|
||||
* path for this ClusterConfig field today... this setter is the seam a
|
||||
* later task wires the real operator-configured value into once that
|
||||
* delivery path exists." This is that seam's controller-side half; the
|
||||
* worker-side receive half is controller_connector.c's new
|
||||
* handle_pushconfig() (task 6.1). */
|
||||
configdata.data.conf4worker->worker_resync_window_ms
|
||||
= c->myconf->cluster_config->worker_resync_window_ms;
|
||||
|
||||
rc = xce_pushconfig_req(ep, c->myhost, &configdata);
|
||||
|
||||
FREEUP(configdata.data.conf4worker);
|
||||
|
|
@ -619,14 +663,118 @@ static int worker_push_config(cluster_t* c, endpoint_t* ep)
|
|||
return rc;
|
||||
}
|
||||
|
||||
/*
|
||||
* worker-rejoin-resync task 7.5 (bug-1755, Req 8.1-8.3): page-size cap for
|
||||
* this controller's own OPCode_jobStateQuery sends, mirroring the worker's
|
||||
* own already-wired XCECD_RESYNC_REPORT_PKTLIMIT (controller_connector.c)
|
||||
* for the report direction -- same constant-driven approach, applied here
|
||||
* to the query direction so xce_job_state_query_req()'s already-implemented
|
||||
* pagination loop (task 1.4) is genuinely exercised in production instead
|
||||
* of always collapsing to a single unbounded page (bug-1755's finding). A
|
||||
* JobStateQuery entry (bare jobid) is smaller on the wire than a
|
||||
* JobStateReport entry (jobid + status + exitCode + finishTimeMs +
|
||||
* queried), so the query direction can safely use the same page size the
|
||||
* report direction already proves safe, without inventing a smaller one.
|
||||
*/
|
||||
#define XCE_RESYNC_QUERY_PKTLIMIT 100
|
||||
|
||||
/*
|
||||
* worker_send_resync_query — worker-rejoin-resync task 6.1 (Req 1.1-1.3,
|
||||
* 2.1, 2.2): sends the round's job-state query to a worker for which
|
||||
* apply_join_state (above) already armed a reconciliation round earlier in
|
||||
* this same join. A worker with no armed round (w->resync == NULL, the
|
||||
* documented Req 1.2 no-op case) gets no query, silently -- this function
|
||||
* simply has nothing to do in that case, not an error.
|
||||
*
|
||||
* Precondition: caller does NOT hold w's lock (on_worker_join's call site,
|
||||
* below, runs after handle_worker_join's own lock scope has already ended)
|
||||
* -- this function acquires w's write lock itself, narrowly, only to copy
|
||||
* the round's queried-job-ID set, then releases it before the actual
|
||||
* (non-blocking, fire-and-forget, no reply awaited -- matching the config
|
||||
* push immediately above it) network send. The jobids array is copied
|
||||
* rather than sent directly from w->resync->queried_jobids so the send can
|
||||
* happen without holding w's lock across it: a concurrent rejoin of the
|
||||
* SAME host on another connection could otherwise call resync_arm_round()
|
||||
* again mid-send (resync_resolve_round() frees the prior round's array),
|
||||
* which would make an unlocked read of the raw pointer a use-after-free.
|
||||
*
|
||||
* XCE_TESTING-exposed (mirrors apply_join_state's/worker_push_config's own
|
||||
* convention above) so tests can drive this directly with a real,
|
||||
* fixture-armed worker_t, independent of get_ep_worker()'s endpoint->worker
|
||||
* resolution (on_worker_join's own concern, not this function's).
|
||||
*/
|
||||
#ifdef XCE_TESTING
|
||||
int worker_send_resync_query(cluster_t* c, worker_t* w, endpoint_t* ep)
|
||||
#else
|
||||
static int worker_send_resync_query(cluster_t* c, worker_t* w, endpoint_t* ep)
|
||||
#endif
|
||||
{
|
||||
xce_job_state_query_t query;
|
||||
uint64_t* ids = NULL;
|
||||
int n = 0;
|
||||
|
||||
if (c == NULL || w == NULL || ep == NULL)
|
||||
return -EINVAL;
|
||||
|
||||
lock(w, RTTI_LOCK_WRITE);
|
||||
if (w->resync != NULL) {
|
||||
n = w->resync->n_queried_jobids;
|
||||
ids = (uint64_t*)malloc((size_t)n * sizeof(uint64_t));
|
||||
if (ids != NULL) {
|
||||
memcpy(ids, w->resync->queried_jobids,
|
||||
(size_t)n * sizeof(uint64_t));
|
||||
} else {
|
||||
ERROR("worker_send_resync_query: worker %s: failed to copy "
|
||||
"%d queried jobid(s) -- query not sent this join (the "
|
||||
"round's own Req 5.3 timeout still resolves it)",
|
||||
(w->host.name != NULL) ? w->host.name : "?", n);
|
||||
n = 0;
|
||||
}
|
||||
}
|
||||
unlock(w);
|
||||
|
||||
if (n <= 0) {
|
||||
free(ids);
|
||||
return 0;
|
||||
}
|
||||
|
||||
memset(&query, 0, sizeof(query));
|
||||
query.n_total = n;
|
||||
query.jobids = ids;
|
||||
|
||||
xce_job_state_query_req(ep, c->myhost, &query, NULL, NULL,
|
||||
XCE_RESYNC_QUERY_PKTLIMIT);
|
||||
|
||||
free(ids);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* consider to push jobs on hosts to */
|
||||
int on_worker_join(void* cls, endpoint_t* ep)
|
||||
{
|
||||
cluster_t* c = (cluster_t*)cls;
|
||||
worker_t* w;
|
||||
int rc = 0;
|
||||
|
||||
rc = worker_push_config(c, ep);
|
||||
|
||||
/* worker-rejoin-resync task 6.1 (Req 1.1-1.3, 2.1, 2.2): if
|
||||
* apply_join_state (rejoin_worker/init_worker) armed a reconciliation
|
||||
* round for this worker earlier in this same join, send its query now
|
||||
* -- strictly AFTER the config push above (design.md File Structure
|
||||
* Plan for worker.c: "config push, then query", both fire-and-forget,
|
||||
* never an extension of the join call itself). get_ep_worker() is the
|
||||
* same ep->worker_t resolution job.c's update_job_rusages() already
|
||||
* relies on (misc.c) -- w == NULL (endpoint not resolvable to a known
|
||||
* worker) degrades to "no query sent," matching worker_send_resync_
|
||||
* query()'s own no-round no-op, not a crash. */
|
||||
w = get_ep_worker(c, ep);
|
||||
if (w != NULL) {
|
||||
worker_send_resync_query(c, w, ep);
|
||||
dereference(w);
|
||||
}
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue