forked from lei.chen/myxce
a793015f74
A worker-reported finish may arrive before the job exists: job
construction is deferred to apply_committed_entry (the submit's WAL
entry), leaving a commit->apply window. on_job_finish dropped such
finishes ('can not find job'), so a fast job that finished in that
window stayed RUN forever (e.g. noexec-test/newcfg-test/verify-fix).
Add an in-memory unknown-finish registry: on_job_finish registers the
finish when the job isn't materialized, and apply_committed_entry's
WAL_JOB_SUBMIT case drains it right after restore_job — appending the
durable WAL_JOB_FINISH and mutating the job to DONE/EXIT immediately so
the scheduler never dispatches a job that already finished.
Design: docs/Job-Finish-Robustness-设计文档.md (Part A of A+B; Part B
worker-liveness + reconnect reconciliation deferred to later phases).
Unit tests (red->green): registry register/take, apply-once-job-
materializes (DONE), exit-code maps to EXIT, drain no-op without
registration.
239 lines
12 KiB
C
239 lines
12 KiB
C
#if !defined(_CLUSTER_H_)
|
|
|
|
#define _CLUSTER_H_ 1
|
|
#include "config.h"
|
|
|
|
#include <pthread.h>
|
|
#include <stdbool.h>
|
|
#include <stdint.h>
|
|
|
|
#include "hook.h"
|
|
#include "htab.h"
|
|
#include "kvmgr.h"
|
|
#include "myconf.h"
|
|
#include "mylock.h"
|
|
#include "netio.h"
|
|
#include "vtaskmgr.h"
|
|
#include "worker.h"
|
|
#include "xce/cluster_def.h"
|
|
#include "xce/types.h"
|
|
#include "xlist.h"
|
|
#include "zarray.h"
|
|
|
|
typedef int (*collect_check_fn_t)(void*);
|
|
|
|
extern int check_controller(void*);
|
|
extern int cluster_init_phase1(cluster_t*);
|
|
extern int cluster_open_listener(cluster_t*);
|
|
extern int cluster_register_listener(cluster_t*);
|
|
extern int cluster_init_phase2(cluster_t*);
|
|
extern int cluster_init(cluster_t*);
|
|
extern int cluster_start(cluster_t*);
|
|
extern int cluster_run(cluster_t*);
|
|
extern void cluster_fini(cluster_t*);
|
|
extern int register_types(cluster_t*);
|
|
extern int worker_ctrl(cluster_t*, xce_hostctrl_t*);
|
|
extern xce_resource_t* get_resource(cluster_t*, char*);
|
|
extern int add_to_xlist(xlist_t*, void*);
|
|
extern int add_worker_to_xlist(xlist_t*, void*);
|
|
extern int add_queue_to_xlist(xlist_t*, void*);
|
|
extern int add_job_to_xlist(xlist_t*, void*);
|
|
extern void resource_killer(void*);
|
|
extern int add_resource(void*, char*, char*, int, xce_order_t, int, int);
|
|
|
|
extern int update_resvals(cluster_t*, worker_t*, int, xce_resval_t*);
|
|
|
|
extern int load_jobs(void*);
|
|
extern int restore_job(cluster_t*, uint64_t, const void*, size_t, uint64_t);
|
|
/* bug-1509: drain a worker finish that arrived before the job was
|
|
* materialized (registered by on_job_finish's unknown-finish path). Called
|
|
* from apply_committed_entry's WAL_JOB_SUBMIT case after restore_job. */
|
|
extern int apply_registered_finish_if_any(cluster_t*, job_t*);
|
|
|
|
#ifdef XCE_TESTING
|
|
/* Direct probes of the bug-1509 unknown-finish registry (test binary only). */
|
|
extern int test_unknown_finish_register(const xce_jobfinish_t*);
|
|
extern int test_unknown_finish_take(uint64_t, xce_jobfinish_t*);
|
|
#endif
|
|
/* job-array-single-wal-record task 3.1 (design.md Array Apply Expansion):
|
|
* shared N-way expansion primitive for a committed WAL_JOB_ARRAY_SUBMIT
|
|
* payload -- see the doc comment on the definition (job.c) for the full
|
|
* atomicity/idempotency contract. out_new_jobs is optional; when non-NULL
|
|
* and the call genuinely creates new elements, it is populated (ownership
|
|
* transferred) with every newly created job_t* in index order. */
|
|
extern int restore_job_array(cluster_t*, const void*, size_t, uint64_t,
|
|
xlist_t*);
|
|
#ifdef XCE_TESTING
|
|
/* job-array-single-wal-record task 3.1: forces restore_job_array()'s
|
|
* scratch-build loop to fail construction at the given 0-based element
|
|
* index (simulating resource exhaustion), so a test can prove the
|
|
* all-or-nothing atomicity contract. Pass a negative index, or call the
|
|
* clear helper, to disable. Compiled out of production. */
|
|
extern void job_array_apply_test_set_force_fail_at(int index);
|
|
extern void job_array_apply_test_clear_force_fail(void);
|
|
/* job-array-single-wal-record task 3.1 remediation: forces
|
|
* restore_job_array()'s BULK-INSERT pass (the key_add(c->jobmgr, ...) loop,
|
|
* distinct from the scratch-build loop above) to fail at the given 0-based
|
|
* index WITHIN THAT PASS, simulating c->jobmgr's backing store hitting a
|
|
* genuine realloc() OOM while growing under the write lock. Lets a test
|
|
* prove this failure mode is reported as a negative errno, never a false
|
|
* built.count success. Pass a negative index, or call the clear helper, to
|
|
* disable. Compiled out of production. */
|
|
extern void job_array_apply_test_set_bulk_insert_force_fail_at(int index);
|
|
extern void job_array_apply_test_clear_bulk_insert_force_fail(void);
|
|
#endif
|
|
extern int update_job(void*, uint64_t, xce_kv_list_t*);
|
|
extern int do_job_started(void*, uint64_t, int, int*, int, int*);
|
|
|
|
extern queue_t* create_queue(cluster_t*, xce_queue_t*);
|
|
extern int build_queues(void*);
|
|
extern void queue_fini(queue_t*);
|
|
extern int queue_ctrl(cluster_t*, xce_queuectrl_t*);
|
|
|
|
extern int xce_update_data(endpoint_t*, cluster_t*, int, int, int*, int,
|
|
xce_kv_t*);
|
|
extern int on_scheduler_join(cluster_t*);
|
|
extern int on_apiserver_join(cluster_t*, endpoint_t*);
|
|
|
|
extern int xce_push_reslist(endpoint_t*, cluster_t*, int, xce_resource_t**);
|
|
extern int xce_push_confvars(endpoint_t*, cluster_t*, int, xce_config_t**);
|
|
extern int xce_push_workerlist(endpoint_t*, cluster_t*, int, xce_worker_t**);
|
|
extern int xce_push_joblist(endpoint_t*, cluster_t*, int, xce_job_t**);
|
|
extern int xce_push_queuelist(endpoint_t*, cluster_t*, int, xce_queue_t**);
|
|
extern int xcesched_push_job(cluster_t*, uint64_t);
|
|
extern int on_worker_join(void*, endpoint_t*);
|
|
extern int get_resval(cluster_t*, char*, char*, xce_val_t*);
|
|
extern int handle_push(struct capn*, cluster_t*, endpoint_t*, struct Request*);
|
|
extern int do_get_op(cluster_t*, endpoint_t*, xce_request_t*);
|
|
extern int do_update_op(cluster_t*, endpoint_t*, xce_request_t*);
|
|
extern int handle_join(cluster_t*, endpoint_t*, xce_request_t*);
|
|
extern int handle_reply(endpoint_t*, struct Request*, cluster_t*);
|
|
extern int handle_jobdispatch(cluster_t*, endpoint_t*, xce_request_t*);
|
|
extern int handle_jobsubmit(cluster_t*, endpoint_t*, xce_request_t*);
|
|
#ifdef XCE_TESTING
|
|
/* bug-1419: clears job.c's process-lifetime in-flight-jobid registry.
|
|
* Tests that deliberately leave a submission's commit continuation
|
|
* unresolved through teardown (no ack, no timeout, no truncation) have no
|
|
* other event to release that registry entry -- call this from fixture
|
|
* teardown so it doesn't outlive the test and spuriously reject an
|
|
* unrelated later test reusing the same jobid range. */
|
|
extern void inflight_submits_reset_for_test(void);
|
|
#endif
|
|
extern int handle_worker_leave(cluster_t*, int);
|
|
extern int mark_worker_unavail(cluster_t*, worker_t*);
|
|
extern int mark_worker_back(cluster_t*, worker_t*);
|
|
extern int handle_scheduler_leave(cluster_t*);
|
|
extern int handle_clusterconf(cluster_t*, endpoint_t*, xce_request_t*);
|
|
|
|
/* controller-status spec (task 2.1): read-only status query.
|
|
* ctrlstatus_fill snapshots local HA/config state into *st (no I/O, no state
|
|
* mutation); ctrlstatus_free_fields releases what fill allocated (the peer
|
|
* pointer array). handle_ctrlstatus is the dispatch entry point that fills,
|
|
* replies via xce_reply, then frees. */
|
|
extern void ctrlstatus_fill(cluster_t*, xce_ctrlstatus_t*);
|
|
extern void ctrlstatus_free_fields(xce_ctrlstatus_t*);
|
|
extern int handle_ctrlstatus(cluster_t*, endpoint_t*, xce_request_t*);
|
|
|
|
extern void collect_confvars(cluster_t*, collect_check_fn_t, xlist_t*);
|
|
extern void collect_queues(cluster_t*, collect_check_fn_t, xlist_t*);
|
|
extern void collect_resources(cluster_t*, collect_check_fn_t, xlist_t*);
|
|
extern void collect_jobs(cluster_t*, collect_check_fn_t, xlist_t*);
|
|
extern void collect_workers(cluster_t*, collect_check_fn_t, xlist_t*);
|
|
|
|
/* ha-convergence-recovery (3.1-3.3): commit-watermark egress gate for
|
|
* scheduler-bound job exports. Defined in dataex.c.
|
|
* job_submit_committed — true iff the job's submit entry is committed
|
|
* (submit_lsn == 0 || submit_lsn <= committed_lsn);
|
|
* one lock-free atomic load + compare.
|
|
* collect_jobs_committed — identical contract to collect_jobs, additionally
|
|
* excluding jobs whose submit entry is uncommitted.
|
|
* For scheduler-bound callers ONLY; status/API read
|
|
* paths keep plain collect_jobs (Req 3.4). */
|
|
extern bool job_submit_committed(cluster_t*, job_t*);
|
|
extern void collect_jobs_committed(cluster_t*, collect_check_fn_t, xlist_t*);
|
|
extern int get_queue_exec(cluster_t*, char*, char**, char**);
|
|
extern void unlock_collect_queues(xlist_t*);
|
|
extern void unlock_collect_jobs(xlist_t*);
|
|
extern void unlock_collect_workers(xlist_t*);
|
|
|
|
/* Outcome of a guarded job state-transition core (raft-apply-decoupling
|
|
* task 2.1, Req 3.3/7.1/7.2): APPLIED means the mutation happened;
|
|
* REJECTED_BACKWARD/NOOP_DUPLICATE mean the monotonic-rank guard held the
|
|
* transition back and no mutation occurred. These cores fire no hook
|
|
* themselves — hook-firing is the caller's responsibility, so it can be
|
|
* relocated (to the apply seam) without touching this guard logic again. */
|
|
typedef enum {
|
|
JOB_TRANSITION_APPLIED = 0,
|
|
JOB_TRANSITION_REJECTED_BACKWARD,
|
|
JOB_TRANSITION_NOOP_DUPLICATE,
|
|
} job_transition_result_t;
|
|
|
|
/* Guarded state-transition cores extracted from mark_job_start/finish/fail/
|
|
* update (job.c). Each applies exactly one fact's mutation to an
|
|
* already-locked job_t, subject to the same monotonic-rank guard the
|
|
* receipt-time wrapper used to apply inline. Callable directly — e.g. by a
|
|
* test, or later by the shared apply seam — without going through the
|
|
* worker-report receipt path. */
|
|
extern job_transition_result_t apply_job_start_transition(job_t*,
|
|
job_pidinfo_t*);
|
|
extern job_transition_result_t apply_job_finish_transition(job_t*,
|
|
xce_jobfinish_t*);
|
|
extern job_transition_result_t apply_job_fail_transition(job_t*,
|
|
job_failinfo_t*);
|
|
extern job_transition_result_t apply_job_update_transition(job_t*, int);
|
|
|
|
/* Guarded state-transition core for a postdone fact (raft-apply-decoupling
|
|
* task 6.1 round 2, Req 3.3/7.1-7.3), extracted from on_job_postdone's old
|
|
* inline mutation. Unlike a generic UPDATE, postdone's formula (conditional
|
|
* EXIT backstop, then OR in JOB_STAT_PDONE) is a pure function of the job's
|
|
* CURRENT state — so, like the other cores above, it is meant to be invoked
|
|
* at apply time against whatever state actually holds then, never against a
|
|
* receipt-time snapshot. No-op (JOB_TRANSITION_NOOP_DUPLICATE) when PDONE is
|
|
* already set and no backstop is needed. Caller must hold `job` under
|
|
* RTTI_LOCK_WRITE. */
|
|
extern job_transition_result_t apply_job_postdone_transition(job_t*);
|
|
|
|
extern int on_job_start(cluster_t*, job_pidinfo_t*);
|
|
extern int on_job_finish(cluster_t*, xce_jobfinish_t*);
|
|
extern int on_job_fail(cluster_t*, job_failinfo_t*);
|
|
extern int on_job_stop(cluster_t*, uint64_t);
|
|
extern int on_job_resume(cluster_t*, uint64_t);
|
|
extern int on_job_postdone(cluster_t*, endpoint_t*, uint64_t);
|
|
extern int on_job_ckpt(cluster_t*, endpoint_t*, xce_ckptinfo_t*);
|
|
extern int notify(cluster_t*, endpoint_t*, xce_notify_data_t*);
|
|
extern int check_worker_(xce_worker_t*);
|
|
extern int update_resval_list(cluster_t*, worker_t*, xce_resval_list_t*);
|
|
extern int do_jobcontrol_op(cluster_t*, endpoint_t*, xce_request_t*);
|
|
extern int job_ctrl(cluster_t*, xce_jobctrl_t*);
|
|
extern int update_job_rusages(cluster_t*, endpoint_t*, xce_request_t*);
|
|
extern int update_host_runusage(xce_job_runusage_t**, char*, int,
|
|
xce_rusage_t**);
|
|
extern int update_pids(job_pidinfo_t**, int, int*);
|
|
extern int update_pgids(job_pidinfo_t**, int, int*);
|
|
extern int get_ep_role(endpoint_t*);
|
|
extern worker_t* get_ep_worker(cluster_t*, endpoint_t*);
|
|
|
|
extern int check_worker_(xce_worker_t*);
|
|
extern int update_resval_list(cluster_t*, worker_t*, xce_resval_list_t*);
|
|
extern int do_jobcontrol_op(cluster_t*, endpoint_t*, xce_request_t*);
|
|
extern int job_ctrl(cluster_t*, xce_jobctrl_t*);
|
|
extern int do_queuecontrol_op(cluster_t*, endpoint_t*, xce_request_t*);
|
|
extern int do_hostcontrol_op(cluster_t*, endpoint_t*, xce_request_t*);
|
|
extern int do_runtasks(cluster_t*, endpoint_t*, xce_request_t*);
|
|
extern int on_task_start(cluster_t*, xce_task_pidinfo_t*);
|
|
extern int on_task_fail(cluster_t*, xce_task_failinfo_t*);
|
|
extern int on_task_finish(cluster_t*, xce_task_finish_t*);
|
|
|
|
extern int xce_controller_res_init(cluster_t*);
|
|
extern int map_resource_to_host(char*, cluster_t*);
|
|
extern int release_resource_from_worker(char*, cluster_t*);
|
|
extern int collect_resval(var, zarray_t*, xlist_t*);
|
|
extern int do_set_preason(cluster_t*, endpoint_t*, xce_request_t*);
|
|
extern int do_clr_preason(cluster_t*, endpoint_t*, xce_request_t*);
|
|
extern int on_fetchfile(cluster_t*, endpoint_t*, xce_request_t*);
|
|
extern void fetchfile_signal(uint64_t jobid, int status, xce_fetchfile_reply_t*);
|
|
extern int handle_apiserver_leave(cluster_t*, endpoint_t*);
|
|
extern int add_apiserver(cluster_t*, char*, endpoint_t*);
|
|
extern int apiserver_init(cluster_t*);
|
|
|
|
#endif
|