Merge branch 'working' into job_peek_lchen
job_peek_lchen: Add fetchfile feature (job output/error peek proxy) working: HA/Raft infrastructure (consensus, replication, config mgmt) Conflict resolution summary: - controller.capnp: fetchfile @41, fetchfilereply @18, fetchfilemsg @30 - cluster.c: bifurcated OPCode_reply for fetchfile + config-push ACK - job.c: kept full fetchfile pending-request subsystem (~183 lines) - request.c: kept both xce_fetchfile_req() and XCE_RETRY_DEFAULTS - helper.c: added missing preVoteRequest/preVoteReply mappings - All 15 new test registrations (test_cbuf.c + 14 HA tests)
This commit is contained in:
Lei Chen
commit
3a2094a461
529 changed files with 345174 additions and 5048 deletions
|
|
@ -1,10 +1,14 @@
|
|||
#include <errno.h>
|
||||
#include <pthread.h>
|
||||
#include <signal.h>
|
||||
#include <stdbool.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
|
||||
#include "cluster.h"
|
||||
#include "controller.capnp.h"
|
||||
#include "jobj.h"
|
||||
#include "persist.h"
|
||||
#include "preason.h"
|
||||
#include "request.h"
|
||||
#include "xce/cluster_def.h"
|
||||
|
|
@ -17,9 +21,27 @@ static int mark_job_finish(cluster_t*, job_t*, xce_jobfinish_t*);
|
|||
static int mark_job_fail(cluster_t*, job_t*, job_failinfo_t*);
|
||||
static int mark_job_update(cluster_t*, job_t*, int);
|
||||
|
||||
#define NJOBS (100)
|
||||
/* A job is terminal once it carries a DONE or EXIT bit. Terminal jobs reject
|
||||
* backward (terminal -> non-terminal) transitions and treat re-applied
|
||||
* terminal transitions as duplicates; forward bit-OR merges (PDONE, PERR,
|
||||
* suspend bits) on a terminal job remain legal. */
|
||||
#define JOB_STATE_TERMINAL_BITS (JOB_STAT_DONE | JOB_STAT_EXIT)
|
||||
|
||||
uint64_t jobid = 100;
|
||||
static inline bool job_state_is_terminal(int state)
|
||||
{
|
||||
return (state & JOB_STATE_TERMINAL_BITS) != 0;
|
||||
}
|
||||
|
||||
/* Shared backward predicate for START and UPDATE (Req 5.2): a transition is
|
||||
* backward when the job is already terminal but the requested next state drops
|
||||
* every terminal bit it currently holds (e.g. START overwriting DONE with a
|
||||
* bare JOB_STAT_RUN, or an UPDATE clearing the terminal bits). A forward merge
|
||||
* that preserves at least one terminal bit (postdone OR'ing PDONE) is NOT
|
||||
* backward. */
|
||||
static inline bool job_state_is_backward(int cur, int next)
|
||||
{
|
||||
return job_state_is_terminal(cur) && !job_state_is_terminal(next);
|
||||
}
|
||||
|
||||
static job_t* create_job(cluster_t* c, xce_job_t* j)
|
||||
{
|
||||
|
|
@ -64,29 +86,70 @@ uint64_t job_submit(void* c, xce_job_submit_t* submit)
|
|||
|
||||
int load_jobs(void* c)
|
||||
{
|
||||
#if 0
|
||||
job_t* job;
|
||||
int i;
|
||||
char cmdline[1024];
|
||||
return 0;
|
||||
}
|
||||
|
||||
for (i = 1; i < NJOBS; i++) {
|
||||
sprintf(cmdline, "echo normal; sleep %d", i);
|
||||
job = create_job(c, jobid++, "normal", "rshen", "/home/rshen", "/tmp",
|
||||
"rshen", (char*)cmdline, NULL, 1, 1);
|
||||
if (job == NULL) {
|
||||
return -errno;
|
||||
}
|
||||
}
|
||||
/* Reconstruct a job from a WAL_JOB_SUBMIT Cap'n Proto payload.
|
||||
* Uses the jobid embedded in the serialized xce_job_t (not the caller's arg).
|
||||
* entry_lsn is the WAL entry LSN of this submit record (ha-convergence-recovery
|
||||
* 3.3): the restored job is tagged with it so its commit status is decidable at
|
||||
* scheduler-export time. 0 = unknown (pre-feature / direct create) = exportable.
|
||||
* Returns -EEXIST if the job is already present (idempotent replay guard). */
|
||||
int restore_job(cluster_t* c, uint64_t jobid_unused,
|
||||
const void* capnp_payload, size_t len, uint64_t entry_lsn)
|
||||
{
|
||||
(void)jobid_unused;
|
||||
|
||||
for (i = 1; i < NJOBS; i++) {
|
||||
job = create_job(c, jobid++, "xce", "rshen", "/home/rshen", "/tmp",
|
||||
"rshen", "echo xce; sleep 30", NULL, 1, 1);
|
||||
if (job == NULL) {
|
||||
return -errno;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
if (!capnp_payload || len == 0)
|
||||
return -EINVAL;
|
||||
|
||||
struct capn c_ctx;
|
||||
capn_init_mem(&c_ctx, (const uint8_t*)capnp_payload, len, 0);
|
||||
Job_ptr jp;
|
||||
jp.p = capn_getp(capn_root(&c_ctx), 0, 1);
|
||||
struct Job cj;
|
||||
xce_job_t j;
|
||||
memset(&j, 0, sizeof(j));
|
||||
read_Job(&cj, jp);
|
||||
decode_Job(&j, &cj);
|
||||
capn_free(&c_ctx);
|
||||
|
||||
/* Safety guard: skip if already present (WAL replay may see duplicates
|
||||
* when recovery restarts from a segment boundary). */
|
||||
job_t* existing = NULL;
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
int found = key_find(c->jobmgr, (void*)j.jobid, (void**)&existing);
|
||||
unlock(c->jobmgr);
|
||||
if (found > 0 && existing != NULL)
|
||||
return -EEXIST;
|
||||
|
||||
job_t* job = create_job(c, &j);
|
||||
if (!job)
|
||||
return -errno;
|
||||
/* ha-convergence-recovery (3.3): tag the restored job with its submit
|
||||
* entry's LSN so the egress watermark predicate can decide its commit
|
||||
* status (job_submit_committed). On a promoted primary, replayed entries
|
||||
* sit at or below the promotion truncation point, so restored jobs are
|
||||
* exportable by construction. */
|
||||
job->submit_lsn = entry_lsn;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Apply a recorded job state transition during WAL recovery.
|
||||
* Looks up the job (which must already exist from WAL_JOB_SUBMIT replay)
|
||||
* and overwrites its state with the value stored in the WAL entry. */
|
||||
int restore_job_state(cluster_t* c, uint64_t jobid, uint32_t state)
|
||||
{
|
||||
job_t* j = NULL;
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&j);
|
||||
unlock(c->jobmgr);
|
||||
if (!j)
|
||||
return -ENOENT;
|
||||
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
j->job.state = (int)state;
|
||||
unlock(j);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -227,6 +290,11 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
|
||||
rc = get_queue_exec(c, j->job.submit->queue, &q_pre_exec, &q_post_exec);
|
||||
if (rc < 0) {
|
||||
unlock(w);
|
||||
dereference(w);
|
||||
dereference(j);
|
||||
free_launchinfo(&launchinfo);
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
|
@ -292,6 +360,15 @@ int job_ctrl(cluster_t* c, xce_jobctrl_t* ctrinfo)
|
|||
return -ENOENT;
|
||||
}
|
||||
|
||||
/* bug-447: key_find returns 1 on found (binary librtti, unnormalizable
|
||||
* upstream). Normalize the found-rc to 0 so the not-started siginfo
|
||||
* branch below (which appends via the mark_job_* hooks but never
|
||||
* reassigns rc) returns 0 to the handler's commit-barrier pattern
|
||||
* instead of leaking 1 as the reply status. Real errors are
|
||||
* untouched: rc < 0 returned above, the j == NULL miss returned
|
||||
* -ENOENT, and every other branch sets rc itself. */
|
||||
rc = 0;
|
||||
|
||||
/* FIXME: eliminate held multiple locks */
|
||||
lock(j, RTTI_LOCK_READ);
|
||||
if (j->job.allocs != NULL) {
|
||||
|
|
@ -310,6 +387,9 @@ int job_ctrl(cluster_t* c, xce_jobctrl_t* ctrinfo)
|
|||
unlock(c->workermgr);
|
||||
|
||||
if (rc < 0) {
|
||||
unlock(j);
|
||||
dereference(j);
|
||||
|
||||
return rc;
|
||||
}
|
||||
if (w != NULL) {
|
||||
|
|
@ -487,16 +567,12 @@ int handle_jobsubmit(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
return -errno;
|
||||
}
|
||||
|
||||
reply.status = 0;
|
||||
reply.data_which = RequestReply_data_jobid;
|
||||
reply.data.jobid = jobid;
|
||||
|
||||
xce_reply(ep, c->myhost, &reply);
|
||||
|
||||
/* Build job list for the AFTER_EVENT hook BEFORE replying to client.
|
||||
* The AFTER_EVENT hook writes WAL and waits for majority replication —
|
||||
* the client must not be acknowledged until the job is durable. */
|
||||
xlist_init(&jlist, submit->array_size);
|
||||
|
||||
for (i = 0; i < submit->array_size; i++) {
|
||||
/* Run hooks to tell scheduler */
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
rc = key_find(c->jobmgr, (void*)(jobid + i), (void**)&j);
|
||||
if (j != NULL) {
|
||||
|
|
@ -516,18 +592,52 @@ int handle_jobsubmit(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
}
|
||||
}
|
||||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||||
HOOK_DETAIL_JOB_NEW, c, &jlist);
|
||||
int hook_rc = xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB,
|
||||
HOOK_AFTER_EVENT,
|
||||
HOOK_DETAIL_JOB_NEW, c, &jlist);
|
||||
|
||||
xlist_iter(&jlist, (xlist_iter_fn_t)xlist_deref_job_item_, NULL);
|
||||
|
||||
xlist_fini(&jlist);
|
||||
|
||||
/* If persistence failed (replication timeout, I/O error), reject the
|
||||
* submission so the client knows the job was not made durable. */
|
||||
if (hook_rc != 0) {
|
||||
WARN("handle_jobsubmit: persistence hook failed (rc=%d) — rejecting job",
|
||||
hook_rc);
|
||||
reply.status = hook_rc;
|
||||
reply.data_which = RequestReply_data_nodata;
|
||||
|
||||
xce_reply(ep, c->myhost, &reply);
|
||||
return hook_rc;
|
||||
}
|
||||
|
||||
/* Reply only AFTER WAL write + majority replication succeeded. */
|
||||
reply.status = 0;
|
||||
reply.data_which = RequestReply_data_jobid;
|
||||
reply.data.jobid = jobid;
|
||||
|
||||
xce_reply(ep, c->myhost, &reply);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int mark_job_start(cluster_t* c, job_t* job, job_pidinfo_t* info)
|
||||
{
|
||||
int cur = job->job.state;
|
||||
|
||||
/* Transition guard (Req 5.2): START sets the non-terminal JOB_STAT_RUN. A
|
||||
* START on an already-terminal job (stale/duplicate worker report after
|
||||
* the job finished) is a backward transition — reject it, preserve the
|
||||
* terminal state, and fire NO hook (no WAL append). Return success-no-op
|
||||
* so the caller (on_job_start) still reports success. */
|
||||
if (job_state_is_backward(cur, JOB_STAT_RUN)) {
|
||||
WARN("job %ld: rejecting backward START on terminal state 0x%x — "
|
||||
"preserving terminal state",
|
||||
job->job.jobid, cur);
|
||||
return 0;
|
||||
}
|
||||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_JOB_START, c, info, job);
|
||||
/* update job information */
|
||||
|
|
@ -583,6 +693,19 @@ int on_job_start(cluster_t* c, job_pidinfo_t* info)
|
|||
|
||||
static int mark_job_finish(cluster_t* c, job_t* job, xce_jobfinish_t* finish)
|
||||
{
|
||||
int cur = job->job.state;
|
||||
|
||||
/* Transition guard (Req 5.1): FINISH drives the job to a terminal state
|
||||
* (DONE on exit_code 0, else EXIT). If the job is already terminal the
|
||||
* FINISH has already been applied (retransmit / WAL replay) — log it and
|
||||
* return a success-no-op WITHOUT firing the AFTER_EVENT hook, so no second
|
||||
* WAL entry is appended and current_lsn does not advance. */
|
||||
if (job_state_is_terminal(cur)) {
|
||||
INFO("job %ld: duplicate FINISH on terminal state 0x%x — no-op",
|
||||
job->job.jobid, cur);
|
||||
return 0;
|
||||
}
|
||||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_JOB_EXIT, c, finish, job);
|
||||
/* update job information */
|
||||
|
|
@ -671,6 +794,26 @@ static int mark_job_fail(cluster_t* c, job_t* job, job_failinfo_t* failinfo)
|
|||
|
||||
static int mark_job_update(cluster_t* c, job_t* job, int state)
|
||||
{
|
||||
int cur = job->job.state;
|
||||
|
||||
/* Transition guard (Req 5.1/5.2), sharing the backward predicate with
|
||||
* START: an UPDATE that clears every terminal bit a terminal job holds is
|
||||
* backward — reject it, preserve the terminal state, fire no hook. A
|
||||
* forward merge that keeps a terminal bit (postdone OR'ing PDONE) is not
|
||||
* backward and proceeds. A no-op update (requested state already current)
|
||||
* is logged and skipped so it appends no duplicate WAL entry. */
|
||||
if (job_state_is_backward(cur, state)) {
|
||||
WARN("job %ld: rejecting backward UPDATE 0x%x -> 0x%x — preserving "
|
||||
"terminal state",
|
||||
job->job.jobid, cur, state);
|
||||
return 0;
|
||||
}
|
||||
if (state == cur) {
|
||||
INFO("job %ld: duplicate UPDATE to state 0x%x — no-op", job->job.jobid,
|
||||
cur);
|
||||
return 0;
|
||||
}
|
||||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_JOB_UPDATE, c, job);
|
||||
/* update job state */
|
||||
|
|
@ -732,8 +875,12 @@ int on_job_stop(cluster_t* c, uint64_t jobid)
|
|||
lock(job, RTTI_LOCK_WRITE);
|
||||
state = job->job.state;
|
||||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_JOB_STOP, c, job);
|
||||
state |= JOB_STAT_USUSP;
|
||||
mark_job_update(c, job, state);
|
||||
job->job.state = state;
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||||
HOOK_DETAIL_JOB_STOP, c, job);
|
||||
unlock(job);
|
||||
|
||||
dereference(job);
|
||||
|
|
@ -762,8 +909,12 @@ int on_job_resume(cluster_t* c, uint64_t jobid)
|
|||
|
||||
lock(job, RTTI_LOCK_WRITE);
|
||||
state = job->job.state;
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_JOB_RESUME, c, job);
|
||||
state &= ~JOB_STAT_USUSP;
|
||||
mark_job_update(c, job, state);
|
||||
job->job.state = state;
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||||
HOOK_DETAIL_JOB_RESUME, c, job);
|
||||
unlock(job);
|
||||
|
||||
dereference(job);
|
||||
|
|
@ -863,7 +1014,20 @@ int do_jobcontrol_op(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
|
||||
init_reply_by_req(&reply, request);
|
||||
|
||||
rc = job_ctrl(c, request->body.jobctrlmsg);
|
||||
rc = job_ctrl(c, request->body.jobctrlmsg);
|
||||
|
||||
/* Commit barrier (ha-commit-barrier-rollout Req 1.1-1.4): the success
|
||||
* reply is sent only after the operation's WAL entry is majority-
|
||||
* committed; barrier failures (-ETIMEDOUT, ...) become the error
|
||||
* reply. Errors from job_ctrl reply immediately — no barrier wait.
|
||||
* job_ctrl released all entity locks before returning, so blocking
|
||||
* here holds no lock. timeout 0 = configured sync_timeout_ms. */
|
||||
if (rc == 0) {
|
||||
rc = persist_commit_barrier(&c->persist, 0);
|
||||
if (rc != 0)
|
||||
WARN("%s commit barrier failed, rc %d", __func__, rc);
|
||||
}
|
||||
|
||||
reply.status = rc;
|
||||
reply.data_which = RequestReply_data_nodata;
|
||||
|
||||
|
|
@ -897,6 +1061,8 @@ int update_job_rusages(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
|
||||
w = get_ep_worker(c, ep);
|
||||
if (w == NULL) {
|
||||
dereference(j);
|
||||
|
||||
return errno;
|
||||
}
|
||||
|
||||
|
|
@ -1040,7 +1206,7 @@ int do_clr_preason(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
FREEUP(job->pend_reasons);
|
||||
}
|
||||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||||
HOOK_DETAIL_CLRPREASON, c, j);
|
||||
unlock(j);
|
||||
|
||||
|
|
@ -1234,3 +1400,7 @@ int on_fetchfile(cluster_t* c, endpoint_t* caller_ep, xce_request_t* request)
|
|||
xce_reply(caller_ep, c->myhost, &reply);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* job_is_terminal and job_should_retain live in persist.c —
|
||||
* they only depend on xce_job_t fields and persist_config_t,
|
||||
* and must be accessible from the unit test binary which links persist.c. */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue