sched_job_exit_fun() (renamed sched_job_finish_fun, sched.c) subscribes
to HOOK_DETAIL_JOB_EXIT, which fires for EVERY job finish, both success
and failure -- record_job_finish_fact() (job.c) decides DONE vs EXIT
from finish->exit_code BEFORE firing this same hook detail either way
(mirrors persist.c's own HOOK_DETAIL_JOB_EXIT -> WAL_JOB_FINISH
mapping). This function discarded info->exit_code and sent only a bare
jobid, forcing sched_handle_notify() (scheduler/src/handler.c) to
unconditionally map EVERY finish -- including a clean, successful one
-- to JOB_STAT_EXIT. A subsequent HOOK_DETAIL_JOB_UPDATE notify usually
self-corrected this moments later via the full job state, but if that
second notify was dropped during a scheduler reconnect gap (the same
window bug-1731/bug-1734 already documented), the scheduler was left
permanently stuck showing EXIT for a job the controller correctly
recorded as DONE.
Live-reproduced on a real 3-node HA cluster + worker: job
0-8-1788143749-0[38] genuinely completed successfully (worker log:
"completed successfully", controller joblist: DONE) but the
scheduler's own replay showed EXIT.
Fix: carry {jobid, exit_code} across the wire via the existing
JobFinishInfo shape (the same struct apiserver_job_exit_fun already
reuses for this identical hook) instead of a bare UInt64, so
sched_handle_notify() can make the identical exit_code==0-vs-not
decision job.c itself makes. Renamed sched_job_exit_fun to
sched_job_finish_fun to reflect what it actually does.
Live-verified: a 51-job run (all exit 0) showed 100% DONE on the
scheduler side after the fix.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019QxaSzHRB7pmYFp4cVnwkA
427 lines
11 KiB
C
427 lines
11 KiB
C
#include <errno.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
|
|
#include "controller.capnp.h"
|
|
#include "helper.h"
|
|
#include "typemap.h"
|
|
#include "xce/types.h"
|
|
|
|
void* xce_request_body(xce_request_t* req)
|
|
{
|
|
switch (req->body_which) {
|
|
case Request_body_nomsg:
|
|
break;
|
|
case Request_body_replymsg:
|
|
return req->body.replymsg;
|
|
case Request_body_hellomsg:
|
|
return req->body.hellomsg;
|
|
case Request_body_joinmsg:
|
|
return req->body.joinmsg;
|
|
case Request_body_configpushmsg:
|
|
return req->body.configpushmsg;
|
|
case Request_body_pushmsg:
|
|
return req->body.pushmsg;
|
|
case Request_body_getmsg:
|
|
return req->body.getmsg;
|
|
case Request_body_datamsg:
|
|
return req->body.datamsg;
|
|
case Request_body_dispatchmsg:
|
|
return req->body.dispatchmsg;
|
|
case Request_body_jobstartmsg:
|
|
return req->body.jobstartmsg;
|
|
case Request_body_notifymsg:
|
|
return req->body.notifymsg;
|
|
case Request_body_jobsubmsg:
|
|
return req->body.jobsubmsg;
|
|
case Request_body_updatemsg:
|
|
return req->body.updatemsg;
|
|
case Request_body_jobctrlmsg:
|
|
return req->body.jobctrlmsg;
|
|
case Request_body_queuectrlmsg:
|
|
return req->body.queuectrlmsg;
|
|
case Request_body_hostctrlmsg:
|
|
return req->body.hostctrlmsg;
|
|
case Request_body_runtaskmsg:
|
|
return req->body.runtaskmsg;
|
|
case Request_body_preasonmsg:
|
|
return req->body.preasonmsg;
|
|
case Request_body_clrpreasonmsg:
|
|
return req->body.clrpreasonmsg;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
char* xce_request_msg_name(xce_request_t* req)
|
|
{
|
|
switch (req->body_which) {
|
|
case Request_body_nomsg:
|
|
return "nomsg";
|
|
case Request_body_replymsg:
|
|
return "replymsg";
|
|
case Request_body_hellomsg:
|
|
return "hellomsg";
|
|
case Request_body_joinmsg:
|
|
return "joinmsg";
|
|
case Request_body_configpushmsg:
|
|
return "configmsg";
|
|
case Request_body_pushmsg:
|
|
return "pushmsg";
|
|
case Request_body_getmsg:
|
|
return "getmsg";
|
|
case Request_body_datamsg:
|
|
return "datamsg";
|
|
case Request_body_dispatchmsg:
|
|
return "dispatchmsg";
|
|
case Request_body_jobstartmsg:
|
|
return "jobstartmsg";
|
|
case Request_body_notifymsg:
|
|
return "notifymsg";
|
|
case Request_body_jobsubmsg:
|
|
return "jobsubmsg";
|
|
case Request_body_updatemsg:
|
|
return "updatemsg";
|
|
case Request_body_jobctrlmsg:
|
|
return "jobctrlmsg";
|
|
case Request_body_queuectrlmsg:
|
|
return "queuectrlmsg";
|
|
case Request_body_hostctrlmsg:
|
|
return "hostctrlmsg";
|
|
case Request_body_runtaskmsg:
|
|
return "runtaskmsg";
|
|
case Request_body_preasonmsg:
|
|
return "preasonmsg";
|
|
case Request_body_clrpreasonmsg:
|
|
return "clrpreasonmsg";
|
|
default:
|
|
break;
|
|
}
|
|
|
|
return "unknown";
|
|
}
|
|
|
|
char* xce_request_op_name(xce_request_t* req)
|
|
{
|
|
switch (req->opcode) {
|
|
case OPCode_nop:
|
|
return "nop";
|
|
case OPCode_reply:
|
|
return "reply";
|
|
case OPCode_hello:
|
|
return "hello";
|
|
case OPCode_join:
|
|
return "join";
|
|
case OPCode_shutdown:
|
|
return "shutdown";
|
|
case OPCode_get:
|
|
return "get";
|
|
case OPCode_push:
|
|
return "push";
|
|
case OPCode_update:
|
|
return "update";
|
|
case OPCode_jobdispatch:
|
|
return "jobdispatch";
|
|
case OPCode_startjob:
|
|
return "startjob";
|
|
case OPCode_notify:
|
|
return "notify";
|
|
case OPCode_jobsub:
|
|
return "jobsub";
|
|
case OPCode_jobstatus:
|
|
return "jobstatus";
|
|
case OPCode_jobrusage:
|
|
return "jobrusage";
|
|
case OPCode_hostcontrol:
|
|
return "hostcontrol";
|
|
case OPCode_jobcontrol:
|
|
return "jobcontrol";
|
|
case OPCode_clusterconf:
|
|
return "clusterconf";
|
|
case OPCode_queuecontrol:
|
|
return "queuecontrol";
|
|
case OPCode_runtasks:
|
|
return "runtasks";
|
|
case OPCode_setpreason:
|
|
return "setpreason";
|
|
case OPCode_clrpreason:
|
|
return "clrpreason";
|
|
case OPCode_heartbeat:
|
|
return "heartbeat";
|
|
case OPCode_redirect:
|
|
return "redirect";
|
|
case OPCode_steppingDown:
|
|
return "steppingDown";
|
|
case OPCode_replEntry:
|
|
return "replEntry";
|
|
case OPCode_replAck:
|
|
return "replAck";
|
|
case OPCode_replSnapshot:
|
|
return "replSnapshot";
|
|
case OPCode_replCatchUpDone:
|
|
return "replCatchUpDone";
|
|
case OPCode_replConfig:
|
|
return "replConfig";
|
|
case OPCode_replConfigPush:
|
|
return "replConfigPush";
|
|
case OPCode_upgradeConfig:
|
|
return "upgradeConfig";
|
|
case OPCode_upgradeConfigAck:
|
|
return "upgradeConfigAck";
|
|
case OPCode_requestVote:
|
|
return "requestVote";
|
|
case OPCode_probeConfigTag:
|
|
return "probeConfigTag";
|
|
case OPCode_walAck:
|
|
return "walAck";
|
|
case OPCode_walCatchupEntry:
|
|
return "walCatchupEntry";
|
|
case OPCode_snapshotTransfer:
|
|
return "snapshotTransfer";
|
|
case OPCode_preVoteRequest:
|
|
return "preVoteRequest";
|
|
case OPCode_preVoteReply:
|
|
return "preVoteReply";
|
|
case OPCode_ctrlstatus:
|
|
return "ctrlstatus";
|
|
case OPCode_fetchfile:
|
|
return "fetchfile";
|
|
default:
|
|
break;
|
|
}
|
|
|
|
return "unknown";
|
|
}
|
|
|
|
void* xce_val_ptr(xce_val_t* v)
|
|
{
|
|
switch (v->which) {
|
|
case Value_u_ivalue32:
|
|
return &(v->u.ivalue32);
|
|
case Value_u_uvalue32:
|
|
return &(v->u.uvalue32);
|
|
case Value_u_ivalue64:
|
|
return &(v->u.ivalue64);
|
|
case Value_u_uvalue64:
|
|
return &(v->u.uvalue64);
|
|
case Value_u_svalue:
|
|
return v->u.svalue;
|
|
case Value_u_fvalue:
|
|
return &(v->u.fvalue);
|
|
case Value_u_bvalue:
|
|
return &(v->u.bvalue);
|
|
default:
|
|
break;
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
void* xce_configdata_ptr(xce_configdata_t* d)
|
|
{
|
|
switch (d->data_which) {
|
|
case ConfigData_data_schedconfig:
|
|
return d->data.conf4sched;
|
|
case ConfigData_data_workerconfig:
|
|
return d->data.conf4worker;
|
|
case ConfigData_data_clusterconfig:
|
|
return d->data.clusterconfig;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
void* xce_data_ptr(xce_data_t* d)
|
|
{
|
|
switch (d->table) {
|
|
case XceData_data_schedjobs:
|
|
return d->data.jobs;
|
|
case XceData_data_schedworkers:
|
|
return d->data.workers;
|
|
case XceData_data_jobs:
|
|
return d->data.jobs;
|
|
case XceData_data_workers:
|
|
return d->data.workers;
|
|
case XceData_data_resources:
|
|
return d->data.resources;
|
|
case XceData_data_resvals:
|
|
return d->data.resvals;
|
|
default:
|
|
break;
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
void* xce_update_data_ptr(xce_update_data_t* d)
|
|
{
|
|
switch (d->table) {
|
|
case Update_data_nodata:
|
|
break;
|
|
case Update_data_resvals:
|
|
return d->data.resvals;
|
|
case Update_data_jobrusages:
|
|
return d->data.jobrusages;
|
|
default:
|
|
break;
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
void* xce_notify_data_ptr(xce_notify_data_t* d)
|
|
{
|
|
switch (d->event) {
|
|
case NotifyData_data_jobPidInfo:
|
|
return d->data.pidinfo;
|
|
case NotifyData_data_jobFailInfo:
|
|
return d->data.failinfo;
|
|
case NotifyData_data_jobDispatchFail:
|
|
return d->data.dispatchfail;
|
|
case NotifyData_data_jobFinishInfo:
|
|
return d->data.finishinfo;
|
|
case NotifyData_data_schedStartJob:
|
|
return (void*)d->data.sched_start_job;
|
|
case NotifyData_data_schedExitJob:
|
|
/* bug-1737: sched_exit_job is now a real xce_jobfinish_t* (was a
|
|
* bare uint64_t jobid cast through void*). */
|
|
return d->data.sched_exit_job;
|
|
case NotifyData_data_schedFailJob:
|
|
return (void*)d->data.sched_fail_job;
|
|
case NotifyData_data_schedUpdateJob:
|
|
return d->data.sched_update_job;
|
|
case NotifyData_data_schedUpdateResvals:
|
|
return d->data.sched_update_resvals;
|
|
case NotifyData_data_stoppedJob:
|
|
return (void*)d->data.stopped_job;
|
|
case NotifyData_data_resumedJob:
|
|
return (void*)d->data.resumed_job;
|
|
case NotifyData_data_ckptinfo:
|
|
return d->data.ckptinfo;
|
|
case NotifyData_data_schedUpdateQueue:
|
|
return d->data.sched_update_queue;
|
|
case NotifyData_data_taskPidInfo:
|
|
return d->data.taskinfo;
|
|
case NotifyData_data_taskFailInfo:
|
|
return d->data.taskfailinfo;
|
|
case NotifyData_data_taskFinish:
|
|
return d->data.taskfinish;
|
|
case NotifyData_data_postDoneJob:
|
|
return (void*)d->data.pdone_job;
|
|
case NotifyData_data_schedJobRusages:
|
|
return d->data.sched_job_rusages;
|
|
case NotifyData_data_updateJob:
|
|
return d->data.update_job;
|
|
default:
|
|
break;
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
void* xce_jobctrl_data_ptr(xce_jobctrl_t* d)
|
|
{
|
|
switch (d->action) {
|
|
case JobControl_u_siginfo:
|
|
return d->u.siginfo;
|
|
case JobControl_u_ckptoption:
|
|
return d->u.ckptoption;
|
|
default:
|
|
break;
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
void* xce_reply_data_ptr(xce_reply_t* reply)
|
|
{
|
|
switch (reply->data_which) {
|
|
case RequestReply_data_nodata:
|
|
break;
|
|
case RequestReply_data_debugmsg:
|
|
return reply->data.debugmsg;
|
|
case RequestReply_data_getdata:
|
|
return reply->data.getdata;
|
|
case RequestReply_data_jobid:
|
|
return (void*)reply->data.jobid;
|
|
case RequestReply_data_clusterconf:
|
|
return reply->data.clusterconf;
|
|
default:
|
|
break;
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
char* xce_dataset_table(int table)
|
|
{
|
|
switch (table) {
|
|
case XceData_data_schedjobs:
|
|
return "schedjobs";
|
|
case XceData_data_jobs:
|
|
return "jobs";
|
|
case XceData_data_workers:
|
|
return "workers";
|
|
case XceData_data_schedworkers:
|
|
return "schedworkers";
|
|
case XceData_data_resources:
|
|
return "resources";
|
|
case XceData_data_resvals:
|
|
return "resvals";
|
|
default:
|
|
break;
|
|
}
|
|
|
|
return "unknown";
|
|
}
|
|
|
|
char* xce_notify_event_name(int event)
|
|
{
|
|
static char buf[32];
|
|
|
|
switch (event) {
|
|
case NotifyData_data_jobPidInfo:
|
|
return "jobPidInfo";
|
|
case NotifyData_data_jobFailInfo:
|
|
return "jobFailInfo";
|
|
case NotifyData_data_jobDispatchFail:
|
|
return "jobDispatchFail";
|
|
case NotifyData_data_jobFinishInfo:
|
|
return "jobFinishInfo";
|
|
case NotifyData_data_schedStartJob:
|
|
return "schedStartJob";
|
|
case NotifyData_data_schedExitJob:
|
|
return "schedExitJob";
|
|
case NotifyData_data_schedFailJob:
|
|
return "schedFailJob";
|
|
case NotifyData_data_schedUpdateJob:
|
|
return "schedUpdateJob";
|
|
case NotifyData_data_schedUpdateResvals:
|
|
return "schedUpdateResvals";
|
|
case NotifyData_data_stoppedJob:
|
|
return "stoppedJob";
|
|
case NotifyData_data_resumedJob:
|
|
return "resumedJob";
|
|
case NotifyData_data_ckptinfo:
|
|
return "ckptinfo";
|
|
case NotifyData_data_schedUpdateQueue:
|
|
return "schedUpdateQueue";
|
|
case NotifyData_data_taskPidInfo:
|
|
return "taskPidInfo";
|
|
case NotifyData_data_taskFailInfo:
|
|
return "taskFailInfo";
|
|
case NotifyData_data_taskFinish:
|
|
return "taskFinish";
|
|
case NotifyData_data_postDoneJob:
|
|
return "postDoneJob";
|
|
case NotifyData_data_schedJobRusages:
|
|
return "schedJobRusages";
|
|
case NotifyData_data_updateJob:
|
|
return "updateJob";
|
|
default:
|
|
break;
|
|
}
|
|
|
|
snprintf(buf, sizeof(buf), "UNKNOWN(%d)", event);
|
|
return buf;
|
|
}
|
|
|