fix(controller,worker): dual-state worker loss + resend actual job state (bug-1509 P3)
Phase 3 (B1 dual-state + B2): a job never lingers as RUN when its worker is lost. Dual-state B1 (user-approved): RDHUP (worker TCP dropped = confirmed dead) marks its RUN jobs EXIT immediately (mark_worker_jobs_exit). Heartbeat timeout (unknown) keeps them UNKWN+TRANSIENT awaiting reconnect reconciliation; the TRANSIENT TTL GC is the backstop. handle_worker_leave now calls mark_worker_jobs_exit (was mark_worker_jobs_unknown in Phase 2). B2 (worker side): resend_job_status now checks the recovered process is alive (kill(pid,0)) before re-reporting RUN — a dead process is reported finished so the controller converges to EXIT instead of a stale RUN. And resend_job_status runs on EVERY rejoin (checking_connection), not just startup, so a reconnecting worker's jobs are reconciled (clears UNKWN via the existing notify_jobstart claim path). Design: docs/Job-Finish-Robustness-设计文档.md §4.4 (dual-state decision). Unit test: worker RDHUP marks jobs EXIT (not UNKWN).
This commit is contained in:
parent
fc091348e8
commit
d3a92f1033
7 changed files with 135 additions and 18 deletions
|
|
@ -2183,6 +2183,69 @@ void mark_all_run_jobs_unknown_in_memory(cluster_t* c)
|
|||
mark_jobs_unknown_core(c, NULL, 0);
|
||||
}
|
||||
|
||||
/* bug-1509 Phase 3 (B1 dual-state): a worker whose TCP connection dropped
|
||||
* (RDHUP) is CONFIRMED dead from the controller's perspective — mark its RUN
|
||||
* jobs EXIT immediately (terminal) instead of leaving them unknown. This is
|
||||
* distinct from the heartbeat-timeout path (mark_worker_jobs_unknown), which
|
||||
* keeps the job UNKWN awaiting reconnect reconciliation. Caller must not hold
|
||||
* any worker/jobmgr lock (B5). */
|
||||
void mark_worker_jobs_exit(cluster_t* c, const char* hostname)
|
||||
{
|
||||
iterator_t it;
|
||||
job_t* j;
|
||||
|
||||
if (c == NULL || hostname == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
if (iter_make(&it, c->jobmgr) < 0) {
|
||||
unlock(c->jobmgr);
|
||||
return;
|
||||
}
|
||||
for (;;) {
|
||||
int rc = iter_next(&it, (void**)&j);
|
||||
if (rc < 0 || rc == RTTI_ITER_NO_DATA) {
|
||||
break;
|
||||
}
|
||||
if (j == NULL) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int on_host = 0;
|
||||
if (j->job.allocs != NULL && j->job.allocs->allocs != NULL) {
|
||||
for (int i = 0; i < j->job.allocs->nhosts; i++) {
|
||||
xce_host_alloc_t* ha = j->job.allocs->allocs[i];
|
||||
if (ha != NULL && ha->host != NULL
|
||||
&& strcmp(ha->host, hostname) == 0) {
|
||||
on_host = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!on_host) {
|
||||
continue;
|
||||
}
|
||||
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
if (JOB_IS_RUN(j->job.state)
|
||||
&& !job_state_is_terminal(j->job.state)) {
|
||||
if (j->job.exit_info == NULL) {
|
||||
j->job.exit_info =
|
||||
(job_exitinfo_t*)calloc(1, sizeof(job_exitinfo_t));
|
||||
}
|
||||
if (j->job.exit_info != NULL) {
|
||||
j->job.exit_info->exitcode = -1; /* worker lost */
|
||||
}
|
||||
j->job.finish_time = time(NULL);
|
||||
mark_job_update(c, j, JOB_STAT_EXIT);
|
||||
}
|
||||
unlock(j);
|
||||
}
|
||||
iter_destroy(&it);
|
||||
unlock(c->jobmgr);
|
||||
}
|
||||
|
||||
/* Guarded state-transition core for a postdone fact, extracted from
|
||||
* on_job_postdone's old inline mutation (raft-apply-decoupling task 6.1
|
||||
* round 2, Req 3.3/7.1-7.3). Unlike a generic UPDATE, postdone's formula
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue