fix(scheduler): stop re-dispatching running jobs (bug-1498) + resval double-free (bug-1497)
bug-1498 (re-dispatch): sched_job_is_ready() was a stub returning 1, so any job lingering in the fcfs pendJobs queue was always elected. A dispatched job was never marked non-PEND (the controller defers its schedStartJob notify to the not-yet-implemented apply pump, controller/job.c), so fcfs_update_job re-enqueued it on every job update and the election thread re-dispatched the running job — the worker got conflicting signals and the job was SIGTERM'd. - sched_job_is_ready: elect only PEND, non-suspended jobs - sched_commit_decision (dispatch): mark the job JOB_STAT_RUN on dispatch so it leaves the PEND set and is no longer re-electable - fcfs_update_job: dedup via ordq_peek before enqueue bug-1497 (double-free): sched_update's resval paths transferred ownership of rv->val to sharedInst/w->resvals but left the request pointer set, so sched_pend_update_killer double-freed it -> use-after-free in build_host_eval_ctx. Clear rv->val = NULL on transfer. Tests: new scheduler/testsuite/test_redispatch.c (8 cases): sched_job_is_ready PEND gating + dispatch-commit marks the job RUN. Full scheduler meson suite green. Live-verified: a `sleep 60` job is elected exactly once, runs the full duration and exits 0 (`completed successfully`); before the fix it was re-dispatched at ~11s and killed (SIGTERM 143). Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
lei.chen
parent
07ab05671d
commit
ee7dee0ed4
5 changed files with 381 additions and 2 deletions
242
src/scheduler/testsuite/test_redispatch.c
Normal file
242
src/scheduler/testsuite/test_redispatch.c
Normal file
|
|
@ -0,0 +1,242 @@
|
|||
/*
|
||||
* test_redispatch.c - white-box regression tests for bug-1498: the scheduler
|
||||
* re-dispatched a job it had already dispatched (running job killed by a
|
||||
* duplicate dispatch).
|
||||
*
|
||||
* Root cause: sched_job_is_ready() was a stub returning 1 unconditionally, so
|
||||
* a job lingering in the fcfs pendJobs queue was always elected; and a
|
||||
* dispatched job was never marked non-PEND, so fcfs_update_job re-enqueued it
|
||||
* on every job update (the controller defers its schedStartJob notify).
|
||||
*
|
||||
* Strategy: pull the REAL scheduler.c into this TU (same white-box pattern as
|
||||
* test_job_update.c pulling schedobj.c), stub the scheduler-internal externs
|
||||
* defined in other .c files not linked here, and drive:
|
||||
* - sched_job_is_ready: PEND electable; RUN/EXIT/DONE/suspended/NULL not.
|
||||
* - sched_commit_decision(XCE_DECISION_DISPATCH): marks the job RUN so it
|
||||
* leaves the PEND set and is no longer electable.
|
||||
*/
|
||||
#define UNIT_TEST 1
|
||||
#include <errno.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdatomic.h>
|
||||
|
||||
#include <unity/unity.h>
|
||||
|
||||
#include "kvmgr.h"
|
||||
#include "scheduler.h"
|
||||
|
||||
/* Pull the real implementation under test. */
|
||||
#include "../src/scheduler.c"
|
||||
|
||||
/* ── Stubs for scheduler.c externs defined in other .c files ─────────────── */
|
||||
|
||||
/* plugin.c — plugin dispatch table & lifecycle (not linked here). */
|
||||
int plugin_job_update(void* s, xce_sched_job_t* j) { (void)s; (void)j; return 0; }
|
||||
int plugin_queue_update(void* s, xce_sched_queue_t* q) { (void)s; (void)q; return 0; }
|
||||
int plugin_worker_update(void* s, xce_sched_worker_t* w) { (void)s; (void)w; return 0; }
|
||||
int plugin_job_state_change(void* s, xce_sched_job_t* j, uint32_t st)
|
||||
{
|
||||
(void)s; (void)j; (void)st; return 0;
|
||||
}
|
||||
int plugin_job_rusage_update(void* s, xce_sched_job_t* j, xce_job_rusages_t* r)
|
||||
{
|
||||
(void)s; (void)j; (void)r; return 0;
|
||||
}
|
||||
int plugin_commit_decision(void* s, xce_decision_t* d) { (void)s; (void)d; return 0; }
|
||||
int plugin_begin_session(void* s) { (void)s; return 0; }
|
||||
int plugin_end_session(void* s) { (void)s; return 0; }
|
||||
int plugin_config_update(void* s, xce_conf4sched_t* c) { (void)s; (void)c; return 0; }
|
||||
int plugin_init(void* s) { (void)s; return 0; }
|
||||
void plugin_unregister(void* s) { (void)s; }
|
||||
int plugin_job_election(void* s, xce_sched_job_t* j) { (void)s; (void)j; return 0; }
|
||||
|
||||
/* schedobj.c / resource.c / client helpers — stubbed for these tests. */
|
||||
void kill_sched_job(void* arg) { (void)arg; }
|
||||
void kill_sched_queue(void* arg) { (void)arg; }
|
||||
void kill_sched_worker(void* arg) { (void)arg; }
|
||||
void kill_sched_user(void* arg) { (void)arg; }
|
||||
void kill_sched_project(void* arg) { (void)arg; }
|
||||
int sched_commit_job_alloc(xce_scheduler_t* s, xce_sched_job_t* j)
|
||||
{
|
||||
(void)s; (void)j; return 0;
|
||||
}
|
||||
int sched_charge_account(xce_scheduler_t* s, xce_job_account_t* a)
|
||||
{
|
||||
(void)s; (void)a; return 0;
|
||||
}
|
||||
int sched_uncharge_account(xce_scheduler_t* s, xce_job_account_t* a)
|
||||
{
|
||||
(void)s; (void)a; return 0;
|
||||
}
|
||||
int sched_apply_job_rusage(xce_scheduler_t* s, xce_job_rusages_t* r)
|
||||
{
|
||||
(void)s; (void)r; return -ENOENT;
|
||||
}
|
||||
int sched_update_existing_job(xce_scheduler_t* s, xce_job_t* j)
|
||||
{
|
||||
(void)s; (void)j; return -ENOENT;
|
||||
}
|
||||
int sched_update_existing_queue(xce_scheduler_t* s, xce_queue_t* q)
|
||||
{
|
||||
(void)s; (void)q; return -ENOENT;
|
||||
}
|
||||
|
||||
/* schedobj.c / decision.c / plugins — additional externs scheduler.c
|
||||
* references (defined in .c files not linked here). */
|
||||
int sched_ctx_init(xce_sched_ctx_t* ctx, void* arg, xce_sched_obj_t* obj)
|
||||
{
|
||||
(void)ctx; (void)arg; (void)obj; return 0;
|
||||
}
|
||||
void sched_ctx_fini(xce_sched_ctx_t* ctx) { (void)ctx; }
|
||||
void sched_ctx_set_primary(xce_sched_ctx_t* ctx, xce_primary_resreq_t* p)
|
||||
{
|
||||
(void)ctx; (void)p;
|
||||
}
|
||||
xce_decision_t* make_pend_decision(xce_pendreason_t* r) { (void)r; return NULL; }
|
||||
xce_decision_t* make_dispatch_decision(xce_job_alloc_t* a) { (void)a; return NULL; }
|
||||
int register_fcfs(void* s) { (void)s; return 0; }
|
||||
int register_limit(void* s) { (void)s; return 0; }
|
||||
int register_seqjob_allocator(void* s) { (void)s; return 0; }
|
||||
int register_parjob_allocator(void* s) { (void)s; return 0; }
|
||||
void kill_sched_resval(xce_val_t* v) { (void)v; }
|
||||
xce_sched_job_t* make_sched_job(void* a, xce_job_t* j) { (void)a; (void)j; return NULL; }
|
||||
xce_sched_queue_t* make_sched_queue(void* a, xce_queue_t* q) { (void)a; (void)q; return NULL; }
|
||||
xce_sched_worker_t* make_sched_worker(void* a, xce_worker_t* w) { (void)a; (void)w; return NULL; }
|
||||
xce_sched_user_t* make_sched_user(void* a, char* n) { (void)a; (void)n; return NULL; }
|
||||
xce_sched_project_t* make_sched_project(void* a, char* n) { (void)a; (void)n; return NULL; }
|
||||
void sched_worker_release_host_ctx(xce_sched_worker_t* w) { (void)w; }
|
||||
|
||||
void setUp(void) { return; }
|
||||
void tearDown(void) { return; }
|
||||
|
||||
/* Note: sched_get_job / sched_add_job / sched_get_queue / ... are defined by
|
||||
* scheduler.c itself (real kvmgr-backed) and come in via the #include — no
|
||||
* stubs needed. They operate on sched->jobs etc., which fixture_init creates. */
|
||||
|
||||
/* ── Fixture ─────────────────────────────────────────────────────────────── */
|
||||
|
||||
static void fixture_init(xce_scheduler_t* sched)
|
||||
{
|
||||
kvmgr_params_t jobs_params = { .key_type = KVMGR_KEY_INT,
|
||||
.nslots = 59,
|
||||
.delta = 16,
|
||||
.datakiller = kill_sched_job };
|
||||
|
||||
memset(sched, 0, sizeof(*sched));
|
||||
resreq_init(1);
|
||||
xstring_table_init(1024 * 16);
|
||||
|
||||
sched->jobs = makeInstance(&xkvmgr_type, &jobs_params);
|
||||
}
|
||||
|
||||
static xce_sched_job_t* seed_job(xce_scheduler_t* sched, uint64_t jobid,
|
||||
uint32_t state)
|
||||
{
|
||||
xce_sched_job_t* j = (xce_sched_job_t*)calloc(1, sizeof(*j));
|
||||
TEST_ASSERT_NOT_NULL(j);
|
||||
j->jobid = jobid;
|
||||
j->state = state;
|
||||
sched_add_job(sched, j);
|
||||
return j;
|
||||
}
|
||||
|
||||
/* ── sched_job_is_ready ──────────────────────────────────────────────────── */
|
||||
|
||||
static void test_ready_null_not_ready(void)
|
||||
{
|
||||
TEST_ASSERT_EQUAL_INT(0, sched_job_is_ready(NULL));
|
||||
}
|
||||
|
||||
static void test_ready_pend_is_ready(void)
|
||||
{
|
||||
xce_sched_job_t j = { .state = JOB_STAT_PEND };
|
||||
TEST_ASSERT_EQUAL_INT(1, sched_job_is_ready(&j));
|
||||
}
|
||||
|
||||
static void test_ready_run_not_ready(void)
|
||||
{
|
||||
xce_sched_job_t j = { .state = JOB_STAT_RUN };
|
||||
TEST_ASSERT_EQUAL_INT(0, sched_job_is_ready(&j));
|
||||
}
|
||||
|
||||
static void test_ready_exit_not_ready(void)
|
||||
{
|
||||
xce_sched_job_t j = { .state = JOB_STAT_EXIT };
|
||||
TEST_ASSERT_EQUAL_INT(0, sched_job_is_ready(&j));
|
||||
}
|
||||
|
||||
static void test_ready_done_not_ready(void)
|
||||
{
|
||||
xce_sched_job_t j = { .state = JOB_STAT_DONE };
|
||||
TEST_ASSERT_EQUAL_INT(0, sched_job_is_ready(&j));
|
||||
}
|
||||
|
||||
static void test_ready_suspended_not_ready(void)
|
||||
{
|
||||
xce_sched_job_t j = { .state = JOB_STAT_PEND | JOB_STAT_USUSP };
|
||||
TEST_ASSERT_EQUAL_INT(0, sched_job_is_ready(&j));
|
||||
}
|
||||
|
||||
/* ── sched_commit_decision dispatch marks the job RUN ────────────────────── */
|
||||
|
||||
static void test_dispatch_commit_marks_job_run(void)
|
||||
{
|
||||
xce_scheduler_t sched;
|
||||
xce_sched_job_t* job;
|
||||
xce_job_alloc_t alloc;
|
||||
xce_decision_t decision;
|
||||
|
||||
fixture_init(&sched);
|
||||
|
||||
job = seed_job(&sched, 42, JOB_STAT_PEND);
|
||||
|
||||
memset(&alloc, 0, sizeof(alloc));
|
||||
alloc.jobid = 42;
|
||||
|
||||
memset(&decision, 0, sizeof(decision));
|
||||
decision.op = XCE_DECISION_DISPATCH;
|
||||
decision.u.alloc = &alloc;
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, sched_commit_decision(&sched, &decision));
|
||||
|
||||
/* The dispatched job leaves PEND: state is RUN and it is not electable. */
|
||||
TEST_ASSERT_EQUAL_UINT(JOB_STAT_RUN, job->state);
|
||||
TEST_ASSERT_EQUAL_INT(0, sched_job_is_ready(job));
|
||||
}
|
||||
|
||||
static void test_dispatch_commit_null_alloc_unchanged(void)
|
||||
{
|
||||
xce_scheduler_t sched;
|
||||
xce_sched_job_t* job;
|
||||
xce_decision_t decision;
|
||||
|
||||
fixture_init(&sched);
|
||||
job = seed_job(&sched, 43, JOB_STAT_PEND);
|
||||
|
||||
memset(&decision, 0, sizeof(decision));
|
||||
decision.op = XCE_DECISION_PEND;
|
||||
decision.u.reason = NULL;
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, sched_commit_decision(&sched, &decision));
|
||||
/* Non-dispatch decision must not touch the job's state. */
|
||||
TEST_ASSERT_EQUAL_UINT(JOB_STAT_PEND, job->state);
|
||||
}
|
||||
|
||||
/* ── Runner ──────────────────────────────────────────────────────────────── */
|
||||
|
||||
int main(int argc, char* argv[])
|
||||
{
|
||||
(void)argc; (void)argv;
|
||||
UNITY_BEGIN();
|
||||
RUN_TEST(test_ready_null_not_ready, 0);
|
||||
RUN_TEST(test_ready_pend_is_ready, 0);
|
||||
RUN_TEST(test_ready_run_not_ready, 0);
|
||||
RUN_TEST(test_ready_exit_not_ready, 0);
|
||||
RUN_TEST(test_ready_done_not_ready, 0);
|
||||
RUN_TEST(test_ready_suspended_not_ready, 0);
|
||||
RUN_TEST(test_dispatch_commit_marks_job_run, 0);
|
||||
RUN_TEST(test_dispatch_commit_null_alloc_unchanged, 0);
|
||||
return UNITY_END();
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue