sched_job_exit_fun() (renamed sched_job_finish_fun, sched.c) subscribes
to HOOK_DETAIL_JOB_EXIT, which fires for EVERY job finish, both success
and failure -- record_job_finish_fact() (job.c) decides DONE vs EXIT
from finish->exit_code BEFORE firing this same hook detail either way
(mirrors persist.c's own HOOK_DETAIL_JOB_EXIT -> WAL_JOB_FINISH
mapping). This function discarded info->exit_code and sent only a bare
jobid, forcing sched_handle_notify() (scheduler/src/handler.c) to
unconditionally map EVERY finish -- including a clean, successful one
-- to JOB_STAT_EXIT. A subsequent HOOK_DETAIL_JOB_UPDATE notify usually
self-corrected this moments later via the full job state, but if that
second notify was dropped during a scheduler reconnect gap (the same
window bug-1731/bug-1734 already documented), the scheduler was left
permanently stuck showing EXIT for a job the controller correctly
recorded as DONE.
Live-reproduced on a real 3-node HA cluster + worker: job
0-8-1788143749-0[38] genuinely completed successfully (worker log:
"completed successfully", controller joblist: DONE) but the
scheduler's own replay showed EXIT.
Fix: carry {jobid, exit_code} across the wire via the existing
JobFinishInfo shape (the same struct apiserver_job_exit_fun already
reuses for this identical hook) instead of a bare UInt64, so
sched_handle_notify() can make the identical exit_code==0-vs-not
decision job.c itself makes. Renamed sched_job_exit_fun to
sched_job_finish_fun to reflect what it actually does.
Live-verified: a 51-job run (all exit 0) showed 100% DONE on the
scheduler side after the fix.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019QxaSzHRB7pmYFp4cVnwkA
324 lines
12 KiB
C
324 lines
12 KiB
C
/*
|
|
* test_handlers.c - white-box tests for the scheduler notify/dataset enqueue
|
|
* handlers (bug-765 G-6, tasks 9.2 + 9.3). Pulls the real (static) handlers in
|
|
* handler.c into this TU; enq_pend_update is local to handler.c, so we drive a
|
|
* real sched->pend_updates dataq and assert the correct op was enqueued (or
|
|
* skipped / -EINVAL) per each handler's contract.
|
|
*/
|
|
#define UNIT_TEST 1
|
|
#include <errno.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include <unity/unity.h>
|
|
|
|
#include "dataq.h"
|
|
#include "mylock.h"
|
|
#include "scheduler.h"
|
|
|
|
/* Pull the real handlers under test into this TU. handler.c is self-contained
|
|
* for the enqueue path: enq_pend_update and sched_job_state_change are both
|
|
* static within it; the dispatchers call lib helpers (linked). The
|
|
* scheduler-internal externs handler.c references but this file does not
|
|
* exercise are stubbed below: sched_init_config (config-init request path)
|
|
* and sched_get_job (job-array-single-wal-record task 5.3's per-element
|
|
* dedup check, XceData_data_jobArray — not tested here, see
|
|
* test_job_array_expand.c for the real kvmgr-backed sched_get_job). */
|
|
int sched_init_config(xce_scheduler_t* s, xce_conf4sched_t* c)
|
|
{
|
|
(void)s;
|
|
(void)c;
|
|
return 0;
|
|
}
|
|
|
|
/* sched_get_job relocated to core.h (scheduler-core spec, task 4.1) -- now
|
|
* operates on xce_sched_core_t*, not xce_scheduler_t*. */
|
|
xce_sched_job_t* sched_get_job(xce_sched_core_t* sched, uint64_t jobid)
|
|
{
|
|
(void)sched;
|
|
(void)jobid;
|
|
return NULL;
|
|
}
|
|
|
|
/* sched_record_status (recording.c, scheduler-core spec, task 9.4) --
|
|
* handler.c's OPCode_schedAdmin/recordStatus path references it; this file
|
|
* never exercises OPCode_schedAdmin (its own scope is notify/dataset
|
|
* enqueue), so a link-satisfying stub is enough, same rationale as
|
|
* sched_init_config above. */
|
|
int sched_record_status(xce_scheduler_t* sched, sched_admin_reply_t* reply)
|
|
{
|
|
(void)sched;
|
|
(void)reply;
|
|
return -EINVAL;
|
|
}
|
|
|
|
#include "../src/handler.c"
|
|
|
|
void setUp(void) { return; }
|
|
void tearDown(void)
|
|
{
|
|
printf("\n-----------------------------------------------------\n");
|
|
return;
|
|
}
|
|
|
|
static void fixture_init(xce_scheduler_t* sched)
|
|
{
|
|
memset(sched, 0, sizeof(*sched));
|
|
mylock_init(&(sched->core.lock));
|
|
sched->core.pend_updates[0] = (dataq_t*)calloc(1, sizeof(dataq_t));
|
|
dataq_init(sched->core.pend_updates[0], NULL);
|
|
sched->core.pend_updates[1] = (dataq_t*)calloc(1, sizeof(dataq_t));
|
|
dataq_init(sched->core.pend_updates[1], NULL);
|
|
}
|
|
|
|
/* Dequeue one pending request (NULL if the queue is empty) and return its op
|
|
* via *op_out; frees the request shell (payloads are test-owned/leaked under
|
|
* the suite's leaks-off convention). Returns 1 if an item was present. */
|
|
static int pop_op(xce_scheduler_t* sched, xce_sched_op_t* op_out)
|
|
{
|
|
data_item_t* it = dataq_deq(sched->core.pend_updates[0], NULL);
|
|
xce_sched_update_request_t* req;
|
|
|
|
if (it == NULL) {
|
|
return 0;
|
|
}
|
|
req = ITPTR_TO_UPDATE_REQ(it);
|
|
if (op_out != NULL) {
|
|
*op_out = req->op;
|
|
}
|
|
free(req);
|
|
return 1;
|
|
}
|
|
|
|
/* Dequeue one pending request and assert it is an XCE_SCHED_OP_JSTATE_CHANGE,
|
|
* returning its jobid/state via jobid_out/state_out (both optional). Frees
|
|
* the request shell. Returns 1 if an item was present, 0 if the queue was
|
|
* empty (scheduler-dispatch-commit task 2.2: verifies the jobDispatchFail
|
|
* notify enqueues the same op the adjacent schedExitJob/schedFailJob cases
|
|
* already use). */
|
|
static int pop_jstate_change(xce_scheduler_t* sched, uint64_t* jobid_out,
|
|
uint32_t* state_out)
|
|
{
|
|
data_item_t* it = dataq_deq(sched->core.pend_updates[0], NULL);
|
|
xce_sched_update_request_t* req;
|
|
|
|
if (it == NULL) {
|
|
return 0;
|
|
}
|
|
req = ITPTR_TO_UPDATE_REQ(it);
|
|
TEST_ASSERT_EQUAL_INT(XCE_SCHED_OP_JSTATE_CHANGE, req->op);
|
|
if (jobid_out != NULL) {
|
|
*jobid_out = req->u.job_state_change.jobid;
|
|
}
|
|
if (state_out != NULL) {
|
|
*state_out = req->u.job_state_change.state;
|
|
}
|
|
free(req);
|
|
return 1;
|
|
}
|
|
|
|
/* ── 9.3: notify single-item handlers (NULL -> -EINVAL, valid -> enqueue) ── */
|
|
|
|
static void test_handle_update_job_enqueues_job_fields(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_sched_op_t op = (xce_sched_op_t)-1;
|
|
fixture_init(&sched);
|
|
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL, sched_handle_sched_update_job(&sched, NULL));
|
|
TEST_ASSERT_EQUAL_INT(0, pop_op(&sched, NULL)); /* NULL enqueued nothing */
|
|
|
|
xce_job_t* job = (xce_job_t*)calloc(1, sizeof(*job));
|
|
job->jobid = 42;
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_sched_update_job(&sched, job));
|
|
TEST_ASSERT_EQUAL_INT(1, pop_op(&sched, &op));
|
|
/* 5.6: notify job update takes the in-place UPDATE_JOB_FIELDS path, NOT
|
|
* the dataset bulk-create UPDATE_JOB path. */
|
|
TEST_ASSERT_EQUAL_INT(XCE_SCHED_OP_UPDATE_JOB_FIELDS, op);
|
|
free(job);
|
|
}
|
|
|
|
static void test_handle_update_queue_enqueues_queue(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_sched_op_t op = (xce_sched_op_t)-1;
|
|
fixture_init(&sched);
|
|
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL,
|
|
sched_handle_sched_update_queue(&sched, NULL));
|
|
|
|
xce_queue_t* q = (xce_queue_t*)calloc(1, sizeof(*q));
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_sched_update_queue(&sched, q));
|
|
TEST_ASSERT_EQUAL_INT(1, pop_op(&sched, &op));
|
|
TEST_ASSERT_EQUAL_INT(XCE_SCHED_OP_UPDATE_QUEUE, op);
|
|
free(q);
|
|
}
|
|
|
|
static void test_handle_job_rusage_enqueues_rusage(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_sched_op_t op = (xce_sched_op_t)-1;
|
|
fixture_init(&sched);
|
|
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL, sched_handle_sched_job_rusage(&sched, NULL));
|
|
|
|
xce_job_rusages_t* ru = (xce_job_rusages_t*)calloc(1, sizeof(*ru));
|
|
ru->jobid = 7;
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_sched_job_rusage(&sched, ru));
|
|
TEST_ASSERT_EQUAL_INT(1, pop_op(&sched, &op));
|
|
TEST_ASSERT_EQUAL_INT(XCE_SCHED_OP_JOB_RUSAGE, op);
|
|
free(ru);
|
|
}
|
|
|
|
static void test_handle_update_resvals_skips_empty_enqueues_nonempty(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_sched_op_t op = (xce_sched_op_t)-1;
|
|
xce_resval_list_t empty = { .count = 0, .vals = NULL };
|
|
xce_resval_list_t full = { .count = 1, .vals = NULL };
|
|
fixture_init(&sched);
|
|
|
|
/* NULL or count==0 -> skip (return 0, nothing enqueued) */
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_sched_update_resvals(&sched, NULL));
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_sched_update_resvals(&sched, &empty));
|
|
TEST_ASSERT_EQUAL_INT(0, pop_op(&sched, NULL));
|
|
|
|
/* count>0 -> enqueue UPDATE_RESVALS */
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_sched_update_resvals(&sched, &full));
|
|
TEST_ASSERT_EQUAL_INT(1, pop_op(&sched, &op));
|
|
TEST_ASSERT_EQUAL_INT(XCE_SCHED_OP_UPDATE_RESVALS, op);
|
|
}
|
|
|
|
/* ── 9.2: dataset handlers (skip on NULL/empty, enqueue on valid) ── */
|
|
|
|
static void test_handle_resources_skips_empty_enqueues_nonempty(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_sched_op_t op = (xce_sched_op_t)-1;
|
|
xce_resource_t* one[1];
|
|
fixture_init(&sched);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_resources(&sched, 0, NULL));
|
|
TEST_ASSERT_EQUAL_INT(0, pop_op(&sched, NULL));
|
|
|
|
one[0] = (xce_resource_t*)calloc(1, sizeof(xce_resource_t));
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_resources(&sched, 1, one));
|
|
TEST_ASSERT_EQUAL_INT(1, pop_op(&sched, &op));
|
|
TEST_ASSERT_EQUAL_INT(XCE_SCHED_OP_UPDATE_RESOURCES, op);
|
|
free(one[0]);
|
|
}
|
|
|
|
static void test_handle_resvals_full_skips_empty_enqueues_nonempty(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_sched_op_t op = (xce_sched_op_t)-1;
|
|
xce_resval_t* one[1];
|
|
fixture_init(&sched);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_resvals(&sched, 0, NULL));
|
|
TEST_ASSERT_EQUAL_INT(0, pop_op(&sched, NULL));
|
|
|
|
one[0] = (xce_resval_t*)calloc(1, sizeof(xce_resval_t));
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_resvals(&sched, 1, one));
|
|
TEST_ASSERT_EQUAL_INT(1, pop_op(&sched, &op));
|
|
TEST_ASSERT_EQUAL_INT(XCE_SCHED_OP_UPDATE_RESVALS_FULL, op);
|
|
free(one[0]);
|
|
}
|
|
|
|
/* ── scheduler-dispatch-commit task 2.2: sched_handle_notify's new
|
|
* jobDispatchFail case (bug-1625's actual fix) ── */
|
|
|
|
static void test_handle_notify_job_dispatch_fail_enqueues_pend(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_notify_data_t notify;
|
|
job_failinfo_t info = { .jobid = 99, .stage = 0, .code = -5,
|
|
.detail = NULL };
|
|
uint64_t jobid = 0;
|
|
uint32_t state = (uint32_t)-1;
|
|
fixture_init(&sched);
|
|
|
|
memset(¬ify, 0, sizeof(notify));
|
|
notify.event = NotifyData_data_jobDispatchFail;
|
|
notify.data.dispatchfail = &info;
|
|
|
|
/* Previously fell to `default: rc = -EINVAL;` and enqueued nothing --
|
|
* this is the actual observable fix for bug-1625: the notify is now
|
|
* processed instead of silently discarded. */
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_notify(&sched, ¬ify));
|
|
TEST_ASSERT_EQUAL_INT(1, pop_jstate_change(&sched, &jobid, &state));
|
|
TEST_ASSERT_EQUAL_UINT64(99, jobid);
|
|
TEST_ASSERT_EQUAL_INT(JOB_STAT_PEND, state);
|
|
|
|
/* No classification logic here: the notify's meaning is exclusively
|
|
* "return this job to pending" regardless of the reported failure
|
|
* code -- verify a different `code` value still resolves to PEND. */
|
|
fixture_init(&sched);
|
|
info.jobid = 123;
|
|
info.code = 0;
|
|
jobid = 0;
|
|
state = (uint32_t)-1;
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_notify(&sched, ¬ify));
|
|
TEST_ASSERT_EQUAL_INT(1, pop_jstate_change(&sched, &jobid, &state));
|
|
TEST_ASSERT_EQUAL_UINT64(123, jobid);
|
|
TEST_ASSERT_EQUAL_INT(JOB_STAT_PEND, state);
|
|
}
|
|
|
|
/* ── bug-1737: sched_handle_notify's schedExitJob case must branch on
|
|
* exit_code (mirroring job.c's own apply_job_finish_transition() decision)
|
|
* instead of unconditionally assuming every job finish is a failure ── */
|
|
|
|
static void test_handle_notify_job_exit_maps_exit_code_to_done_or_exit(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_notify_data_t notify;
|
|
xce_jobfinish_t finish = { .jobid = 200, .exit_code = 0 };
|
|
uint64_t jobid = 0;
|
|
uint32_t state = (uint32_t)-1;
|
|
fixture_init(&sched);
|
|
|
|
memset(¬ify, 0, sizeof(notify));
|
|
notify.event = NotifyData_data_schedExitJob;
|
|
notify.data.sched_exit_job = &finish;
|
|
|
|
/* exit_code == 0 -- clean finish -- must resolve to JOB_STAT_DONE, not
|
|
* the previous unconditional JOB_STAT_EXIT. */
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_notify(&sched, ¬ify));
|
|
TEST_ASSERT_EQUAL_INT(1, pop_jstate_change(&sched, &jobid, &state));
|
|
TEST_ASSERT_EQUAL_UINT64(200, jobid);
|
|
TEST_ASSERT_EQUAL_INT(JOB_STAT_DONE, state);
|
|
|
|
/* exit_code != 0 -- a real failure -- still resolves to JOB_STAT_EXIT. */
|
|
fixture_init(&sched);
|
|
finish.jobid = 201;
|
|
finish.exit_code = 7;
|
|
jobid = 0;
|
|
state = (uint32_t)-1;
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_notify(&sched, ¬ify));
|
|
TEST_ASSERT_EQUAL_INT(1, pop_jstate_change(&sched, &jobid, &state));
|
|
TEST_ASSERT_EQUAL_UINT64(201, jobid);
|
|
TEST_ASSERT_EQUAL_INT(JOB_STAT_EXIT, state);
|
|
|
|
/* NULL payload -- malformed notify -- must fail cleanly, not crash. */
|
|
fixture_init(&sched);
|
|
notify.data.sched_exit_job = NULL;
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL, sched_handle_notify(&sched, ¬ify));
|
|
TEST_ASSERT_EQUAL_INT(0, pop_op(&sched, NULL));
|
|
}
|
|
|
|
int main(int argc, char* argv[])
|
|
{
|
|
(void)argc;
|
|
(void)argv;
|
|
UNITY_BEGIN();
|
|
RUN_TEST(test_handle_update_job_enqueues_job_fields, 0);
|
|
RUN_TEST(test_handle_update_queue_enqueues_queue, 0);
|
|
RUN_TEST(test_handle_job_rusage_enqueues_rusage, 0);
|
|
RUN_TEST(test_handle_update_resvals_skips_empty_enqueues_nonempty, 0);
|
|
RUN_TEST(test_handle_resources_skips_empty_enqueues_nonempty, 0);
|
|
RUN_TEST(test_handle_resvals_full_skips_empty_enqueues_nonempty, 0);
|
|
RUN_TEST(test_handle_notify_job_dispatch_fail_enqueues_pend, 0);
|
|
RUN_TEST(test_handle_notify_job_exit_maps_exit_code_to_done_or_exit, 0);
|
|
return UNITY_END();
|
|
}
|