Merge follow-up fixes found during comprehensive unit-test + live-cluster
verification of the scheduler-framework merge:
- apiserver/backend/clib.go (ControlJob): Rongsong's clib.go had reverted
our bug-1490 fix — ctrl.action must be the capnp JobControl union
DISCRIMINANT (0=siginfo,1=ckptoption), not the models.JOB_* constant
(1/2/3/8), or every kill/stop/resume fails -EINVAL. Restored
ctrl.action = C.int(jobCtrlDiscriminant(action)); verified live (job
kill RUN->EXIT).
- controller/persist.c (fsync_worker): the merged bug-1563 no-progress
backoff used the PRE-FLUSH `pending` snapshot, so a no-op flush (another
path advanced durable_lsn to tip mid-flush) also slept wal_sync_ms and
raced commit_engine_fini in teardown, hanging the p2_integration suite
~60% (bug-1515). Re-evaluate the level AFTER the flush (pending_after)
so only a genuine sustained fdatasync failure backoffs. 20/20 clean p2
runs (was 3/5 hang). Also carries the bug-1509 B3 RUN->UNKWN marking in
persist_recover.
- scheduler/worker testsuite meson.build: add unity_dep parent includedir
(unity.pc Cflags only add -I${includedir}/unity, so <unity/unity.h> was
unresolvable) — same fix as src/lib/meson.build.
- test_handlers.c: re-point notify.data.dispatchfail after the first
sched_handle_notify NULLs it on success (code=0 branch was -EINVAL).
- test_controller_connector.c: give the rejoin-success fixtures a valid
worker_list pointer (my bug-1509 resend_job_status deref was NULL-
crashing).
- test_wal_job_dispatch_fact.c / test_dispatch_commit.c: Rongsong's
WAL_JOB_DISPATCH replay tests asserted JOB_STAT_RUN after persist_recover,
conflicting with bug-1509 B3 (RUN->UNKWN on restart). Assert JOB_IS_RUN
(base RUN recovered, incl. bug-1678 allocs) + UNKWN flag instead.
All suites green: controller 1145/0, scheduler 57/57, worker 15/15,
lib 303/0, apiserver CGO build ok.
282 lines
10 KiB
C
282 lines
10 KiB
C
/*
|
|
* test_handlers.c - white-box tests for the scheduler notify/dataset enqueue
|
|
* handlers (bug-765 G-6, tasks 9.2 + 9.3). Pulls the real (static) handlers in
|
|
* handler.c into this TU; enq_pend_update is local to handler.c, so we drive a
|
|
* real sched->pend_updates dataq and assert the correct op was enqueued (or
|
|
* skipped / -EINVAL) per each handler's contract.
|
|
*/
|
|
#define UNIT_TEST 1
|
|
#include <errno.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include <unity/unity.h>
|
|
|
|
#include "dataq.h"
|
|
#include "mylock.h"
|
|
#include "scheduler.h"
|
|
|
|
/* Pull the real handlers under test into this TU. handler.c is self-contained
|
|
* for the enqueue path: enq_pend_update and sched_job_state_change are both
|
|
* static within it; the dispatchers call lib helpers (linked). The
|
|
* scheduler-internal externs handler.c references but this file does not
|
|
* exercise are stubbed below: sched_init_config (config-init request path)
|
|
* and sched_get_job (job-array-single-wal-record task 5.3's per-element
|
|
* dedup check, XceData_data_jobArray — not tested here, see
|
|
* test_job_array_expand.c for the real kvmgr-backed sched_get_job). */
|
|
int sched_init_config(xce_scheduler_t* s, xce_conf4sched_t* c)
|
|
{
|
|
(void)s;
|
|
(void)c;
|
|
return 0;
|
|
}
|
|
|
|
/* sched_get_job relocated to core.h (scheduler-core spec, task 4.1) -- now
|
|
* operates on xce_sched_core_t*, not xce_scheduler_t*. */
|
|
xce_sched_job_t* sched_get_job(xce_sched_core_t* sched, uint64_t jobid)
|
|
{
|
|
(void)sched;
|
|
(void)jobid;
|
|
return NULL;
|
|
}
|
|
|
|
/* sched_record_status (recording.c, scheduler-core spec, task 9.4) --
|
|
* handler.c's OPCode_schedAdmin/recordStatus path references it; this file
|
|
* never exercises OPCode_schedAdmin (its own scope is notify/dataset
|
|
* enqueue), so a link-satisfying stub is enough, same rationale as
|
|
* sched_init_config above. */
|
|
int sched_record_status(xce_scheduler_t* sched, sched_admin_reply_t* reply)
|
|
{
|
|
(void)sched;
|
|
(void)reply;
|
|
return -EINVAL;
|
|
}
|
|
|
|
#include "../src/handler.c"
|
|
|
|
void setUp(void) { return; }
|
|
void tearDown(void)
|
|
{
|
|
printf("\n-----------------------------------------------------\n");
|
|
return;
|
|
}
|
|
|
|
static void fixture_init(xce_scheduler_t* sched)
|
|
{
|
|
memset(sched, 0, sizeof(*sched));
|
|
mylock_init(&(sched->core.lock));
|
|
sched->core.pend_updates[0] = (dataq_t*)calloc(1, sizeof(dataq_t));
|
|
dataq_init(sched->core.pend_updates[0], NULL);
|
|
sched->core.pend_updates[1] = (dataq_t*)calloc(1, sizeof(dataq_t));
|
|
dataq_init(sched->core.pend_updates[1], NULL);
|
|
}
|
|
|
|
/* Dequeue one pending request (NULL if the queue is empty) and return its op
|
|
* via *op_out; frees the request shell (payloads are test-owned/leaked under
|
|
* the suite's leaks-off convention). Returns 1 if an item was present. */
|
|
static int pop_op(xce_scheduler_t* sched, xce_sched_op_t* op_out)
|
|
{
|
|
data_item_t* it = dataq_deq(sched->core.pend_updates[0], NULL);
|
|
xce_sched_update_request_t* req;
|
|
|
|
if (it == NULL) {
|
|
return 0;
|
|
}
|
|
req = ITPTR_TO_UPDATE_REQ(it);
|
|
if (op_out != NULL) {
|
|
*op_out = req->op;
|
|
}
|
|
free(req);
|
|
return 1;
|
|
}
|
|
|
|
/* Dequeue one pending request and assert it is an XCE_SCHED_OP_JSTATE_CHANGE,
|
|
* returning its jobid/state via jobid_out/state_out (both optional). Frees
|
|
* the request shell. Returns 1 if an item was present, 0 if the queue was
|
|
* empty (scheduler-dispatch-commit task 2.2: verifies the jobDispatchFail
|
|
* notify enqueues the same op the adjacent schedExitJob/schedFailJob cases
|
|
* already use). */
|
|
static int pop_jstate_change(xce_scheduler_t* sched, uint64_t* jobid_out,
|
|
uint32_t* state_out)
|
|
{
|
|
data_item_t* it = dataq_deq(sched->core.pend_updates[0], NULL);
|
|
xce_sched_update_request_t* req;
|
|
|
|
if (it == NULL) {
|
|
return 0;
|
|
}
|
|
req = ITPTR_TO_UPDATE_REQ(it);
|
|
TEST_ASSERT_EQUAL_INT(XCE_SCHED_OP_JSTATE_CHANGE, req->op);
|
|
if (jobid_out != NULL) {
|
|
*jobid_out = req->u.job_state_change.jobid;
|
|
}
|
|
if (state_out != NULL) {
|
|
*state_out = req->u.job_state_change.state;
|
|
}
|
|
free(req);
|
|
return 1;
|
|
}
|
|
|
|
/* ── 9.3: notify single-item handlers (NULL -> -EINVAL, valid -> enqueue) ── */
|
|
|
|
static void test_handle_update_job_enqueues_job_fields(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_sched_op_t op = (xce_sched_op_t)-1;
|
|
fixture_init(&sched);
|
|
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL, sched_handle_sched_update_job(&sched, NULL));
|
|
TEST_ASSERT_EQUAL_INT(0, pop_op(&sched, NULL)); /* NULL enqueued nothing */
|
|
|
|
xce_job_t* job = (xce_job_t*)calloc(1, sizeof(*job));
|
|
job->jobid = 42;
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_sched_update_job(&sched, job));
|
|
TEST_ASSERT_EQUAL_INT(1, pop_op(&sched, &op));
|
|
/* 5.6: notify job update takes the in-place UPDATE_JOB_FIELDS path, NOT
|
|
* the dataset bulk-create UPDATE_JOB path. */
|
|
TEST_ASSERT_EQUAL_INT(XCE_SCHED_OP_UPDATE_JOB_FIELDS, op);
|
|
free(job);
|
|
}
|
|
|
|
static void test_handle_update_queue_enqueues_queue(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_sched_op_t op = (xce_sched_op_t)-1;
|
|
fixture_init(&sched);
|
|
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL,
|
|
sched_handle_sched_update_queue(&sched, NULL));
|
|
|
|
xce_queue_t* q = (xce_queue_t*)calloc(1, sizeof(*q));
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_sched_update_queue(&sched, q));
|
|
TEST_ASSERT_EQUAL_INT(1, pop_op(&sched, &op));
|
|
TEST_ASSERT_EQUAL_INT(XCE_SCHED_OP_UPDATE_QUEUE, op);
|
|
free(q);
|
|
}
|
|
|
|
static void test_handle_job_rusage_enqueues_rusage(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_sched_op_t op = (xce_sched_op_t)-1;
|
|
fixture_init(&sched);
|
|
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL, sched_handle_sched_job_rusage(&sched, NULL));
|
|
|
|
xce_job_rusages_t* ru = (xce_job_rusages_t*)calloc(1, sizeof(*ru));
|
|
ru->jobid = 7;
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_sched_job_rusage(&sched, ru));
|
|
TEST_ASSERT_EQUAL_INT(1, pop_op(&sched, &op));
|
|
TEST_ASSERT_EQUAL_INT(XCE_SCHED_OP_JOB_RUSAGE, op);
|
|
free(ru);
|
|
}
|
|
|
|
static void test_handle_update_resvals_skips_empty_enqueues_nonempty(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_sched_op_t op = (xce_sched_op_t)-1;
|
|
xce_resval_list_t empty = { .count = 0, .vals = NULL };
|
|
xce_resval_list_t full = { .count = 1, .vals = NULL };
|
|
fixture_init(&sched);
|
|
|
|
/* NULL or count==0 -> skip (return 0, nothing enqueued) */
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_sched_update_resvals(&sched, NULL));
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_sched_update_resvals(&sched, &empty));
|
|
TEST_ASSERT_EQUAL_INT(0, pop_op(&sched, NULL));
|
|
|
|
/* count>0 -> enqueue UPDATE_RESVALS */
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_sched_update_resvals(&sched, &full));
|
|
TEST_ASSERT_EQUAL_INT(1, pop_op(&sched, &op));
|
|
TEST_ASSERT_EQUAL_INT(XCE_SCHED_OP_UPDATE_RESVALS, op);
|
|
}
|
|
|
|
/* ── 9.2: dataset handlers (skip on NULL/empty, enqueue on valid) ── */
|
|
|
|
static void test_handle_resources_skips_empty_enqueues_nonempty(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_sched_op_t op = (xce_sched_op_t)-1;
|
|
xce_resource_t* one[1];
|
|
fixture_init(&sched);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_resources(&sched, 0, NULL));
|
|
TEST_ASSERT_EQUAL_INT(0, pop_op(&sched, NULL));
|
|
|
|
one[0] = (xce_resource_t*)calloc(1, sizeof(xce_resource_t));
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_resources(&sched, 1, one));
|
|
TEST_ASSERT_EQUAL_INT(1, pop_op(&sched, &op));
|
|
TEST_ASSERT_EQUAL_INT(XCE_SCHED_OP_UPDATE_RESOURCES, op);
|
|
free(one[0]);
|
|
}
|
|
|
|
static void test_handle_resvals_full_skips_empty_enqueues_nonempty(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_sched_op_t op = (xce_sched_op_t)-1;
|
|
xce_resval_t* one[1];
|
|
fixture_init(&sched);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_resvals(&sched, 0, NULL));
|
|
TEST_ASSERT_EQUAL_INT(0, pop_op(&sched, NULL));
|
|
|
|
one[0] = (xce_resval_t*)calloc(1, sizeof(xce_resval_t));
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_resvals(&sched, 1, one));
|
|
TEST_ASSERT_EQUAL_INT(1, pop_op(&sched, &op));
|
|
TEST_ASSERT_EQUAL_INT(XCE_SCHED_OP_UPDATE_RESVALS_FULL, op);
|
|
free(one[0]);
|
|
}
|
|
|
|
/* ── scheduler-dispatch-commit task 2.2: sched_handle_notify's new
|
|
* jobDispatchFail case (bug-1625's actual fix) ── */
|
|
|
|
static void test_handle_notify_job_dispatch_fail_enqueues_pend(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_notify_data_t notify;
|
|
job_failinfo_t info = { .jobid = 99, .stage = 0, .code = -5,
|
|
.detail = NULL };
|
|
uint64_t jobid = 0;
|
|
uint32_t state = (uint32_t)-1;
|
|
fixture_init(&sched);
|
|
|
|
memset(¬ify, 0, sizeof(notify));
|
|
notify.event = NotifyData_data_jobDispatchFail;
|
|
notify.data.dispatchfail = &info;
|
|
|
|
/* Previously fell to `default: rc = -EINVAL;` and enqueued nothing --
|
|
* this is the actual observable fix for bug-1625: the notify is now
|
|
* processed instead of silently discarded. */
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_notify(&sched, ¬ify));
|
|
TEST_ASSERT_EQUAL_INT(1, pop_jstate_change(&sched, &jobid, &state));
|
|
TEST_ASSERT_EQUAL_UINT64(99, jobid);
|
|
TEST_ASSERT_EQUAL_INT(JOB_STAT_PEND, state);
|
|
|
|
/* A code of 0 is the controller's SUCCESS ack (the job was dispatched
|
|
* and is already RUN, committed at dispatch) — it must NOT be returned
|
|
* to PEND (bug-1512: re-PENDing the success ack re-elects and
|
|
* re-dispatches a job whose original instance keeps running, spawning
|
|
* duplicate instances). The ack is consumed as a no-op. */
|
|
fixture_init(&sched);
|
|
info.jobid = 123;
|
|
info.code = 0;
|
|
jobid = 0;
|
|
state = (uint32_t)-1;
|
|
notify.data.dispatchfail = &info; /* first notify NULLed it on success */
|
|
TEST_ASSERT_EQUAL_INT(0, sched_handle_notify(&sched, ¬ify));
|
|
TEST_ASSERT_EQUAL_INT(0, pop_jstate_change(&sched, &jobid, &state));
|
|
}
|
|
|
|
int main(int argc, char* argv[])
|
|
{
|
|
(void)argc;
|
|
(void)argv;
|
|
UNITY_BEGIN();
|
|
RUN_TEST(test_handle_update_job_enqueues_job_fields, 0);
|
|
RUN_TEST(test_handle_update_queue_enqueues_queue, 0);
|
|
RUN_TEST(test_handle_job_rusage_enqueues_rusage, 0);
|
|
RUN_TEST(test_handle_update_resvals_skips_empty_enqueues_nonempty, 0);
|
|
RUN_TEST(test_handle_resources_skips_empty_enqueues_nonempty, 0);
|
|
RUN_TEST(test_handle_resvals_full_skips_empty_enqueues_nonempty, 0);
|
|
RUN_TEST(test_handle_notify_job_dispatch_fail_enqueues_pend, 0);
|
|
return UNITY_END();
|
|
}
|