/* * test_config_push_deferred.c - bug-1846 part 2 regression: a config push * that arrives while a scheduling session is in progress must not end that * session early. * * The controller answers every scheduler join (on_scheduler_join(), * controller/sched.c) by re-pushing config, workers and jobs. The scheduler * re-joins from sched_check_pending_rejoin(), which sched_publish_retry_loop() * calls after a successful publish -- i.e. from inside the driver's decision * drain loop, mid-round. handler.c used to apply the config push right there * on the handler thread: sched_init_config() -> sched_init_queues() -> * builtin_fcfs on_queue_update(), which nulls the picker's cursor when it * re-inserts the queue the cursor is resting on. next_job() then reports * "no more candidates" and the round ends with the rest of the queue never * considered. Captured in scheduler-replay-20260923-085356.replay as 9 * rounds cut off after 11-12 decisions, each followed by the rejoin's * workersPush + full 50-job jobsPush (the config push itself was never * recorded, since it bypassed the pending-update queue). * * This fixture puts the real fcfs picker mid-session, delivers a config * push through the real scheduler_req_handler(), and requires the session * to keep walking the queue; the config must then take effect when the * driver applies it between rounds. */ #include #include #include #include #include #include #include #if defined(__SANITIZE_ADDRESS__) #include #endif #include "controller.capnp.h" #include "plugin.h" #include "resobj.h" #include "sched/types.h" #include "scheduler.h" #include "xstring.h" /* handler.c's request entry point -- no header declares it (main.c passes * it to sched_init() by name); "extern declared where consumed", the same * idiom test_sim_contention_limit_driver_parity.c uses. */ extern int scheduler_req_handler(xce_request_t* request, void* args); #define QUEUE_NAME "normal" #define TEST_RESREQ ((char*)"select[type==any] rusage[mem=1]") #define JOBA_ID ((uint64_t)518471ULL) #define JOBB_ID ((uint64_t)518472ULL) #define JOBC_ID ((uint64_t)518473ULL) void setUp(void) { return; } void tearDown(void) { return; } static void register_slots_resource(var resmgr) { xce_resource_t res; memset(&res, 0, sizeof(res)); res.name = strdup("slots"); res.valtype = XCE_VAL_NUMERIC; res.ordtype = XCE_ORD_DECR; res.flags = XCE_RESF_LIMIT | XCE_RESF_CONSUMABLE | XCE_RESF_RELEASE | XCE_RESF_DEFAULT_HOST; TEST_ASSERT_NOT_NULL(makeResource(resmgr, &res)); } static void push_job(xce_scheduler_t* sched, xce_sched_plugin_t* picker, uint64_t jobid, uint64_t queue_time) { job_submit_data_t submit; xce_job_t j; xce_sched_job_t* job; int rc; memset(&submit, 0, sizeof(submit)); submit.queue = (char*)QUEUE_NAME; submit.user = (char*)"cfgpush_user"; submit.project_name = (char*)"default"; submit.resreq = TEST_RESREQ; submit.mintasks = 1; submit.maxtasks = 1; memset(&j, 0, sizeof(j)); j.jobid = jobid; j.state = JOB_STAT_PEND; j.sub_time = queue_time; j.queue_time = queue_time; j.submit = &submit; job = make_sched_job(sched, &j); TEST_ASSERT_NOT_NULL(job); rc = sched_add_job(&(sched->core), job); TEST_ASSERT_TRUE(rc >= 0); picker->picker_iface->on_job_push(picker, job); } /* The same shape on_scheduler_join()'s xcesched_push_configs() sends: a * Conf4Sched naming the queue the picker is already walking. */ static void deliver_config_push(xce_scheduler_t* sched) { xce_queue_t* q = (xce_queue_t*)calloc(1, sizeof(*q)); xce_queue_t** queues = (xce_queue_t**)calloc(1, sizeof(*queues)); xce_conf4sched_t* config = (xce_conf4sched_t*)calloc(1, sizeof(*config)); xce_configdata_t configdata; xce_request_t request; int rc; TEST_ASSERT_NOT_NULL(q); TEST_ASSERT_NOT_NULL(queues); TEST_ASSERT_NOT_NULL(config); q->queue = strdup(QUEUE_NAME); queues[0] = q; config->n_queues = 1; config->queues = queues; memset(&configdata, 0, sizeof(configdata)); configdata.data.conf4sched = config; memset(&request, 0, sizeof(request)); request.opcode = OPCode_push; request.body_which = Request_body_configpushmsg; request.body.configpushmsg = &configdata; rc = scheduler_req_handler(&request, sched); TEST_ASSERT_TRUE(rc >= 0); } static xce_scheduler_t g_sched; static void test_config_push_mid_session_does_not_end_round(void) { xce_sched_plugin_t* fcfs = NULL; xce_sched_queue_t* queue; xce_sched_job_t* job; int configver_before; int rc; memset(&g_sched, 0, sizeof(g_sched)); rc = xce_sched_core_init(&(g_sched.core)); TEST_ASSERT_EQUAL_INT(0, rc); register_slots_resource(g_sched.core.resmgr); fcfs = builtin_fcfs_new_plugin(); TEST_ASSERT_NOT_NULL(fcfs); rc = plugin_register(&g_sched, fcfs); TEST_ASSERT_TRUE(rc >= 0); g_sched.core.job_pick_plugin = (char*)"builtin_fcfs"; queue = (xce_sched_queue_t*)calloc(1, sizeof(*queue)); TEST_ASSERT_NOT_NULL(queue); queue->name = makeString(QUEUE_NAME); rc = sched_add_queue(&(g_sched.core), queue); TEST_ASSERT_TRUE(rc >= 0); fcfs->picker_iface->on_queue_update(fcfs, queue); push_job(&g_sched, fcfs, JOBA_ID, 1000ULL); push_job(&g_sched, fcfs, JOBB_ID, 1001ULL); push_job(&g_sched, fcfs, JOBC_ID, 1002ULL); /* Mid-session: the picker has handed out job A and rests on job B. */ rc = fcfs->picker_iface->begin_session(fcfs); TEST_ASSERT_EQUAL_INT(0, rc); job = fcfs->picker_iface->next_job(fcfs); TEST_ASSERT_NOT_NULL(job); TEST_ASSERT_EQUAL_UINT64(JOBA_ID, job->jobid); configver_before = g_sched.core.configver; /* A rejoin's config push lands now, on the handler thread. */ deliver_config_push(&g_sched); job = fcfs->picker_iface->next_job(fcfs); TEST_ASSERT_NOT_NULL_MESSAGE( job, "a config push delivered mid-session must not end the round -- " "job B was never considered"); TEST_ASSERT_EQUAL_UINT64(JOBB_ID, job->jobid); job = fcfs->picker_iface->next_job(fcfs); TEST_ASSERT_NOT_NULL(job); TEST_ASSERT_EQUAL_UINT64(JOBC_ID, job->jobid); TEST_ASSERT_NULL(fcfs->picker_iface->next_job(fcfs)); TEST_ASSERT_EQUAL_INT_MESSAGE( configver_before, g_sched.core.configver, "the pushed config must not be applied while the session runs"); fcfs->picker_iface->end_session(fcfs); /* Between rounds the driver applies it -- exactly once. */ TEST_ASSERT_EQUAL_INT(1, sched_apply_pending_config(&g_sched)); TEST_ASSERT_EQUAL_INT_MESSAGE(configver_before + 1, g_sched.core.configver, "the stashed config must take effect when " "applied between rounds"); TEST_ASSERT_EQUAL_INT(0, sched_apply_pending_config(&g_sched)); TEST_ASSERT_EQUAL_INT(configver_before + 1, g_sched.core.configver); printf("test_config_push_mid_session_does_not_end_round: PASS\n"); } /* The same deferral makes the push recordable: it is now applied at a * round boundary on the driver thread, exactly where the recorder writes * the round's other controller events. Before, it bypassed the pending- * update queue entirely and never reached a .replay file, so a replay * spanning a rejoin (or a genuine config change) never saw it. */ static xce_scheduler_t g_rec_sched; static void test_applied_config_push_is_recorded(void) { xce_queue_t* q = (xce_queue_t*)calloc(1, sizeof(*q)); xce_queue_t** queues = (xce_queue_t**)calloc(1, sizeof(*queues)); xce_conf4sched_t* config = (xce_conf4sched_t*)calloc(1, sizeof(*config)); FILE* f; int rc; memset(&g_rec_sched, 0, sizeof(g_rec_sched)); rc = xce_sched_core_init(&(g_rec_sched.core)); TEST_ASSERT_EQUAL_INT(0, rc); f = tmpfile(); TEST_ASSERT_NOT_NULL(f); g_rec_sched.recording = true; g_rec_sched.record_file = f; q->queue = strdup(QUEUE_NAME); queues[0] = q; config->n_queues = 1; config->queues = queues; TEST_ASSERT_EQUAL_INT(0, sched_stash_config(&g_rec_sched, config)); TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, g_rec_sched.record_event_count, "stashing alone must not record"); TEST_ASSERT_EQUAL_INT(1, sched_apply_pending_config(&g_rec_sched)); TEST_ASSERT_EQUAL_UINT64_MESSAGE( 1, g_rec_sched.record_event_count, "an applied config push must be written to the recording as one " "configPush event"); TEST_ASSERT_TRUE(ftell(f) > 0); g_rec_sched.recording = false; g_rec_sched.record_file = NULL; fclose(f); printf("test_applied_config_push_is_recorded: PASS\n"); } /* bug-1886: stashes and applies a config carrying one queue, QUEUE_NAME, * at `priority`; returns the applied config. */ static xce_conf4sched_t* apply_queue_config(xce_scheduler_t* sched, int32_t priority) { xce_queue_t* q = (xce_queue_t*)calloc(1, sizeof(*q)); xce_queue_t** queues = (xce_queue_t**)calloc(1, sizeof(*queues)); xce_conf4sched_t* config = (xce_conf4sched_t*)calloc(1, sizeof(*config)); TEST_ASSERT_NOT_NULL(q); TEST_ASSERT_NOT_NULL(queues); TEST_ASSERT_NOT_NULL(config); q->queue = strdup(QUEUE_NAME); q->priority = priority; queues[0] = q; config->n_queues = 1; config->queues = queues; TEST_ASSERT_EQUAL_INT(0, sched_stash_config(sched, config)); TEST_ASSERT_EQUAL_INT(1, sched_apply_pending_config(sched)); TEST_ASSERT_EQUAL_PTR(config, sched->core.config); return config; } /* * bug-1886: re-applying a config updates the registered queue in place * (the rejected duplicate used to be built, handed to the picker and * leaked, while the registered queue kept its old settings), and the * config it replaces is freed. */ static xce_scheduler_t g_reapply_sched; static void test_config_reapply_updates_queue_in_place_and_frees_old(void) { xce_sched_plugin_t* fcfs = NULL; xce_sched_queue_t* q_first; xce_sched_queue_t* q_second; xce_conf4sched_t* first; int rc; memset(&g_reapply_sched, 0, sizeof(g_reapply_sched)); rc = xce_sched_core_init(&(g_reapply_sched.core)); TEST_ASSERT_EQUAL_INT(0, rc); fcfs = builtin_fcfs_new_plugin(); TEST_ASSERT_NOT_NULL(fcfs); rc = plugin_register(&g_reapply_sched, fcfs); TEST_ASSERT_TRUE(rc >= 0); g_reapply_sched.core.job_pick_plugin = (char*)"builtin_fcfs"; first = apply_queue_config(&g_reapply_sched, 1); q_first = sched_get_queue(&(g_reapply_sched.core), QUEUE_NAME); TEST_ASSERT_NOT_NULL(q_first); TEST_ASSERT_EQUAL_INT(1, q_first->priority); (void)apply_queue_config(&g_reapply_sched, 7); q_second = sched_get_queue(&(g_reapply_sched.core), QUEUE_NAME); TEST_ASSERT_EQUAL_PTR(q_first, q_second); TEST_ASSERT_EQUAL_INT_MESSAGE( 7, q_second->priority, "a re-applied config must update the registered queue in place"); #if defined(__SANITIZE_ADDRESS__) TEST_ASSERT_TRUE_MESSAGE(__asan_address_is_poisoned(first), "the replaced config must be freed"); #else (void)first; #endif } int main(void) { UNITY_BEGIN(); RUN_TEST(test_config_push_mid_session_does_not_end_round); RUN_TEST(test_applied_config_push_is_recorded); RUN_TEST(test_config_reapply_updates_queue_in_place_and_frees_old); return UNITY_END(); }