Rongsong Shen
3224e90683
Cause: live applies every pushed config through sched_init_config(), whose plugin_config_update() runs each plugin's config_hook (the limit plugin builds its limits there). The simulator seeded the limit plugin only at bootstrap (sim_bootstrap_limit_plugin()); the core's CONFIG_PUSH event loaded groups, resources and queues but never told the plugins, so a limit added during a replay never existed in sim. Fix: one shared fan-out, plugin_config_update_core(core, config), which live's plugin_config_update() now delegates to. The core's CONFIG_PUSH calls a driver-registered xce_core_config_cb_t after loading groups (live order), keeping the core free of plugin.c; the simulator registers the fan-out in sim_bootstrap_limit_plugin(). limit_update_config() reads the honestly-typed plugin->core instead of casting plugin->sched, so the hook is safe on the sim core; the hand-built plugin fixtures in test_limit_plugin.c/test_repl_limit_table.c now set plugin->core as plugin_register() does. test_sim_limit_config_push_parity.c covers it. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
380 lines
20 KiB
C
380 lines
20 KiB
C
#if !defined(_PLUGIN_H_)
|
|
|
|
#define _PLUGIN_H_ 1
|
|
|
|
#include <stdint.h>
|
|
#include <stdlib.h>
|
|
|
|
#include "decision.h"
|
|
#include "resreq.h"
|
|
#include "sched/types.h"
|
|
|
|
typedef void* plugin_data_t;
|
|
|
|
/* hooker of plugin, the first parameter should be xce_sched_plugin_t */
|
|
typedef int (*plugin_update_config_hook_t)(void*, xce_conf4sched_t*);
|
|
typedef int (*plugin_update_queue_hook_t)(void*, xce_sched_queue_t*);
|
|
typedef int (*plugin_update_job_hook_t)(void*, xce_sched_job_t*);
|
|
typedef int (*plugin_update_worker_hook_t)(void*, xce_sched_worker_t*);
|
|
typedef int (*plugin_job_state_change_hook_t)(void*, xce_sched_job_t*,
|
|
uint32_t);
|
|
typedef int (*plugin_commit_decision_hook_t)(void*, xce_decision_t*);
|
|
typedef int (*plugin_job_election_hook_t)(void*, xce_sched_job_t*);
|
|
typedef int (*plugin_prepare_allocation_hook_t)(void*, xce_sched_ctx_t*);
|
|
typedef int (*plugin_job_rusage_hook_t)(void*, xce_sched_job_t*,
|
|
xce_job_rusages_t*);
|
|
|
|
/* initialize function of plugin */
|
|
typedef int (*plugin_init_fn_t)(void*);
|
|
|
|
typedef void (*plugin_fini_fn_t)(void*);
|
|
|
|
/* plugin function to elect job to be scheduler */
|
|
typedef void* (*plugin_job_election_fn_t)(void*);
|
|
|
|
/* plugin callback when a job has been elected */
|
|
|
|
typedef int (*plugin_begin_session_fn_t)(void*);
|
|
|
|
typedef int (*plugin_end_session_fn_t)(void*);
|
|
|
|
/* allocation function of job */
|
|
typedef int (*plugin_job_allocate_fn_t)(void*, xce_sched_ctx_t*);
|
|
|
|
typedef struct {
|
|
plugin_update_config_hook_t config_hook;
|
|
plugin_update_queue_hook_t queue_hook;
|
|
plugin_update_job_hook_t job_hook;
|
|
plugin_job_state_change_hook_t jstate_hook;
|
|
plugin_update_worker_hook_t worker_hook;
|
|
plugin_commit_decision_hook_t commit_hook;
|
|
plugin_job_election_hook_t job_election_hook;
|
|
plugin_prepare_allocation_hook_t prepare_allocation_hook;
|
|
plugin_job_rusage_hook_t rusage_hook;
|
|
} plugin_hooks_t;
|
|
|
|
/* Forward declaration -- xce_job_picker_iface_t is fully defined below
|
|
* (Req 1 AC 2). Only needed here as a pointer field on xce_sched_plugin_t
|
|
* (picker plugin registration category, scheduler-core task 1.2);
|
|
* re-declared as the same typedef once the struct body is complete below
|
|
* -- same idiom as cluster_t across myconf.h/persist.h/ha.h/cluster_def.h. */
|
|
typedef struct xce_job_picker_iface_s xce_job_picker_iface_t;
|
|
|
|
/* Forward declaration -- xce_sched_allocator_iface_t (the allocator plugin
|
|
* contract, scheduler-allocator-runner Req 1.2) is fully defined in
|
|
* sched/alloc_iter.h, next to the plan-iterator contract it wraps. Only
|
|
* needed here as a pointer field on xce_sched_plugin_t. */
|
|
typedef struct xce_sched_allocator_iface_s xce_sched_allocator_iface_t;
|
|
|
|
/* xce_sched_core_t is defined in core.h -- forward-declared here (re-declared
|
|
* again, harmlessly, near plugin_register_core()'s own declaration below)
|
|
* so plugin_begin_session()/plugin_end_session() can be declared against it
|
|
* ahead of that later block. */
|
|
typedef struct xce_sched_core_s xce_sched_core_t;
|
|
|
|
typedef struct {
|
|
char* name;
|
|
void* sched;
|
|
uint32_t plugin_id;
|
|
plugin_init_fn_t init_fn;
|
|
plugin_fini_fn_t fini_fn;
|
|
plugin_begin_session_fn_t begin_session_fn;
|
|
plugin_end_session_fn_t end_session_fn;
|
|
plugin_job_election_fn_t job_elect_fn;
|
|
plugin_job_allocate_fn_t job_alloc_fn;
|
|
plugin_hooks_t hooks;
|
|
plugin_data_t data;
|
|
/* Picker plugin registration category (task 1.2): non-NULL once this
|
|
* plugin participates in the xce_job_picker_iface_t contract -- either
|
|
* because it was authored against the new interface directly, or
|
|
* because plugin_register() adapted it from the deprecated job_elect_fn
|
|
* hook via the compatibility wrapper (Req 5 "Migration from old
|
|
* job_elect_fn"). Stays NULL for plugin categories that are neither
|
|
* (allocator/limit plugins never set job_elect_fn). */
|
|
xce_job_picker_iface_t* picker_iface;
|
|
/* Allocator plugin registration category (scheduler-allocator-runner
|
|
* Req 1.2 / 1.6): the plugin's allocator contract. Same ownership as
|
|
* picker_iface -- static storage owned by the plugin file (a `static
|
|
* const` object there), copied by pointer on registration and never
|
|
* freed by the registry. NULL for non-allocator plugins. */
|
|
const xce_sched_allocator_iface_t* allocator_iface;
|
|
/* scheduler-sim-live-allocation spec, task 1.2 ("Plugin Hook
|
|
* Dispatchers" / "Limit Plugin Core-Safe Matching"): an honestly-typed
|
|
* reference to the shared core state, set by BOTH plugin_register()
|
|
* (`plugin->core = &(sched->core);`) and plugin_register_core()
|
|
* (`plugin->core = core;`) -- non-NULL after either registration path.
|
|
* This is a NEW, separate field from `sched` above -- `sched` keeps its
|
|
* existing meaning and consumers (still xce_scheduler_t*-flavored,
|
|
* still set only by plugin_register(), still read by plugins/fcfs.c's
|
|
* legacy job_elect_fn path) untouched. A plugin hook that needs shared
|
|
* core state (e.g. plugins/limit.c's job-election/prepare-allocation
|
|
* matching) reads `core`, never `sched`, so it works identically
|
|
* whichever registration path registered it. */
|
|
xce_sched_core_t* core;
|
|
/* bug-1874: set by plugin_register() when it adapted a job_elect_fn-only
|
|
* plugin via the legacy wrapper; the deprecation is reported (once,
|
|
* legacy_picker_warned) when such a plugin is SELECTED as the picker,
|
|
* not merely registered. */
|
|
int legacy_picker;
|
|
int legacy_picker_warned;
|
|
} xce_sched_plugin_t;
|
|
|
|
extern int plugin_register(void*, void*);
|
|
extern void plugin_unregister(void*);
|
|
extern int plugin_config_update(void*, xce_conf4sched_t*);
|
|
extern int plugin_config_update_core(xce_sched_core_t*, xce_conf4sched_t*);
|
|
extern int plugin_queue_update(void*, xce_sched_queue_t*);
|
|
extern int plugin_job_update(void*, xce_sched_job_t*);
|
|
extern int plugin_worker_update(void*, xce_sched_worker_t*);
|
|
extern int plugin_job_state_change(void*, xce_sched_job_t*, uint32_t);
|
|
/* plugin_begin_session()/plugin_end_session() - relocated to xce_sched_core_t*
|
|
* (bug-1681, mirroring sched_account_jobs()'s identical scheduler-sim-
|
|
* decision-parity task 1.1 relocation): the per-plugin begin/end_session_fn
|
|
* iteration only ever touches core->plugins, never anything else on
|
|
* xce_scheduler_t* -- so, like plugin_register_core() below, these are
|
|
* callable from a driver with no xce_scheduler_t* (the simulation driver).
|
|
* See driver.c's xce_sched_driver_run_normal_mode_round() for the normal-
|
|
* mode call site and sim_loop.c's sched_sim_driver_run() for the mirrored
|
|
* simulation-mode one. */
|
|
extern int plugin_begin_session(xce_sched_core_t*);
|
|
extern int plugin_end_session(xce_sched_core_t*);
|
|
/* plugin_commit_decision()/plugin_prepare_allocation()/plugin_job_election()
|
|
* - relocated to xce_sched_core_t* (scheduler-sim-live-allocation spec,
|
|
* task 1.2, mirroring plugin_begin_session()/plugin_end_session()'s own
|
|
* identical bug-1681 relocation immediately above): each dispatch loop only
|
|
* ever touches core->plugins, never anything else on xce_scheduler_t* -- so,
|
|
* like plugin_register_core() below, these are callable from a driver with
|
|
* no xce_scheduler_t* (the simulation driver). Does NOT change
|
|
* plugin_hooks_t's own hook signatures (job_election_hook/
|
|
* prepare_allocation_hook/commit_hook keep their existing `(void* arg, ...)`
|
|
* shape -- `arg` there is the plugin instance, not the driver/core struct). */
|
|
extern int plugin_commit_decision(xce_sched_core_t*, xce_decision_t*);
|
|
extern int plugin_prepare_allocation(xce_sched_core_t*, xce_sched_ctx_t*);
|
|
extern int plugin_job_election(xce_sched_core_t*, xce_sched_job_t*);
|
|
extern int plugin_job_rusage_update(void*, xce_sched_job_t*,
|
|
xce_job_rusages_t*);
|
|
extern int register_fcfs(void*);
|
|
extern int register_builtin_fcfs(void*);
|
|
extern int register_limit(void*);
|
|
|
|
/*
|
|
* limit_new_plugin - factory half of register_limit()'s template-build
|
|
* step (plugins/limit.c), extracted so a caller with no xce_scheduler_t*
|
|
* (e.g. the simulation driver, via plugin_register_core() below) can
|
|
* obtain a freshly calloc'd, fully populated `xce_sched_plugin_t` for the
|
|
* limit plugin -- same hooks/init_fn/fini_fn template register_limit()
|
|
* itself uses, just not yet registered anywhere. Mirrors
|
|
* builtin_fcfs_new_plugin()'s identical extraction (below) exactly
|
|
* (scheduler-sim-diagnostics spec, task 6; bug-1825). Returns NULL on
|
|
* allocation failure. Caller owns the returned plugin (frees it only on a
|
|
* subsequent registration failure -- once registered, the plugin registry
|
|
* owns it, same lifetime rule every other plugin category in this
|
|
* codebase already follows).
|
|
*/
|
|
extern xce_sched_plugin_t* limit_new_plugin(void);
|
|
|
|
/*
|
|
* limit_seed_config - bug-1825 remediation round 2 (scheduler-sim-
|
|
* diagnostics spec, task 6): honestly-typed config-seed entry point for a
|
|
* caller that holds a real xce_sched_core_t* but no real xce_scheduler_t*
|
|
* -- the simulation driver (sim_limit.c's sim_bootstrap_limit_plugin()),
|
|
* whose xce_sched_sim_driver_t (sim.h) has no xce_scheduler_t to point
|
|
* plugin->sched at. Takes `core` directly with no cast and never touches
|
|
* plugin->sched -- see the function's own doc comment (plugins/limit.c)
|
|
* for why a plugin->sched type-pun is unsafe here (core.h's
|
|
* xce_core_cache_clear_cb_t doc comment documents the real production bug
|
|
* this avoids repeating). Delegates to the same seed logic
|
|
* limit_update_config() (the normal-mode config_hook) itself uses, so the
|
|
* two entry points cannot silently diverge in behavior.
|
|
*/
|
|
extern int limit_seed_config(xce_sched_plugin_t* plugin,
|
|
xce_sched_core_t* core,
|
|
xce_conf4sched_t* config);
|
|
|
|
/* limit_get_configured_limits/limit_get_usage_instances - scheduler-sim-
|
|
* diagnostics spec, Requirement Set 1 (Limit Plugin Accessor), task 1.
|
|
* Read-only crossing of the plugin-private-state boundary: each looks up
|
|
* the limit plugin ("xce_sched_limit") in core->plugins and returns the
|
|
* requested field of its private limit_plugin_data_t (plugins/limit.c) --
|
|
* a raw kvmgr handle, iterable via iter_make()/iter_next() -- or NULL if
|
|
* the plugin is not registered or not yet initialized (plugin->data ==
|
|
* NULL). Never allocates, mutates, or triggers plugin hooks; performs no
|
|
* locking beyond what key_find() already does internally. NULL is a
|
|
* semantic signal only ("nothing is configured yet"), not an error -- the
|
|
* caller decides what to display for it. */
|
|
extern var limit_get_configured_limits(xce_sched_core_t* core);
|
|
extern var limit_get_usage_instances(xce_sched_core_t* core);
|
|
extern int register_seqjob_allocator(void*);
|
|
extern int register_parjob_allocator(void*);
|
|
|
|
/*
|
|
* seqjob_allocator_new_plugin/parjob_allocator_new_plugin -
|
|
* scheduler-sim-live-allocation spec, task 4.3: factory half of
|
|
* register_seqjob_allocator()'s/register_parjob_allocator()'s own
|
|
* template-build step, extracted so a caller with no xce_scheduler_t*
|
|
* (the simulation driver, via plugin_register_core() -- see sim_alloc.c)
|
|
* can obtain a freshly calloc'd, fully populated xce_sched_plugin_t for
|
|
* the seqjob/parjob allocator plugin -- same hooks/init_fn/fini_fn
|
|
* template register_seqjob_allocator()/register_parjob_allocator()
|
|
* themselves use, just not yet registered anywhere. Mirrors
|
|
* limit_new_plugin()'s (plugins/limit.c) identical extraction exactly
|
|
* (scheduler-sim-diagnostics spec, task 6; bug-1825), closing bug-1829's
|
|
* tracked follow-up: seqjob_allocator_init()/parjob_allocator_init() (and
|
|
* the prepare_allocation_hook/job_alloc_fn hooks these plugins register)
|
|
* now read the honestly-typed plugin->core field (task 1.2) instead of
|
|
* plugin->sched, so a plugin built here and registered via
|
|
* plugin_register_core(&(sim->core), plugin) directly reaches the SAME
|
|
* core->plugins registry every other sim-mode plugin (the picker, the
|
|
* limit plugin) is registered in -- no separate wrapper registry, unlike
|
|
* task 4.1's now-superseded fix. Returns NULL on allocation failure.
|
|
* Caller owns the returned plugin (frees it only on a subsequent
|
|
* registration failure -- once registered, the plugin registry owns it).
|
|
*/
|
|
extern xce_sched_plugin_t* seqjob_allocator_new_plugin(void);
|
|
extern xce_sched_plugin_t* parjob_allocator_new_plugin(void);
|
|
|
|
/* ---------------------------------------------------------------------
|
|
* Job picker plugin interface (xce_job_picker_iface_t)
|
|
*
|
|
* scheduler-core spec, Req 1 AC 2 / AC 5, Req 2 AC 6.
|
|
*
|
|
* Supersedes plugin_job_election_hook_t / plugin_job_election_fn_t
|
|
* (job_elect_fn above) as the job-selection contract: the core scheduling
|
|
* loop calls picker->next_job(picker) to get the next schedulable job (or
|
|
* NULL) and carries zero knowledge of project/queue/job iteration or
|
|
* ordering — all of that lives behind this interface (Req 1 AC 1).
|
|
* job_elect_fn-only plugins keep working during the transition via a
|
|
* compatibility wrapper (registered in plugin.c, scheduler-core task 1.2);
|
|
* new pickers implement xce_job_picker_iface_t directly.
|
|
* --------------------------------------------------------------------- */
|
|
|
|
/* xce_sched_core_t is defined in core.h, added by the scheduler-core spec's
|
|
* Phase 2 core/driver extraction. Forward-declared here (same idiom as
|
|
* cluster_t in myconf.h/persist.h/ha.h) to avoid a circular include —
|
|
* init() below only ever holds/passes a pointer to it. */
|
|
typedef struct xce_sched_core_s xce_sched_core_t;
|
|
|
|
/*
|
|
* builtin_fcfs_new_plugin - factory half of register_builtin_fcfs()'s
|
|
* template-build step (plugins/builtin_fcfs.c), extracted so a caller with
|
|
* no xce_scheduler_t* (e.g. the simulation driver, via
|
|
* plugin_register_core() below) can obtain a freshly calloc'd, fully
|
|
* populated `xce_sched_plugin_t` for builtin_fcfs -- same picker_iface/
|
|
* init_fn/fini_fn template register_builtin_fcfs() itself uses, just not
|
|
* yet registered anywhere. Returns NULL on allocation failure. Caller owns
|
|
* the returned plugin (frees it only on a subsequent registration failure
|
|
* -- once registered, the plugin registry owns it, same lifetime rule
|
|
* every other plugin category in this codebase already follows).
|
|
*/
|
|
extern xce_sched_plugin_t* builtin_fcfs_new_plugin(void);
|
|
|
|
/*
|
|
* plugin_register_core - Req 5 AC 17 (scheduler-core spec, task 11.2): the
|
|
* "core.h-scoped plugin registration path" sim.h's own doc comment on
|
|
* sched_sim_driver_start() anticipated but did not build (task 11.1's
|
|
* documented "Known scope boundary"). plugin_register() above requires a
|
|
* genuine `xce_scheduler_t*` (it dereferences `sched->core.plugins` and
|
|
* stores `plugin->sched = sched` for plugin_unregister()'s later use) --
|
|
* unavailable to a driver like the simulation driver that deliberately has
|
|
* no such type. This is the SAME registration mechanism minus that one
|
|
* dependency: run the plugin's init_fn (same "driver thread, at plugin
|
|
* register" timing plugin.h's own hook-category doc comment documents),
|
|
* then key_add() it into `core->plugins` directly -- identical steps
|
|
* plugin_register() itself performs, in the same order, just addressed at
|
|
* xce_sched_core_t* instead of xce_scheduler_t*. `plugin->sched` is left
|
|
* NULL (no xce_scheduler_t* exists to store) -- safe, since
|
|
* plugin_unregister() is a no-op on a NULL `plugin->sched` and nothing
|
|
* calls it for a plugin registered this way (no sim-driver teardown path
|
|
* exists yet). Returns the assigned plugin_id (>= 0) on success, or a
|
|
* negative errno: -EINVAL for a NULL core/plugin, whatever init_fn
|
|
* returned if it failed, or key_add()'s own negative return.
|
|
*/
|
|
extern int plugin_register_core(xce_sched_core_t*, xce_sched_plugin_t*);
|
|
|
|
/*
|
|
* Hook categories and the thread each category runs in (Req 1 AC 3):
|
|
*
|
|
* Lifecycle hooks (MUST) — driver thread, at plugin register/teardown:
|
|
* init One-time setup; receives a core reference for state queries.
|
|
* Called exactly once, before any other hook.
|
|
* fini Teardown; frees all picker-owned resources.
|
|
*
|
|
* State-change hooks (MUST) — driver thread; called synchronously, only
|
|
* from inside core_push_events(), which itself is only invoked between
|
|
* sessions (picker thread blocked on cbuf_get(&wbuf), allocator thread
|
|
* blocked on cbuf_get(&jbuf)). No mid-session event injection:
|
|
* on_queue_update Add/update a queue in the picker's internal
|
|
* ordering. <- QUEUE_UPDATE event
|
|
* on_job_push Add a newly pending job to the candidate set.
|
|
* <- JOBS_PUSH event
|
|
* on_job_state_change Drop the job from candidates if it left pending;
|
|
* re-add if it returned to pending.
|
|
* <- JOB_STATE_CHANGE event
|
|
* on_job_update Reorder the job in candidates if its priority or
|
|
* queue changed. <- JOB_UPDATE event
|
|
*
|
|
* Session hooks (MUST) — picker thread, at scheduling-round boundaries:
|
|
* begin_session Reset the iteration cursor. Called at the start of
|
|
* every scheduling round. Session-aware requirement: a
|
|
* picker MUST reset all iteration state here — never
|
|
* carry it over from the previous session.
|
|
* next_job Return the next schedulable job per the picker's
|
|
* ordering, or NULL once exhausted. Cancel-safe
|
|
* requirement: next_job() runs in the dedicated picker
|
|
* thread, which is torn down via pthread_cancel — the
|
|
* iteration loop inside next_job() (and any blocking
|
|
* point it reaches, e.g. cbuf_get(&wbuf) between
|
|
* sessions) MUST have defined pthread_cancel
|
|
* cancellation points (pthread_testcancel() or an
|
|
* existing cancellation-point call).
|
|
* end_session End-of-round notification; called before the core
|
|
* sleeps between rounds.
|
|
*
|
|
* Post-allocation hooks (MUST) — allocator thread, after each allocation
|
|
* attempt for a job next_job() returned:
|
|
* on_dispatch Job was allocated; remove it from candidates
|
|
* permanently.
|
|
* on_pend Job was pended (or preempted back to pend); keep it for
|
|
* the next round, may deprioritize.
|
|
*
|
|
* Optional hooks (MAY be NULL) — same thread as the triggering event
|
|
* above (on_config: driver thread inside core_push_events(); on_job_rusage:
|
|
* driver thread inside core_push_events()):
|
|
* on_config Rebuild picker strategy from config. NULL if the
|
|
* picker is config-agnostic.
|
|
* on_job_rusage Track per-job resource usage. NULL if the picker is
|
|
* not resource-aware.
|
|
*
|
|
* Contract (Req 1 AC 5, Req 2 AC 6):
|
|
* - Cancel-safe: next_job() has defined pthread_cancel cancellation
|
|
* points.
|
|
* - Session-aware: begin_session() resets all iteration state.
|
|
* - Hook implementations never block indefinitely — the picker thread
|
|
* holds no lock the allocator thread needs.
|
|
* - next_job() never returns the same job twice within one session.
|
|
*
|
|
* Preconditions / postconditions:
|
|
* - init() is called exactly once before any other hook.
|
|
* - begin_session() precedes every next_job() sequence.
|
|
* - next_job() returns NULL exactly when no ready job remains for the
|
|
* current session.
|
|
*/
|
|
typedef struct xce_job_picker_iface_s {
|
|
int (*init)(void* picker, xce_sched_core_t* core);
|
|
void (*fini)(void* picker);
|
|
int (*begin_session)(void* picker);
|
|
xce_sched_job_t* (*next_job)(void* picker);
|
|
int (*end_session)(void* picker);
|
|
int (*on_dispatch)(void* picker, uint64_t jobid);
|
|
int (*on_pend)(void* picker, uint64_t jobid);
|
|
int (*on_queue_update)(void* picker, xce_sched_queue_t* queue);
|
|
int (*on_job_push)(void* picker, xce_sched_job_t* job);
|
|
int (*on_job_state_change)(void* picker, xce_sched_job_t* job,
|
|
uint32_t state);
|
|
int (*on_job_update)(void* picker, xce_sched_job_t* job);
|
|
int (*on_config)(void* picker, xce_conf4sched_t* config); /* MAY be NULL */
|
|
int (*on_job_rusage)(void* picker, xce_sched_job_t* job,
|
|
xce_job_rusages_t* rusages); /* MAY be NULL */
|
|
} xce_job_picker_iface_t;
|
|
|
|
#endif
|