Rongsong Shen
7ee9a16f53
Registers builtin_fcfs at scheduler startup and makes it the default job_pick_plugin, replacing the hardcoded "xce_sched_fcfs" literal in sched_job_picker() with a fail-loud lookup through sched_resolve_job_picker(). The picker thread now branches legacy job_elect_fn plugins (delegated unchanged, preserving their self-contained wbuf-driven loop) from native xce_job_picker_iface_t pickers (driven by a new per-session loop), and the allocator now calls on_dispatch/on_pend for every picker as Req 1 AC 4 requires. Closes task 2.3's flagged picker/allocator concurrency race with a mutex on fcfs_plugin_data_t, and adds cancel points to next_job().
212 lines
10 KiB
C
212 lines
10 KiB
C
#if !defined(_PLUGIN_H_)
|
|
|
|
#define _PLUGIN_H_ 1
|
|
|
|
#include <stdint.h>
|
|
#include <stdlib.h>
|
|
|
|
#include "decision.h"
|
|
#include "resreq.h"
|
|
#include "sched/types.h"
|
|
|
|
typedef void* plugin_data_t;
|
|
|
|
/* hooker of plugin, the first parameter should be xce_sched_plugin_t */
|
|
typedef int (*plugin_update_config_hook_t)(void*, xce_conf4sched_t*);
|
|
typedef int (*plugin_update_queue_hook_t)(void*, xce_sched_queue_t*);
|
|
typedef int (*plugin_update_job_hook_t)(void*, xce_sched_job_t*);
|
|
typedef int (*plugin_update_worker_hook_t)(void*, xce_sched_worker_t*);
|
|
typedef int (*plugin_job_state_change_hook_t)(void*, xce_sched_job_t*,
|
|
uint32_t);
|
|
typedef int (*plugin_commit_decision_hook_t)(void*, xce_decision_t*);
|
|
typedef int (*plugin_job_election_hook_t)(void*, xce_sched_job_t*);
|
|
typedef int (*plugin_prepare_allocation_hook_t)(void*, xce_sched_ctx_t*);
|
|
typedef int (*plugin_job_rusage_hook_t)(void*, xce_sched_job_t*,
|
|
xce_job_rusages_t*);
|
|
|
|
/* initialize function of plugin */
|
|
typedef int (*plugin_init_fn_t)(void*);
|
|
|
|
typedef void (*plugin_fini_fn_t)(void*);
|
|
|
|
/* plugin function to elect job to be scheduler */
|
|
typedef void* (*plugin_job_election_fn_t)(void*);
|
|
|
|
/* plugin callback when a job has been elected */
|
|
|
|
typedef int (*plugin_begin_session_fn_t)(void*);
|
|
|
|
typedef int (*plugin_end_session_fn_t)(void*);
|
|
|
|
/* allocation function of job */
|
|
typedef int (*plugin_job_allocate_fn_t)(void*, xce_sched_ctx_t*);
|
|
|
|
typedef struct {
|
|
plugin_update_config_hook_t config_hook;
|
|
plugin_update_queue_hook_t queue_hook;
|
|
plugin_update_job_hook_t job_hook;
|
|
plugin_job_state_change_hook_t jstate_hook;
|
|
plugin_update_worker_hook_t worker_hook;
|
|
plugin_commit_decision_hook_t commit_hook;
|
|
plugin_job_election_hook_t job_election_hook;
|
|
plugin_prepare_allocation_hook_t prepare_allocation_hook;
|
|
plugin_job_rusage_hook_t rusage_hook;
|
|
} plugin_hooks_t;
|
|
|
|
/* Forward declaration -- xce_job_picker_iface_t is fully defined below
|
|
* (Req 1 AC 2). Only needed here as a pointer field on xce_sched_plugin_t
|
|
* (picker plugin registration category, scheduler-core task 1.2);
|
|
* re-declared as the same typedef once the struct body is complete below
|
|
* -- same idiom as cluster_t across myconf.h/persist.h/ha.h/cluster_def.h. */
|
|
typedef struct xce_job_picker_iface_s xce_job_picker_iface_t;
|
|
|
|
typedef struct {
|
|
char* name;
|
|
void* sched;
|
|
uint32_t plugin_id;
|
|
plugin_init_fn_t init_fn;
|
|
plugin_fini_fn_t fini_fn;
|
|
plugin_begin_session_fn_t begin_session_fn;
|
|
plugin_end_session_fn_t end_session_fn;
|
|
plugin_job_election_fn_t job_elect_fn;
|
|
plugin_job_allocate_fn_t job_alloc_fn;
|
|
plugin_hooks_t hooks;
|
|
plugin_data_t data;
|
|
/* Picker plugin registration category (task 1.2): non-NULL once this
|
|
* plugin participates in the xce_job_picker_iface_t contract -- either
|
|
* because it was authored against the new interface directly, or
|
|
* because plugin_register() adapted it from the deprecated job_elect_fn
|
|
* hook via the compatibility wrapper (Req 5 "Migration from old
|
|
* job_elect_fn"). Stays NULL for plugin categories that are neither
|
|
* (allocator/limit plugins never set job_elect_fn). */
|
|
xce_job_picker_iface_t* picker_iface;
|
|
} xce_sched_plugin_t;
|
|
|
|
extern int plugin_register(void*, void*);
|
|
extern void plugin_unregister(void*);
|
|
extern int plugin_config_update(void*, xce_conf4sched_t*);
|
|
extern int plugin_queue_update(void*, xce_sched_queue_t*);
|
|
extern int plugin_job_update(void*, xce_sched_job_t*);
|
|
extern int plugin_worker_update(void*, xce_sched_worker_t*);
|
|
extern int plugin_job_state_change(void*, xce_sched_job_t*, uint32_t);
|
|
extern int plugin_begin_session(void*);
|
|
extern int plugin_end_session(void*);
|
|
extern int plugin_commit_decision(void*, xce_decision_t*);
|
|
extern int plugin_prepare_allocation(void*, xce_sched_ctx_t*);
|
|
extern int plugin_job_election(void*, xce_sched_job_t*);
|
|
extern int plugin_job_rusage_update(void*, xce_sched_job_t*,
|
|
xce_job_rusages_t*);
|
|
extern int register_fcfs(void*);
|
|
extern int register_builtin_fcfs(void*);
|
|
extern int register_limit(void*);
|
|
extern int register_seqjob_allocator(void*);
|
|
extern int register_parjob_allocator(void*);
|
|
|
|
/* ---------------------------------------------------------------------
|
|
* Job picker plugin interface (xce_job_picker_iface_t)
|
|
*
|
|
* scheduler-core spec, Req 1 AC 2 / AC 5, Req 2 AC 6.
|
|
*
|
|
* Supersedes plugin_job_election_hook_t / plugin_job_election_fn_t
|
|
* (job_elect_fn above) as the job-selection contract: the core scheduling
|
|
* loop calls picker->next_job(picker) to get the next schedulable job (or
|
|
* NULL) and carries zero knowledge of project/queue/job iteration or
|
|
* ordering — all of that lives behind this interface (Req 1 AC 1).
|
|
* job_elect_fn-only plugins keep working during the transition via a
|
|
* compatibility wrapper (registered in plugin.c, scheduler-core task 1.2);
|
|
* new pickers implement xce_job_picker_iface_t directly.
|
|
* --------------------------------------------------------------------- */
|
|
|
|
/* xce_sched_core_t is defined in core.h, added by the scheduler-core spec's
|
|
* Phase 2 core/driver extraction. Forward-declared here (same idiom as
|
|
* cluster_t in myconf.h/persist.h/ha.h) to avoid a circular include —
|
|
* init() below only ever holds/passes a pointer to it. */
|
|
typedef struct xce_sched_core_s xce_sched_core_t;
|
|
|
|
/*
|
|
* Hook categories and the thread each category runs in (Req 1 AC 3):
|
|
*
|
|
* Lifecycle hooks (MUST) — driver thread, at plugin register/teardown:
|
|
* init One-time setup; receives a core reference for state queries.
|
|
* Called exactly once, before any other hook.
|
|
* fini Teardown; frees all picker-owned resources.
|
|
*
|
|
* State-change hooks (MUST) — driver thread; called synchronously, only
|
|
* from inside core_push_events(), which itself is only invoked between
|
|
* sessions (picker thread blocked on cbuf_get(&wbuf), allocator thread
|
|
* blocked on cbuf_get(&jbuf)). No mid-session event injection:
|
|
* on_queue_update Add/update a queue in the picker's internal
|
|
* ordering. <- QUEUE_UPDATE event
|
|
* on_job_push Add a newly pending job to the candidate set.
|
|
* <- JOBS_PUSH event
|
|
* on_job_state_change Drop the job from candidates if it left pending;
|
|
* re-add if it returned to pending.
|
|
* <- JOB_STATE_CHANGE event
|
|
* on_job_update Reorder the job in candidates if its priority or
|
|
* queue changed. <- JOB_UPDATE event
|
|
*
|
|
* Session hooks (MUST) — picker thread, at scheduling-round boundaries:
|
|
* begin_session Reset the iteration cursor. Called at the start of
|
|
* every scheduling round. Session-aware requirement: a
|
|
* picker MUST reset all iteration state here — never
|
|
* carry it over from the previous session.
|
|
* next_job Return the next schedulable job per the picker's
|
|
* ordering, or NULL once exhausted. Cancel-safe
|
|
* requirement: next_job() runs in the dedicated picker
|
|
* thread, which is torn down via pthread_cancel — the
|
|
* iteration loop inside next_job() (and any blocking
|
|
* point it reaches, e.g. cbuf_get(&wbuf) between
|
|
* sessions) MUST have defined pthread_cancel
|
|
* cancellation points (pthread_testcancel() or an
|
|
* existing cancellation-point call).
|
|
* end_session End-of-round notification; called before the core
|
|
* sleeps between rounds.
|
|
*
|
|
* Post-allocation hooks (MUST) — allocator thread, after each allocation
|
|
* attempt for a job next_job() returned:
|
|
* on_dispatch Job was allocated; remove it from candidates
|
|
* permanently.
|
|
* on_pend Job was pended (or preempted back to pend); keep it for
|
|
* the next round, may deprioritize.
|
|
*
|
|
* Optional hooks (MAY be NULL) — same thread as the triggering event
|
|
* above (on_config: driver thread inside core_push_events(); on_job_rusage:
|
|
* driver thread inside core_push_events()):
|
|
* on_config Rebuild picker strategy from config. NULL if the
|
|
* picker is config-agnostic.
|
|
* on_job_rusage Track per-job resource usage. NULL if the picker is
|
|
* not resource-aware.
|
|
*
|
|
* Contract (Req 1 AC 5, Req 2 AC 6):
|
|
* - Cancel-safe: next_job() has defined pthread_cancel cancellation
|
|
* points.
|
|
* - Session-aware: begin_session() resets all iteration state.
|
|
* - Hook implementations never block indefinitely — the picker thread
|
|
* holds no lock the allocator thread needs.
|
|
* - next_job() never returns the same job twice within one session.
|
|
*
|
|
* Preconditions / postconditions:
|
|
* - init() is called exactly once before any other hook.
|
|
* - begin_session() precedes every next_job() sequence.
|
|
* - next_job() returns NULL exactly when no ready job remains for the
|
|
* current session.
|
|
*/
|
|
typedef struct xce_job_picker_iface_s {
|
|
int (*init)(void* picker, xce_sched_core_t* core);
|
|
void (*fini)(void* picker);
|
|
int (*begin_session)(void* picker);
|
|
xce_sched_job_t* (*next_job)(void* picker);
|
|
int (*end_session)(void* picker);
|
|
int (*on_dispatch)(void* picker, uint64_t jobid);
|
|
int (*on_pend)(void* picker, uint64_t jobid);
|
|
int (*on_queue_update)(void* picker, xce_sched_queue_t* queue);
|
|
int (*on_job_push)(void* picker, xce_sched_job_t* job);
|
|
int (*on_job_state_change)(void* picker, xce_sched_job_t* job,
|
|
uint32_t state);
|
|
int (*on_job_update)(void* picker, xce_sched_job_t* job);
|
|
int (*on_config)(void* picker, xce_conf4sched_t* config); /* MAY be NULL */
|
|
int (*on_job_rusage)(void* picker, xce_sched_job_t* job,
|
|
xce_job_rusages_t* rusages); /* MAY be NULL */
|
|
} xce_job_picker_iface_t;
|
|
|
|
#endif
|