lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

job submit and support run in worker

Browse source
This commit is contained in:
Rongsong Shen 2025-11-28 17:24:31 +08:00
parent 3630e38a07
commit e497dd1b69
18 changed files with 502 additions and 236 deletions
Download patch file Download diff file Expand all files Collapse all files

15
src/worker/include/xcecd_utils.h
View file

@ -8,9 +8,16 @@
/* This header file has some utilities and structure for compatibility with
* other scheduler systems. */
#define XCE_DEFAULT_SOCKS 15
#define MAXPATHLEN 1024
#define MAXHOSTNAMELEN 256
#define XCE_DEFAULT_SOCKS 15
#if !defined(MAXPATHLEN)
#define MAXPATHLEN 1024
#endif
#if !defined(MAXHOSTNAMELEN)
#define MAXHOSTNAMELEN 256
#endif
#define MAXLINELEN 4096
#define MAXDATASIZE (16 * 1024)
#define MAXXCENAMELEN 40
@ -133,4 +140,6 @@ typedef struct {
void block_signal(int sig);
void unblock_signal(int sig);
extern int make_job_sd_unit_name(char*, size_t, const char*, uint64_t);
#endif

45
src/worker/src/main.c
View file

@ -1,6 +1,9 @@
#include "sqlite3.h"
#include "xcecd.h"
#include <errno.h>
#include <libgen.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@ -216,6 +219,44 @@ static void master_ckpt_result_callback(uint64_t job_id, int status,
xcecd_job_ckpted(job_id, status, restore_cmd);
}
static int check_xce_server_dir(char* myfile)
{
char* xce_server_dir = getenv("XCE_SERVER_DIR");
char myfullpath[XCE_MAXPATH_LEN + 1];
if (xce_server_dir != NULL) {
return 0;
}
if (realpath(myfile, myfullpath) != NULL) {
char* mydir = dirname(myfullpath);
char workerpath[XCE_MAXPATH_LEN + 1];
snprintf(workerpath, sizeof(workerpath) - 1, "%s/xcecd_worker", mydir);
if (access(workerpath, X_OK) == 0) {
/* setup XCE_SERVER_DIR */
XCECD_LOG(XCE_LOG_INFO, "XCE_SERVER_DIR set to `%s'", mydir);
setenv("XCE_SERVER_DIR", mydir, 0);
return 1;
}
}
XCECD_LOG(XCE_LOG_INFO, "XCE_SERVER_DIR set to `/opt/xce/bin'");
setenv("XCE_SERVER_DIR", "/opt/xce/bin", 0);
return 1;
}
static int check_user(void)
{
if (geteuid() != 0) {
fprintf(stderr, "FATAL: xcecd should be run as root");
exit(-1);
}
return 1;
}
int main(int argc, char* argv[])
{
int opt;
@ -225,6 +266,8 @@ int main(int argc, char* argv[])
char* custom_config_path = NULL;
__attribute__((unused)) char worker_log[1024] = { 0 };
check_user();
signal(SIGPIPE, SIG_IGN);
struct sigaction sa;
@ -392,6 +435,8 @@ int main(int argc, char* argv[])
}
config.log_file = logfile;
check_xce_server_dir(argv[0]);
/* array to store exit worker pids */
init_static_array(&pids, sizeof(int));

11
src/worker/src/process_manager.c
View file

@ -20,6 +20,8 @@
#include <unistd.h>
#endif
#include "utils.h"
#define MAX_PROCESSES 1024
#define BUFFER_SIZE 4096
@ -136,8 +138,11 @@ int process_manager_start_process(process_manager_t* manager,
time(&now);
struct tm* timeinfo = localtime(&now);
if (jobid != 0) {
snprintf(log_path, sizeof(log_path), "/tmp/job_%lu_%lu.out",
(unsigned long)now, jobid);
char idstring[128];
fmt_jobid(idstring, sizeof(idstring) - 1, jobid);
snprintf(log_path, sizeof(log_path), "/tmp/job_%lu_%s.out",
(unsigned long)now, idstring);
XCECD_WORKER_LOG(XCE_LOG_INFO, "Job output file path is: %s", log_path);
} else {
snprintf(log_path, sizeof(log_path),
@ -606,4 +611,4 @@ void process_manager_free(process_manager_t* manager)
FREEUP(manager->processes);
FREEUP(manager);
XCECD_LOG(XCE_LOG_DEBUG, "Process manager freed successfully");
}
}

27
src/worker/src/uv_proc.c
View file

@ -1,5 +1,8 @@
#include "uv_proc.h"
#include "utils.h"
#include "xcecd_utils.h"
#include "xcelog.h"
#include <errno.h>
#include <signal.h>
#include <stdio.h>
@ -8,6 +11,23 @@
#include <sys/wait.h>
#include <unistd.h>
static int make_job_output_path(char* buf, size_t bufsize, const char* dir,
uint64_t jobid, int task)
{
char idstring[128];
if (buf == NULL) {
return -EINVAL;
}
if (dir == NULL) {
dir = "/tmp";
}
fmt_jobid(idstring, sizeof(idstring) - 1, jobid);
return snprintf(buf, bufsize, "%s/job_%s_task%d.out", dir, idstring, task);
}
static void on_process_exit(uv_process_t* process, int64_t exit_status,
int term_signal)
{
@ -241,8 +261,9 @@ int uv_proc_exec(uv_proc_manager_t* manager, const char* command,
}
char job_output[256];
snprintf(job_output, sizeof(job_output), "/tmp/job_%lu_task%d.out",
manager->job_id, i);
make_job_output_path(job_output, sizeof(job_output) - 1, "/tmp",
manager->job_id, i);
uv_stdio_container_t stdio[3];
uv_fs_t file_req;
@ -491,4 +512,4 @@ int uv_proc_exec_single(uv_proc_manager_t* manager, const char* command,
void* user_data)
{
return uv_proc_exec(manager, command, args, exit_cb, user_data, 1);
}
}

8
src/worker/src/xcecd.c
View file

@ -13,6 +13,7 @@
#include <sys/un.h>
#include "process_worker_manager.h"
#include "utils.h"
xcecd_context_t xcecd_ctx = { 0 };
hostattrs_t host_attrs = { 0 };
@ -31,7 +32,7 @@ static int virtual_pid = 1000;
void xcecd_process_command(void* arg)
{
pid_t pid = 0;
char unit_name[256];
char unit_name[256];
fork_task_t* ctx = (fork_task_t*)arg;
if (!ctx || !ctx->startinfo || !ctx->startinfo->cmdline) {
@ -126,9 +127,8 @@ void xcecd_process_command(void* arg)
return;
}
snprintf(unit_name, sizeof(unit_name), "xcecd-worker-%lu.service",
ctx->xce_jobid);
make_job_sd_unit_name(unit_name, sizeof(unit_name) - 1, "xcecd-worker",
ctx->xce_jobid);
/* get worker pid */
pid = dbus_get_worker_pid(unit_name);
#else

9
src/worker/src/xcecd_cgroups.c
View file

@ -27,10 +27,9 @@ char* start_systemd_job(const char* command, const res_limits_t* limits,
char service_name[256];
// 生成唯一的unit名
const char* prefix = unit_name_prefix ? unit_name_prefix : "xcecd-worker";
snprintf(service_name, sizeof(service_name), "%s-%lu.service", prefix,
job_id);
make_job_sd_unit_name(service_name, sizeof(service_name) - 1,
unit_name_prefix ? unit_name_prefix : "xcecd-worker",
job_id);
// 连接到system bus
int r = sd_bus_open_system(&bus);
if (r < 0) {
@ -1225,4 +1224,4 @@ char* get_systemd_job_status_text(const char* unit_name)
sd_bus_unref(bus);
return status_text;
}
}

20
src/worker/src/xcecd_dbus_manager.c
View file

@ -3,6 +3,7 @@
* @brief This file contains the implementation of the D-Bus manager functions.
*/
#include "xcecd_dbus_manager.h"
#include "utils.h"
#include "xcecd_utils.h"
#include "xcelog.h"
@ -62,7 +63,8 @@ bool dbus_start_worker(const res_limits_t* limits, uint64_t job_id,
}
char myname[256];
snprintf(myname, sizeof(myname), "xcecd-worker-%lu.service", job_id);
make_job_sd_unit_name(myname, sizeof(myname) - 1, "xcecd-worker", job_id);
/* checking if service name is exist before start service and it's a
* negative case */
dbus_stop_worker(myname);
@ -207,8 +209,9 @@ int dbus_workers_update(xcecd_worker_t** worker_list, xcecd_job_t* job,
}
char unit_name[256];
snprintf(unit_name, sizeof(unit_name), "xcecd-worker-%lu.service",
worker->job_info.jobid);
make_job_sd_unit_name(unit_name, sizeof(unit_name) - 1, "xcecd-worker",
worker->job_info.jobid);
if (!dbus_is_worker_active(unit_name)) {
#ifndef VIRTUAL_CLUSTER
@ -350,8 +353,9 @@ bool dbus_worker_clean(xcecd_worker_t* worker, xcecd_job_t* job)
}
char unit_name[256];
snprintf(unit_name, sizeof(unit_name), "xcecd-worker-%lu.service",
worker->job_info.jobid);
make_job_sd_unit_name(unit_name, sizeof(unit_name) - 1, "xcecd-worker",
worker->job_info.jobid);
if (dbus_is_worker_active(unit_name)) {
stop_systemd_job(unit_name);
@ -380,9 +384,9 @@ void dbus_stop_all_workers(xcecd_worker_t** worker_list, sqlite3* db)
xcecd_worker_t* worker = *worker_list;
while (worker) {
char unit_name[256];
snprintf(unit_name, sizeof(unit_name), "xcecd-worker-%lu.service",
worker->job_info.jobid);
make_job_sd_unit_name(unit_name, sizeof(unit_name) - 1, "xcecd-worker",
worker->job_info.jobid);
job.job_id = worker->job_info.jobid;
job.job_pid = worker->worker_pid;
job.job_status = JOB_EXITED;
@ -395,4 +399,4 @@ void dbus_stop_all_workers(xcecd_worker_t** worker_list, sqlite3* db)
worker = worker->next;
}
}
}

20
src/worker/src/xcecd_exdata.c
View file

@ -9,6 +9,9 @@
#include "xcelog.h"
#include <errno.h>
#include <stdint.h>
#include <sys/resource.h>
#include "utils.h"
/**
* @brief Parses the JSON string, populates an environment variable linked list,
@ -279,8 +282,15 @@ bool parse_limits(const char* exdata, res_limits_t* limits)
bool success = false;
cJSON* limits_obj = cJSON_GetObjectItemCaseSensitive(root, "limits");
if (!limits_obj || !cJSON_IsObject(limits_obj)) {
// limits not exist
if (limits_obj == NULL) {
success = true;
memset(limits, 0x0, sizeof(*limits));
goto cleanup;
}
if (!cJSON_IsObject(limits_obj)) {
// invalidate limits
goto cleanup;
}
@ -404,7 +414,7 @@ void apply_env_vars(env_var_t** env_vars, uint64_t jobid)
// Set some default XCE environment variables
char jobid_str[32];
snprintf(jobid_str, sizeof(jobid_str), "%lu", jobid);
snprintf(jobid_str, sizeof(jobid_str), "%lu", array_jobid(jobid));
if (setenv("XCE_JOBID", jobid_str, 1) != 0) {
XCECD_WORKER_LOG(XCE_LOG_ERROR,
"Error setting environment variable: %s",
@ -416,7 +426,7 @@ void apply_env_vars(env_var_t** env_vars, uint64_t jobid)
}
char jobindex_str[8];
snprintf(jobindex_str, sizeof(jobindex_str), "%d", 0);
snprintf(jobindex_str, sizeof(jobindex_str), "%lu", array_idx(jobid));
if (setenv("XCE_JOBINDEX", jobindex_str, 1) != 0) {
XCECD_WORKER_LOG(XCE_LOG_ERROR,
"Error setting environment variable: %s",
@ -491,4 +501,4 @@ char* generate_exdata(const start_exdata_t* start_info)
}
return exdata_str;
}
}

4
src/worker/src/xcecd_worker.c
View file

@ -1168,8 +1168,8 @@ static void do_criu_ckpt(uint64_t job_id, int kill)
XCECD_WORKER_LOG(XCE_LOG_DEBUG3, "Checkpoint directory created: %s",
ckpt_dir);
char criu_cmd[MAXLINELEN];
char info[MAXLINELEN];
char criu_cmd[MAXLINELEN*2];
char info[MAXLINELEN*2];
if (kill) {
snprintf(criu_cmd, sizeof(criu_cmd),
"cd %s; %s dump -t %d -vvv -o dump.log", ckpt_dir, path,

16
src/worker/src/xcecd_worker_utils.c
View file

@ -1,4 +1,5 @@
#include "xcecd_worker_utils.h"
#include "utils.h"
#include "uv_ipc.h"
#include "xcecd_dbus_manager.h"
#include "xcecd_utils.h"
@ -512,4 +513,17 @@ bool check_worker_sleep_time_elapsed(xcecd_worker_t* worker)
return false;
}
}
#endif
#endif
int make_job_sd_unit_name(char* buf, size_t bufsize, const char* prefix,
uint64_t jobid)
{
char idstring[128];
if (buf == NULL) {
return -EINVAL;
}
fmt_jobid_ex(idstring, sizeof(idstring) - 1, jobid, '_', '_');
return snprintf(buf, bufsize, "%s-%s.service", prefix, idstring);
}