diff --git a/src/autoscaler/scaler/include/obj_mgr.h b/src/autoscaler/scaler/include/obj_mgr.h index f9a80ba8..48f9983e 100644 --- a/src/autoscaler/scaler/include/obj_mgr.h +++ b/src/autoscaler/scaler/include/obj_mgr.h @@ -17,19 +17,22 @@ extern scaler_provider_t* provider_get(scaler_obj_mgr_t*, char*); extern int template_add(scaler_obj_mgr_t*, scaler_template_t*); extern scaler_template_t* template_get(scaler_obj_mgr_t*, char*); -extern scaler_template_t* template_get_by_id(scaler_obj_mgr_t*, scaler_template_id_t*); +extern scaler_template_t* template_get_by_id(scaler_obj_mgr_t*, + scaler_template_id_t*); extern int template_del(scaler_obj_mgr_t*, char*); extern int template_cmp(const void* a, const void* b); -extern int merge_templates(scaler_obj_mgr_t*, scaler_obj_mgr_t*, char*); +extern int merge_templates(scaler_obj_mgr_t*, scaler_obj_mgr_t*, char*); extern int request_add(scaler_obj_mgr_t*, scaler_request_t*); extern scaler_request_t* request_get(scaler_obj_mgr_t*, char*); -extern int request_update_status(scaler_obj_mgr_t*, char*, scaler_req_status_t, time_t); -extern int request_del(scaler_obj_mgr_t*, char*); +extern int request_update_status(scaler_obj_mgr_t*, char*, scaler_req_status_t, + time_t); +extern int request_del(scaler_obj_mgr_t*, char*); extern int instance_add(scaler_obj_mgr_t*, scaler_instance_t*); extern scaler_instance_t* instance_get(scaler_obj_mgr_t*, char*); -extern int instance_update_status(scaler_obj_mgr_t*, char*, scaler_alloc_status_t, char*, char*); +extern int instance_update_status(scaler_obj_mgr_t*, char*, + scaler_alloc_status_t, char*, char*); extern int instance_del(scaler_obj_mgr_t*, char*); #endif \ No newline at end of file diff --git a/src/autoscaler/scaler/include/plug_mgr.h b/src/autoscaler/scaler/include/plug_mgr.h index 1a2100a9..2a14ca09 100644 --- a/src/autoscaler/scaler/include/plug_mgr.h +++ b/src/autoscaler/scaler/include/plug_mgr.h @@ -1,8 +1,8 @@ #ifndef _PLUG_MGR_H_ #define _PLUG_MGR_H_ -#include #include "scaler/types.h" +#include extern int get_available_templates(char* prov_name); extern int trigger_create(scaler_demand_t* demand); diff --git a/src/autoscaler/scaler/include/prov_conf.h b/src/autoscaler/scaler/include/prov_conf.h index 9d7e1817..4c44f26e 100644 --- a/src/autoscaler/scaler/include/prov_conf.h +++ b/src/autoscaler/scaler/include/prov_conf.h @@ -15,7 +15,7 @@ typedef struct { // Structure for YAML providers configuration typedef struct { yaml_provider_t* providers; - size_t providers_count; + size_t providers_count; } yaml_providers_config_t; extern int load_providers(const char*); diff --git a/src/autoscaler/scaler/include/scaler.h b/src/autoscaler/scaler/include/scaler.h index 207a70d4..96079fa0 100644 --- a/src/autoscaler/scaler/include/scaler.h +++ b/src/autoscaler/scaler/include/scaler.h @@ -5,7 +5,6 @@ #include #include -#include #include "config.h" #include "htab.h" #include "mylock.h" @@ -13,6 +12,7 @@ #include "xce/types.h" #include "xlist.h" #include "zarray.h" +#include // in seconds #define DEFAULT_CHECK_INTERVAL 30 @@ -44,7 +44,7 @@ typedef struct { pthread_t threads[2]; int running; #define SCALER_MENU_CHANGED 0x1 - int flag; + int flag; } autoscaler_context_t; extern autoscaler_context_t g_scaler_ctx; diff --git a/src/autoscaler/scaler/include/scaler/types.h b/src/autoscaler/scaler/include/scaler/types.h index 6e62155a..57918335 100644 --- a/src/autoscaler/scaler/include/scaler/types.h +++ b/src/autoscaler/scaler/include/scaler/types.h @@ -48,7 +48,8 @@ typedef struct { } v; } scaler_template_attr_t; -// Template available number of quota and current pricing (for spot/bid instances) +// Template available number of quota and current pricing (for spot/bid +// instances) typedef struct { char* id; // unique id: __as__provider#template scaler_template_id_t tpl_id; @@ -65,9 +66,11 @@ typedef enum { ALLOC_STATUS_NONE = 0, ALLOC_STATUS_ALLOCATED, /* get a running instance from cloud */ ALLOC_STATUS_RECLAIMED, /* cloud reclaimed this instance */ - ALLOC_STATUS_RELINQUISH_REQ_SENT, /* request is sent to scheduler that it should relinquish this instance */ + ALLOC_STATUS_RELINQUISH_REQ_SENT, /* request is sent to scheduler that it + should relinquish this instance */ ALLOC_STATUS_RELINQUISHED, /* scheduler relinquished this instance */ - ALLOC_STATUS_DEALLOCATE_REQ_SENT, /* release/deallocate request is sent to cloud to terminate this instance */ + ALLOC_STATUS_DEALLOCATE_REQ_SENT, /* release/deallocate request is sent to + cloud to terminate this instance */ ALLOC_STATUS_DONE } scaler_alloc_status_t; diff --git a/src/autoscaler/scaler/include/scaler_json.h b/src/autoscaler/scaler/include/scaler_json.h index fdf5e48c..19fc776c 100644 --- a/src/autoscaler/scaler/include/scaler_json.h +++ b/src/autoscaler/scaler/include/scaler_json.h @@ -1,18 +1,19 @@ #ifndef __JSON_UTILS_H__ #define __JSON_UTILS_H__ +#include "scaler/types.h" #include #include -#include "scaler/types.h" -extern int write_json_to_file(const char* filename, cJSON* json); -extern int read_json_file(const char* filename, cJSON** json); -extern cJSON* s2j_demand_t(scaler_demand_t* demand); -extern cJSON* s2j_instance_id_list_t(instance_id_list_t* instance_ids); -extern cJSON* s2j_request_id_list_t(scaler_request_t* req); -extern scaler_instance_t* j2s_instance_t(cJSON* json, scaler_req_type_t req_type); +extern int write_json_to_file(const char* filename, cJSON* json); +extern int read_json_file(const char* filename, cJSON** json); +extern cJSON* s2j_demand_t(scaler_demand_t* demand); +extern cJSON* s2j_instance_id_list_t(instance_id_list_t* instance_ids); +extern cJSON* s2j_request_id_list_t(scaler_request_t* req); +extern scaler_instance_t* j2s_instance_t(cJSON* json, + scaler_req_type_t req_type); extern scaler_instance_id_t* j2s_instance_id_t(cJSON* json); extern scaler_template_t* j2s_template_t(char* provider_name, cJSON* json); -extern cJSON* s2j_create_empty_obj(); +extern cJSON* s2j_create_empty_obj(); extern char* attrs_to_str(scaler_template_attr_t* attrs, int num_attr); extern int str_to_attr(char* str, scaler_template_attr_t** out_attrs); diff --git a/src/autoscaler/scaler/include/scaler_misc.h b/src/autoscaler/scaler/include/scaler_misc.h index 9ff501fa..14359818 100644 --- a/src/autoscaler/scaler/include/scaler_misc.h +++ b/src/autoscaler/scaler/include/scaler_misc.h @@ -8,7 +8,7 @@ extern int my_snprintf(char* buf, size_t size, const char* fmt, ...); extern int create_temp_file(char* file_name); extern int my_strcmp(const char* p1, const char* p2); extern char* my_strdup(char*); -extern int cmp_template_id(scaler_template_id_t* p1, scaler_template_id_t* p2); +extern int cmp_template_id(scaler_template_id_t* p1, scaler_template_id_t* p2); extern void print_demand_list(scaler_demand_array_t*); extern void print_request(void*); diff --git a/src/autoscaler/scaler/include/scaler_obj.h b/src/autoscaler/scaler/include/scaler_obj.h index 75a39338..4b935487 100644 --- a/src/autoscaler/scaler/include/scaler_obj.h +++ b/src/autoscaler/scaler/include/scaler_obj.h @@ -12,8 +12,9 @@ extern void template_fields_fini(void*); extern void template_fini(void*); extern int template_cpy(scaler_template_t*, scaler_template_t*); -extern void request_fini(void*); -extern scaler_request_t* new_create_request(const char*, const char*, int, const char*); +extern void request_fini(void*); +extern scaler_request_t* new_create_request(const char*, const char*, int, + const char*); extern scaler_request_t* new_release_request(const char*, char*); extern void instance_id_fields_fini(void*); diff --git a/src/autoscaler/scaler/obj_mgr.c b/src/autoscaler/scaler/obj_mgr.c index 7f866ee6..df34c2c0 100644 --- a/src/autoscaler/scaler/obj_mgr.c +++ b/src/autoscaler/scaler/obj_mgr.c @@ -41,10 +41,7 @@ int scaler_obj_mgr_count(scaler_obj_mgr_t* mgr) return count; } -static int _obj_mgr_add(scaler_obj_mgr_t* mgr, - void* obj, - char* key, - int* idx) +static int _obj_mgr_add(scaler_obj_mgr_t* mgr, void* obj, char* key, int* idx) { hent_t* e = NULL; int i; @@ -132,10 +129,8 @@ static void* obj_mgr_get_ex(scaler_obj_mgr_t* mgr, char* key) return _obj_mgr_get(mgr, key); } -static int obj_mgr_add_ex(scaler_obj_mgr_t* mgr, - void* obj, - char* key, - char* type) +static int obj_mgr_add_ex(scaler_obj_mgr_t* mgr, void* obj, char* key, + char* type) { int idx = -1; int ret; @@ -198,7 +193,8 @@ scaler_template_t* template_get(scaler_obj_mgr_t* mgr, char* key) return (scaler_template_t*)obj_mgr_get_ex(mgr, key); } -scaler_template_t* template_get_by_id(scaler_obj_mgr_t* mgr, scaler_template_id_t* tid) +scaler_template_t* template_get_by_id(scaler_obj_mgr_t* mgr, + scaler_template_id_t* tid) { if (!mgr || !tid) { return NULL; @@ -212,7 +208,8 @@ int template_del(scaler_obj_mgr_t* mgr, char* id) { return obj_mgr_del_ex(mgr, id, SC_TEMPLATE); } -static int attribute_cmp(const scaler_template_attr_t* a1, const scaler_template_attr_t* a2) +static int attribute_cmp(const scaler_template_attr_t* a1, + const scaler_template_attr_t* a2) { if (!a1 || !a2) { return -EINVAL; @@ -349,7 +346,7 @@ static int _add_not_found_template_id(void* arg, void* data) static int _clean_not_found_template(void* data, void* arg) { char* id = (char*)data; - if (! id) { + if (!id) { return 0; } template_del(&g_scaler_ctx.templates, id); @@ -358,7 +355,8 @@ static int _clean_not_found_template(void* data, void* arg) return 0; } -int merge_templates(scaler_obj_mgr_t* tgt, scaler_obj_mgr_t* src, char* provider_name) +int merge_templates(scaler_obj_mgr_t* tgt, scaler_obj_mgr_t* src, + char* provider_name) { tpl_arg_t tmp; xlist_t l; @@ -395,7 +393,8 @@ scaler_request_t* request_get(scaler_obj_mgr_t* mgr, char* key) return (scaler_request_t*)obj_mgr_get_ex(mgr, key); } -int request_update_status(scaler_obj_mgr_t* mgr, char* req_id, scaler_req_status_t status, time_t now) +int request_update_status(scaler_obj_mgr_t* mgr, char* req_id, + scaler_req_status_t status, time_t now) { scaler_request_t* req_in_mgr; @@ -435,11 +434,9 @@ scaler_instance_t* instance_get(scaler_obj_mgr_t* mgr, char* key) return (scaler_instance_t*)obj_mgr_get_ex(mgr, key); } -int instance_update_status(scaler_obj_mgr_t* mgr, - char* instance_id, - scaler_alloc_status_t alloc_status, - char* inst_state, - char* relinquish_id) +int instance_update_status(scaler_obj_mgr_t* mgr, char* instance_id, + scaler_alloc_status_t alloc_status, char* inst_state, + char* relinquish_id) { scaler_instance_t* inst; diff --git a/src/autoscaler/scaler/plug_mgr.c b/src/autoscaler/scaler/plug_mgr.c index ec6a23d4..98e72297 100644 --- a/src/autoscaler/scaler/plug_mgr.c +++ b/src/autoscaler/scaler/plug_mgr.c @@ -1,11 +1,11 @@ // Main functions: -#include #include "obj_mgr.h" #include "scaler.h" #include "scaler_json.h" #include "scaler_misc.h" #include "scaler_obj.h" #include "utils.h" +#include #include #include #include @@ -13,11 +13,13 @@ #include #include -static cJSON* invoke(scaler_provider_t* prov, INTERFACE_ACTION ACTION, cJSON* in_json); +static cJSON* invoke(scaler_provider_t* prov, INTERFACE_ACTION ACTION, + cJSON* in_json); /* - * This function call invoke() to execute shell script of provider->scripts[GET_AVAILABLE_TEMPLATES] - * Get all available templates from the provider. Add the templates to g_scaler_ctx.templates. + * This function call invoke() to execute shell script of + * provider->scripts[GET_AVAILABLE_TEMPLATES] Get all available templates from + * the provider. Add the templates to g_scaler_ctx.templates. */ int get_available_templates(char* prov_name) { @@ -116,8 +118,7 @@ int trigger_create(scaler_demand_t* demand) if (req_id_item && cJSON_IsString(req_id_item)) { strncpy(req_id, req_id_item->valuestring, sizeof(req_id) - 1); req = new_create_request(demand->tpl_id.provider_name, - demand->tpl_id.template_id, - demand->count, + demand->tpl_id.template_id, demand->count, req_id); ret = request_add(&g_scaler_ctx.requests, req); if (ret < 0) { @@ -179,11 +180,9 @@ int trigger_release(char* prov_name, instance_id_list_t* inst_ids) return ret; } for (int i = 0; i < inst_ids->num; i++) { - ret = instance_update_status(&g_scaler_ctx.instances, - inst_ids->instance_ids[i].instance_id, - ALLOC_STATUS_DEALLOCATE_REQ_SENT, - NULL, - req_id); + ret = instance_update_status( + &g_scaler_ctx.instances, inst_ids->instance_ids[i].instance_id, + ALLOC_STATUS_DEALLOCATE_REQ_SENT, NULL, req_id); if (ret < 0) { ERROR("Failed to mark instance %s as RELINQUISHED", inst_ids->instance_ids[i].instance_id); @@ -264,10 +263,12 @@ int get_request_status(scaler_request_t* req) // Safe string duplication with null checks if (req->tpl_id.provider_name) { - inst->tpl_id.provider_name = strdup(req->tpl_id.provider_name); + inst->tpl_id.provider_name + = strdup(req->tpl_id.provider_name); } if (req->tpl_id.template_id) { - inst->tpl_id.template_id = strdup(req->tpl_id.template_id); + inst->tpl_id.template_id + = strdup(req->tpl_id.template_id); } if (req->request_id) { inst->request_id = strdup(req->request_id); @@ -275,16 +276,15 @@ int get_request_status(scaler_request_t* req) int ret = instance_add(&g_scaler_ctx.instances, inst); if (ret < 0 && ret != -EEXIST) { - ERROR("Failed to add instance %s, ret %d", inst->instance_id, ret); + ERROR("Failed to add instance %s, ret %d", + inst->instance_id, ret); instance_fini(inst); } } else if (req->type == SCALER_REQ_TYPE_RET) { inst->relinquish_id = strdup(req->request_id); - instance_update_status(&g_scaler_ctx.instances, - inst->instance_id, - inst->alloc_status, - inst->inst_state, - NULL); + instance_update_status( + &g_scaler_ctx.instances, inst->instance_id, + inst->alloc_status, inst->inst_state, NULL); instance_fini(inst); } } @@ -295,7 +295,8 @@ int get_request_status(scaler_request_t* req) status = REQ_STATUS_COMPLETED; } else if (strcmp(status_item->valuestring, "running") == 0) { status = REQ_STATUS_RUNNING; - } else if (strcmp(status_item->valuestring, "complete_with_error") == 0) { + } else if (strcmp(status_item->valuestring, "complete_with_error") + == 0) { status = REQ_STATUS_COMPLETE_WITH_ERROR; } else { ERROR("Invalid request status %s", status_item->valuestring); @@ -308,7 +309,8 @@ int get_request_status(scaler_request_t* req) req->request_id, status, req->status); continue; } - ret = request_update_status(&g_scaler_ctx.requests, req->request_id, status, now); + ret = request_update_status(&g_scaler_ctx.requests, req->request_id, + status, now); if (ret < 0) { ERROR("Failed to update request status for %s, ret %d", req->request_id, ret); @@ -368,11 +370,8 @@ int get_return_request(char* prov_name, instance_id_list_t* inst_ids) ERROR("Failed to parse instance ID from request object"); continue; } - instance_update_status(&g_scaler_ctx.instances, - inst_id->instance_id, - ALLOC_STATUS_RECLAIMED, - NULL, - NULL); + instance_update_status(&g_scaler_ctx.instances, inst_id->instance_id, + ALLOC_STATUS_RECLAIMED, NULL, NULL); instance_id_fini(inst_id); } @@ -418,7 +417,8 @@ static int write_input_json(const char* filename, cJSON* json) * It takes a JSON input, invokes the script, and returns a JSON output. * All 5 interfaces can be called in this way. */ -static cJSON* invoke(scaler_provider_t* prov, INTERFACE_ACTION ACTION, cJSON* in_json) +static cJSON* invoke(scaler_provider_t* prov, INTERFACE_ACTION ACTION, + cJSON* in_json) { char in_file[64] = "/tmp/sc_in_XXXXXX"; char out_file[64] = "/tmp/sc_out_XXXXXX"; @@ -452,10 +452,7 @@ static cJSON* invoke(scaler_provider_t* prov, INTERFACE_ACTION ACTION, cJSON* in exit(EXIT_FAILURE); } // Build command to call the script - my_snprintf(cmd, - sizeof(cmd), - "%s/scripts/%s", - prov->scriptPath, + my_snprintf(cmd, sizeof(cmd), "%s/scripts/%s", prov->scriptPath, prov->scripts[ACTION]); // Redirect stdout and stderr to file diff --git a/src/autoscaler/scaler/scaler.c b/src/autoscaler/scaler/scaler.c index cf2008d6..a0f52429 100644 --- a/src/autoscaler/scaler/scaler.c +++ b/src/autoscaler/scaler/scaler.c @@ -167,8 +167,7 @@ typedef struct { instance_id_list_t* id_list; } prov_inst_id_list_t; -static int -_add_relinquished_inst_id(void* arg, void* data) +static int _add_relinquished_inst_id(void* arg, void* data) { prov_inst_id_list_t* l = (prov_inst_id_list_t*)arg; scaler_instance_t* inst = (scaler_instance_t*)data; @@ -205,8 +204,7 @@ static int _grab_relinquished_hosts_list(prov_inst_id_list_t* l) return 0; } -static int -_add_allocated_inst_id(void* arg, void* data) +static int _add_allocated_inst_id(void* arg, void* data) { prov_inst_id_list_t* l = (prov_inst_id_list_t*)arg; scaler_instance_t* inst = (scaler_instance_t*)data; @@ -258,8 +256,8 @@ static int _get_reclaimed_hosts(void* data, void* arg) return -ENOMEM; } id_list->num = 0; - id_list->instance_ids = (scaler_instance_id_t*)calloc(g_scaler_ctx.instances.array.count, - sizeof(scaler_instance_id_t)); + id_list->instance_ids = (scaler_instance_id_t*)calloc( + g_scaler_ctx.instances.array.count, sizeof(scaler_instance_id_t)); l.provider_name = p->name; l.id_list = id_list; @@ -275,8 +273,7 @@ static int _get_reclaimed_hosts(void* data, void* arg) return 0; } -static int -_add_done_instance_id(void* arg, void* data) +static int _add_done_instance_id(void* arg, void* data) { xlist_t* l = (xlist_t*)arg; scaler_instance_t* inst = (scaler_instance_t*)data; @@ -327,8 +324,7 @@ typedef struct { int reclaimed; } demand_count_t; -static int -_count_allocated_instances(void* arg1, void* arg2) +static int _count_allocated_instances(void* arg1, void* arg2) { demand_count_t* counts = (demand_count_t*)arg1; scaler_instance_t* inst = (scaler_instance_t*)arg2; @@ -408,12 +404,14 @@ static int calculate(scaler_demand_t* demand) // Count allocated instances mylock_lock(&(g_scaler_ctx.instances.lock)); - zarray_iter(&(g_scaler_ctx.instances.array), _count_allocated_instances, &t_count); + zarray_iter(&(g_scaler_ctx.instances.array), _count_allocated_instances, + &t_count); mylock_unlock(&(g_scaler_ctx.instances.lock)); // Count requesting instances mylock_lock(&(g_scaler_ctx.requests.lock)); - zarray_iter(&(g_scaler_ctx.requests.array), _count_requesting_instances, &t_count); + zarray_iter(&(g_scaler_ctx.requests.array), _count_requesting_instances, + &t_count); mylock_unlock(&(g_scaler_ctx.requests.lock)); template_max = _get_template_max_number(&demand->tpl_id); @@ -422,7 +420,8 @@ static int calculate(scaler_demand_t* demand) left = template_max - total; if (left <= 0) { - INFO("Template %s reached max number %d, no new instances can be created", + INFO("Template %s reached max number %d, no new instances can be " + "created", demand->tpl_id.template_id, template_max); demand->count = 0; return 0; @@ -437,9 +436,10 @@ static int calculate(scaler_demand_t* demand) new_demand = 0; } - INFO("Demand %s: requested %d, allocated %d, requesting %d, reclaimed %d, new demand %d", - demand->tpl_id.template_id, demand->count, t_count.allocated, t_count.requesting, - t_count.reclaimed, new_demand); + INFO("Demand %s: requested %d, allocated %d, requesting %d, reclaimed %d, " + "new demand %d", + demand->tpl_id.template_id, demand->count, t_count.allocated, + t_count.requesting, t_count.reclaimed, new_demand); demand->count = new_demand; return demand->count; } @@ -459,8 +459,8 @@ static int _return_relinquished_hosts(void* data, void* arg) return -ENOMEM; } id_list->num = 0; - id_list->instance_ids = (scaler_instance_id_t*)calloc(g_scaler_ctx.instances.array.count, - sizeof(scaler_instance_id_t)); + id_list->instance_ids = (scaler_instance_id_t*)calloc( + g_scaler_ctx.instances.array.count, sizeof(scaler_instance_id_t)); l.provider_name = p->name; l.id_list = id_list; @@ -507,19 +507,19 @@ int request_delete_instances(instance_id_list_t* inst_ids) ERROR("Null instance ID list"); return -EINVAL; } - // Do not directly trigger release. Instead, mark the instances as RELINQUISHED + // Do not directly trigger release. Instead, mark the instances as + // RELINQUISHED for (int i = 0; i < inst_ids->num; i++) { ret = instance_update_status(&g_scaler_ctx.instances, inst_ids->instance_ids[i].instance_id, - ALLOC_STATUS_RELINQUISHED, - NULL, - NULL); + ALLOC_STATUS_RELINQUISHED, NULL, NULL); if (ret < 0) { ERROR("Failed to mark instance %s as RELINQUISHED", inst_ids->instance_ids[i].instance_id); continue; } - TRACE("Marked instance %s as RELINQUISHED", inst_ids->instance_ids[i].instance_id); + TRACE("Marked instance %s as RELINQUISHED", + inst_ids->instance_ids[i].instance_id); } return 0; } @@ -593,7 +593,7 @@ int scaler_start() } demand_thread = &g_scaler_ctx.threads[1]; - ret = pthread_create(demand_thread, NULL, scaler_demand_handler, NULL); + ret = pthread_create(demand_thread, NULL, scaler_demand_handler, NULL); if (ret != 0) { ERROR("Failed to create demand handler thread: %s", strerror(ret)); goto Err_Exit; diff --git a/src/autoscaler/scaler/scaler_ctx.c b/src/autoscaler/scaler/scaler_ctx.c index c545376e..51ceab04 100644 --- a/src/autoscaler/scaler/scaler_ctx.c +++ b/src/autoscaler/scaler/scaler_ctx.c @@ -14,7 +14,8 @@ int scaler_ctx_init(char* prv_file) memset(&g_scaler_ctx, 0, sizeof(g_scaler_ctx)); if (prv_file) { - strncpy(g_scaler_ctx.configs.providers_file, prv_file, XCE_MAXPATH_LEN - 1); + strncpy(g_scaler_ctx.configs.providers_file, prv_file, + XCE_MAXPATH_LEN - 1); } else { return -EINVAL; } @@ -66,7 +67,7 @@ int scaler_ctx_is_menu_changed() int flag = 0; mylock_lock(&g_scaler_ctx.lock); - flag = g_scaler_ctx.flag & SCALER_MENU_CHANGED; + flag = g_scaler_ctx.flag & SCALER_MENU_CHANGED; mylock_unlock(&g_scaler_ctx.lock); return flag; @@ -79,4 +80,3 @@ int scaler_ctx_reset_menu_changed() mylock_unlock(&g_scaler_ctx.lock); return 0; } - diff --git a/src/autoscaler/scaler/scaler_json.c b/src/autoscaler/scaler/scaler_json.c index 2c939faa..ac2a48f4 100644 --- a/src/autoscaler/scaler/scaler_json.c +++ b/src/autoscaler/scaler/scaler_json.c @@ -63,8 +63,7 @@ int read_json_file(const char* filename, cJSON** json) return 0; } -cJSON* -s2j_demand_t(scaler_demand_t* demand) +cJSON* s2j_demand_t(scaler_demand_t* demand) { if (!demand) return NULL; @@ -89,8 +88,7 @@ s2j_demand_t(scaler_demand_t* demand) return json; } -cJSON* -s2j_instance_id_list_t(instance_id_list_t* instance_ids) +cJSON* s2j_instance_id_list_t(instance_id_list_t* instance_ids) { if (!instance_ids) return NULL; @@ -111,8 +109,10 @@ s2j_instance_id_list_t(instance_id_list_t* instance_ids) ERROR("Failed to create machine JSON object"); return NULL; } - cJSON_AddStringToObject(machine, "name", instance_ids->instance_ids[i].hostname); - cJSON_AddStringToObject(machine, "machineId", instance_ids->instance_ids[i].instance_id); + cJSON_AddStringToObject(machine, "name", + instance_ids->instance_ids[i].hostname); + cJSON_AddStringToObject(machine, "machineId", + instance_ids->instance_ids[i].instance_id); cJSON_AddItemToArray(machines, machine); } cJSON_AddItemToObject(json, "machines", machines); @@ -121,10 +121,10 @@ s2j_instance_id_list_t(instance_id_list_t* instance_ids) } /* - * This is the abstracted function to convert scaler_request_t to JSON. It only save the request_id. + * This is the abstracted function to convert scaler_request_t to JSON. It only + * save the request_id. */ -cJSON* -s2j_request_id_list_t(scaler_request_t* req) +cJSON* s2j_request_id_list_t(scaler_request_t* req) { if (!req) return NULL; @@ -150,10 +150,7 @@ s2j_request_id_list_t(scaler_request_t* req) return json; } -cJSON* s2j_create_empty_obj() -{ - return cJSON_CreateObject(); -} +cJSON* s2j_create_empty_obj() { return cJSON_CreateObject(); } // convert scaler_template_attr_t array to cJSON object cJSON* s2j_attribute_t(scaler_template_attr_t* attrs, int num_attr) @@ -175,7 +172,9 @@ cJSON* s2j_attribute_t(scaler_template_attr_t* attrs, int num_attr) switch (attrs[i].type) { case SCALER_ATTR_TYPE_BOOL: cJSON_AddItemToArray(attr_array, cJSON_CreateString("Boolean")); - cJSON_AddItemToArray(attr_array, cJSON_CreateString(attrs[i].v.bool_val ? "1" : "0")); + cJSON_AddItemToArray( + attr_array, + cJSON_CreateString(attrs[i].v.bool_val ? "1" : "0")); break; case SCALER_ATTR_TYPE_NUM: { char buf[32]; @@ -185,7 +184,9 @@ cJSON* s2j_attribute_t(scaler_template_attr_t* attrs, int num_attr) } break; case SCALER_ATTR_TYPE_STR: cJSON_AddItemToArray(attr_array, cJSON_CreateString("String")); - cJSON_AddItemToArray(attr_array, cJSON_CreateString(attrs[i].v.str_val ? attrs[i].v.str_val : "")); + cJSON_AddItemToArray( + attr_array, cJSON_CreateString( + attrs[i].v.str_val ? attrs[i].v.str_val : "")); break; default: cJSON_AddItemToArray(attr_array, cJSON_CreateString("Null")); @@ -223,12 +224,10 @@ static int j2s_attribute_t(cJSON* attrs_obj, scaler_template_attr_t** out_attrs) cJSON* attr = NULL; int count = 0; int idx = 0; - cJSON_ArrayForEach(attr, attrs_obj) - { - count++; - } + cJSON_ArrayForEach(attr, attrs_obj) { count++; } num_attrs = count; - attrs = (scaler_template_attr_t*)calloc(count, sizeof(scaler_template_attr_t)); + attrs = (scaler_template_attr_t*)calloc(count, + sizeof(scaler_template_attr_t)); cJSON_ArrayForEach(attr, attrs_obj) { @@ -241,13 +240,15 @@ static int j2s_attribute_t(cJSON* attrs_obj, scaler_template_attr_t** out_attrs) if (cJSON_IsArray(attr)) { cJSON* type_item = cJSON_GetArrayItem(attr, 0); cJSON* value_item = cJSON_GetArrayItem(attr, 1); - if (type_item && cJSON_IsString(type_item) - && value_item && cJSON_IsString(value_item)) { + if (type_item && cJSON_IsString(type_item) && value_item + && cJSON_IsString(value_item)) { const char* type_str = type_item->valuestring; const char* value_str = value_item->valuestring; if (strcmp(type_str, "Boolean") == 0) { - attrs[idx].type = SCALER_ATTR_TYPE_BOOL; - attrs[idx].v.bool_val = (strcmp(value_str, "1") == 0 || strcasecmp(value_str, "true") == 0); + attrs[idx].type = SCALER_ATTR_TYPE_BOOL; + attrs[idx].v.bool_val + = (strcmp(value_str, "1") == 0 + || strcasecmp(value_str, "true") == 0); } else if (strcmp(type_str, "Numeric") == 0) { attrs[idx].type = SCALER_ATTR_TYPE_NUM; attrs[idx].v.num_val = atoi(value_str); @@ -304,7 +305,8 @@ scaler_template_t* j2s_template_t(char* provider_name, cJSON* json) if (item && cJSON_IsString(item)) { char key[256]; tpl->tpl_id.template_id = strdup(item->valuestring); - my_snprintf(key, sizeof(key), "%s#%s", tpl->tpl_id.provider_name, tpl->tpl_id.template_id); + my_snprintf(key, sizeof(key), "%s#%s", tpl->tpl_id.provider_name, + tpl->tpl_id.template_id); tpl->id = strdup(key); } else { ERROR("No templateId define in json."); @@ -347,7 +349,8 @@ scaler_instance_id_t* j2s_instance_id_t(cJSON* json) if (!json) return NULL; - scaler_instance_id_t* inst_id = (scaler_instance_id_t*)calloc(1, sizeof(scaler_instance_id_t)); + scaler_instance_id_t* inst_id + = (scaler_instance_id_t*)calloc(1, sizeof(scaler_instance_id_t)); if (!inst_id) return NULL; @@ -378,7 +381,8 @@ scaler_instance_t* j2s_instance_t(cJSON* json, scaler_req_type_t req_type) return NULL; item = cJSON_GetObjectItem(json, "result"); - if (item && cJSON_IsString(item) && strcmp(item->valuestring, "succeed") == 0) { + if (item && cJSON_IsString(item) + && strcmp(item->valuestring, "succeed") == 0) { inst = (scaler_instance_t*)calloc(1, sizeof(scaler_instance_t)); if (req_type == SCALER_REQ_TYPE_REQ) { inst->alloc_status = ALLOC_STATUS_ALLOCATED; diff --git a/src/autoscaler/scaler/scaler_misc.c b/src/autoscaler/scaler/scaler_misc.c index 4e3c0685..a692b255 100644 --- a/src/autoscaler/scaler/scaler_misc.c +++ b/src/autoscaler/scaler/scaler_misc.c @@ -21,7 +21,8 @@ int my_snprintf(char* buf, size_t size, const char* fmt, ...) return ret; } if ((size_t)ret >= size) { - WARN("my_snprintf: output truncated for format: %s (needed %d bytes, buffer size %zu)", + WARN("my_snprintf: output truncated for format: %s (needed %d bytes, " + "buffer size %zu)", fmt, ret + 1, size); } return ret; @@ -33,14 +34,16 @@ int my_snprintf(char* buf, size_t size, const char* fmt, ...) * like "/tmp/as_in_XXXXXX.json". The 'X's will be replaced with a unique * sequence to ensure the file name is unique. * - * Returns 0 on success, -1 on failure. file_name will contain the name of the created file. + * Returns 0 on success, -1 on failure. file_name will contain the name of the + * created file. */ int create_temp_file(char* file_name) { // Call mkstemp to create a temporary file int fd = mkstemp(file_name); if (fd == -1) { - ERROR("Failed to create temporary file, errno %d : %s.", errno, strerror(errno)); + ERROR("Failed to create temporary file, errno %d : %s.", errno, + strerror(errno)); return -1; } close(fd); @@ -87,10 +90,8 @@ void print_demand_list(scaler_demand_array_t* d_list) INFO("Number of demands: %d", d_list->num); for (int i = 0; i < d_list->num; i++) { scaler_demand_t* demand = &d_list->data[i]; - INFO("Demand %d: provider=%s, template=%s, count=%d", - i, - demand->tpl_id.provider_name, - demand->tpl_id.template_id, + INFO("Demand %d: provider=%s, template=%s, count=%d", i, + demand->tpl_id.provider_name, demand->tpl_id.template_id, demand->count); } } @@ -112,9 +113,9 @@ void print_request(void* data) return; } - INFO("Request: req_id=%s, type=%d, status=%d, provider=%s, template=%s, count=%d", - request->request_id ? request->request_id : "NULL", - request->type, + INFO("Request: req_id=%s, type=%d, status=%d, provider=%s, template=%s, " + "count=%d", + request->request_id ? request->request_id : "NULL", request->type, request->status, request->tpl_id.provider_name ? request->tpl_id.provider_name : "NULL", request->tpl_id.template_id ? request->tpl_id.template_id : "NULL", @@ -153,10 +154,8 @@ void print_instance(void* p) inst->instance_id ? inst->instance_id : "NULL", inst->hostname ? inst->hostname : "NULL", inst->private_ip ? inst->private_ip : "NULL", - inst->public_ip ? inst->public_ip : "NULL", - (long)inst->launch_time, - inst->alloc_status, - inst->inst_state ? inst->inst_state : "NULL", + inst->public_ip ? inst->public_ip : "NULL", (long)inst->launch_time, + inst->alloc_status, inst->inst_state ? inst->inst_state : "NULL", inst->tpl_id.provider_name ? inst->tpl_id.provider_name : "NULL", inst->tpl_id.template_id ? inst->tpl_id.template_id : "NULL", inst->request_id ? inst->request_id : "NULL", @@ -193,7 +192,4 @@ int float_cmp_ex(float a, float b, float ep) return 1; } -int float_cmp(float a, float b) -{ - return float_cmp_ex(a, b, DEFAULT_EPSILON); -} +int float_cmp(float a, float b) { return float_cmp_ex(a, b, DEFAULT_EPSILON); } diff --git a/src/autoscaler/scaler/scaler_obj.c b/src/autoscaler/scaler/scaler_obj.c index c0c55d0d..bd95b19f 100644 --- a/src/autoscaler/scaler/scaler_obj.c +++ b/src/autoscaler/scaler/scaler_obj.c @@ -4,17 +4,17 @@ typedef void (*obj_fields_fini_t)(void*); -#define OBJ_ARRAY_FINI(arr_ptr, field_fini_func, type) \ - do { \ - type* _arr = (type*)(arr_ptr); \ - if (!_arr) { \ - break; \ - } \ - for (int i = 0; i < _arr->num; ++i) { \ - field_fini_func(&(_arr->data[i])); \ - } \ - FREEUP(_arr->data); \ - _arr->num = 0; \ +#define OBJ_ARRAY_FINI(arr_ptr, field_fini_func, type) \ + do { \ + type* _arr = (type*)(arr_ptr); \ + if (!_arr) { \ + break; \ + } \ + for (int i = 0; i < _arr->num; ++i) { \ + field_fini_func(&(_arr->data[i])); \ + } \ + FREEUP(_arr->data); \ + _arr->num = 0; \ } while (0) int provider_init(scaler_provider_t* p, char* name, char* type) @@ -86,7 +86,8 @@ void template_fields_fini(void* p) if (tpl->attrs[i].name) { FREEUP(tpl->attrs[i].name); } - if (tpl->attrs[i].type == SCALER_ATTR_TYPE_STR && tpl->attrs[i].v.str_val) { + if (tpl->attrs[i].type == SCALER_ATTR_TYPE_STR + && tpl->attrs[i].v.str_val) { FREEUP(tpl->attrs[i].v.str_val); } } @@ -124,7 +125,8 @@ int template_cpy(scaler_template_t* tgt, scaler_template_t* src) // Allocate and copy attributes tgt->num_attrs = src->num_attrs; if (src->num_attrs > 0) { - tgt->attrs = (scaler_template_attr_t*)calloc(src->num_attrs, sizeof(scaler_template_attr_t)); + tgt->attrs = (scaler_template_attr_t*)calloc( + src->num_attrs, sizeof(scaler_template_attr_t)); if (!tgt->attrs) { return -ENOMEM; } @@ -158,12 +160,12 @@ int template_cpy(scaler_template_t* tgt, scaler_template_t* src) } scaler_request_t* new_create_request(const char* provider_name, - const char* template_id, - int count, + const char* template_id, int count, const char* request_id) { - scaler_request_t* r = (scaler_request_t*)calloc(1, sizeof(scaler_request_t)); - time_t now = time(NULL); + scaler_request_t* r + = (scaler_request_t*)calloc(1, sizeof(scaler_request_t)); + time_t now = time(NULL); if (!r) { ERROR("Memory allocation failed for scaler_request_t"); @@ -196,10 +198,12 @@ scaler_request_t* new_create_request(const char* provider_name, return r; } -scaler_request_t* new_release_request(const char* request_id, char* provider_name) +scaler_request_t* new_release_request(const char* request_id, + char* provider_name) { - scaler_request_t* r = (scaler_request_t*)calloc(1, sizeof(scaler_request_t)); - time_t now = time(NULL); + scaler_request_t* r + = (scaler_request_t*)calloc(1, sizeof(scaler_request_t)); + time_t now = time(NULL); if (!r) { ERROR("Memory allocation failed for scaler_request_t"); diff --git a/src/autoscaler/scaler/testsuite/test.c b/src/autoscaler/scaler/testsuite/test.c index 97608493..95a4f2d1 100644 --- a/src/autoscaler/scaler/testsuite/test.c +++ b/src/autoscaler/scaler/testsuite/test.c @@ -7,15 +7,9 @@ extern void test_plug_mgr(void); extern void test_scale_misc(void); -void setUp(void) -{ - return; -} +void setUp(void) { return; } -void tearDown(void) -{ - return; -} +void tearDown(void) { return; } int main(int argc, char* argv[]) { diff --git a/src/autoscaler/scaler/testsuite/test_plug_mgr.c b/src/autoscaler/scaler/testsuite/test_plug_mgr.c index b96174d4..f5f41231 100644 --- a/src/autoscaler/scaler/testsuite/test_plug_mgr.c +++ b/src/autoscaler/scaler/testsuite/test_plug_mgr.c @@ -29,16 +29,20 @@ static void global_init(void) scaler_obj_mgr_init(&g_scaler_ctx.requests, request_fini); scaler_obj_mgr_init(&g_scaler_ctx.instances, instance_fini); - scaler_provider_t* prov = (scaler_provider_t*)malloc(sizeof(scaler_provider_t)); + scaler_provider_t* prov + = (scaler_provider_t*)malloc(sizeof(scaler_provider_t)); provider_init(prov, TEST_PROVIDER, TEST_PROVIDER_TYPE); - prov->confPath = strdup("./testsuite"); - prov->scriptPath = strdup("./testsuite"); - prov->scripts[GET_AVAILABLE_TEMPLATES] = strdup(PROV_SCRIPTS[GET_AVAILABLE_TEMPLATES]); - prov->scripts[REQUEST_MACHINES] = strdup(PROV_SCRIPTS[REQUEST_MACHINES]); - prov->scripts[GET_REQUEST_STATUS] = strdup(PROV_SCRIPTS[GET_REQUEST_STATUS]); - prov->scripts[GET_RETURN_REQUEST] = strdup(PROV_SCRIPTS[GET_RETURN_REQUEST]); - prov->scripts[RETURN_MACHINES] = strdup(PROV_SCRIPTS[RETURN_MACHINES]); + prov->confPath = strdup("./testsuite"); + prov->scriptPath = strdup("./testsuite"); + prov->scripts[GET_AVAILABLE_TEMPLATES] + = strdup(PROV_SCRIPTS[GET_AVAILABLE_TEMPLATES]); + prov->scripts[REQUEST_MACHINES] = strdup(PROV_SCRIPTS[REQUEST_MACHINES]); + prov->scripts[GET_REQUEST_STATUS] + = strdup(PROV_SCRIPTS[GET_REQUEST_STATUS]); + prov->scripts[GET_RETURN_REQUEST] + = strdup(PROV_SCRIPTS[GET_RETURN_REQUEST]); + prov->scripts[RETURN_MACHINES] = strdup(PROV_SCRIPTS[RETURN_MACHINES]); provider_add(&g_scaler_ctx.providers, prov); } @@ -73,17 +77,16 @@ static void test_trigger_create(void) int ret = trigger_create(&demand); TEST_ASSERT_EQUAL_INT(0, ret); TEST_ASSERT_EQUAL_INT(1, g_scaler_ctx.requests.array.count); - scaler_request_t* req = request_get(&g_scaler_ctx.requests, TEST_REQUEST_ID_CREATE); + scaler_request_t* req + = request_get(&g_scaler_ctx.requests, TEST_REQUEST_ID_CREATE); TEST_ASSERT_NOT_NULL(req); TEST_ASSERT_EQUAL_STRING(TEST_PROVIDER, req->tpl_id.provider_name); } static void test_trigger_release(void) { - scaler_instance_id_t inst_id = { - .instance_id = TEST_INSTANCE_ID, - .hostname = TEST_HOSTNAME - }; + scaler_instance_id_t inst_id + = { .instance_id = TEST_INSTANCE_ID, .hostname = TEST_HOSTNAME }; instance_id_list_t inst_ids; inst_ids.num = 1; inst_ids.instance_ids = &inst_id; @@ -91,14 +94,16 @@ static void test_trigger_release(void) int ret = trigger_release(TEST_PROVIDER, &inst_ids); TEST_ASSERT_EQUAL_INT(0, ret); TEST_ASSERT_EQUAL_INT(2, g_scaler_ctx.requests.array.count); - scaler_request_t* req = request_get(&g_scaler_ctx.requests, TEST_REQUEST_ID_RELEASE); + scaler_request_t* req + = request_get(&g_scaler_ctx.requests, TEST_REQUEST_ID_RELEASE); TEST_ASSERT_NOT_NULL(req); TEST_ASSERT_EQUAL(SCALER_REQ_TYPE_RET, req->type); } static void test_get_create_request_status(void) { - scaler_request_t* req = new_create_request(TEST_PROVIDER, TEST_TEMPLATE, 1, TEST_REQUEST_ID_CREATE); + scaler_request_t* req = new_create_request(TEST_PROVIDER, TEST_TEMPLATE, 1, + TEST_REQUEST_ID_CREATE); int ret = get_request_status(req); request_fini(req); @@ -106,7 +111,8 @@ static void test_get_create_request_status(void) TEST_ASSERT_EQUAL_INT(0, ret); TEST_ASSERT_EQUAL(REQ_STATUS_COMPLETED, req->status); TEST_ASSERT_EQUAL_INT(2, g_scaler_ctx.instances.array.count); - scaler_instance_t* inst = instance_get(&g_scaler_ctx.instances, TEST_INSTANCE_ID); + scaler_instance_t* inst + = instance_get(&g_scaler_ctx.instances, TEST_INSTANCE_ID); TEST_ASSERT_NOT_NULL(inst); TEST_ASSERT_EQUAL_STRING(TEST_PRIVATE_IP, inst->private_ip); TEST_ASSERT_EQUAL(ALLOC_STATUS_ALLOCATED, inst->alloc_status); @@ -114,14 +120,16 @@ static void test_get_create_request_status(void) static void test_get_release_request_status(void) { - scaler_request_t* req = new_release_request(TEST_REQUEST_ID_RELEASE, TEST_PROVIDER); - int ret = get_request_status(req); + scaler_request_t* req + = new_release_request(TEST_REQUEST_ID_RELEASE, TEST_PROVIDER); + int ret = get_request_status(req); request_fini(req); req = request_get(&g_scaler_ctx.requests, TEST_REQUEST_ID_CREATE); TEST_ASSERT_EQUAL_INT(0, ret); TEST_ASSERT_EQUAL(REQ_STATUS_COMPLETED, req->status); TEST_ASSERT_EQUAL_INT(2, g_scaler_ctx.instances.array.count); - scaler_instance_t* inst = instance_get(&g_scaler_ctx.instances, TEST_INSTANCE_ID); + scaler_instance_t* inst + = instance_get(&g_scaler_ctx.instances, TEST_INSTANCE_ID); TEST_ASSERT_NOT_NULL(inst); TEST_ASSERT_EQUAL_STRING(TEST_PRIVATE_IP, inst->private_ip); TEST_ASSERT_EQUAL(ALLOC_STATUS_DONE, inst->alloc_status); @@ -129,16 +137,19 @@ static void test_get_release_request_status(void) static void test_get_return_request(void) { - instance_id_list_t* inst_id_list = (instance_id_list_t*)malloc(sizeof(instance_id_list_t)); - inst_id_list->num = 1; - inst_id_list->instance_ids = (scaler_instance_id_t*)malloc(sizeof(scaler_instance_id_t)); + instance_id_list_t* inst_id_list + = (instance_id_list_t*)malloc(sizeof(instance_id_list_t)); + inst_id_list->num = 1; + inst_id_list->instance_ids + = (scaler_instance_id_t*)malloc(sizeof(scaler_instance_id_t)); inst_id_list->instance_ids[0].instance_id = strdup(TEST_INSTANCE_ID); inst_id_list->instance_ids[0].hostname = strdup(TEST_HOSTNAME); int ret = get_return_request(TEST_PROVIDER, inst_id_list); instance_id_list_fini(inst_id_list); TEST_ASSERT_EQUAL_INT(0, ret); - scaler_instance_t* inst = instance_get(&g_scaler_ctx.instances, TEST_INSTANCE_ID); + scaler_instance_t* inst + = instance_get(&g_scaler_ctx.instances, TEST_INSTANCE_ID); TEST_ASSERT_NOT_NULL(inst); TEST_ASSERT_EQUAL_STRING(TEST_HOSTNAME, inst->hostname); TEST_ASSERT_EQUAL_STRING(TEST_INSTANCE_ID, inst->instance_id); diff --git a/src/autoscaler/scaler/testsuite/test_scale_misc.c b/src/autoscaler/scaler/testsuite/test_scale_misc.c index 6c557c55..52791b75 100644 --- a/src/autoscaler/scaler/testsuite/test_scale_misc.c +++ b/src/autoscaler/scaler/testsuite/test_scale_misc.c @@ -9,28 +9,29 @@ #include "unity.h" #ifndef TEST_ASSERT_NOT_EQUAL_STRING -#define TEST_ASSERT_NOT_EQUAL_STRING(a, b) \ - do { \ - if (strcmp(a, b) == 0) { \ - TEST_FAIL_MESSAGE(#a " and " #b " are equal"); \ - } \ +#define TEST_ASSERT_NOT_EQUAL_STRING(a, b) \ + do { \ + if (strcmp(a, b) == 0) { \ + TEST_FAIL_MESSAGE(#a " and " #b " are equal"); \ + } \ } while (0) #endif -static void -test_my_snprintf(void) +static void test_my_snprintf(void) { char buffer[50]; int result; // Test normal formatting - result = my_snprintf(buffer, sizeof(buffer), "Hello %s, number: %d", "World", 42); + result = my_snprintf(buffer, sizeof(buffer), "Hello %s, number: %d", + "World", 42); TEST_ASSERT_EQUAL_STRING("Hello World, number: 42", buffer); TEST_ASSERT_EQUAL_INT(strlen("Hello World, number: 42"), result); // Test buffer size limit char small_buffer[10]; - result = my_snprintf(small_buffer, sizeof(small_buffer), "This is a very long string"); + result = my_snprintf(small_buffer, sizeof(small_buffer), + "This is a very long string"); TEST_ASSERT_EQUAL_INT(strlen("This is a very long string"), result); // Test empty format @@ -90,25 +91,17 @@ static void test_my_strcmp(void) static void test_cmp_template_id(void) { - scaler_template_id_t id1 = { - .provider_name = "aws", - .template_id = "t2.micro" - }; + scaler_template_id_t id1 + = { .provider_name = "aws", .template_id = "t2.micro" }; - scaler_template_id_t id2 = { - .provider_name = "aws", - .template_id = "t2.micro" - }; + scaler_template_id_t id2 + = { .provider_name = "aws", .template_id = "t2.micro" }; - scaler_template_id_t id3 = { - .provider_name = "azure", - .template_id = "t2.micro" - }; + scaler_template_id_t id3 + = { .provider_name = "azure", .template_id = "t2.micro" }; - scaler_template_id_t id4 = { - .provider_name = "aws", - .template_id = "t2.small" - }; + scaler_template_id_t id4 + = { .provider_name = "aws", .template_id = "t2.small" }; // Test both NULL TEST_ASSERT_TRUE(cmp_template_id(NULL, NULL)); @@ -127,9 +120,12 @@ static void test_cmp_template_id(void) TEST_ASSERT_FALSE(cmp_template_id(&id1, &id4)); // Test with NULL fields - scaler_template_id_t id_null1 = { .provider_name = NULL, .template_id = "test" }; - scaler_template_id_t id_null2 = { .provider_name = "test", .template_id = NULL }; - scaler_template_id_t id_null3 = { .provider_name = NULL, .template_id = NULL }; + scaler_template_id_t id_null1 + = { .provider_name = NULL, .template_id = "test" }; + scaler_template_id_t id_null2 + = { .provider_name = "test", .template_id = NULL }; + scaler_template_id_t id_null3 + = { .provider_name = NULL, .template_id = NULL }; TEST_ASSERT_FALSE(cmp_template_id(&id1, &id_null1)); TEST_ASSERT_FALSE(cmp_template_id(&id1, &id_null2)); diff --git a/src/controller/cluster.c b/src/controller/cluster.c index 87275800..0b198ae3 100644 --- a/src/controller/cluster.c +++ b/src/controller/cluster.c @@ -5,6 +5,7 @@ #include #include "cluster.h" +#include "dataq.h" #include "htab.h" #include "mylock.h" #include "request.h" @@ -188,6 +189,12 @@ static int myreq_handler(vtask_t* task) case OPCode_runtasks: rc = do_runtasks(c, ep, &request); break; + case OPCode_setpreason: + rc = do_set_preason(c, ep, &request); + break; + case OPCode_clrpreason: + rc = do_clr_preason(c, ep, &request); + break; default: rc = -E_XCE_BAD_OPCODE; ERROR("Unknown opcode: %d", request.opcode); @@ -447,6 +454,11 @@ static int xce_install_hooks(cluster_t* c) return 0; } +static int run_delay_tasks(delay_task_entry_fn_t entry, void* arg) +{ + return entry(arg); +} + int cluster_init(cluster_t* c) { int i, rc; @@ -524,12 +536,90 @@ int cluster_init(cluster_t* c) vtaskmgr_init(&(c->vtaskmgr), c->myconf->cluster_config->vcpus); + delaymgr_init(&(c->delaymgr), 2, run_delay_tasks); + xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_CLUSTER, HOOK_AFTER_EVENT, HOOK_DETAIL_CLUSTER_INIT, c); return 0; } +static int make_heartbeat(cluster_t* c, endpoint_t* ep) +{ + return xce_nop_req(ep); +} + +static int check_worker_timeout_(void* arg1, void* arg2) +{ + cluster_t* c = (cluster_t*)arg1; + worker_t* w = (worker_t*)arg2; + endpoint_t* ep; + time_t now = time(NULL); + int timeout; + cluster_config_t* config; + + if (w == NULL) { + return 0; + } + + if (c == NULL) { + return -EINVAL; + } + + ep = w->ep; + if (ep == NULL) { + return 0; + } + + config = c->myconf->cluster_config; + timeout = config->heartbeat_interval * config->inactive_heartbeat_count; + + if (now - ep->ts_last_in > timeout) { + TRACE("%s worker %s becomes UNAVAIL as heartbeat timeout", __func__, + w->host.name); + + mark_worker_unavail(c, w); + + return 0; + } else { + if (IS_WORKER_UNAVAIL(w->host.status)) { + mark_worker_back(c, w); + } + } + + if (now - ep->ts_last_out >= config->heartbeat_interval) { + if (DATAQ_EMPTY(&(ep->outq))) { + int rc = 0; + + TRACE("%s send heartbeat to %s", __func__, w->host.name); + + rc = make_heartbeat(c, ep); + + if (rc < 0) { + ERROR("%s make_heartbeat to %s failed", __func__, w->host.name); + return rc; + } + } + } + + return 0; +} + +static int check_workers_timeout(void* args) +{ + xlist_t lst; + int rc; + cluster_t* c = (cluster_t*)args; + + xlist_init(&lst, 3); + xlist_store(&lst, check_worker_timeout_); + rc = workermgr_iter(&(c->workermgr), &lst, c); + + xlist_fini(&lst); + + return rc; +} + int cluster_start(cluster_t* c) { int rc; @@ -546,6 +636,10 @@ int cluster_start(cluster_t* c) iomgr_start(&(c->iomgr)); vtaskmgr_start(&(c->vtaskmgr)); + delaymgr_start(&(c->delaymgr)); + + delay_task(&(c->delaymgr), c->myconf->cluster_config->heartbeat_interval, + -1, (delay_task_entry_fn_t)check_workers_timeout, (void*)c); c->running = 1; return 0; @@ -565,6 +659,9 @@ int cluster_run(cluster_t* c) xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_CLUSTER, HOOK_BEFORE_EVENT, HOOK_DETAIL_CLUSTER_SHUTDOWN, c); c->running = 0; + + delaymgr_stop(&(c->delaymgr)); + iomgr_unmonitor(&(c->iomgr), &(c->ep)); iomgr_stop(&(c->iomgr)); @@ -573,6 +670,7 @@ int cluster_run(cluster_t* c) void cluster_fini(cluster_t* c) { + delaymgr_fini(&(c->delaymgr)); vtaskmgr_fini(&(c->vtaskmgr)); endpoint_fini(&(c->ep)); diff --git a/src/controller/include/cluster.h b/src/controller/include/cluster.h index 76b6f9d9..da1fc5af 100644 --- a/src/controller/include/cluster.h +++ b/src/controller/include/cluster.h @@ -98,6 +98,8 @@ extern int handle_reply(endpoint_t*, struct Request*, cluster_t*); extern int handle_jobdispatch(cluster_t*, endpoint_t*, xce_request_t*); extern int handle_jobsubmit(cluster_t*, endpoint_t*, xce_request_t*); extern int handle_worker_leave(cluster_t*, int); +extern int mark_worker_unavail(cluster_t*, worker_t*); +extern int mark_worker_back(cluster_t*, worker_t*); extern int handle_scheduler_leave(cluster_t*); extern int handle_clusterconf(cluster_t*, endpoint_t*, xce_request_t*); @@ -151,5 +153,7 @@ extern int xce_controller_res_init(cluster_t*); extern int map_resource_to_host(char*, cluster_t*); extern int release_resource_from_worker(char*, cluster_t*); extern int collect_resval_with_idx(zarray_t*, int*, xce_val_with_idx_t***); +extern int do_set_preason(cluster_t*, endpoint_t*, xce_request_t*); +extern int do_clr_preason(cluster_t*, endpoint_t*, xce_request_t*); #endif diff --git a/src/controller/job.c b/src/controller/job.c index da6bf5ac..6c757201 100644 --- a/src/controller/job.c +++ b/src/controller/job.c @@ -3,6 +3,7 @@ #include #include "cluster.h" +#include "preason.h" #include "xce/cluster_def.h" #include "xce/errnox.h" #include "xce/types.h" @@ -457,7 +458,10 @@ int handle_jobsubmit(cluster_t* c, endpoint_t* ep, xce_request_t* request) /* Run hooks to tell scheduler */ j = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_READ); if (j == NULL) { - ERROR("%s fail to lock job %lu", __func__, jobid); + char idstring[128]; + + fmt_jobid(idstring, sizeof(idstring) - 1, jobid); + ERROR("%s fail to lock job %s (%lu)", __func__, idstring, jobid); } else { xlist_init(&jlist, 2); if (xlist_store(&jlist, j) >= 0) { @@ -777,6 +781,9 @@ int update_job_rusages(cluster_t* c, endpoint_t* ep, xce_request_t* request) TRACE("%s trying to update job(%ld) rusage", __func__, rusages->jobid); + xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT, + HOOK_DETAIL_JOB_RUSAGE_UPDATE, c, rusages); + j = jobmgr_lockjob(&(c->jobmgr), rusages->jobid, OBJREF_UPDATE); if (j == NULL) { PERROR("jobmgr_lockjob"); @@ -815,5 +822,106 @@ int update_job_rusages(cluster_t* c, endpoint_t* ep, xce_request_t* request) HOOK_DETAIL_JOB_UPDATE, c, j); obj_deref(j); + xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, + HOOK_DETAIL_JOB_RUSAGE_UPDATE, c, rusages); + + return 0; +} + +static int set_preason(cluster_t* c, xce_pendreason_t* reasons) +{ + job_t* j; + xce_job_t* job; + + TRACE("%s jobid %ld", __func__, reasons->jobid); + + j = jobmgr_lockjob(&(c->jobmgr), reasons->jobid, OBJREF_UPDATE); + if (j == NULL) { + PERROR("jobmgr_lockjob"); + return -ENOENT; + } + + xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT, + HOOK_DETAIL_SETPREASON, c, j); + job = &(j->job); + if (job->pend_reasons != NULL) { + erase_job_preason(job->pend_reasons); + FREEUP(job->pend_reasons); + } + + MOVE_PTR(job->pend_reasons, reasons->reasons); + xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, + HOOK_DETAIL_SETPREASON, c, j); + obj_deref(j); + + return 0; +} + +int do_set_preason(cluster_t* c, endpoint_t* ep, xce_request_t* request) +{ + int i; + xce_preason_set_t* reasons = request->body.preasonmsg; + xce_reply_t reply; + + init_reply_by_req(&reply, request); + reply.status = 0; + reply.data_which = RequestReply_data_nodata; + + xce_reply(ep, c->myhost, &reply); + + for (i = 0; i < reasons->nreasons; i++) { + int rc = 0; + + if (reasons->reasons[i] == NULL) { + continue; + } + + rc = set_preason(c, reasons->reasons[i]); + if (rc < 0) { + PERROR("set_preason"); + } + } + + return 0; +} + +int do_clr_preason(cluster_t* c, endpoint_t* ep, xce_request_t* request) +{ + int i; + xce_reply_t reply; + xce_clear_reason_t* clreason = request->body.clrpreasonmsg; + + init_reply_by_req(&reply, request); + reply.status = 0; + reply.data_which = RequestReply_data_nodata; + + xce_reply(ep, c->myhost, &reply); + + for (i = 0; i < clreason->njobs; i++) { + uint64_t jobid = clreason->jobs[i]; + job_t* j; + xce_job_t* job; + + j = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_UPDATE); + if (j == NULL) { + PERROR("jobmgr_lockjob"); + continue; + } + + xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT, + HOOK_DETAIL_CLRPREASON, c, j); + + job = &(j->job); + if (job->pend_reasons != NULL) { + erase_job_preason(job->pend_reasons); + FREEUP(job->pend_reasons); + } + + xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT, + HOOK_DETAIL_CLRPREASON, c, j); + + obj_deref(j); + } + return 0; } diff --git a/src/controller/opget.c b/src/controller/opget.c index 1ef089ab..90e928c1 100644 --- a/src/controller/opget.c +++ b/src/controller/opget.c @@ -143,10 +143,28 @@ static int do_reply_joblist(cluster_t* c, endpoint_t* ep, xce_data_t getdata = { .table = request->body.getmsg->table, .data = { .jobs = &jobdata } }; int rc = 0; + xce_match_t* match = request->body.getmsg->condition; init_reply_by_req(&reply, request); - collect_jobs(c, (collect_check_fn_t)check_job_, &(jobdata.n_total), - &(jobdata.jobs)); + + if (match->type == MatchCondition_condition_matchjid) { + job_t* j; + + jobdata.n_total = 0; + jobdata.jobs = NULL; + + j = jobmgr_lockjob(&(c->jobmgr), match->condition.jobid, OBJREF_READ); + if (j != NULL) { + jobdata.jobs = (xce_job_t**)calloc(1, sizeof(xce_job_t*)); + if (jobdata.jobs != NULL) { + jobdata.jobs[0] = &(j->job); + jobdata.n_total = 1; + } + } + } else { + collect_jobs(c, (collect_check_fn_t)check_job_, &(jobdata.n_total), + &(jobdata.jobs)); + } reply.status = 0; reply.data_which = RequestReply_data_getdata; diff --git a/src/controller/sched.c b/src/controller/sched.c index a5866bed..e6eeaa7b 100644 --- a/src/controller/sched.c +++ b/src/controller/sched.c @@ -314,6 +314,37 @@ static int sched_job_update_fun(int type, int situation, void* data, va_list ap) return rc; } +static int sched_job_rusage_update_fun(int type, int situation, void* data, + va_list ap) +{ + cluster_t* c = va_arg(ap, cluster_t*); + xce_job_rusages_t* rusages = va_arg(ap, xce_job_rusages_t*); + int rc = 0; + /* clang-format off */ + xce_notify_data_t xdata = { + .event = NotifyData_data_schedJobRusages, + .data = { + .sched_job_rusages = rusages + } + }; + /* clang-format on */ + + mylock_lock(&(c->sched.lock)); + if (c->sched.ep != NULL) { + switch (situation) { + case HOOK_AFTER_EVENT: + rc = notify(c, c->sched.ep, &xdata); + break; + default: + break; + } + } + + mylock_unlock(&(c->sched.lock)); + + return rc; +} + static int sched_job_hook_fun(int type, int situation, int detail, void* data, va_list ap) { @@ -328,6 +359,8 @@ static int sched_job_hook_fun(int type, int situation, int detail, void* data, return sched_job_update_fun(type, situation, data, ap); case HOOK_DETAIL_JOB_FAIL: return sched_job_fail_fun(type, situation, data, ap); + case HOOK_DETAIL_JOB_RUSAGE_UPDATE: + return sched_job_rusage_update_fun(type, situation, data, ap); } return 0; diff --git a/src/controller/task.c b/src/controller/task.c index 7f089634..bdcef8a8 100644 --- a/src/controller/task.c +++ b/src/controller/task.c @@ -51,6 +51,8 @@ static int check_task_hosts(cluster_t* c, xce_taskrun_t* runmsg) obj_deref(w); return -ENOENT; } + + obj_deref(w); } return 0; diff --git a/src/controller/testsuite/test.c b/src/controller/testsuite/test.c index b2769033..a369cec5 100644 --- a/src/controller/testsuite/test.c +++ b/src/controller/testsuite/test.c @@ -17,7 +17,9 @@ #include "client.h" #include "controller.capnp.h" +#include "delay.h" #include "idgen.h" +#include "preason.h" #include "xce/types.h" typedef int (*cmd_func_t)(int, char*[]); @@ -73,6 +75,7 @@ typedef struct { #define MAX_TASK_HOSTS (5) #define MAX_RUN_TASKS (50) #define RUSAGE_UPDATE_INTERVAL (60) +#define MAX_LINE_LEN (40960) #define S2J_JSON_SET_var_string_ELEMENT(to_json, from_struct, _element) \ if (((from_struct) != NULL) && (((from_struct)->_element) != NULL)) { \ @@ -109,14 +112,12 @@ extern int cmd_jobkill(int, char*[]); extern int cmd_jobckpt(int, char*[]); extern int cmd_qctrl(int, char*[]); extern int cmd_runtasks(int, char*[]); +extern int cmd_setpend(int, char*[]); +extern int cmd_jobinfo(int, char*[]); +static int worker_house_keeping(void*); -static void handler(int, siginfo_t*, void*); - -#define CLOCKID CLOCK_REALTIME -#define SIG SIGRTMIN - -static timer_t timerid; static idgen_ctx_t idctx; +delaymgr_t delaymgr; static setting_t settings[] = { { (char*)"host", (char*)NULL, (char*)"localhost" }, @@ -168,6 +169,7 @@ static command_t client_commands[] = { { "jobsub", cmd_jobsub, "submit a job" }, { "jobkill", cmd_jobkill, "signal a job" }, { "jobckpt", cmd_jobckpt, "checkpoint a job" }, + { "jobinfo", cmd_jobinfo, "job information" }, { "resval", cmd_resval, "get value of res" }, { "taskrun", cmd_runtasks, "run tasks" }, { "help", cmd_help, "Display this text" }, @@ -186,6 +188,7 @@ static command_t scheduler_commands[] = { { "quit", cmd_quit, "Quit xce test" }, { "join", cmd_join_sched, "Join the cluster" }, { "getconf", cmd_getconf, "Get conf of cluster" }, + { "setpend", cmd_setpend, "set job pend reason" }, { (char*)NULL, (cmd_func_t)NULL, (char*)NULL } }; @@ -278,7 +281,7 @@ int leave_ctx(command_ctx_t* ctx) } if (ctx == &workerctx) { - timer_delete(timerid); + delaymgr_fini(&delaymgr); } xcecli_wakeup(active_ctx->ctx, (int)-EINTR); @@ -392,10 +395,7 @@ int cmd_up(int argc, char* argv[]) int cmd_worker(int argc, char* argv[]) { - int rc; - struct sigevent sev; - struct sigaction sa; - struct itimerspec its; + int rc; if ((rc = enter_ctx(&workerctx)) < 0) { return rc; @@ -403,22 +403,10 @@ int cmd_worker(int argc, char* argv[]) active_ctx = &workerctx; - sa.sa_flags = SA_SIGINFO; - sa.sa_sigaction = handler; - sigemptyset(&(sa.sa_mask)); - sigaction(SIG, &sa, NULL); - - sev.sigev_notify = SIGEV_SIGNAL; - sev.sigev_signo = SIG; - sev.sigev_value.sival_ptr = &timerid; - - timer_create(CLOCKID, &sev, &timerid); - - memset(&its, 0x0, sizeof(its)); - its.it_value.tv_sec = 1; - - timer_settime(timerid, 0, &its, NULL); - + delaymgr_init(&delaymgr, 2, NULL); + delaymgr_start(&delaymgr); + delay_task(&delaymgr, 5, -1, (delay_task_entry_fn_t)worker_house_keeping, + &workerctx); return 0; } @@ -631,11 +619,94 @@ int cmd_joblist(int argc, char* argv[]) jobs[i]->submit->start_info->cmdline); } + for (i = 0; i < rc; i++) { + free_Job_ptr(&(jobs[i])); + } + free(jobs); return 0; } +static void print_jobinfo(xce_job_t* job) +{ + char idstring[128]; + + if (job == NULL) { + return; + } + + fmt_jobid(idstring, sizeof(idstring) - 1, job->jobid); + printf("JobID: %s\n", idstring); + printf("state: %s\n", job_state_str(job->state)); + printf("Queue: %s\n", job->submit->queue); + printf("User: %s\n", job->submit->user); + printf("Slots: %d,%d\n", job->submit->mintasks, job->submit->maxtasks); + printf("Command: %s\n", job->submit->start_info->cmdline); + + if (job->pend_reasons != NULL) { + printf("Pend Reason:\n"); + if (job->pend_reasons->main_reason != 0) { + printf("\tmain Reason: %d\n", job->pend_reasons->main_reason); + } else { + if (job->pend_reasons->nhost_reasons > 0) { + int i; + + printf("\tHost Reasons:\n"); + for (i = 0; i < job->pend_reasons->nhost_reasons; i++) { + printf("\t\t(%s, %d)\n", + job->pend_reasons->host_based_reasons[i]->host, + job->pend_reasons->host_based_reasons[i]->reason); + } + } + + if (job->pend_reasons->nres_reasons > 0) { + int i; + + printf("\tResource Reasons:\n"); + for (i = 0; i < job->pend_reasons->nres_reasons; i++) { + printf("\t\t(%s, %s, %d)\n", + job->pend_reasons->res_based_reasons[i]->resource, + job->pend_reasons->res_based_reasons[i]->host, + job->pend_reasons->res_based_reasons[i]->reason); + } + } + } + } +} + +int cmd_jobinfo(int argc, char* argv[]) +{ + int opt; + + optind = 1; + + while ((opt = getopt(argc, argv, "j:")) != -1) { + uint64_t jobid; + xce_job_t* job; + int rc = 0; + + switch (opt) { + case 'j': + parse_jobid(&jobid, optarg); + + if (jobid > 0) { + rc = xcecli_jobinfo(active_ctx->ctx, jobid, &job); + if (rc < 0) { + perror("xcecli_jobinfo"); + continue; + } + + print_jobinfo(job); + free_Job_ptr(&job); + } + + break; + } + } + return 0; +} + static int fill_job_alloc(xce_job_alloc_t* jalloc, int nhosts, host_task_t* hosts) { @@ -1256,6 +1327,91 @@ int cmd_runtasks(int argc, char* argv[]) return rc; } +static int add_pendreason(xce_job_preason_t* preason, char* s) +{ + char s1[MAX_LINE_LEN]; + char* pcode = NULL; + char* phost = NULL; + char* pres = NULL; + char* p; + int code; + + if ((preason == NULL) || (s == NULL)) { + fprintf(stderr, "bad parameter\n"); + return -1; + } + + if ((p = strrchr(s, (int)':')) == NULL) { + pcode = s; + } else { + pcode = p + 1; + memset(s1, 0x0, sizeof(s1)); + strncpy(s1, s, p - s); + + if ((p = strrchr(s1, (int)':')) == NULL) { + phost = s1; + } else { + *p = 0x0; + phost = p + 1; + pres = s1; + } + } + + code = atoi(pcode); + + return add_job_preason(preason, code, phost, pres); +} + +int cmd_setpend(int argc, char* argv[]) +{ + int opt; + xce_job_preason_t jreasons; + xce_pendreason_t preason = { .reasons = &jreasons }; + xce_preason_set_t reasons + = { .nreasons = 1, .reasons = (xce_pendreason_t*[]) { &preason } }; + int rc = 0; + + optind = 1; + + while ((opt = getopt(argc, argv, "j:p:")) != -1) { + int rc = 0; + switch (opt) { + case 'j': + parse_jobid(&(preason.jobid), optarg); + break; + case 'p': + rc = add_pendreason(&jreasons, optarg); + if (rc < 0) { + perror("add_pendreason"); + return rc; + } + break; + } + } + + if (preason.jobid == 0) { + fprintf(stderr, "no or incorrect jobid specified\n"); + return -1; + } + + if ((jreasons.main_reason == 0) && (jreasons.nhost_reasons == 0) + && (jreasons.nres_reasons == 0)) { + fprintf(stderr, "no or incorrect pend reason set\n"); + return -1; + } + + rc = xcecli_set_pend_reason(active_ctx->ctx, &reasons); + if (rc < 0) { + perror("xcecli_set_pend_reason"); + erase_job_preason(&jreasons); + return rc; + } + + printf("ok\n"); + + return 0; +} + int cmd_join_worker(int argc, char* argv[]) { int rc; @@ -2263,18 +2419,24 @@ static void check_run_tasks() } } -static void handler(int sig, siginfo_t* si, void* uc) +#define HEARTBEAT_INTERVAL (60) + +static int worker_house_keeping(void* arg) { - struct itimerspec its; + command_ctx_t* ctx = (command_ctx_t*)arg; + int rc = 0; + + xce_use(ctx); check_run_jobs(); check_run_tasks(); - memset(&its, 0x0, sizeof(its)); + rc = xcecli_heartbeat(ctx->ctx, HEARTBEAT_INTERVAL); + if (rc < 0) { + perror("xcecli_heartbeat"); + } - its.it_value.tv_sec = 1; - - timer_settime(timerid, 0, &its, NULL); + return rc; } int main(int argc, char* argv[]) diff --git a/src/controller/worker.c b/src/controller/worker.c index 31f38d7c..d7d87909 100644 --- a/src/controller/worker.c +++ b/src/controller/worker.c @@ -476,6 +476,28 @@ int get_resval(cluster_t* c, char* resname, char* host, xce_val_t* val) return 1; } +int mark_worker_unavail(cluster_t* c, worker_t* w) +{ + xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_WORKER, HOOK_BEFORE_EVENT, + HOOK_DETAIL_WORKER_DEAD, c, w); + w->host.status |= XCE_WORKER_UNAVAIL; + + xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_WORKER, HOOK_AFTER_EVENT, + HOOK_DETAIL_WORKER_DEAD, c, w); + return 0; +} + +int mark_worker_back(cluster_t* c, worker_t* w) +{ + xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_WORKER, HOOK_BEFORE_EVENT, + HOOK_DETAIL_WORKER_UPDATE, c, w); + w->host.status &= ~XCE_WORKER_UNAVAIL; + + xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_WORKER, HOOK_AFTER_EVENT, + HOOK_DETAIL_WORKER_UPDATE, c, w); + return 0; +} + int handle_worker_leave(cluster_t* c, int idx) { worker_t* w = NULL; @@ -499,13 +521,8 @@ int handle_worker_leave(cluster_t* c, int idx) return -errno; } - xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_WORKER, HOOK_BEFORE_EVENT, - HOOK_DETAIL_WORKER_DEAD, c, w); - w->host.status |= XCE_WORKER_UNAVAIL; - w->ep = NULL; + mark_worker_unavail(c, w); - xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_WORKER, HOOK_AFTER_EVENT, - HOOK_DETAIL_WORKER_DEAD, c, w); obj_deref(w); return 0; } @@ -539,7 +556,6 @@ int worker_ctrl(cluster_t* c, xce_hostctrl_t* ctrl) } w->host.status |= XCE_WORKER_CLOSED; break; - break; default: rc = -1; break; diff --git a/src/controller/xceobjs.c b/src/controller/xceobjs.c index 3a576d65..867117b6 100644 --- a/src/controller/xceobjs.c +++ b/src/controller/xceobjs.c @@ -5,6 +5,7 @@ #include "cluster.h" #include "objmgr.h" +#include "preason.h" #include "utils.h" #include "xce/cluster_def.h" #include "xstring.h" @@ -296,6 +297,11 @@ void job_obj_killer(job_t* job) FREEUP(job->job.run_usage); } + if (job->job.pend_reasons != NULL) { + erase_job_preason(job->job.pend_reasons); + FREEUP(job->job.pend_reasons); + } + FREEUP(job->job.exit_info); memset(obj, 0x0, sizeof(*obj)); diff --git a/src/include/client.h b/src/include/client.h index 5b301a32..89c8fe5e 100644 --- a/src/include/client.h +++ b/src/include/client.h @@ -30,6 +30,8 @@ extern int xcecli_start(xcecli_ctx_t); extern int xcecli_stop(xcecli_ctx_t); extern int xcecli_wakeup(xcecli_ctx_t, int); extern int xcecli_hello(xcecli_ctx_t, char*); +extern int xcecli_nop(xcecli_ctx_t); +extern int xcecli_heartbeat(xcecli_ctx_t, int); extern int xcecli_shutdown(xcecli_ctx_t); extern int xcecli_reslist(xcecli_ctx_t, xce_resource_t***); extern int xcecli_join(xcecli_ctx_t, int, hostattrs_t*); @@ -60,6 +62,9 @@ extern endpoint_t* xcecli_get_my_endpoint(xcecli_ctx_t); extern xce_peerinfo_t* xcecli_get_myinfo(xcecli_ctx_t); extern void xcecli_block_tasks(xcecli_ctx_t); extern void xcecli_unblock_tasks(xcecli_ctx_t); +extern int xcecli_set_pend_reason(xcecli_ctx_t, xce_preason_set_t*); +extern int xcecli_clear_reason(xcecli_ctx_t, int, uint64_t*); +extern int xcecli_jobinfo(xcecli_ctx_t, uint64_t, xce_job_t**); #define xcecli_free_ress(nres, ress) xce_free_ress(nres, ress) #define xcecli_free_workers(nworkers, workers) \ diff --git a/src/include/clusterconfig.h b/src/include/clusterconfig.h index 5ae899de..13af38d3 100644 --- a/src/include/clusterconfig.h +++ b/src/include/clusterconfig.h @@ -28,6 +28,14 @@ typedef struct { int port; } master_candidate_t; +typedef struct { + char* name; + char* prov_type; + char* conf_path; + char* script_path; + char* script_option; +} scaler_config_t; + typedef struct { char* clustername; int port; @@ -53,6 +61,8 @@ typedef struct { qmgr_config_t* qmgr; resmgr_config_t* resmgr; xstrings_config_t* xstrings; + int nhproviders; + scaler_config_t** hproviders; } cluster_config_t; #endif diff --git a/src/include/delay.h b/src/include/delay.h new file mode 100644 index 00000000..07ec4e85 --- /dev/null +++ b/src/include/delay.h @@ -0,0 +1,37 @@ +#if !defined(_DELAY_H_) + +#define _DELAY_H_ 1 + +#include +#include +#include +#include + +#include "ordq.h" + +typedef int (*delay_task_entry_fn_t)(void*); +typedef int (*task_run_fn_t)(delay_task_entry_fn_t, void*); + +typedef struct { + int run_period; + int runcount; + delay_task_entry_fn_t entry; + void* args; +} delay_task_t; + +typedef struct { + pthread_t worker; + int running; + int precision; + task_run_fn_t runner_fn; + timer_t timerid; + ordq_t tasks; +} delaymgr_t; + +extern int delaymgr_init(delaymgr_t*, int, task_run_fn_t); +extern void delaymgr_stop(delaymgr_t*); +extern void delaymgr_fini(delaymgr_t*); +extern int delaymgr_start(delaymgr_t*); +extern int delay_task(delaymgr_t*, int, int, delay_task_entry_fn_t, void*); + +#endif diff --git a/src/include/netio.h b/src/include/netio.h index a56e142d..6d84a332 100644 --- a/src/include/netio.h +++ b/src/include/netio.h @@ -79,6 +79,8 @@ typedef struct { xce_peerinfo_t* peer; msg_handler_t msg_handler; err_handler_t err_handler; + time_t ts_last_in; + time_t ts_last_out; } endpoint_t; #define INVOKE_MSG_HANDLER(ep) \ diff --git a/src/include/ordq.h b/src/include/ordq.h new file mode 100644 index 00000000..f064de11 --- /dev/null +++ b/src/include/ordq.h @@ -0,0 +1,35 @@ +#if !defined(_ORDQ_H_) + +#define _ORDQ_H_ 1 + +#include + +#include "mylock.h" + +typedef void* ordkey_t; + +typedef struct _ordq_item { + struct _ordq_item* next; + ordkey_t key; + void* data; +} ordq_item_t; + +typedef int (*ordq_cmp_fn_t)(ordkey_t, ordkey_t); +typedef void (*ordq_data_killer_t)(void*); + +typedef struct { + mylock_t lock; + ordq_item_t* head; + ordq_data_killer_t killer; + ordq_cmp_fn_t cmp; +} ordq_t; + +#define mk_ordkey(k) (ordkey_t)((long)(k)) + +extern int ordq_init(ordq_t*, ordq_data_killer_t, ordq_cmp_fn_t); +extern void ordq_fini(ordq_t* q); +extern int ordq_put(ordq_t*, ordkey_t, void*); +extern ordq_item_t* ordq_get(ordq_t*); +extern ordq_item_t* ordq_peek(ordq_t*); + +#endif diff --git a/src/include/preason.h b/src/include/preason.h new file mode 100644 index 00000000..296ea048 --- /dev/null +++ b/src/include/preason.h @@ -0,0 +1,10 @@ +#if !defined(_PREASON_H_) + +#define _PREASON_H_ 1 + +#include "xce/types.h" + +extern void erase_job_preason(xce_job_preason_t*); +extern int add_job_preason(xce_job_preason_t*, int, char*, char*); + +#endif diff --git a/src/include/request.h b/src/include/request.h index aa784a7b..f2ad3bc3 100644 --- a/src/include/request.h +++ b/src/include/request.h @@ -55,5 +55,7 @@ extern int xce_update_req(endpoint_t*, xce_update_data_t*); extern int xce_jobctrl_req(endpoint_t*, xce_jobctrl_t*); extern int xce_queuectrl_req(endpoint_t*, xce_queuectrl_t*); extern int xce_runtask_req(endpoint_t*, xce_taskrun_t*); +extern int xce_set_pend_reason_req(endpoint_t*, xce_preason_set_t*); +extern int xce_clear_reason_req(endpoint_t*, xce_clear_reason_t*); #endif diff --git a/src/include/typemap.h b/src/include/typemap.h index cba95bab..c9e77bd8 100644 --- a/src/include/typemap.h +++ b/src/include/typemap.h @@ -141,6 +141,7 @@ typedef struct { xce_task_failinfo_t* taskfailinfo; xce_task_finish_t* taskfinish; uint64_t pdone_job; + xce_job_rusages_t* sched_job_rusages; } data; } xce_notify_data_t; @@ -186,6 +187,8 @@ typedef struct { xce_jobctrl_t* jobctrlmsg; xce_queuectrl_t* queuectrlmsg; xce_taskrun_t* runtaskmsg; + xce_preason_set_t* preasonmsg; + xce_clear_reason_t* clrpreasonmsg; } body; } xce_request_t; diff --git a/src/include/utils.h b/src/include/utils.h index f1c0351f..c74a6994 100644 --- a/src/include/utils.h +++ b/src/include/utils.h @@ -130,5 +130,7 @@ extern int fmt_jobid(char*, size_t, uint64_t); extern int parse_jobid(uint64_t*, char*); extern uint64_t array_jobid(uint64_t); extern uint64_t array_idx(uint64_t); +extern void unblock_sig(int); +extern void block_sig(int); #endif diff --git a/src/include/xce/cluster_def.h b/src/include/xce/cluster_def.h index d46121d5..96b50be5 100644 --- a/src/include/xce/cluster_def.h +++ b/src/include/xce/cluster_def.h @@ -6,6 +6,7 @@ #include #include +#include "delay.h" #include "hook.h" #include "htab.h" #include "kvmgr.h" @@ -39,6 +40,8 @@ enum { HOOK_DETAIL_JOB_FAIL, HOOK_DETAIL_JOB_UPDATE, HOOK_DETAIL_JOB_CKPT, + HOOK_DETAIL_SETPREASON, + HOOK_DETAIL_CLRPREASON, HOOK_DETAIL_WORKER_JOIN, HOOK_DETAIL_WORKER_DEAD, HOOK_DETAIL_WORKER_UPDATE, @@ -50,7 +53,8 @@ enum { HOOK_DETAIL_RESOURCE_LOC_UPDATE, HOOK_DETAIL_QUEUE_UPDATE, HOOK_DETAIL_USERGROUP_UPDATE, - HOOK_DETAIL_HOSTGROUP_UPDATE + HOOK_DETAIL_HOSTGROUP_UPDATE, + HOOK_DETAIL_JOB_RUSAGE_UPDATE }; #define XCE_NS_CONFIG (0) @@ -125,6 +129,7 @@ typedef struct { int myidx; int master; int ncontrollers; + delaymgr_t delaymgr; scheduler_t sched; jobmgr_t jobmgr; resmgr_t resmgr; diff --git a/src/include/xce/types.h b/src/include/xce/types.h index 00dabe6b..7b8e7556 100644 --- a/src/include/xce/types.h +++ b/src/include/xce/types.h @@ -230,16 +230,16 @@ typedef struct { } xce_launch_t; typedef struct { - char* exechome; - char* execcwd; - char* execuser; - char* cmdline; - char* exdata; - char* launch_type; - char* q_pre_exec; - char* q_post_exec; - char* job_pre_exec; - char* job_post_exec; + char* exechome; + char* execcwd; + char* execuser; + char* cmdline; + char* exdata; + char* launch_type; + char* q_pre_exec; + char* q_post_exec; + char* job_pre_exec; + char* job_post_exec; } job_startinfo_t; typedef struct { @@ -389,6 +389,30 @@ typedef struct { char* ckptinfo; } xce_ckptinfo_t; +typedef struct { + char* host; + int reason; +} xce_host_reason_t; + +typedef struct { + char* resource; + char* host; + int reason; +} xce_res_reason_t; + +typedef struct { + int njobs; + uint64_t* jobs; +} xce_clear_reason_t; + +typedef struct { + int main_reason; + int nhost_reasons; + xce_host_reason_t** host_based_reasons; + int nres_reasons; + xce_res_reason_t** res_based_reasons; +} xce_job_preason_t; + typedef struct { uint64_t jobid; job_submit_data_t* submit; @@ -405,6 +429,7 @@ typedef struct { job_exitinfo_t* exit_info; char* ckptinfo; int64_t pending_ctrl; + xce_job_preason_t* pend_reasons; } xce_job_t; typedef struct { @@ -447,6 +472,7 @@ typedef struct { int64_t pending_ctrl; char* pre_exec; char* post_exec; + char* autopscaler; } xce_queue_t; typedef struct { @@ -615,4 +641,14 @@ typedef struct { float value; } xce_rusage_t; +typedef struct { + uint64_t jobid; + xce_job_preason_t* reasons; +} xce_pendreason_t; + +typedef struct { + int nreasons; + xce_pendreason_t** reasons; +} xce_preason_set_t; + #endif diff --git a/src/include/xceapi.h b/src/include/xceapi.h index ac3c384f..4667aaa4 100644 --- a/src/include/xceapi.h +++ b/src/include/xceapi.h @@ -8,6 +8,8 @@ #include "xcectx.h" extern int xce_hello(xce_ctx_t*, char*); +extern int xce_nop(xce_ctx_t*); +extern int xce_heartbeat(xce_ctx_t*, int); extern int xce_shutdown(xce_ctx_t*); extern int xce_reslist(xce_ctx_t*, xce_resource_t***); extern int xce_join(xce_ctx_t*, int, hostattrs_t*); @@ -27,5 +29,8 @@ extern int xce_queuectrl(xce_ctx_t*, xce_queuectrl_t*); extern int xce_runtask(xce_ctx_t*, xce_taskrun_t*); extern int xce_update_jusage(xce_ctx_t*, uint64_t, int, int*, int, int*, int, xce_rusage_t**); +extern int xce_set_pend_reason(xce_ctx_t*, xce_preason_set_t*); +extern int xce_clear_reason(xce_ctx_t*, xce_clear_reason_t*); +extern int xce_jobinfo(xce_ctx_t*, uint64_t, xce_job_t**); #endif diff --git a/src/lib/client.c b/src/lib/client.c index 4a58ec55..977c89fa 100644 --- a/src/lib/client.c +++ b/src/lib/client.c @@ -178,6 +178,28 @@ int xcecli_hello(xcecli_ctx_t c, char* msg) return xce_hello(&(ctx->ctx), msg); } +int xcecli_nop(xcecli_ctx_t c) +{ + api_ctx_t* ctx = (api_ctx_t*)c; + + if (c == NULL) { + return -EINVAL; + } + + return xce_nop(&(ctx->ctx)); +} + +int xcecli_heartbeat(xcecli_ctx_t c, int hb_interval) +{ + api_ctx_t* ctx = (api_ctx_t*)c; + + if (c == NULL) { + return -EINVAL; + } + + return xce_heartbeat(&(ctx->ctx), hb_interval); +} + int xcecli_shutdown(xcecli_ctx_t c) { api_ctx_t* ctx = (api_ctx_t*)c; @@ -589,3 +611,52 @@ void xcecli_unblock_tasks(xcecli_ctx_t c) xce_unblock_tasks(&(ctx->ctx)); } + +int xcecli_set_pend_reason(xcecli_ctx_t c, xce_preason_set_t* reasons) +{ + api_ctx_t* ctx = (api_ctx_t*)c; + int rc; + + if ((ctx == NULL) || (reasons == NULL)) { + return -EINVAL; + } + + rc = xce_set_pend_reason(&(ctx->ctx), reasons); + return rc; +} + +int xcecli_clear_reason(xcecli_ctx_t c, int njobs, uint64_t* jobs) +{ + api_ctx_t* ctx = (api_ctx_t*)c; + int rc; + /* clang-format off */ + xce_clear_reason_t reason = { + .njobs = njobs, + .jobs = jobs + }; + /* clang-format on */ + + if ((ctx == NULL) || ((njobs > 0) && (jobs == NULL))) { + return -EINVAL; + } + + if (njobs == 0) { + return 0; + } + + rc = xce_clear_reason(&(ctx->ctx), &reason); + return rc; +} + +int xcecli_jobinfo(xcecli_ctx_t c, uint64_t jobid, xce_job_t** job) +{ + api_ctx_t* ctx = (api_ctx_t*)c; + int rc; + + if ((ctx == NULL) || (job == NULL)) { + return -EINVAL; + } + + rc = xce_jobinfo(&(ctx->ctx), jobid, job); + return rc; +} diff --git a/src/lib/delay.c b/src/lib/delay.c new file mode 100644 index 00000000..df56a242 --- /dev/null +++ b/src/lib/delay.c @@ -0,0 +1,192 @@ +#include +#include +#include +#include +#include + +#include "delay.h" +#include "utils.h" + +static void* delay_worker(void* args) +{ + delaymgr_t* mgr = (delaymgr_t*)args; + + while (1) { + ordq_item_t* it; + delay_task_t* t; + time_t now; + + pthread_testcancel(); + sleep(mgr->precision); + + it = ordq_peek(&(mgr->tasks)); + if (it == NULL) { + continue; + } + + now = time(NULL); + if ((time_t)it->key > now) { + continue; + } + + it = ordq_get(&(mgr->tasks)); + t = (delay_task_t*)it->data; + + mgr->runner_fn(t->entry, t->args); + if (t->runcount > 0) { + t->runcount--; + } + + if (t->runcount == 0) { + FREEUP(t); + } else { + time_t nextrun = now + t->run_period; + + /* put back to taskq as it need to be run again */ + ordq_put(&(mgr->tasks), (void*)nextrun, t); + } + + FREEUP(it); + } + + return NULL; +} + +static int time_cmp(time_t k1, time_t k2) +{ + if (k1 > k2) { + return 1; + } + + if (k1 == k2) { + return 0; + } + + return -1; +} + +static void delay_task_killer(delay_task_t* t) +{ + if (t == NULL) { + return; + } + + FREEUP(t); +} + +static int default_runner_fn(delay_task_entry_fn_t entry, void* arg) +{ + if (entry == NULL) { + return 0; + } + + return entry(arg); +} + +int delaymgr_init(delaymgr_t* mgr, int precision, task_run_fn_t runner_fn) +{ + if ((mgr == NULL) || (precision <= 0)) { + return -EINVAL; + } + + if (runner_fn == NULL) { + runner_fn = default_runner_fn; + } + + mgr->precision = precision; + mgr->running = 0; + mgr->runner_fn = runner_fn; + mgr->worker = 0; + + ordq_init(&(mgr->tasks), (ordq_data_killer_t)delay_task_killer, + (ordq_cmp_fn_t)time_cmp); + + return 0; +} + +void delaymgr_stop(delaymgr_t* mgr) +{ + int rc; + + if (mgr == NULL) { + return; + } + + if (!(mgr->running)) { + return; + } + + if (mgr->worker != 0) { + rc = pthread_cancel(mgr->worker); + if (rc == 0) { + void* res = NULL; + + pthread_join(mgr->worker, &res); + + if (res == PTHREAD_CANCELED) { + mgr->worker = 0; + } + } + } + + return; +} + +void delaymgr_fini(delaymgr_t* mgr) +{ + if (mgr == NULL) { + return; + } + + if (mgr->running) { + delaymgr_stop(mgr); + } + + ordq_fini(&(mgr->tasks)); +} + +int delaymgr_start(delaymgr_t* mgr) +{ + int rc; + + if (mgr == NULL) { + return -EINVAL; + } + + rc = pthread_create(&(mgr->worker), NULL, &delay_worker, mgr); + if (rc == 0) { + mgr->running = 1; + + } else { + SET_ERROR(errno); + return -errno; + } + + return 0; +} + +int delay_task(delaymgr_t* mgr, int delay, int runcount, + delay_task_entry_fn_t entry, void* args) +{ + delay_task_t* t; + time_t now = time(NULL); + time_t ts_nextrun; + + if ((mgr == NULL) || (delay <= 0) || (entry == NULL)) { + return -EINVAL; + } + + t = (delay_task_t*)calloc(1, sizeof(*t)); + if (t == NULL) { + return -ENOMEM; + } + + ts_nextrun = now + delay; + + t->runcount = runcount; + t->run_period = delay; + t->entry = entry; + t->args = args; + + return ordq_put(&(mgr->tasks), (void*)ts_nextrun, t); +} diff --git a/src/lib/meson.build b/src/lib/meson.build index 63cf70ed..1a1d745b 100644 --- a/src/lib/meson.build +++ b/src/lib/meson.build @@ -27,7 +27,10 @@ libxce_src = [ gen_srcs[0], 'xstring.c', 'zarray.c', 'wrap.c', - 'idgen.c' + 'idgen.c', + 'delay.c', + 'ordq.c', + 'preason.c' ] incdir = include_directories('../include') @@ -61,7 +64,10 @@ libxce_test_src = ['testsuite/test.c', 'testsuite/test_xstring.c', 'testsuite/test_zarray.c', 'testsuite/test_config.c', - 'testsuite/test_idgen.c' + 'testsuite/test_idgen.c', + 'testsuite/test_delay.c', + 'testsuite/test_ordq.c', + 'testsuite/test_preason.c' ] libxce_test_deps = [libxce_deps, unity_dep, libxce_dep] diff --git a/src/lib/mylock.c b/src/lib/mylock.c index 810e6152..a030396f 100644 --- a/src/lib/mylock.c +++ b/src/lib/mylock.c @@ -161,6 +161,30 @@ int cbuf_get(cbuf_t* b, void* it) return 0; } +int cbuf_tryget(cbuf_t* b, void* it) +{ + int rc = 0; + + if ((b == NULL) || (it == NULL)) { + return -EINVAL; + } + + pthread_mutex_lock(&(b->mutex)); + if (b->len == 0) { + rc = -EAGAIN; + } else { + memcpy(it, (void*)((char*)b->data) + b->isize * b->head, b->isize); + + b->head = (b->head + 1) % b->ncap; + b->len--; + + pthread_cond_signal(&(b->condput)); + } + pthread_mutex_unlock(&(b->mutex)); + + return rc; +} + int my_rwlock_init(my_rwlock_t* l) { if (l == NULL) { diff --git a/src/lib/netio.c b/src/lib/netio.c index 558647eb..ad761da4 100644 --- a/src/lib/netio.c +++ b/src/lib/netio.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include "dataq.h" @@ -355,6 +356,7 @@ static int handle_input(endpoint_t* ep) } io_fsm_start(&(ep->fsm_in), pkt); + ep->ts_last_in = time(NULL); } rc = io_fsm_perform(&(ep->fsm_in)); @@ -388,6 +390,8 @@ static int handle_output(endpoint_t* ep) if (rc < 0) { return rc; } + + ep->ts_last_out = time(NULL); } rc = io_fsm_perform(&(ep->fsm_out)); diff --git a/src/lib/ordq.c b/src/lib/ordq.c new file mode 100644 index 00000000..a930a3bb --- /dev/null +++ b/src/lib/ordq.c @@ -0,0 +1,107 @@ +#include +#include +#include +#include + +#include "mylock.h" +#include "ordq.h" + +int ordq_init(ordq_t* q, ordq_data_killer_t killer, ordq_cmp_fn_t cmp) +{ + if ((q == NULL) || (cmp == NULL)) { + return -EINVAL; + } + + mylock_init(&(q->lock)); + q->head = NULL; + q->killer = killer; + q->cmp = cmp; + + return 0; +} + +static int ordq_put_(ordq_t* q, ordq_item_t* it) +{ + ordq_item_t** p = &(q->head); + + while ((*p) != NULL) { + if (q->cmp((*p)->key, it->key) > 0) { + /* we found the location to insert */ + break; + } + + p = &((*p)->next); + } + + it->next = *p; + *p = it; + + return 0; +} + +int ordq_put(ordq_t* q, ordkey_t key, void* data) +{ + int rc = 0; + ordq_item_t* it; + + it = (ordq_item_t*)calloc(1, sizeof(*it)); + if (it == NULL) { + return -ENOMEM; + } + + it->key = key; + it->data = data; + + mylock_lock(&(q->lock)); + rc = ordq_put_(q, it); + mylock_unlock(&(q->lock)); + + return rc; +} + +void ordq_fini(ordq_t* q) +{ + ordq_item_t* p = q->head; + + while (p != NULL) { + ordq_item_t* p1 = p->next; + + if (p->data != NULL) { + q->killer(p->data); + } + + p->data = NULL; + free(p); + + p = p1; + } + + mylock_fini(&(q->lock)); +} + +ordq_item_t* ordq_get(ordq_t* q) +{ + ordq_item_t* it; + + mylock_lock(&(q->lock)); + + it = q->head; + if (it != NULL) { + q->head = it->next; + } + + mylock_unlock(&(q->lock)); + + return it; +} + +ordq_item_t* ordq_peek(ordq_t* q) +{ + ordq_item_t* it = NULL; + + mylock_lock(&(q->lock)); + it = q->head; + mylock_unlock(&(q->lock)); + + return it; +} diff --git a/src/lib/preason.c b/src/lib/preason.c new file mode 100644 index 00000000..91f72cf9 --- /dev/null +++ b/src/lib/preason.c @@ -0,0 +1,151 @@ +#include + +#include "preason.h" +#include "utils.h" + +void erase_job_preason(xce_job_preason_t* reasons) +{ + if (reasons == NULL) { + return; + } + + if (reasons->host_based_reasons != NULL) { + int i; + + for (i = 0; i < reasons->nhost_reasons; i++) { + if (reasons->host_based_reasons[i] != NULL) { + FREEUP(reasons->host_based_reasons[i]->host); + FREEUP(reasons->host_based_reasons[i]); + } + } + + FREEUP(reasons->host_based_reasons); + reasons->nhost_reasons = 0; + } + + if (reasons->res_based_reasons != NULL) { + int i; + + for (i = 0; i < reasons->nres_reasons; i++) { + if (reasons->res_based_reasons[i] == NULL) { + continue; + } + + FREEUP(reasons->res_based_reasons[i]->resource); + FREEUP(reasons->res_based_reasons[i]->host); + FREEUP(reasons->res_based_reasons[i]); + } + + FREEUP(reasons->res_based_reasons); + reasons->nres_reasons = 0; + } + + reasons->main_reason = 0; +} + +static int add_host_preason(xce_job_preason_t* reasons, int code, char* host) +{ + xce_host_reason_t** p; + xce_host_reason_t* p1; + char* h; + int count; + + if ((reasons == NULL) || (host == NULL)) { + return -EINVAL; + } + + h = strdup(host); + if (h == NULL) { + return -ENOMEM; + } + + p1 = (xce_host_reason_t*)calloc(1, sizeof(*p1)); + if (p1 == NULL) { + FREEUP(h); + return -ENOMEM; + } + + p1->reason = code; + p1->host = h; + + count = reasons->nhost_reasons; + p = (xce_host_reason_t**)realloc(reasons->host_based_reasons, + (count + 1) * sizeof(xce_host_reason_t*)); + if (p == NULL) { + FREEUP(h); + FREEUP(p1); + return -ENOMEM; + } + + reasons->host_based_reasons = p; + reasons->host_based_reasons[count] = p1; + reasons->nhost_reasons++; + + return reasons->nhost_reasons; +} + +static int add_res_preason(xce_job_preason_t* reasons, int code, char* host, + char* res) +{ + xce_res_reason_t** p; + char* h; + char* r; + int count; + xce_res_reason_t* p1; + + if ((reasons == NULL) || (host == NULL) || (res == NULL)) { + return -EINVAL; + } + + h = strdup(host); + r = strdup(res); + p1 = (xce_res_reason_t*)calloc(1, sizeof(*p1)); + + if ((h == NULL) || (r == NULL) || (p1 == NULL)) { + FREEUP(h); + FREEUP(r); + FREEUP(p1); + return -ENOMEM; + } + + p1->host = h; + p1->resource = r; + p1->reason = code; + + count = reasons->nres_reasons; + p = (xce_res_reason_t**)realloc(reasons->res_based_reasons, + (count + 1) * sizeof(*p)); + if (p == NULL) { + FREEUP(h); + FREEUP(r); + FREEUP(p1); + + return -ENOMEM; + } + + reasons->res_based_reasons = p; + reasons->res_based_reasons[count] = p1; + + reasons->nres_reasons++; + + return reasons->nres_reasons; +} + +int add_job_preason(xce_job_preason_t* reasons, int code, char* host, char* res) +{ + int rc = 0; + + if ((host == NULL) && (res == NULL)) { + /* set main reason */ + erase_job_preason(reasons); + reasons->main_reason = code; + } else if (res == NULL) { + /* set host reason */ + rc = add_host_preason(reasons, code, host); + } else { + /* set resource reason */ + rc = add_res_preason(reasons, code, host, res); + } + + return rc; +} diff --git a/src/lib/request.c b/src/lib/request.c index aa9a2c97..74ca549b 100644 --- a/src/lib/request.c +++ b/src/lib/request.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include "mylock.h" @@ -850,3 +851,41 @@ int xce_runtask_req(endpoint_t* ep, xce_taskrun_t* runinfo) return xce_request(ep, &request); } + +int xce_set_pend_reason_req(endpoint_t* ep, xce_preason_set_t* reasons) +{ + /* clang-format off */ + xce_request_t request = { + .opcode = OPCode_setpreason, + .body_which = Request_body_preasonmsg, + .body = { + .preasonmsg = reasons + } + }; + /* clang-format on */ + + if ((ep == NULL) || (reasons == NULL)) { + return -EINVAL; + } + + return xce_request(ep, &request); +} + +int xce_clear_reason_req(endpoint_t* ep, xce_clear_reason_t* reason) +{ + /* clang-format off */ + xce_request_t request = { + .opcode = OPCode_clrpreason, + .body_which = Request_body_clrpreasonmsg, + .body = { + .clrpreasonmsg = reason + } + }; + /* clang-format on */ + + if ((ep == NULL) || (reason == NULL)) { + return -EINVAL; + } + + return xce_request(ep, &request); +} diff --git a/src/lib/testsuite/test.c b/src/lib/testsuite/test.c index 30258417..f3669b22 100644 --- a/src/lib/testsuite/test.c +++ b/src/lib/testsuite/test.c @@ -12,6 +12,9 @@ extern void test_error(void); extern void test_utils(void); extern void test_config(void); extern void test_idgen(void); +extern void test_preason(void); +extern void test_ordq(void); +extern void test_delay(void); void setUp(void) { return; } @@ -28,6 +31,9 @@ int main(int argc, char* argv[]) test_xstrings(); test_error(); test_utils(); + test_preason(); + test_ordq(); + test_delay(); return UNITY_END(); } diff --git a/src/lib/testsuite/test_delay.c b/src/lib/testsuite/test_delay.c new file mode 100644 index 00000000..08214fe5 --- /dev/null +++ b/src/lib/testsuite/test_delay.c @@ -0,0 +1,56 @@ +#include +#include +#include +#include + +#include "delay.h" +#include "unity.h" + +static int delay_work(void* arg) +{ + time_t now; + int* pcount = (int*)arg; + + now = time(NULL); + (*pcount)++; + + printf("delay task run at %ld\n", now); + return 0; +} + +static void test_delay1() +{ + delaymgr_t mgr; + int count = 0; + int runcount = 15; + int delay_period = 2; + time_t t_start = time(NULL); + + delaymgr_init(&mgr, 1, NULL); + delaymgr_start(&mgr); + delay_task(&mgr, delay_period, runcount, (delay_task_entry_fn_t)delay_work, + &count); + + while (1) { + time_t now = time(NULL); + + if (now - t_start > (runcount + 2) * delay_period) { + break; + } + + sleep(5); + } + + TEST_ASSERT_TRUE(count == runcount); + + delaymgr_fini(&mgr); + + return; +} + +void test_delay(void) +{ + printf("\n>> delay test\n\n"); + + RUN_TEST(test_delay1); +} diff --git a/src/lib/testsuite/test_ordq.c b/src/lib/testsuite/test_ordq.c new file mode 100644 index 00000000..64b459e7 --- /dev/null +++ b/src/lib/testsuite/test_ordq.c @@ -0,0 +1,82 @@ +#include +#include +#include +#include + +#include "ordq.h" +#include "unity.h" +#include "utils.h" + +static int long_cmp(ordkey_t k1, ordkey_t k2) +{ + if ((long)k1 < (long)k2) { + return -1; + } else if ((long)k1 > (long)k2) { + return 1; + } + + return 0; +} + +static void test_ordq_get(void) +{ + int rc; + ordq_t q; + ordq_item_t* it; + int keys[8] = { 9, 8, 1, 4, 7, 2, 6, 2 }; + int val[8] = { 1, 2, 2, 4, 6, 7, 8, 9 }; + int i; + + rc = ordq_init(&q, NULL, (ordq_cmp_fn_t)long_cmp); + TEST_ASSERT_TRUE(rc >= 0); + + for (i = 0; i < 8; i++) { + ordq_put(&q, mk_ordkey(keys[i]), (void*)NULL); + } + + for (i = 0; i < 8; i++) { + it = ordq_get(&q); + + TEST_ASSERT_TRUE((it != NULL) && (it->key == mk_ordkey(val[i]))); + free(it); + } + + ordq_fini(&q); + + return; +} + +static void test_ordq_peek(void) +{ + int rc; + ordq_t q; + ordq_item_t* it; + int keys[8] = { 9, 8, 1, 4, 7, 2, 6, 2 }; + int val[8] = { 1, 2, 2, 4, 6, 7, 8, 9 }; + int i; + + rc = ordq_init(&q, NULL, (ordq_cmp_fn_t)long_cmp); + TEST_ASSERT_TRUE(rc >= 0); + + for (i = 0; i < 8; i++) { + ordq_put(&q, mk_ordkey(keys[i]), (void*)NULL); + } + + for (i = 0; i < 8; i++) { + it = ordq_peek(&q); /* ordq_peek() will not remove item from q */ + + TEST_ASSERT_TRUE((it != NULL) && (it->key == mk_ordkey(val[0]))); + } + + ordq_fini(&q); + + return; +} + +void test_ordq(void) +{ + printf("\n>> ordq test\n\n"); + + RUN_TEST(test_ordq_get); + RUN_TEST(test_ordq_peek); +} diff --git a/src/lib/testsuite/test_preason.c b/src/lib/testsuite/test_preason.c new file mode 100644 index 00000000..0a8ed29d --- /dev/null +++ b/src/lib/testsuite/test_preason.c @@ -0,0 +1,168 @@ +#include +#include +#include + +#include "preason.h" +#include "unity.h" + +static void test_main_reason() +{ + xce_job_preason_t reason; + + memset(&reason, 0x0, sizeof(reason)); + + add_job_preason(&reason, 1, NULL, NULL); + TEST_ASSERT_TRUE(reason.main_reason == 1); + TEST_ASSERT_TRUE((reason.nhost_reasons == 0) + && (reason.host_based_reasons == NULL)); + TEST_ASSERT_TRUE((reason.nres_reasons == 0) + && (reason.res_based_reasons == NULL)); +} + +static void test_add_host_reason(void) +{ + xce_job_preason_t reason; + + memset(&reason, 0x0, sizeof(reason)); + + add_job_preason(&reason, 1, "h1", NULL); + TEST_ASSERT_TRUE(reason.main_reason == 0); + TEST_ASSERT_TRUE((reason.nhost_reasons == 1) + && (reason.host_based_reasons != NULL) + && (reason.host_based_reasons[0] != NULL)); + TEST_ASSERT_TRUE( + (reason.host_based_reasons[0]->reason == 1) + && (strcmp(reason.host_based_reasons[0]->host, "h1") == 0)); + TEST_ASSERT_TRUE((reason.nres_reasons == 0) + && (reason.res_based_reasons == NULL)); + erase_job_preason(&reason); +} + +static void test_add_res_reason(void) +{ + xce_job_preason_t reason; + + memset(&reason, 0x0, sizeof(reason)); + + add_job_preason(&reason, 1, "h1", "res1"); + TEST_ASSERT_TRUE(reason.main_reason == 0); + TEST_ASSERT_TRUE((reason.nres_reasons == 1) + && (reason.res_based_reasons != NULL) + && (reason.res_based_reasons[0] != NULL)); + TEST_ASSERT_TRUE( + (reason.res_based_reasons[0]->reason == 1) + && (strcmp(reason.res_based_reasons[0]->host, "h1") == 0) + && (strcmp(reason.res_based_reasons[0]->resource, "res1") == 0)); + TEST_ASSERT_TRUE((reason.nhost_reasons == 0) + && (reason.host_based_reasons == NULL)); + erase_job_preason(&reason); +} + +static void test_add_res_host_reason(void) +{ + xce_job_preason_t reason; + + memset(&reason, 0x0, sizeof(reason)); + + add_job_preason(&reason, 1, "h1", "res1"); + add_job_preason(&reason, 2, "h1", NULL); + + TEST_ASSERT_TRUE(reason.main_reason == 0); + TEST_ASSERT_TRUE((reason.nres_reasons == 1) + && (reason.res_based_reasons != NULL) + && (reason.res_based_reasons[0] != NULL)); + TEST_ASSERT_TRUE( + (reason.res_based_reasons[0]->reason == 1) + && (strcmp(reason.res_based_reasons[0]->host, "h1") == 0) + && (strcmp(reason.res_based_reasons[0]->resource, "res1") == 0)); + TEST_ASSERT_TRUE((reason.nhost_reasons == 1) + && (reason.host_based_reasons != NULL) + && (reason.host_based_reasons[0] != NULL)); + TEST_ASSERT_TRUE( + (reason.host_based_reasons[0]->reason == 2) + && (strcmp(reason.host_based_reasons[0]->host, "h1") == 0)); + erase_job_preason(&reason); +} + +static void test_add_main_reset_other(void) +{ + xce_job_preason_t reason; + + memset(&reason, 0x0, sizeof(reason)); + + add_job_preason(&reason, 1, "h1", "res1"); + add_job_preason(&reason, 2, "h1", NULL); + add_job_preason(&reason, 3, NULL, NULL); + + TEST_ASSERT_TRUE(reason.main_reason == 3); + TEST_ASSERT_TRUE((reason.nres_reasons == 0) + && (reason.res_based_reasons == NULL)); + TEST_ASSERT_TRUE((reason.nhost_reasons == 0) + && (reason.host_based_reasons == NULL)); + erase_job_preason(&reason); +} + +static void test_add_more_host_reason(void) +{ + xce_job_preason_t reason; + + memset(&reason, 0x0, sizeof(reason)); + + add_job_preason(&reason, 1, "h1", NULL); + add_job_preason(&reason, 2, "h2", NULL); + + TEST_ASSERT_TRUE(reason.main_reason == 0); + TEST_ASSERT_TRUE((reason.nhost_reasons == 2) + && (reason.host_based_reasons != NULL) + && (reason.host_based_reasons[0] != NULL) + && (reason.host_based_reasons[1] != NULL)); + TEST_ASSERT_TRUE( + (reason.host_based_reasons[0]->reason == 1) + && (strcmp(reason.host_based_reasons[0]->host, "h1") == 0) + && (reason.host_based_reasons[1]->reason == 2) + && (strcmp(reason.host_based_reasons[1]->host, "h2") == 0)); + TEST_ASSERT_TRUE((reason.nres_reasons == 0) + && (reason.res_based_reasons == NULL)); + erase_job_preason(&reason); +} + +static void test_add_more_res_reason(void) +{ + xce_job_preason_t reason; + + memset(&reason, 0x0, sizeof(reason)); + + add_job_preason(&reason, 1, "h1", "res1"); + add_job_preason(&reason, 2, "h2", "res2"); + + TEST_ASSERT_TRUE(reason.main_reason == 0); + TEST_ASSERT_TRUE((reason.nres_reasons == 2) + && (reason.res_based_reasons != NULL) + && (reason.res_based_reasons[0] != NULL) + && (reason.res_based_reasons[1] != NULL)); + TEST_ASSERT_TRUE( + (reason.res_based_reasons[0]->reason == 1) + && (strcmp(reason.res_based_reasons[0]->host, "h1") == 0) + && (strcmp(reason.res_based_reasons[0]->resource, "res1") == 0)); + TEST_ASSERT_TRUE( + (reason.res_based_reasons[1]->reason == 2) + && (strcmp(reason.res_based_reasons[1]->host, "h2") == 0) + && (strcmp(reason.res_based_reasons[1]->resource, "res2") == 0)); + + TEST_ASSERT_TRUE((reason.nhost_reasons == 0) + && (reason.host_based_reasons == NULL)); + erase_job_preason(&reason); +} + +void test_preason(void) +{ + printf("\n>> preason test\n\n"); + + RUN_TEST(test_main_reason); + RUN_TEST(test_add_host_reason); + RUN_TEST(test_add_res_reason); + RUN_TEST(test_add_res_host_reason); + RUN_TEST(test_add_main_reset_other); + RUN_TEST(test_add_more_host_reason); + RUN_TEST(test_add_more_res_reason); +} diff --git a/src/lib/utils.c b/src/lib/utils.c index d57766bf..b3110191 100644 --- a/src/lib/utils.c +++ b/src/lib/utils.c @@ -455,7 +455,7 @@ int fmt_jobid_ex(char* buf, size_t sz, uint64_t jobid, char lc, char rc) id_idx = EXTRACT_ARRAY_IDX(jobid); snprintf(data, sizeof(data) - 1, "%d-%d-%d-%d%c%d%c", id_cluster, - id_apiserver, id_timestamp, id_job, lc,id_idx, rc); + id_apiserver, id_timestamp, id_job, lc, id_idx, rc); } else { uint32_t id_seq; @@ -473,7 +473,8 @@ int fmt_jobid_ex(char* buf, size_t sz, uint64_t jobid, char lc, char rc) return len; } -int fmt_jobid(char* buf, size_t sz, uint64_t jobid) { +int fmt_jobid(char* buf, size_t sz, uint64_t jobid) +{ return fmt_jobid_ex(buf, sz, jobid, '[', ']'); } @@ -546,7 +547,8 @@ int parse_jobid(uint64_t* jobid, char* buf) return -EINVAL; } -uint64_t array_jobid(uint64_t jobid) { +uint64_t array_jobid(uint64_t jobid) +{ uint32_t tag = EXTRACT_ID_TAG(jobid); if (tag == 0x00) { @@ -556,7 +558,8 @@ uint64_t array_jobid(uint64_t jobid) { return EXTRACT_ARRAY_BASEID(jobid); } -uint64_t array_idx(uint64_t jobid) { +uint64_t array_idx(uint64_t jobid) +{ uint32_t tag = EXTRACT_ID_TAG(jobid); if (tag == 0x00) { @@ -565,3 +568,22 @@ uint64_t array_idx(uint64_t jobid) { return EXTRACT_ARRAY_IDX(jobid); } + +void unblock_sig(int signo) +{ + sigset_t mask; + + sigemptyset(&mask); + sigaddset(&mask, signo); + + sigprocmask(SIG_UNBLOCK, &mask, NULL); +} + +void block_sig(int signo) +{ + sigset_t mask; + + sigemptyset(&mask); + sigaddset(&mask, signo); + sigprocmask(SIG_BLOCK, &mask, NULL); +} diff --git a/src/lib/xceapi.c b/src/lib/xceapi.c index 81ee7f80..a913f999 100644 --- a/src/lib/xceapi.c +++ b/src/lib/xceapi.c @@ -27,6 +27,48 @@ int xce_hello(xce_ctx_t* ctx, char* msg) return status; } +int xce_nop(xce_ctx_t* ctx) +{ + int rc; + + if (ctx == NULL) { + return -EINVAL; + } + + if (ctx->state != XCE_CTX_RUN) { + return -EINVAL; + } + + rc = xce_nop_req(ctx->ep); + + /* no need to wait reply, just as heartbeat */ + return rc; +} + +int xce_heartbeat(xce_ctx_t* ctx, int hb_interval) +{ + time_t now; + + if (ctx == NULL) { + return -EINVAL; + } + + if (ctx->state != XCE_CTX_RUN) { + return -EINVAL; + } + + if (ctx->ep == NULL) { + return -EINVAL; + } + + now = time(NULL); + if (now - ctx->ep->ts_last_out < hb_interval) { + return 0; + } + + return xce_nop(ctx); +} + int xce_shutdown(xce_ctx_t* ctx) { int rc; @@ -45,13 +87,11 @@ int xce_shutdown(xce_ctx_t* ctx) return rc; } -int xce_get(xce_ctx_t* ctx, int table, void*** data) +int xce_get_with_condition(xce_ctx_t* ctx, xce_get_t* get, void*** data) { - int rc; - int status; - xce_match_t condition = { .type = MatchCondition_condition_matchall }; - xce_get_t get = { .table = table, .condition = &condition }; - dataset_t getreply = { .table = table, .nrows = 0, .rows = NULL }; + int rc; + int status; + dataset_t getreply; if (ctx == NULL) { return -EINVAL; @@ -61,8 +101,12 @@ int xce_get(xce_ctx_t* ctx, int table, void*** data) return -EINVAL; } + if ((get == NULL) || (data == NULL)) { + return -EINVAL; + } + ctx->reply.u.get = &getreply; - rc = xce_get_req(ctx->ep, &get); + rc = xce_get_req(ctx->ep, get); if (rc < 0) { return rc; } @@ -84,6 +128,21 @@ int xce_get(xce_ctx_t* ctx, int table, void*** data) return rc; } +int xce_get(xce_ctx_t* ctx, int table, void*** data) +{ + /* clang-format off */ + xce_match_t condition = { + .type = MatchCondition_condition_matchall + }; + xce_get_t get = { + .table = table, + .condition = &condition + }; + /* clang-format on */ + + return xce_get_with_condition(ctx, &get, data); +} + int xce_reslist(xce_ctx_t* ctx, xce_resource_t*** res) { int table = Data_data_resources; @@ -524,3 +583,88 @@ int xce_runtask(xce_ctx_t* ctx, xce_taskrun_t* runinfo) return status; } + +int xce_set_pend_reason(xce_ctx_t* ctx, xce_preason_set_t* reasons) +{ + int rc; + int status; + + if (ctx == NULL) { + return -EINVAL; + } + + if (ctx->state != XCE_CTX_RUN) { + return -EINVAL; + } + + rc = xce_set_pend_reason_req(ctx->ep, reasons); + if (rc < 0) { + return rc; + } + + rc = cbuf_get(&(ctx->wbuf), &status); + if (rc < 0) { + return rc; + } + + return status; +} + +int xce_clear_reason(xce_ctx_t* ctx, xce_clear_reason_t* reasons) +{ + int rc; + int status; + + if (ctx == NULL) { + return -EINVAL; + } + + if (ctx->state != XCE_CTX_RUN) { + return -EINVAL; + } + + rc = xce_clear_reason_req(ctx->ep, reasons); + if (rc < 0) { + return rc; + } + + rc = cbuf_get(&(ctx->wbuf), &status); + if (rc < 0) { + return rc; + } + + return status; +} + +int xce_jobinfo(xce_ctx_t* ctx, uint64_t jobid, xce_job_t** job) +{ + /* clang-format off */ + xce_match_t condition = { + .type = MatchCondition_condition_matchjid, + .condition = { + .jobid = jobid + } + }; + xce_get_t get = { + .table = Data_data_jobs, + .condition = &condition + }; + /* clang-format on */ + int rc; + xce_job_t** jobs; + + if ((ctx == NULL) || (job == NULL)) { + return -EINVAL; + } + + rc = xce_get_with_condition(ctx, &get, (void***)&jobs); + + if (rc <= 0) { + return rc; + } + + *job = jobs[0]; + FREEUP(jobs); + + return 1; +} diff --git a/src/scheduler/include/scheduler.h b/src/scheduler/include/scheduler.h index 40611fe4..c6877198 100644 --- a/src/scheduler/include/scheduler.h +++ b/src/scheduler/include/scheduler.h @@ -15,7 +15,7 @@ typedef enum { typedef struct { xce_decision_op_t op; union { - xce_job_alloc_t *alloc; + xce_job_alloc_t* alloc; } u; } xce_decision_t; diff --git a/src/schema/controller.capnp b/src/schema/controller.capnp index 3a8de61e..6fdd1bb7 100644 --- a/src/schema/controller.capnp +++ b/src/schema/controller.capnp @@ -28,6 +28,8 @@ enum OPCode { clusterconf @17; queuecontrol @18; runtasks @19; + setpreason @20; + clrpreason @21; } struct PeerInfo $C.mapname("xce_peerinfo_t") { @@ -35,6 +37,23 @@ struct PeerInfo $C.mapname("xce_peerinfo_t") { key @1: Text; } +struct HostBasedReason $C.mapname("xce_host_reason_t") { + host @0: Text; + reason @1: UInt32; +} + +struct ResourceBasedReason $C.mapname("xce_res_reason_t") { + resource @0: Text; + host @1: Text; + reason @2: UInt32; +} + +struct JobPendReason $C.mapname("xce_job_preason_t") { + mainReason @0: UInt32 $C.mapname("main_reason"); + hostReasons @1: List(HostBasedReason) $C.mapname("host_based_reasons") $C.maplistcount("nhost_reasons"); + resReasons @2: List(ResourceBasedReason) $C.mapname("res_based_reasons") $C.maplistcount("nres_reasons"); +} + struct StrList $C.mapname("xce_strlist_t") { data @0: List(Text) $C.mapname("data") $C.maplistcount("count"); } @@ -242,6 +261,7 @@ struct Job $C.mapname("xce_job_t"){ allocs @8: JobAlloc $C.mapname("allocs"); runtime @9: UInt32 $C.mapname("run_time"); pidinfo @10: JobPIDInfo; + pendreason @11: JobPendReason $C.mapname("pend_reasons"); } struct SchedJob $C.mapname("xce_job_t"){ @@ -257,6 +277,11 @@ struct SchedJob $C.mapname("xce_job_t"){ runtime @9: UInt32 $C.mapname("run_time"); } +struct SchedJobRusages $C.mapname("xce_job_rusages_t") { + jobid @0: UInt64 $C.mapname("jobid"); + usages @1: List(ResUsage) $C.mapname("usages") $C.maplistcount("count"); +} + struct JobData $C.mapname("xce_jobs_data_t") { ntotal @0: UInt32 $C.mapname("n_total"); pos @1: UInt32; @@ -422,6 +447,7 @@ struct NotifyData $C.mapname("xce_notify_data_t"){ taskFailInfo @14: TaskFailInfo $C.mapname("taskfailinfo"); taskFinish @15: TaskFinish $C.mapname("taskfinish"); postDoneJob @16: UInt64 $C.mapname("pdone_job"); + schedJobRusages @17: SchedJobRusages $C.mapname("sched_job_rusages"); } } @@ -510,6 +536,19 @@ struct RunTask $C.mapname("xce_taskrun_t") { jobid @2: UInt64 $C.mapname("jobid"); } +struct PendReason $C.mapname("xce_pendreason_t") { + jobid @0: UInt64 $C.mapname("jobid"); + reasons @1: JobPendReason $C.mapname("reasons"); +} + +struct PendReasonSet $C.mapname("xce_preason_set_t") { + reason @0: List(PendReason) $C.mapname("reasons") $C.maplistcount("nreasons"); +} + +struct ClearPReason $C.mapname("xce_clear_reason_t") { + jobs @0: List(UInt64) $C.mapname("jobs") $C.maplistcount("njobs"); +} + struct Request $C.mapname("xce_request_t"){ opcode @0: OPCode; seq @1: UInt64; @@ -531,6 +570,8 @@ struct Request $C.mapname("xce_request_t"){ jobctrlmsg @16: JobControl; queuectrlmsg @17: QueueControl; runtaskmsg @18: RunTask; + preasonmsg @19: PendReasonSet; + clrpreasonmsg @20: ClearPReason; } } @@ -561,6 +602,14 @@ struct HostConfig $C.mapname("xce_hostconfig_t") { dispatchwindows @6: TimeWindow $C.mapname("dispatchwindow"); } +struct ScalerConfig $C.mapname("scaler_config_t") { + name @0: Text $C.mapname("name"); + type @1: Text $C.mapname("prov_type"); + confPath @2: Text $C.mapname("conf_path"); + scriptPath @3: Text $C.mapname("script_path"); + scriptOption @4: Text $C.mapname("script_option"); +} + struct ClusterConfig $C.mapname("cluster_config_t") { clusterName @0: Text $C.mapname("clustername"); port @1: UInt32 $C.mapname("port"); @@ -580,4 +629,5 @@ struct ClusterConfig $C.mapname("cluster_config_t") { qmgr @15: HashZArrayConfig; resmgr @16: ResmgrConfig; xstrings @17: HashZArrayConfig; + hproviders @18: List(ScalerConfig) $C.mapname("hproviders") $C.maplistcount("nhproviders"); } diff --git a/src/worker/include/controller_connector.h b/src/worker/include/controller_connector.h index 47b381d3..b60625c8 100644 --- a/src/worker/include/controller_connector.h +++ b/src/worker/include/controller_connector.h @@ -52,8 +52,10 @@ int xcecd_job_ckpted(uint64_t jobid, int status, const char* info); int xcecd_job_remote_start(uint64_t jobid, const char* hostname, const char* command); const cc_callbacks_t* get_cc_callbacks(void); -int xcecd_task_fail(uint64_t jobid, int clientidx, int taskidx, int code); -int xcecd_jobpost_done(uint64_t jobid); -int xcecd_notify_job_fail(uint64_t jobid, int stage, int code); +int xcecd_task_fail(uint64_t jobid, int clientidx, int taskidx, int code); +int xcecd_jobpost_done(uint64_t jobid); +int xcecd_notify_job_fail(uint64_t jobid, int stage, int code); +int xcecd_jobrusage_update(xce_job_rusages_t* rusages); +bool xcecd_send_load_indices(xce_resval_t** vals, int count); #endif // CONTROL_CONNECTOR_H diff --git a/src/worker/include/node_information.h b/src/worker/include/node_information.h index 9b99d8ce..05a42adb 100644 --- a/src/worker/include/node_information.h +++ b/src/worker/include/node_information.h @@ -18,99 +18,23 @@ #define DEFAULT_AFTER_HOUR "19:00-7:00 5:19:00-1:7:00" struct time_window { - char* window_name; // 窗口名称 + char* window_name; windows_t* week[8]; time_t window_edge; }; -struct node_status { - short node_id; // 节点ID - int max_cpu_cores; // CPU核心数 - int max_memory; // 内存大小 - int num_disk; // 磁盘数量 - int max_swap; // 交换空间大小 - int max_network; // 网络带宽 - unsigned short port_number; // 端口号 - char node_type[MAXXCENAMELEN]; // 节点类型 - char node_arch[MAXXCENAMELEN]; // 节点架构 -}; - -typedef struct host_node host_node_t; -struct host_node { - char* node_name; // 节点名称 - short node_number; // 节点编号 - short node_model_number; // 节点型号编号 - short node_type_number; // 节点类型编号 - in_addr_t* node_ip; // 节点IP地址 - struct node_status node_status; // 节点状态 - char info_valid; // 信息是否有效 - unsigned char protocol_version; // 协议版本 - short available_high; // 高可用性 - short available_low; // 低可用性 - short use; // 使用情况 - char* windows; // 时间窗口 - windows_t* week[8]; // 一周的时间窗口 - time_t window_edge; // 时间窗口边界 - time_t last_jack_time; // 最后一次插入时间 - int node_inactivity_count; // 节点不活动计数 - int* status; // 状态 - float* busy_threshold; // 繁忙阈值 - float* load_index; // 负载指数 - float* user_load_index; // 用户负载指数 - int info_mask; // 信息掩码 - int load_mask; // 负载掩码 - int* resource_bitmap; // 资源位图 - int* dynamic_resource_bitmap; // 动态资源位图 - int number_of_instances; // 实例数量 - host_node_t* next; // 下一个节点 - time_t expire_time; // 过期时间 - uint8_t migrant; -}; - -struct cluster_node { - short cluster_number; // 集群编号 - char* cluster_name; // 集群名称 - int status; // 状态 - in_addr_t candidate_address_list[MAXCANDHOSTS]; // 候选地址列表 - char* primary_node_name; // 主节点名称 - unsigned short primary_address; // 主地址 - unsigned short primary_port; // 主端口 - char primary_known; // 主节点可识别 - int primary_inactivity_count; // 主节点不活动计数 - host_node_t* primary_pointer; // 主节点指针 - host_node_t* previous_primary_pointer; // 上一个主节点指针 - unsigned short checksum; // 校验和 - int number_of_nodes; // 节点数量 - int number_of_clients; // 客户端数量 - host_node_t* node_list; // 节点列表 - host_node_t* client_list; // 客户端列表 - int manager_id; // 管理器ID - char* manager_name; // 管理器名称 - int channel_fd; // 通道文件描述符 - int number_of_index; // 索引数量 - int number_of_user_index; // 用户索引数量 - int user_index_class; // 用户索引类 - char** load_index_names; // 负载指数名称 - int number_of_admins; // 管理员数量 - int* admin_ids; // 管理员ID - char** admin_names; // 管理员名称 - int number_of_resouces; // 资源数量 - int* resource_bitmaps; // 资源位图 - int* hosttype_bitmaps; // 主机类型位图 - int* hostmodel_bitmaps; // 主机模型位图 - int number_of_shared_resources; // 共享资源数量 - char** shared_resources; // 共享资源 - struct cluster_node* next; -}; - -// client_node 和 minSLimConfData 数据结构需要添加吗？ - -host_node_t* host_node_create(void); - -int host_node_update(host_node_t* node_ptr, host_node_t* new_node); - -void free_host_node(host_node_t* node_ptr); - int get_cpu_usage(int total_cores, int used_cores); +char* node_info_get_15s(void); +char* node_info_get_1m(void); +char* node_info_get_15m(void); +char* node_info_get_ut(void); +char* node_info_get_pg(void); +char* node_info_get_ls(void); +char* node_info_get_it(void); +char* node_info_get_swap(void); +char* node_info_get_mem(void); +char* node_info_get_tmp(void); +char* node_info_get_io(void); + #endif // NODE_INFORMATION_H_ diff --git a/src/worker/include/resource_sync.h b/src/worker/include/resource_sync.h index b4b7e096..e271b11c 100644 --- a/src/worker/include/resource_sync.h +++ b/src/worker/include/resource_sync.h @@ -1,29 +1,22 @@ -#ifndef RESOURCE_SYNC_H -#define RESOURCE_SYNC_H - -#include "hardware_monitor.h" -#include "node_information.h" -#include - -// 资源同步模块结构体 -typedef struct resource_sync resource_sync_t; - -// 创建资源同步模块 -resource_sync_t* resource_sync_create(void); - -// 销毁资源同步模块 -void resource_sync_destroy(resource_sync_t* sync); - -// 启动资源同步模块 -bool resource_sync_start(resource_sync_t* sync); - -// 停止资源同步模块 -void resource_sync_stop(resource_sync_t* sync); - -// 更新资源同步模块 -bool resource_sync_update(resource_sync_t* sync); - -// 读取资源同步模块 -void resource_sync_read(resource_sync_t* sync, host_node_t** node); - +#ifndef RESOURCE_SYNC_H +#define RESOURCE_SYNC_H + +#include "hardware_monitor.h" +#include "node_information.h" +#include + +typedef struct resource_sync resource_sync_t; + +resource_sync_t* resource_sync_create(void); +void resource_sync_destroy(resource_sync_t* sync); +bool resource_sync_start(resource_sync_t* sync); +void resource_sync_stop(resource_sync_t* sync); +bool resource_sync_update(resource_sync_t* sync); +void resource_sync_read(resource_sync_t* sync, void* data); + +bool resource_sync_register_load_index(const char* name, int valtype, + char* (*getter_func)(void), + bool is_builtin); +bool resource_sync_unregister_load_index(const char* name); + #endif // RESOURCE_SYNC_H \ No newline at end of file diff --git a/src/worker/include/uv_ipc.h b/src/worker/include/uv_ipc.h index 9e2f8190..35526294 100644 --- a/src/worker/include/uv_ipc.h +++ b/src/worker/include/uv_ipc.h @@ -11,12 +11,13 @@ extern "C" { #endif -#define HOSTNAME_LENGTH 256 -#define MESSAGE_LENGTH 256 -#define COMMAND_LENGTH 4096 -#define METHOD_LENGTH 64 -#define PATH_LENGTH 1024 -#define EXDATA_LENGTH 16 * 1024 +#define HOSTNAME_LENGTH 256 +#define MESSAGE_LENGTH 256 +#define COMMAND_LENGTH 4096 +#define METHOD_LENGTH 64 +#define PATH_LENGTH 1024 +#define EXDATA_LENGTH 16 * 1024 +#define MAX_RUSAGE_PAYLOAD 4096 typedef enum { MSG_GET_JOB, @@ -30,10 +31,22 @@ typedef enum { MSG_JOB_CKPT_RESULT, MSG_REMOTE_START_JOB, MSG_REMOTE_START_JOB_ACK, + MSG_JOB_RUSAGES, MSG_HEARTBEAT, MSG_ERROR } message_type_t; +/** + * @brief IPC rusage message structure + * As rusage data is variable-sized, we define a maximum payload size. + * This structure will be serialized and sent. + */ +typedef struct { + uint64_t job_id; + size_t payload_size; + char payload[MAX_RUSAGE_PAYLOAD]; +} ipc_job_rusages_t; + typedef struct { int type; } heartbeat_message_t; @@ -103,6 +116,7 @@ typedef struct { job_ckpt_result_t ckpt_result; remote_start_job_t remote_start; remote_start_job_ack_t remote_start_ack; + ipc_job_rusages_t rusages; char error_message[MESSAGE_LENGTH]; } data; int worker_pid; @@ -129,6 +143,7 @@ typedef void (*job_ckpt_callback_t)(uint64_t job_id, int kill, const char* method); typedef void (*job_ckpt_result_callback_t)(uint64_t job_id, int status, const char* restore_cmd); +typedef void (*job_rusages_callback_t)(xce_job_rusages_t* rusages); bool ipc_master_init(const char* socket_path); void ipc_master_run(uv_run_mode mode); @@ -141,6 +156,7 @@ bool ipc_master_send_job_signal(uint64_t job_id, int signal, bool force); bool ipc_master_send_job_ckpt(uint64_t job_id, int kill, const char* method); void ipc_master_set_ckpt_callback(job_ckpt_callback_t callback); void ipc_master_set_ckpt_result_callback(job_ckpt_result_callback_t callback); +void ipc_master_set_rusages_callback(job_rusages_callback_t callback); bool ipc_worker_init(const char* socket_path); void ipc_worker_connect(connection_callback_t callback); @@ -160,6 +176,7 @@ bool ipc_worker_send_ckpt_ack(uint64_t job_id, int kill, const char* method); void ipc_worker_set_ckpt_callback(job_ckpt_callback_t callback); bool ipc_worker_send_ckpt_result(uint64_t job_id, int status, const char* restore_cmd); +bool ipc_worker_send_rusages(xce_job_rusages_t* rusages); void on_write_complete(uv_write_t* req, int status); diff --git a/src/worker/include/xcecd.h b/src/worker/include/xcecd.h index ce661173..3c6d566c 100644 --- a/src/worker/include/xcecd.h +++ b/src/worker/include/xcecd.h @@ -92,7 +92,6 @@ int xcecd_init(const xcecd_config_t* config); int xcecd_start(void); void xcecd_stop(void); void xcecd_cleanup(void); -int send_status_and_receive_tasks(void); void xcecd_process_command(void* arg); const char* handle_network_request(const char* request, void* user_data); bool xcecd_config_is_valid(const xcecd_config_t* config); diff --git a/src/worker/include/xcecd_cgroups.h b/src/worker/include/xcecd_cgroups.h index 180d2275..58350722 100644 --- a/src/worker/include/xcecd_cgroups.h +++ b/src/worker/include/xcecd_cgroups.h @@ -1,6 +1,7 @@ #ifndef _XCECD_CGROUPS_H_ #define _XCECD_CGROUPS_H_ +#include "xcecd_utils.h" #include #include #include @@ -93,4 +94,13 @@ int get_systemd_job_exit_code(const char* unit_name); */ char* get_systemd_job_status_text(const char* unit_name); +/** + * @brief Collect resource usage statistics for a systemd job. + * @param job_id The ID of the job. + * @return A pointer to an xce_job_rusages_t struct, or NULL on failure. + * The caller is responsible for freeing the returned struct + * using free_xce_job_rusages(). + */ +xce_job_rusages_t* collect_job_rusages(uint64_t job_id); + #endif // _XCECD_CGROUPS_H_ \ No newline at end of file diff --git a/src/worker/include/xcecd_exdata.h b/src/worker/include/xcecd_exdata.h index e5c39335..0197fadb 100644 --- a/src/worker/include/xcecd_exdata.h +++ b/src/worker/include/xcecd_exdata.h @@ -1,8 +1,7 @@ #ifndef _XCECD_EXDATA_H #define _XCECD_EXDATA_H -#include - +#include "cjson/cJSON.h" #include "xcecd_cgroups.h" #include #include diff --git a/src/worker/meson.build b/src/worker/meson.build index 2c5c25aa..364c6ada 100644 --- a/src/worker/meson.build +++ b/src/worker/meson.build @@ -4,7 +4,22 @@ xcecd_worker_srcs = [ 'src/xcecd_exdata.c', 'src/uv_ipc.c', 'src/uv_proc.c', - 'src/static_array.c' + 'src/static_array.c', + 'src/xcecd_cgroups.c', + 'src/xcecd_worker_utils.c', + 'src/controller_connector.c', + 'src/xcecd_dbus_manager.c', + 'src/xcecd_db.c', + 'src/xcecd.c', + 'src/connection_handler.c', + 'src/resource_sync.c', + 'src/node_information.c', + 'src/thread_pool.c', + 'src/hardware_monitor.c', + 'src/system_monitor.c', + 'src/process_manager.c', + 'src/process_worker_manager.c', + 'src/xcecd_sig.c', ] incdir = include_directories('.', 'include', '../include', '../lib') @@ -13,10 +28,12 @@ libuv_dep = dependency('libuv') elogind_dep = dependency('libelogind') sqlite3_dep = dependency('sqlite3') libcgroup_dep = dependency('libcgroup') +hwloc_dep = dependency('hwloc') xcecd_worker_deps = [ libxce_dep, libcgroup_dep, + hwloc_dep, libuv_dep, elogind_dep, sqlite3_dep, @@ -69,9 +86,7 @@ xcecd_srcs = [ 'src/job_information.c' ] -hwloc_dep = dependency('hwloc') - -xcecd_deps = [xcecd_worker_deps,hwloc_dep] +xcecd_deps = [xcecd_worker_deps] xcecd = executable('xcecd', [xcecd_srcs,'src/main.c'], c_args: ['-DDBUS_MANAGER'], @@ -85,47 +100,3 @@ xcecd_virtual = executable('xcecd_virtual', include_directories: [incdir], dependencies:[xcecd_deps], install:true) - -include_dir_test = include_directories('test') - -xce_test_deps = xcecd_deps -xce_sched_test = executable('xce_sched_test', - [xcecd_srcs, 'test/xce_sched_test.c'], - include_directories: [incdir,include_dir_test], - dependencies: [xce_test_deps], - install: true) - -unity_deps = dependency('unity') -xcecd_daemon_test = executable('xcecd_daemon_test', - [xcecd_srcs, 'test/xcecd_daemon_test.c'], - include_directories: [incdir, include_dir_test], - dependencies: [xce_test_deps, unity_deps], - install: true) - -xcecd_worker_test = executable('xcecd_worker_test', - [xcecd_srcs, 'test/xcecd_worker_test.c'], - include_directories: [incdir, include_dir_test], - dependencies: [xce_test_deps, unity_deps], - install: true) - -# cgroup_srcs = [ -# 'src/xcecd_cgroups.c', -# 'test/cgroup_test.c' -# ] - -# cgroup_test = executable('cgroup_test', -# cgroup_srcs, -# include_directories: [incdir, include_dir_test], -# dependencies: [xce_test_deps, unity_deps], -# install: true) - -limit_test_srcs = [ - 'test/limit_test.c' -] - - -limit_test = executable('limit_test', - limit_test_srcs, - include_directories: [incdir, include_dir_test], - dependencies: [xce_test_deps, unity_deps, math_dep], - install: true) diff --git a/src/worker/src/controller_connector.c b/src/worker/src/controller_connector.c old mode 100644 new mode 100755 index 6d5484af..6ccc3f16 --- a/src/worker/src/controller_connector.c +++ b/src/worker/src/controller_connector.c @@ -1514,3 +1514,51 @@ int xcecd_jobpost_done(uint64_t jobid) "Job post done and calling xcecli_notify_jobpostdone..."); return xcecli_notify_jobpostdone(xcecli_ctx, jobid); } + +int xcecd_jobrusage_update(xce_job_rusages_t* rusages) +{ + if (!rusages) { + XCECD_LOG(XCE_LOG_ERROR, + "Invalid rusage data."); + return -1; + } + + XCECD_LOG(XCE_LOG_DEBUG, + "Updating job rusage to controller for job %lu", rusages->jobid); + + return xcecli_update_jusage(xcecli_ctx, + rusages->jobid, + rusages->npids, + rusages->pids, + rusages->npgids, + rusages->pgids, + rusages->count, + rusages->usages); +} + +bool xcecd_send_load_indices(xce_resval_t** vals, int count) { + if (vals == NULL || count <= 0) { + XCECD_LOG(XCE_LOG_ERROR, + "Invalid parameters for sending load indices"); + return false; + } + + if (xcecli_ctx == NULL) { + XCECD_LOG(XCE_LOG_ERROR, + "xcecli_ctx is not initialized"); + return false; + } + + int result = xcecli_update_resvals(xcecli_ctx, count, &vals[0]); + + if (result != 0) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to send load indices to controller, error code: %d", + result); + return false; + } + + XCECD_LOG(XCE_LOG_DEBUG, + "Successfully sent %d load indices to controller", count); + return true; +} diff --git a/src/worker/src/hardware_monitor.c b/src/worker/src/hardware_monitor.c index 046bb5ae..c1e54ff6 100644 --- a/src/worker/src/hardware_monitor.c +++ b/src/worker/src/hardware_monitor.c @@ -25,22 +25,13 @@ struct hardware_monitor { time_t last_update_time; }; -static void write_to_acct_file(hardware_monitor_t* monitor, int cpu_cores) +bool hwloc_disabled = true; +bool hwloc_available = false; + +static int get_cpu_cores_fallback(void) { - XCECD_LOG(XCE_LOG_INFO, "Writing to acct file..."); - FILE* file = fopen(monitor->acct_file, "a"); - if (!file) { - return; - } - - time_t now; - time(&now); - struct tm* timeinfo = localtime(&now); - char timestamp[64]; - strftime(timestamp, sizeof(timestamp), "%Y-%m-%d %H:%M:%S", timeinfo); - - fprintf(file, "%s\t%d\n", timestamp, cpu_cores); - fclose(file); + long cores = sysconf(_SC_NPROCESSORS_ONLN); + return (cores > 0) ? (int)cores : 1; } hardware_monitor_t* hardware_monitor_create(const char* acct_file) @@ -75,16 +66,20 @@ bool hardware_monitor_update(hardware_monitor_t* monitor) } hwloc_topology_t topology; - if (hwloc_topology_init(&topology) < 0 - || hwloc_topology_load(topology) < 0) { - return false; + int current_cores = -1; + + if (!hwloc_disabled && hwloc_topology_init(&topology) >= 0 + && hwloc_topology_load(topology) >= 0) { + current_cores = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_CORE); + hwloc_available = true; + } else { + XCECD_LOG( + XCE_LOG_DEBUG3, + "hwloc disabled or not available, using fallback for CPU cores"); + current_cores = get_cpu_cores_fallback(); } - int current_cores = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_CORE); - - // 如果CPU核心数发生变化，记录到文件 if (current_cores != monitor->last_cpu_cores) { - write_to_acct_file(monitor, current_cores); monitor->last_cpu_cores = current_cores; } @@ -94,7 +89,10 @@ bool hardware_monitor_update(hardware_monitor_t* monitor) monitor->free_memory = free_memory; } - hwloc_topology_destroy(topology); + if (hwloc_available) { + hwloc_topology_destroy(topology); + } + monitor->last_update_time = time(NULL); return true; } @@ -136,23 +134,31 @@ bool hardware_monitor_get_memory_info(int64_t* total_memory, return false; } - hwloc_topology_t topology; - if (hwloc_topology_init(&topology) < 0 - || hwloc_topology_load(topology) < 0) { - return false; - } + if (!hwloc_disabled && hwloc_available) { + hwloc_topology_t topology; + if (hwloc_topology_init(&topology) < 0 + || hwloc_topology_load(topology) < 0) { + return false; + } - hwloc_obj_t root = hwloc_get_root_obj(topology); - *total_memory = root->total_memory; + hwloc_obj_t root = hwloc_get_root_obj(topology); + *total_memory = root->total_memory; + + hwloc_topology_destroy(topology); + } struct sysinfo info; if (sysinfo(&info) != 0) { XCECD_LOG(XCE_LOG_ERROR, "Failed to get sysinfo: %s", strerror(errno)); return false; } - *free_memory = (size_t)info.freeram * info.mem_unit; - hwloc_topology_destroy(topology); + if (!hwloc_available) { + *total_memory = (int64_t)info.totalram * info.mem_unit; + } + + *free_memory = (int64_t)info.freeram * info.mem_unit; + return true; } #endif diff --git a/src/worker/src/main.c b/src/worker/src/main.c index 1c13168f..1b10ae66 100644 --- a/src/worker/src/main.c +++ b/src/worker/src/main.c @@ -1,9 +1,6 @@ #include "sqlite3.h" #include "xcecd.h" - #include -#include -#include #include #include #include @@ -15,16 +12,13 @@ #include #else #include +#include #include #include #include #include #endif -#if !defined(DBUS_MANAGER) -#include "process_worker_manager.h" -#endif - /* TEMP SOLUTION: use config later */ #define LOG_SIZE (1024 * 1024 * 100) // 100 MB #define LOG_LEVEL 2 // DEBUG level @@ -116,6 +110,27 @@ static void handle_exited_children(process_manager_t* manager, StaticArray pids) clear_static(&pids); } +static void free_xce_job_rusages(xce_job_rusages_t* rusages) +{ + if (!rusages) { + return; + } + + FREEUP(rusages->pids); + FREEUP(rusages->pgids); + + if (rusages->usages) { + for (int i = 0; i < rusages->count; i++) { + if (rusages->usages[i]) { + FREEUP(rusages->usages[i]->resname); + FREEUP(rusages->usages[i]); + } + } + FREEUP(rusages->usages); + } + FREEUP(rusages); +} + static void job_status_cleanup(xcecd_job_t* job) { job->job_id = 0; @@ -219,6 +234,20 @@ static void master_ckpt_result_callback(uint64_t job_id, int status, xcecd_job_ckpted(job_id, status, restore_cmd); } +static void master_rusages_callback(xce_job_rusages_t* rusages) +{ + if (!rusages) { + return; + } + + XCECD_LOG(XCE_LOG_DEBUG, "Received rusage for job %lu with %d metrics.", + rusages->jobid, rusages->count); + + xcecd_jobrusage_update(rusages); + + free_xce_job_rusages(rusages); +} + static int check_xce_server_dir(char* myfile) { char* xce_server_dir = getenv("XCE_SERVER_DIR"); @@ -242,7 +271,6 @@ static int check_xce_server_dir(char* myfile) } XCECD_LOG(XCE_LOG_INFO, "XCE_SERVER_DIR set to `/opt/xce/bin'"); - setenv("XCE_SERVER_DIR", "/opt/xce/bin", 0); return 1; } @@ -525,6 +553,7 @@ int main(int argc, char* argv[]) ipc_master_set_heartbeat_callback(master_heartbeat_callback); ipc_master_set_disconnect_callback(master_disconnect_callback); ipc_master_set_ckpt_callback(master_ckpt_result_callback); + ipc_master_set_rusages_callback(master_rusages_callback); XCECD_LOG(XCE_LOG_INFO, "XCECD started successfully."); diff --git a/src/worker/src/node_information.c b/src/worker/src/node_information.c index d513925c..7a69adb2 100644 --- a/src/worker/src/node_information.c +++ b/src/worker/src/node_information.c @@ -1,13 +1,31 @@ #include "node_information.h" #include "xcelog.h" +#include #include #include #include #include #include #include +#include +#include +#include +#include +#include -// 获取当前CPU使用情况 +// Static variables for 15-second load average calculation +static double last_15s_load = 0.0; +static time_t last_update_time = 0; +static int sample_count = 0; +static double load_sum = 0.0; +static pthread_mutex_t load_15s_mutex = PTHREAD_MUTEX_INITIALIZER; + +/** + * @brief get available CPU cores + * @param total_cores Total CPU cores + * @param used_cores Used CPU cores + * @return Available CPU cores + */ int get_cpu_usage(int total_cores, int used_cores) { if (used_cores == 0) { @@ -22,69 +40,980 @@ int get_cpu_usage(int total_cores, int used_cores) return total_cores - used_cores; } - // 其它状况 return 0; } -host_node_t* host_node_create(void) +/** + * @brief Read CPU utilization from /proc/stat + * @param ut Pointer to store CPU utilization percentage + * @return 0 on success, -1 on failure + */ +static int read_cpu_utilization(double* ut) { - host_node_t* node_ptr = (host_node_t*)calloc(1, sizeof(host_node_t)); - if (node_ptr == NULL) { - return NULL; - } + static unsigned long long prev_total = 0; + static unsigned long long prev_idle = 0; + static time_t last_read_time = 0; - node_ptr->node_name = NULL; - node_ptr->node_number = 0; - node_ptr->node_model_number = 0; - node_ptr->node_type_number = 0; - node_ptr->node_ip = NULL; + FILE* fp; + unsigned long long user, nice, system, idle, iowait, irq, softirq, steal; + unsigned long long total, total_idle, total_diff, idle_diff; + time_t current_time = time(NULL); - node_ptr->node_status.node_id = 0; - node_ptr->node_status.max_cpu_cores = 0; - node_ptr->node_status.max_memory = 0; - node_ptr->node_status.num_disk = 0; - node_ptr->node_status.max_swap = 0; - node_ptr->node_status.max_network = 0; - node_ptr->node_status.port_number = 0; - - /* TODO: initialize load_index and user_load_index */ - - node_ptr->use = -1; - node_ptr->expire_time = -1; - - return node_ptr; -} - -int host_node_update(host_node_t* node_ptr, host_node_t* new_node) -{ - if (node_ptr == NULL || new_node == NULL) { + if (!ut) { + XCECD_LOG(XCE_LOG_ERROR, + "Invalid parameter for CPU utilization reading"); return -1; } - /* TODO: add implementation here */ + fp = fopen("/proc/stat", "r"); + if (fp == NULL) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to open /proc/stat: %s", + strerror(errno)); + return -1; + } + + // Read the first line of /proc/stat (cpu total) + if (fscanf(fp, "cpu %llu %llu %llu %llu %llu %llu %llu %llu", &user, &nice, + &system, &idle, &iowait, &irq, &softirq, &steal) + != 8) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to parse CPU stats from /proc/stat"); + fclose(fp); + return -1; + } + fclose(fp); + + // Calculate total CPU time and idle time + total = user + nice + system + idle + iowait + irq + softirq + steal; + total_idle = idle + iowait; + + // If this is the first read, just store the values and return 0 + if (prev_total == 0 || last_read_time == 0 + || (current_time - last_read_time) < 1) { + prev_total = total; + prev_idle = total_idle; + last_read_time = current_time; + *ut = 0.0; + return 0; + } + + // Calculate differences + total_diff = total - prev_total; + idle_diff = total_idle - prev_idle; + + // Calculate CPU utilization percentage + if (total_diff == 0) { + *ut = 0.0; + } else { + *ut = ((double)(total_diff - idle_diff) / total_diff) * 100.0; + } + + // Update previous values + prev_total = total; + prev_idle = total_idle; + last_read_time = current_time; + return 0; } -void free_host_node(host_node_t* node_ptr) +/** + * @brief Read virtual memory paging rate from /proc/vmstat + * @param pg Pointer to store paging rate in pages per second + * @return 0 on success, -1 on failure + */ +static int read_paging_rate(double* pg) { - if (node_ptr == NULL) { - return; + static unsigned long long prev_pgpgin = 0; + static unsigned long long prev_pgpgout = 0; + static time_t last_read_time = 0; + + FILE* fp; + char line[256]; + unsigned long long pgpgin = 0, pgpgout = 0; + unsigned long long total_pages, prev_total, page_diff; + time_t current_time = time(NULL); + int found_pgpgin = 0, found_pgpgout = 0; + + if (!pg) { + XCECD_LOG(XCE_LOG_ERROR, "Invalid parameter for paging rate reading"); + return -1; } - FREEUP(node_ptr->node_name); - FREEUP(node_ptr->node_ip); - FREEUP(node_ptr->windows); - - for (int i = 0; i < 8; i++) { - FREEUP(node_ptr->week[i]); + fp = fopen("/proc/vmstat", "r"); + if (fp == NULL) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to open /proc/vmstat: %s", + strerror(errno)); + return -1; } - FREEUP(node_ptr->load_index); - FREEUP(node_ptr->user_load_index); - FREEUP(node_ptr->status); - FREEUP(node_ptr->busy_threshold); - FREEUP(node_ptr->resource_bitmap); - FREEUP(node_ptr->dynamic_resource_bitmap); + // Parse /proc/vmstat to find pgpgin and pgpgout values + while (fgets(line, sizeof(line), fp)) { + if (sscanf(line, "pgpgin %llu", &pgpgin) == 1) { + found_pgpgin = 1; + } else if (sscanf(line, "pgpgout %llu", &pgpgout) == 1) { + found_pgpgout = 1; + } - FREEUP(node_ptr); + // Break early if we found both values + if (found_pgpgin && found_pgpgout) { + break; + } + } + fclose(fp); + + if (!found_pgpgin || !found_pgpgout) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to find paging statistics in /proc/vmstat"); + return -1; + } + + // Calculate total pages (in + out) + total_pages = pgpgin + pgpgout; + + // If this is the first read, just store the values and return 0 + if ((prev_pgpgin == 0 && prev_pgpgout == 0) || last_read_time == 0 + || (current_time - last_read_time) < 1) { + prev_pgpgin = pgpgin; + prev_pgpgout = pgpgout; + last_read_time = current_time; + *pg = 0.0; + return 0; + } + + // Calculate differences + prev_total = prev_pgpgin + prev_pgpgout; + page_diff = total_pages - prev_total; + + // Calculate paging rate per second + time_t time_diff = current_time - last_read_time; + if (time_diff > 0) { + *pg = (double)page_diff / time_diff; + } else { + *pg = 0.0; + } + + // Update previous values + prev_pgpgin = pgpgin; + prev_pgpgout = pgpgout; + last_read_time = current_time; + + return 0; +} + +/** + * @brief Read system load averages from /proc/loadavg + * @param load1 Pointer to store 1-minute load average + * @param load5 Pointer to store 5-minute load average + * @param load15 Pointer to store 15-minute load average + * @return 0 on success, -1 on failure + */ +static int read_system_loadavg(double* load1, double* load5, double* load15) +{ + FILE* fp; + int ret; + + if (!load1 || !load5 || !load15) { + XCECD_LOG(XCE_LOG_ERROR, "Invalid parameters for load average reading"); + return -1; + } + + fp = fopen("/proc/loadavg", "r"); + if (fp == NULL) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to open /proc/loadavg: %s", + strerror(errno)); + return -1; + } + + // Read load averages: 1min 5min 15min running/total_processes last_pid + ret = fscanf(fp, "%lf %lf %lf", load1, load5, load15); + fclose(fp); + + if (ret != 3) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to parse load average from /proc/loadavg"); + return -1; + } + + return 0; +} + +/** + * @brief Get the 15 seconds average CPU run queue lengths. + * @return A string representing the 15 seconds load average. + */ +char* node_info_get_15s(void) +{ + pthread_mutex_lock(&load_15s_mutex); + + time_t current_time = time(NULL); + char* result = malloc(32); + double load1, load5, load15; + + if (result == NULL) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to allocate memory for 15s load average string"); + pthread_mutex_unlock(&load_15s_mutex); + return NULL; + } + + // Read current system load + if (read_system_loadavg(&load1, &load5, &load15) != 0) { + free(result); + pthread_mutex_unlock(&load_15s_mutex); + return NULL; + } + + // Initialize or reset if more than 15 seconds have passed + if (last_update_time == 0 || (current_time - last_update_time) >= 15) { + last_15s_load = load1; // Use 1-minute load as base for 15s calculation + last_update_time = current_time; + sample_count = 1; + load_sum = load1; + } else { + // Accumulate samples for averaging over 15 seconds + sample_count++; + load_sum += load1; + last_15s_load = load_sum / sample_count; + } + + int ret = snprintf(result, 32, "%.2f", last_15s_load); + if (ret < 0 || ret >= 32) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to format 15s load average string"); + free(result); + pthread_mutex_unlock(&load_15s_mutex); + return NULL; + } + + pthread_mutex_unlock(&load_15s_mutex); + return result; +} + +/** + * @brief Get the 1 minute average CPU run queue lengths. + * @return A string representing the 1 minute load average. + */ +char* node_info_get_1m(void) +{ + char* result = malloc(32); + double load1, load5, load15; + + if (result == NULL) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to allocate memory for 1m load average string"); + return NULL; + } + + // Read system load averages + if (read_system_loadavg(&load1, &load5, &load15) != 0) { + free(result); + return NULL; + } + + // Format the 1-minute load average as a string + int ret = snprintf(result, 32, "%.2f", load1); + if (ret < 0 || ret >= 32) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to format 1m load average string"); + free(result); + return NULL; + } + + return result; +} + +/** + * @brief Get the 15 minutes average CPU run queue lengths. + * @return A string representing the 15 minutes load average. + */ +char* node_info_get_15m(void) +{ + char* result = malloc(32); + double load1, load5, load15; + + if (result == NULL) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to allocate memory for 15m load average string"); + return NULL; + } + + // Read system load averages + if (read_system_loadavg(&load1, &load5, &load15) != 0) { + free(result); + return NULL; + } + + // Format the 15-minute load average as a string + int ret = snprintf(result, 32, "%.2f", load15); + if (ret < 0 || ret >= 32) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to format 15m load average string"); + free(result); + return NULL; + } + + return result; +} + +/** + * @brief The ut index measures CPU utilization, which is the percentage of + * time spent running system and user code. A host with no process running has a + * ut value of 0 percent; a host on which the CPU is completely loaded has + * a ut of 100 percent. + * @return A string representing the CPU utilization percentage. + */ +char* node_info_get_ut(void) +{ + char* result = malloc(32); + double ut; + + if (result == NULL) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to allocate memory for CPU utilization string"); + return NULL; + } + + // Read CPU utilization + if (read_cpu_utilization(&ut) != 0) { + free(result); + return NULL; + } + + // Format the CPU utilization as a string + int ret = snprintf(result, 32, "%.2f", ut); + if (ret < 0 || ret >= 32) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to format CPU utilization string"); + free(result); + return NULL; + } + + return result; +} + +/** + * @brief The pg index gives the virtual memory paging rate in pages per + * second. + * @return A string representing the paging rate. + */ +char* node_info_get_pg(void) +{ + char* result = malloc(32); + double pg; + + if (result == NULL) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to allocate memory for paging rate string"); + return NULL; + } + + // Read paging rate + if (read_paging_rate(&pg) != 0) { + free(result); + return NULL; + } + + // Format the paging rate as a string + int ret = snprintf(result, 32, "%.2f", pg); + if (ret < 0 || ret >= 32) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to format paging rate string"); + free(result); + return NULL; + } + + return result; +} + +/** + * @brief Read the number of logged-in users + * @param ls Pointer to store the number of logged-in users + * @return 0 on success, -1 on failure + */ +static int read_logged_in_users(int* ls) +{ + struct utmp* utmp_entry; + char* unique_users[1024]; + int unique_count = 0; + int i, found; + + if (!ls) { + XCECD_LOG(XCE_LOG_ERROR, + "Invalid parameter for logged-in users reading"); + return -1; + } + + setutent(); + + while ((utmp_entry = getutent()) != NULL) { + if (utmp_entry->ut_type == USER_PROCESS) { + // Create a null-terminated copy of the username + char username[UT_NAMESIZE + 1]; + strncpy(username, utmp_entry->ut_user, UT_NAMESIZE); + username[UT_NAMESIZE] = '\0'; + + // Skip empty usernames + if (strlen(username) == 0) { + continue; + } + + found = 0; + for (i = 0; i < unique_count; i++) { + if (strcmp(unique_users[i], username) == 0) { + found = 1; + break; + } + } + + if (!found && unique_count < 1024) { + unique_users[unique_count] = strdup(username); + if (unique_users[unique_count] != NULL) { + unique_count++; + } + } + } + } + + endutent(); + + *ls = unique_count; + + for (i = 0; i < unique_count; i++) { + free(unique_users[i]); + } + + return 0; +} + +/** + * @brief The ls index gives the number of users logged in. Each user is + * counted once, no matter how many times they have logged into the host. + * @return A string representing the number of logged-in users. + */ +char* node_info_get_ls(void) +{ + char* result = malloc(32); + int ls; + + if (result == NULL) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to allocate memory for logged-in users string"); + return NULL; + } + + // Read number of logged-in users + if (read_logged_in_users(&ls) != 0) { + free(result); + return NULL; + } + + // Format the number of logged-in users as a string + int ret = snprintf(result, 32, "%d", ls); + if (ret < 0 || ret >= 32) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to format logged-in users string"); + free(result); + return NULL; + } + + return result; +} + +/** + * @brief Read the interactive idle time of the host + * @param it Pointer to store the interactive idle time in minutes + * @return 0 on success, -1 on failure + */ +static int read_interactive_idle_time(double* it) +{ + struct utmp* utmp_entry; + time_t current_time = time(NULL); + time_t min_idle_time = current_time; + time_t last_activity = 0; + int found_interactive_user = 0; + + if (!it) { + XCECD_LOG(XCE_LOG_ERROR, + "Invalid parameter for interactive idle time reading"); + return -1; + } + + setutent(); + + while ((utmp_entry = getutent()) != NULL) { + if (utmp_entry->ut_type == USER_PROCESS) { + char tty_path[256]; + struct stat tty_stat; + + if (strncmp(utmp_entry->ut_line, "pts/", 4) == 0) { + snprintf(tty_path, sizeof(tty_path), "/dev/%s", + utmp_entry->ut_line); + } else if (strncmp(utmp_entry->ut_line, "tty", 3) == 0) { + snprintf(tty_path, sizeof(tty_path), "/dev/%s", + utmp_entry->ut_line); + } else { + snprintf(tty_path, sizeof(tty_path), "/dev/%s", + utmp_entry->ut_line); + } + + if (stat(tty_path, &tty_stat) == 0) { + found_interactive_user = 1; + + time_t tty_activity = tty_stat.st_atime > tty_stat.st_mtime + ? tty_stat.st_atime + : tty_stat.st_mtime; + + if (tty_activity > last_activity) { + last_activity = tty_activity; + } + + time_t idle_time = current_time - tty_activity; + if (idle_time + < (min_idle_time - current_time + min_idle_time)) { + min_idle_time = tty_activity; + } + } + } + } + + endutent(); + + if (!found_interactive_user) { + FILE* fp = fopen("/proc/uptime", "r"); + if (fp != NULL) { + double uptime; + if (fscanf(fp, "%lf", &uptime) == 1) { + *it = uptime / 60.0; // convert to minutes + } else { + *it = 0.0; + } + fclose(fp); + } else { + *it = 0.0; + } + } else { + // Calculate minimum idle time (minutes) + double idle_minutes = (double)(current_time - last_activity) / 60.0; + *it = idle_minutes < 0 ? 0.0 : idle_minutes; + } + + return 0; +} + +/** + * @brief The it index is the interactive idle time of the host, in minutes. + * @return A string representing the interactive idle time. + */ +char* node_info_get_it(void) +{ + char* result = malloc(32); + double it; + + if (result == NULL) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to allocate memory for interactive idle time string"); + return NULL; + } + + // Read interactive idle time + if (read_interactive_idle_time(&it) != 0) { + free(result); + return NULL; + } + + // Format the interactive idle time as a string + int ret = snprintf(result, 32, "%.2f", it); + if (ret < 0 || ret >= 32) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to format interactive idle time string"); + free(result); + return NULL; + } + + return result; +} + +/** + * @brief Read swap space information from /proc/meminfo + * @param swap_total Pointer to store total swap space in MB + * @param swap_free Pointer to store free swap space in MB + * @param swap_used Pointer to store used swap space in MB + * @return 0 on success, -1 on failure + */ +static int read_swap_info(double* swap_total, double* swap_free, + double* swap_used) +{ + FILE* fp; + char line[256]; + unsigned long long total_kb = 0, free_kb = 0; + int found_total = 0, found_free = 0; + + if (!swap_total || !swap_free || !swap_used) { + XCECD_LOG(XCE_LOG_ERROR, + "Invalid parameters for swap information reading"); + return -1; + } + + fp = fopen("/proc/meminfo", "r"); + if (fp == NULL) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to open /proc/meminfo: %s", + strerror(errno)); + return -1; + } + + // Parse /proc/meminfo to find SwapTotal and SwapFree + while (fgets(line, sizeof(line), fp)) { + if (sscanf(line, "SwapTotal: %llu kB", &total_kb) == 1) { + found_total = 1; + } else if (sscanf(line, "SwapFree: %llu kB", &free_kb) == 1) { + found_free = 1; + } + + // Break early if we found both values + if (found_total && found_free) { + break; + } + } + fclose(fp); + + if (!found_total || !found_free) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to find swap information in /proc/meminfo"); + return -1; + } + + // Convert from KB to MB + *swap_total = (double)total_kb / 1024.0; + *swap_free = (double)free_kb / 1024.0; + *swap_used = *swap_total - *swap_free; + + return 0; +} + +/** + * @brief The swp index gives the currently available virtual memory + * (swap space) in MB. + * @return A string representing the available swap space in megabytes. + */ +char* node_info_get_swap(void) +{ + char* result = malloc(64); + double swap_total, swap_free, swap_used; + + if (result == NULL) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to allocate memory for swap information string"); + return NULL; + } + + // Read swap information + if (read_swap_info(&swap_total, &swap_free, &swap_used) != 0) { + free(result); + return NULL; + } + + // Format the available swap space as a string (in MB) + int ret = snprintf(result, 64, "%.2f", swap_free); + if (ret < 0 || ret >= 64) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to format swap information string"); + free(result); + return NULL; + } + + return result; +} + +/** + * @brief Read physical memory information from /proc/meminfo + * @param mem_total Pointer to store total physical memory in MB + * @param mem_free Pointer to store free physical memory in MB + * @param mem_used Pointer to store used physical memory in MB + * @return 0 on success, -1 on failure + */ +static int read_memory_info(double* mem_total, double* mem_free, + double* mem_used) +{ + FILE* fp; + char line[256]; + unsigned long long total_kb = 0, free_kb = 0, available_kb = 0, + buffers_kb = 0, cached_kb = 0; + int found_total = 0, found_free = 0, found_available = 0, found_buffers = 0, + found_cached = 0; + + if (!mem_total || !mem_free || !mem_used) { + XCECD_LOG(XCE_LOG_ERROR, + "Invalid parameters for memory information reading"); + return -1; + } + + fp = fopen("/proc/meminfo", "r"); + if (fp == NULL) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to open /proc/meminfo: %s", + strerror(errno)); + return -1; + } + + // Parse /proc/meminfo to find memory statistics + while (fgets(line, sizeof(line), fp)) { + if (sscanf(line, "MemTotal: %llu kB", &total_kb) == 1) { + found_total = 1; + } else if (sscanf(line, "MemFree: %llu kB", &free_kb) == 1) { + found_free = 1; + } else if (sscanf(line, "MemAvailable: %llu kB", &available_kb) == 1) { + found_available = 1; + } else if (sscanf(line, "Buffers: %llu kB", &buffers_kb) == 1) { + found_buffers = 1; + } else if (sscanf(line, "Cached: %llu kB", &cached_kb) == 1) { + found_cached = 1; + } + + // Break early if we found all required values + if (found_total && found_free && found_available && found_buffers + && found_cached) { + break; + } + } + fclose(fp); + + if (!found_total || !found_free) { + XCECD_LOG( + XCE_LOG_ERROR, + "Failed to find required memory information in /proc/meminfo"); + return -1; + } + + // Convert from KB to MB + *mem_total = (double)total_kb / 1024.0; + + // Use MemAvailable if available (Linux 3.14+), otherwise calculate + if (found_available) { + *mem_free = (double)available_kb / 1024.0; + } else { + // Fallback calculation: MemFree + Buffers + Cached + unsigned long long estimated_available = free_kb; + if (found_buffers) { + estimated_available += buffers_kb; + } + if (found_cached) { + estimated_available += cached_kb; + } + *mem_free = (double)estimated_available / 1024.0; + } + + *mem_used = *mem_total - *mem_free; + + return 0; +} + +/** + * @brief The mem index gives the currently available physical memory in MB. + * @return A string representing the available physical memory in megabytes. + */ +char* node_info_get_mem(void) +{ + char* result = malloc(64); + double mem_total, mem_free, mem_used; + + if (result == NULL) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to allocate memory for memory information string"); + return NULL; + } + + // Read memory information + if (read_memory_info(&mem_total, &mem_free, &mem_used) != 0) { + free(result); + return NULL; + } + + // Format the available memory as a string (in MB) + int ret = snprintf(result, 64, "%.2f", mem_free); + if (ret < 0 || ret >= 64) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to format memory information string"); + free(result); + return NULL; + } + + return result; +} + +/** + * @brief The tmp index is the space available in MB or in units set + * on the file system that contains the temporary directory. + * @return A string representing the available space in temporary directory. + */ +char* node_info_get_tmp(void) +{ + char* result = malloc(64); + struct statvfs fs_stat; + + if (result == NULL) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to allocate memory for tmp space string"); + return NULL; + } + + // Get filesystem statistics for /tmp + if (statvfs("/tmp", &fs_stat) != 0) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to get filesystem information for /tmp: %s", + strerror(errno)); + free(result); + return NULL; + } + + // Calculate free space in bytes + unsigned long long free_bytes + = (unsigned long long)fs_stat.f_bavail * fs_stat.f_frsize; + + // Convert to MB + double tmp_free_mb = (double)free_bytes / (1024.0 * 1024.0); + + // Format the available tmp space as a string (in MB) + int ret = snprintf(result, 64, "%.2f", tmp_free_mb); + if (ret < 0 || ret >= 64) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to format tmp space string"); + free(result); + return NULL; + } + + return result; +} + +/** + * @brief Read disk I/O statistics from /proc/diskstats + * @param io_rate Pointer to store I/O rate in KB per second + * @return 0 on success, -1 on failure + */ +static int read_disk_io_rate(double* io_rate) +{ + static unsigned long long prev_read_sectors = 0; + static unsigned long long prev_write_sectors = 0; + static time_t last_read_time = 0; + + FILE* fp; + char line[512]; + unsigned long long total_read_sectors = 0, total_write_sectors = 0; + unsigned long long prev_total, current_total, sector_diff; + time_t current_time = time(NULL); + int found_disk = 0; + + if (!io_rate) { + XCECD_LOG(XCE_LOG_ERROR, "Invalid parameter for I/O rate reading"); + return -1; + } + + fp = fopen("/proc/diskstats", "r"); + if (fp == NULL) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to open /proc/diskstats: %s", + strerror(errno)); + return -1; + } + + // Parse /proc/diskstats to accumulate I/O statistics for all physical disks + while (fgets(line, sizeof(line), fp)) { + int major, minor; + char device_name[64]; + unsigned long long reads_completed, reads_merged, sectors_read, + read_time; + unsigned long long writes_completed, writes_merged, sectors_written, + write_time; + unsigned long long ios_in_progress, total_ios_time, weighted_ios_time; + + // Parse diskstats line format + int parsed = sscanf( + line, + "%d %d %s %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu", + &major, &minor, device_name, &reads_completed, &reads_merged, + §ors_read, &read_time, &writes_completed, &writes_merged, + §ors_written, &write_time, &ios_in_progress, &total_ios_time, + &weighted_ios_time); + + if (parsed >= 14) { + // Only count physical disks (skip partitions and virtual devices) + // Physical disks typically have names like sda, sdb, hda, etc. (no + // numbers at end) Skip loop devices, ram disks, and partitions + if (strncmp(device_name, "loop", 4) == 0 + || strncmp(device_name, "ram", 3) == 0 + || strncmp(device_name, "dm-", 3) == 0 + || strstr(device_name, "p") != NULL + || // nvme partitions (nvme0n1p1) + (strlen(device_name) > 3 + && device_name[strlen(device_name) - 1] >= '0' + && device_name[strlen(device_name) - 1] <= '9')) { + continue; + } + + total_read_sectors += sectors_read; + total_write_sectors += sectors_written; + found_disk = 1; + } + } + fclose(fp); + + if (!found_disk) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to find any disk statistics in /proc/diskstats"); + return -1; + } + + current_total = total_read_sectors + total_write_sectors; + + // If this is the first read, just store the values and return 0 + if ((prev_read_sectors == 0 && prev_write_sectors == 0) + || last_read_time == 0 || (current_time - last_read_time) < 1) { + prev_read_sectors = total_read_sectors; + prev_write_sectors = total_write_sectors; + last_read_time = current_time; + *io_rate = 0.0; + return 0; + } + + // Calculate differences + prev_total = prev_read_sectors + prev_write_sectors; + sector_diff = current_total - prev_total; + + // Calculate I/O rate in KB per second + // Each sector is typically 512 bytes + time_t time_diff = current_time - last_read_time; + if (time_diff > 0) { + *io_rate = (double)(sector_diff * 512) + / (1024.0 * time_diff); // Convert to KB/s + } else { + *io_rate = 0.0; + } + + // Update previous values + prev_read_sectors = total_read_sectors; + prev_write_sectors = total_write_sectors; + last_read_time = current_time; + + return 0; +} + +/** + * @brief The io index measures I/O throughput to disks attached directly to + * this host, in KB per second. It does not include I/O to disks that are + * mounted from other hosts. + * @return A string representing the I/O throughput in KB per second. + */ +char* node_info_get_io(void) +{ + char* result = malloc(32); + double io_rate; + + if (result == NULL) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to allocate memory for I/O rate string"); + return NULL; + } + + // Read disk I/O rate + if (read_disk_io_rate(&io_rate) != 0) { + free(result); + return NULL; + } + + // Format the I/O rate as a string (in KB/s) + int ret = snprintf(result, 32, "%.2f", io_rate); + if (ret < 0 || ret >= 32) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to format I/O rate string"); + free(result); + return NULL; + } + + return result; } \ No newline at end of file diff --git a/src/worker/src/resource_sync.c b/src/worker/src/resource_sync.c index 26c28f89..d852463c 100644 --- a/src/worker/src/resource_sync.c +++ b/src/worker/src/resource_sync.c @@ -10,27 +10,326 @@ #include #include -#define SYNC_INTERVAL 1 +#define SYNC_INTERVAL 1 +#define LOAD_INDICES_INTERVAL_DEFAULT 15 +#define MAX_LOAD_INDICES 64 +#define BUILTIN_LOAD_INDICES_COUNT 11 struct resource_sync { - pthread_t thread; - _Atomic bool running; - _Atomic bool updating; - struct cluster_node* my_cluster; - struct host_node* my_node; - struct client_node* my_client; + pthread_t thread; + _Atomic bool running; + _Atomic bool updating; + int load_report_counter; }; +typedef struct { + char* name; + int valtype; + char* (*getter_func)(void); + bool is_builtin; +} load_index_info_t; + +static load_index_info_t* g_load_indices = NULL; +static int g_load_indices_count = 0; +static int g_load_indices_capacity = 0; +static pthread_mutex_t g_load_indices_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void free_resvals(xce_resval_t* vals, int count) +{ + if (vals == NULL) + return; + + for (int i = 0; i < count; i++) { + if (vals[i].name) { + FREEUP(vals[i].name); + } + if (vals[i].host) { + FREEUP(vals[i].host); + } + if (vals[i].val) { + if (vals[i].val->which == Value_u_svalue && vals[i].val->u.svalue) { + FREEUP(vals[i].val->u.svalue); + } + FREEUP(vals[i].val); + } + } + FREEUP(vals); +} + +static void init_builtin_load_indices(void) +{ + static bool initialized = false; + if (initialized) + return; + + g_load_indices_capacity = MAX_LOAD_INDICES; + g_load_indices = (load_index_info_t*)calloc(g_load_indices_capacity, + sizeof(load_index_info_t)); + + // register built-in load indices + resource_sync_register_load_index("r15s", Value_u_fvalue, node_info_get_15s, + true); + resource_sync_register_load_index("r1m", Value_u_fvalue, node_info_get_1m, + true); + resource_sync_register_load_index("r15m", Value_u_fvalue, node_info_get_15m, + true); + resource_sync_register_load_index("ut", Value_u_fvalue, node_info_get_ut, + true); + resource_sync_register_load_index("pg", Value_u_fvalue, node_info_get_pg, + true); + resource_sync_register_load_index("ls", Value_u_fvalue, node_info_get_ls, + true); + resource_sync_register_load_index("it", Value_u_fvalue, node_info_get_it, + true); + resource_sync_register_load_index("swp", Value_u_fvalue, node_info_get_swap, + true); + resource_sync_register_load_index("mem", Value_u_fvalue, node_info_get_mem, + true); + resource_sync_register_load_index("tmp", Value_u_fvalue, node_info_get_tmp, + true); + resource_sync_register_load_index("io", Value_u_fvalue, node_info_get_io, + true); + + initialized = true; +} + +/** + * @brief Registers a new load index. + * @param name The name of the load index. + * @param valtype The type of the load index value. + * @param getter_func Function pointer to retrieve the load index value. + * @param is_builtin Whether the load index is built-in or custom. + * @return true on success, false on failure. + */ +bool resource_sync_register_load_index(const char* name, int valtype, + char* (*getter_func)(void), + bool is_builtin) +{ + if (name == NULL || getter_func == NULL) { + return false; + } + + pthread_mutex_lock(&g_load_indices_mutex); + + for (int i = 0; i < g_load_indices_count; i++) { + if (strcmp(g_load_indices[i].name, name) == 0) { + XCECD_LOG(XCE_LOG_WARN, "Load index '%s' already exists", name); + pthread_mutex_unlock(&g_load_indices_mutex); + return false; + } + } + + if (g_load_indices_count >= g_load_indices_capacity) { + XCECD_LOG(XCE_LOG_ERROR, "Maximum load indices capacity reached (%d)", + g_load_indices_capacity); + pthread_mutex_unlock(&g_load_indices_mutex); + return false; + } + + g_load_indices[g_load_indices_count].name = strdup(name); + g_load_indices[g_load_indices_count].valtype = valtype; + g_load_indices[g_load_indices_count].getter_func = getter_func; + g_load_indices[g_load_indices_count].is_builtin = is_builtin; + g_load_indices_count++; + + pthread_mutex_unlock(&g_load_indices_mutex); + + XCECD_LOG(XCE_LOG_DEBUG3, + "Registered load index '%s' (type: %d, builtin: %s)", name, + valtype, is_builtin ? "yes" : "no"); + return true; +} + +/** + * @brief Unregisters a load index. + * @param name The name of the load index to unregister. + * @return true on success, false on failure. + */ +bool resource_sync_unregister_load_index(const char* name) +{ + if (name == NULL) { + return false; + } + + pthread_mutex_lock(&g_load_indices_mutex); + + for (int i = 0; i < g_load_indices_count; i++) { + if (strcmp(g_load_indices[i].name, name) == 0) { + if (g_load_indices[i].is_builtin) { + XCECD_LOG(XCE_LOG_WARN, + "Cannot unregister builtin load index '%s'", name); + pthread_mutex_unlock(&g_load_indices_mutex); + return false; + } + + FREEUP(g_load_indices[i].name); + // Shift remaining indices + for (int j = i; j < g_load_indices_count - 1; j++) { + g_load_indices[j] = g_load_indices[j + 1]; + } + g_load_indices_count--; + + pthread_mutex_unlock(&g_load_indices_mutex); + XCECD_LOG(XCE_LOG_DEBUG3, "Unregistered load index '%s'", name); + return true; + } + } + + pthread_mutex_unlock(&g_load_indices_mutex); + XCECD_LOG(XCE_LOG_WARN, "Load index '%s' not found", name); + return false; +} + +static xce_resval_t* collect_load_indices(int* count) +{ + if (g_load_indices == NULL) { + init_builtin_load_indices(); + } + + pthread_mutex_lock(&g_load_indices_mutex); + + int rescount = g_load_indices_count; + if (rescount == 0) { + pthread_mutex_unlock(&g_load_indices_mutex); + *count = 0; + return NULL; + } + + xce_resval_t* vals = (xce_resval_t*)calloc(rescount, sizeof(xce_resval_t)); + if (vals == NULL) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to allocate memory for load indices"); + pthread_mutex_unlock(&g_load_indices_mutex); + *count = 0; + return NULL; + } + + char hostname[256]; + if (gethostname(hostname, sizeof(hostname)) != 0) { + strcpy(hostname, "unknown"); + } + + for (int i = 0; i < rescount; i++) { + vals[i].name = strdup(g_load_indices[i].name); + vals[i].host = strdup(hostname); + vals[i].val = (xce_val_t*)calloc(1, sizeof(xce_val_t)); + + if (vals[i].name == NULL || vals[i].host == NULL + || vals[i].val == NULL) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to allocate memory for load index %s", + g_load_indices[i].name); + free_resvals(vals, i + 1); + pthread_mutex_unlock(&g_load_indices_mutex); + *count = 0; + return NULL; + } + + char* val_str = g_load_indices[i].getter_func(); + + switch (g_load_indices[i].valtype) { + case Value_u_fvalue: + vals[i].val->which = g_load_indices[i].valtype; + if (val_str == NULL) { + vals[i].val->u.fvalue = 0.0; + } else { + vals[i].val->u.fvalue = atof(val_str); + FREEUP(val_str); + } + break; + case Value_u_ivalue32: + vals[i].val->which = g_load_indices[i].valtype; + if (val_str == NULL) { + vals[i].val->u.ivalue32 = 0; + } else { + vals[i].val->u.ivalue32 = atoi(val_str); + FREEUP(val_str); + } + break; + case Value_u_svalue: + vals[i].val->which = g_load_indices[i].valtype; + if (val_str == NULL) { + vals[i].val->u.svalue = strdup(""); + } else { + vals[i].val->u.svalue = val_str; + } + break; + default: + XCECD_LOG(XCE_LOG_WARN, "Unsupported value type for %s", + g_load_indices[i].name); + vals[i].val->which = Value_u_fvalue; + vals[i].val->u.fvalue = 0.0; + if (val_str) + FREEUP(val_str); + break; + } + } + + pthread_mutex_unlock(&g_load_indices_mutex); + + *count = rescount; + return vals; +} + +/** + * @brief Cleans up all registered load indices. + */ +static void resource_sync_cleanup_load_indices(void) +{ + pthread_mutex_lock(&g_load_indices_mutex); + + if (g_load_indices != NULL) { + for (int i = 0; i < g_load_indices_count; i++) { + FREEUP(g_load_indices[i].name); + } + FREEUP(g_load_indices); + g_load_indices = NULL; + g_load_indices_count = 0; + g_load_indices_capacity = 0; + } + + pthread_mutex_unlock(&g_load_indices_mutex); +} + static void* resource_sync_thread(void* arg) { - resource_sync_t* sync = (resource_sync_t*)arg; + resource_sync_t* sync = (resource_sync_t*)arg; + xce_resval_t* vals = NULL; + xce_resval_t** valptrs = NULL; + int vals_count = 0; while (atomic_load(&sync->running)) { - /* TODO：资源更新，添加逻辑 */ - resource_sync_update(sync); - /* TODO：资源汇总，添加逻辑 */ + sync->load_report_counter++; + if (sync->load_report_counter >= LOAD_INDICES_INTERVAL_DEFAULT) { + vals = collect_load_indices(&vals_count); + if (vals == NULL || vals_count <= 0) { + XCECD_LOG(XCE_LOG_ERROR, "No load indices to report"); + sync->load_report_counter = 0; + sleep(SYNC_INTERVAL); + continue; + } + + valptrs = (xce_resval_t**)calloc(vals_count, sizeof(xce_resval_t*)); + if (valptrs == NULL) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to allocate memory for value pointers"); + free_resvals(vals, vals_count); + sync->load_report_counter = 0; + sleep(SYNC_INTERVAL); + continue; + } + + for (int i = 0; i < vals_count; i++) { + valptrs[i] = &vals[i]; + } + + xcecd_send_load_indices(valptrs, vals_count); + + FREEUP(valptrs); + free_resvals(vals, vals_count); + sync->load_report_counter = 0; + } sleep(SYNC_INTERVAL); } @@ -48,10 +347,8 @@ resource_sync_t* resource_sync_create(void) } atomic_store(&sync->running, false); atomic_store(&sync->updating, false); - sync->my_cluster = NULL; - sync->my_node = host_node_create(); - sync->my_client = NULL; - sync->thread = 0; + sync->thread = 0; + sync->load_report_counter = 0; XCECD_LOG(XCE_LOG_DEBUG, "Resource sync module created successfully"); return sync; @@ -64,9 +361,9 @@ void resource_sync_destroy(resource_sync_t* sync) XCECD_LOG(XCE_LOG_ERROR, "Invalid resource sync module"); return; } - - /* 资源销毁 TODO：先释放host，client等结构 */ - FREEUP(sync->my_node); + resource_sync_stop(sync); + resource_sync_cleanup_load_indices(); + XCECD_LOG(XCE_LOG_DEBUG, "Resource sync module destroyed successfully"); FREEUP(sync); } @@ -104,7 +401,6 @@ void resource_sync_stop(resource_sync_t* sync) bool resource_sync_update(resource_sync_t* sync) { - // XCECD_LOG(XCE_LOG_DEBUG, "Updating resource sync module..."); if (sync == NULL) { XCECD_LOG(XCE_LOG_ERROR, "Invalid resource sync module"); return false; @@ -112,29 +408,23 @@ bool resource_sync_update(resource_sync_t* sync) atomic_store(&sync->updating, true); - /* TODO：资源更新，添加逻辑 */ - sync->my_node->node_status.max_cpu_cores - = hardware_monitor_get_cpu_cores(xcecd_ctx.hardware_monitor); + /* Currently there is no resource need to update */ atomic_store(&sync->updating, false); - // XCECD_LOG(XCE_LOG_DEBUG, "Resource sync module updated successfully"); return true; } -void resource_sync_read(resource_sync_t* sync, host_node_t** node) +void resource_sync_read(resource_sync_t* sync, void* data) { if (!sync) { return; } while (atomic_load(&sync->updating)) { - /* 等待更新完成 */ + /* wait sync to complete */ usleep(1000); } - /* 读取资源同步状态 - * TODO: 添加读取逻辑 - */ - *node = sync->my_node; + /* Currently there is no resource need to read */ } \ No newline at end of file diff --git a/src/worker/src/uv_ipc.c b/src/worker/src/uv_ipc.c index 2089d722..7db55421 100644 --- a/src/worker/src/uv_ipc.c +++ b/src/worker/src/uv_ipc.c @@ -32,6 +32,7 @@ static worker_reconnect_callback_t worker_reconnect_callback = NULL; static job_signal_callback_t worker_signal_callback = NULL; static job_ckpt_callback_t worker_ckpt_callback = NULL; static job_ckpt_result_callback_t master_ckpt_result_callback = NULL; +static job_rusages_callback_t master_rusages_callback = NULL; static uv_timer_t* exit_timer = NULL; @@ -506,6 +507,61 @@ static void on_master_read(uv_stream_t* client, ssize_t nread, msg->data.ckpt_result.status, msg->data.ckpt_result.restore_cmd); } + } else if (msg->type == MSG_JOB_RUSAGES) { + XCECD_LOG(XCE_LOG_DEBUG, "Received rusage info for job %lu", + msg->data.rusages.job_id); + if (master_rusages_callback) { + // deserialize payload + xce_job_rusages_t* rusages + = (xce_job_rusages_t*)calloc(1, sizeof(xce_job_rusages_t)); + if (rusages) { + char* p = msg->data.rusages.payload; + rusages->jobid = msg->data.rusages.job_id; + + memcpy(&rusages->npids, p, sizeof(int)); + p += sizeof(int); + if (rusages->npids > 0) { + rusages->pids + = (int*)calloc(rusages->npids, sizeof(int)); + memcpy(rusages->pids, p, rusages->npids * sizeof(int)); + p += rusages->npids * sizeof(int); + } + + memcpy(&rusages->npgids, p, sizeof(int)); + p += sizeof(int); + if (rusages->npgids > 0) { + rusages->pgids + = (int*)calloc(rusages->npgids, sizeof(int)); + memcpy(rusages->pgids, p, + rusages->npgids * sizeof(int)); + p += rusages->npgids * sizeof(int); + } + + memcpy(&rusages->count, p, sizeof(int)); + p += sizeof(int); + if (rusages->count > 0) { + rusages->usages = (xce_rusage_t**)calloc( + rusages->count, sizeof(xce_rusage_t*)); + for (int i = 0; i < rusages->count; i++) { + rusages->usages[i] = (xce_rusage_t*)calloc( + 1, sizeof(xce_rusage_t)); + size_t name_len; + memcpy(&name_len, p, sizeof(size_t)); + p += sizeof(size_t); + rusages->usages[i]->resname + = (char*)calloc(name_len + 1, sizeof(char)); + memcpy(rusages->usages[i]->resname, p, name_len); + p += name_len; + memcpy(&rusages->usages[i]->value, p, + sizeof(float)); + p += sizeof(float); + } + } + master_rusages_callback(rusages); + // note: master_rusages_callback needs to free rusages and + // its internal pointers + } + } } else if (msg->type == MSG_REMOTE_START_JOB) { XCECD_LOG(XCE_LOG_DEBUG, "Received remote start request for host '%s'", @@ -1361,7 +1417,21 @@ void ipc_master_set_ckpt_callback(job_ckpt_callback_t callback) } /** - * + * @brief master set rusages callback + * @param callback + * @return void + */ +void ipc_master_set_rusages_callback(job_rusages_callback_t callback) +{ + master_rusages_callback = callback; +} + +/** + * @brief worker send checkpoint result + * @param job_id - job ID + * @param status - checkpoint status (0 success, non-zero failure) + * @param restore_cmd - command to restore the job + * @return bool - true if sent successfully */ bool ipc_worker_send_ckpt_result(uint64_t job_id, int status, const char* restore_cmd) @@ -1396,3 +1466,79 @@ bool ipc_worker_send_ckpt_result(uint64_t job_id, int status, return true; } + +/** + * @brief worker send rusage info to master + * @param rusages - rusage data + * @return bool - true if sent successfully + */ +bool ipc_worker_send_rusages(xce_job_rusages_t* rusages) +{ + if (!worker_connected || !rusages) + return false; + + ipc_message_t msg; + memset(&msg, 0, sizeof(msg)); + msg.type = MSG_JOB_RUSAGES; + msg.data.rusages.job_id = rusages->jobid; + + // serialize payload + char* p = msg.data.rusages.payload; + char* end = p + MAX_RUSAGE_PAYLOAD; + + // serialize npids and pids + memcpy(p, &rusages->npids, sizeof(int)); + p += sizeof(int); + if (p + rusages->npids * sizeof(int) > end) + return false; // check buffer overflow + if (rusages->npids > 0) { + memcpy(p, rusages->pids, rusages->npids * sizeof(int)); + p += rusages->npids * sizeof(int); + } + + // serialize npgids and pgids + memcpy(p, &rusages->npgids, sizeof(int)); + p += sizeof(int); + if (p + rusages->npgids * sizeof(int) > end) + return false; + if (rusages->npgids > 0) { + memcpy(p, rusages->pgids, rusages->npgids * sizeof(int)); + p += rusages->npgids * sizeof(int); + } + + // serialize count and usages + memcpy(p, &rusages->count, sizeof(int)); + p += sizeof(int); + for (int i = 0; i < rusages->count; i++) { + xce_rusage_t* usage = rusages->usages[i]; + size_t name_len = strlen(usage->resname); + if (p + sizeof(size_t) + name_len + sizeof(float) > end) + return false; + + memcpy(p, &name_len, sizeof(size_t)); + p += sizeof(size_t); + memcpy(p, usage->resname, name_len); + p += name_len; + memcpy(p, &usage->value, sizeof(float)); + p += sizeof(float); + } + + msg.data.rusages.payload_size = p - msg.data.rusages.payload; + + uv_write_t* req = (uv_write_t*)calloc(1, sizeof(uv_write_t)); + if (req) { + uv_buf_t buf = uv_buf_init((char*)&msg, sizeof(msg)); + int r = uv_write(req, (uv_stream_t*)&worker_pipe, &buf, 1, + on_write_complete); + if (r < 0) { + XCECD_WORKER_LOG(XCE_LOG_ERROR, "Failed to send rusage info: %s", + uv_strerror(r)); + free(req); + return false; + } + } + + XCECD_WORKER_LOG(XCE_LOG_DEBUG, "Sent rusage info for job %lu", + rusages->jobid); + return true; +} diff --git a/src/worker/src/uv_proc.c b/src/worker/src/uv_proc.c index be3a6077..2ddc5aa8 100644 --- a/src/worker/src/uv_proc.c +++ b/src/worker/src/uv_proc.c @@ -1,8 +1,5 @@ #include "uv_proc.h" -#include "utils.h" -#include "xcecd_utils.h" #include "xcelog.h" - #include #include #include @@ -11,6 +8,8 @@ #include #include +#include "utils.h" + static int make_job_output_path(char* buf, size_t bufsize, const char* dir, uint64_t jobid, int task) { @@ -25,6 +24,7 @@ static int make_job_output_path(char* buf, size_t bufsize, const char* dir, } fmt_jobid(idstring, sizeof(idstring) - 1, jobid); + return snprintf(buf, bufsize, "%s/job_%s_task%d.out", dir, idstring, task); } @@ -261,9 +261,13 @@ int uv_proc_exec(uv_proc_manager_t* manager, const char* command, } char job_output[256]; - +#ifdef AUTOSCALER_TEST + make_job_output_path(job_output, sizeof(job_output) - 1, "/shared/log", + manager->job_id, i); +#else make_job_output_path(job_output, sizeof(job_output) - 1, "/tmp", manager->job_id, i); +#endif uv_stdio_container_t stdio[3]; uv_fs_t file_req; diff --git a/src/worker/src/xcecd.c b/src/worker/src/xcecd.c index 20414c6c..0142957b 100644 --- a/src/worker/src/xcecd.c +++ b/src/worker/src/xcecd.c @@ -12,8 +12,9 @@ #include #include +#if !defined(DBUS_MANAGER) #include "process_worker_manager.h" -#include "utils.h" +#endif xcecd_context_t xcecd_ctx = { 0 }; hostattrs_t host_attrs = { 0 }; @@ -32,7 +33,7 @@ static int virtual_pid = 1000; void xcecd_process_command(void* arg) { pid_t pid = 0; - char unit_name[256]; + char unit_name[256]; fork_task_t* ctx = (fork_task_t*)arg; if (!ctx || !ctx->startinfo || !ctx->startinfo->cmdline) { @@ -129,6 +130,7 @@ void xcecd_process_command(void* arg) make_job_sd_unit_name(unit_name, sizeof(unit_name) - 1, "xcecd-worker", ctx->xce_jobid); + /* get worker pid */ pid = dbus_get_worker_pid(unit_name); #else @@ -245,136 +247,13 @@ const char* handle_network_request(const char* request, void* user_data) return strdup("Task submitted successfully\n"); } -// TODO: next phase task - send workload and host info to controller -int send_status_and_receive_tasks(void) -{ - // disabled currently - return 0; - - int sockfd; - struct sockaddr_in server_addr; - char buffer[BUFFER_SIZE]; - static int error_count = 0; - int error_log_interval = 10; - - // 创建套接字 - sockfd = socket(AF_INET, SOCK_STREAM, 0); - if (sockfd < 0) { - XCECD_LOG(XCE_LOG_ERROR, "Failed to create socket"); - return -1; - } - - // 设置服务器地址 - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(xcecd_ctx.remote_port); - - // 打印远程主机名和端口以进行调试 - XCECD_LOG(XCE_LOG_DEBUG, "Connecting to remote host: %s, port: %d", - xcecd_ctx.remote_host, xcecd_ctx.remote_port); - - // 使用getaddrinfo解析主机名 - struct addrinfo hints, *res; - memset(&hints, 0, sizeof(struct addrinfo)); - hints.ai_family = AF_INET; - hints.ai_socktype = SOCK_STREAM; - - int err = getaddrinfo(xcecd_ctx.remote_host, NULL, &hints, &res); - if (err != 0) { - XCECD_LOG(XCE_LOG_ERROR, "getaddrinfo: %s", gai_strerror(err)); - close(sockfd); - return -1; - } - - struct sockaddr_in* addr_in = (struct sockaddr_in*)res->ai_addr; - server_addr.sin_addr = addr_in->sin_addr; - freeaddrinfo(res); - - // 打印解析后的IP地址 - char ip_str[INET_ADDRSTRLEN]; - inet_ntop(AF_INET, &(server_addr.sin_addr), ip_str, INET_ADDRSTRLEN); - XCECD_LOG(XCE_LOG_DEBUG, "Resolved IP address: %s", ip_str); - - // 连接到服务器 - if (connect(sockfd, (struct sockaddr*)&server_addr, sizeof(server_addr)) - < 0) { - error_count++; - if ((error_count == 1) || (error_count % error_log_interval == 0)) { - XCECD_LOG(XCE_LOG_ERROR, - "Failed connecting to remote host: %s (attempt %d)", - strerror(errno), error_count); - } - close(sockfd); - return -1; - } - - // 准备状态信息 - int running_processes = get_process_count(xcecd_ctx.process_manager); - host_node_t* node = NULL; - resource_sync_read(xcecd_ctx.resource_sync, &node); - - int slots - = get_cpu_usage(node->node_status.max_cpu_cores, running_processes); - snprintf(buffer, BUFFER_SIZE, "{\"slots\":%d}", slots); - - // 发送状态信息 - if (send(sockfd, buffer, strlen(buffer), 0) < 0) { - XCECD_LOG(XCE_LOG_ERROR, "Failed to send status"); - close(sockfd); - return -1; - } - - /* 接收任务请求 */ - ssize_t bytes_received; - while ((bytes_received = recv(sockfd, buffer, BUFFER_SIZE - 1, 0)) > 0) { - buffer[bytes_received] = '\0'; - XCECD_LOG(XCE_LOG_DEBUG, "Received task: %s", buffer); - - /* 创建任务上下文并添加到fork线程池队列 */ - fork_task_t* ctx = calloc(1, sizeof(fork_task_t)); - if (ctx) { - ctx->startinfo = calloc(1, sizeof(job_startinfo_t)); - if (!ctx->startinfo) { - FREEUP(ctx); - XCECD_LOG(XCE_LOG_ERROR, - "Failed to allocate memory for start_info"); - continue; - } - ctx->startinfo->cmdline = strdup(buffer); - if (!ctx->startinfo->cmdline) { - FREEUP(ctx->startinfo); - FREEUP(ctx); - XCECD_LOG(XCE_LOG_ERROR, - "Failed to allocate memory for command"); - continue; - } -#ifndef DBUS_MANAGER - ctx->manager = (void*)xcecd_ctx.process_manager; -#endif - ctx->func = xcecd_process_command; - - if (submit_fork_task(xcecd_ctx.thread_pool, ctx) != 0) { - XCECD_LOG(XCE_LOG_ERROR, "Failed to add task to thread pool"); - FREEUP(ctx->startinfo->cmdline); - FREEUP(ctx->startinfo); - FREEUP(ctx); - } else { - XCECD_LOG(XCE_LOG_DEBUG, "Task added to thread pool queue"); - } - } - } - - close(sockfd); - return 0; -} - static void connector_to_controller(void* arg) { while (xcecd_ctx.running && cc_ctx->running) { checking_connection(); if (xcecd_ctx.running && cc_ctx->running) { - send_status_and_receive_tasks(); + // continue running } else { XCECD_LOG(XCE_LOG_ERROR, "XCECD daemon is not running, exiting thread"); @@ -612,7 +491,10 @@ int xcecd_start(void) if (sqlite3_open(xcecd_ctx.db_path, &xcecd_ctx.db) != SQLITE_OK) { XCECD_LOG(XCE_LOG_ERROR, "Failed to open database: %s", sqlite3_errmsg(xcecd_ctx.db)); - FREEUP(xcecd_ctx.db); + + if (xcecd_ctx.db != NULL) { + sqlite3_close(xcecd_ctx.db); + } return false; } diff --git a/src/worker/src/xcecd_cgroups.c b/src/worker/src/xcecd_cgroups.c index 55b7d205..c27b6a45 100644 --- a/src/worker/src/xcecd_cgroups.c +++ b/src/worker/src/xcecd_cgroups.c @@ -1,5 +1,4 @@ #include "xcecd_cgroups.h" -#include "xcecd_utils.h" #include "xcelog.h" #include @@ -27,9 +26,11 @@ char* start_systemd_job(const char* command, const res_limits_t* limits, char service_name[256]; // 生成唯一的unit名 - make_job_sd_unit_name(service_name, sizeof(service_name) - 1, - unit_name_prefix ? unit_name_prefix : "xcecd-worker", + const char* prefix = unit_name_prefix ? unit_name_prefix : "xcecd-worker"; + + make_job_sd_unit_name(service_name, sizeof(service_name) - 1, prefix, job_id); + // 连接到system bus int r = sd_bus_open_system(&bus); if (r < 0) { @@ -241,6 +242,56 @@ char* start_systemd_job(const char* command, const res_limits_t* limits, if (r < 0) goto finish; + // 设置 TasksAccounting = yes + r = sd_bus_message_open_container(m, 'r', "sv"); + if (r < 0) + goto finish; + + r = sd_bus_message_append(m, "s", "TasksAccounting"); + if (r < 0) + goto finish; + + r = sd_bus_message_open_container(m, 'v', "b"); + if (r < 0) + goto finish; + + r = sd_bus_message_append(m, "b", 1); + if (r < 0) + goto finish; + + r = sd_bus_message_close_container(m); // 关闭变体 + if (r < 0) + goto finish; + + r = sd_bus_message_close_container(m); // 关闭属性 + if (r < 0) + goto finish; + + // 设置 MemoryAccounting = yes + r = sd_bus_message_open_container(m, 'r', "sv"); + if (r < 0) + goto finish; + + r = sd_bus_message_append(m, "s", "MemoryAccounting"); + if (r < 0) + goto finish; + + r = sd_bus_message_open_container(m, 'v', "b"); + if (r < 0) + goto finish; + + r = sd_bus_message_append(m, "b", 1); + if (r < 0) + goto finish; + + r = sd_bus_message_close_container(m); // 关闭变体 + if (r < 0) + goto finish; + + r = sd_bus_message_close_container(m); // 关闭属性 + if (r < 0) + goto finish; + // 如果提供了资源限制，设置相应的属性 if (limits) { @@ -1225,3 +1276,454 @@ char* get_systemd_job_status_text(const char* unit_name) return status_text; } + +/** + * @brief Comparison function for qsort. + */ +static int compare_ints(const void* a, const void* b) +{ + int int_a = *((int*)a); + int int_b = *((int*)b); + if (int_a < int_b) + return -1; + if (int_a > int_b) + return 1; + return 0; +} + +/** + * @brief Get a specific property from a systemd unit. + * This is a generic helper function. + * @param bus The sd_bus connection. + * @param unit_path The object path of the unit. + * @param interface The D-Bus interface. + * @param property The name of the property to get. + * @param type The expected D-Bus type character. + * @param ret A pointer to store the result. + * @return 0 on success, negative error code on failure. + */ +static int get_unit_property(sd_bus* bus, const char* unit_path, + const char* interface, const char* property, + char type, void* ret) +{ + sd_bus_error error = SD_BUS_ERROR_NULL; + int r = sd_bus_get_property_trivial(bus, "org.freedesktop.systemd1", + unit_path, interface, property, &error, + type, ret); + if (r < 0) { + XCECD_LOG(XCE_LOG_DEBUG, "Failed to get property '%s': %s", property, + error.message); + sd_bus_error_free(&error); + } + return r; +} + +/** + * @brief From cgroup.procs read all PIDs + * @param control_group for example "/system.slice/xcecd-worker-123.service" + */ +static int read_pids_from_cgroup(const char* control_group, int** pids_out, + int* npids_out) +{ + if (!control_group || !pids_out || !npids_out) + return -EINVAL; + + char path[MAXPATHLEN]; + snprintf(path, sizeof(path), "/sys/fs/cgroup%s/cgroup.procs", + control_group); + + FILE* fp = fopen(path, "r"); + if (!fp) { + XCECD_LOG(XCE_LOG_DEBUG, "Open %s failed: %s", path, strerror(errno)); + return -errno; + } + + int capacity = 32; + int* arr = (int*)calloc(capacity, sizeof(int)); + if (!arr) { + fclose(fp); + return -ENOMEM; + } + + int pid, count = 0; + while (fscanf(fp, "%d", &pid) == 1) { + if (count >= capacity) { + capacity *= 2; + int* tmp = (int*)realloc(arr, capacity * sizeof(int)); + if (!tmp) { + FREEUP(arr); + fclose(fp); + return -ENOMEM; + } + arr = tmp; + } + arr[count++] = pid; + } + fclose(fp); + + if (count == 0) { + FREEUP(arr); + return 1; // no pid + } + + *pids_out = arr; + *npids_out = count; + return 0; +} + +/** + * @brief Collect resource usage statistics for a systemd job. + * @param job_id The ID of the job. + * @return A pointer to an xce_job_rusages_t struct, or NULL on failure. + * The caller is responsible for freeing the returned struct + * using free_xce_job_rusages(). + */ +xce_job_rusages_t* collect_job_rusages(uint64_t job_id) +{ + sd_bus* bus = NULL; + int r; + char unit_name[256]; + char unit_path[512]; + xce_job_rusages_t* rusages = NULL; + + rusages = (xce_job_rusages_t*)calloc(1, sizeof(xce_job_rusages_t)); + if (!rusages) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to allocate memory for rusages struct."); + return NULL; + } + rusages->jobid = job_id; + + r = sd_bus_open_system(&bus); + if (r < 0) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to connect to system bus: %s", + strerror(-r)); + FREEUP(rusages); + return NULL; + } + + snprintf(unit_name, sizeof(unit_name), "xcecd-worker-%lu.service", job_id); + char* escaped_unit_name = replace_chars(unit_name); + if (!escaped_unit_name) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to escape unit name."); + sd_bus_unref(bus); + FREEUP(rusages); + return NULL; + } + snprintf(unit_path, sizeof(unit_path), "/org/freedesktop/systemd1/unit/%s", + escaped_unit_name); + FREEUP(escaped_unit_name); + + const char* res_names[] + = { "cpu_time_sec", "memory_rss", "memory_swap", "tasks_current" }; + const int num_res = sizeof(res_names) / sizeof(res_names[0]); + rusages->count = num_res; + rusages->usages = (xce_rusage_t**)calloc(num_res, sizeof(xce_rusage_t*)); + if (!rusages->usages) { + XCECD_LOG(XCE_LOG_ERROR, "Failed to allocate memory for usages array."); + goto cleanup; + } + + for (int i = 0; i < num_res; i++) { + rusages->usages[i] = (xce_rusage_t*)calloc(1, sizeof(xce_rusage_t)); + if (!rusages->usages[i]) { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to allocate memory for rusage item."); + goto cleanup; + } + rusages->usages[i]->resname = strdup(res_names[i]); + } + + uint64_t u64_val = 0; + + // CPU Time (CPUUsageNSec) + if (get_unit_property(bus, unit_path, "org.freedesktop.systemd1.Service", + "CPUUsageNSec", 't', &u64_val) + >= 0) { + rusages->usages[0]->value = (float)u64_val / 1e9; // 从纳秒转换为秒 + } + + // Memory RSS (MemoryCurrent) + if (get_unit_property(bus, unit_path, "org.freedesktop.systemd1.Service", + "MemoryCurrent", 't', &u64_val) + >= 0) { + rusages->usages[1]->value + = (float)u64_val / (1024 * 1024); // 从字节转换为MB + } + + // Memory Swap (MemorySwapCurrent) + if (get_unit_property(bus, unit_path, "org.freedesktop.systemd1.Service", + "MemorySwapCurrent", 't', &u64_val) + >= 0) { + rusages->usages[2]->value + = (float)u64_val / (1024 * 1024); // 从字节转换为MB + } + + // Tasks/Threads (TasksCurrent) + if (get_unit_property(bus, unit_path, "org.freedesktop.systemd1.Service", + "TasksCurrent", 't', &u64_val) + >= 0) { + rusages->usages[3]->value = (float)u64_val; + } + + // get all PIDs from cgroup.procs + char* control_group = NULL; + sd_bus_error cg_error = SD_BUS_ERROR_NULL; + sd_bus_message* reply = NULL; + r = sd_bus_call_method(bus, "org.freedesktop.systemd1", unit_path, + "org.freedesktop.DBus.Properties", "Get", &cg_error, + &reply, "ss", "org.freedesktop.systemd1.Service", + "ControlGroup"); + if (r < 0) { + XCECD_LOG(XCE_LOG_DEBUG, "Failed to call Get ControlGroup: %s", + cg_error.message ? cg_error.message : "unknown"); + sd_bus_error_free(&cg_error); + sd_bus_message_unref(reply); + reply = NULL; + } else { + const char* cgroup_str = NULL; + r = sd_bus_message_enter_container(reply, 'v', "s"); + if (r >= 0) { + r = sd_bus_message_read(reply, "s", &cgroup_str); + if (r >= 0 && cgroup_str) { + control_group = strdup(cgroup_str); + XCECD_LOG(XCE_LOG_DEBUG, "Got ControlGroup: %s", control_group); + sd_bus_message_exit_container(reply); + } else { + XCECD_LOG(XCE_LOG_DEBUG, + "Failed to read ControlGroup from reply: %s", + strerror(-r)); + } + } else { + XCECD_LOG(XCE_LOG_DEBUG, "Failed to enter variant container: %s", + strerror(-r)); + } + sd_bus_message_unref(reply); + reply = NULL; + } + + if (control_group) { + r = read_pids_from_cgroup(control_group, &rusages->pids, + &rusages->npids); + + if (r == 0 && rusages->npids > 0) { + // filter out xcecd_worker processes + int* filtered_pids = (int*)calloc(rusages->npids, sizeof(int)); + int filtered_count = 0; + + if (filtered_pids) { + for (int i = 0; i < rusages->npids; i++) { + pid_t pid = rusages->pids[i]; + + // check if xcecd_worker exists in cmdline + char proc_path[256]; + snprintf(proc_path, sizeof(proc_path), "/proc/%d/cmdline", + pid); + + FILE* fp = fopen(proc_path, "r"); + if (fp) { + char cmdline[1024] = { 0 }; + size_t len = fread(cmdline, 1, sizeof(cmdline) - 1, fp); + fclose(fp); + + // replace \0 with space for searching + for (size_t j = 0; j < len; j++) { + if (cmdline[j] == '\0') + cmdline[j] = ' '; + } + + // if xcecd_worker is not found, keep this PID + if (!strstr(cmdline, "xcecd_worker")) { + filtered_pids[filtered_count++] = pid; + XCECD_LOG(XCE_LOG_DEBUG3, "Including PID %d: %s", + pid, cmdline); + } else { + XCECD_LOG(XCE_LOG_DEBUG, + "Excluding xcecd_worker PID %d", pid); + } + } else { + // If cmdline cannot be read, include this PID for + // safety + filtered_pids[filtered_count++] = pid; + XCECD_LOG( + XCE_LOG_DEBUG3, + "Cannot read cmdline for PID %d, including it", + pid); + } + } + + // Replace original PIDs array + FREEUP(rusages->pids); + rusages->pids = filtered_pids; + int original_count = rusages->npids; + rusages->npids = filtered_count; + + XCECD_LOG(XCE_LOG_DEBUG, + "Filtered PIDs: %d -> %d (excluded %d xcecd_worker " + "processes)", + original_count, filtered_count, + original_count - filtered_count); + } + + // collect PGID (based on filtered PID) + if (rusages->npids > 0) { + int* all_pgids = (int*)calloc(rusages->npids, sizeof(int)); + if (all_pgids) { + int valid = 0; + + XCECD_LOG(XCE_LOG_DEBUG3, + "Collecting PGIDs for %d processes", + rusages->npids); + + for (int i = 0; i < rusages->npids; i++) { + pid_t pid = rusages->pids[i]; + + // Read PGID from /proc//stat + char stat_path[256]; + snprintf(stat_path, sizeof(stat_path), "/proc/%d/stat", + pid); + FILE* fp = fopen(stat_path, "r"); + if (fp) { + int stat_pid, stat_ppid, stat_pgrp; + char comm[256]; + char state; + if (fscanf(fp, "%d %s %c %d %d", &stat_pid, comm, + &state, &stat_ppid, &stat_pgrp) + >= 5) { + all_pgids[valid++] = stat_pgrp; + XCECD_LOG(XCE_LOG_DEBUG3, "PID %d -> PGID %d", + pid, stat_pgrp); + } + fclose(fp); + } else { + pid_t pgid = getpgid(pid); + if (pgid != -1) { + all_pgids[valid++] = (int)pgid; + XCECD_LOG(XCE_LOG_DEBUG3, + "PID %d -> PGID %d (getpgid)", pid, + (int)pgid); + } else { + XCECD_LOG(XCE_LOG_DEBUG, + "Failed to get PGID for PID %d: %s", + pid, strerror(errno)); + } + } + } + + if (valid > 0) { + qsort(all_pgids, valid, sizeof(int), compare_ints); + + int unique = 1; + for (int j = 1; j < valid; j++) { + if (all_pgids[j] != all_pgids[j - 1]) { + unique++; + } + } + + rusages->pgids = (int*)calloc(unique, sizeof(int)); + if (rusages->pgids) { + rusages->npgids = unique; + rusages->pgids[0] = all_pgids[0]; + int k = 1; + for (int j = 1; j < valid; j++) { + if (all_pgids[j] != all_pgids[j - 1]) { + rusages->pgids[k++] = all_pgids[j]; + } + } + + XCECD_LOG(XCE_LOG_DEBUG, + "Found %d unique PGIDs from %d PIDs", + unique, valid); + } else { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to allocate memory for PGIDs"); + } + } else { + XCECD_LOG(XCE_LOG_DEBUG, + "No valid PGIDs found after filtering"); + } + FREEUP(all_pgids); + } else { + XCECD_LOG(XCE_LOG_ERROR, + "Failed to allocate memory for PGID collection"); + } + } + } else if (r < 0) { + XCECD_LOG(XCE_LOG_DEBUG, "Read cgroup.procs failed (r=%d) for %s", + r, control_group); + } + FREEUP(control_group); + } else { + XCECD_LOG(XCE_LOG_DEBUG, + "ControlGroup is null, skipping PID collection"); + } + + sd_bus_error_free(&cg_error); + + /* consider to disable this in product for performance reasons */ +#if 1 + char pids_str[1024] = ""; + char pgids_str[1024] = ""; + + if (rusages->npids > 0 && rusages->pids) { + char temp[32]; + for (int i = 0; i < rusages->npids; i++) { + if (i > 0) + strcat(pids_str, ","); + snprintf(temp, sizeof(temp), "%d", rusages->pids[i]); + strcat(pids_str, temp); + + if (strlen(pids_str) > 900) { + strcat(pids_str, "..."); + break; + } + } + } else { + strcpy(pids_str, "none"); + } + + if (rusages->npgids > 0 && rusages->pgids) { + char temp[32]; + for (int i = 0; i < rusages->npgids; i++) { + if (i > 0) + strcat(pgids_str, ","); + snprintf(temp, sizeof(temp), "%d", rusages->pgids[i]); + strcat(pgids_str, temp); + + if (strlen(pgids_str) > 900) { + strcat(pgids_str, "..."); + break; + } + } + } else { + strcpy(pgids_str, "none"); + } + + XCECD_LOG(XCE_LOG_DEBUG3, + "Job %lu rusage: CPU: %.2fs, Mem: %.2fMB, Swap: %.2fMB, Tasks: " + "%d, PIDs: %d [%s], PGIDs: %d [%s]", + job_id, rusages->usages[0]->value, rusages->usages[1]->value, + rusages->usages[2]->value, (int)rusages->usages[3]->value, + rusages->npids, pids_str, rusages->npgids, pgids_str); +#endif + + sd_bus_unref(bus); + return rusages; + +cleanup: + if (rusages) { + if (rusages->usages) { + for (int i = 0; i < num_res; i++) { + if (rusages->usages[i]) { + FREEUP(rusages->usages[i]->resname); + FREEUP(rusages->usages[i]); + } + } + FREEUP(rusages->usages); + } + FREEUP(rusages->pids); + FREEUP(rusages); + } + sd_bus_unref(bus); + return NULL; +} diff --git a/src/worker/src/xcecd_db.c b/src/worker/src/xcecd_db.c index 996e48d6..d2a789b9 100644 --- a/src/worker/src/xcecd_db.c +++ b/src/worker/src/xcecd_db.c @@ -31,6 +31,7 @@ bool xcecd_db_init(xcecd_db_t** db, const char* db_path) if (sqlite3_open(db_path, db) != SQLITE_OK) { XCECD_LOG(XCE_LOG_ERROR, "Failed to open database: %s", sqlite3_errmsg(*db)); + if ((*db) != NULL) { sqlite3_close(*db); } diff --git a/src/worker/src/xcecd_dbus_manager.c b/src/worker/src/xcecd_dbus_manager.c index 4e0c028c..c5fda459 100644 --- a/src/worker/src/xcecd_dbus_manager.c +++ b/src/worker/src/xcecd_dbus_manager.c @@ -3,7 +3,6 @@ * @brief This file contains the implementation of the D-Bus manager functions. */ #include "xcecd_dbus_manager.h" -#include "utils.h" #include "xcecd_utils.h" #include "xcelog.h" @@ -65,6 +64,7 @@ bool dbus_start_worker(const res_limits_t* limits, uint64_t job_id, char myname[256]; make_job_sd_unit_name(myname, sizeof(myname) - 1, "xcecd-worker", job_id); + /* checking if service name is exist before start service and it's a * negative case */ dbus_stop_worker(myname); @@ -387,6 +387,7 @@ void dbus_stop_all_workers(xcecd_worker_t** worker_list, sqlite3* db) make_job_sd_unit_name(unit_name, sizeof(unit_name) - 1, "xcecd-worker", worker->job_info.jobid); + job.job_id = worker->job_info.jobid; job.job_pid = worker->worker_pid; job.job_status = JOB_EXITED; diff --git a/src/worker/src/xcecd_exdata.c b/src/worker/src/xcecd_exdata.c index da5b7808..f08c574b 100644 --- a/src/worker/src/xcecd_exdata.c +++ b/src/worker/src/xcecd_exdata.c @@ -9,9 +9,6 @@ #include "xcelog.h" #include #include -#include - -#include "utils.h" /** * @brief Parses the JSON string, populates an environment variable linked list, @@ -282,15 +279,9 @@ bool parse_limits(const char* exdata, res_limits_t* limits) bool success = false; cJSON* limits_obj = cJSON_GetObjectItemCaseSensitive(root, "limits"); - if (limits_obj == NULL) { + if (!limits_obj || !cJSON_IsObject(limits_obj)) { + // limits not exist success = true; - - memset(limits, 0x0, sizeof(*limits)); - goto cleanup; - } - - if (!cJSON_IsObject(limits_obj)) { - // invalidate limits goto cleanup; } @@ -414,7 +405,7 @@ void apply_env_vars(env_var_t** env_vars, uint64_t jobid) // Set some default XCE environment variables char jobid_str[32]; - snprintf(jobid_str, sizeof(jobid_str), "%lu", array_jobid(jobid)); + snprintf(jobid_str, sizeof(jobid_str), "%lu", jobid); if (setenv("XCE_JOBID", jobid_str, 1) != 0) { XCECD_WORKER_LOG(XCE_LOG_ERROR, "Error setting environment variable: %s", @@ -426,7 +417,7 @@ void apply_env_vars(env_var_t** env_vars, uint64_t jobid) } char jobindex_str[8]; - snprintf(jobindex_str, sizeof(jobindex_str), "%lu", array_idx(jobid)); + snprintf(jobindex_str, sizeof(jobindex_str), "%d", 0); if (setenv("XCE_JOBINDEX", jobindex_str, 1) != 0) { XCECD_WORKER_LOG(XCE_LOG_ERROR, "Error setting environment variable: %s", @@ -501,4 +492,4 @@ char* generate_exdata(const start_exdata_t* start_info) } return exdata_str; -} +} \ No newline at end of file diff --git a/src/worker/src/xcecd_worker.c b/src/worker/src/xcecd_worker.c index e52f7eff..e9772bef 100644 --- a/src/worker/src/xcecd_worker.c +++ b/src/worker/src/xcecd_worker.c @@ -3,6 +3,7 @@ #include "static_array.h" #include "uv_ipc.h" #include "uv_proc.h" +#include "xcecd_cgroups.h" #include "xcecd_exdata.h" #include "xcecd_utils.h" #include "xcelog.h" @@ -45,6 +46,8 @@ static int return_code = 0; static job_runtime_info_t job_runtime_info = { 0 }; char* log_file = NULL; start_exdata_t* start_exdata = NULL; +static uv_timer_t rusage_timer; +static int job_rusage_collect_interval = 10000; // 10 seconds // process manager will share the same event loop with ipc uv_loop_t* uv_loop = NULL; @@ -56,6 +59,57 @@ static void xjob_stop_vnc(); static bool check_process_status(pid_t target_pid, int expected_signal, int timeout_seconds); +static void free_xce_job_rusages(xce_job_rusages_t* rusages) +{ + if (!rusages) + return; + FREEUP(rusages->pids); + FREEUP(rusages->pgids); + if (rusages->usages) { + for (int i = 0; i < rusages->count; i++) { + if (rusages->usages[i]) { + FREEUP(rusages->usages[i]->resname); + FREEUP(rusages->usages[i]); + } + } + FREEUP(rusages->usages); + } + FREEUP(rusages); +} + +static void on_rusage_timer(uv_timer_t* timer) +{ + if (jobid > 0) { + XCECD_WORKER_LOG(XCE_LOG_DEBUG, "Collecting rusage for job %lu", jobid); + + xce_job_rusages_t* rusages = collect_job_rusages(jobid); + + if (rusages) { + ipc_worker_send_rusages(rusages); + free_xce_job_rusages(rusages); + } else { + XCECD_WORKER_LOG(XCE_LOG_ERROR, "Failed to collect job rusages."); + } + } +} + +static void start_rusage_collection() +{ + uv_loop_t* loop = ipc_get_event_loop(); + if (loop) { + uv_timer_init(loop, &rusage_timer); + uv_timer_start(&rusage_timer, on_rusage_timer, + job_rusage_collect_interval, + job_rusage_collect_interval); + } +} + +static void stop_rusage_collection() +{ + uv_timer_stop(&rusage_timer); + XCECD_WORKER_LOG(XCE_LOG_DEBUG3, "Job rusage collection stoped."); +} + static void signal_children_recursive(int signo, pid_t target_pid) { DIR* dir; @@ -412,6 +466,8 @@ static void process_exit_callback(int pid, int exit_status, int term_signal, { check_and_reconnect(); + stop_rusage_collection(); + if (term_signal == 0) { XCECD_WORKER_LOG(XCE_LOG_INFO, "Child %d exited with status %d\n", pid, @@ -540,14 +596,14 @@ static void xjob_start_vnc() if (access(XJOB_PRE_EXEC, F_OK) != 0) { XCECD_WORKER_LOG(XCE_LOG_DEBUG, "VNC pre-exec command not found: %s", XJOB_PRE_EXEC); - char* server_dir = getenv("XCE_SERVICE_DIR"); + char* server_dir = getenv("XCE_SERVER_DIR"); if (server_dir) { snprintf(vnc_start_cmd, sizeof(vnc_start_cmd), "%s/startvnc %d", server_dir, port_info.xjob_port); } else { XCECD_WORKER_LOG( XCE_LOG_ERROR, - "XCE_SERVICE_DIR not set, cannot find VNC pre-exec command"); + "XCE_SERVER_DIR not set, cannot find VNC pre-exec command"); exit(1); } } else { @@ -608,14 +664,14 @@ static void xjob_stop_vnc() if (access(XJOB_POST_EXEC, F_OK) != 0) { XCECD_WORKER_LOG(XCE_LOG_DEBUG, "VNC post-exec command not found: %s", XJOB_POST_EXEC); - char* server_dir = getenv("XCE_SERVICE_DIR"); + char* server_dir = getenv("XCE_SERVER_DIR"); if (server_dir) { snprintf(vnc_stop_cmd, sizeof(vnc_stop_cmd), "%s/stopvnc %d", server_dir, port_info.xjob_port); } else { XCECD_WORKER_LOG( XCE_LOG_ERROR, - "XCE_SERVICE_DIR not set, cannot find VNC post-exec command"); + "XCE_SERVER_DIR not set, cannot find VNC post-exec command"); exit(1); } } else { @@ -778,6 +834,9 @@ static void worker_job_callback(job_assignment_t* info) "Process started successfully to run task: %s", info->cmd); + start_rusage_collection(); + XCECD_WORKER_LOG(XCE_LOG_DEBUG3, "Starting job rusage collection..."); + if (mydata.transfer_info->interactive_mode > 0) { url = generate_url(port_info, mydata.transfer_info->interactive_mode); @@ -1168,16 +1227,18 @@ static void do_criu_ckpt(uint64_t job_id, int kill) XCECD_WORKER_LOG(XCE_LOG_DEBUG3, "Checkpoint directory created: %s", ckpt_dir); - char criu_cmd[MAXLINELEN*2]; - char info[MAXLINELEN*2]; + char criu_cmd[MAXLINELEN * 2]; + char info[MAXLINELEN * 2]; if (kill) { snprintf(criu_cmd, sizeof(criu_cmd), - "cd %s; %s dump -t %d -vvv -o dump.log", ckpt_dir, path, - target_pid); + "cd %s; %s dump -t %d -vvv --tcp-established --manage-cgroups " + "-o dump.log", + ckpt_dir, path, target_pid); } else { snprintf(criu_cmd, sizeof(criu_cmd), - "cd %s; %s dump -t %d -R -vvv -o dump.log", ckpt_dir, path, - target_pid); + "cd %s; %s dump -t %d -R -vvv --tcp-established " + "--manage-cgroups -o dump.log", + ckpt_dir, path, target_pid); } int ret = system(criu_cmd); @@ -1228,7 +1289,7 @@ static void do_criu_ckpt(uint64_t job_id, int kill) snprintf(info, sizeof(info), "{\"ckpt_pid\": %d, \"restart_command\": \"cd %s; %s restore -vvv " - "-o restore.log\"}", + "--tcp-established --manage-cgroups -o restore.log\"}", target_pid, ckpt_dir, path); ipc_worker_send_ckpt_result(job_id, 0, info); diff --git a/src/worker/src/xcecd_worker_utils.c b/src/worker/src/xcecd_worker_utils.c index 8b0d64f4..47b99e91 100644 --- a/src/worker/src/xcecd_worker_utils.c +++ b/src/worker/src/xcecd_worker_utils.c @@ -1,5 +1,4 @@ #include "xcecd_worker_utils.h" -#include "utils.h" #include "uv_ipc.h" #include "xcecd_dbus_manager.h" #include "xcecd_utils.h" @@ -525,5 +524,6 @@ int make_job_sd_unit_name(char* buf, size_t bufsize, const char* prefix, } fmt_jobid_ex(idstring, sizeof(idstring) - 1, jobid, '_', '_'); + return snprintf(buf, bufsize, "%s-%s.service", prefix, idstring); } diff --git a/src/worker/test/limit_test.c b/src/worker/test/limit_test.c deleted file mode 100644 index 27031e34..00000000 --- a/src/worker/test/limit_test.c +++ /dev/null @@ -1,795 +0,0 @@ -/** - * @file test_xcecd_exdata.c - * @brief Test program for resource limits in xcecd worker - * - * This program tests resource limits by gradually increasing resource usage - * and reporting progress. It can be used to test limits parsing functionality - * in xcecd_exdata.c. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// Default values -#define DEFAULT_STEP_SIZE_KB 1024 // Default memory step size (1MB) -#define DEFAULT_STEP_INTERVAL_MS 100 // 100ms between steps -#define DEFAULT_FILE_PATH "/tmp/xcecd_test_file" -#define MAX_THREADS 10000 - -// Global variables for cleanup -volatile sig_atomic_t terminate = 0; -void** memory_blocks = NULL; -int num_memory_blocks = 0; -pthread_t* thread_list = NULL; -int num_threads = 0; -FILE* test_file = NULL; -char* file_path = NULL; -pid_t* process_list = NULL; -int num_processes = 0; - -// Thread function prototype -void* thread_function(void* arg); -void process_function(void); - -// Signal handler -void signal_handler(int sig) -{ - terminate = 1; - printf("\nReceived signal %d, cleaning up and exiting...\n", sig); -} - -// Function to get current timestamp in milliseconds -long long current_time_ms() -{ - struct timeval tv; - gettimeofday(&tv, NULL); - return (long long)(tv.tv_sec) * 1000 + (tv.tv_usec / 1000); -} - -// Helper function to print a specific resource limit -static void print_limit(const char* name, int resource) -{ - struct rlimit rlim; - if (getrlimit(resource, &rlim) == 0) { - printf(" - %-25s: ", name); - if (rlim.rlim_cur == RLIM_INFINITY) { - printf("soft (unlimited), "); - } else { - printf("soft (%-10lld), ", (long long)rlim.rlim_cur); - } - if (rlim.rlim_max == RLIM_INFINITY) { - printf("hard (unlimited)\n"); - } else { - printf("hard (%lld)\n", (long long)rlim.rlim_max); - } - } else { - printf(" - %-25s: Failed to get limit (%s)\n", name, strerror(errno)); - } -} - -// Helper function to print all relevant limits -static void print_all_limits() -{ - printf("System Resource Limits:\n"); - print_limit("CPU time (seconds)", RLIMIT_CPU); - print_limit("Max file size (bytes)", RLIMIT_FSIZE); - print_limit("Max data segment (bytes)", RLIMIT_DATA); - print_limit("Max stack size (bytes)", RLIMIT_STACK); - print_limit("Max core file size (bytes)", RLIMIT_CORE); - print_limit("Max resident set (bytes)", RLIMIT_RSS); - print_limit("Max processes", RLIMIT_NPROC); - print_limit("Max open files", RLIMIT_NOFILE); - print_limit("Max locked memory (bytes)", RLIMIT_MEMLOCK); - print_limit("Max address space (bytes)", RLIMIT_AS); - printf("\n"); -} - -// Function to print current resource usage -void print_resource_usage() -{ - // Print limits only on the first call - static int first_call = 1; - if (first_call) { - print_all_limits(); - first_call = 0; - } - - struct rusage usage; - if (getrusage(RUSAGE_SELF, &usage) == 0) { - printf("Resource usage - User CPU: %.2fs, System CPU: %.2fs, RSS: %ld " - "KB\n", - usage.ru_utime.tv_sec + usage.ru_utime.tv_usec / 1000000.0, - usage.ru_stime.tv_sec + usage.ru_stime.tv_usec / 1000000.0, - usage.ru_maxrss); - } -} - -// Memory test function -void test_memory(size_t target_mb, size_t step_kb, int interval_ms) -{ - printf("Starting memory test: Target=%zu MB, Step=%zu KB, Interval=%d ms\n", - target_mb, step_kb, interval_ms); - - size_t target_kb = target_mb * 1024; - size_t allocated_kb = 0; - long long start_time = current_time_ms(); - long long next_step_time = start_time; - - // Pre-allocate array for memory block pointers - num_memory_blocks = (target_kb / step_kb) + 1; - memory_blocks = calloc(num_memory_blocks, sizeof(void*)); - if (!memory_blocks) { - printf("Failed to allocate memory for tracking blocks\n"); - exit(EXIT_FAILURE); - } - - int block_index = 0; - - while (allocated_kb < target_kb && !terminate) { - long long current_time = current_time_ms(); - - // Check if it's time for the next allocation - if (current_time >= next_step_time) { - // Allocate memory block - void* block = malloc(step_kb * 1024); - if (block) { - // Write to the memory to ensure it's actually allocated - memset(block, 1, step_kb * 1024); - - // Store pointer to free later - if (block_index < num_memory_blocks) { - memory_blocks[block_index++] = block; - } - - allocated_kb += step_kb; - - // Print progress - printf("[%6.2f s] Memory allocated: %zu KB (%zu MB)\n", - (current_time - start_time) / 1000.0, allocated_kb, - allocated_kb / 1024); - - // Update next step time - next_step_time = current_time + interval_ms; - - // Print resource usage every 10MB - if (allocated_kb % (10 * 1024) == 0) { - print_resource_usage(); - } - } else { - printf("Memory allocation failed at %zu KB\n", allocated_kb); - exit(EXIT_FAILURE); - } - } - - // Small sleep to prevent CPU hogging - usleep(10000); // 10ms - } - - printf("Memory test completed: Allocated %zu KB (%zu MB)\n", allocated_kb, - allocated_kb / 1024); - print_resource_usage(); -} - -// CPU test function -void test_cpu(int duration_seconds) -{ - printf("Starting CPU test: Duration=%d seconds\n", duration_seconds); - - long long start_time = current_time_ms(); - long long end_time = start_time + (duration_seconds * 1000); - long long next_report_time = start_time + 1000; // Report every second - - // Variables for busy work - double result = 0.0; - unsigned long iterations = 0; - - while (current_time_ms() < end_time && !terminate) { - long long current_time = current_time_ms(); - - // Do some CPU-intensive calculation - for (int i = 0; i < 1000000 && !terminate; i++) { - result += sin(result + i) * cos(result + i); - iterations++; - } - - // Check if it's time to report progress - if (current_time >= next_report_time) { - double elapsed_seconds = (current_time - start_time) / 1000.0; - double remaining_seconds = duration_seconds - elapsed_seconds; - - printf("[%6.2f s] CPU test progress: %.1f%% complete (%.1f seconds " - "remaining)\n", - elapsed_seconds, - (elapsed_seconds / duration_seconds) * 100.0, - remaining_seconds); - - print_resource_usage(); - - next_report_time = current_time + 1000; // Next report in 1 second - } - } - - printf("CPU test completed: Performed %lu iterations\n", iterations); - print_resource_usage(); -} - -// Thread creation function -void* thread_function(void* arg) -{ - int thread_id = *(int*)arg; - free(arg); // Free the memory allocated for the argument - - printf("Thread %d started\n", thread_id); - - // Keep the thread alive until termination signal - while (!terminate) { - sleep(1); - } - - printf("Thread %d exiting\n", thread_id); - return NULL; -} - -// Thread test function -void test_threads(int target_threads, int interval_ms) -{ - printf("Starting thread test: Target=%d threads, Interval=%d ms\n", - target_threads, interval_ms); - - int created_threads = 0; - long long start_time = current_time_ms(); - long long next_step_time = start_time; - - // Allocate array for thread IDs - thread_list = calloc(target_threads, sizeof(pthread_t)); - if (!thread_list) { - printf("Failed to allocate memory for thread tracking\n"); - exit(EXIT_FAILURE); - } - - while (created_threads < target_threads && !terminate) { - long long current_time = current_time_ms(); - - // Check if it's time to create a new thread - if (current_time >= next_step_time) { - // Allocate memory for thread ID argument - int* thread_id = malloc(sizeof(int)); - if (!thread_id) { - printf("Failed to allocate memory for thread ID\n"); - break; - } - *thread_id = created_threads + 1; - - // Create a new thread - if (pthread_create(&thread_list[created_threads], NULL, - thread_function, thread_id) - == 0) { - created_threads++; - - // Print progress - printf("[%6.2f s] Threads created: %d\n", - (current_time - start_time) / 1000.0, created_threads); - - // Update next step time - next_step_time = current_time + interval_ms; - - // Print resource usage every 10 threads - if (created_threads % 10 == 0) { - print_resource_usage(); - } - } else { - printf("Thread creation failed at %d threads: %s\n", - created_threads, strerror(errno)); - free(thread_id); // Free the ID memory if thread creation failed - exit(EXIT_FAILURE); - } - } - - // Small sleep to prevent CPU hogging - usleep(10000); // 10ms - } - - printf("Thread test completed: Created %d threads\n", created_threads); - print_resource_usage(); - - num_threads = created_threads; -} - -void process_function() -{ - printf("Child process %d started\n", getpid()); - - volatile sig_atomic_t local_terminate = 0; - - void child_signal_handler(int sig) - { - local_terminate = 1; - printf("Child process %d received signal %d, exiting...\n", getpid(), - sig); - } - - struct sigaction sa; - sa.sa_handler = child_signal_handler; - sigemptyset(&sa.sa_mask); - sa.sa_flags = 0; - sigaction(SIGTERM, &sa, NULL); - sigaction(SIGINT, &sa, NULL); - - while (!local_terminate) { - sleep(1); - } - - printf("Child process %d exiting\n", getpid()); - exit(EXIT_SUCCESS); -} - -void test_processes(int target_processes, int interval_ms) -{ - printf("Starting process test: Target=%d processes, Interval=%d ms\n", - target_processes, interval_ms); - - int created_processes = 0; - long long start_time = current_time_ms(); - long long next_step_time = start_time; - - process_list = calloc(target_processes, sizeof(pid_t)); - if (!process_list) { - printf("Failed to allocate memory for process tracking\n"); - exit(EXIT_FAILURE); - } - - while (created_processes < target_processes && !terminate) { - long long current_time = current_time_ms(); - - if (current_time >= next_step_time) { - pid_t pid = fork(); - - if (pid == -1) { - printf("Process creation failed at %d processes: %s\n", - created_processes, strerror(errno)); - exit(EXIT_FAILURE); - } else if (pid == 0) { - process_function(); - exit(EXIT_SUCCESS); - } else { - process_list[created_processes] = pid; - created_processes++; - - printf("[%6.2f s] Processes created: %d\n", - (current_time - start_time) / 1000.0, created_processes); - - next_step_time = current_time + interval_ms; - - if (created_processes % 10 == 0) { - print_resource_usage(); - } - } - } - - usleep(10000); // 10ms - } - - printf("Process test completed: Created %d processes\n", created_processes); - print_resource_usage(); - - num_processes = created_processes; -} - -// File size test function -void test_file_size(size_t target_mb, size_t step_kb, int interval_ms) -{ - printf( - "Starting file size test: Target=%zu MB, Step=%zu KB, Interval=%d ms\n", - target_mb, step_kb, interval_ms); - - size_t target_kb = target_mb * 1024; - size_t written_kb = 0; - long long start_time = current_time_ms(); - long long next_step_time = start_time; - - // Create buffer for writing - char* buffer = malloc(step_kb * 1024); - if (!buffer) { - printf("Failed to allocate buffer for file writing\n"); - exit(EXIT_FAILURE); - } - - // Fill buffer with random data - memset(buffer, 'A', step_kb * 1024); - - // Open file for writing - test_file = fopen(file_path, "wb"); - if (!test_file) { - printf("Failed to open file %s: %s\n", file_path, strerror(errno)); - free(buffer); - exit(EXIT_FAILURE); - } - - while (written_kb < target_kb && !terminate) { - long long current_time = current_time_ms(); - - // Check if it's time for the next write - if (current_time >= next_step_time) { - // Write to file - size_t bytes_written = fwrite(buffer, 1, step_kb * 1024, test_file); - if (bytes_written > 0) { - written_kb += bytes_written / 1024; - - // Flush to ensure data is actually written - fflush(test_file); - - // Print progress - printf("[%6.2f s] File size: %zu KB (%zu MB)\n", - (current_time - start_time) / 1000.0, written_kb, - written_kb / 1024); - - // Update next step time - next_step_time = current_time + interval_ms; - - // Print resource usage every 10MB - if (written_kb % (10 * 1024) == 0) { - print_resource_usage(); - } - } else { - printf("File write failed at %zu KB: %s\n", written_kb, - strerror(errno)); - exit(EXIT_FAILURE); - } - } - - // Small sleep to prevent CPU hogging - usleep(10000); // 10ms - } - - free(buffer); - printf("File size test completed: Wrote %zu KB (%zu MB)\n", written_kb, - written_kb / 1024); - print_resource_usage(); -} - -// Runtime test function (simple sleep) -void test_runtime(int duration_seconds) -{ - printf("Starting runtime test: Duration=%d seconds\n", duration_seconds); - - long long start_time = current_time_ms(); - long long end_time = start_time + (duration_seconds * 1000); - long long next_report_time = start_time + 1000; // Report every second - - while (current_time_ms() < end_time && !terminate) { - long long current_time = current_time_ms(); - - // Check if it's time to report progress - if (current_time >= next_report_time) { - double elapsed_seconds = (current_time - start_time) / 1000.0; - double remaining_seconds = duration_seconds - elapsed_seconds; - - printf("[%6.2f s] Runtime test progress: %.1f%% complete (%.1f " - "seconds remaining)\n", - elapsed_seconds, - (elapsed_seconds / duration_seconds) * 100.0, - remaining_seconds); - - print_resource_usage(); - - next_report_time = current_time + 1000; // Next report in 1 second - } - - // Sleep for a short time - usleep(100000); // 100ms - } - - printf("Runtime test completed: Slept for %.2f seconds\n", - (current_time_ms() - start_time) / 1000.0); - print_resource_usage(); -} - -// Cleanup function -void cleanup() -{ - printf("Cleaning up resources...\n"); - - // Free memory blocks - if (memory_blocks) { - for (int i = 0; i < num_memory_blocks && memory_blocks[i]; i++) { - free(memory_blocks[i]); - } - free(memory_blocks); - memory_blocks = NULL; - } - - // Wait for threads to terminate - if (thread_list) { - for (int i = 0; i < num_threads; i++) { - pthread_join(thread_list[i], NULL); - } - free(thread_list); - thread_list = NULL; - } - - // Terminate and wait for child processes - if (process_list) { - printf("Terminating %d child processes...\n", num_processes); - for (int i = 0; i < num_processes; i++) { - if (process_list[i] > 0) { - kill(process_list[i], SIGTERM); - } - } - - printf("Waiting for child processes to exit...\n"); - - int timeout_seconds = 5; - time_t start_wait = time(NULL); - - int remaining = num_processes; - while (remaining > 0 && time(NULL) - start_wait < timeout_seconds) { - for (int i = 0; i < num_processes; i++) { - if (process_list[i] > 0) { - int status; - pid_t result = waitpid(process_list[i], &status, WNOHANG); - - if (result > 0) { - if (WIFEXITED(status)) { - printf("Process %d exited with status %d\n", - (int)process_list[i], WEXITSTATUS(status)); - } else if (WIFSIGNALED(status)) { - printf("Process %d killed by signal %d\n", - (int)process_list[i], WTERMSIG(status)); - } - process_list[i] = -1; - remaining--; - } - } - } - usleep(100000); - } - - if (remaining > 0) { - printf("Force killing %d remaining processes...\n", remaining); - for (int i = 0; i < num_processes; i++) { - if (process_list[i] > 0) { - kill(process_list[i], SIGKILL); - waitpid(process_list[i], NULL, 0); - printf("Process %d force killed\n", (int)process_list[i]); - } - } - } - - free(process_list); - process_list = NULL; - } - - // Close and remove test file - if (test_file) { - fclose(test_file); - test_file = NULL; - if (file_path) { - unlink(file_path); - } - } - - // Free file path - if (file_path && strcasecmp(file_path, DEFAULT_FILE_PATH) != 0) { - free(file_path); - file_path = NULL; - } - - printf("Cleanup completed\n"); -} - -// Print usage -void print_usage(const char* program_name) -{ - printf("Usage: %s [options]\n", program_name); - printf(" Required: Duration to run in seconds\n"); - printf("Options:\n"); - printf(" -m, --memory MB Target memory usage in MB\n"); - printf(" -c, --cpu Enable CPU stress test\n"); - printf(" -t, --threads N Number of threads to create\n"); - printf(" -p, --processes N Number of processes to create\n"); - printf(" -f, --filesize MB Target file size in MB\n"); - printf(" -P, --filepath PATH Path for file size test (default: %s)\n", - DEFAULT_FILE_PATH); - printf(" -s, --step KB Step size in KB (default: %d)\n", - DEFAULT_STEP_SIZE_KB); - printf(" -i, --interval MS Interval between steps in ms (default: %d)\n", - DEFAULT_STEP_INTERVAL_MS); - printf(" -h, --help Show this help message\n"); - printf("\nExample:\n"); - printf(" %s 60 -m 500 -t 50 -p 10 -s 1024 -i 100\n", program_name); - printf(" (Run for 60 seconds, allocate up to 500MB memory, create 50 " - "threads,\n"); - printf(" create 10 processes, with step size of 1MB every 100ms)\n"); -} - -int main(int argc, char* argv[]) -{ - // Default parameters - int sleep_time = 0; - size_t memory_mb = 0; - int cpu_test = 0; - int thread_count = 0; - int process_count = 0; - size_t file_size_mb = 0; - size_t step_size_kb = DEFAULT_STEP_SIZE_KB; - int step_interval_ms = DEFAULT_STEP_INTERVAL_MS; - file_path = (char*)DEFAULT_FILE_PATH; - - // Define long options - static struct option long_options[] = { - { "memory", required_argument, 0, 'm' }, - { "cpu", no_argument, 0, 'c' }, - { "threads", required_argument, 0, 't' }, - { "processes", required_argument, 0, 'p' }, - { "filesize", required_argument, 0, 'f' }, - { "filepath", required_argument, 0, 'P' }, - { "step", required_argument, 0, 's' }, - { "interval", required_argument, 0, 'i' }, - { "help", no_argument, 0, 'h' }, - { 0, 0, 0, 0 } - }; - - // Check if sleep_time is provided - if (argc < 2) { - print_usage(argv[0]); - return EXIT_FAILURE; - } - - // Parse the sleep_time argument - sleep_time = atoi(argv[1]); - if (sleep_time <= 0) { - printf("Error: Sleep time must be a positive integer\n"); - print_usage(argv[0]); - return EXIT_FAILURE; - } - - // Shift the arguments to make getopt work - for (int i = 1; i < argc - 1; i++) { - argv[i] = argv[i + 1]; - } - argc--; - - // Parse options - int option_index = 0; - int opt; - while ((opt = getopt_long(argc, argv, "m:ct:p:f:P:s:i:h", long_options, - &option_index)) - != -1) { - switch (opt) { - case 'm': - memory_mb = atoi(optarg); - break; - case 'c': - cpu_test = 1; - break; - case 't': - thread_count = atoi(optarg); - break; - case 'p': - process_count = atoi(optarg); - break; - case 'f': - file_size_mb = atoi(optarg); - break; - case 'P': - file_path = strdup(optarg); - break; - case 's': - step_size_kb = atoi(optarg); - break; - case 'i': - step_interval_ms = atoi(optarg); - break; - case 'h': - print_usage(argv[0]); - return EXIT_SUCCESS; - case '?': - // getopt_long already printed an error message - print_usage(argv[0]); - return EXIT_FAILURE; - default: - break; - } - } - - // Validate parameters - if (step_size_kb <= 0) { - printf("Warning: Invalid step size, using default %d KB\n", - DEFAULT_STEP_SIZE_KB); - step_size_kb = DEFAULT_STEP_SIZE_KB; - } - - if (step_interval_ms <= 0) { - printf("Warning: Invalid interval, using default %d ms\n", - DEFAULT_STEP_INTERVAL_MS); - step_interval_ms = DEFAULT_STEP_INTERVAL_MS; - } - - // Set up signal handlers - signal(SIGINT, signal_handler); - signal(SIGTERM, signal_handler); - - // Set up cleanup handler - atexit(cleanup); - - // Print test configuration - printf("Resource Limit Test Configuration:\n"); - printf("- Sleep time: %d seconds\n", sleep_time); - printf("- Memory test: %s", memory_mb > 0 ? "Enabled" : "Disabled"); - if (memory_mb > 0) - printf(" (Target: %zu MB)", memory_mb); - printf("\n"); - - printf("- CPU test: %s\n", cpu_test ? "Enabled" : "Disabled"); - - printf("- Thread test: %s", thread_count > 0 ? "Enabled" : "Disabled"); - if (thread_count > 0) - printf(" (Target: %d threads)", thread_count); - printf("\n"); - - printf("- Process test: %s", process_count > 0 ? "Enabled" : "Disabled"); - if (process_count > 0) - printf(" (Target: %d processes)", process_count); - printf("\n"); - - printf("- File size test: %s", file_size_mb > 0 ? "Enabled" : "Disabled"); - if (file_size_mb > 0) - printf(" (Target: %zu MB, Path: %s)", file_size_mb, file_path); - printf("\n"); - - printf("- Step size: %zu KB\n", step_size_kb); - printf("- Step interval: %d ms\n", step_interval_ms); - printf("\n"); - - // Record start time - long long start_time = current_time_ms(); - - // Run tests based on configuration - if (memory_mb > 0) { - test_memory(memory_mb, step_size_kb, step_interval_ms); - } - - if (thread_count > 0) { - test_threads(thread_count, step_interval_ms); - } - - if (process_count > 0) { - test_processes(process_count, step_interval_ms); - } - - if (file_size_mb > 0) { - test_file_size(file_size_mb, step_size_kb, step_interval_ms); - } - - if (cpu_test) { - test_cpu(sleep_time); - } else { - // If no CPU test, just sleep for the specified duration - test_runtime(sleep_time); - } - - terminate = 1; - - if (thread_count > 0) { - printf("Waiting for threads to exit...\n"); - usleep(500000); - } - - // Print overall runtime - printf("\nTest completed in %.2f seconds\n", - (current_time_ms() - start_time) / 1000.0); - - return EXIT_SUCCESS; -} \ No newline at end of file diff --git a/src/worker/test/test_ipc.c b/src/worker/test/test_ipc.c deleted file mode 100644 index 30466db8..00000000 --- a/src/worker/test/test_ipc.c +++ /dev/null @@ -1,390 +0,0 @@ -#include "uv_ipc.h" -#include -#include -#include -#include -#include -#include -#include -#include - -#define SOCKET_PATH "/tmp/ipc_test.sock" -#define CHILD_PROCESS_COUNT 5 - -// 进程类型 -typedef enum { - PROCESS_MASTER, - PROCESS_WORKER -} process_type_t; - -static process_type_t process_type; - -// 全局变量，需要从uv_ipc.c中访问 -int child_processes = 0; -int active_jobs = 0; -int exit_flag = 0; - -void run_worker(); - -// 处理闹钟超时 -void alarm_handler(int sig) -{ - if (sig == SIGALRM) { - printf("\n[主进程] 警告：程序运行超时，强制退出\n"); - - // 强制杀死所有子进程 - kill(0, SIGKILL); - - // 打印当前状态 - printf("[主进程] 强制退出时状态：child_processes=%d, active_jobs=%d, " - "exit_flag=%d\n", - child_processes, active_jobs, exit_flag); - - // 直接退出，不进行任何清理 - _exit(1); - } -} - -// 主进程: 处理子进程任务状态回调 -void master_job_status_callback(uint64_t job_id, int status_code, - const char* status_message) -{ - if (!status_message) { - printf("[主进程] 警告: 收到空消息，job_id=%lu, status_code=%d\n", - job_id, status_code); - return; - } - - printf("[主进程] 收到任务状态: Job ID %lu, 状态码 %d, 消息: %s\n", job_id, - status_code, status_message); - - // 任务完成的情况有两种: - // 1. MSG_JOB_COMPLETED 消息(status_code=0, message="Completed") - // 2. 最后一个状态消息(status_code=1, message="Completed") - if ((status_code == 0 && strncmp(status_message, "Completed", 9) == 0) - || (status_code == 1 && strncmp(status_message, "Completed", 9) == 0)) { - - // 使用静态变量跟踪已完成的任务，避免重复计数 - static uint64_t completed_jobs[1024] = { 0 }; - static int completed_count = 0; - - // 检查是否已记录过这个任务 - bool already_completed = false; - for (int i = 0; i < completed_count; i++) { - if (completed_jobs[i] == job_id) { - already_completed = true; - break; - } - } - - if (!already_completed && completed_count < 1024) { - completed_jobs[completed_count++] = job_id; - active_jobs--; - printf("[主进程] 任务 %lu 已完成, 剩余活跃任务: %d\n", job_id, - active_jobs); - } - } -} - -// 主进程: 处理子进程断开连接回调 -void master_disconnect_callback(uint64_t client_id) -{ - printf("[主进程] 子进程 %lu 已断开连接\n", client_id); - child_processes--; - - printf("[主进程] 剩余子进程: %d, 剩余活跃任务: %d\n", child_processes, - active_jobs); - - // 如果所有子进程都已断开连接，可以考虑设置退出标志 - if (child_processes <= 0 && active_jobs <= 0) { - printf("[主进程] 所有子进程已断开连接且没有活跃任务，设置退出标志\n"); - exit_flag = 1; - } -} - -// 子进程: 连接回调 -void worker_connection_callback(bool success) -{ - if (success) { - printf("[子进程 %d] 已连接到主进程\n", getpid()); - // 连接成功后请求任务 - ipc_worker_request_job(); - } else { - printf("[子进程 %d] 连接到主进程失败\n", getpid()); - _exit(1); - } -} - -// 子进程: 任务回调 -void worker_job_callback(uint64_t job_id, const char* cmd) -{ - printf("[子进程 %d] 收到任务: Job ID %lu, 命令: %s\n", getpid(), job_id, - cmd); - - // 模拟任务进行 - printf("[子进程 %d] 开始执行任务 %lu\n", getpid(), job_id); - - // 发送状态更新 - ipc_worker_send_status(job_id, 1, "Running"); - - // 模拟随机执行时间（较短以便于测试） - // int execution_time = rand() % 3 + 1; - int execution_time = 3; - printf("[子进程 %d] 任务 %lu 将运行 %d 秒\n", getpid(), job_id, - execution_time); - - // 模拟任务进度报告 - for (int i = 1; i <= execution_time; i++) { - usleep(10 * 1000); // 10ms - char status[256]; - int progress = (i * 100) / execution_time; - - // 清除status缓冲区，避免垃圾数据 - memset(status, 0, sizeof(status)); - - if (progress < 100) { - snprintf(status, sizeof(status), "Progress %d%%", progress); - ipc_worker_send_status(job_id, 1, status); - } - } - - // 发送任务完成消息 - 使用 MSG_JOB_COMPLETED 类型 - printf("[子进程 %d] 任务 %lu 已完成\n", getpid(), job_id); - - // 先发送一个最终状态，确保主进程知道我们完成了 - ipc_worker_send_status(job_id, 1, "Completed"); - - // 等待一小段时间确保上面的消息已发送 - usleep(50000); // 50ms - - // 发送正式的完成消息 - ipc_worker_job_completed(job_id); - - // 等待另一小段时间确保完成消息被发送 - usleep(100000); // 100ms - - // 关闭连接，模拟子进程完成后退出 - printf("[子进程 %d] 准备断开连接并退出\n", getpid()); - - // 停止libuv相关资源（不会尝试关闭或删除事件循环） - ipc_worker_stop(); - - printf("[子进程 %d] 即将退出\n", getpid()); - fflush(stdout); // 确保所有输出都被写出 - - // 最后一次等待，确保所有消息都已发送 - usleep(200000); // 200ms - - _exit(0); // 立即退出，不执行任何清理 -} - -void signal_handler(int sig) -{ - if (sig == SIGINT || sig == SIGTERM) { - printf("\n收到信号 %d，准备退出\n", sig); - fflush(stdout); - - if (process_type == PROCESS_MASTER) { - exit_flag = 1; - - // 向所有子进程发送中断信号 - printf("[主进程] 向子进程发送终止信号\n"); - kill(0, SIGTERM); // 使用SIGTERM而不是SIGINT - - // 如果没有子进程，立即停止事件循环 - if (child_processes <= 0) { - printf("[主进程] 没有子进程，直接退出\n"); - if (loop) - uv_stop(loop); - return; - } - } else { - // 子进程直接退出，不做任何清理 - printf("[子进程 %d] 收到信号，直接退出\n", getpid()); - fflush(stdout); - _exit(0); - } - } -} - -void run_master() -{ - process_type = PROCESS_MASTER; - - printf("[主进程] 启动中...\n"); - - // 设置随机种子 - srand((unsigned int)time(NULL)); - - // 初始化主进程 - if (!ipc_master_init(SOCKET_PATH)) { - fprintf(stderr, "[主进程] 初始化失败\n"); - exit(1); - } - - // 设置回调 - ipc_master_set_status_callback(master_job_status_callback); - ipc_master_set_disconnect_callback(master_disconnect_callback); - - printf("[主进程] 初始化完成，等待子进程连接...\n"); - - // 启动子进程 - for (int i = 0; i < CHILD_PROCESS_COUNT; i++) { - pid_t pid = fork(); - - if (pid < 0) { - fprintf(stderr, "[主进程] 创建子进程失败\n"); - exit(1); - } else if (pid == 0) { - // 子进程 - 确保不继承父进程的状态 - child_processes = 0; // 重置子进程中的计数器 - active_jobs = 0; - exit_flag = 0; - - run_worker(); - _exit(0); // 使用_exit确保不执行父进程的atexit处理程序 - } else { - // 主进程 - 记录子进程 - printf("[主进程] 子进程 %d 已创建\n", pid); - child_processes++; - active_jobs++; - } - } - - // 运行主进程事件循环 - ipc_master_run(); - - printf("[主进程] 事件循环已退出，等待子进程结束...\n"); - - // 确保所有子进程退出 - int status; - pid_t wpid; - - // 先等待一小段时间，让子进程有机会自然退出 - usleep(100000); // 等待100ms - - // 等待所有子进程退出 - int remaining = 0; - while ((wpid = waitpid(-1, &status, WNOHANG)) > 0) { - printf("[主进程] 子进程 %d 已退出，状态码: %d\n", wpid, - WIFEXITED(status) ? WEXITSTATUS(status) : -1); - } - - // 检查是否还有子进程 - pid_t* child_pids = malloc(sizeof(pid_t) * 100); - if (child_pids) { - int remaining = 0; - FILE* fp = popen("ps -o pid -p $$ | tail -n +2", "r"); // 使用ps更可靠 - if (fp) { - char line[32]; - while (fgets(line, sizeof(line), fp)) { - pid_t child_pid = atoi(line); - if (child_pid > 0) { - printf("[主进程] 发现仍在运行的子进程: %d，发送终止信号\n", - child_pid); - kill(child_pid, SIGTERM); - remaining++; - } - } - pclose(fp); - } - free(child_pids); - - if (remaining > 0) { - printf("[主进程] 等待 %d 个子进程终止...\n", remaining); - sleep(1); // 给子进程一些时间来处理SIGTERM - - // 使用ps再次检查并用SIGKILL终止顽固的进程 - fp = popen("ps -o pid -p $$ | tail -n +2", "r"); - if (fp) { - char line[32]; - while (fgets(line, sizeof(line), fp)) { - pid_t child_pid = atoi(line); - if (child_pid > 0) { - printf( - "[主进程] 子进程 %d 未响应 SIGTERM, 发送 SIGKILL\n", - child_pid); - kill(child_pid, SIGKILL); - } - } - pclose(fp); - } - } - } - - printf("[主进程] 所有子进程已退出，测试完成\n"); -} - -void run_worker() -{ - process_type = PROCESS_WORKER; - - // 设置随机种子，使不同子进程行为不同 - srand(getpid() + (unsigned int)time(NULL)); - - printf("[子进程 %d] 启动中...\n", getpid()); - - // 初始化子进程 - if (!ipc_worker_init(SOCKET_PATH)) { - fprintf(stderr, "[子进程 %d] 初始化失败\n", getpid()); - _exit(1); - } - - // 设置回调 - ipc_worker_set_job_callback(worker_job_callback); - - // 连接到主进程 - ipc_worker_connect(worker_connection_callback); - - // 运行子进程事件循环 - ipc_worker_run(); - - printf("[子进程 %d] 事件循环已退出\n", getpid()); - - // 确保子进程立即退出，不执行任何清理操作 - fflush(stdout); - _exit(0); -} - -int main() -{ - // 初始化全局变量 - child_processes = 0; - active_jobs = 0; - exit_flag = 0; - - // 忽略SIGPIPE信号，它可能在写入已关闭的pipe时发生 - signal(SIGPIPE, SIG_IGN); - - // 设置信号处理 - struct sigaction sa; - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = signal_handler; - sa.sa_flags = SA_RESTART; // 使被中断的系统调用自动重启 - sigaction(SIGINT, &sa, NULL); - sigaction(SIGTERM, &sa, NULL); - - // 设置闹钟信号处理 - struct sigaction sa_alrm; - memset(&sa_alrm, 0, sizeof(sa_alrm)); - sa_alrm.sa_handler = alarm_handler; - sigaction(SIGALRM, &sa_alrm, NULL); - - // 设置超时机制，防止程序永久挂起 - alarm(30); // 如果30秒后程序仍未退出，将发送SIGALRM - - // 处理子进程退出，避免僵尸进程 - struct sigaction sa_chld; - memset(&sa_chld, 0, sizeof(sa_chld)); - sa_chld.sa_handler = SIG_DFL; - sa_chld.sa_flags = SA_NOCLDWAIT; // 不创建僵尸进程 - sigaction(SIGCHLD, &sa_chld, NULL); - - // 作为主进程运行 - run_master(); - - // 确保清理任何残留资源 - printf("[主进程] 程序退出\n"); - - return 0; -} diff --git a/src/worker/test/xce_sched_test.c b/src/worker/test/xce_sched_test.c deleted file mode 100644 index b6936935..00000000 --- a/src/worker/test/xce_sched_test.c +++ /dev/null @@ -1,142 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define BUFFER_SIZE 1024 - -static volatile int running = 1; -static int server_fd = -1; - -static void signal_handler(int signo) -{ - if (signo == SIGINT || signo == SIGTERM) { - printf("\nReceived signal %d, shutting down...\n", signo); - running = 0; - if (server_fd != -1) { - close(server_fd); - } - } -} - -// 获取当前时间戳字符串 -static void get_timestamp(char* timestamp, size_t size) -{ - time_t now; - struct tm* timeinfo; - - time(&now); - timeinfo = localtime(&now); - strftime(timestamp, size, "%Y-%m-%d %H:%M:%S", timeinfo); -} - -// 从JSON字符串中解析CPU核心数 -static int parse_cpu_cores(const char* json) -{ - int cores = 0; - if (sscanf(json, "{\"slots\":%d}", &cores) == 1) { - return cores; - } - return -1; -} - -int main() -{ - int client_fd; - struct sockaddr_in address; - int opt = 1; - socklen_t addrlen = sizeof(address); - char buffer[BUFFER_SIZE] = { 0 }; - char timestamp[64]; - const char* tasks[] = { "sleep 7", "sleep 5", "sleep 6", "date" }; - - // 设置随机数种子 - srand(time(NULL)); - - // 设置信号处理 - signal(SIGINT, signal_handler); - signal(SIGTERM, signal_handler); - - // 创建服务器套接字 - server_fd = socket(AF_INET, SOCK_STREAM, 0); - if (server_fd == 0) { - perror("socket failed"); - exit(EXIT_FAILURE); - } - - // 设置套接字选项 - if (setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR | SO_REUSEPORT, &opt, - sizeof(opt))) { - perror("setsockopt failed"); - close(server_fd); - exit(EXIT_FAILURE); - } - - // 绑定套接字 - address.sin_family = AF_INET; - address.sin_addr.s_addr = INADDR_ANY; - address.sin_port = htons(6789); - - if (bind(server_fd, (struct sockaddr*)&address, sizeof(address)) < 0) { - perror("bind failed"); - close(server_fd); - exit(EXIT_FAILURE); - } - - // 监听连接 - if (listen(server_fd, 3) < 0) { - perror("listen failed"); - close(server_fd); - exit(EXIT_FAILURE); - } - - printf("Server listening on port 6789\n"); - - while (running) { - client_fd = accept(server_fd, (struct sockaddr*)&address, &addrlen); - if (client_fd < 0) { - if (running) { - perror("accept failed"); - } - continue; - } - - // 处理客户端请求 - int bytes_read = read(client_fd, buffer, BUFFER_SIZE); - if (bytes_read > 0) { - buffer[bytes_read] = '\0'; - get_timestamp(timestamp, sizeof(timestamp)); - printf("[%s] Received: %s\n", timestamp, buffer); - - // 解析CPU核心数 - int cpu_cores = parse_cpu_cores(buffer); - if (cpu_cores > 0) { - printf("Parsed CPU cores: %d\n", cpu_cores); - - // 随机选择一个任务 - const char* task = tasks[rand() % 4]; - printf("Sending task: %s\n", task); - - // 发送任务到远端 - if (send(client_fd, task, strlen(task), 0) < 0) { - perror("send failed"); - } - } else if (cpu_cores == 0) { - printf("The server is running at full capacity\n"); - } else { - printf("Failed to parse CPU cores\n"); - } - } - - close(client_fd); - } - - close(server_fd); - printf("Server shut down\n"); - return 0; -} \ No newline at end of file diff --git a/src/worker/test/xcecd_daemon_test.c b/src/worker/test/xcecd_daemon_test.c deleted file mode 100644 index a1f22b90..00000000 --- a/src/worker/test/xcecd_daemon_test.c +++ /dev/null @@ -1,165 +0,0 @@ -#include "../include/connection_handler.h" -#include "../include/process_manager.h" -#include "../include/system_monitor.h" -#include "../include/thread_pool.h" -#include "../include/xcecd.h" -#include -#include -#include -#include - -// Mock function declarations -static int mock_network_start_called = 0; -static int mock_thread_pool_create_called = 0; -static bool mock_network_start_return = true; - -void setUp(void) -{ - mock_network_start_called = 0; - mock_thread_pool_create_called = 0; - mock_network_start_return = true; -} - -void tearDown(void) { xcecd_cleanup(); } - -// Test successful xcecd initialization -void test_xcecd_init_success(void) -{ - xcecd_config_t config = { .thread_pool_size = 4, - .resource_update_interval = 5, - .job_status_update_interval = 1, - .port = 8080, - .acct_file = "/tmp/xcecd.acct", - .log_file = "/tmp/xcecd.log", - .domain_sock_file = "/tmp/xcecd.sock", - .remote_host = "localhost", - .remote_port = 6789, - .update_interval = 2 }; - int result = xcecd_init(&config); - TEST_ASSERT_EQUAL_INT(0, result); -} - -// Test xcecd initialization with null config -void test_xcecd_init_null_config(void) -{ - int result = xcecd_init(NULL); - TEST_ASSERT_EQUAL_INT(-1, result); -} - -// Test xcecd initialization with invalid parameters -void test_xcecd_init_invalid_params(void) -{ - xcecd_config_t config = { .thread_pool_size = 0, // Invalid size - .resource_update_interval = 0, - .job_status_update_interval = 0, - .port = 0, - .acct_file = NULL, - .log_file = NULL, - .domain_sock_file = NULL, - .remote_host = NULL, - .remote_port = 0, - .update_interval = 0 }; - int result = xcecd_init(&config); - TEST_ASSERT_EQUAL_INT(-1, result); -} - -// Test xcecd start without initialization -void test_xcecd_start_without_init(void) -{ - int result = xcecd_start(); - TEST_ASSERT_EQUAL_INT(-1, result); -} - -// Test xcecd stop functionality -void test_xcecd_stop(void) -{ - xcecd_config_t config = { .thread_pool_size = 4, - .resource_update_interval = 5, - .job_status_update_interval = 1, - .port = 8080, - .acct_file = "/tmp/xcecd.acct", - .log_file = "/tmp/xcecd.log", - .domain_sock_file = "/tmp/xcecd.sock", - .remote_host = "localhost", - .remote_port = 6789, - .update_interval = 2 }; - xcecd_init(&config); - xcecd_stop(); - // Verify xcecd is stopped by trying to start it - int result = xcecd_start(); - TEST_ASSERT_EQUAL_INT(-1, result); -} - -// Test cleanup functionality -void test_xcecd_cleanup(void) -{ - xcecd_config_t config = { .thread_pool_size = 4, - .resource_update_interval = 5, - .job_status_update_interval = 1, - .port = 8080, - .acct_file = "/tmp/xcecd.acct", - .log_file = "/tmp/xcecd.log", - .domain_sock_file = "/tmp/xcecd.sock", - .remote_host = "localhost", - .remote_port = 6789, - .update_interval = 2 }; - xcecd_init(&config); - xcecd_cleanup(); - // Verify cleanup was successful by trying to start - int result = xcecd_start(); - TEST_ASSERT_EQUAL_INT(-1, result); -} - -// Test multiple init/cleanup cycles -void test_xcecd_multiple_init_cleanup(void) -{ - xcecd_config_t config = { .thread_pool_size = 4, - .resource_update_interval = 5, - .job_status_update_interval = 1, - .port = 8080, - .acct_file = "/tmp/xcecd.acct", - .log_file = "/tmp/xcecd.log", - .domain_sock_file = "/tmp/xcecd.sock", - .remote_host = "localhost", - .remote_port = 6789, - .update_interval = 2 }; - - for (int i = 0; i < 5; i++) { - int init_result = xcecd_init(&config); - TEST_ASSERT_EQUAL_INT(0, init_result); - xcecd_cleanup(); - } -} - -// Test initialization with minimum values -void test_xcecd_init_minimum_values(void) -{ - xcecd_config_t config = { .thread_pool_size = 1, - .resource_update_interval = 1, - .job_status_update_interval = 1, - .port = 1024, - .acct_file = "/tmp/xcecd.acct", - .log_file = "/tmp/xcecd.log", - .domain_sock_file = "/tmp/xcecd.sock", - .remote_host = "localhost", - .remote_port = 6789, - .update_interval = 1 }; - int result = xcecd_init(&config); - TEST_ASSERT_EQUAL_INT(0, result); -} - -int main(void) -{ - UNITY_BEGIN(); - - RUN_TEST(test_xcecd_init_success); - RUN_TEST(test_xcecd_init_null_config); - RUN_TEST(test_xcecd_init_invalid_params); - RUN_TEST(test_xcecd_start_without_init); - RUN_TEST(test_xcecd_stop); - RUN_TEST(test_xcecd_cleanup); - RUN_TEST(test_xcecd_multiple_init_cleanup); - RUN_TEST(test_xcecd_init_minimum_values); - - return UNITY_END(); -} diff --git a/src/worker/test/xcecd_worker_test.c b/src/worker/test/xcecd_worker_test.c deleted file mode 100644 index e90a0e5c..00000000 --- a/src/worker/test/xcecd_worker_test.c +++ /dev/null @@ -1,322 +0,0 @@ -#include "../include/process_manager.h" -#include "../include/xcecd.h" -#include "../include/xcecd_utils.h" -#include "unity.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define SOCKET_PATH "/tmp/xcecd.sock" -#define BUFFER_SIZE 4096 - -static bool reported_flag = false; - -// Mock function declarations -static int mock_socket_fd = -1; -static int mock_connect_called = 0; -static int mock_send_called = 0; -static int mock_recv_called = 0; -static int mock_close_called = 0; -static int mock_select_called = 0; -static int mock_waitpid_called = 0; -static int mock_process_manager_start_process_called = 0; - -static int mock_connect_return = 0; -static int mock_send_return = 0; -static int mock_recv_return = 0; -static int mock_select_return = 0; -static int mock_waitpid_return = 0; -static int mock_process_manager_start_process_return = 0; - -extern int main(void); - -void setUp(void) -{ - mock_socket_fd = -1; - mock_connect_called = 0; - mock_send_called = 0; - mock_recv_called = 0; - mock_close_called = 0; - mock_select_called = 0; - mock_waitpid_called = 0; - mock_process_manager_start_process_called = 0; - - mock_connect_return = 0; - mock_send_return = 0; - mock_recv_return = 0; - mock_select_return = 0; - mock_waitpid_return = 0; - mock_process_manager_start_process_return = 0; -} - -void tearDown(void) -{ - // 清理资源 -} - -// Mock functions -int socket(int domain, int type, int protocol) { return mock_socket_fd; } - -int connect(int sockfd, const struct sockaddr* addr, socklen_t addrlen) -{ - mock_connect_called++; - return mock_connect_return; -} - -ssize_t send(int sockfd, const void* buf, size_t len, int flags) -{ - mock_send_called++; - return mock_send_return; -} - -ssize_t recv(int sockfd, void* buf, size_t len, int flags) -{ - mock_recv_called++; - if (mock_recv_return > 0) { - strncpy(buf, "sleep 5", len); - } - return mock_recv_return; -} - -int close(int fd) -{ - mock_close_called++; - return 0; -} - -int select(int nfds, fd_set* readfds, fd_set* writefds, fd_set* exceptfds, - struct timeval* timeout) -{ - mock_select_called++; - return mock_select_return; -} - -pid_t waitpid(pid_t pid, int* status, int options) -{ - mock_waitpid_called++; - *status = 0; - return mock_waitpid_return; -} - -__attribute__((weak)) int -process_manager_start_process(process_manager_t* manager, const char* command) -{ - mock_process_manager_start_process_called++; - return mock_process_manager_start_process_return; -} - -static char* get_task_command(void) -{ - int sockfd; - struct sockaddr_un addr; - - if ((sockfd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) { - perror("socket error"); - return NULL; - } - - memset(&addr, 0, sizeof(addr)); - addr.sun_family = AF_UNIX; - strncpy(addr.sun_path, SOCKET_PATH, sizeof(addr.sun_path) - 1); - - if (connect(sockfd, (struct sockaddr*)&addr, sizeof(addr)) == -1) { - perror("connect error"); - close(sockfd); - return NULL; - } - - // int flags = fcntl(sockfd, F_GETFL, 0); - // fcntl(sockfd, F_SETFL, flags | O_NONBLOCK); - - fd_set read_fds; - FD_ZERO(&read_fds); - FD_SET(sockfd, &read_fds); - - struct timeval timeout = { 1, 0 }; - - int select_ret = select(sockfd + 1, &read_fds, NULL, NULL, &timeout); - if (select_ret > 0 && FD_ISSET(sockfd, &read_fds)) { - char command[BUFFER_SIZE] = { 0 }; - ssize_t bytes_read = read(sockfd, command, sizeof(command) - 1); - if (bytes_read > 0) { - command[bytes_read] = '\0'; - close(sockfd); - return strndup(command, sizeof(command)); - } else if (bytes_read == -1 && errno != EAGAIN - && errno != EWOULDBLOCK) { - perror("read error"); - } - } else if (select_ret == -1) { - perror("select error"); - } else { - printf("Socket not ready for reading\n"); - } - - close(sockfd); - return NULL; -} - -static void report_status_to_master(int pid, int status) -{ - int sockfd; - struct sockaddr_un addr; - - if (reported_flag) { - return; - } - - if ((sockfd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) { - perror("socket error"); - return; - } - - memset(&addr, 0, sizeof(addr)); - addr.sun_family = AF_UNIX; - strncpy(addr.sun_path, SOCKET_PATH, sizeof(addr.sun_path) - 1); - - if (connect(sockfd, (struct sockaddr*)&addr, sizeof(addr)) == -1) { - perror("connect error"); - close(sockfd); - return; - } - - // int flags = fcntl(sockfd, F_GETFL, 0); - // fcntl(sockfd, F_SETFL, flags | O_NONBLOCK); - - fd_set write_fds; - FD_ZERO(&write_fds); - FD_SET(sockfd, &write_fds); - - struct timeval timeout = { 1, 0 }; - - int select_ret = select(sockfd + 1, NULL, &write_fds, NULL, &timeout); - if (select_ret > 0 && FD_ISSET(sockfd, &write_fds)) { - char buffer[BUFFER_SIZE] = { 0 }; - snprintf(buffer, sizeof(buffer), - "WORKER_PID: %d JOB_PID: %d STATUS: %d", getpid(), pid, - status); - ssize_t bytes_written - = send(sockfd, buffer, strlen(buffer), MSG_NOSIGNAL); - if (bytes_written < 0) { - perror("send error"); - } else { - printf("Reported status to master\n"); - fflush(stdout); - reported_flag = true; - } - } else if (select_ret == -1) { - perror("select error"); - } else { - printf("Socket not ready for writing\n"); - } - - close(sockfd); -} - -static void handle_task_output(bool peek) -{ - if (!peek) { - return; - } - - printf("Succeed!"); - fflush(stdout); -} - -// Test cases -void test_xcecd_worker_init(void) -{ - // 测试初始化 - process_manager_t* manager = process_manager_create(); - TEST_ASSERT_NOT_NULL(manager); - process_manager_destroy(manager); -} - -void test_xcecd_worker_get_task_command(void) -{ - // 测试获取任务命令 - mock_socket_fd = 1; - mock_connect_return = 0; - mock_recv_return = 8; - mock_select_return = 1; - - char* command = get_task_command(); - TEST_ASSERT_NOT_NULL(command); - TEST_ASSERT_EQUAL_STRING("sleep 5", command); - free(command); - - TEST_ASSERT_EQUAL_INT(1, mock_connect_called); - TEST_ASSERT_EQUAL_INT(1, mock_recv_called); - TEST_ASSERT_EQUAL_INT(1, mock_select_called); - TEST_ASSERT_EQUAL_INT(1, mock_close_called); -} - -void test_xcecd_worker_handle_task_output(void) -{ - // 测试处理任务输出 - process_manager_t* manager = process_manager_create(); - process_info_t proc; - proc.output_fd = 1; - manager->processes = &proc; - manager->process_count = 1; - - handle_task_output(true); - - process_manager_destroy(manager); -} - -void test_xcecd_worker_report_status_to_master(void) -{ - // 测试报告状态到主进程 - mock_socket_fd = 1; - mock_connect_return = 0; - mock_send_return = 20; - mock_select_return = 1; - - report_status_to_master(1234, WORKER_DONE); - - TEST_ASSERT_EQUAL_INT(1, mock_connect_called); - TEST_ASSERT_EQUAL_INT(1, mock_send_called); - TEST_ASSERT_EQUAL_INT(1, mock_select_called); - TEST_ASSERT_EQUAL_INT(1, mock_close_called); -} - -void test_xcecd_worker_main(void) -{ - // 测试worker主函数 - mock_socket_fd = 1; - mock_connect_return = 0; - mock_recv_return = 8; - mock_select_return = 1; - mock_process_manager_start_process_return = 0; - mock_waitpid_return = 1234; - - int result = main(); - TEST_ASSERT_EQUAL_INT(0, result); - - TEST_ASSERT_EQUAL_INT(1, mock_connect_called); - TEST_ASSERT_EQUAL_INT(1, mock_recv_called); - TEST_ASSERT_EQUAL_INT(1, mock_select_called); - TEST_ASSERT_EQUAL_INT(1, mock_close_called); - TEST_ASSERT_EQUAL_INT(1, mock_process_manager_start_process_called); - TEST_ASSERT_EQUAL_INT(1, mock_waitpid_called); -} - -int main(void) -{ - UNITY_BEGIN(); - - RUN_TEST(test_xcecd_worker_init); - RUN_TEST(test_xcecd_worker_get_task_command); - RUN_TEST(test_xcecd_worker_handle_task_output); - RUN_TEST(test_xcecd_worker_report_status_to_master); - RUN_TEST(test_xcecd_worker_main); - - return UNITY_END(); -}