diff --git a/.claude/skills/xce-build/SKILL.md b/.claude/skills/xce-build/SKILL.md new file mode 100644 index 00000000..6f21082e --- /dev/null +++ b/.claude/skills/xce-build/SKILL.md @@ -0,0 +1,13 @@ +--- +name: xce-build +description: Build the xce source code +context: fork +disable-model-invocation: true +allowed-tools: Bash(*) +--- + +## build xce source code + +```bash +cd src && nix develop --command "rm -Rf xce.build && meson setup xce.build --buildtype=debug -Db_sanitize=address && meson compile -C build" +``` diff --git a/CLAUDE.md b/CLAUDE.md index 8868b2ca..04ab1d40 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,6 +11,9 @@ XCE is an HPC cluster management system providing job scheduling, queue manageme ## Development Commands ```bash +# Full dev build — all components (debug + ASAN) +/xce-build # Preferred: builds everything via nix develop + meson + # From src/frontend/ make build-server # Build apiserver binary → /tmp/apiserver make build-cli # Build xcecli binary → /tmp/xce_static (CGO_ENABLED=0) @@ -18,10 +21,6 @@ make test # Unit tests (no live server required) make test-integration # Regression suite (requires XCE_BIN and SRV_URL env vars) make test-all # Unit + integration tests -# Full dev build — all components (from src/) -nix develop # Enter dev environment -meson setup build && ninja -C build # Build everything (all components) - # Nix package builds (from src/) nix build --option sandbox false .#xcecore # C daemons + frontend/apiserver nix build --option sandbox false .#xcecore_debug # Same with debug/ASAN diff --git a/src/controller/config_repl.c b/src/controller/config_repl.c index 09ae4d8d..abf2750b 100644 --- a/src/controller/config_repl.c +++ b/src/controller/config_repl.c @@ -498,14 +498,14 @@ int config_contains_reserved_name(cluster_config_t *cfg) for (int i = 0; i < cfg->nqueues; i++) { if (cfg->queues[i] && cfg->queues[i]->queue && - strcmp(cfg->queues[i]->queue, "lost_and_found") == 0) + strcmp(cfg->queues[i]->queue, XCE_LOST_AND_FOUND_NAME) == 0) return 1; } /* Check host config names — xce_hostconfig_t.name */ for (int i = 0; i < cfg->nhconfigs; i++) { if (cfg->hconfigs[i] && cfg->hconfigs[i]->name && - strcmp(cfg->hconfigs[i]->name, "lost_and_found") == 0) + strcmp(cfg->hconfigs[i]->name, XCE_LOST_AND_FOUND_NAME) == 0) return 1; } return 0; @@ -551,9 +551,9 @@ void create_lost_and_found(cluster_t *c) * unknown queue IDs here instead of failing. */ { void *existing = NULL; - if (key_find(c->qmgr, (void *)"lost_and_found", &existing) <= 0) { + if (key_find(c->qmgr, (void *)XCE_LOST_AND_FOUND_NAME, &existing) <= 0) { xce_queue_t lf = {0}; - lf.queue = "lost_and_found"; + lf.queue = XCE_LOST_AND_FOUND_NAME; queue_t *q = create_queue(c, &lf); if (q) INFO("Created synthetic 'lost_and_found' queue"); @@ -567,9 +567,11 @@ void create_lost_and_found(cluster_t *c) * for this sentinel host. */ { void *existing = NULL; - if (key_find(c->workermgr, (void *)"lost_and_found", &existing) <= 0) { - xce_worker_t lf_host = {0}; - lf_host.name = "lost_and_found"; + if (key_find(c->workermgr, (void *)XCE_LOST_AND_FOUND_NAME, &existing) <= 0) { + hostattrs_t lf_attrs = { .hosttype = XCE_LOST_AND_FOUND_NAME, + .hostmodel = XCE_LOST_AND_FOUND_NAME }; + xce_worker_t lf_host = { .name = XCE_LOST_AND_FOUND_NAME, + .attrs = &lf_attrs }; worker_t *w = makeWorker(c->workermgr, &lf_host); if (w) INFO("Created synthetic 'lost_and_found' host"); @@ -780,15 +782,13 @@ typedef struct { const uint8_t *bytes; size_t blen; const uint8_t *sha256; - atomic_int *stop; int acked; + int started; } push_ctx_t; static void *push_thread(void *arg) { push_ctx_t *ctx = (push_ctx_t *)arg; - if (atomic_load(ctx->stop)) - return NULL; int rc = xce_replconfigpush_req_raw( ctx->ep, ctx->from, @@ -810,71 +810,51 @@ int push_config_to_standbys(cluster_t *c, const uint8_t *bytes, size_t blen, if (!c->ha.standby_endpoints) return 0; - /* Snapshot connected standbys under lock */ pthread_mutex_lock(&c->ha.lock); int n = c->ha.standby_endpoints->count; if (n == 0) { pthread_mutex_unlock(&c->ha.lock); return 0; } - endpoint_t **eps = malloc((size_t)n * sizeof(endpoint_t *)); - if (!eps) { + push_ctx_t *ctxs = calloc((size_t)n, sizeof(push_ctx_t)); + if (!ctxs) { pthread_mutex_unlock(&c->ha.lock); return 0; } - for (int i = 0; i < n; i++) - eps[i] = (endpoint_t *)zarray_get(c->ha.standby_endpoints, i); - pthread_mutex_unlock(&c->ha.lock); - - atomic_int stop = 0; - push_ctx_t *ctxs = calloc((size_t)n, sizeof(push_ctx_t)); - pthread_t *threads = malloc((size_t)n * sizeof(pthread_t)); - if (!ctxs || !threads) { - free(ctxs); free(threads); free(eps); - return 0; - } - for (int i = 0; i < n; i++) { - ctxs[i].ep = eps[i]; + ctxs[i].ep = (endpoint_t *)zarray_get(c->ha.standby_endpoints, i); ctxs[i].from = c->myhost; ctxs[i].new_tag = new_tag; ctxs[i].bytes = bytes; ctxs[i].blen = blen; ctxs[i].sha256 = sha256; - ctxs[i].stop = &stop; - ctxs[i].acked = 0; - pthread_create(&threads[i], NULL, push_thread, &ctxs[i]); + } + pthread_mutex_unlock(&c->ha.lock); + + pthread_t *threads = malloc((size_t)n * sizeof(pthread_t)); + if (!threads) { + free(ctxs); + return 0; } - /* Aggregate wait: CONF_PUSH_TIMEOUT_MS total, not per-standby */ - struct timespec deadline; - clock_gettime(CLOCK_MONOTONIC, &deadline); - deadline.tv_sec += CONF_PUSH_TIMEOUT_MS / 1000; - deadline.tv_nsec += (CONF_PUSH_TIMEOUT_MS % 1000) * 1000000L; - if (deadline.tv_nsec >= 1000000000L) { - deadline.tv_sec++; - deadline.tv_nsec -= 1000000000L; - } - while (1) { - struct timespec now; - clock_gettime(CLOCK_MONOTONIC, &now); - if (now.tv_sec > deadline.tv_sec || - (now.tv_sec == deadline.tv_sec && now.tv_nsec >= deadline.tv_nsec)) - break; - struct timespec sl = { .tv_sec = 0, .tv_nsec = 5 * 1000000L }; - nanosleep(&sl, NULL); + for (int i = 0; i < n; i++) { + ctxs[i].started = (pthread_create(&threads[i], NULL, push_thread, &ctxs[i]) == 0); + if (!ctxs[i].started) + WARN("push_config_to_standbys: pthread_create failed for standby %d", i); } - atomic_store(&stop, 1); int acked = 0; for (int i = 0; i < n; i++) { + if (!ctxs[i].started) + continue; pthread_join(threads[i], NULL); if (ctxs[i].acked) acked++; } INFO("Config pushed to %d/%d standbys", acked, n); - free(ctxs); free(threads); free(eps); + free(ctxs); + free(threads); return acked; } diff --git a/src/controller/config_repl.h b/src/controller/config_repl.h index 11423015..a4d23873 100644 --- a/src/controller/config_repl.h +++ b/src/controller/config_repl.h @@ -13,6 +13,9 @@ /* Max config blob size — reject before allocating */ #define CONF_MAX_SIZE (1 * 1024 * 1024) /* 1 MiB */ +/* Sentinel name for the synthetic lost-and-found queue and host */ +#define XCE_LOST_AND_FOUND_NAME "lost_and_found" + /* Compile-time gate for OP_upgradeConfig (Phase 2, default disabled) */ #ifndef CONF_ENABLE_UPGRADE_CONFIG #define CONF_ENABLE_UPGRADE_CONFIG 0 diff --git a/src/controller/dataex.c b/src/controller/dataex.c index 9a9c6464..02d35ed0 100644 --- a/src/controller/dataex.c +++ b/src/controller/dataex.c @@ -1,6 +1,7 @@ #include #include "cluster.h" +#include "config_repl.h" #include "request.h" #include "xce/cluster_def.h" #include "xce/types.h" @@ -235,6 +236,10 @@ void collect_queues(cluster_t* c, collect_check_fn_t fn, xlist_t* l) if (q == NULL) { continue; } + if (q->queue.queue != NULL + && strcmp(q->queue.queue, XCE_LOST_AND_FOUND_NAME) == 0) { + continue; + } if ((fn == NULL) || fn(q)) { reference(q); xlist_store(l, &(q->queue)); @@ -350,6 +355,11 @@ void collect_workers(cluster_t* c, collect_check_fn_t fn, xlist_t* l) continue; } + if (w->host.name != NULL + && strcmp(w->host.name, XCE_LOST_AND_FOUND_NAME) == 0) { + continue; + } + if ((fn == NULL) || (fn(w))) { reference(w); xlist_store(l, &(w->host)); diff --git a/src/controller/worker.c b/src/controller/worker.c index 0c25f88e..80fb12ff 100644 --- a/src/controller/worker.c +++ b/src/controller/worker.c @@ -369,16 +369,26 @@ int add_worker(void* cls, endpoint_t* ep, const char* host, hostattrs_t* attrs) return idx; } +static void reset_hostattrs_unknown(hostattrs_t* a) +{ + dereference(a->hosttype); + a->hosttype = makeString("unknown"); + dereference(a->hostmodel); + a->hostmodel = makeString("unknown"); + a->cpufactor = 0; + a->maxcpus = 0; + a->maxmem = 0; + a->maxswap = 0; + a->maxtmp = 0; + a->ndisks = 0; +} + int restore_worker(void* cls, xce_worker_t host) { cluster_t* c = (cluster_t*)cls; int idx = 0; worker_t* w = NULL; - w = calloc(1, sizeof(*w)); - if (w == NULL) { - return -ENOMEM; - } w = makeWorker(c->workermgr, &host); if (w == NULL) { return -errno; @@ -386,6 +396,8 @@ int restore_worker(void* cls, xce_worker_t host) lock(w, RTTI_LOCK_WRITE); + if (w->host.attrs != NULL) + reset_hostattrs_unknown(w->host.attrs); add_hostattrs_resvals(c, w); idx = w->idx; w->host.status |= XCE_WORKER_UNAVAIL; @@ -527,6 +539,12 @@ int mark_worker_unavail(cluster_t* c, worker_t* w) if (w->host.dead_since == 0) w->host.dead_since = time(NULL); + if (w->host.attrs != NULL) { + reset_hostattrs_unknown(w->host.attrs); + add_hostattrs_resvals(c, w); + set_host_slots(c, w, 0); + } + xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_WORKER, HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_DEAD, c, w); return 0;