From 805288d74708dd2b6f56a3c7a3656fa02c3359ff Mon Sep 17 00:00:00 2001 From: "lei.chen" Date: Tue, 27 Oct 2026 14:00:33 +0000 Subject: [PATCH] =?UTF-8?q?E2=20Phase=202:=20external-license-resources=20?= =?UTF-8?q?=E8=BF=90=E7=BB=B4=E9=9D=A2=20=E2=80=94=20stale=20=E6=A0=87?= =?UTF-8?q?=E8=AE=B0=E3=80=81last-updated=20=E5=B1=95=E7=A4=BA=E3=80=81xce?= =?UTF-8?q?admin=20extres=20status?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit design.md 3.5 Phase 2（PEN D reason 专用名为可选项，复用 PREASON_NO_RESOURCE 已带资源名，跳过）： ① esd stale 标记与 WARN - esd_source_t 增 stale_limit/last_ok_time；esd_source_is_stale(): 从未有值 不算 stale、时钟超前钳到 0、严格大于（interval==limit 不抖动）、 ESD_STALE_LIMIT_DEFAULT 150；fail-open 保持旧值不变 - 采集失败首轮一次性 WARN，过限后每轮升级为 STALE WARN（带 age/limit/kept 值），恢复即静默 - status 文件 esd_status_write(): daemon/pid/started_at/generated_at/ pushes ok/fail/sources[{name,last_value,last_ok_time,stale}]，tmp+rename 原子写；写失败仅状态转换时告警一次，绝不拖垮 poll 循环 ② `xce resources` 展示 last-updated - controller worker.c: EXTERNAL 共享值被接受时盖 last_report——同时盖 live resmgr 对象与 myconf 配置树副本（apiserver 定义 store 只由 join 时 config push 供给，串行的是配置树，缺后者 CLI 永远显示 never） - scheduler core_events.c: RESOURCES_PUSH 原地更新拷贝 last_report（纯 metadata，resobj.c 比较本就排除） - apiserver: model 带 last_report + 默认列（SelectFields 只投影默认列， 缺失即 API/CLI 全链不可见）；convert cResourceToModel 透传 - apiserver handler.go: OnResourceLocationUpdated 对 host=="" 且定义带 EXTERNAL(0x10) 的推送刷新 store 内 LastReport——config push 快照之外 的活刷新；host 来源值与非 EXTERNAL 资源绝不触碰 - xcecli: LAST_UPDATED 列（-l verbose 表格/详情/JSON 三态），0 显示 "-" ④ `xceadmin extres status` - 本机读 status 文件（远程读取留待 PR-2，需新 wire 协议；LAST_UPDATED 已覆盖远程新鲜度语义）——--status-file 显式路径或 /var/run/xce/esd-status.json 默认，无 controller 连接依赖 - XceExtresStatusError 本地文件类错误；click default=None 装饰期冻结坑 用调用期解析 TDD：test_extres_resvals +2（配置树盖戳/非 EXTERNAL 不盖戳）、test_esd_core +4（stale 判式/状态文件）、apiserver models +2（默认列/json tag）、backend integration +2（EXTERNAL 刷新/非 EXTERNAL 不刷新）、xceadmin +11（渲染/ 本地读/容错）。全绿：controller 1360、scheduler 100、apiserver（cgo+nocgo）、 esd 11、xceadmin 385。 t1 E2E 实证：esd join 即推送→API last_report=join 时刻→CLI LAST_UPDATED "Oct 27 13:52"；破坏采集器后 60s 过限转 per-round STALE WARN + status stale:true + fail-open 保值，恢复脚本即回 false；liccheck 作业提交→运行→ DONE，五 daemon 全健康。 Co-Authored-By: Claude Code --- .../testsuite/test_extres_resvals.c | 95 ++++++ src/controller/worker.c | 29 ++ src/esd/include/esd.h | 40 +++ src/esd/meson.build | 1 + src/esd/src/esd.c | 51 +++- src/esd/src/esd_config.c | 33 ++- src/esd/src/esd_status.c | 123 ++++++++ src/esd/testsuite/test_esd_core.c | 192 ++++++++++++ src/frontend/apiserver/backend/convert.go | 13 +- .../apiserver/backend/convert_pure.go | 14 +- src/frontend/apiserver/backend/handler.go | 18 ++ .../apiserver/backend/integration_test.go | 69 +++++ src/frontend/apiserver/models/resources.go | 15 +- .../apiserver/models/resources_test.go | 47 +++ .../common/output/page_print_resources.go | 7 + src/frontend/xcecli/models/resource.go | 46 +-- src/scheduler/src/core_events.c | 14 +- src/xceadmin/src/xceadmin/cli.py | 124 ++++++++ src/xceadmin/src/xcecommon/errors.py | 20 ++ src/xceadmin/tests/test_cli_extres.py | 275 ++++++++++++++++++ 20 files changed, 1179 insertions(+), 47 deletions(-) create mode 100644 src/esd/src/esd_status.c create mode 100644 src/frontend/apiserver/models/resources_test.go create mode 100644 src/xceadmin/tests/test_cli_extres.py diff --git a/src/controller/testsuite/test_extres_resvals.c b/src/controller/testsuite/test_extres_resvals.c index 966c9735..980190e8 100644 --- a/src/controller/testsuite/test_extres_resvals.c +++ b/src/controller/testsuite/test_extres_resvals.c @@ -18,6 +18,12 @@ * endpoint, so the seam under test is worker.c's list applier, the same * XCE_TESTING-exposure convention as apply_join_state in * test_worker_join.c). + * + * external-license-resources Phase 2 adds rule 3: an EXTERNAL shared + * acceptance also stamps last_report on the myconf cluster-config tree's + * definition -- the apiserver's resource-definition store is fed ONLY by + * the join-time config push (myconf tree, not the live resmgr object), so + * without this the CLI's LAST_UPDATED column would stay "never" forever. */ /* rtti.h must come first — it defines 'var' and 'typeinfo_t' used by kvmgr.h */ @@ -33,6 +39,7 @@ #include "cluster.h" #include "controller.capnp.h" #include "kvmgr.h" +#include "myconf.h" #include "resobj.h" #include "worker.h" #include "xce/types.h" @@ -52,6 +59,12 @@ int update_resval_list(cluster_t* c, worker_t* w, xce_resval_list_t* l, typedef struct { cluster_t c; var resmgr; + /* myconf fixture: one cluster_config_t owning a static resources[] tree + * (the config-push wire source, distinct from the resmgr objects). */ + myconf_t myconf; + cluster_config_t ccfg; + xce_resource_t* cfg_res[2]; + xce_resource_t cfg_defs[2]; } test_cluster_t; static test_cluster_t g_te; @@ -90,6 +103,23 @@ static void setup_resmgr(void) g_te.c.resmgr = g_te.resmgr; zarray_init(&g_te.c.sharedinst, 64, NULL); + + /* the myconf tree mirrors the resmgr registrations by name (the live + * shape: cluster_build_resources applies the config tree INTO resmgr, + * so names match; objects are distinct allocations) */ + { + static const char* names[2] = { "lic", "slots" }; + + for (int i = 0; i < 2; i++) { + memset(&g_te.cfg_defs[i], 0, sizeof(g_te.cfg_defs[i])); + g_te.cfg_defs[i].name = (char*)names[i]; + g_te.cfg_res[i] = &g_te.cfg_defs[i]; + } + g_te.ccfg.nres = 2; + g_te.ccfg.resources = g_te.cfg_res; + g_te.myconf.cluster_config = &g_te.ccfg; + g_te.c.myconf = &g_te.myconf; + } } static void teardown_resmgr(void) @@ -192,6 +222,69 @@ static void test_extres_nonshared_resval_refused_eperm(void) free(applied_vals); } +/* Rule 3 (Phase 2): an EXTERNAL shared acceptance stamps last_report on the + * myconf cluster-config tree's definition with the current time — the + * config push the apiserver receives on (re)join serializes THAT tree, not + * the live resmgr object, so an unstamped tree would show "never" in the + * CLI's LAST_UPDATED column even while the daemon pushes normally. */ +static void test_extres_external_accept_stamps_config_tree(void) +{ + static const char* names[1] = { "lic" }; + static const float values[1] = { 4.0f }; + xce_resval_list_t* l; + time_t before; + time_t after; + + setup_resmgr(); + + /* mark the definition EXTERNAL and confirm both copies start unstamped + * (config decode leaves last_report 0 for YAML-sourced definitions) */ + { + xce_resource_t* live = NULL; + + lock(g_te.resmgr, RTTI_LOCK_READ); + (void)key_find(g_te.resmgr, (void*)"lic", (void**)&live); + unlock(g_te.resmgr); + TEST_ASSERT_NOT_NULL(live); + live->flags |= XCE_RESF_EXTERNAL; + g_te.cfg_defs[0].flags |= XCE_RESF_EXTERNAL; + + TEST_ASSERT_EQUAL_INT(0, live->last_report); + TEST_ASSERT_EQUAL_INT(0, g_te.cfg_defs[0].last_report); + } + + time(&before); + l = make_resval_list(names, values, 1); + TEST_ASSERT_EQUAL_INT(0, update_resval_list(&g_te.c, NULL, l, NULL)); + time(&after); + + TEST_ASSERT_EQUAL_INT((int)before, (int)g_te.cfg_defs[0].last_report); + TEST_ASSERT_TRUE(g_te.cfg_defs[0].last_report >= before); + TEST_ASSERT_TRUE(g_te.cfg_defs[0].last_report <= after); + + teardown_resmgr(); +} + +/* The stamp is EXTERNAL-gated: a plain SHARED (non-EXTERNAL) acceptance + * leaves the config tree untouched — last_report means "external value + * last accepted", and worker-sourced values must never set it. */ +static void test_extres_plain_shared_accept_leaves_config_tree(void) +{ + static const char* names[1] = { "lic" }; + static const float values[1] = { 4.0f }; + xce_resval_list_t* l; + + setup_resmgr(); + + /* SHARED but NOT EXTERNAL */ + l = make_resval_list(names, values, 1); + TEST_ASSERT_EQUAL_INT(0, update_resval_list(&g_te.c, NULL, l, NULL)); + + TEST_ASSERT_EQUAL_INT(0, g_te.cfg_defs[0].last_report); + + teardown_resmgr(); +} + /* ── suite entry point ─────────────────────────────────────────────── */ void test_extres_resvals(void) @@ -199,4 +292,6 @@ void test_extres_resvals(void) UnitySetTestFile(__FILE__); RUN_TEST(test_extres_shared_resval_lands_in_sharedinst); RUN_TEST(test_extres_nonshared_resval_refused_eperm); + RUN_TEST(test_extres_external_accept_stamps_config_tree); + RUN_TEST(test_extres_plain_shared_accept_leaves_config_tree); } diff --git a/src/controller/worker.c b/src/controller/worker.c index a69a12e7..32cb3aca 100644 --- a/src/controller/worker.c +++ b/src/controller/worker.c @@ -74,6 +74,35 @@ static int update_val_res(cluster_t* c, worker_t* w, xce_resource_t* r, if (r->flags & XCE_RESF_SHARED) { /* update shared resource */ array = &(c->sharedinst); + /* external-license-resources Phase 2: the descriptor's reserved + * last_report becomes the "external value freshness" stamp -- the + * extres daemon's accepted pushes are the only writers of an + * EXTERNAL shared value, so every acceptance refreshes it and + * `xce resources` / a stale read can tell "reported recently" + * from "kept old value" (design.md 3.5 运维面). The apiserver's + * definition store is fed by the join-time CONFIG push, which + * serializes c->myconf->cluster_config -- a DIFFERENT copy from + * this live resmgr object -- so that tree's same-named definition + * gets the same stamp or the CLI's LAST_UPDATED column would stay + * "never" between apiserver (re)joins. */ + if (r->flags & XCE_RESF_EXTERNAL) { + time_t now = time(NULL); + + r->last_report = now; + if ((c->myconf != NULL) && (c->myconf->cluster_config != NULL)) { + cluster_config_t* cc = c->myconf->cluster_config; + + for (int i = 0; i < cc->nres; i++) { + xce_resource_t* cr = cc->resources[i]; + + if ((cr != NULL) && (cr->name != NULL) + && (strcmp(cr->name, r->name) == 0)) { + cr->last_report = now; + break; + } + } + } + } } else if (w == NULL) { /* external-license-resources Phase 1: a NULL worker is the extres * daemon (HostRole_extres join, no workermgr slot). It may only diff --git a/src/esd/include/esd.h b/src/esd/include/esd.h index 0b367f72..15894cd1 100644 --- a/src/esd/include/esd.h +++ b/src/esd/include/esd.h @@ -29,7 +29,10 @@ typedef struct esd_source_s { char* name; /* resource name, same as resources.yaml */ char* command; /* script collector: the command to run */ int interval; /* poll interval, seconds (0: use the daemon default) */ + long stale_limit; /* seconds a kept value stays trustworthy (0: use + * the daemon default); design.md 3.4 */ double last_value; /* last successfully-parsed value */ + time_t last_ok_time; /* when last_value was collected (0: never) */ bool have_value; /* last_value is valid (any push has succeeded) */ bool last_failed; /* last poll failed (stale-value policy input) */ } esd_source_t; @@ -39,10 +42,22 @@ typedef struct esd_config_s { int controller_port; char* peers; /* --peers style candidate list, or NULL */ int poll_interval; /* daemon default poll interval, seconds */ + long stale_limit; /* daemon default stale limit, seconds */ + char* status_file; /* status JSON path (NULL: no file) */ int nsrc; esd_source_t* src; /* array of nsrc sources */ } esd_config_t; +/* Daemon-wide counters and identity for the status file (Phase 2, + * design.md 3.5 "xceadmin extres status"): the daemon fills one and hands + * it to esd_status_write each poll round. */ +typedef struct esd_stats_s { + time_t started_at; /* daemon start, epoch seconds */ + time_t now; /* snapshot time (filled by esd_status_write when 0) */ + long pushes_ok; /* successful resval pushes since start */ + long pushes_fail; /* failed pushes since start */ +} esd_stats_t; + /* * Parse esd.yaml: * @@ -87,4 +102,29 @@ extern double esd_script_collect(const char* command); * it is killed (default 10s; main.c overrides from the CLI). */ extern int esd_script_timeout_ms; +/* Default stale limit for sources (and the config default) -- design.md + * 3.4's example: 150s. A source whose collector keeps failing past this + * many seconds since its last good value reports stale=true in the status + * file (the daemon itself keeps the last value -- fail-open). */ +#define ESD_STALE_LIMIT_DEFAULT 150 + +/* Stale predicate (Phase 2): true when the source's last good value is + * older than its stale limit -- `now` vs src->last_ok_time, the source's + * own stale_limit when set, else the daemon default. A source with no + * value yet is never stale (there is nothing being kept alive; admission + * already treats the pool as empty through the normal NO_RESOURCE path). + * A last_ok_time in the future (clock skew) is not stale. */ +extern bool esd_source_is_stale(const esd_source_t* src, time_t now, + long default_limit); + +/* Write the daemon status JSON (Phase 2, "xceadmin extres status"): one + * object with the daemon identity, push counters, and per-source rows + * (name, last_value, last_ok_time, stale). `stats->now == 0` is filled + * with time(NULL). Written atomically (tmp + rename) so a reader never + * sees a partial file. Returns 0, or -errno-shaped negative (the caller + * logs; the status file is best-effort observability and must never take + * the poll loop down). */ +extern int esd_status_write(const char* path, const esd_config_t* cfg, + esd_stats_t* stats); + #endif diff --git a/src/esd/meson.build b/src/esd/meson.build index 3e878a83..445af1ed 100644 --- a/src/esd/meson.build +++ b/src/esd/meson.build @@ -5,6 +5,7 @@ esd_srcs = [ 'src/esd_config.c', 'src/esd_script.c', + 'src/esd_status.c', 'src/esd.c', ] diff --git a/src/esd/src/esd.c b/src/esd/src/esd.c index abdad1b5..364c8f81 100644 --- a/src/esd/src/esd.c +++ b/src/esd/src/esd.c @@ -27,6 +27,7 @@ typedef struct esd_daemon_s { const char* from; /* join identity ("xceesd") */ int running; /* 0 once the main loop should exit */ xce_cluster_config_t cluster_cfg; /* validated peers list (HA mode) */ + esd_stats_t stats; /* status-file counters (Phase 2) */ } esd_daemon_t; static esd_daemon_t g_esd; @@ -61,8 +62,10 @@ int esd_push_value(const char* name, double value) rc = xcecli_update_resvals(g_esd.apictx, 1, &valptr); if (rc < 0) { ERROR("esd push %s=%f failed (rc=%d)", name, value, rc); + g_esd.stats.pushes_fail++; } else { TRACE("esd pushed %s=%f", name, value); + g_esd.stats.pushes_ok++; } return rc; @@ -72,11 +75,18 @@ int esd_push_value(const char* name, double value) * push only what CHANGED (the debounce -- design.md step 6 "轮询去抖"). * A failed collection keeps the previous value and does not push (fail- * open: the controller keeps scheduling against the last known good - * count; design.md 3.4 stale policy default). */ + * count; design.md 3.4 stale policy default). + * + * Phase 2: every source records last_ok_time on a good collect (the + * stale predicate's input, exposed via the status file); a failure past + * the source's stale limit escalates from the one-shot WARN to a per- + * round stale WARN -- the operator sees a kept-old-value daemon without + * having to guess from the absence of "pushed" lines. */ int esd_poll_round(void) { int i; int pushed = 0; + time_t now = time(NULL); for (i = 0; i < g_esd.cfg.nsrc; i++) { esd_source_t* src = &(g_esd.cfg.src[i]); @@ -89,12 +99,22 @@ int esd_poll_round(void) WARN("source %s: collection failed -- keeping last known " "value %f (fail-open)", src->name, src->have_value ? src->last_value : 0.0); + } else if (esd_source_is_stale(src, now, + g_esd.cfg.stale_limit)) { + WARN("source %s: STALE for %lds (limit %lds) -- last good " + "value %f kept (fail-open); check the collector", + src->name, + (long)(now - src->last_ok_time), + (src->stale_limit > 0) ? src->stale_limit + : g_esd.cfg.stale_limit, + src->last_value); } src->last_failed = true; continue; } - src->last_failed = false; + src->last_failed = false; + src->last_ok_time = now; if (src->have_value && (src->last_value == v)) { continue; /* unchanged -- the debounce */ @@ -262,6 +282,26 @@ int esd_run(void) (void)esd_poll_round(); + /* Phase 2: refresh the status file every round (best-effort; the + * poll loop must survive a bad path). */ + if (g_esd.cfg.status_file != NULL) { + /* one WARN per failure would spam a broken path every round; + * log on the transition only */ + static bool status_warned = false; + int src = esd_status_write(g_esd.cfg.status_file, &g_esd.cfg, + &g_esd.stats); + if (src < 0) { + if (!status_warned) { + WARN("esd: cannot write status file %s (rc=%d) -- " + "continuing without it", + g_esd.cfg.status_file, src); + status_warned = true; + } + } else { + status_warned = false; + } + } + /* sleep in 1s slices so a signal stop lands promptly */ for (i = 0; (i < g_esd.cfg.poll_interval) && g_esd.running; i++) { sleep(1); @@ -282,7 +322,8 @@ void esd_stop(void) void esd_init(esd_config_t* cfg, const char* from) { memset(&g_esd, 0, sizeof(g_esd)); - g_esd.cfg = *cfg; - g_esd.from = from; - g_esd.running = 1; + g_esd.cfg = *cfg; + g_esd.from = from; + g_esd.running = 1; + g_esd.stats.started_at = time(NULL); } diff --git a/src/esd/src/esd_config.c b/src/esd/src/esd_config.c index 5c8936bd..5ad145c7 100644 --- a/src/esd/src/esd_config.c +++ b/src/esd/src/esd_config.c @@ -17,13 +17,16 @@ typedef struct yaml_esd_source_s { char* name; char* collector; char* command; - long interval; /* 0 = daemon default */ + long interval; /* 0 = daemon default */ + long stale_limit; /* 0 = daemon default (Phase 2, design.md 3.4) */ } yaml_esd_source_t; typedef struct yaml_esd_config_s { char* controller; char* peers; long poll_interval; + long stale_limit; /* 0 = ESD_STALE_LIMIT_DEFAULT */ + char* status_file; /* NULL = no status file */ yaml_esd_source_t* resources; unsigned resources_count; } yaml_esd_config_t; @@ -37,6 +40,8 @@ static const cyaml_schema_field_t esd_source_fields_schema[] = { command, 1, CYAML_UNLIMITED), CYAML_FIELD_INT("interval", CYAML_FLAG_OPTIONAL, yaml_esd_source_t, interval), + CYAML_FIELD_INT("stale_limit", CYAML_FLAG_OPTIONAL, yaml_esd_source_t, + stale_limit), CYAML_FIELD_END }; @@ -52,6 +57,10 @@ static const cyaml_schema_field_t esd_config_fields_schema[] = { peers, 1, CYAML_UNLIMITED), CYAML_FIELD_INT("poll_interval", CYAML_FLAG_OPTIONAL, yaml_esd_config_t, poll_interval), + CYAML_FIELD_INT("stale_limit", CYAML_FLAG_OPTIONAL, yaml_esd_config_t, + stale_limit), + CYAML_FIELD_STRING_PTR("status_file", CYAML_FLAG_OPTIONAL, + yaml_esd_config_t, status_file, 1, CYAML_UNLIMITED), CYAML_FIELD_SEQUENCE("resources", CYAML_FLAG_POINTER, yaml_esd_config_t, resources, &esd_source_schema, 0, CYAML_UNLIMITED), CYAML_FIELD_END @@ -173,9 +182,23 @@ int esd_config_load_path(const char* dir, const char* name, cfg.peers = (ycfg->peers != NULL) ? strdup(ycfg->peers) : NULL; cfg.poll_interval = (int)ycfg->poll_interval; + cfg.stale_limit = ycfg->stale_limit; + cfg.status_file = (ycfg->status_file != NULL) ? strdup(ycfg->status_file) + : NULL; cfg.nsrc = (int)ycfg->resources_count; cfg.src = NULL; + if ((cfg.peers == NULL) && (ycfg->peers != NULL)) { + esd_config_fini(&cfg); + cyaml_free(&cyaml_cfg, &esd_config_schema, ycfg, 0); + return -ENOMEM; + } + if ((cfg.status_file == NULL) && (ycfg->status_file != NULL)) { + esd_config_fini(&cfg); + cyaml_free(&cyaml_cfg, &esd_config_schema, ycfg, 0); + return -ENOMEM; + } + if (cfg.nsrc > 0) { cfg.src = (esd_source_t*)calloc((size_t)cfg.nsrc, sizeof(esd_source_t)); @@ -195,9 +218,10 @@ int esd_config_load_path(const char* dir, const char* name, break; } - cfg.src[i].name = strdup(ys->name); - cfg.src[i].command = strdup(ys->command); - cfg.src[i].interval = (int)ys->interval; + cfg.src[i].name = strdup(ys->name); + cfg.src[i].command = strdup(ys->command); + cfg.src[i].interval = (int)ys->interval; + cfg.src[i].stale_limit = ys->stale_limit; if ((cfg.src[i].name == NULL) || (cfg.src[i].command == NULL)) { rc = -ENOMEM; @@ -275,6 +299,7 @@ void esd_config_fini(esd_config_t* cfg) free(cfg->controller_host); free(cfg->peers); + free(cfg->status_file); for (i = 0; i < cfg->nsrc; i++) { free(cfg->src[i].name); diff --git a/src/esd/src/esd_status.c b/src/esd/src/esd_status.c new file mode 100644 index 00000000..9a39fbba --- /dev/null +++ b/src/esd/src/esd_status.c @@ -0,0 +1,123 @@ +/* esd_status.c -- xceesd status file (external-license-resources Phase 2, + * design.md 3.5 "xceadmin extres status"). The daemon writes one JSON + * snapshot per poll round to a configured path; xceadmin reads it on the + * daemon host. No new wire opcode: the PR-2 proxy formalizes a status + * channel later, and a local file keeps this observability surface + * strictly read-only and crash-safe. + * + * Atomicity: write to .tmp then rename(2) -- a concurrent reader + * (xceadmin) either sees the previous complete file or the new one, never + * a partial write (the fake-license E2E's lic_state lesson: a reader + * observing a truncated file mistakes it for data loss). + */ +#include +#include +#include +#include +#include +#include + +#include "esd.h" + +bool esd_source_is_stale(const esd_source_t* src, time_t now, + long default_limit) +{ + long limit; + long age; + + if ((src == NULL) || (!src->have_value) || (src->last_ok_time == 0)) { + return false; + } + + /* Future last_ok_time (caller clock skew): age would go negative and + * read as fresh -- that is the honest reading, keep it by clamping. */ + age = (now > src->last_ok_time) ? (long)(now - src->last_ok_time) : 0; + + limit = (src->stale_limit > 0) ? src->stale_limit : default_limit; + if (limit <= 0) { + limit = ESD_STALE_LIMIT_DEFAULT; + } + + return age > limit; +} + +int esd_status_write(const char* path, const esd_config_t* cfg, + esd_stats_t* stats) +{ + char tmp[1024]; + FILE* f; + int i; + time_t now; + int rc = 0; + + if ((path == NULL) || (cfg == NULL) || (stats == NULL)) { + return -EINVAL; + } + + now = (stats->now != 0) ? stats->now : time(NULL); + + snprintf(tmp, sizeof(tmp), "%s.tmp", path); + f = fopen(tmp, "w"); + if (f == NULL) { + return -errno; + } + + if (fprintf(f, + "{\n" + " \"daemon\": \"xceesd\",\n" + " \"pid\": %ld,\n" + " \"started_at\": %ld,\n" + " \"generated_at\": %ld,\n" + " \"pushes_ok\": %ld,\n" + " \"pushes_fail\": %ld,\n" + " \"sources\": [\n", + (long)getpid(), (long)stats->started_at, (long)now, + stats->pushes_ok, stats->pushes_fail) < 0) { + rc = -EIO; + goto out; + } + + for (i = 0; i < cfg->nsrc; i++) { + const esd_source_t* src = &(cfg->src[i]); + bool stale = + esd_source_is_stale(src, now, cfg->stale_limit); + char valbuf[64]; + + if (src->have_value) { + snprintf(valbuf, sizeof(valbuf), "%.17g", src->last_value); + } else { + snprintf(valbuf, sizeof(valbuf), "null"); + } + + if (fprintf(f, + " {\"name\": \"%s\", \"last_value\": %s, " + "\"last_ok_time\": %ld, \"stale\": %s}%s\n", + (src->name != NULL) ? src->name : "", + valbuf, + (long)src->last_ok_time, stale ? "true" : "false", + (i + 1 < cfg->nsrc) ? "," : "") < 0) { + rc = -EIO; + goto out; + } + } + + if (fprintf(f, " ]\n}\n") < 0) { + rc = -EIO; + goto out; + } + +out: + if (fclose(f) != 0) { + rc = (rc == 0) ? -EIO : rc; + } + if (rc == 0) { + if (rename(tmp, path) != 0) { + rc = -errno; + } + } + if (rc != 0) { + unlink(tmp); /* never leave the .tmp behind for a reader to trip on */ + } + + return rc; +} diff --git a/src/esd/testsuite/test_esd_core.c b/src/esd/testsuite/test_esd_core.c index 95f6809b..b12a1f0b 100644 --- a/src/esd/testsuite/test_esd_core.c +++ b/src/esd/testsuite/test_esd_core.c @@ -23,6 +23,9 @@ * the spawn half (esd_script_collect) is exercised by the t1 E2E with a * real fake-license script, not here. */ #include "../src/esd_script.c" +/* White-box: the stale predicate and the status-file writer (Phase 2, + * design.md 3.5 "运维面"). */ +#include "../src/esd_status.c" static char g_tmpdir[256]; @@ -58,6 +61,35 @@ static void write_file(const char* dir, const char* name, const char* content) fclose(f); } +static char* slurp(const char* path) +{ + FILE* f; + long len; + char* buf; + + f = fopen(path, "r"); + if (f == NULL) { + return NULL; + } + fseek(f, 0, SEEK_END); + len = ftell(f); + fseek(f, 0, SEEK_SET); + buf = (char*)malloc((size_t)len + 1); + if (buf == NULL) { + fclose(f); + return NULL; + } + if (fread(buf, 1, (size_t)len, f) != (size_t)len) { + fclose(f); + free(buf); + return NULL; + } + buf[len] = '\0'; + fclose(f); + + return buf; +} + /* ── esd_config_load ─────────────────────────────────────────────────── */ static void test_config_load_full_file(void) @@ -185,6 +217,162 @@ static void test_parse_output_garbage_is_nan(void) TEST_ASSERT_TRUE(isnan(v)); } +/* ── Phase 2: stale_limit config parsing ────────────────────────────── */ + +static void test_config_stale_limit_parse(void) +{ + esd_config_t cfg; + int rc; + + memset(&cfg, 0, sizeof(cfg)); + + /* per-source override wins; a source without one takes the daemon + * default; the daemon default itself is optional with a sane built-in + * (design.md 3.4: stale_limit in esd.yaml, fail-open keep-old-value). */ + write_file(g_tmpdir, "esd.yaml", + "controller: 172.31.1.30:9990\n" + "poll_interval: 10\n" + "stale_limit: 200\n" + "resources:\n" + " - name: lic_a\n" + " collector: script\n" + " command: /bin/a.sh\n" + " stale_limit: 60\n" + " - name: lic_b\n" + " collector: script\n" + " command: /bin/b.sh\n"); + + rc = esd_config_load_path(g_tmpdir, "esd.yaml", &cfg); + TEST_ASSERT_EQUAL_INT(0, rc); + TEST_ASSERT_EQUAL_INT(200, cfg.stale_limit); + TEST_ASSERT_EQUAL_INT(60, cfg.src[0].stale_limit); + TEST_ASSERT_EQUAL_INT(0, cfg.src[1].stale_limit); /* 0 = use default */ + + esd_config_fini(&cfg); +} + +/* ── Phase 2: stale predicate ───────────────────────────────────────── */ + +static void test_stale_predicate(void) +{ + esd_source_t src; + + memset(&src, 0, sizeof(src)); + src.name = (char*)"lic"; + src.have_value = true; + src.last_value = 3.0; + + /* Never report a value -> nothing to call stale; admission treats the + * pool as 0 through the normal NO_RESOURCE path. */ + src.last_ok_time = 0; + TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60)); + + /* Fresh success (10s ago, limit 60) -> not stale. */ + src.last_ok_time = 90; + TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60)); + + /* Exactly at the limit -> not stale (strictly-greater rule, so an + * interval == stale_limit config never flaps). */ + src.last_ok_time = 40; + TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60)); + + /* Past the limit -> stale. */ + src.last_ok_time = 39; + TEST_ASSERT_TRUE(esd_source_is_stale(&src, 100, 60)); + + /* A source whose own stale_limit override is set uses it. */ + src.stale_limit = 10; + TEST_ASSERT_TRUE(esd_source_is_stale(&src, 100, 60)); + src.last_ok_time = 95; + TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60)); + + /* last_ok_time newer than `now` (clock skew between test harness and + * caller) must not read as stale. */ + src.last_ok_time = 105; + TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60)); +} + +/* ── Phase 2: status file ───────────────────────────────────────────── */ + +static void test_status_write_basic(void) +{ + esd_config_t cfg; + esd_stats_t stats; + char path[512]; + char* text; + time_t now = time(NULL); + char expect_lvt[128]; + + memset(&cfg, 0, sizeof(cfg)); + cfg.nsrc = 1; + cfg.src = (esd_source_t*)calloc(1, sizeof(esd_source_t)); + TEST_ASSERT_NOT_NULL(cfg.src); + cfg.src[0].name = (char*)"extlic"; + cfg.src[0].have_value = true; + cfg.src[0].last_value = 3.0; + /* fresh success: last good value 5s ago, limit default 150 -> not + * stale (the marks-stale test below covers the other side) */ + cfg.src[0].last_ok_time = now - 5; + cfg.stale_limit = ESD_STALE_LIMIT_DEFAULT; + + memset(&stats, 0, sizeof(stats)); + stats.started_at = 1000; + stats.pushes_ok = 7; + stats.pushes_fail = 1; + + snprintf(path, sizeof(path), "%s/status.json", g_tmpdir); + TEST_ASSERT_EQUAL_INT(0, esd_status_write(path, &cfg, &stats)); + + text = slurp(path); + TEST_ASSERT_NOT_NULL(text); + + /* The fields xceadmin extres status reads, and the stale flag driven + * by the same predicate the tests above pin. */ + TEST_ASSERT_TRUE(strstr(text, "\"daemon\": \"xceesd\"") != NULL); + TEST_ASSERT_TRUE(strstr(text, "\"name\": \"extlic\"") != NULL); + snprintf(expect_lvt, sizeof(expect_lvt), "\"last_value\": %.17g", 3.0); + TEST_ASSERT_TRUE(strstr(text, expect_lvt) != NULL); + TEST_ASSERT_TRUE(strstr(text, "\"last_ok_time\": ") != NULL); + TEST_ASSERT_TRUE(strstr(text, "\"stale\": false") != NULL); + TEST_ASSERT_TRUE(strstr(text, "\"pushes_ok\": 7") != NULL); + TEST_ASSERT_TRUE(strstr(text, "\"pushes_fail\": 1") != NULL); + + free(text); + free(cfg.src); +} + +static void test_status_write_marks_stale(void) +{ + esd_config_t cfg; + esd_stats_t stats; + char path[512]; + char* text; + time_t now = time(NULL); + + memset(&cfg, 0, sizeof(cfg)); + cfg.nsrc = 1; + cfg.src = (esd_source_t*)calloc(1, sizeof(esd_source_t)); + TEST_ASSERT_NOT_NULL(cfg.src); + cfg.src[0].name = (char*)"extlic"; + cfg.src[0].have_value = true; + cfg.src[0].last_value = 2.0; + /* last success 1 hour ago, limit 150s -> stale. */ + cfg.src[0].last_ok_time = now - 3600; + cfg.src[0].stale_limit = 150; + + memset(&stats, 0, sizeof(stats)); + + snprintf(path, sizeof(path), "%s/status.json", g_tmpdir); + TEST_ASSERT_EQUAL_INT(0, esd_status_write(path, &cfg, &stats)); + + text = slurp(path); + TEST_ASSERT_NOT_NULL(text); + TEST_ASSERT_TRUE(strstr(text, "\"stale\": true") != NULL); + + free(text); + free(cfg.src); +} + int main(void) { UNITY_BEGIN(); @@ -196,6 +384,10 @@ int main(void) RUN_TEST(test_parse_output_name_value); RUN_TEST(test_parse_output_whitespace_and_negative); RUN_TEST(test_parse_output_garbage_is_nan); + RUN_TEST(test_config_stale_limit_parse); + RUN_TEST(test_stale_predicate); + RUN_TEST(test_status_write_basic); + RUN_TEST(test_status_write_marks_stale); return UNITY_END(); } diff --git a/src/frontend/apiserver/backend/convert.go b/src/frontend/apiserver/backend/convert.go index 5df2b2af..0993535b 100644 --- a/src/frontend/apiserver/backend/convert.go +++ b/src/frontend/apiserver/backend/convert.go @@ -286,12 +286,13 @@ func cQueueToModel(queue *C.xce_queue_t) *models.QueueInfo { // cResourceToModel extracts C xce_resource_t fields and maps them to ResourceDefinitionInfo. func cResourceToModel(res *C.xce_resource_t) *models.ResourceDefinitionInfo { return rawResourceToModel(rawResourceData{ - name: goString(res.name), - desc: goString(res.desc), - valType: int(res.valtype), - ordType: int(res.ordtype), - flags: int(res.flags), - interval: int(res.interval), + name: goString(res.name), + desc: goString(res.desc), + valType: int(res.valtype), + ordType: int(res.ordtype), + flags: int(res.flags), + interval: int(res.interval), + lastReport: int64(res.last_report), }) } diff --git a/src/frontend/apiserver/backend/convert_pure.go b/src/frontend/apiserver/backend/convert_pure.go index feb0cf8d..5c6a26f7 100644 --- a/src/frontend/apiserver/backend/convert_pure.go +++ b/src/frontend/apiserver/backend/convert_pure.go @@ -578,12 +578,13 @@ type rawQueueData struct { // rawResourceData mirrors the C xce_resource_t fields used by the Go model. type rawResourceData struct { - name string - desc string - valType int - ordType int - flags int - interval int + name string + desc string + valType int + ordType int + flags int + interval int + lastReport int64 } // interactiveURLFromExdata extracts the interactive session URL from a job @@ -828,6 +829,7 @@ func rawResourceToModel(r rawResourceData) *models.ResourceDefinitionInfo { OrderType: r.ordType, Flag: r.flags, Interval: r.interval, + LastReport: r.lastReport, } } diff --git a/src/frontend/apiserver/backend/handler.go b/src/frontend/apiserver/backend/handler.go index d484f839..bd6140fe 100644 --- a/src/frontend/apiserver/backend/handler.go +++ b/src/frontend/apiserver/backend/handler.go @@ -5,6 +5,7 @@ import ( "apiserver/store" logger "apiserver/utils" "sync" + "time" "go.uber.org/zap" ) @@ -249,6 +250,23 @@ func (h *DefaultHandler) OnResourceLocationUpdated(res *models.ResourceDistribut key := res.ResourceName + ":" + res.Location store.ResourceLocations.Put(key, *res) + // external-license-resources Phase 2: the definition store is fed ONLY + // by the join-time config push, whose snapshot carries whatever the + // config tree had at push time -- so an EXTERNAL pool's last_report + // would stay 0 ("never") for the whole apiserver process lifetime even + // while the extres daemon pushes normally. A shared-value push (the + // daemon's landing shape: Location "") for an EXTERNAL definition is + // exactly an accepted external report at the controller, so refresh the + // stamp here. Host-sourced values (Location != "") and non-EXTERNAL + // resources must not touch it: last_report means "external value last + // accepted", and worker reports are a different freshness domain. + if res.Location == "" { + if def, ok := store.Resources.Get(res.ResourceName); ok && (def.Flag&models.EXTERNAL) != 0 { + def.LastReport = time.Now().Unix() + store.Resources.Put(def.ResourceName, def) + } + } + // resource-value-verification task 4.2 (Requirements 5.1, 5.2, 5.4): trace // the stored value at its point of application, mirroring the scheduler's // sibling trace log added in task 4.1 (core_events.c). One typed "value" diff --git a/src/frontend/apiserver/backend/integration_test.go b/src/frontend/apiserver/backend/integration_test.go index baa1cd98..ef5ac6fb 100644 --- a/src/frontend/apiserver/backend/integration_test.go +++ b/src/frontend/apiserver/backend/integration_test.go @@ -8,6 +8,7 @@ import ( "apiserver/utils" "sync/atomic" "testing" + "time" "go.uber.org/zap" ) @@ -516,3 +517,71 @@ func TestIntegration_CrossFeature_SequentialConfigPushes(t *testing.T) { t.Errorf("heartbeatIntervalSecs = %d, want 60", hb) } } + +// --- 5.6 Phase 2 (external-license-resources): a shared-value push refreshes +// the stored EXTERNAL definition's last_report. The definition store is fed +// only by the join-time config push, whose snapshot carries whatever the +// config tree had at push time — without this refresh the CLI's LAST_UPDATED +// column would show "never" for the whole apiserver process lifetime even +// while the extres daemon pushes normally. The daemon landing shape for a +// cluster-level value is Location=="" (host:"" on the wire). Worker-sourced +// host values must NOT touch the stamp. --- +func TestIntegration_ExternalSharedPushRefreshesDefinitionLastReport(t *testing.T) { + resetIntegrationStores() + h := &DefaultHandler{} + + // config push seeds the definition store: extlic is EXTERNAL (0x10), + // mem is a plain host-based resource. + dispatchConfigPush(h, nil, []models.ResourceDefinitionInfo{ + {ResourceName: "extlic", Flag: 0x80 | 0x10, LastReport: 0}, + {ResourceName: "mem", Flag: 0x80, LastReport: 0}, + }, nil, &models.ClusterInfo{ClusterName: "c1"}) + + before := time.Now().Unix() + + // daemon-shaped shared push for extlic (Location "") + h.OnResourceLocationUpdated(&models.ResourceDistributionInfo{ + ResourceName: "extlic", Location: "", NumericValue: 4.0, + ValueType: int(models.NUMERIC), IsDynamic: true, + }) + + got, ok := store.Resources.Get("extlic") + if !ok { + t.Fatal("extlic definition should exist after config push") + } + if got.LastReport < before { + t.Errorf("extlic LastReport = %d, want >= %d (refreshed by the shared push)", got.LastReport, before) + } + + // host-sourced push for mem must not stamp its definition + h.OnResourceLocationUpdated(&models.ResourceDistributionInfo{ + ResourceName: "mem", Location: "node01", NumericValue: 65536.0, + ValueType: int(models.NUMERIC), IsDynamic: true, + }) + gotMem, _ := store.Resources.Get("mem") + if gotMem.LastReport != 0 { + t.Errorf("mem LastReport = %d, want 0 (host values never stamp)", gotMem.LastReport) + } +} + +// The refresh only applies to EXTERNAL definitions: a shared push for a +// non-EXTERNAL resource (e.g. a dynamic shared pool reported by workers) +// leaves last_report untouched. +func TestIntegration_SharedPushDoesNotStampNonExternalDefinition(t *testing.T) { + resetIntegrationStores() + h := &DefaultHandler{} + + dispatchConfigPush(h, nil, []models.ResourceDefinitionInfo{ + {ResourceName: "plainshared", Flag: 0x80 | 0x08, LastReport: 0}, + }, nil, &models.ClusterInfo{ClusterName: "c1"}) + + h.OnResourceLocationUpdated(&models.ResourceDistributionInfo{ + ResourceName: "plainshared", Location: "", NumericValue: 2.0, + ValueType: int(models.NUMERIC), IsDynamic: true, + }) + + got, _ := store.Resources.Get("plainshared") + if got.LastReport != 0 { + t.Errorf("plainshared LastReport = %d, want 0 (not EXTERNAL)", got.LastReport) + } +} diff --git a/src/frontend/apiserver/models/resources.go b/src/frontend/apiserver/models/resources.go index e6afde0c..dca0441d 100644 --- a/src/frontend/apiserver/models/resources.go +++ b/src/frontend/apiserver/models/resources.go @@ -67,6 +67,10 @@ var RESOURCE_DEFINITION_DEFAULT_COLUMN = []string{ "order_type", "flag", "interval", + // external-license-resources Phase 2: SelectFields drops row keys not + // listed here, so the external value's last-report timestamp must be a + // default column or it never reaches API clients / the CLI. + "last_report", } var ResourceFieldMappings = map[string]string{ @@ -77,8 +81,12 @@ var ResourceFieldMappings = map[string]string{ var ResourcesAvailableTypes = []string{"static", "dynamic"} const ( - BUILTIN = 0x01 - DYNAMIC = 0x02 + BUILTIN = 0x01 + DYNAMIC = 0x02 + // external-license-resources: EXTERNAL=0x10 mirrors C's XCE_RESF_EXTERNAL + // (src/include/xce/types.h) — the value comes from an external daemon, + // not from workers. + EXTERNAL = 0x10 RELEASE = 0x20 CONSUMABLE = 0x80 ) @@ -132,6 +140,9 @@ type ResourceDefinitionInfo struct { OrderType int `json:"order_type"` Flag int `json:"flag"` Interval int `json:"interval"` + // external-license-resources Phase 2: epoch seconds the external value + // was last accepted by the controller (0 = never / not external). + LastReport int64 `json:"last_report"` } var RESOURCE_DISTRIBUTION_SUPPORT_SORT = []string{ diff --git a/src/frontend/apiserver/models/resources_test.go b/src/frontend/apiserver/models/resources_test.go new file mode 100644 index 00000000..175d1051 --- /dev/null +++ b/src/frontend/apiserver/models/resources_test.go @@ -0,0 +1,47 @@ +package models + +import ( + "encoding/json" + "testing" +) + +// external-license-resources Phase 2: last_report must be part of the default +// column list for /resources/definition — SelectFields drops any row key not +// listed there, so without it the API response (and the CLI LAST_UPDATED +// column) never sees the timestamp even though the model carries it. +func TestResourceDefinitionDefaultColumnIncludesLastReport(t *testing.T) { + found := false + for _, col := range RESOURCE_DEFINITION_DEFAULT_COLUMN { + if col == "last_report" { + found = true + break + } + } + if !found { + t.Errorf("RESOURCE_DEFINITION_DEFAULT_COLUMN = %v, missing \"last_report\"", RESOURCE_DEFINITION_DEFAULT_COLUMN) + } +} + +// ResourceDefinitionInfo must serialize last_report with the json tag the CLI +// model (xcecli/models/resource.go) unmarshals. +func TestResourceDefinitionInfoJSONCarriesLastReport(t *testing.T) { + info := ResourceDefinitionInfo{ + ResourceName: "extlic", + LastReport: 1800000000, + } + data, err := json.Marshal(info) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + var m map[string]interface{} + if err := json.Unmarshal(data, &m); err != nil { + t.Fatalf("json.Unmarshal: %v", err) + } + v, ok := m["last_report"] + if !ok { + t.Fatalf("serialized body %s has no last_report key", data) + } + if v.(float64) != 1800000000 { + t.Errorf("last_report = %v, want 1800000000", v) + } +} diff --git a/src/frontend/xcecli/common/output/page_print_resources.go b/src/frontend/xcecli/common/output/page_print_resources.go index 32469cf4..eb9386a8 100644 --- a/src/frontend/xcecli/common/output/page_print_resources.go +++ b/src/frontend/xcecli/common/output/page_print_resources.go @@ -3,6 +3,7 @@ package output import ( "fmt" "strings" + "time" "xcecli/models" ) @@ -143,6 +144,8 @@ func (p *ResourcesPagePrinter) initColumnWidthsForDef(resourcesDef *models.Resou width = len(resourceDef.ResourceType) case models.HEADER_RESOURCE_DEF_DESC: width = len(resourceDef.Description) + case models.HEADER_RESOURCE_DEF_LAST_REPORT: + width = len(formatJobTime(time.Unix(resourceDef.LastReport, 0))) } if !p.wideFormat && width > 15 { @@ -270,6 +273,8 @@ func (p *ResourcesPagePrinter) formatRowsForDef(resourcesDef *models.ResourceDef row = appendOrDefault(row, FormatStringField(builtinValue, width, truncatFlag, true), "-") case models.HEADER_RESOURCE_DEF_INTERVAL: row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, resourceDef.Interval), "0") + case models.HEADER_RESOURCE_DEF_LAST_REPORT: + row = appendOrDefault(row, FormatStringField(fmtUnixTime(resourceDef.LastReport), width, truncatFlag, true), "-") } } rows = append(rows, row) @@ -354,6 +359,8 @@ func printResourcesDefVerboseOutput(resourceDef *models.ResourceDefResponse) { val = releaseValue case models.HEADER_RESOURCE_DEF_INTERVAL: val = fmt.Sprintf("%d", resourceDef.Interval) + case models.HEADER_RESOURCE_DEF_LAST_REPORT: + val = fmtUnixTime(resourceDef.LastReport) } widths1[i] = max(len(h), len(val)) row1 = append(row1, FormatStringField(val, widths1[i], false, true)) diff --git a/src/frontend/xcecli/models/resource.go b/src/frontend/xcecli/models/resource.go index cd72068f..34927b8d 100644 --- a/src/frontend/xcecli/models/resource.go +++ b/src/frontend/xcecli/models/resource.go @@ -7,17 +7,18 @@ import ( ) const ( - HEADER_RESOURCE_NAME = "RESOURCE" - HEADER_RESOURCE_VALUE = "VALUE" - HEADER_RESOURCE_LOC = "LOCATION" - HEADER_RESOURCE_DEF_TYPE = "TYPE" - HEADER_RESOURCE_DEF_ORDER = "ORDER" - HEADER_RESOURCE_DEF_BUILTIN = "BUILTIN" - HEADER_RESOURCE_DEF_CONSUMABLE = "CONSUMABLE" - HEADER_RESOURCE_DEF_DYNAMIC = "DYNAMIC" - HEADER_RESOURCE_DEF_RELEASE = "RELEASE" - HEADER_RESOURCE_DEF_INTERVAL = "INTERVAL" - HEADER_RESOURCE_DEF_DESC = "DESCRIPTION" + HEADER_RESOURCE_NAME = "RESOURCE" + HEADER_RESOURCE_VALUE = "VALUE" + HEADER_RESOURCE_LOC = "LOCATION" + HEADER_RESOURCE_DEF_TYPE = "TYPE" + HEADER_RESOURCE_DEF_ORDER = "ORDER" + HEADER_RESOURCE_DEF_BUILTIN = "BUILTIN" + HEADER_RESOURCE_DEF_CONSUMABLE = "CONSUMABLE" + HEADER_RESOURCE_DEF_DYNAMIC = "DYNAMIC" + HEADER_RESOURCE_DEF_RELEASE = "RELEASE" + HEADER_RESOURCE_DEF_INTERVAL = "INTERVAL" + HEADER_RESOURCE_DEF_LAST_REPORT = "LAST_UPDATED" + HEADER_RESOURCE_DEF_DESC = "DESCRIPTION" ) var ResourceDistFieldsMap = map[string]string{ @@ -28,21 +29,23 @@ var ResourceDistFieldsMap = map[string]string{ // ResourceDefFieldsMap maps resource definition field headers to API field names var ResourceDefFieldsMap = map[string]string{ - HEADER_RESOURCE_NAME: "res_name", - HEADER_RESOURCE_DEF_DESC: "description", - HEADER_RESOURCE_DEF_TYPE: "type", - HEADER_RESOURCE_DEF_ORDER: "order", - HEADER_RESOURCE_DEF_BUILTIN: "builtin", - HEADER_RESOURCE_DEF_CONSUMABLE: "consumable", - HEADER_RESOURCE_DEF_DYNAMIC: "dynamic", - HEADER_RESOURCE_DEF_RELEASE: "release", - HEADER_RESOURCE_DEF_INTERVAL: "interval", + HEADER_RESOURCE_NAME: "res_name", + HEADER_RESOURCE_DEF_DESC: "description", + HEADER_RESOURCE_DEF_TYPE: "type", + HEADER_RESOURCE_DEF_ORDER: "order", + HEADER_RESOURCE_DEF_BUILTIN: "builtin", + HEADER_RESOURCE_DEF_CONSUMABLE: "consumable", + HEADER_RESOURCE_DEF_DYNAMIC: "dynamic", + HEADER_RESOURCE_DEF_RELEASE: "release", + HEADER_RESOURCE_DEF_INTERVAL: "interval", + HEADER_RESOURCE_DEF_LAST_REPORT: "last_report", } var ResourceDefDetailHeaders = []string{ HEADER_RESOURCE_DEF_TYPE, HEADER_RESOURCE_DEF_ORDER, HEADER_RESOURCE_DEF_INTERVAL, + HEADER_RESOURCE_DEF_LAST_REPORT, HEADER_RESOURCE_DEF_BUILTIN, HEADER_RESOURCE_DEF_DYNAMIC, HEADER_RESOURCE_DEF_RELEASE, @@ -177,6 +180,7 @@ type ResourceDef struct { OrderType string `json:"-"` // populated from order_type int Flag map[string]interface{} `json:"flag"` Interval int `json:"interval"` + LastReport int64 `json:"last_report"` // Unix seconds of the last external value report (0 = never) } func (r *ResourceDef) UnmarshalJSON(data []byte) error { @@ -187,6 +191,7 @@ func (r *ResourceDef) UnmarshalJSON(data []byte) error { OrderType string `json:"order_type"` Flag map[string]interface{} `json:"flag"` Interval int `json:"interval"` + LastReport int64 `json:"last_report"` } var a Alias if err := UnmarshalOrderMapRow(data, &a); err != nil { @@ -198,5 +203,6 @@ func (r *ResourceDef) UnmarshalJSON(data []byte) error { r.OrderType = a.OrderType r.Flag = a.Flag r.Interval = a.Interval + r.LastReport = a.LastReport return nil } diff --git a/src/scheduler/src/core_events.c b/src/scheduler/src/core_events.c index 836d8ed1..47a4caec 100644 --- a/src/scheduler/src/core_events.c +++ b/src/scheduler/src/core_events.c @@ -1184,10 +1184,16 @@ int xce_core_push_events(xce_sched_core_t* core, xce_core_event_t* events, if ((key_find(core->resmgr, res->name, (void**)&found) == 0) && (found != NULL)) { - found->valtype = res->valtype; - found->ordtype = res->ordtype; - found->flags = res->flags; - found->interval = res->interval; + found->valtype = res->valtype; + found->ordtype = res->ordtype; + found->flags = res->flags; + found->interval = res->interval; + /* external-license-resources Phase 2: the extres + * freshness stamp (controller stamps it at each + * accepted push); resobj.c's makeResource excludes + * last_report from definition comparison, so this + * is metadata, not an identity change. */ + found->last_report = res->last_report; } else { WARN("%s RESOURCES_PUSH: makeResource failed for " "%s and lookup not found", diff --git a/src/xceadmin/src/xceadmin/cli.py b/src/xceadmin/src/xceadmin/cli.py index 471feabb..c4bb0a1f 100644 --- a/src/xceadmin/src/xceadmin/cli.py +++ b/src/xceadmin/src/xceadmin/cli.py @@ -27,11 +27,14 @@ traceback. from __future__ import annotations +import json import os +import time from collections.abc import Callable from datetime import datetime, timezone from importlib.metadata import PackageNotFoundError from importlib.metadata import version as _package_version +from pathlib import Path import click from click.core import ParameterSource @@ -43,6 +46,7 @@ from xcecommon.errors import ( SchedAdminError, TooManyRedirectsError, XceAdminError, + XceExtresStatusError, ) from xceadmin.build import build_cluster_config, dump_cluster_config, write_output from xceadmin.push import probe_config, push_config @@ -372,6 +376,126 @@ def scheduler_dump(file: str) -> None: click.echo(text, nl=False) # dump() already ends its own text with "\n" (5.20). +# --------------------------------------------------------------------------- +# extres status (external-license-resources Phase 2 item 4, design.md 3.5): +# read the status JSON xceesd writes on its own host and render it for the +# operator. A LOCAL-file command like scheduler dump -- no --controller, no +# wire: the daemon's status file is the observability surface until the +# PR-2 proxy formalizes a status channel (design.md 3.6; a temporary +# remote protocol would be redesigned out from under us by PR-2's +# provider-lease model). Run it on the daemon's host. +# --------------------------------------------------------------------------- + +# Where the deployed esd.yaml's status_file default points. Matches +# esd_config.c's status_file default and the deploy docs; overridable with +# --status-file (the t1 test cluster uses a /tmp path instead). +DEFAULT_STATUS_FILE = "/var/run/xce/esd-status.json" + + +def _format_extres_status(payload: dict, now: int | None = None) -> str: + """Renders one decoded esd status object as operator text. + + ``now`` pins the wall clock for deterministic tests (None = time.time()). + Rendering rules: + - daemon identity line: name + pid + - uptime and push counters on their own lines + - per-source rows: NAME value= age=s [-- STALE] or + "no value yet" when last_value is null. + """ + if now is None: + now = int(time.time()) + + lines = [ + f"daemon: {payload.get('daemon', 'xceesd')} (pid {payload.get('pid', '?')})", + ] + + started_at = payload.get("started_at", 0) + if isinstance(started_at, int) and started_at > 0: + uptime = max(now - started_at, 0) + started_str = datetime.fromtimestamp(started_at, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%S") + lines.append(f"started at: {started_str} (up {uptime}s)") + + lines.append( + f"pushes: {payload.get('pushes_ok', 0)} ok, " + f"{payload.get('pushes_fail', 0)} failed" + ) + + sources = payload.get("sources", []) + if not sources: + lines.append("sources: (none configured)") + return "\n".join(lines) + + lines.append("sources:") + for src in sources: + name = src.get("name", "?") + if src.get("last_value") is None: + lines.append(f" {name}: no value yet") + continue + last_ok = src.get("last_ok_time", 0) + age = max(now - last_ok, 0) if isinstance(last_ok, int) and last_ok > 0 else 0 + stale = " -- STALE" if src.get("stale") else "" + lines.append(f" {name}: value={src.get('last_value')} age={age}s{stale}") + + return "\n".join(lines) + + +def _read_extres_status(path: str) -> dict: + """Reads and validates one esd status file. Raises + :class:`XceExtresStatusError` for every local failure mode (unreadable + path, unparseable JSON, JSON that is not an esd status object) so the + CLI layer never sees a raw OSError/JSONDecodeError. + """ + try: + raw = Path(path).read_text() + except OSError as exc: + raise XceExtresStatusError( + f"cannot read extres status file {path!r}: {exc}", path=path, reason=str(exc) + ) from exc + + try: + payload = json.loads(raw) + except json.JSONDecodeError as exc: + raise XceExtresStatusError( + f"extres status file {path!r} is not valid JSON: {exc}", path=path, reason=str(exc) + ) from exc + + if not isinstance(payload, dict) or "sources" not in payload or "daemon" not in payload: + raise XceExtresStatusError( + f"extres status file {path!r} is not an xceesd status object " + "(missing 'daemon' or 'sources')", + path=path, + reason="wrong schema", + ) + return payload + + +@cli.group() +def extres() -> None: + """Inspect the external-resources daemon (xceesd).""" + + +@extres.command("status") +@click.option( + "--status-file", + default=None, + help=f"Path of the xceesd status JSON (default: {DEFAULT_STATUS_FILE}).", +) +def extres_status(status_file: str | None) -> None: + """Report xceesd's status from its local status file (run on the daemon's host).""" + # Resolved at call time, not decoration time, so tests (and wrappers) + # can re-point the default -- click freezes a non-None default= into + # the command's own default_value at decoration. + if status_file is None: + status_file = DEFAULT_STATUS_FILE + try: + payload = _read_extres_status(status_file) + click.echo(_format_extres_status(payload)) + except XceAdminError as e: + raise click.ClickException(e.message) from e + except KeyboardInterrupt: + raise click.ClickException("cancelled") from None + + def main() -> None: """Console-script entry point (see pyproject.toml [project.scripts]).""" cli() diff --git a/src/xceadmin/src/xcecommon/errors.py b/src/xceadmin/src/xcecommon/errors.py index 480af1c7..7bec4230 100644 --- a/src/xceadmin/src/xcecommon/errors.py +++ b/src/xceadmin/src/xcecommon/errors.py @@ -222,3 +222,23 @@ class SchedAdminError(XceAdminError): self.file = file self.event_count = event_count self.started_at = started_at + + +class XceExtresStatusError(XceAdminError): + """The xceesd status file (``xceadmin extres status``) could not be + read or decoded (external-license-resources Phase 2 item 4, design.md + 3.5; raised by ``xceadmin.cli._read_extres_status``). + + Covers every local-file failure mode: the path doesn't exist / isn't + readable, the bytes aren't valid JSON, or the JSON parses but isn't an + esd status object (missing ``daemon``/``sources`` -- e.g. someone + pointed ``--status-file`` at some other JSON file). Same local-file + concern as :class:`ReplayFileError`/:class:`ClusterBinFileError`: no + network framing involved -- ``extres status`` reads the status JSON + xceesd writes on its own host. + """ + + def __init__(self, message: str, path: str, reason: str) -> None: + super().__init__(message) + self.path = path + self.reason = reason diff --git a/src/xceadmin/tests/test_cli_extres.py b/src/xceadmin/tests/test_cli_extres.py new file mode 100644 index 00000000..a8c4c8e6 --- /dev/null +++ b/src/xceadmin/tests/test_cli_extres.py @@ -0,0 +1,275 @@ +"""Tests for the ``extres`` command group (external-license-resources +Phase 2 item 4, design.md 3.5 "``xceadmin extres status``"). + +``extres status`` is a LOCAL-file command -- the inverse slice of +``scheduler dump``: where ``dump()`` proved xceadmin can be a local-file +decoder with no network at all, ``extres status`` reads the status JSON +``xceesd`` writes on its own host (``esd_status_write``, +``src/esd/src/esd_status.c``) and formats it for the operator. No wire, +no ``--controller``, no TLS -- the daemon's status file is deliberately +the observability surface until the PR-2 proxy formalizes a status +channel (design.md 3.6; a temporary remote protocol would be +redesigned out from under us by PR-2's provider-lease model). + +Split into its own file (rather than appended to ``test_cli.py``, 1187 +lines at the time of writing) to keep both files under the project's +800-line guideline -- the same split precedent as +``test_cli_scheduler_mock.py``. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from xceadmin import cli as cli_module +from xceadmin.cli import cli + + +# --------------------------------------------------------------------------- +# Fixtures: realistic status JSON in the exact shape esd_status_write() +# emits (field order aside -- we parse, we never substring-match). +# --------------------------------------------------------------------------- + +_NOW = 1_800_000_000 # a fixed "now" for deterministic age rendering + + +@pytest.fixture +def status_file(tmp_path: Path) -> Path: + """A status file exactly as esd_status_write() writes it for two + sources: one fresh, one stale (last_ok_time 3600s in the past of + _NOW against the daemon's 150s default limit). + """ + path = tmp_path / "esd-status.json" + payload = { + "daemon": "xceesd", + "pid": 12345, + "started_at": _NOW - 7200, + "generated_at": _NOW, + "pushes_ok": 142, + "pushes_fail": 3, + "sources": [ + { + "name": "Verilog_XL", + "last_value": 2.0, + "last_ok_time": _NOW - 5, + "stale": False, + }, + { + "name": "VCS", + "last_value": 7.0, + "last_ok_time": _NOW - 3600, + "stale": True, + }, + ], + } + path.write_text(json.dumps(payload)) + return path + + +def _freeze_now(monkeypatch: pytest.MonkeyPatch) -> None: + """Pin time.time() to _NOW so age rendering is deterministic.""" + monkeypatch.setattr(cli_module.time, "time", lambda: _NOW) + + +# --------------------------------------------------------------------------- +# Formatting helper: pure functions tested directly (the _format_record_ +# result precedent in test_cli.py). +# --------------------------------------------------------------------------- + + +def test_format_status_renders_daemon_identity_and_counters() -> None: + from xceadmin.cli import _format_extres_status + + payload = json.loads( + (Path(__file__).parent / "data" / "esd-status-sample.json").read_text() + ) if (Path(__file__).parent / "data" / "esd-status-sample.json").exists() else { + "daemon": "xceesd", + "pid": 12345, + "started_at": _NOW - 7200, + "generated_at": _NOW, + "pushes_ok": 142, + "pushes_fail": 3, + "sources": [], + } + + text = _format_extres_status(payload, now=_NOW) + + assert "xceesd" in text + assert "12345" in text + assert "142" in text + assert "3" in text + + +def test_format_status_marks_stale_source_with_age() -> None: + from xceadmin.cli import _format_extres_status + + payload = { + "daemon": "xceesd", + "pid": 1, + "started_at": _NOW - 100, + "generated_at": _NOW, + "pushes_ok": 0, + "pushes_fail": 0, + "sources": [ + { + "name": "VCS", + "last_value": 7.0, + "last_ok_time": _NOW - 3600, + "stale": True, + } + ], + } + + text = _format_extres_status(payload, now=_NOW) + + assert "VCS" in text + assert "STALE" in text + assert "3600s" in text + + +def test_format_status_fresh_source_shows_age_only() -> None: + from xceadmin.cli import _format_extres_status + + payload = { + "daemon": "xceesd", + "pid": 1, + "started_at": _NOW - 100, + "generated_at": _NOW, + "pushes_ok": 0, + "pushes_fail": 0, + "sources": [ + { + "name": "Verilog_XL", + "last_value": 2.0, + "last_ok_time": _NOW - 5, + "stale": False, + } + ], + } + + text = _format_extres_status(payload, now=_NOW) + + assert "Verilog_XL" in text + assert "STALE" not in text + assert "5s" in text + + +def test_format_status_source_with_no_value_yet() -> None: + from xceadmin.cli import _format_extres_status + + payload = { + "daemon": "xceesd", + "pid": 1, + "started_at": _NOW - 100, + "generated_at": _NOW, + "pushes_ok": 0, + "pushes_fail": 0, + "sources": [ + { + "name": "Verilog_XL", + "last_value": None, + "last_ok_time": 0, + "stale": False, + } + ], + } + + text = _format_extres_status(payload, now=_NOW) + + assert "Verilog_XL" in text + assert "no value yet" in text + + +# --------------------------------------------------------------------------- +# CLI wiring: arg parsing, file reading, error translation. +# --------------------------------------------------------------------------- + + +def test_cli_help_shows_extres_command() -> None: + runner = CliRunner() + result = runner.invoke(cli, ["--help"]) + + assert result.exit_code == 0 + assert "extres" in result.output + + +def test_extres_status_reads_local_file(status_file: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _freeze_now(monkeypatch) + + runner = CliRunner() + result = runner.invoke(cli, ["extres", "status", "--status-file", str(status_file)]) + + assert result.exit_code == 0, result.output + assert "xceesd" in result.output + assert "Verilog_XL" in result.output + assert "VCS" in result.output + + +def test_extres_status_defaults_to_etc_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """With no --status-file, the default path is the deployed default + (/var/run/xce/esd-status.json -- where esd.yaml's production + status_file default points); a missing default file is a clean + error, not a traceback. + """ + monkeypatch.setattr( + cli_module, "DEFAULT_STATUS_FILE", str(tmp_path / "nonexistent.json") + ) + + runner = CliRunner() + result = runner.invoke(cli, ["extres", "status"]) + + assert result.exit_code != 0 + assert "nonexistent.json" in result.output + + +def test_extres_status_missing_file_clean_error(tmp_path: Path) -> None: + runner = CliRunner() + result = runner.invoke( + cli, ["extres", "status", "--status-file", str(tmp_path / "gone.json")] + ) + + assert result.exit_code != 0 + assert "gone.json" in result.output + assert "Traceback" not in result.output + + +def test_extres_status_invalid_json_clean_error(tmp_path: Path) -> None: + bad = tmp_path / "bad.json" + bad.write_text("{not json") + + runner = CliRunner() + result = runner.invoke(cli, ["extres", "status", "--status-file", str(bad)]) + + assert result.exit_code != 0 + assert "Traceback" not in result.output + + +def test_extres_status_wrong_schema_clean_error(tmp_path: Path) -> None: + """JSON that parses but is not an esd status object (e.g. someone + pointed --status-file at some other JSON) fails loud and clean. + """ + bad = tmp_path / "other.json" + bad.write_text(json.dumps({"hello": "world"})) + + runner = CliRunner() + result = runner.invoke(cli, ["extres", "status", "--status-file", str(bad)]) + + assert result.exit_code != 0 + assert "Traceback" not in result.output + + +def test_extres_status_no_controller_needed(monkeypatch: pytest.MonkeyPatch, status_file: Path) -> None: + """The command must not touch the controller: XCE_CONTROLLER_PEERS + pointing at a dead endpoint changes nothing (the file is local). + """ + monkeypatch.setenv("XCE_CONTROLLER_PEERS", "127.0.0.1:1") + monkeypatch.setenv("XCEADMIN_CONTROLLER", "127.0.0.1:1") + + runner = CliRunner() + result = runner.invoke(cli, ["extres", "status", "--status-file", str(status_file)]) + + assert result.exit_code == 0, result.output