lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

E2 Phase 2: external-license-resources 运维面 — stale 标记、last-updated 展示、xceadmin extres status

Browse source 
design.md 3.5 Phase 2（PEN D reason 专用名为可选项，复用 PREASON_NO_RESOURCE
已带资源名，跳过）：

① esd stale 标记与 WARN
  - esd_source_t 增 stale_limit/last_ok_time；esd_source_is_stale(): 从未有值
    不算 stale、时钟超前钳到 0、严格大于（interval==limit 不抖动）、
    ESD_STALE_LIMIT_DEFAULT 150；fail-open 保持旧值不变
  - 采集失败首轮一次性 WARN，过限后每轮升级为 STALE WARN（带 age/limit/kept
    值），恢复即静默
  - status 文件 esd_status_write(): daemon/pid/started_at/generated_at/
    pushes ok/fail/sources[{name,last_value,last_ok_time,stale}]，tmp+rename
    原子写；写失败仅状态转换时告警一次，绝不拖垮 poll 循环

② `xce resources` 展示 last-updated
  - controller worker.c: EXTERNAL 共享值被接受时盖 last_report——同时盖
    live resmgr 对象与 myconf 配置树副本（apiserver 定义 store 只由 join 时
    config push 供给，串行的是配置树，缺后者 CLI 永远显示 never）
  - scheduler core_events.c: RESOURCES_PUSH 原地更新拷贝 last_report（纯
    metadata，resobj.c 比较本就排除）
  - apiserver: model 带 last_report + 默认列（SelectFields 只投影默认列，
    缺失即 API/CLI 全链不可见）；convert cResourceToModel 透传
  - apiserver handler.go: OnResourceLocationUpdated 对 host=="" 且定义带
    EXTERNAL(0x10) 的推送刷新 store 内 LastReport——config push 快照之外
    的活刷新；host 来源值与非 EXTERNAL 资源绝不触碰
  - xcecli: LAST_UPDATED 列（-l verbose 表格/详情/JSON 三态），0 显示 "-"

④ `xceadmin extres status`
  - 本机读 status 文件（远程读取留待 PR-2，需新 wire 协议；LAST_UPDATED
    已覆盖远程新鲜度语义）——--status-file 显式路径或
    /var/run/xce/esd-status.json 默认，无 controller 连接依赖
  - XceExtresStatusError 本地文件类错误；click default=None 装饰期冻结坑
    用调用期解析

TDD：test_extres_resvals +2（配置树盖戳/非 EXTERNAL 不盖戳）、test_esd_core
+4（stale 判式/状态文件）、apiserver models +2（默认列/json tag）、backend
integration +2（EXTERNAL 刷新/非 EXTERNAL 不刷新）、xceadmin +11（渲染/
本地读/容错）。全绿：controller 1360、scheduler 100、apiserver（cgo+nocgo）、
esd 11、xceadmin 385。

t1 E2E 实证：esd join 即推送→API last_report=join 时刻→CLI LAST_UPDATED
"Oct 27 13:52"；破坏采集器后 60s 过限转 per-round STALE WARN + status
stale:true + fail-open 保值，恢复脚本即回 false；liccheck 作业提交→运行→
DONE，五 daemon 全健康。

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
lei.chen 2026-10-27 14:00:33 +00:00
parent 3eb1b9716d
commit 805288d747
20 changed files with 1179 additions and 47 deletions
Download patch file Download diff file Expand all files Collapse all files

95
src/controller/testsuite/test_extres_resvals.c
View file

@ -18,6 +18,12 @@
* endpoint, so the seam under test is worker.c's list applier, the same
* XCE_TESTING-exposure convention as apply_join_state in
* test_worker_join.c).
*
* external-license-resources Phase 2 adds rule 3: an EXTERNAL shared
* acceptance also stamps last_report on the myconf cluster-config tree's
* definition -- the apiserver's resource-definition store is fed ONLY by
* the join-time config push (myconf tree, not the live resmgr object), so
* without this the CLI's LAST_UPDATED column would stay "never" forever.
*/
/* rtti.h must come first — it defines 'var' and 'typeinfo_t' used by kvmgr.h */
@ -33,6 +39,7 @@
#include "cluster.h"
#include "controller.capnp.h"
#include "kvmgr.h"
#include "myconf.h"
#include "resobj.h"
#include "worker.h"
#include "xce/types.h"
@ -52,6 +59,12 @@ int update_resval_list(cluster_t* c, worker_t* w, xce_resval_list_t* l,
typedef struct {
cluster_t c;
var resmgr;
/* myconf fixture: one cluster_config_t owning a static resources[] tree
* (the config-push wire source, distinct from the resmgr objects). */
myconf_t myconf;
cluster_config_t ccfg;
xce_resource_t* cfg_res[2];
xce_resource_t cfg_defs[2];
} test_cluster_t;
static test_cluster_t g_te;
@ -90,6 +103,23 @@ static void setup_resmgr(void)
g_te.c.resmgr = g_te.resmgr;
zarray_init(&g_te.c.sharedinst, 64, NULL);
/* the myconf tree mirrors the resmgr registrations by name (the live
* shape: cluster_build_resources applies the config tree INTO resmgr,
* so names match; objects are distinct allocations) */
{
static const char* names[2] = { "lic", "slots" };
for (int i = 0; i < 2; i++) {
memset(&g_te.cfg_defs[i], 0, sizeof(g_te.cfg_defs[i]));
g_te.cfg_defs[i].name = (char*)names[i];
g_te.cfg_res[i] = &g_te.cfg_defs[i];
}
g_te.ccfg.nres = 2;
g_te.ccfg.resources = g_te.cfg_res;
g_te.myconf.cluster_config = &g_te.ccfg;
g_te.c.myconf = &g_te.myconf;
}
}
static void teardown_resmgr(void)
@ -192,6 +222,69 @@ static void test_extres_nonshared_resval_refused_eperm(void)
free(applied_vals);
}
/* Rule 3 (Phase 2): an EXTERNAL shared acceptance stamps last_report on the
* myconf cluster-config tree's definition with the current time — the
* config push the apiserver receives on (re)join serializes THAT tree, not
* the live resmgr object, so an unstamped tree would show "never" in the
* CLI's LAST_UPDATED column even while the daemon pushes normally. */
static void test_extres_external_accept_stamps_config_tree(void)
{
static const char* names[1] = { "lic" };
static const float values[1] = { 4.0f };
xce_resval_list_t* l;
time_t before;
time_t after;
setup_resmgr();
/* mark the definition EXTERNAL and confirm both copies start unstamped
* (config decode leaves last_report 0 for YAML-sourced definitions) */
{
xce_resource_t* live = NULL;
lock(g_te.resmgr, RTTI_LOCK_READ);
(void)key_find(g_te.resmgr, (void*)"lic", (void**)&live);
unlock(g_te.resmgr);
TEST_ASSERT_NOT_NULL(live);
live->flags |= XCE_RESF_EXTERNAL;
g_te.cfg_defs[0].flags |= XCE_RESF_EXTERNAL;
TEST_ASSERT_EQUAL_INT(0, live->last_report);
TEST_ASSERT_EQUAL_INT(0, g_te.cfg_defs[0].last_report);
}
time(&before);
l = make_resval_list(names, values, 1);
TEST_ASSERT_EQUAL_INT(0, update_resval_list(&g_te.c, NULL, l, NULL));
time(&after);
TEST_ASSERT_EQUAL_INT((int)before, (int)g_te.cfg_defs[0].last_report);
TEST_ASSERT_TRUE(g_te.cfg_defs[0].last_report >= before);
TEST_ASSERT_TRUE(g_te.cfg_defs[0].last_report <= after);
teardown_resmgr();
}
/* The stamp is EXTERNAL-gated: a plain SHARED (non-EXTERNAL) acceptance
* leaves the config tree untouched — last_report means "external value
* last accepted", and worker-sourced values must never set it. */
static void test_extres_plain_shared_accept_leaves_config_tree(void)
{
static const char* names[1] = { "lic" };
static const float values[1] = { 4.0f };
xce_resval_list_t* l;
setup_resmgr();
/* SHARED but NOT EXTERNAL */
l = make_resval_list(names, values, 1);
TEST_ASSERT_EQUAL_INT(0, update_resval_list(&g_te.c, NULL, l, NULL));
TEST_ASSERT_EQUAL_INT(0, g_te.cfg_defs[0].last_report);
teardown_resmgr();
}
/* ── suite entry point ─────────────────────────────────────────────── */
void test_extres_resvals(void)
@ -199,4 +292,6 @@ void test_extres_resvals(void)
UnitySetTestFile(__FILE__);
RUN_TEST(test_extres_shared_resval_lands_in_sharedinst);
RUN_TEST(test_extres_nonshared_resval_refused_eperm);
RUN_TEST(test_extres_external_accept_stamps_config_tree);
RUN_TEST(test_extres_plain_shared_accept_leaves_config_tree);
}

29
src/controller/worker.c
View file

@ -74,6 +74,35 @@ static int update_val_res(cluster_t* c, worker_t* w, xce_resource_t* r,
if (r->flags & XCE_RESF_SHARED) {
/* update shared resource */
array = &(c->sharedinst);
/* external-license-resources Phase 2: the descriptor's reserved
* last_report becomes the "external value freshness" stamp -- the
* extres daemon's accepted pushes are the only writers of an
* EXTERNAL shared value, so every acceptance refreshes it and
* `xce resources` / a stale read can tell "reported recently"
* from "kept old value" (design.md 3.5 运维面). The apiserver's
* definition store is fed by the join-time CONFIG push, which
* serializes c->myconf->cluster_config -- a DIFFERENT copy from
* this live resmgr object -- so that tree's same-named definition
* gets the same stamp or the CLI's LAST_UPDATED column would stay
* "never" between apiserver (re)joins. */
if (r->flags & XCE_RESF_EXTERNAL) {
time_t now = time(NULL);
r->last_report = now;
if ((c->myconf != NULL) && (c->myconf->cluster_config != NULL)) {
cluster_config_t* cc = c->myconf->cluster_config;
for (int i = 0; i < cc->nres; i++) {
xce_resource_t* cr = cc->resources[i];
if ((cr != NULL) && (cr->name != NULL)
&& (strcmp(cr->name, r->name) == 0)) {
cr->last_report = now;
break;
}
}
}
}
} else if (w == NULL) {
/* external-license-resources Phase 1: a NULL worker is the extres
* daemon (HostRole_extres join, no workermgr slot). It may only

40
src/esd/include/esd.h
View file

@ -29,7 +29,10 @@ typedef struct esd_source_s {
char* name; /* resource name, same as resources.yaml */
char* command; /* script collector: the command to run */
int interval; /* poll interval, seconds (0: use the daemon default) */
long stale_limit; /* seconds a kept value stays trustworthy (0: use
* the daemon default); design.md 3.4 */
double last_value; /* last successfully-parsed value */
time_t last_ok_time; /* when last_value was collected (0: never) */
bool have_value; /* last_value is valid (any push has succeeded) */
bool last_failed; /* last poll failed (stale-value policy input) */
} esd_source_t;
@ -39,10 +42,22 @@ typedef struct esd_config_s {
int controller_port;
char* peers; /* --peers style candidate list, or NULL */
int poll_interval; /* daemon default poll interval, seconds */
long stale_limit; /* daemon default stale limit, seconds */
char* status_file; /* status JSON path (NULL: no file) */
int nsrc;
esd_source_t* src; /* array of nsrc sources */
} esd_config_t;
/* Daemon-wide counters and identity for the status file (Phase 2,
* design.md 3.5 "xceadmin extres status"): the daemon fills one and hands
* it to esd_status_write each poll round. */
typedef struct esd_stats_s {
time_t started_at; /* daemon start, epoch seconds */
time_t now; /* snapshot time (filled by esd_status_write when 0) */
long pushes_ok; /* successful resval pushes since start */
long pushes_fail; /* failed pushes since start */
} esd_stats_t;
/*
* Parse esd.yaml:
*
@ -87,4 +102,29 @@ extern double esd_script_collect(const char* command);
* it is killed (default 10s; main.c overrides from the CLI). */
extern int esd_script_timeout_ms;
/* Default stale limit for sources (and the config default) -- design.md
* 3.4's example: 150s. A source whose collector keeps failing past this
* many seconds since its last good value reports stale=true in the status
* file (the daemon itself keeps the last value -- fail-open). */
#define ESD_STALE_LIMIT_DEFAULT 150
/* Stale predicate (Phase 2): true when the source's last good value is
* older than its stale limit -- `now` vs src->last_ok_time, the source's
* own stale_limit when set, else the daemon default. A source with no
* value yet is never stale (there is nothing being kept alive; admission
* already treats the pool as empty through the normal NO_RESOURCE path).
* A last_ok_time in the future (clock skew) is not stale. */
extern bool esd_source_is_stale(const esd_source_t* src, time_t now,
long default_limit);
/* Write the daemon status JSON (Phase 2, "xceadmin extres status"): one
* object with the daemon identity, push counters, and per-source rows
* (name, last_value, last_ok_time, stale). `stats->now == 0` is filled
* with time(NULL). Written atomically (tmp + rename) so a reader never
* sees a partial file. Returns 0, or -errno-shaped negative (the caller
* logs; the status file is best-effort observability and must never take
* the poll loop down). */
extern int esd_status_write(const char* path, const esd_config_t* cfg,
esd_stats_t* stats);
#endif

1
src/esd/meson.build
View file

@ -5,6 +5,7 @@
esd_srcs = [
'src/esd_config.c',
'src/esd_script.c',
'src/esd_status.c',
'src/esd.c',
]

51
src/esd/src/esd.c
View file

@ -27,6 +27,7 @@ typedef struct esd_daemon_s {
const char* from; /* join identity ("xceesd") */
int running; /* 0 once the main loop should exit */
xce_cluster_config_t cluster_cfg; /* validated peers list (HA mode) */
esd_stats_t stats; /* status-file counters (Phase 2) */
} esd_daemon_t;
static esd_daemon_t g_esd;
@ -61,8 +62,10 @@ int esd_push_value(const char* name, double value)
rc = xcecli_update_resvals(g_esd.apictx, 1, &valptr);
if (rc < 0) {
ERROR("esd push %s=%f failed (rc=%d)", name, value, rc);
g_esd.stats.pushes_fail++;
} else {
TRACE("esd pushed %s=%f", name, value);
g_esd.stats.pushes_ok++;
}
return rc;
@ -72,11 +75,18 @@ int esd_push_value(const char* name, double value)
* push only what CHANGED (the debounce -- design.md step 6 "轮询去抖").
* A failed collection keeps the previous value and does not push (fail-
* open: the controller keeps scheduling against the last known good
* count; design.md 3.4 stale policy default). */
* count; design.md 3.4 stale policy default).
*
* Phase 2: every source records last_ok_time on a good collect (the
* stale predicate's input, exposed via the status file); a failure past
* the source's stale limit escalates from the one-shot WARN to a per-
* round stale WARN -- the operator sees a kept-old-value daemon without
* having to guess from the absence of "pushed" lines. */
int esd_poll_round(void)
{
int i;
int pushed = 0;
time_t now = time(NULL);
for (i = 0; i < g_esd.cfg.nsrc; i++) {
esd_source_t* src = &(g_esd.cfg.src[i]);
@ -89,12 +99,22 @@ int esd_poll_round(void)
WARN("source %s: collection failed -- keeping last known "
"value %f (fail-open)",
src->name, src->have_value ? src->last_value : 0.0);
} else if (esd_source_is_stale(src, now,
g_esd.cfg.stale_limit)) {
WARN("source %s: STALE for %lds (limit %lds) -- last good "
"value %f kept (fail-open); check the collector",
src->name,
(long)(now - src->last_ok_time),
(src->stale_limit > 0) ? src->stale_limit
: g_esd.cfg.stale_limit,
src->last_value);
}
src->last_failed = true;
continue;
}
src->last_failed = false;
src->last_failed = false;
src->last_ok_time = now;
if (src->have_value && (src->last_value == v)) {
continue; /* unchanged -- the debounce */
@ -262,6 +282,26 @@ int esd_run(void)
(void)esd_poll_round();
/* Phase 2: refresh the status file every round (best-effort; the
* poll loop must survive a bad path). */
if (g_esd.cfg.status_file != NULL) {
/* one WARN per failure would spam a broken path every round;
* log on the transition only */
static bool status_warned = false;
int src = esd_status_write(g_esd.cfg.status_file, &g_esd.cfg,
&g_esd.stats);
if (src < 0) {
if (!status_warned) {
WARN("esd: cannot write status file %s (rc=%d) -- "
"continuing without it",
g_esd.cfg.status_file, src);
status_warned = true;
}
} else {
status_warned = false;
}
}
/* sleep in 1s slices so a signal stop lands promptly */
for (i = 0; (i < g_esd.cfg.poll_interval) && g_esd.running; i++) {
sleep(1);
@ -282,7 +322,8 @@ void esd_stop(void)
void esd_init(esd_config_t* cfg, const char* from)
{
memset(&g_esd, 0, sizeof(g_esd));
g_esd.cfg = *cfg;
g_esd.from = from;
g_esd.running = 1;
g_esd.cfg = *cfg;
g_esd.from = from;
g_esd.running = 1;
g_esd.stats.started_at = time(NULL);
}

33
src/esd/src/esd_config.c
View file

@ -17,13 +17,16 @@ typedef struct yaml_esd_source_s {
char* name;
char* collector;
char* command;
long interval; /* 0 = daemon default */
long interval; /* 0 = daemon default */
long stale_limit; /* 0 = daemon default (Phase 2, design.md 3.4) */
} yaml_esd_source_t;
typedef struct yaml_esd_config_s {
char* controller;
char* peers;
long poll_interval;
long stale_limit; /* 0 = ESD_STALE_LIMIT_DEFAULT */
char* status_file; /* NULL = no status file */
yaml_esd_source_t* resources;
unsigned resources_count;
} yaml_esd_config_t;
@ -37,6 +40,8 @@ static const cyaml_schema_field_t esd_source_fields_schema[] = {
command, 1, CYAML_UNLIMITED),
CYAML_FIELD_INT("interval", CYAML_FLAG_OPTIONAL, yaml_esd_source_t,
interval),
CYAML_FIELD_INT("stale_limit", CYAML_FLAG_OPTIONAL, yaml_esd_source_t,
stale_limit),
CYAML_FIELD_END
};
@ -52,6 +57,10 @@ static const cyaml_schema_field_t esd_config_fields_schema[] = {
peers, 1, CYAML_UNLIMITED),
CYAML_FIELD_INT("poll_interval", CYAML_FLAG_OPTIONAL, yaml_esd_config_t,
poll_interval),
CYAML_FIELD_INT("stale_limit", CYAML_FLAG_OPTIONAL, yaml_esd_config_t,
stale_limit),
CYAML_FIELD_STRING_PTR("status_file", CYAML_FLAG_OPTIONAL,
yaml_esd_config_t, status_file, 1, CYAML_UNLIMITED),
CYAML_FIELD_SEQUENCE("resources", CYAML_FLAG_POINTER, yaml_esd_config_t,
resources, &esd_source_schema, 0, CYAML_UNLIMITED),
CYAML_FIELD_END
@ -173,9 +182,23 @@ int esd_config_load_path(const char* dir, const char* name,
cfg.peers = (ycfg->peers != NULL) ? strdup(ycfg->peers) : NULL;
cfg.poll_interval = (int)ycfg->poll_interval;
cfg.stale_limit = ycfg->stale_limit;
cfg.status_file = (ycfg->status_file != NULL) ? strdup(ycfg->status_file)
: NULL;
cfg.nsrc = (int)ycfg->resources_count;
cfg.src = NULL;
if ((cfg.peers == NULL) && (ycfg->peers != NULL)) {
esd_config_fini(&cfg);
cyaml_free(&cyaml_cfg, &esd_config_schema, ycfg, 0);
return -ENOMEM;
}
if ((cfg.status_file == NULL) && (ycfg->status_file != NULL)) {
esd_config_fini(&cfg);
cyaml_free(&cyaml_cfg, &esd_config_schema, ycfg, 0);
return -ENOMEM;
}
if (cfg.nsrc > 0) {
cfg.src = (esd_source_t*)calloc((size_t)cfg.nsrc,
sizeof(esd_source_t));
@ -195,9 +218,10 @@ int esd_config_load_path(const char* dir, const char* name,
break;
}
cfg.src[i].name = strdup(ys->name);
cfg.src[i].command = strdup(ys->command);
cfg.src[i].interval = (int)ys->interval;
cfg.src[i].name = strdup(ys->name);
cfg.src[i].command = strdup(ys->command);
cfg.src[i].interval = (int)ys->interval;
cfg.src[i].stale_limit = ys->stale_limit;
if ((cfg.src[i].name == NULL) || (cfg.src[i].command == NULL)) {
rc = -ENOMEM;
@ -275,6 +299,7 @@ void esd_config_fini(esd_config_t* cfg)
free(cfg->controller_host);
free(cfg->peers);
free(cfg->status_file);
for (i = 0; i < cfg->nsrc; i++) {
free(cfg->src[i].name);

123
src/esd/src/esd_status.c Normal file
View file

@ -0,0 +1,123 @@
/* esd_status.c -- xceesd status file (external-license-resources Phase 2,
* design.md 3.5 "xceadmin extres status"). The daemon writes one JSON
* snapshot per poll round to a configured path; xceadmin reads it on the
* daemon host. No new wire opcode: the PR-2 proxy formalizes a status
* channel later, and a local file keeps this observability surface
* strictly read-only and crash-safe.
*
* Atomicity: write to <path>.tmp then rename(2) -- a concurrent reader
* (xceadmin) either sees the previous complete file or the new one, never
* a partial write (the fake-license E2E's lic_state lesson: a reader
* observing a truncated file mistakes it for data loss).
*/
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include "esd.h"
bool esd_source_is_stale(const esd_source_t* src, time_t now,
long default_limit)
{
long limit;
long age;
if ((src == NULL) || (!src->have_value) || (src->last_ok_time == 0)) {
return false;
}
/* Future last_ok_time (caller clock skew): age would go negative and
* read as fresh -- that is the honest reading, keep it by clamping. */
age = (now > src->last_ok_time) ? (long)(now - src->last_ok_time) : 0;
limit = (src->stale_limit > 0) ? src->stale_limit : default_limit;
if (limit <= 0) {
limit = ESD_STALE_LIMIT_DEFAULT;
}
return age > limit;
}
int esd_status_write(const char* path, const esd_config_t* cfg,
esd_stats_t* stats)
{
char tmp[1024];
FILE* f;
int i;
time_t now;
int rc = 0;
if ((path == NULL) || (cfg == NULL) || (stats == NULL)) {
return -EINVAL;
}
now = (stats->now != 0) ? stats->now : time(NULL);
snprintf(tmp, sizeof(tmp), "%s.tmp", path);
f = fopen(tmp, "w");
if (f == NULL) {
return -errno;
}
if (fprintf(f,
"{\n"
" \"daemon\": \"xceesd\",\n"
" \"pid\": %ld,\n"
" \"started_at\": %ld,\n"
" \"generated_at\": %ld,\n"
" \"pushes_ok\": %ld,\n"
" \"pushes_fail\": %ld,\n"
" \"sources\": [\n",
(long)getpid(), (long)stats->started_at, (long)now,
stats->pushes_ok, stats->pushes_fail) < 0) {
rc = -EIO;
goto out;
}
for (i = 0; i < cfg->nsrc; i++) {
const esd_source_t* src = &(cfg->src[i]);
bool stale =
esd_source_is_stale(src, now, cfg->stale_limit);
char valbuf[64];
if (src->have_value) {
snprintf(valbuf, sizeof(valbuf), "%.17g", src->last_value);
} else {
snprintf(valbuf, sizeof(valbuf), "null");
}
if (fprintf(f,
" {\"name\": \"%s\", \"last_value\": %s, "
"\"last_ok_time\": %ld, \"stale\": %s}%s\n",
(src->name != NULL) ? src->name : "",
valbuf,
(long)src->last_ok_time, stale ? "true" : "false",
(i + 1 < cfg->nsrc) ? "," : "") < 0) {
rc = -EIO;
goto out;
}
}
if (fprintf(f, " ]\n}\n") < 0) {
rc = -EIO;
goto out;
}
out:
if (fclose(f) != 0) {
rc = (rc == 0) ? -EIO : rc;
}
if (rc == 0) {
if (rename(tmp, path) != 0) {
rc = -errno;
}
}
if (rc != 0) {
unlink(tmp); /* never leave the .tmp behind for a reader to trip on */
}
return rc;
}

192
src/esd/testsuite/test_esd_core.c
View file

@ -23,6 +23,9 @@
* the spawn half (esd_script_collect) is exercised by the t1 E2E with a
* real fake-license script, not here. */
#include "../src/esd_script.c"
/* White-box: the stale predicate and the status-file writer (Phase 2,
* design.md 3.5 "运维面"). */
#include "../src/esd_status.c"
static char g_tmpdir[256];
@ -58,6 +61,35 @@ static void write_file(const char* dir, const char* name, const char* content)
fclose(f);
}
static char* slurp(const char* path)
{
FILE* f;
long len;
char* buf;
f = fopen(path, "r");
if (f == NULL) {
return NULL;
}
fseek(f, 0, SEEK_END);
len = ftell(f);
fseek(f, 0, SEEK_SET);
buf = (char*)malloc((size_t)len + 1);
if (buf == NULL) {
fclose(f);
return NULL;
}
if (fread(buf, 1, (size_t)len, f) != (size_t)len) {
fclose(f);
free(buf);
return NULL;
}
buf[len] = '\0';
fclose(f);
return buf;
}
/* ── esd_config_load ─────────────────────────────────────────────────── */
static void test_config_load_full_file(void)
@ -185,6 +217,162 @@ static void test_parse_output_garbage_is_nan(void)
TEST_ASSERT_TRUE(isnan(v));
}
/* ── Phase 2: stale_limit config parsing ────────────────────────────── */
static void test_config_stale_limit_parse(void)
{
esd_config_t cfg;
int rc;
memset(&cfg, 0, sizeof(cfg));
/* per-source override wins; a source without one takes the daemon
* default; the daemon default itself is optional with a sane built-in
* (design.md 3.4: stale_limit in esd.yaml, fail-open keep-old-value). */
write_file(g_tmpdir, "esd.yaml",
"controller: 172.31.1.30:9990\n"
"poll_interval: 10\n"
"stale_limit: 200\n"
"resources:\n"
" - name: lic_a\n"
" collector: script\n"
" command: /bin/a.sh\n"
" stale_limit: 60\n"
" - name: lic_b\n"
" collector: script\n"
" command: /bin/b.sh\n");
rc = esd_config_load_path(g_tmpdir, "esd.yaml", &cfg);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(200, cfg.stale_limit);
TEST_ASSERT_EQUAL_INT(60, cfg.src[0].stale_limit);
TEST_ASSERT_EQUAL_INT(0, cfg.src[1].stale_limit); /* 0 = use default */
esd_config_fini(&cfg);
}
/* ── Phase 2: stale predicate ───────────────────────────────────────── */
static void test_stale_predicate(void)
{
esd_source_t src;
memset(&src, 0, sizeof(src));
src.name = (char*)"lic";
src.have_value = true;
src.last_value = 3.0;
/* Never report a value -> nothing to call stale; admission treats the
* pool as 0 through the normal NO_RESOURCE path. */
src.last_ok_time = 0;
TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60));
/* Fresh success (10s ago, limit 60) -> not stale. */
src.last_ok_time = 90;
TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60));
/* Exactly at the limit -> not stale (strictly-greater rule, so an
* interval == stale_limit config never flaps). */
src.last_ok_time = 40;
TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60));
/* Past the limit -> stale. */
src.last_ok_time = 39;
TEST_ASSERT_TRUE(esd_source_is_stale(&src, 100, 60));
/* A source whose own stale_limit override is set uses it. */
src.stale_limit = 10;
TEST_ASSERT_TRUE(esd_source_is_stale(&src, 100, 60));
src.last_ok_time = 95;
TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60));
/* last_ok_time newer than `now` (clock skew between test harness and
* caller) must not read as stale. */
src.last_ok_time = 105;
TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60));
}
/* ── Phase 2: status file ───────────────────────────────────────────── */
static void test_status_write_basic(void)
{
esd_config_t cfg;
esd_stats_t stats;
char path[512];
char* text;
time_t now = time(NULL);
char expect_lvt[128];
memset(&cfg, 0, sizeof(cfg));
cfg.nsrc = 1;
cfg.src = (esd_source_t*)calloc(1, sizeof(esd_source_t));
TEST_ASSERT_NOT_NULL(cfg.src);
cfg.src[0].name = (char*)"extlic";
cfg.src[0].have_value = true;
cfg.src[0].last_value = 3.0;
/* fresh success: last good value 5s ago, limit default 150 -> not
* stale (the marks-stale test below covers the other side) */
cfg.src[0].last_ok_time = now - 5;
cfg.stale_limit = ESD_STALE_LIMIT_DEFAULT;
memset(&stats, 0, sizeof(stats));
stats.started_at = 1000;
stats.pushes_ok = 7;
stats.pushes_fail = 1;
snprintf(path, sizeof(path), "%s/status.json", g_tmpdir);
TEST_ASSERT_EQUAL_INT(0, esd_status_write(path, &cfg, &stats));
text = slurp(path);
TEST_ASSERT_NOT_NULL(text);
/* The fields xceadmin extres status reads, and the stale flag driven
* by the same predicate the tests above pin. */
TEST_ASSERT_TRUE(strstr(text, "\"daemon\": \"xceesd\"") != NULL);
TEST_ASSERT_TRUE(strstr(text, "\"name\": \"extlic\"") != NULL);
snprintf(expect_lvt, sizeof(expect_lvt), "\"last_value\": %.17g", 3.0);
TEST_ASSERT_TRUE(strstr(text, expect_lvt) != NULL);
TEST_ASSERT_TRUE(strstr(text, "\"last_ok_time\": ") != NULL);
TEST_ASSERT_TRUE(strstr(text, "\"stale\": false") != NULL);
TEST_ASSERT_TRUE(strstr(text, "\"pushes_ok\": 7") != NULL);
TEST_ASSERT_TRUE(strstr(text, "\"pushes_fail\": 1") != NULL);
free(text);
free(cfg.src);
}
static void test_status_write_marks_stale(void)
{
esd_config_t cfg;
esd_stats_t stats;
char path[512];
char* text;
time_t now = time(NULL);
memset(&cfg, 0, sizeof(cfg));
cfg.nsrc = 1;
cfg.src = (esd_source_t*)calloc(1, sizeof(esd_source_t));
TEST_ASSERT_NOT_NULL(cfg.src);
cfg.src[0].name = (char*)"extlic";
cfg.src[0].have_value = true;
cfg.src[0].last_value = 2.0;
/* last success 1 hour ago, limit 150s -> stale. */
cfg.src[0].last_ok_time = now - 3600;
cfg.src[0].stale_limit = 150;
memset(&stats, 0, sizeof(stats));
snprintf(path, sizeof(path), "%s/status.json", g_tmpdir);
TEST_ASSERT_EQUAL_INT(0, esd_status_write(path, &cfg, &stats));
text = slurp(path);
TEST_ASSERT_NOT_NULL(text);
TEST_ASSERT_TRUE(strstr(text, "\"stale\": true") != NULL);
free(text);
free(cfg.src);
}
int main(void)
{
UNITY_BEGIN();
@ -196,6 +384,10 @@ int main(void)
RUN_TEST(test_parse_output_name_value);
RUN_TEST(test_parse_output_whitespace_and_negative);
RUN_TEST(test_parse_output_garbage_is_nan);
RUN_TEST(test_config_stale_limit_parse);
RUN_TEST(test_stale_predicate);
RUN_TEST(test_status_write_basic);
RUN_TEST(test_status_write_marks_stale);
return UNITY_END();
}

13
src/frontend/apiserver/backend/convert.go
View file

@ -286,12 +286,13 @@ func cQueueToModel(queue *C.xce_queue_t) *models.QueueInfo {
// cResourceToModel extracts C xce_resource_t fields and maps them to ResourceDefinitionInfo.
func cResourceToModel(res *C.xce_resource_t) *models.ResourceDefinitionInfo {
return rawResourceToModel(rawResourceData{
name: goString(res.name),
desc: goString(res.desc),
valType: int(res.valtype),
ordType: int(res.ordtype),
flags: int(res.flags),
interval: int(res.interval),
name: goString(res.name),
desc: goString(res.desc),
valType: int(res.valtype),
ordType: int(res.ordtype),
flags: int(res.flags),
interval: int(res.interval),
lastReport: int64(res.last_report),
})
}

14
src/frontend/apiserver/backend/convert_pure.go
View file

@ -578,12 +578,13 @@ type rawQueueData struct {
// rawResourceData mirrors the C xce_resource_t fields used by the Go model.
type rawResourceData struct {
name string
desc string
valType int
ordType int
flags int
interval int
name string
desc string
valType int
ordType int
flags int
interval int
lastReport int64
}
// interactiveURLFromExdata extracts the interactive session URL from a job
@ -828,6 +829,7 @@ func rawResourceToModel(r rawResourceData) *models.ResourceDefinitionInfo {
OrderType: r.ordType,
Flag: r.flags,
Interval: r.interval,
LastReport: r.lastReport,
}
}

18
src/frontend/apiserver/backend/handler.go
View file

@ -5,6 +5,7 @@ import (
"apiserver/store"
logger "apiserver/utils"
"sync"
"time"
"go.uber.org/zap"
)
@ -249,6 +250,23 @@ func (h *DefaultHandler) OnResourceLocationUpdated(res *models.ResourceDistribut
key := res.ResourceName + ":" + res.Location
store.ResourceLocations.Put(key, *res)
// external-license-resources Phase 2: the definition store is fed ONLY
// by the join-time config push, whose snapshot carries whatever the
// config tree had at push time -- so an EXTERNAL pool's last_report
// would stay 0 ("never") for the whole apiserver process lifetime even
// while the extres daemon pushes normally. A shared-value push (the
// daemon's landing shape: Location "") for an EXTERNAL definition is
// exactly an accepted external report at the controller, so refresh the
// stamp here. Host-sourced values (Location != "") and non-EXTERNAL
// resources must not touch it: last_report means "external value last
// accepted", and worker reports are a different freshness domain.
if res.Location == "" {
if def, ok := store.Resources.Get(res.ResourceName); ok && (def.Flag&models.EXTERNAL) != 0 {
def.LastReport = time.Now().Unix()
store.Resources.Put(def.ResourceName, def)
}
}
// resource-value-verification task 4.2 (Requirements 5.1, 5.2, 5.4): trace
// the stored value at its point of application, mirroring the scheduler's
// sibling trace log added in task 4.1 (core_events.c). One typed "value"

69
src/frontend/apiserver/backend/integration_test.go
View file

@ -8,6 +8,7 @@ import (
"apiserver/utils"
"sync/atomic"
"testing"
"time"
"go.uber.org/zap"
)
@ -516,3 +517,71 @@ func TestIntegration_CrossFeature_SequentialConfigPushes(t *testing.T) {
t.Errorf("heartbeatIntervalSecs = %d, want 60", hb)
}
}
// --- 5.6 Phase 2 (external-license-resources): a shared-value push refreshes
// the stored EXTERNAL definition's last_report. The definition store is fed
// only by the join-time config push, whose snapshot carries whatever the
// config tree had at push time — without this refresh the CLI's LAST_UPDATED
// column would show "never" for the whole apiserver process lifetime even
// while the extres daemon pushes normally. The daemon landing shape for a
// cluster-level value is Location=="" (host:"" on the wire). Worker-sourced
// host values must NOT touch the stamp. ---
func TestIntegration_ExternalSharedPushRefreshesDefinitionLastReport(t *testing.T) {
resetIntegrationStores()
h := &DefaultHandler{}
// config push seeds the definition store: extlic is EXTERNAL (0x10),
// mem is a plain host-based resource.
dispatchConfigPush(h, nil, []models.ResourceDefinitionInfo{
{ResourceName: "extlic", Flag: 0x80 | 0x10, LastReport: 0},
{ResourceName: "mem", Flag: 0x80, LastReport: 0},
}, nil, &models.ClusterInfo{ClusterName: "c1"})
before := time.Now().Unix()
// daemon-shaped shared push for extlic (Location "")
h.OnResourceLocationUpdated(&models.ResourceDistributionInfo{
ResourceName: "extlic", Location: "", NumericValue: 4.0,
ValueType: int(models.NUMERIC), IsDynamic: true,
})
got, ok := store.Resources.Get("extlic")
if !ok {
t.Fatal("extlic definition should exist after config push")
}
if got.LastReport < before {
t.Errorf("extlic LastReport = %d, want >= %d (refreshed by the shared push)", got.LastReport, before)
}
// host-sourced push for mem must not stamp its definition
h.OnResourceLocationUpdated(&models.ResourceDistributionInfo{
ResourceName: "mem", Location: "node01", NumericValue: 65536.0,
ValueType: int(models.NUMERIC), IsDynamic: true,
})
gotMem, _ := store.Resources.Get("mem")
if gotMem.LastReport != 0 {
t.Errorf("mem LastReport = %d, want 0 (host values never stamp)", gotMem.LastReport)
}
}
// The refresh only applies to EXTERNAL definitions: a shared push for a
// non-EXTERNAL resource (e.g. a dynamic shared pool reported by workers)
// leaves last_report untouched.
func TestIntegration_SharedPushDoesNotStampNonExternalDefinition(t *testing.T) {
resetIntegrationStores()
h := &DefaultHandler{}
dispatchConfigPush(h, nil, []models.ResourceDefinitionInfo{
{ResourceName: "plainshared", Flag: 0x80 | 0x08, LastReport: 0},
}, nil, &models.ClusterInfo{ClusterName: "c1"})
h.OnResourceLocationUpdated(&models.ResourceDistributionInfo{
ResourceName: "plainshared", Location: "", NumericValue: 2.0,
ValueType: int(models.NUMERIC), IsDynamic: true,
})
got, _ := store.Resources.Get("plainshared")
if got.LastReport != 0 {
t.Errorf("plainshared LastReport = %d, want 0 (not EXTERNAL)", got.LastReport)
}
}

15
src/frontend/apiserver/models/resources.go
View file

@ -67,6 +67,10 @@ var RESOURCE_DEFINITION_DEFAULT_COLUMN = []string{
"order_type",
"flag",
"interval",
// external-license-resources Phase 2: SelectFields drops row keys not
// listed here, so the external value's last-report timestamp must be a
// default column or it never reaches API clients / the CLI.
"last_report",
}
var ResourceFieldMappings = map[string]string{
@ -77,8 +81,12 @@ var ResourceFieldMappings = map[string]string{
var ResourcesAvailableTypes = []string{"static", "dynamic"}
const (
BUILTIN = 0x01
DYNAMIC = 0x02
BUILTIN = 0x01
DYNAMIC = 0x02
// external-license-resources: EXTERNAL=0x10 mirrors C's XCE_RESF_EXTERNAL
// (src/include/xce/types.h) — the value comes from an external daemon,
// not from workers.
EXTERNAL = 0x10
RELEASE = 0x20
CONSUMABLE = 0x80
)
@ -132,6 +140,9 @@ type ResourceDefinitionInfo struct {
OrderType int `json:"order_type"`
Flag int `json:"flag"`
Interval int `json:"interval"`
// external-license-resources Phase 2: epoch seconds the external value
// was last accepted by the controller (0 = never / not external).
LastReport int64 `json:"last_report"`
}
var RESOURCE_DISTRIBUTION_SUPPORT_SORT = []string{

47
src/frontend/apiserver/models/resources_test.go Normal file
View file

@ -0,0 +1,47 @@
package models
import (
"encoding/json"
"testing"
)
// external-license-resources Phase 2: last_report must be part of the default
// column list for /resources/definition — SelectFields drops any row key not
// listed there, so without it the API response (and the CLI LAST_UPDATED
// column) never sees the timestamp even though the model carries it.
func TestResourceDefinitionDefaultColumnIncludesLastReport(t *testing.T) {
found := false
for _, col := range RESOURCE_DEFINITION_DEFAULT_COLUMN {
if col == "last_report" {
found = true
break
}
}
if !found {
t.Errorf("RESOURCE_DEFINITION_DEFAULT_COLUMN = %v, missing \"last_report\"", RESOURCE_DEFINITION_DEFAULT_COLUMN)
}
}
// ResourceDefinitionInfo must serialize last_report with the json tag the CLI
// model (xcecli/models/resource.go) unmarshals.
func TestResourceDefinitionInfoJSONCarriesLastReport(t *testing.T) {
info := ResourceDefinitionInfo{
ResourceName: "extlic",
LastReport: 1800000000,
}
data, err := json.Marshal(info)
if err != nil {
t.Fatalf("json.Marshal: %v", err)
}
var m map[string]interface{}
if err := json.Unmarshal(data, &m); err != nil {
t.Fatalf("json.Unmarshal: %v", err)
}
v, ok := m["last_report"]
if !ok {
t.Fatalf("serialized body %s has no last_report key", data)
}
if v.(float64) != 1800000000 {
t.Errorf("last_report = %v, want 1800000000", v)
}
}

7
src/frontend/xcecli/common/output/page_print_resources.go
View file

@ -3,6 +3,7 @@ package output
import (
"fmt"
"strings"
"time"
"xcecli/models"
)
@ -143,6 +144,8 @@ func (p *ResourcesPagePrinter) initColumnWidthsForDef(resourcesDef *models.Resou
width = len(resourceDef.ResourceType)
case models.HEADER_RESOURCE_DEF_DESC:
width = len(resourceDef.Description)
case models.HEADER_RESOURCE_DEF_LAST_REPORT:
width = len(formatJobTime(time.Unix(resourceDef.LastReport, 0)))
}
if !p.wideFormat && width > 15 {
@ -270,6 +273,8 @@ func (p *ResourcesPagePrinter) formatRowsForDef(resourcesDef *models.ResourceDef
row = appendOrDefault(row, FormatStringField(builtinValue, width, truncatFlag, true), "-")
case models.HEADER_RESOURCE_DEF_INTERVAL:
row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, resourceDef.Interval), "0")
case models.HEADER_RESOURCE_DEF_LAST_REPORT:
row = appendOrDefault(row, FormatStringField(fmtUnixTime(resourceDef.LastReport), width, truncatFlag, true), "-")
}
}
rows = append(rows, row)
@ -354,6 +359,8 @@ func printResourcesDefVerboseOutput(resourceDef *models.ResourceDefResponse) {
val = releaseValue
case models.HEADER_RESOURCE_DEF_INTERVAL:
val = fmt.Sprintf("%d", resourceDef.Interval)
case models.HEADER_RESOURCE_DEF_LAST_REPORT:
val = fmtUnixTime(resourceDef.LastReport)
}
widths1[i] = max(len(h), len(val))
row1 = append(row1, FormatStringField(val, widths1[i], false, true))

46
src/frontend/xcecli/models/resource.go
View file

@ -7,17 +7,18 @@ import (
)
const (
HEADER_RESOURCE_NAME = "RESOURCE"
HEADER_RESOURCE_VALUE = "VALUE"
HEADER_RESOURCE_LOC = "LOCATION"
HEADER_RESOURCE_DEF_TYPE = "TYPE"
HEADER_RESOURCE_DEF_ORDER = "ORDER"
HEADER_RESOURCE_DEF_BUILTIN = "BUILTIN"
HEADER_RESOURCE_DEF_CONSUMABLE = "CONSUMABLE"
HEADER_RESOURCE_DEF_DYNAMIC = "DYNAMIC"
HEADER_RESOURCE_DEF_RELEASE = "RELEASE"
HEADER_RESOURCE_DEF_INTERVAL = "INTERVAL"
HEADER_RESOURCE_DEF_DESC = "DESCRIPTION"
HEADER_RESOURCE_NAME = "RESOURCE"
HEADER_RESOURCE_VALUE = "VALUE"
HEADER_RESOURCE_LOC = "LOCATION"
HEADER_RESOURCE_DEF_TYPE = "TYPE"
HEADER_RESOURCE_DEF_ORDER = "ORDER"
HEADER_RESOURCE_DEF_BUILTIN = "BUILTIN"
HEADER_RESOURCE_DEF_CONSUMABLE = "CONSUMABLE"
HEADER_RESOURCE_DEF_DYNAMIC = "DYNAMIC"
HEADER_RESOURCE_DEF_RELEASE = "RELEASE"
HEADER_RESOURCE_DEF_INTERVAL = "INTERVAL"
HEADER_RESOURCE_DEF_LAST_REPORT = "LAST_UPDATED"
HEADER_RESOURCE_DEF_DESC = "DESCRIPTION"
)
var ResourceDistFieldsMap = map[string]string{
@ -28,21 +29,23 @@ var ResourceDistFieldsMap = map[string]string{
// ResourceDefFieldsMap maps resource definition field headers to API field names
var ResourceDefFieldsMap = map[string]string{
HEADER_RESOURCE_NAME: "res_name",
HEADER_RESOURCE_DEF_DESC: "description",
HEADER_RESOURCE_DEF_TYPE: "type",
HEADER_RESOURCE_DEF_ORDER: "order",
HEADER_RESOURCE_DEF_BUILTIN: "builtin",
HEADER_RESOURCE_DEF_CONSUMABLE: "consumable",
HEADER_RESOURCE_DEF_DYNAMIC: "dynamic",
HEADER_RESOURCE_DEF_RELEASE: "release",
HEADER_RESOURCE_DEF_INTERVAL: "interval",
HEADER_RESOURCE_NAME: "res_name",
HEADER_RESOURCE_DEF_DESC: "description",
HEADER_RESOURCE_DEF_TYPE: "type",
HEADER_RESOURCE_DEF_ORDER: "order",
HEADER_RESOURCE_DEF_BUILTIN: "builtin",
HEADER_RESOURCE_DEF_CONSUMABLE: "consumable",
HEADER_RESOURCE_DEF_DYNAMIC: "dynamic",
HEADER_RESOURCE_DEF_RELEASE: "release",
HEADER_RESOURCE_DEF_INTERVAL: "interval",
HEADER_RESOURCE_DEF_LAST_REPORT: "last_report",
}
var ResourceDefDetailHeaders = []string{
HEADER_RESOURCE_DEF_TYPE,
HEADER_RESOURCE_DEF_ORDER,
HEADER_RESOURCE_DEF_INTERVAL,
HEADER_RESOURCE_DEF_LAST_REPORT,
HEADER_RESOURCE_DEF_BUILTIN,
HEADER_RESOURCE_DEF_DYNAMIC,
HEADER_RESOURCE_DEF_RELEASE,
@ -177,6 +180,7 @@ type ResourceDef struct {
OrderType string `json:"-"` // populated from order_type int
Flag map[string]interface{} `json:"flag"`
Interval int `json:"interval"`
LastReport int64 `json:"last_report"` // Unix seconds of the last external value report (0 = never)
}
func (r *ResourceDef) UnmarshalJSON(data []byte) error {
@ -187,6 +191,7 @@ func (r *ResourceDef) UnmarshalJSON(data []byte) error {
OrderType string `json:"order_type"`
Flag map[string]interface{} `json:"flag"`
Interval int `json:"interval"`
LastReport int64 `json:"last_report"`
}
var a Alias
if err := UnmarshalOrderMapRow(data, &a); err != nil {
@ -198,5 +203,6 @@ func (r *ResourceDef) UnmarshalJSON(data []byte) error {
r.OrderType = a.OrderType
r.Flag = a.Flag
r.Interval = a.Interval
r.LastReport = a.LastReport
return nil
}

14
src/scheduler/src/core_events.c
View file

@ -1184,10 +1184,16 @@ int xce_core_push_events(xce_sched_core_t* core, xce_core_event_t* events,
if ((key_find(core->resmgr, res->name, (void**)&found)
== 0)
&& (found != NULL)) {
found->valtype = res->valtype;
found->ordtype = res->ordtype;
found->flags = res->flags;
found->interval = res->interval;
found->valtype = res->valtype;
found->ordtype = res->ordtype;
found->flags = res->flags;
found->interval = res->interval;
/* external-license-resources Phase 2: the extres
* freshness stamp (controller stamps it at each
* accepted push); resobj.c's makeResource excludes
* last_report from definition comparison, so this
* is metadata, not an identity change. */
found->last_report = res->last_report;
} else {
WARN("%s RESOURCES_PUSH: makeResource failed for "
"%s and lookup not found",

124
src/xceadmin/src/xceadmin/cli.py
View file

@ -27,11 +27,14 @@ traceback.
from __future__ import annotations
import json
import os
import time
from collections.abc import Callable
from datetime import datetime, timezone
from importlib.metadata import PackageNotFoundError
from importlib.metadata import version as _package_version
from pathlib import Path
import click
from click.core import ParameterSource
@ -43,6 +46,7 @@ from xcecommon.errors import (
SchedAdminError,
TooManyRedirectsError,
XceAdminError,
XceExtresStatusError,
)
from xceadmin.build import build_cluster_config, dump_cluster_config, write_output
from xceadmin.push import probe_config, push_config
@ -372,6 +376,126 @@ def scheduler_dump(file: str) -> None:
click.echo(text, nl=False) # dump() already ends its own text with "\n" (5.20).
# ---------------------------------------------------------------------------
# extres status (external-license-resources Phase 2 item 4, design.md 3.5):
# read the status JSON xceesd writes on its own host and render it for the
# operator. A LOCAL-file command like scheduler dump -- no --controller, no
# wire: the daemon's status file is the observability surface until the
# PR-2 proxy formalizes a status channel (design.md 3.6; a temporary
# remote protocol would be redesigned out from under us by PR-2's
# provider-lease model). Run it on the daemon's host.
# ---------------------------------------------------------------------------
# Where the deployed esd.yaml's status_file default points. Matches
# esd_config.c's status_file default and the deploy docs; overridable with
# --status-file (the t1 test cluster uses a /tmp path instead).
DEFAULT_STATUS_FILE = "/var/run/xce/esd-status.json"
def _format_extres_status(payload: dict, now: int | None = None) -> str:
"""Renders one decoded esd status object as operator text.
``now`` pins the wall clock for deterministic tests (None = time.time()).
Rendering rules:
- daemon identity line: name + pid
- uptime and push counters on their own lines
- per-source rows: NAME value=<v|-> age=<n>s [-- STALE] or
"no value yet" when last_value is null.
"""
if now is None:
now = int(time.time())
lines = [
f"daemon: {payload.get('daemon', 'xceesd')} (pid {payload.get('pid', '?')})",
]
started_at = payload.get("started_at", 0)
if isinstance(started_at, int) and started_at > 0:
uptime = max(now - started_at, 0)
started_str = datetime.fromtimestamp(started_at, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
lines.append(f"started at: {started_str} (up {uptime}s)")
lines.append(
f"pushes: {payload.get('pushes_ok', 0)} ok, "
f"{payload.get('pushes_fail', 0)} failed"
)
sources = payload.get("sources", [])
if not sources:
lines.append("sources: (none configured)")
return "\n".join(lines)
lines.append("sources:")
for src in sources:
name = src.get("name", "?")
if src.get("last_value") is None:
lines.append(f" {name}: no value yet")
continue
last_ok = src.get("last_ok_time", 0)
age = max(now - last_ok, 0) if isinstance(last_ok, int) and last_ok > 0 else 0
stale = " -- STALE" if src.get("stale") else ""
lines.append(f" {name}: value={src.get('last_value')} age={age}s{stale}")
return "\n".join(lines)
def _read_extres_status(path: str) -> dict:
"""Reads and validates one esd status file. Raises
:class:`XceExtresStatusError` for every local failure mode (unreadable
path, unparseable JSON, JSON that is not an esd status object) so the
CLI layer never sees a raw OSError/JSONDecodeError.
"""
try:
raw = Path(path).read_text()
except OSError as exc:
raise XceExtresStatusError(
f"cannot read extres status file {path!r}: {exc}", path=path, reason=str(exc)
) from exc
try:
payload = json.loads(raw)
except json.JSONDecodeError as exc:
raise XceExtresStatusError(
f"extres status file {path!r} is not valid JSON: {exc}", path=path, reason=str(exc)
) from exc
if not isinstance(payload, dict) or "sources" not in payload or "daemon" not in payload:
raise XceExtresStatusError(
f"extres status file {path!r} is not an xceesd status object "
"(missing 'daemon' or 'sources')",
path=path,
reason="wrong schema",
)
return payload
@cli.group()
def extres() -> None:
"""Inspect the external-resources daemon (xceesd)."""
@extres.command("status")
@click.option(
"--status-file",
default=None,
help=f"Path of the xceesd status JSON (default: {DEFAULT_STATUS_FILE}).",
)
def extres_status(status_file: str | None) -> None:
"""Report xceesd's status from its local status file (run on the daemon's host)."""
# Resolved at call time, not decoration time, so tests (and wrappers)
# can re-point the default -- click freezes a non-None default= into
# the command's own default_value at decoration.
if status_file is None:
status_file = DEFAULT_STATUS_FILE
try:
payload = _read_extres_status(status_file)
click.echo(_format_extres_status(payload))
except XceAdminError as e:
raise click.ClickException(e.message) from e
except KeyboardInterrupt:
raise click.ClickException("cancelled") from None
def main() -> None:
"""Console-script entry point (see pyproject.toml [project.scripts])."""
cli()

20
src/xceadmin/src/xcecommon/errors.py
View file

@ -222,3 +222,23 @@ class SchedAdminError(XceAdminError):
self.file = file
self.event_count = event_count
self.started_at = started_at
class XceExtresStatusError(XceAdminError):
"""The xceesd status file (``xceadmin extres status``) could not be
read or decoded (external-license-resources Phase 2 item 4, design.md
3.5; raised by ``xceadmin.cli._read_extres_status``).
Covers every local-file failure mode: the path doesn't exist / isn't
readable, the bytes aren't valid JSON, or the JSON parses but isn't an
esd status object (missing ``daemon``/``sources`` -- e.g. someone
pointed ``--status-file`` at some other JSON file). Same local-file
concern as :class:`ReplayFileError`/:class:`ClusterBinFileError`: no
network framing involved -- ``extres status`` reads the status JSON
xceesd writes on its own host.
"""
def __init__(self, message: str, path: str, reason: str) -> None:
super().__init__(message)
self.path = path
self.reason = reason

275
src/xceadmin/tests/test_cli_extres.py Normal file
View file

@ -0,0 +1,275 @@
"""Tests for the ``extres`` command group (external-license-resources
Phase 2 item 4, design.md 3.5 "``xceadmin extres status``").
``extres status`` is a LOCAL-file command -- the inverse slice of
``scheduler dump``: where ``dump()`` proved xceadmin can be a local-file
decoder with no network at all, ``extres status`` reads the status JSON
``xceesd`` writes on its own host (``esd_status_write``,
``src/esd/src/esd_status.c``) and formats it for the operator. No wire,
no ``--controller``, no TLS -- the daemon's status file is deliberately
the observability surface until the PR-2 proxy formalizes a status
channel (design.md 3.6; a temporary remote protocol would be
redesigned out from under us by PR-2's provider-lease model).
Split into its own file (rather than appended to ``test_cli.py``, 1187
lines at the time of writing) to keep both files under the project's
800-line guideline -- the same split precedent as
``test_cli_scheduler_mock.py``.
"""
from __future__ import annotations
import json
from pathlib import Path
import pytest
from click.testing import CliRunner
from xceadmin import cli as cli_module
from xceadmin.cli import cli
# ---------------------------------------------------------------------------
# Fixtures: realistic status JSON in the exact shape esd_status_write()
# emits (field order aside -- we parse, we never substring-match).
# ---------------------------------------------------------------------------
_NOW = 1_800_000_000 # a fixed "now" for deterministic age rendering
@pytest.fixture
def status_file(tmp_path: Path) -> Path:
"""A status file exactly as esd_status_write() writes it for two
sources: one fresh, one stale (last_ok_time 3600s in the past of
_NOW against the daemon's 150s default limit).
"""
path = tmp_path / "esd-status.json"
payload = {
"daemon": "xceesd",
"pid": 12345,
"started_at": _NOW - 7200,
"generated_at": _NOW,
"pushes_ok": 142,
"pushes_fail": 3,
"sources": [
{
"name": "Verilog_XL",
"last_value": 2.0,
"last_ok_time": _NOW - 5,
"stale": False,
},
{
"name": "VCS",
"last_value": 7.0,
"last_ok_time": _NOW - 3600,
"stale": True,
},
],
}
path.write_text(json.dumps(payload))
return path
def _freeze_now(monkeypatch: pytest.MonkeyPatch) -> None:
"""Pin time.time() to _NOW so age rendering is deterministic."""
monkeypatch.setattr(cli_module.time, "time", lambda: _NOW)
# ---------------------------------------------------------------------------
# Formatting helper: pure functions tested directly (the _format_record_
# result precedent in test_cli.py).
# ---------------------------------------------------------------------------
def test_format_status_renders_daemon_identity_and_counters() -> None:
from xceadmin.cli import _format_extres_status
payload = json.loads(
(Path(__file__).parent / "data" / "esd-status-sample.json").read_text()
) if (Path(__file__).parent / "data" / "esd-status-sample.json").exists() else {
"daemon": "xceesd",
"pid": 12345,
"started_at": _NOW - 7200,
"generated_at": _NOW,
"pushes_ok": 142,
"pushes_fail": 3,
"sources": [],
}
text = _format_extres_status(payload, now=_NOW)
assert "xceesd" in text
assert "12345" in text
assert "142" in text
assert "3" in text
def test_format_status_marks_stale_source_with_age() -> None:
from xceadmin.cli import _format_extres_status
payload = {
"daemon": "xceesd",
"pid": 1,
"started_at": _NOW - 100,
"generated_at": _NOW,
"pushes_ok": 0,
"pushes_fail": 0,
"sources": [
{
"name": "VCS",
"last_value": 7.0,
"last_ok_time": _NOW - 3600,
"stale": True,
}
],
}
text = _format_extres_status(payload, now=_NOW)
assert "VCS" in text
assert "STALE" in text
assert "3600s" in text
def test_format_status_fresh_source_shows_age_only() -> None:
from xceadmin.cli import _format_extres_status
payload = {
"daemon": "xceesd",
"pid": 1,
"started_at": _NOW - 100,
"generated_at": _NOW,
"pushes_ok": 0,
"pushes_fail": 0,
"sources": [
{
"name": "Verilog_XL",
"last_value": 2.0,
"last_ok_time": _NOW - 5,
"stale": False,
}
],
}
text = _format_extres_status(payload, now=_NOW)
assert "Verilog_XL" in text
assert "STALE" not in text
assert "5s" in text
def test_format_status_source_with_no_value_yet() -> None:
from xceadmin.cli import _format_extres_status
payload = {
"daemon": "xceesd",
"pid": 1,
"started_at": _NOW - 100,
"generated_at": _NOW,
"pushes_ok": 0,
"pushes_fail": 0,
"sources": [
{
"name": "Verilog_XL",
"last_value": None,
"last_ok_time": 0,
"stale": False,
}
],
}
text = _format_extres_status(payload, now=_NOW)
assert "Verilog_XL" in text
assert "no value yet" in text
# ---------------------------------------------------------------------------
# CLI wiring: arg parsing, file reading, error translation.
# ---------------------------------------------------------------------------
def test_cli_help_shows_extres_command() -> None:
runner = CliRunner()
result = runner.invoke(cli, ["--help"])
assert result.exit_code == 0
assert "extres" in result.output
def test_extres_status_reads_local_file(status_file: Path, monkeypatch: pytest.MonkeyPatch) -> None:
_freeze_now(monkeypatch)
runner = CliRunner()
result = runner.invoke(cli, ["extres", "status", "--status-file", str(status_file)])
assert result.exit_code == 0, result.output
assert "xceesd" in result.output
assert "Verilog_XL" in result.output
assert "VCS" in result.output
def test_extres_status_defaults_to_etc_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""With no --status-file, the default path is the deployed default
(/var/run/xce/esd-status.json -- where esd.yaml's production
status_file default points); a missing default file is a clean
error, not a traceback.
"""
monkeypatch.setattr(
cli_module, "DEFAULT_STATUS_FILE", str(tmp_path / "nonexistent.json")
)
runner = CliRunner()
result = runner.invoke(cli, ["extres", "status"])
assert result.exit_code != 0
assert "nonexistent.json" in result.output
def test_extres_status_missing_file_clean_error(tmp_path: Path) -> None:
runner = CliRunner()
result = runner.invoke(
cli, ["extres", "status", "--status-file", str(tmp_path / "gone.json")]
)
assert result.exit_code != 0
assert "gone.json" in result.output
assert "Traceback" not in result.output
def test_extres_status_invalid_json_clean_error(tmp_path: Path) -> None:
bad = tmp_path / "bad.json"
bad.write_text("{not json")
runner = CliRunner()
result = runner.invoke(cli, ["extres", "status", "--status-file", str(bad)])
assert result.exit_code != 0
assert "Traceback" not in result.output
def test_extres_status_wrong_schema_clean_error(tmp_path: Path) -> None:
"""JSON that parses but is not an esd status object (e.g. someone
pointed --status-file at some other JSON) fails loud and clean.
"""
bad = tmp_path / "other.json"
bad.write_text(json.dumps({"hello": "world"}))
runner = CliRunner()
result = runner.invoke(cli, ["extres", "status", "--status-file", str(bad)])
assert result.exit_code != 0
assert "Traceback" not in result.output
def test_extres_status_no_controller_needed(monkeypatch: pytest.MonkeyPatch, status_file: Path) -> None:
"""The command must not touch the controller: XCE_CONTROLLER_PEERS
pointing at a dead endpoint changes nothing (the file is local).
"""
monkeypatch.setenv("XCE_CONTROLLER_PEERS", "127.0.0.1:1")
monkeypatch.setenv("XCEADMIN_CONTROLLER", "127.0.0.1:1")
runner = CliRunner()
result = runner.invoke(cli, ["extres", "status", "--status-file", str(status_file)])
assert result.exit_code == 0, result.output