A controller restart dropped xceesd's resval pushes forever: the fresh endpoint is created by handle_connection() with priv->role = HostRole_client, and only OPCode_join sets HostRole_extres -- but the old esd_on_reconnect cleared only g_needs_full_push, so the daemon re-PUSHED onto an un-joined endpoint and every push landed in do_update_resvals's neither-branch, refused -EPERM forever (live sighting: pushes_ok frozen, rc=-1 every 10s). Mirror the worker's needs_rejoin contract: the reconnect handler now sets BOTH g_needs_rejoin and g_needs_full_push; the main loop consumes the re-join first (xcecli_join(HostRole_extres) on the main thread, outside the failover lock -- the xcecd_on_reconnect no-I/O constraint), retrying next tick on failure, then does the full re-push. Live t1/t2 proof: controller restart -> "esd: re-joined after reconnect (rc=0)" + "full re-push: 1/1 source(s)" + extlic back in `xcecli resources distribution list`. Two new white-box unit tests pin the both-flags contract (13/13 in the esd suite). Co-Authored-By: Claude Code <noreply@anthropic.com>
434 lines
14 KiB
C
434 lines
14 KiB
C
/* test_esd_core.c -- external-license-resources Phase 1, step 6 TDD: the
|
|
* xceesd skeleton's pure core -- esd.yaml parsing and the script
|
|
* collector's output parsing -- white-box #include of the sources under
|
|
* test (same pattern as the worker's resource_sync_test /
|
|
* controller_connector_test). The join/reconnect wiring lives in esd.c
|
|
* against the live client library and is exercised by the t1 E2E, not
|
|
* here; everything this file drives must be deterministic. */
|
|
#include <errno.h>
|
|
#include <math.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <sys/stat.h>
|
|
#include <unistd.h>
|
|
|
|
#include <unity/unity.h>
|
|
|
|
#include "esd.h"
|
|
|
|
/* White-box: reach the config schema/parse internals. */
|
|
#include "../src/esd_config.c"
|
|
/* White-box: the script collector's parse half (esd_parse_script_output);
|
|
* the spawn half (esd_script_collect) is exercised by the t1 E2E with a
|
|
* real fake-license script, not here. */
|
|
#include "../src/esd_script.c"
|
|
/* White-box: the stale predicate and the status-file writer (Phase 2,
|
|
* design.md 3.5 "运维面"). */
|
|
#include "../src/esd_status.c"
|
|
/* White-box: the reconnect-flag contract (bug-1978). esd.c's main loop
|
|
* drives the live client library, but the reconnect handler's flag contract
|
|
* -- set BOTH rejoin and full-push, and the loop consuming rejoin first --
|
|
* is unit-testable here without a controller: esd_run() itself is not
|
|
* linked (its join sequence would need a live primary); we drive
|
|
* esd_on_reconnect and read the loop's guard statics directly. */
|
|
#include "../src/esd.c"
|
|
|
|
static char g_tmpdir[256];
|
|
|
|
void setUp(void)
|
|
{
|
|
snprintf(g_tmpdir, sizeof(g_tmpdir), "/tmp/xce_esd_testXXXXXX");
|
|
TEST_ASSERT_NOT_NULL(mkdtemp(g_tmpdir));
|
|
}
|
|
|
|
void tearDown(void)
|
|
{
|
|
if (g_tmpdir[0] != '\0') {
|
|
char path[300];
|
|
|
|
/* tests only write flat files inside g_tmpdir */
|
|
snprintf(path, sizeof(path), "rm -rf %s", g_tmpdir);
|
|
if (system(path) != 0) {
|
|
/* best effort */
|
|
}
|
|
g_tmpdir[0] = '\0';
|
|
}
|
|
}
|
|
|
|
static void write_file(const char* dir, const char* name, const char* content)
|
|
{
|
|
char path[512];
|
|
FILE* f;
|
|
|
|
snprintf(path, sizeof(path), "%s/%s", dir, name);
|
|
f = fopen(path, "w");
|
|
TEST_ASSERT_NOT_NULL(f);
|
|
fputs(content, f);
|
|
fclose(f);
|
|
}
|
|
|
|
static char* slurp(const char* path)
|
|
{
|
|
FILE* f;
|
|
long len;
|
|
char* buf;
|
|
|
|
f = fopen(path, "r");
|
|
if (f == NULL) {
|
|
return NULL;
|
|
}
|
|
fseek(f, 0, SEEK_END);
|
|
len = ftell(f);
|
|
fseek(f, 0, SEEK_SET);
|
|
buf = (char*)malloc((size_t)len + 1);
|
|
if (buf == NULL) {
|
|
fclose(f);
|
|
return NULL;
|
|
}
|
|
if (fread(buf, 1, (size_t)len, f) != (size_t)len) {
|
|
fclose(f);
|
|
free(buf);
|
|
return NULL;
|
|
}
|
|
buf[len] = '\0';
|
|
fclose(f);
|
|
|
|
return buf;
|
|
}
|
|
|
|
/* ── esd_config_load ─────────────────────────────────────────────────── */
|
|
|
|
static void test_config_load_full_file(void)
|
|
{
|
|
esd_config_t cfg;
|
|
int rc;
|
|
|
|
memset(&cfg, 0, sizeof(cfg));
|
|
|
|
write_file(g_tmpdir, "esd.yaml",
|
|
"controller: 172.31.1.30:9990\n"
|
|
"poll_interval: 30\n"
|
|
"resources:\n"
|
|
" - name: Verilog_XL\n"
|
|
" collector: script\n"
|
|
" command: /etc/xce/lic-report.sh\n"
|
|
" interval: 15\n"
|
|
" - name: cloud_gpu_quota\n"
|
|
" collector: script\n"
|
|
" command: /etc/xce/quota-report.sh\n");
|
|
|
|
rc = esd_config_load_path(g_tmpdir, "esd.yaml", &cfg);
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
TEST_ASSERT_EQUAL_STRING("172.31.1.30", cfg.controller_host);
|
|
TEST_ASSERT_EQUAL_INT(9990, cfg.controller_port);
|
|
TEST_ASSERT_EQUAL_INT(30, cfg.poll_interval);
|
|
TEST_ASSERT_EQUAL_INT(2, cfg.nsrc);
|
|
|
|
TEST_ASSERT_NOT_NULL(cfg.src);
|
|
TEST_ASSERT_EQUAL_STRING("Verilog_XL", cfg.src[0].name);
|
|
TEST_ASSERT_EQUAL_STRING("/etc/xce/lic-report.sh", cfg.src[0].command);
|
|
TEST_ASSERT_EQUAL_INT(15, cfg.src[0].interval);
|
|
TEST_ASSERT_EQUAL_STRING("cloud_gpu_quota", cfg.src[1].name);
|
|
TEST_ASSERT_EQUAL_STRING("/etc/xce/quota-report.sh", cfg.src[1].command);
|
|
TEST_ASSERT_EQUAL_INT(0, cfg.src[1].interval); /* no override */
|
|
|
|
esd_config_fini(&cfg);
|
|
}
|
|
|
|
static void test_config_load_missing_file_is_enoent(void)
|
|
{
|
|
esd_config_t cfg;
|
|
int rc;
|
|
|
|
memset(&cfg, 0, sizeof(cfg));
|
|
|
|
rc = esd_config_load_path(g_tmpdir, "no-such-file.yaml", &cfg);
|
|
TEST_ASSERT_EQUAL_INT(-ENOENT, rc);
|
|
}
|
|
|
|
static void test_config_load_bad_shape_is_einval(void)
|
|
{
|
|
esd_config_t cfg;
|
|
int rc;
|
|
|
|
memset(&cfg, 0, sizeof(cfg));
|
|
|
|
/* a source without a name is structurally invalid -- the controller
|
|
* would refuse the push and the operator would have no idea which
|
|
* entry died; fail loudly at load. */
|
|
write_file(g_tmpdir, "esd.yaml",
|
|
"controller: 172.31.1.30:9990\n"
|
|
"resources:\n"
|
|
" - collector: script\n"
|
|
" command: /bin/true\n");
|
|
|
|
rc = esd_config_load_path(g_tmpdir, "esd.yaml", &cfg);
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL, rc);
|
|
|
|
/* a source without a command can never report -- same rule. */
|
|
memset(&cfg, 0, sizeof(cfg));
|
|
write_file(g_tmpdir, "esd.yaml",
|
|
"controller: 172.31.1.30:9990\n"
|
|
"resources:\n"
|
|
" - name: lic\n"
|
|
" collector: script\n");
|
|
|
|
rc = esd_config_load_path(g_tmpdir, "esd.yaml", &cfg);
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL, rc);
|
|
}
|
|
|
|
/* ── script collector output parsing (parse contract, not the spawn) ── */
|
|
|
|
static void test_parse_output_bare_number(void)
|
|
{
|
|
double v = esd_parse_script_output("4\n");
|
|
|
|
TEST_ASSERT_FALSE(isnan(v));
|
|
TEST_ASSERT_EQUAL_FLOAT((float)4.0, (float)v);
|
|
}
|
|
|
|
static void test_parse_output_name_value(void)
|
|
{
|
|
/* elim-style "<name> <value>" -- name may mismatch; the source it is
|
|
* configured for is authoritative, the name is decorative. */
|
|
double v = esd_parse_script_output("Verilog_XL 7\n");
|
|
|
|
TEST_ASSERT_FALSE(isnan(v));
|
|
TEST_ASSERT_EQUAL_FLOAT((float)7.0, (float)v);
|
|
}
|
|
|
|
static void test_parse_output_whitespace_and_negative(void)
|
|
{
|
|
double v = esd_parse_script_output(" \t 3.5 \n");
|
|
|
|
TEST_ASSERT_FALSE(isnan(v));
|
|
TEST_ASSERT_EQUAL_FLOAT((float)3.5, (float)v);
|
|
|
|
v = esd_parse_script_output("lic -1\n");
|
|
TEST_ASSERT_FALSE(isnan(v));
|
|
TEST_ASSERT_EQUAL_FLOAT((float)-1.0, (float)v);
|
|
}
|
|
|
|
static void test_parse_output_garbage_is_nan(void)
|
|
{
|
|
double v;
|
|
|
|
v = esd_parse_script_output("");
|
|
TEST_ASSERT_TRUE(isnan(v));
|
|
v = esd_parse_script_output("\n");
|
|
TEST_ASSERT_TRUE(isnan(v));
|
|
v = esd_parse_script_output("no digits here\n");
|
|
TEST_ASSERT_TRUE(isnan(v));
|
|
v = esd_parse_script_output(NULL);
|
|
TEST_ASSERT_TRUE(isnan(v));
|
|
}
|
|
|
|
/* ── Phase 2: stale_limit config parsing ────────────────────────────── */
|
|
|
|
static void test_config_stale_limit_parse(void)
|
|
{
|
|
esd_config_t cfg;
|
|
int rc;
|
|
|
|
memset(&cfg, 0, sizeof(cfg));
|
|
|
|
/* per-source override wins; a source without one takes the daemon
|
|
* default; the daemon default itself is optional with a sane built-in
|
|
* (design.md 3.4: stale_limit in esd.yaml, fail-open keep-old-value). */
|
|
write_file(g_tmpdir, "esd.yaml",
|
|
"controller: 172.31.1.30:9990\n"
|
|
"poll_interval: 10\n"
|
|
"stale_limit: 200\n"
|
|
"resources:\n"
|
|
" - name: lic_a\n"
|
|
" collector: script\n"
|
|
" command: /bin/a.sh\n"
|
|
" stale_limit: 60\n"
|
|
" - name: lic_b\n"
|
|
" collector: script\n"
|
|
" command: /bin/b.sh\n");
|
|
|
|
rc = esd_config_load_path(g_tmpdir, "esd.yaml", &cfg);
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
TEST_ASSERT_EQUAL_INT(200, cfg.stale_limit);
|
|
TEST_ASSERT_EQUAL_INT(60, cfg.src[0].stale_limit);
|
|
TEST_ASSERT_EQUAL_INT(0, cfg.src[1].stale_limit); /* 0 = use default */
|
|
|
|
esd_config_fini(&cfg);
|
|
}
|
|
|
|
/* ── Phase 2: stale predicate ───────────────────────────────────────── */
|
|
|
|
static void test_stale_predicate(void)
|
|
{
|
|
esd_source_t src;
|
|
|
|
memset(&src, 0, sizeof(src));
|
|
src.name = (char*)"lic";
|
|
src.have_value = true;
|
|
src.last_value = 3.0;
|
|
|
|
/* Never report a value -> nothing to call stale; admission treats the
|
|
* pool as 0 through the normal NO_RESOURCE path. */
|
|
src.last_ok_time = 0;
|
|
TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60));
|
|
|
|
/* Fresh success (10s ago, limit 60) -> not stale. */
|
|
src.last_ok_time = 90;
|
|
TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60));
|
|
|
|
/* Exactly at the limit -> not stale (strictly-greater rule, so an
|
|
* interval == stale_limit config never flaps). */
|
|
src.last_ok_time = 40;
|
|
TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60));
|
|
|
|
/* Past the limit -> stale. */
|
|
src.last_ok_time = 39;
|
|
TEST_ASSERT_TRUE(esd_source_is_stale(&src, 100, 60));
|
|
|
|
/* A source whose own stale_limit override is set uses it. */
|
|
src.stale_limit = 10;
|
|
TEST_ASSERT_TRUE(esd_source_is_stale(&src, 100, 60));
|
|
src.last_ok_time = 95;
|
|
TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60));
|
|
|
|
/* last_ok_time newer than `now` (clock skew between test harness and
|
|
* caller) must not read as stale. */
|
|
src.last_ok_time = 105;
|
|
TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60));
|
|
}
|
|
|
|
/* ── Phase 2: status file ───────────────────────────────────────────── */
|
|
|
|
static void test_status_write_basic(void)
|
|
{
|
|
esd_config_t cfg;
|
|
esd_stats_t stats;
|
|
char path[512];
|
|
char* text;
|
|
time_t now = time(NULL);
|
|
char expect_lvt[128];
|
|
|
|
memset(&cfg, 0, sizeof(cfg));
|
|
cfg.nsrc = 1;
|
|
cfg.src = (esd_source_t*)calloc(1, sizeof(esd_source_t));
|
|
TEST_ASSERT_NOT_NULL(cfg.src);
|
|
cfg.src[0].name = (char*)"extlic";
|
|
cfg.src[0].have_value = true;
|
|
cfg.src[0].last_value = 3.0;
|
|
/* fresh success: last good value 5s ago, limit default 150 -> not
|
|
* stale (the marks-stale test below covers the other side) */
|
|
cfg.src[0].last_ok_time = now - 5;
|
|
cfg.stale_limit = ESD_STALE_LIMIT_DEFAULT;
|
|
|
|
memset(&stats, 0, sizeof(stats));
|
|
stats.started_at = 1000;
|
|
stats.pushes_ok = 7;
|
|
stats.pushes_fail = 1;
|
|
|
|
snprintf(path, sizeof(path), "%s/status.json", g_tmpdir);
|
|
TEST_ASSERT_EQUAL_INT(0, esd_status_write(path, &cfg, &stats));
|
|
|
|
text = slurp(path);
|
|
TEST_ASSERT_NOT_NULL(text);
|
|
|
|
/* The fields xceadmin extres status reads, and the stale flag driven
|
|
* by the same predicate the tests above pin. */
|
|
TEST_ASSERT_TRUE(strstr(text, "\"daemon\": \"xceesd\"") != NULL);
|
|
TEST_ASSERT_TRUE(strstr(text, "\"name\": \"extlic\"") != NULL);
|
|
snprintf(expect_lvt, sizeof(expect_lvt), "\"last_value\": %.17g", 3.0);
|
|
TEST_ASSERT_TRUE(strstr(text, expect_lvt) != NULL);
|
|
TEST_ASSERT_TRUE(strstr(text, "\"last_ok_time\": ") != NULL);
|
|
TEST_ASSERT_TRUE(strstr(text, "\"stale\": false") != NULL);
|
|
TEST_ASSERT_TRUE(strstr(text, "\"pushes_ok\": 7") != NULL);
|
|
TEST_ASSERT_TRUE(strstr(text, "\"pushes_fail\": 1") != NULL);
|
|
|
|
free(text);
|
|
free(cfg.src);
|
|
}
|
|
|
|
static void test_status_write_marks_stale(void)
|
|
{
|
|
esd_config_t cfg;
|
|
esd_stats_t stats;
|
|
char path[512];
|
|
char* text;
|
|
time_t now = time(NULL);
|
|
|
|
memset(&cfg, 0, sizeof(cfg));
|
|
cfg.nsrc = 1;
|
|
cfg.src = (esd_source_t*)calloc(1, sizeof(esd_source_t));
|
|
TEST_ASSERT_NOT_NULL(cfg.src);
|
|
cfg.src[0].name = (char*)"extlic";
|
|
cfg.src[0].have_value = true;
|
|
cfg.src[0].last_value = 2.0;
|
|
/* last success 1 hour ago, limit 150s -> stale. */
|
|
cfg.src[0].last_ok_time = now - 3600;
|
|
cfg.src[0].stale_limit = 150;
|
|
|
|
memset(&stats, 0, sizeof(stats));
|
|
|
|
snprintf(path, sizeof(path), "%s/status.json", g_tmpdir);
|
|
TEST_ASSERT_EQUAL_INT(0, esd_status_write(path, &cfg, &stats));
|
|
|
|
text = slurp(path);
|
|
TEST_ASSERT_NOT_NULL(text);
|
|
TEST_ASSERT_TRUE(strstr(text, "\"stale\": true") != NULL);
|
|
|
|
free(text);
|
|
free(cfg.src);
|
|
}
|
|
|
|
/* ── bug-1978: the reconnect-flag contract ───────────────────────────── */
|
|
|
|
static void test_reconnect_sets_both_rejoin_and_full_push(void)
|
|
{
|
|
g_needs_rejoin = 0;
|
|
g_needs_full_push = 0;
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, esd_on_reconnect((xcecli_ctx_t)0x1, NULL));
|
|
|
|
/* BOTH flags: the re-join must happen before any re-push (an un-joined
|
|
* push is refused -EPERM on the fresh endpoint), and the re-push must
|
|
* still happen (the fresh controller knows no values). The bug-1978
|
|
* defect cleared only g_needs_full_push -- a daemon that re-pushed onto
|
|
* an un-joined HostRole_client endpoint, every push -EPERM forever. */
|
|
TEST_ASSERT_EQUAL_INT(1, g_needs_rejoin);
|
|
TEST_ASSERT_EQUAL_INT(1, g_needs_full_push);
|
|
}
|
|
|
|
static void test_reconnect_flags_recover_from_any_previous_state(void)
|
|
{
|
|
/* The handler is level-set, not level-toggle: regardless of what a
|
|
* previous (already consumed) reconnect left behind, a new callback
|
|
* invocation re-arms both flags. */
|
|
g_needs_rejoin = 0;
|
|
g_needs_full_push = 1; /* stale leftover from an unconsumed cycle */
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, esd_on_reconnect((xcecli_ctx_t)0x2, NULL));
|
|
|
|
TEST_ASSERT_EQUAL_INT(1, g_needs_rejoin);
|
|
TEST_ASSERT_EQUAL_INT(1, g_needs_full_push);
|
|
}
|
|
|
|
int main(void)
|
|
{
|
|
UNITY_BEGIN();
|
|
|
|
RUN_TEST(test_config_load_full_file);
|
|
RUN_TEST(test_config_load_missing_file_is_enoent);
|
|
RUN_TEST(test_config_load_bad_shape_is_einval);
|
|
RUN_TEST(test_parse_output_bare_number);
|
|
RUN_TEST(test_parse_output_name_value);
|
|
RUN_TEST(test_parse_output_whitespace_and_negative);
|
|
RUN_TEST(test_parse_output_garbage_is_nan);
|
|
RUN_TEST(test_config_stale_limit_parse);
|
|
RUN_TEST(test_stale_predicate);
|
|
RUN_TEST(test_status_write_basic);
|
|
RUN_TEST(test_status_write_marks_stale);
|
|
RUN_TEST(test_reconnect_sets_both_rejoin_and_full_push);
|
|
RUN_TEST(test_reconnect_flags_recover_from_any_previous_state);
|
|
|
|
return UNITY_END();
|
|
}
|