myxce/src/esd/testsuite/test_esd_core.c
lei.chen b471ce31ca fix(esd): re-join after transparent reconnect -- bug-1978
A controller restart dropped xceesd's resval pushes forever: the fresh
endpoint is created by handle_connection() with priv->role =
HostRole_client, and only OPCode_join sets HostRole_extres -- but the
old esd_on_reconnect cleared only g_needs_full_push, so the daemon
re-PUSHED onto an un-joined endpoint and every push landed in
do_update_resvals's neither-branch, refused -EPERM forever (live
sighting: pushes_ok frozen, rc=-1 every 10s).

Mirror the worker's needs_rejoin contract: the reconnect handler now
sets BOTH g_needs_rejoin and g_needs_full_push; the main loop consumes
the re-join first (xcecli_join(HostRole_extres) on the main thread,
outside the failover lock -- the xcecd_on_reconnect no-I/O constraint),
retrying next tick on failure, then does the full re-push.

Live t1/t2 proof: controller restart -> "esd: re-joined after
reconnect (rc=0)" + "full re-push: 1/1 source(s)" + extlic back in
`xcecli resources distribution list`. Two new white-box unit tests pin
the both-flags contract (13/13 in the esd suite).

Co-Authored-By: Claude Code <noreply@anthropic.com>
2026-10-28 14:07:41 +00:00

434 lines
14 KiB
C

/* test_esd_core.c -- external-license-resources Phase 1, step 6 TDD: the
* xceesd skeleton's pure core -- esd.yaml parsing and the script
* collector's output parsing -- white-box #include of the sources under
* test (same pattern as the worker's resource_sync_test /
* controller_connector_test). The join/reconnect wiring lives in esd.c
* against the live client library and is exercised by the t1 E2E, not
* here; everything this file drives must be deterministic. */
#include <errno.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#include <unity/unity.h>
#include "esd.h"
/* White-box: reach the config schema/parse internals. */
#include "../src/esd_config.c"
/* White-box: the script collector's parse half (esd_parse_script_output);
* the spawn half (esd_script_collect) is exercised by the t1 E2E with a
* real fake-license script, not here. */
#include "../src/esd_script.c"
/* White-box: the stale predicate and the status-file writer (Phase 2,
* design.md 3.5 "运维面"). */
#include "../src/esd_status.c"
/* White-box: the reconnect-flag contract (bug-1978). esd.c's main loop
* drives the live client library, but the reconnect handler's flag contract
* -- set BOTH rejoin and full-push, and the loop consuming rejoin first --
* is unit-testable here without a controller: esd_run() itself is not
* linked (its join sequence would need a live primary); we drive
* esd_on_reconnect and read the loop's guard statics directly. */
#include "../src/esd.c"
static char g_tmpdir[256];
void setUp(void)
{
snprintf(g_tmpdir, sizeof(g_tmpdir), "/tmp/xce_esd_testXXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(g_tmpdir));
}
void tearDown(void)
{
if (g_tmpdir[0] != '\0') {
char path[300];
/* tests only write flat files inside g_tmpdir */
snprintf(path, sizeof(path), "rm -rf %s", g_tmpdir);
if (system(path) != 0) {
/* best effort */
}
g_tmpdir[0] = '\0';
}
}
static void write_file(const char* dir, const char* name, const char* content)
{
char path[512];
FILE* f;
snprintf(path, sizeof(path), "%s/%s", dir, name);
f = fopen(path, "w");
TEST_ASSERT_NOT_NULL(f);
fputs(content, f);
fclose(f);
}
static char* slurp(const char* path)
{
FILE* f;
long len;
char* buf;
f = fopen(path, "r");
if (f == NULL) {
return NULL;
}
fseek(f, 0, SEEK_END);
len = ftell(f);
fseek(f, 0, SEEK_SET);
buf = (char*)malloc((size_t)len + 1);
if (buf == NULL) {
fclose(f);
return NULL;
}
if (fread(buf, 1, (size_t)len, f) != (size_t)len) {
fclose(f);
free(buf);
return NULL;
}
buf[len] = '\0';
fclose(f);
return buf;
}
/* ── esd_config_load ─────────────────────────────────────────────────── */
static void test_config_load_full_file(void)
{
esd_config_t cfg;
int rc;
memset(&cfg, 0, sizeof(cfg));
write_file(g_tmpdir, "esd.yaml",
"controller: 172.31.1.30:9990\n"
"poll_interval: 30\n"
"resources:\n"
" - name: Verilog_XL\n"
" collector: script\n"
" command: /etc/xce/lic-report.sh\n"
" interval: 15\n"
" - name: cloud_gpu_quota\n"
" collector: script\n"
" command: /etc/xce/quota-report.sh\n");
rc = esd_config_load_path(g_tmpdir, "esd.yaml", &cfg);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_STRING("172.31.1.30", cfg.controller_host);
TEST_ASSERT_EQUAL_INT(9990, cfg.controller_port);
TEST_ASSERT_EQUAL_INT(30, cfg.poll_interval);
TEST_ASSERT_EQUAL_INT(2, cfg.nsrc);
TEST_ASSERT_NOT_NULL(cfg.src);
TEST_ASSERT_EQUAL_STRING("Verilog_XL", cfg.src[0].name);
TEST_ASSERT_EQUAL_STRING("/etc/xce/lic-report.sh", cfg.src[0].command);
TEST_ASSERT_EQUAL_INT(15, cfg.src[0].interval);
TEST_ASSERT_EQUAL_STRING("cloud_gpu_quota", cfg.src[1].name);
TEST_ASSERT_EQUAL_STRING("/etc/xce/quota-report.sh", cfg.src[1].command);
TEST_ASSERT_EQUAL_INT(0, cfg.src[1].interval); /* no override */
esd_config_fini(&cfg);
}
static void test_config_load_missing_file_is_enoent(void)
{
esd_config_t cfg;
int rc;
memset(&cfg, 0, sizeof(cfg));
rc = esd_config_load_path(g_tmpdir, "no-such-file.yaml", &cfg);
TEST_ASSERT_EQUAL_INT(-ENOENT, rc);
}
static void test_config_load_bad_shape_is_einval(void)
{
esd_config_t cfg;
int rc;
memset(&cfg, 0, sizeof(cfg));
/* a source without a name is structurally invalid -- the controller
* would refuse the push and the operator would have no idea which
* entry died; fail loudly at load. */
write_file(g_tmpdir, "esd.yaml",
"controller: 172.31.1.30:9990\n"
"resources:\n"
" - collector: script\n"
" command: /bin/true\n");
rc = esd_config_load_path(g_tmpdir, "esd.yaml", &cfg);
TEST_ASSERT_EQUAL_INT(-EINVAL, rc);
/* a source without a command can never report -- same rule. */
memset(&cfg, 0, sizeof(cfg));
write_file(g_tmpdir, "esd.yaml",
"controller: 172.31.1.30:9990\n"
"resources:\n"
" - name: lic\n"
" collector: script\n");
rc = esd_config_load_path(g_tmpdir, "esd.yaml", &cfg);
TEST_ASSERT_EQUAL_INT(-EINVAL, rc);
}
/* ── script collector output parsing (parse contract, not the spawn) ── */
static void test_parse_output_bare_number(void)
{
double v = esd_parse_script_output("4\n");
TEST_ASSERT_FALSE(isnan(v));
TEST_ASSERT_EQUAL_FLOAT((float)4.0, (float)v);
}
static void test_parse_output_name_value(void)
{
/* elim-style "<name> <value>" -- name may mismatch; the source it is
* configured for is authoritative, the name is decorative. */
double v = esd_parse_script_output("Verilog_XL 7\n");
TEST_ASSERT_FALSE(isnan(v));
TEST_ASSERT_EQUAL_FLOAT((float)7.0, (float)v);
}
static void test_parse_output_whitespace_and_negative(void)
{
double v = esd_parse_script_output(" \t 3.5 \n");
TEST_ASSERT_FALSE(isnan(v));
TEST_ASSERT_EQUAL_FLOAT((float)3.5, (float)v);
v = esd_parse_script_output("lic -1\n");
TEST_ASSERT_FALSE(isnan(v));
TEST_ASSERT_EQUAL_FLOAT((float)-1.0, (float)v);
}
static void test_parse_output_garbage_is_nan(void)
{
double v;
v = esd_parse_script_output("");
TEST_ASSERT_TRUE(isnan(v));
v = esd_parse_script_output("\n");
TEST_ASSERT_TRUE(isnan(v));
v = esd_parse_script_output("no digits here\n");
TEST_ASSERT_TRUE(isnan(v));
v = esd_parse_script_output(NULL);
TEST_ASSERT_TRUE(isnan(v));
}
/* ── Phase 2: stale_limit config parsing ────────────────────────────── */
static void test_config_stale_limit_parse(void)
{
esd_config_t cfg;
int rc;
memset(&cfg, 0, sizeof(cfg));
/* per-source override wins; a source without one takes the daemon
* default; the daemon default itself is optional with a sane built-in
* (design.md 3.4: stale_limit in esd.yaml, fail-open keep-old-value). */
write_file(g_tmpdir, "esd.yaml",
"controller: 172.31.1.30:9990\n"
"poll_interval: 10\n"
"stale_limit: 200\n"
"resources:\n"
" - name: lic_a\n"
" collector: script\n"
" command: /bin/a.sh\n"
" stale_limit: 60\n"
" - name: lic_b\n"
" collector: script\n"
" command: /bin/b.sh\n");
rc = esd_config_load_path(g_tmpdir, "esd.yaml", &cfg);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(200, cfg.stale_limit);
TEST_ASSERT_EQUAL_INT(60, cfg.src[0].stale_limit);
TEST_ASSERT_EQUAL_INT(0, cfg.src[1].stale_limit); /* 0 = use default */
esd_config_fini(&cfg);
}
/* ── Phase 2: stale predicate ───────────────────────────────────────── */
static void test_stale_predicate(void)
{
esd_source_t src;
memset(&src, 0, sizeof(src));
src.name = (char*)"lic";
src.have_value = true;
src.last_value = 3.0;
/* Never report a value -> nothing to call stale; admission treats the
* pool as 0 through the normal NO_RESOURCE path. */
src.last_ok_time = 0;
TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60));
/* Fresh success (10s ago, limit 60) -> not stale. */
src.last_ok_time = 90;
TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60));
/* Exactly at the limit -> not stale (strictly-greater rule, so an
* interval == stale_limit config never flaps). */
src.last_ok_time = 40;
TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60));
/* Past the limit -> stale. */
src.last_ok_time = 39;
TEST_ASSERT_TRUE(esd_source_is_stale(&src, 100, 60));
/* A source whose own stale_limit override is set uses it. */
src.stale_limit = 10;
TEST_ASSERT_TRUE(esd_source_is_stale(&src, 100, 60));
src.last_ok_time = 95;
TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60));
/* last_ok_time newer than `now` (clock skew between test harness and
* caller) must not read as stale. */
src.last_ok_time = 105;
TEST_ASSERT_FALSE(esd_source_is_stale(&src, 100, 60));
}
/* ── Phase 2: status file ───────────────────────────────────────────── */
static void test_status_write_basic(void)
{
esd_config_t cfg;
esd_stats_t stats;
char path[512];
char* text;
time_t now = time(NULL);
char expect_lvt[128];
memset(&cfg, 0, sizeof(cfg));
cfg.nsrc = 1;
cfg.src = (esd_source_t*)calloc(1, sizeof(esd_source_t));
TEST_ASSERT_NOT_NULL(cfg.src);
cfg.src[0].name = (char*)"extlic";
cfg.src[0].have_value = true;
cfg.src[0].last_value = 3.0;
/* fresh success: last good value 5s ago, limit default 150 -> not
* stale (the marks-stale test below covers the other side) */
cfg.src[0].last_ok_time = now - 5;
cfg.stale_limit = ESD_STALE_LIMIT_DEFAULT;
memset(&stats, 0, sizeof(stats));
stats.started_at = 1000;
stats.pushes_ok = 7;
stats.pushes_fail = 1;
snprintf(path, sizeof(path), "%s/status.json", g_tmpdir);
TEST_ASSERT_EQUAL_INT(0, esd_status_write(path, &cfg, &stats));
text = slurp(path);
TEST_ASSERT_NOT_NULL(text);
/* The fields xceadmin extres status reads, and the stale flag driven
* by the same predicate the tests above pin. */
TEST_ASSERT_TRUE(strstr(text, "\"daemon\": \"xceesd\"") != NULL);
TEST_ASSERT_TRUE(strstr(text, "\"name\": \"extlic\"") != NULL);
snprintf(expect_lvt, sizeof(expect_lvt), "\"last_value\": %.17g", 3.0);
TEST_ASSERT_TRUE(strstr(text, expect_lvt) != NULL);
TEST_ASSERT_TRUE(strstr(text, "\"last_ok_time\": ") != NULL);
TEST_ASSERT_TRUE(strstr(text, "\"stale\": false") != NULL);
TEST_ASSERT_TRUE(strstr(text, "\"pushes_ok\": 7") != NULL);
TEST_ASSERT_TRUE(strstr(text, "\"pushes_fail\": 1") != NULL);
free(text);
free(cfg.src);
}
static void test_status_write_marks_stale(void)
{
esd_config_t cfg;
esd_stats_t stats;
char path[512];
char* text;
time_t now = time(NULL);
memset(&cfg, 0, sizeof(cfg));
cfg.nsrc = 1;
cfg.src = (esd_source_t*)calloc(1, sizeof(esd_source_t));
TEST_ASSERT_NOT_NULL(cfg.src);
cfg.src[0].name = (char*)"extlic";
cfg.src[0].have_value = true;
cfg.src[0].last_value = 2.0;
/* last success 1 hour ago, limit 150s -> stale. */
cfg.src[0].last_ok_time = now - 3600;
cfg.src[0].stale_limit = 150;
memset(&stats, 0, sizeof(stats));
snprintf(path, sizeof(path), "%s/status.json", g_tmpdir);
TEST_ASSERT_EQUAL_INT(0, esd_status_write(path, &cfg, &stats));
text = slurp(path);
TEST_ASSERT_NOT_NULL(text);
TEST_ASSERT_TRUE(strstr(text, "\"stale\": true") != NULL);
free(text);
free(cfg.src);
}
/* ── bug-1978: the reconnect-flag contract ───────────────────────────── */
static void test_reconnect_sets_both_rejoin_and_full_push(void)
{
g_needs_rejoin = 0;
g_needs_full_push = 0;
TEST_ASSERT_EQUAL_INT(0, esd_on_reconnect((xcecli_ctx_t)0x1, NULL));
/* BOTH flags: the re-join must happen before any re-push (an un-joined
* push is refused -EPERM on the fresh endpoint), and the re-push must
* still happen (the fresh controller knows no values). The bug-1978
* defect cleared only g_needs_full_push -- a daemon that re-pushed onto
* an un-joined HostRole_client endpoint, every push -EPERM forever. */
TEST_ASSERT_EQUAL_INT(1, g_needs_rejoin);
TEST_ASSERT_EQUAL_INT(1, g_needs_full_push);
}
static void test_reconnect_flags_recover_from_any_previous_state(void)
{
/* The handler is level-set, not level-toggle: regardless of what a
* previous (already consumed) reconnect left behind, a new callback
* invocation re-arms both flags. */
g_needs_rejoin = 0;
g_needs_full_push = 1; /* stale leftover from an unconsumed cycle */
TEST_ASSERT_EQUAL_INT(0, esd_on_reconnect((xcecli_ctx_t)0x2, NULL));
TEST_ASSERT_EQUAL_INT(1, g_needs_rejoin);
TEST_ASSERT_EQUAL_INT(1, g_needs_full_push);
}
int main(void)
{
UNITY_BEGIN();
RUN_TEST(test_config_load_full_file);
RUN_TEST(test_config_load_missing_file_is_enoent);
RUN_TEST(test_config_load_bad_shape_is_einval);
RUN_TEST(test_parse_output_bare_number);
RUN_TEST(test_parse_output_name_value);
RUN_TEST(test_parse_output_whitespace_and_negative);
RUN_TEST(test_parse_output_garbage_is_nan);
RUN_TEST(test_config_stale_limit_parse);
RUN_TEST(test_stale_predicate);
RUN_TEST(test_status_write_basic);
RUN_TEST(test_status_write_marks_stale);
RUN_TEST(test_reconnect_sets_both_rejoin_and_full_push);
RUN_TEST(test_reconnect_flags_recover_from_any_previous_state);
return UNITY_END();
}