From 9466b716919191885a1a0383a81a0d45719e809e Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Fri, 19 Jun 2026 16:29:26 +0000 Subject: [PATCH] feat(xce-test-e2e-primitives): wait-primary [timeout_ms] (task 6.1) --- .kiro/specs/xce-test-e2e-primitives/tasks.md | 3 +- src/controller/testsuite/test.c | 99 ++++++++++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/.kiro/specs/xce-test-e2e-primitives/tasks.md b/.kiro/specs/xce-test-e2e-primitives/tasks.md index 206251db..ad361774 100644 --- a/.kiro/specs/xce-test-e2e-primitives/tasks.md +++ b/.kiro/specs/xce-test-e2e-primitives/tasks.md @@ -61,7 +61,7 @@ - _Depends: 5.1, 3.1, 4.1_ - [ ] 6. Wait for elected primary -- [ ] 6.1 `wait-primary [timeout_ms]` +- [x] 6.1 `wait-primary [timeout_ms]` - Add `cmd_wait_primary`: loop a cheap, failover-aware client op (the existing `joblist`/probe path via `active_ctx`) until it succeeds or the timeout elapses (monotonic clock); default timeout when the arg is omitted; return 0 on success, `<0` on timeout. - Observable: against a healthy cluster `wait-primary` returns immediately; after a primary kill it blocks until a new primary serves, then succeeds; with no primary within the timeout it fails. - _Requirements: 4.1, 4.2, 4.3, 4.4_ @@ -92,3 +92,4 @@ - 3.1: introduced `g_last_jobid[XCE_MAXNAME_LEN]={0}` (empty = unset) and `g_last_jobcount=-1` (-1 = unset) file-scope state, POPULATED later by 4.1 (jobsub) / 5.1 (joblist). `substitute_autovars(line,out,sz)` does whole-token replace of `$?`/`$jobid`/`$jobcount`; unset `$jobid`/`$jobcount` → command fails (`autovar: $X referenced before set`), no garbage. `!` shell-escape lines are NOT substituted (raw to shell — the shell owns its `$`). Added `#include "config.h"` for XCE_MAXNAME_LEN. - 4.1 (bug-786 fixed): `cmd_jobsub` inits local `jobid=0`, prints `jobsubmit ok` + captures `g_last_jobid` ONLY after BOTH gates: `rc>=0` AND `jobid!=0`; else typed error (`E_XCE_NOMASTER/REDIRECT/NOT_READY`/timeout switch via `xce_strerror_r`) + `return <0`. **`$jobid` capture uses `fmt_jobid(g_last_jobid, ..., jobid)` — the FULL jobid, byte-identical to how `cmd_joblist` formats `jobs[i]->jobid`** (NOT `array_jobid()`, which the display-only `ok:` line uses). This is REQUIRED so task 5.3 `expect-job $jobid` matches the joblist column for array jobs. Also moved the error check AFTER the FREEUP block → fixed a pre-existing error-path leak. Live success-path ($jobid==joblist entry) verified by code reading; runtime-verified via soak (7.2). - 5.1: extracted DRY print-free helper `static int joblist_query(const char* find_jobid, int* contains, int print_table)` — single `xcecli_joblist` call + internal free; returns count (>=0) or `<0`; membership via `strcmp(fmt_jobid(buf,...,jobs[i]->jobid), find_jobid)` (SAME full-jobid form as 4.1's `$jobid`). `cmd_joblist` delegates `joblist_query(NULL,NULL,1)` and sets `g_last_jobcount=count` ONLY on success (no fake count on failure). 5.2/5.3 assertion commands call `joblist_query` directly. Live `$jobcount`-after-success needs a controller (soak 7.2). +- 5.2/5.3/6.1: `expect-jobs`, `expect-job`, `wait-primary` are all CLIENT-context commands registered in `client_commands[]` (they need `active_ctx` to query jobs) — a script must `client` first. All return `<0` on failure → `preprocess_and_run` fails the session. Implemented directly (no flag macro); RED shown via unknown-command absence. Key invariant across assertions: a FAILED joblist query is an assertion failure (distinct message), NEVER silently "0 jobs"/"absent". `wait-primary`: monotonic `clock_gettime(CLOCK_MONOTONIC)`, default `WAIT_PRIMARY_DEFAULT_MS=30000`, poll `WAIT_PRIMARY_POLL_MS=200`, bounded (no hang) — do NOT run no-arg `wait-primary` against a dead cluster in a test (blocks 30s); pass a small explicit timeout. diff --git a/src/controller/testsuite/test.c b/src/controller/testsuite/test.c index 3a4acef6..6b50cd12 100644 --- a/src/controller/testsuite/test.c +++ b/src/controller/testsuite/test.c @@ -119,6 +119,7 @@ extern int cmd_jobinfo(int, char*[]); extern int cmd_echo(int, char*[]); extern int cmd_expect_jobs(int, char*[]); extern int cmd_expect_job(int, char*[]); +extern int cmd_wait_primary(int, char*[]); static int worker_house_keeping(void*); /* Session verdict / loop state (xce-test-e2e-primitives task 1.1). @@ -205,6 +206,8 @@ static command_t client_commands[] = { { "joblist", cmd_joblist, "list jobs" }, { "expect-jobs", cmd_expect_jobs, "assert current job count" }, { "expect-job", cmd_expect_job, "assert a JobId is present" }, + { "wait-primary", cmd_wait_primary, + "wait until a primary serves [timeout_ms]" }, { "shutdown", cmd_shutdown, "shutdown xce controller" }, { "jobrun", cmd_jobrun, "run a job" }, { "jobsub", cmd_jobsub, "submit a job" }, @@ -1072,6 +1075,102 @@ int cmd_expect_job(int argc, char* argv[]) return 0; } +/* `wait-primary [timeout_ms]` (xce-test-e2e-primitives task 6.1, Req 4.1-4.4, + * design.md cmd_wait_primary). + * + * Blocks until a cheap, failover-aware client op succeeds, or the timeout + * elapses. The probe is the print-free joblist_query(NULL, NULL, 0): it drives + * active_ctx + libxce master-rotation/failover and returns a job count (>= 0) + * only when a primary actually served the request, or < 0 on error (Req 4.1). + * + * On the first successful probe we return 0 (Req 4.2). Otherwise we sleep a + * short poll interval and retry, re-checking a MONOTONIC deadline between + * attempts so the command returns within a bounded time of the timeout and + * never hangs indefinitely when there is no primary (Req 4.3). The deadline is + * derived from CLOCK_MONOTONIC, never wall-clock, so a clock step cannot skew + * it. + * + * The timeout is argv[1] in milliseconds when supplied, else + * WAIT_PRIMARY_DEFAULT_MS (Req 4.4). A malformed timeout argument is a usage + * error (return < 0) rather than a silent fallback to the default. */ +#define WAIT_PRIMARY_DEFAULT_MS 30000L /* default max wait when arg omitted */ +#define WAIT_PRIMARY_POLL_MS 200L /* sleep between probe attempts */ +#define MS_PER_SEC 1000L +#define NS_PER_MS 1000000L +#define US_PER_MS 1000L + +static long monotonic_ms(void) +{ + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + + return ((long)ts.tv_sec * MS_PER_SEC) + (ts.tv_nsec / NS_PER_MS); +} + +int cmd_wait_primary(int argc, char* argv[]) +{ + long timeout_ms = WAIT_PRIMARY_DEFAULT_MS; + long start_ms, elapsed_ms; + + if (argc > 2) { + printf("wait-primary [timeout_ms]\n"); + return -EINVAL; + } + + if (argc == 2) { + char* end; + + errno = 0; + timeout_ms = strtol(argv[1], &end, 10); + + if ((errno != 0) || (end == argv[1]) || (*end != '\0') + || (timeout_ms < 0)) { + printf("wait-primary: invalid timeout '%s' (expected a " + "non-negative integer in ms)\n", + argv[1]); + return -EINVAL; + } + } + + start_ms = monotonic_ms(); + + /* Probe, then re-check the monotonic deadline between attempts. Each probe + * may itself take time (libxce failover has its own internal retry budget); + * we re-read the clock after every probe returns so the overall loop honors + * the deadline and returns failure in bounded time. */ + for (;;) { + int rc = joblist_query(NULL, NULL, 0); + + if (rc >= 0) { + elapsed_ms = monotonic_ms() - start_ms; + printf("wait-primary: primary serving after %ld ms\n", elapsed_ms); + return 0; + } + + elapsed_ms = monotonic_ms() - start_ms; + + if (elapsed_ms >= timeout_ms) { + printf("wait-primary: no primary served within %ld ms (waited " + "%ld ms)\n", + timeout_ms, elapsed_ms); + return -ETIMEDOUT; + } + + /* Don't oversleep past the deadline: cap the poll to the time left. */ + { + long remaining_ms = timeout_ms - elapsed_ms; + long sleep_ms = (remaining_ms < WAIT_PRIMARY_POLL_MS) + ? remaining_ms + : WAIT_PRIMARY_POLL_MS; + + if (sleep_ms > 0) { + usleep((useconds_t)(sleep_ms * US_PER_MS)); + } + } + } +} + static void print_jobinfo(xce_job_t* job) { char idstring[128];