feat(xce-test-e2e-primitives): wait-primary [timeout_ms] (task 6.1)

.kiro/specs/xce-test-e2e-primitives/tasks.md

@@ -61,7 +61,7 @@
   - _Depends: 5.1, 3.1, 4.1_
 
 - [ ] 6. Wait for elected primary
-- [ ] 6.1 `wait-primary [timeout_ms]`
+- [x] 6.1 `wait-primary [timeout_ms]`
   - Add `cmd_wait_primary`: loop a cheap, failover-aware client op (the existing `joblist`/probe path via `active_ctx`) until it succeeds or the timeout elapses (monotonic clock); default timeout when the arg is omitted; return 0 on success, `<0` on timeout.
   - Observable: against a healthy cluster `wait-primary` returns immediately; after a primary kill it blocks until a new primary serves, then succeeds; with no primary within the timeout it fails.
   - _Requirements: 4.1, 4.2, 4.3, 4.4_
@@ -92,3 +92,4 @@
 - 3.1: introduced `g_last_jobid[XCE_MAXNAME_LEN]={0}` (empty = unset) and `g_last_jobcount=-1` (-1 = unset) file-scope state, POPULATED later by 4.1 (jobsub) / 5.1 (joblist). `substitute_autovars(line,out,sz)` does whole-token replace of `$?`/`$jobid`/`$jobcount`; unset `$jobid`/`$jobcount` → command fails (`autovar: $X referenced before set`), no garbage. `!` shell-escape lines are NOT substituted (raw to shell — the shell owns its `$`). Added `#include "config.h"` for XCE_MAXNAME_LEN.
 - 4.1 (bug-786 fixed): `cmd_jobsub` inits local `jobid=0`, prints `jobsubmit ok` + captures `g_last_jobid` ONLY after BOTH gates: `rc>=0` AND `jobid!=0`; else typed error (`E_XCE_NOMASTER/REDIRECT/NOT_READY`/timeout switch via `xce_strerror_r`) + `return <0`. **`$jobid` capture uses `fmt_jobid(g_last_jobid, ..., jobid)` — the FULL jobid, byte-identical to how `cmd_joblist` formats `jobs[i]->jobid`** (NOT `array_jobid()`, which the display-only `ok:` line uses). This is REQUIRED so task 5.3 `expect-job $jobid` matches the joblist column for array jobs. Also moved the error check AFTER the FREEUP block → fixed a pre-existing error-path leak. Live success-path ($jobid==joblist entry) verified by code reading; runtime-verified via soak (7.2).
 - 5.1: extracted DRY print-free helper `static int joblist_query(const char* find_jobid, int* contains, int print_table)` — single `xcecli_joblist` call + internal free; returns count (>=0) or `<0`; membership via `strcmp(fmt_jobid(buf,...,jobs[i]->jobid), find_jobid)` (SAME full-jobid form as 4.1's `$jobid`). `cmd_joblist` delegates `joblist_query(NULL,NULL,1)` and sets `g_last_jobcount=count` ONLY on success (no fake count on failure). 5.2/5.3 assertion commands call `joblist_query` directly. Live `$jobcount`-after-success needs a controller (soak 7.2).
+- 5.2/5.3/6.1: `expect-jobs`, `expect-job`, `wait-primary` are all CLIENT-context commands registered in `client_commands[]` (they need `active_ctx` to query jobs) — a script must `client` first. All return `<0` on failure → `preprocess_and_run` fails the session. Implemented directly (no flag macro); RED shown via unknown-command absence. Key invariant across assertions: a FAILED joblist query is an assertion failure (distinct message), NEVER silently "0 jobs"/"absent". `wait-primary`: monotonic `clock_gettime(CLOCK_MONOTONIC)`, default `WAIT_PRIMARY_DEFAULT_MS=30000`, poll `WAIT_PRIMARY_POLL_MS=200`, bounded (no hang) — do NOT run no-arg `wait-primary` against a dead cluster in a test (blocks 30s); pass a small explicit timeout.

src/controller/testsuite/test.c

@@ -119,6 +119,7 @@ extern int cmd_jobinfo(int, char*[]);
 extern int cmd_echo(int, char*[]);
 extern int cmd_expect_jobs(int, char*[]);
 extern int cmd_expect_job(int, char*[]);
+extern int cmd_wait_primary(int, char*[]);
 static int worker_house_keeping(void*);
 
 /* Session verdict / loop state (xce-test-e2e-primitives task 1.1).
@@ -205,6 +206,8 @@ static command_t client_commands[] = {
     { "joblist",    cmd_joblist,      "list jobs"                   },
     { "expect-jobs", cmd_expect_jobs, "assert current job count"    },
     { "expect-job", cmd_expect_job,   "assert a JobId is present"   },
+    { "wait-primary", cmd_wait_primary,
+      "wait until a primary serves [timeout_ms]"                    },
     { "shutdown",   cmd_shutdown,     "shutdown xce controller"     },
     { "jobrun",     cmd_jobrun,       "run a job"                   },
     { "jobsub",     cmd_jobsub,       "submit a job"                },
@@ -1072,6 +1075,102 @@ int cmd_expect_job(int argc, char* argv[])
     return 0;
 }
 
+/* `wait-primary [timeout_ms]` (xce-test-e2e-primitives task 6.1, Req 4.1-4.4,
+ * design.md cmd_wait_primary).
+ *
+ * Blocks until a cheap, failover-aware client op succeeds, or the timeout
+ * elapses. The probe is the print-free joblist_query(NULL, NULL, 0): it drives
+ * active_ctx + libxce master-rotation/failover and returns a job count (>= 0)
+ * only when a primary actually served the request, or < 0 on error (Req 4.1).
+ *
+ * On the first successful probe we return 0 (Req 4.2). Otherwise we sleep a
+ * short poll interval and retry, re-checking a MONOTONIC deadline between
+ * attempts so the command returns within a bounded time of the timeout and
+ * never hangs indefinitely when there is no primary (Req 4.3). The deadline is
+ * derived from CLOCK_MONOTONIC, never wall-clock, so a clock step cannot skew
+ * it.
+ *
+ * The timeout is argv[1] in milliseconds when supplied, else
+ * WAIT_PRIMARY_DEFAULT_MS (Req 4.4). A malformed timeout argument is a usage
+ * error (return < 0) rather than a silent fallback to the default. */
+#define WAIT_PRIMARY_DEFAULT_MS 30000L /* default max wait when arg omitted */
+#define WAIT_PRIMARY_POLL_MS    200L   /* sleep between probe attempts       */
+#define MS_PER_SEC              1000L
+#define NS_PER_MS               1000000L
+#define US_PER_MS               1000L
+
+static long monotonic_ms(void)
+{
+    struct timespec ts;
+
+    clock_gettime(CLOCK_MONOTONIC, &ts);
+
+    return ((long)ts.tv_sec * MS_PER_SEC) + (ts.tv_nsec / NS_PER_MS);
+}
+
+int cmd_wait_primary(int argc, char* argv[])
+{
+    long timeout_ms = WAIT_PRIMARY_DEFAULT_MS;
+    long start_ms, elapsed_ms;
+
+    if (argc > 2) {
+        printf("wait-primary [timeout_ms]\n");
+        return -EINVAL;
+    }
+
+    if (argc == 2) {
+        char* end;
+
+        errno      = 0;
+        timeout_ms = strtol(argv[1], &end, 10);
+
+        if ((errno != 0) || (end == argv[1]) || (*end != '\0')
+            || (timeout_ms < 0)) {
+            printf("wait-primary: invalid timeout '%s' (expected a "
+                   "non-negative integer in ms)\n",
+                   argv[1]);
+            return -EINVAL;
+        }
+    }
+
+    start_ms = monotonic_ms();
+
+    /* Probe, then re-check the monotonic deadline between attempts. Each probe
+     * may itself take time (libxce failover has its own internal retry budget);
+     * we re-read the clock after every probe returns so the overall loop honors
+     * the deadline and returns failure in bounded time. */
+    for (;;) {
+        int rc = joblist_query(NULL, NULL, 0);
+
+        if (rc >= 0) {
+            elapsed_ms = monotonic_ms() - start_ms;
+            printf("wait-primary: primary serving after %ld ms\n", elapsed_ms);
+            return 0;
+        }
+
+        elapsed_ms = monotonic_ms() - start_ms;
+
+        if (elapsed_ms >= timeout_ms) {
+            printf("wait-primary: no primary served within %ld ms (waited "
+                   "%ld ms)\n",
+                   timeout_ms, elapsed_ms);
+            return -ETIMEDOUT;
+        }
+
+        /* Don't oversleep past the deadline: cap the poll to the time left. */
+        {
+            long remaining_ms = timeout_ms - elapsed_ms;
+            long sleep_ms     = (remaining_ms < WAIT_PRIMARY_POLL_MS)
+                                    ? remaining_ms
+                                    : WAIT_PRIMARY_POLL_MS;
+
+            if (sleep_ms > 0) {
+                usleep((useconds_t)(sleep_ms * US_PER_MS));
+            }
+        }
+    }
+}
+
 static void print_jobinfo(xce_job_t* job)
 {
     char idstring[128];