feat(scheduler-dispatch-commit): add controller dispatch-failure classification (task 1.1)

Define xce_dispatch_fail_site_t/xce_dispatch_fail_class_t and
xce_classify_dispatch_fail() in job.c, mapping each of dispatch_a_job_()'s
six post-lookup failure points to retryable or terminal per Requirement
4.1/4.2. Standalone, not yet wired into dispatch_a_job_() (deferred to
tasks 3.2/3.3). Also adds this spec's requirements/design/research/tasks
docs (first commit touching this spec).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

src/controller/job.c

@@ -1,3 +1,4 @@
+#include <assert.h>
 #include <errno.h>
 #include <pthread.h>
 #include <signal.h>
@@ -828,6 +829,96 @@ static int mk_launchinfo(xce_launch_t* launch, xce_job_alloc_t* alloc)
     return rc;
 }
 
+/*
+ * scheduler-dispatch-commit task 1.1 (Req 4.1, 4.2): classifies one of
+ * dispatch_a_job_()'s six post-lookup failure points as retryable
+ * (transient/environmental -- return the job to PEND for a later retry) or
+ * terminal (a locally invalid decision that retrying will not fix -- resolve
+ * the job to a terminal failed state). Declared static/file-local: nothing
+ * outside job.c consumes this classification (design.md's own placement-
+ * correction note). dispatch_a_job_() itself is wired to call this at each
+ * of its six failure points by a later task; this task only adds the
+ * disconnected, independently-testable classification step --
+ * dispatch_a_job_() below is unmodified by this task.
+ */
+typedef enum {
+    XCE_DISPATCH_FAIL_SITE_WORKER_LOOKUP = 0, /* workermgr key_find() rc<0 */
+    XCE_DISPATCH_FAIL_SITE_WORKER_NULL   = 1, /* worker not found (w==NULL) */
+    XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY = 2, /* E_XCE_BAD_WORKER */
+    XCE_DISPATCH_FAIL_SITE_INVALID_ALLOC    = 3, /* nc <= 0 */
+    XCE_DISPATCH_FAIL_SITE_QUEUE_EXEC       = 4, /* get_queue_exec() failure */
+    XCE_DISPATCH_FAIL_SITE_RPC_FAILURE      = 5, /* xce_startjob_req() rc<0 */
+} xce_dispatch_fail_site_t;
+
+typedef enum {
+    XCE_DISPATCH_FAIL_RETRYABLE = 0,
+    XCE_DISPATCH_FAIL_TERMINAL  = 1,
+} xce_dispatch_fail_class_t;
+
+/* Keyed on WHICH failure point was hit (site identity), not on the raw rc
+ * that point returned: workermgr's key_find() and get_queue_exec() are two
+ * independently-implemented failure sources not guaranteed to return
+ * disjoint error codes (e.g. both could plausibly return -ENOENT for
+ * unrelated reasons), so classifying by rc value alone would be ambiguous
+ * whenever two sites happen to collide.
+ *
+ * Every named site is handled explicitly by its own case. The `default:`
+ * arm below exists for the impossible-by-construction case of a `site` with
+ * no matching enumerator (every current/planned call site passes a literal
+ * enumerator, so this can only be reached by a genuine programming-invariant
+ * violation, not by any operational/environmental condition) -- a plain
+ * default-less switch does NOT get compiler-enforced exhaustiveness in C
+ * (verified directly: GCC emits only a non-fatal `-Wreturn-type` "control
+ * reaches end of non-void function" warning here, and this project sets no
+ * -Werror anywhere), so falling off the end would silently return an
+ * indeterminate value instead. This mirrors persist.c's wal_classify()
+ * default arm exactly: ERROR-log and return a safe-side classification
+ * (TERMINAL -- never risk an unbounded retry loop on an unrecognized site)
+ * unconditionally, and additionally assert() only under XCE_TESTING, so
+ * xce_controller itself (built without -DXCE_TESTING) can never self-
+ * terminate on this path (CLAUDE.md's controller fail-loud/never-self-
+ * terminate discipline: assert()/abort is reserved for exactly this kind of
+ * impossible-state bug, but production must still never exit on it), while
+ * controller_unit_test (built WITH -DXCE_TESTING) crashes loud and
+ * immediately if it's ever hit. */
+static xce_dispatch_fail_class_t
+xce_classify_dispatch_fail(xce_dispatch_fail_site_t site)
+{
+    switch (site) {
+    case XCE_DISPATCH_FAIL_SITE_WORKER_LOOKUP:
+    case XCE_DISPATCH_FAIL_SITE_WORKER_NULL:
+    case XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY:
+    case XCE_DISPATCH_FAIL_SITE_RPC_FAILURE:
+        return XCE_DISPATCH_FAIL_RETRYABLE;
+    case XCE_DISPATCH_FAIL_SITE_INVALID_ALLOC:
+    case XCE_DISPATCH_FAIL_SITE_QUEUE_EXEC:
+        return XCE_DISPATCH_FAIL_TERMINAL;
+    default:
+        ERROR("xce_classify_dispatch_fail: unrecognized failure site %d -- "
+              "TERMINAL fallback; add it to the classification table",
+              (int)site);
+#ifdef XCE_TESTING
+        assert(0
+               && "unrecognized dispatch-fail site -- add it to "
+                  "xce_classify_dispatch_fail");
+#endif
+        return XCE_DISPATCH_FAIL_TERMINAL;
+    }
+}
+
+#ifdef XCE_TESTING
+/* Test-only accessor: xce_classify_dispatch_fail() and its enum types are
+ * intentionally undeclared in any header (nothing outside job.c consumes
+ * them), so the unit test reaches it through this thin, int-typed wrapper
+ * instead -- mirroring job_array_apply_test_set_force_fail_at()'s existing
+ * XCE_TESTING-only-export pattern earlier in this file (xce_controller
+ * itself is built without -DXCE_TESTING, so this never ships). */
+int xce_classify_dispatch_fail_test(int site)
+{
+    return (int)xce_classify_dispatch_fail((xce_dispatch_fail_site_t)site);
+}
+#endif /* XCE_TESTING */
+
 static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
 {
     int                rc = 0;

src/controller/meson.build

@@ -176,6 +176,7 @@ controller_unit_test_src = [
   'testsuite/test_job_array_apply.c',
   'testsuite/test_job_array_fence.c',
   'testsuite/test_job_array_replay_fence.c',
+  'testsuite/test_job_dispatch.c',
   'testsuite/test_upgrade_snapshot_restore.c',
   'testsuite/test_upgrade_snapshot_apply.c',
   'testsuite/test_bug857_regression.c',

src/controller/testsuite/controller_test.c

@@ -94,6 +94,7 @@ extern void test_upgrade_snapshot_restore(void);
 extern void test_upgrade_snapshot_apply(void);
 extern void test_bug857_regression(void);
 extern void test_sched_admin_dispatch(void);
+extern void test_job_dispatch(void);
 
 void setUp(void) { return; }
 
@@ -257,6 +258,7 @@ int main(int argc, char* argv[])
     TEST(test_upgrade_snapshot_apply);
     TEST(test_bug857_regression);
     TEST(test_sched_admin_dispatch);
+    TEST(test_job_dispatch);
 
     return UNITY_END();
 }

src/controller/testsuite/test_job_dispatch.c

@@ -0,0 +1,114 @@
+/* test_job_dispatch.c -- scheduler-dispatch-commit task 1.1
+ *
+ * Unit tests for the Controller Dispatch-Failure Classification component
+ * (design.md, "Controller Dispatch-Failure Classification"): a pure, static,
+ * file-local function in job.c that maps each of dispatch_a_job_()'s six
+ * post-lookup failure points (xce_dispatch_fail_site_t) to a retryable-vs-
+ * terminal classification (xce_dispatch_fail_class_t), per Requirements
+ * 4.1/4.2.
+ *
+ * xce_classify_dispatch_fail() and its enum types are deliberately NOT
+ * declared in any header (design.md's own placement-correction note: nothing
+ * outside job.c consumes them). This test file reaches the static function
+ * through a thin, int-typed, XCE_TESTING-only wrapper exported from job.c
+ * (xce_classify_dispatch_fail_test), the same pattern job.c already uses for
+ * job_array_apply_test_set_force_fail_at()/friends -- and, like test_job_
+ * array_fence.c's ha_arm_promotion_fence_timer_for_test, the extern
+ * declaration lives directly in this test file rather than in a shared
+ * header.
+ *
+ * No cluster_t/persist/worker fixture is needed: the classification is a
+ * pure decision with no side effects, independently testable against every
+ * known failure case without a live dispatch attempt (task 1.1's own
+ * Observable-completion clause).
+ */
+
+#include <stdio.h>
+
+#include <unity/unity.h>
+
+/* Mirrors design.md's Service Interface enum values exactly (job.c's own
+ * xce_dispatch_fail_site_t/xce_dispatch_fail_class_t are file-local and not
+ * shared via a header -- this test drives the wrapper by the same plain-int
+ * values documented there, not by including a private type). */
+#define SITE_WORKER_LOOKUP    0
+#define SITE_WORKER_NULL      1
+#define SITE_WORKER_NOT_READY 2
+#define SITE_INVALID_ALLOC    3
+#define SITE_QUEUE_EXEC       4
+#define SITE_RPC_FAILURE      5
+
+#define CLASS_RETRYABLE 0
+#define CLASS_TERMINAL  1
+
+extern int xce_classify_dispatch_fail_test(int site);
+
+static void test_worker_lookup_is_retryable(void)
+{
+    TEST_ASSERT_EQUAL_INT(CLASS_RETRYABLE,
+                          xce_classify_dispatch_fail_test(SITE_WORKER_LOOKUP));
+    printf("test_worker_lookup_is_retryable: PASS\n");
+}
+
+static void test_worker_null_is_retryable(void)
+{
+    TEST_ASSERT_EQUAL_INT(CLASS_RETRYABLE,
+                          xce_classify_dispatch_fail_test(SITE_WORKER_NULL));
+    printf("test_worker_null_is_retryable: PASS\n");
+}
+
+static void test_worker_not_ready_is_retryable(void)
+{
+    TEST_ASSERT_EQUAL_INT(CLASS_RETRYABLE, xce_classify_dispatch_fail_test(
+                                               SITE_WORKER_NOT_READY));
+    printf("test_worker_not_ready_is_retryable: PASS\n");
+}
+
+static void test_rpc_failure_is_retryable(void)
+{
+    TEST_ASSERT_EQUAL_INT(CLASS_RETRYABLE,
+                          xce_classify_dispatch_fail_test(SITE_RPC_FAILURE));
+    printf("test_rpc_failure_is_retryable: PASS\n");
+}
+
+static void test_invalid_alloc_is_terminal(void)
+{
+    TEST_ASSERT_EQUAL_INT(CLASS_TERMINAL,
+                          xce_classify_dispatch_fail_test(SITE_INVALID_ALLOC));
+    printf("test_invalid_alloc_is_terminal: PASS\n");
+}
+
+static void test_queue_exec_is_terminal(void)
+{
+    TEST_ASSERT_EQUAL_INT(CLASS_TERMINAL,
+                          xce_classify_dispatch_fail_test(SITE_QUEUE_EXEC));
+    printf("test_queue_exec_is_terminal: PASS\n");
+}
+
+/* Invariant (design.md Service Interface Postconditions): the same site
+ * always classifies the same way -- no hidden state, no time-dependence. */
+static void test_classification_is_stable_across_repeated_calls(void)
+{
+    int first  = xce_classify_dispatch_fail_test(SITE_WORKER_NOT_READY);
+    int second = xce_classify_dispatch_fail_test(SITE_WORKER_NOT_READY);
+    int third  = xce_classify_dispatch_fail_test(SITE_QUEUE_EXEC);
+    int fourth = xce_classify_dispatch_fail_test(SITE_QUEUE_EXEC);
+
+    TEST_ASSERT_EQUAL_INT(first, second);
+    TEST_ASSERT_EQUAL_INT(third, fourth);
+    printf("test_classification_is_stable_across_repeated_calls: PASS\n");
+}
+
+void test_job_dispatch(void)
+{
+    UnitySetTestFile(__FILE__);
+    printf("\n>> dispatch-failure classification tests (Req 4.1, 4.2)\n\n");
+
+    RUN_TEST(test_worker_lookup_is_retryable);
+    RUN_TEST(test_worker_null_is_retryable);
+    RUN_TEST(test_worker_not_ready_is_retryable);
+    RUN_TEST(test_rpc_failure_is_retryable);
+    RUN_TEST(test_invalid_alloc_is_terminal);
+    RUN_TEST(test_queue_exec_is_terminal);
+    RUN_TEST(test_classification_is_stable_across_repeated_calls);
+}