fix(controller): apply worker finish arriving before job materializes (bug-1509)

A worker-reported finish may arrive before the job exists: job
construction is deferred to apply_committed_entry (the submit's WAL
entry), leaving a commit->apply window. on_job_finish dropped such
finishes ('can not find job'), so a fast job that finished in that
window stayed RUN forever (e.g. noexec-test/newcfg-test/verify-fix).

Add an in-memory unknown-finish registry: on_job_finish registers the
finish when the job isn't materialized, and apply_committed_entry's
WAL_JOB_SUBMIT case drains it right after restore_job — appending the
durable WAL_JOB_FINISH and mutating the job to DONE/EXIT immediately so
the scheduler never dispatches a job that already finished.

Design: docs/Job-Finish-Robustness-设计文档.md (Part A of A+B; Part B
worker-liveness + reconnect reconciliation deferred to later phases).

Unit tests (red->green): registry register/take, apply-once-job-
materializes (DONE), exit-code maps to EXIT, drain no-op without
registration.

src/controller/job.c

@@ -1759,6 +1759,140 @@ static int record_job_finish_fact(cluster_t* c, job_t* job,
     return 0;
 }
 
+/* ── Unknown-finish registry (bug-1509) ────────────────────────────────────
+ *
+ * A worker-reported finish may arrive at the controller BEFORE the job is
+ * materialized: job construction is deferred to apply_committed_entry (the
+ * submit's WAL entry), so there is a commit→apply window during which
+ * key_find(jobmgr) misses the job. on_job_finish used to drop such finishes
+ * ("can not find job"), leaving the job RUN forever. This registry holds the
+ * finish until the job materializes; apply_committed_entry's WAL_JOB_SUBMIT
+ * case then drains it via apply_registered_finish_if_any().
+ *
+ * The registry is process-local and NOT durable across failover (a finish
+ * registered but unconsumed when the primary fails over is lost; the new
+ * primary replays the submit with the job stuck PEND). Documented residual
+ * risk (design §3.6/A3) — the commit→apply window is small. */
+typedef struct unknown_finish {
+    uint64_t               jobid;
+    xce_jobfinish_t        finish;    /* owned copy (2 scalar fields) */
+    time_t                 queued_at;
+    struct unknown_finish* next;
+} unknown_finish_t;
+
+static pthread_mutex_t   g_unknown_finish_mu = PTHREAD_MUTEX_INITIALIZER;
+static unknown_finish_t* g_unknown_finish_list = NULL;
+
+#define UNKNOWN_FINISH_TTL_S (60) /* must exceed max commit→apply lag */
+
+static void unknown_finish_sweep_locked(void)
+{
+    unknown_finish_t** pp = &g_unknown_finish_list;
+    time_t            now = time(NULL);
+    while (*pp != NULL) {
+        if (now - (*pp)->queued_at > UNKNOWN_FINISH_TTL_S) {
+            unknown_finish_t* dead = *pp;
+            *pp = dead->next;
+            free(dead);
+        } else {
+            pp = &(*pp)->next;
+        }
+    }
+}
+
+static int unknown_finish_register(const xce_jobfinish_t* finish)
+{
+    unknown_finish_t* uf;
+
+    if (finish == NULL) {
+        return -EINVAL;
+    }
+    uf = (unknown_finish_t*)calloc(1, sizeof(*uf));
+    if (uf == NULL) {
+        return -ENOMEM;
+    }
+    uf->jobid     = finish->jobid;
+    uf->finish    = *finish;
+    uf->queued_at = time(NULL);
+
+    pthread_mutex_lock(&g_unknown_finish_mu);
+    unknown_finish_sweep_locked(); /* opportunistic TTL bound */
+    uf->next = g_unknown_finish_list;
+    g_unknown_finish_list = uf;
+    pthread_mutex_unlock(&g_unknown_finish_mu);
+    return 0;
+}
+
+/* Take (and remove) the registered finish for jobid. Returns 1 if found, 0
+ * if not. `out` (optional) receives a copy. */
+static int unknown_finish_take(uint64_t jobid, xce_jobfinish_t* out)
+{
+    unknown_finish_t** pp = &g_unknown_finish_list;
+    int               found = 0;
+
+    pthread_mutex_lock(&g_unknown_finish_mu);
+    while (*pp != NULL) {
+        if ((*pp)->jobid == jobid) {
+            unknown_finish_t* uf = *pp;
+            *pp = uf->next;
+            if (out != NULL) {
+                *out = uf->finish;
+            }
+            free(uf);
+            found = 1;
+            break;
+        }
+        pp = &(*pp)->next;
+    }
+    pthread_mutex_unlock(&g_unknown_finish_mu);
+    return found;
+}
+
+/* Called from apply_committed_entry's WAL_JOB_SUBMIT case right after the job
+ * is materialized: if a worker finish was registered for this jobid while the
+ * job did not yet exist, append the durable WAL_JOB_FINISH and mutate the job
+ * to its terminal state immediately — so the scheduler never sees a
+ * PEND/RUN job that already finished. The later WAL apply of the appended
+ * entry is a terminal→terminal no-op. Returns 1 if a deferred finish was
+ * applied, 0 otherwise. */
+int apply_registered_finish_if_any(cluster_t* c, job_t* j)
+{
+    xce_jobfinish_t finish;
+
+    if (c == NULL || j == NULL) {
+        return 0;
+    }
+    if (!unknown_finish_take(j->job.jobid, &finish)) {
+        return 0;
+    }
+
+    lock(j, RTTI_LOCK_WRITE);
+    /* Job is PEND here (not terminal), so record_job_finish_fact appends the
+     * durable WAL_JOB_FINISH rather than skipping it as a duplicate. */
+    record_job_finish_fact(c, j, &finish);
+    /* Mutate to terminal immediately; later WAL apply is a no-op. */
+    apply_job_finish_transition(j, &finish);
+    unlock(j);
+
+    INFO("job %ld: applied deferred worker finish (exit_code=%d) at submit "
+         "apply",
+         j->job.jobid, finish.exit_code);
+    return 1;
+}
+
+#ifdef XCE_TESTING
+/* Exposed for the controller unit test binary (compiled with -DXCE_TESTING):
+ * direct probes of the bug-1509 unknown-finish registry. */
+int test_unknown_finish_register(const xce_jobfinish_t* f)
+{
+    return unknown_finish_register(f);
+}
+int test_unknown_finish_take(uint64_t jobid, xce_jobfinish_t* out)
+{
+    return unknown_finish_take(jobid, out);
+}
+#endif
+
 int on_job_finish(cluster_t* c, xce_jobfinish_t* finish)
 {
     job_t* job = NULL;
@@ -1774,6 +1908,17 @@ int on_job_finish(cluster_t* c, xce_jobfinish_t* finish)
     unlock(c->jobmgr);
 
     if (job == NULL) {
+        /* bug-1509: finish arrived before the job was materialized (submit
+         * committed but not yet applied — job construction is deferred to
+         * apply_committed_entry). Register it in the unknown-finish registry;
+         * the submit's apply drains it and converges the job to DONE/EXIT.
+         * Previously this finish was dropped, leaving the job RUN forever. */
+        if (unknown_finish_register(finish) == 0) {
+            INFO("job %ld: finish for un-materialized job queued in "
+                 "unknown-finish registry",
+                 finish->jobid);
+            return 0;
+        }
         ERROR("can not find job %ld", finish->jobid);
         return -errno;
     }