merge: sync Rongsong's scheduler-framework into chatbot

Full merge of origin/scheduler-framework (scheduler-core restructure,
failover-adoption, WAL_JOB_DISPATCH, apiserver backend health/heartbeat,
build-time versioning, bug-1673/1676/1678/1679/1680).

Conflict resolution:
- scheduler: adopt Rongsong's scheduler-core architecture (driver loop,
  queue project/picker fields, resource bug-1569 fix)
- worker/controller_connector.c: keep Rongsong's reconnect/rejoin
  structure, PORT our bug-1509 resend_job_status onto the rejoin success
  path (his reconnect didn't re-report job state)
- apiserver backend: take Rongsong's health/heartbeat (supersedes our
  e505066 circuit breaker)
- group.c: keep our bug-1513 comment
- misc: take theirs for additive fixes (go_build versioning,
  test_group.c, persist.c RUSAGE_UPDATE, FilterRows bug-1680)

NOTE: our scheduler fixes (bug-1494 family dispatch-fail requeue,
PEND-reason classification) are NOT in Rongsong's branch — needs
verification/porting onto scheduler-core (separate follow-up).

Co-Authored-By: Claude <noreply@anthropic.com>

src/controller/testsuite/test_apply_pump.c

@@ -68,6 +68,7 @@
 #include <errno.h>
 #include <pthread.h>
 #include <signal.h>
+#include <stdatomic.h>
 #include <stdbool.h>
 #include <stdio.h>
 #include <string.h>
@@ -1406,6 +1407,116 @@ static void test_apply_pump_late_pending_apply_registration_self_heals(void)
           "PASS\n");
 }
 
+/* bug-1428: a narrower TOCTOU residual of the SAME self-heal mechanism the
+ * test above proves. There, the register call arrives after the walk has
+ * ALREADY finished applying `lsn` -- the well-known bug-1415 case, closed
+ * by the fast-path self-heal check. This test targets the tighter,
+ * previously-unclosed window: the register call's check runs BEFORE the
+ * walk applies `lsn` (sees "not yet applied") but the walk's own
+ * advance-then-take sequence completes entirely BEFORE the register call
+ * reaches its insert -- so the walk's persist_take_pending_apply() finds
+ * nothing (register hasn't inserted yet), and the register call then
+ * inserts a node the walk will never revisit. Permanently orphaned,
+ * identical failure mode to bug-1415, different trigger window.
+ *
+ * Reproduced deterministically (no real-time race dependent on timing)
+ * by driving the exact bad ordering through controlled state, mirroring
+ * how the register thread's old code observed state before its lock and
+ * how the walk observed/mutated it independently:
+ *   1. Main thread holds pending_applies_lock (standing in for "the walk's
+ *      advance-then-take sequence is in flight, about to complete").
+ *   2. A second thread calls the real persist_register_pending_apply() for
+ *      an lsn NOT YET applied (last_applied_lsn is still below it) -- under
+ *      the pre-fix code this thread's unlocked check would already have
+ *      run and concluded "not yet applied" by this point; under the fixed
+ *      code it blocks entirely on the lock, check included.
+ *   3. Main thread (still holding the lock) advances last_applied_lsn past
+ *      lsn -- simulating the walk finishing its apply for this lsn while
+ *      the register call is blocked/in-flight.
+ *   4. Main thread releases the lock.
+ *   5. The second thread's persist_register_pending_apply() call proceeds.
+ *
+ * Post-fix: the register call's self-heal check runs AFTER acquiring the
+ * lock in step 5, observes the now-current last_applied_lsn from step 3,
+ * and fires on_applied directly -- no insert, no orphan. Pre-fix, the
+ * check would have already run (unlocked) back in step 2 against the
+ * STALE last_applied_lsn, concluded "not yet applied", and inserted
+ * unconditionally once the lock freed in step 5 -- orphaning it forever
+ * (on_applied never fires; the node stays linked on pending_applies).
+ */
+typedef struct {
+    persist_t* p;
+    uint64_t   lsn;
+    int        fired;
+    uint64_t   fired_lsn;
+} trpa_ctx_t;
+
+static void trpa_on_applied(void* ctx, uint64_t lsn)
+{
+    trpa_ctx_t* c = (trpa_ctx_t*)ctx;
+    c->fired      = 1;
+    c->fired_lsn  = lsn;
+}
+
+static void* trpa_register_thread(void* arg)
+{
+    trpa_ctx_t* c = (trpa_ctx_t*)arg;
+    persist_register_pending_apply(c->p, c->lsn, trpa_on_applied, c);
+    return NULL;
+}
+
+static void test_register_pending_apply_race_against_concurrent_apply_self_heals(void)
+{
+    persist_t p;
+    memset(&p, 0, sizeof(p));
+    TEST_ASSERT_EQUAL_INT(0, pthread_mutex_init(&p.pending_applies_lock, NULL));
+    atomic_store_explicit(&p.last_applied_lsn, 40, memory_order_relaxed);
+
+    const uint64_t target_lsn = 42; /* not yet applied at the moment the
+                                      * register thread's call begins */
+
+    trpa_ctx_t rctx = { .p = &p, .lsn = target_lsn, .fired = 0, .fired_lsn = 0 };
+
+    pthread_mutex_lock(&p.pending_applies_lock);
+
+    pthread_t th;
+    TEST_ASSERT_EQUAL_INT(0,
+        pthread_create(&th, NULL, trpa_register_thread, &rctx));
+
+    /* Give the register thread every opportunity to run past an unlocked
+     * check (pre-fix) before the walk's simulated advance below — a bounded
+     * sleep is fine here since the correctness assertion is deterministic
+     * regardless of exactly how far it got; this only maximizes pre-fix
+     * failure detection sensitivity. */
+    usleep(20000);
+
+    /* Simulate the walk finishing its apply of target_lsn while the
+     * register call is blocked on (or, pre-fix, already past) the check. */
+    atomic_store_explicit(&p.last_applied_lsn, target_lsn, memory_order_relaxed);
+
+    pthread_mutex_unlock(&p.pending_applies_lock);
+
+    struct timespec ts = { .tv_sec = 0, .tv_nsec = 200000000L };
+    nanosleep(&ts, NULL);
+    TEST_ASSERT_EQUAL_INT(0, pthread_join(th, NULL));
+
+    int has_pending = persist_has_pending_apply(&p, target_lsn);
+
+    pthread_mutex_destroy(&p.pending_applies_lock);
+
+    TEST_ASSERT_EQUAL_INT_MESSAGE(1, rctx.fired,
+        "bug-1428: a registration racing a concurrent apply of the same lsn "
+        "must still fire its hand-off (self-heal) exactly once -- 0 means "
+        "the TOCTOU orphaned it permanently");
+    TEST_ASSERT_EQUAL_UINT64_MESSAGE(target_lsn, rctx.fired_lsn,
+        "the hand-off must fire for the correct lsn");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0, has_pending,
+        "no orphaned registration may remain linked on pending_applies");
+
+    printf("test_register_pending_apply_race_against_concurrent_apply_self_heals: "
+          "PASS\n");
+}
+
 /* ── 6. Standby-ingest commit-then-apply (2026-07-10 fix) ────────────────
  *
  * Regression coverage for ha.c's standby WAL-replication ingest path.
@@ -1878,6 +1989,7 @@ void test_apply_pump(void)
     RUN_TEST(test_apply_pump_guard_does_not_break_job_ctrl_live_append);
     RUN_TEST(test_apply_pump_fail_racing_finish_merges_against_current_state);
     RUN_TEST(test_apply_pump_late_pending_apply_registration_self_heals);
+    RUN_TEST(test_register_pending_apply_race_against_concurrent_apply_self_heals);
     RUN_TEST(test_standby_apply_deferred_until_committed_then_materializes);
     RUN_TEST(test_standby_truncate_before_commit_leaves_no_phantom_job);
 }