fix(scheduler): record config pushes, free hlimits lists, quiet benign picker WARN (bug-1847/1848/1849)

Follow-ups from the bug-1846 investigation.

bug-1847: config pushes never reached a .replay recording -- handler.c
applied them synchronously, bypassing the pending-update queue the
recorder writes from. With bug-1846's deferral they're now applied at a
round boundary on the driver thread, so sched_apply_pending_config()
writes a CONFIG_PUSH event via the existing sched_record_write_events()
(no-op when not recording), ahead of the round's other recorded events.
Test: test_config_push_deferred.c's test_applied_config_push_is_recorded
(RED: 0 events; GREEN: 1). Fidelity limit, pre-existing: replay applies
configPush through core_events.c's CONFIG_PUSH path, which doesn't re-run
groupmgr_load_configs()/plugin_config_update(), so a limits/groups change
still won't take effect in sim.

bug-1848: xce_sched_obj_t.hlimits was created with a NULL zarray killer
at all three constructors, so reset_sched_obj() and schedobj teardown
leaked the per-worker xlist_t match_job_host_limit_() callocs. New
kill_hlimits_list() (schedobj.c) frees the list, never its vres entries
(owned by the limit plugin); core_events.c and snapshot_load.c use
identical static local copies, since both avoid sharing helpers with
schedobj.c and several test binaries link them without it. Test:
test_limit_host_dedup.c's test_hlimits_zarray_releases_its_lists (RED:
killer NULL; GREEN).

bug-1849: every --sim run warned "failed to resolve job picker plugin ...
picker rebuild will be skipped". Benign: the snapshot is loaded before
the sim registers its picker, and sim_bootstrap_picker() replays the same
queue/job walk afterward. Downgraded to TRACE with an explanation.

Buglog also records bug-1850 (open): the ~100s reconnect churn behind
bug-1846 part 2 -- xcecli_retry_next() rotates/reconnects on every retry
(ETIMEDOUT after the 10s reply deadline, E_XCE_NOT_READY, redirects), so
a slow reply from a healthy primary forces a rejoin + full re-push.
Needs the live scheduler.log and a libxce spec.

Full scheduler suite 87/87 (asan). Replay of the captured file: same 184
mismatches, all attributable to the recording's pre-fix binary.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

src/scheduler/testsuite/test_config_push_deferred.c

@@ -23,6 +23,8 @@
  * driver applies it between rounds.
  */
 #include <errno.h>
+#include <stdbool.h>
+#include <stdio.h>
 #include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
@@ -200,9 +202,57 @@ static void test_config_push_mid_session_does_not_end_round(void)
     printf("test_config_push_mid_session_does_not_end_round: PASS\n");
 }
 
+/* The same deferral makes the push recordable: it is now applied at a
+ * round boundary on the driver thread, exactly where the recorder writes
+ * the round's other controller events. Before, it bypassed the pending-
+ * update queue entirely and never reached a .replay file, so a replay
+ * spanning a rejoin (or a genuine config change) never saw it. */
+static xce_scheduler_t g_rec_sched;
+
+static void test_applied_config_push_is_recorded(void)
+{
+    xce_queue_t*      q      = (xce_queue_t*)calloc(1, sizeof(*q));
+    xce_queue_t**     queues = (xce_queue_t**)calloc(1, sizeof(*queues));
+    xce_conf4sched_t* config = (xce_conf4sched_t*)calloc(1, sizeof(*config));
+    FILE*             f;
+    int               rc;
+
+    memset(&g_rec_sched, 0, sizeof(g_rec_sched));
+    rc = xce_sched_core_init(&(g_rec_sched.core));
+    TEST_ASSERT_EQUAL_INT(0, rc);
+
+    f = tmpfile();
+    TEST_ASSERT_NOT_NULL(f);
+    g_rec_sched.recording   = true;
+    g_rec_sched.record_file = f;
+
+    q->queue         = strdup(QUEUE_NAME);
+    queues[0]        = q;
+    config->n_queues = 1;
+    config->queues   = queues;
+
+    TEST_ASSERT_EQUAL_INT(0, sched_stash_config(&g_rec_sched, config));
+    TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, g_rec_sched.record_event_count,
+                                     "stashing alone must not record");
+
+    TEST_ASSERT_EQUAL_INT(1, sched_apply_pending_config(&g_rec_sched));
+    TEST_ASSERT_EQUAL_UINT64_MESSAGE(
+        1, g_rec_sched.record_event_count,
+        "an applied config push must be written to the recording as one "
+        "configPush event");
+    TEST_ASSERT_TRUE(ftell(f) > 0);
+
+    g_rec_sched.recording   = false;
+    g_rec_sched.record_file = NULL;
+    fclose(f);
+
+    printf("test_applied_config_push_is_recorded: PASS\n");
+}
+
 int main(void)
 {
     UNITY_BEGIN();
     RUN_TEST(test_config_push_mid_session_does_not_end_round);
+    RUN_TEST(test_applied_config_push_is_recorded);
     return UNITY_END();
 }

src/scheduler/testsuite/test_limit_host_dedup.c

@@ -252,9 +252,38 @@ static void test_repeated_host_attempts_stage_host_limit_once(void)
     printf("test_repeated_host_attempts_stage_host_limit_once: PASS\n");
 }
 
+/* reset_sched_obj() (limit_job_election()'s config-version rebuild) and
+ * schedobj teardown release obj->hlimits through zarray_fini(), which only
+ * frees items when the zarray has an item killer. The per-worker entries
+ * are xlist_t's calloc'd by match_job_host_limit_(), and the zarray used to
+ * be created with a NULL killer, so every rebuild/teardown leaked one list
+ * per worker the job had tried. (The vres entries inside each list are
+ * owned by the limit plugin's instance table and must not be freed.) A
+ * direct LeakSanitizer check isn't usable here -- the fixture's own resreq
+ * parsing leaks independently -- so this pins the construction contract. */
+static void test_hlimits_zarray_releases_its_lists(void)
+{
+    xce_sched_job_t* job = sched_get_job(&(g_sched.core), JOB_ID);
+
+    TEST_ASSERT_NOT_NULL_MESSAGE(job, "runs after the dedup test's fixture");
+    TEST_ASSERT_EQUAL_PTR_MESSAGE(
+        (void*)kill_hlimits_list, (void*)job->schedobj->hlimits.killer,
+        "obj->hlimits must be created with kill_hlimits_list so "
+        "reset_sched_obj()/teardown free the per-worker lists");
+
+    /* and a reset must leave it reusable with the same killer */
+    reset_sched_obj(job->schedobj);
+    TEST_ASSERT_EQUAL_PTR((void*)kill_hlimits_list,
+                          (void*)job->schedobj->hlimits.killer);
+    TEST_ASSERT_NULL(zarray_get(&(job->schedobj->hlimits), 0));
+
+    printf("test_hlimits_zarray_releases_its_lists: PASS\n");
+}
+
 int main(void)
 {
     UNITY_BEGIN();
     RUN_TEST(test_repeated_host_attempts_stage_host_limit_once);
+    RUN_TEST(test_hlimits_zarray_releases_its_lists);
     return UNITY_END();
 }