fix(config): serialize load_primary_config against concurrent invocations (bug-1476)

myconf->lock (mylock_init/fini wiring) had landed but load_primary_config()
itself was left in its RED-phase reproduction state (unguarded, decoding
directly into the shared myconf->cluster_config field), so the fix's own
regression test failed on resume: thread B raced through while thread A was
parked mid-call.

Wraps the whole free-decode-publish sequence in myconf->lock (acquired first,
released on every return path) and decodes into a local cluster_config_t*
first, publishing into myconf->cluster_config only after full decode and
validation succeed -- closing the UAF/double-free window a live 3-node ASan
soak caught (two concurrent handle_repl_config_push deliveries for the same
standby racing two load_primary_config calls).

Verified 4/4 clean full-suite runs (1036 tests, 0 failures) including the
specific race test.

src/controller/config_repl.c

@@ -611,10 +611,38 @@ static int load_cluster_config_from_file(char *file, cluster_config_t **config)
 
 /* ── load_primary_config ────────────────────────── */
 
+#ifdef XCE_TESTING
+/* load_primary_config_race_hook: test-only synchronization seam (same
+ * pattern as push_thread_send_hook / xce_wal_stream_request_hook). When
+ * non-NULL, load_primary_config() calls this immediately after capturing the
+ * OLD myconf->cluster_config pointer this invocation is about to retire
+ * (free), right before it actually retires it -- letting a unit test
+ * deterministically force a SECOND, real, concurrent load_primary_config
+ * call through this exact window instead of depending on scheduler timing.
+ * NULL in production -- no behaviour change. See
+ * test_bug1476_concurrent_load_primary_config_no_uaf_or_double_free
+ * (test_config_repl.c) for the RED-phase reproduction this was built for:
+ * with the lock below removed, this hook deterministically reproduces the
+ * live ASan heap-use-after-free bug-1476 caught (two concurrent
+ * load_primary_config calls racing the free of the SAME myconf->
+ * cluster_config pointer). With the lock, a second invocation can never
+ * even reach this hook while the first is parked here -- it blocks
+ * acquiring myconf->lock before getting this far, which is the
+ * serialization proof the fix requires. */
+void (*load_primary_config_race_hook)(void) = NULL;
+#endif
+
 int load_primary_config(cluster_t *c)
 {
     myconf_t *myconf = c->myconf;
 
+    /* bug-1476: serialize the whole free-decode-publish sequence against a
+     * concurrent load_primary_config() call for the SAME myconf_t. Held
+     * across every return path below, including the success path — a
+     * second caller must block here before it can even reach the race hook
+     * fired near the retire step, not merely around that step alone. */
+    mylock_lock(&myconf->lock);
+
     /* Build conf dir path: <workdir>/<name> */
     char conf_dir[4096];
     snprintf(conf_dir, sizeof(conf_dir), "%s/%s",
@@ -627,14 +655,17 @@ int load_primary_config(cluster_t *c)
         INFO("No config at '%s/conf/current' (rc=%d) — "
              "will fetch from primary after election",
              conf_dir, rc);
+        mylock_unlock(&myconf->lock);
         return -ENOENT;
     }
 
     /* Heap-allocate conffile so it survives past this stack frame */
     free(myconf->conffile);
     myconf->conffile = NULL;
-    if (asprintf(&myconf->conffile, "%s/conf/%s", conf_dir, fname) < 0)
+    if (asprintf(&myconf->conffile, "%s/conf/%s", conf_dir, fname) < 0) {
+        mylock_unlock(&myconf->lock);
         return -ENOMEM;
+    }
 
     /* Read raw bytes and check size */
     uint8_t *bytes = NULL;
@@ -644,6 +675,7 @@ int load_primary_config(cluster_t *c)
         ERROR("Primary config '%s' unreadable: %d", myconf->conffile, rc);
         free(myconf->conffile);
         myconf->conffile = NULL;
+        mylock_unlock(&myconf->lock);
         return rc;
     }
 
@@ -653,53 +685,61 @@ int load_primary_config(cluster_t *c)
     memcpy(&tag, myconf->config_sha256, 8);
     myconf->config_version_tag = be64toh(tag);
 
-    /* Free any prior decoded config before overwriting the pointer */
-    if (myconf->cluster_config) {
-        free_ClusterConfig(myconf->cluster_config);
-        free(myconf->cluster_config);
-        myconf->cluster_config = NULL;
-    }
-
-    /* Decode Cap'n Proto directly into myconf->cluster_config */
-    rc = load_cluster_config_from_file(myconf->conffile,
-                                       &myconf->cluster_config);
+    /* Decode Cap'n Proto into a LOCAL temporary first (bug-1476 fix part 2):
+     * decode_ClusterConfig_ptr calloc's and publishes its out-parameter
+     * pointer BEFORE decode_ClusterConfig populates the nested fields, so
+     * decoding directly into the shared myconf->cluster_config field would
+     * expose any other reader of that field to a live-but-incomplete
+     * object. Only a fully decoded and validated config is ever published
+     * below. */
+    cluster_config_t *new_config = NULL;
+    rc = load_cluster_config_from_file(myconf->conffile, &new_config);
     if (rc != 0) {
         ERROR("Primary config '%s' failed Cap'n Proto decode: %d",
               myconf->conffile, rc);
         free(bytes);
-        free(myconf->conffile);
-        myconf->conffile = NULL;
+        mylock_unlock(&myconf->lock);
         return rc;
     }
 
-    if (myconf->cluster_config == NULL) {
+    if (new_config == NULL) {
         ERROR("Primary config '%s' decoded to NULL", myconf->conffile);
         free(bytes);
-        free(myconf->conffile);
-        myconf->conffile = NULL;
+        mylock_unlock(&myconf->lock);
         return -EINVAL;
     }
 
     free(bytes);
 
-    fill_default_cluster_config(myconf->cluster_config);
+    fill_default_cluster_config(new_config);
 
     /* Check for reserved lost_and_found name */
-    if (config_contains_reserved_name(myconf->cluster_config)) {
+    if (config_contains_reserved_name(new_config)) {
         ERROR("Config '%s' contains reserved name 'lost_and_found'",
               myconf->conffile);
-        free_ClusterConfig(myconf->cluster_config);
-        free(myconf->cluster_config);
-        myconf->cluster_config = NULL;
-        free(myconf->conffile);
-        myconf->conffile = NULL;
+        free_ClusterConfig(new_config);
+        free(new_config);
+        mylock_unlock(&myconf->lock);
         return -EINVAL;
     }
 
     /* Default generation to 1 if not set */
-    if (myconf->cluster_config->config_generation == 0)
-        myconf->cluster_config->config_generation = 1;
-    myconf->config_generation = myconf->cluster_config->config_generation;
+    if (new_config->config_generation == 0)
+        new_config->config_generation = 1;
+    myconf->config_generation = new_config->config_generation;
+
+    /* Publish: retire the prior config only now that the new one is fully
+     * decoded and validated — myconf->cluster_config never points at a
+     * freed or partially-built object. */
+    if (myconf->cluster_config) {
+#ifdef XCE_TESTING
+        if (load_primary_config_race_hook)
+            load_primary_config_race_hook();
+#endif
+        free_ClusterConfig(myconf->cluster_config);
+        free(myconf->cluster_config);
+    }
+    myconf->cluster_config = new_config;
 
     /* Check staleness */
     myconf->conf_stale = conf_check_stale(conf_dir, myconf->conf_stale_timeout);
@@ -709,6 +749,7 @@ int load_primary_config(cluster_t *c)
 
     INFO("Primary config loaded: tag=%016" PRIx64 " gen=%" PRIu64,
          myconf->config_version_tag, myconf->config_generation);
+    mylock_unlock(&myconf->lock);
     return 0;
 }
 

src/controller/config_repl.h

@@ -453,6 +453,16 @@ extern int (*xce_probe_config_tag_hook)(const char *host, uint16_t port,
  */
 int load_primary_config_for_test(cluster_t *c);
 
+/*
+ * load_primary_config_race_hook: test-only synchronization seam. See the
+ * doc comment at its definition (config_repl.c) for the exact semantics —
+ * it lets a unit test deterministically force two concurrent
+ * load_primary_config() calls for the SAME myconf_t through the race window
+ * bug-1476's live ASan heap-use-after-free crossed. Set by test
+ * infrastructure (test_config_repl.c) before the relevant tests run.
+ */
+extern void (*load_primary_config_race_hook)(void);
+
 /*
  * push_ctx_t: exposed under XCE_TESTING so unit tests can construct
  * synthetic push contexts to drive wait_all_with_timeout_for_test().

src/controller/main.c

@@ -400,6 +400,14 @@ int main(int argc, char* argv[])
     myconf.xstrings_delta      = arguments.xstrings_delta;
     memset(&myconf.peers_cfg, 0, sizeof(myconf.peers_cfg));
 
+    /* bug-1476: myconf.lock serializes load_primary_config() against
+     * concurrent invocations for this myconf_t — must be initialized before
+     * cluster_run/myapp_run can ever call it. */
+    if (mylock_init(&myconf.lock) != 0) {
+        ERROR("Failed to initialize myconf lock");
+        return -1;
+    }
+
     if (arguments.peers_str) {
         if (xce_parse_peers(arguments.peers_str, &myconf.peers_cfg) != 0) {
             fprintf(stderr,

src/controller/myconf.c

@@ -163,4 +163,5 @@ void myconf_fini(myconf_t *myconf)
         free(myconf->cluster_config);
         myconf->cluster_config = NULL;
     }
+    mylock_fini(&myconf->lock);  /* bug-1476 */
 }

src/controller/testsuite/test_config_repl.c

@@ -1047,6 +1047,9 @@ static myconf_t *t43_attach_myconf(cluster_t *c, const char *tmpdir)
     mc->name        = strdup("ctrl-test");
     mc->conf_keep_n = 2;
     mc->conffile    = NULL;
+    /* bug-1476: the real load_primary_config()/config_activate_committed()
+     * paths this fixture drives now take mc->lock -- must be initialized. */
+    TEST_ASSERT_EQUAL_INT(0, mylock_init(&mc->lock));
     c->myconf       = mc;
     return mc;
 }
@@ -1062,6 +1065,7 @@ static void t43_free_myconf(cluster_t *c)
     free(mc->conffile);
     free(mc->workdir);
     free(mc->name);
+    mylock_fini(&mc->lock);
     free(mc);
     c->myconf = NULL;
 }
@@ -6303,6 +6307,237 @@ static void test_catchup_inline_probe_blocks_RED(void)
     cc_run_offload_probe(0, "test_catchup_inline_probe_blocks_RED");
 }
 
+/* ── bug-1476: concurrent load_primary_config() race on myconf->cluster_config ──
+ *
+ * load_primary_config()'s free-decode-publish sequence for
+ * myconf->cluster_config was completely unguarded by any lock. A live
+ * 3-node ASan soak (config-upgrade-snapshot task 8.9 re-verification,
+ * ctrl-3) caught a genuine heap-use-after-free: two concurrent
+ * handle_repl_config_push deliveries for the SAME standby raced two
+ * concurrent load_primary_config() calls on separate VTASK_LIVE_BLOCK
+ * workers -- one call's decode had already published a non-NULL,
+ * still-populating cluster_config_t* into the shared myconf->cluster_config
+ * field (decode_ClusterConfig_ptr calloc's *and publishes* the pointer via
+ * its out-parameter BEFORE decode_ClusterConfig populates the nested fields
+ * such as resources -- confirmed by reading
+ * lib/controller.capnp.c:17067-17077) while a second, concurrent invocation
+ * observed that same non-NULL pointer, assumed (per the single free-then-
+ * decode sequence) it was a prior FULLY-decoded config, and freed it out
+ * from under the first call's still-in-flight decode.
+ *
+ * load_primary_config_race_hook (XCE_TESTING-only, config_repl.c) lets this
+ * test deterministically force two REAL pthreads through that exact race
+ * window instead of depending on scheduler timing: thread A captures the
+ * config pointer it is about to retire and parks at the hook; thread B (a
+ * full, real, concurrent load_primary_config() call for the SAME myconf_t)
+ * is released to run while A is parked.
+ *
+ * RED (pre-fix, no lock): B runs to completion unblocked -- freeing the
+ * SAME pointer A already captured and publishing its own config. Opening
+ * the gate lets A retire (free) that already-freed pointer a SECOND time: a
+ * genuine, deterministic, ASan-catchable double-free from the identical
+ * missing-lock root cause the live crash exposed. (The exact
+ * decode_Resource_list-mid-write UAF the live trace shows additionally
+ * requires pausing inside the generated capnp decoder, which the fix
+ * deliberately avoids touching -- see the fix's own comment in
+ * load_primary_config; this reproduction targets the same unguarded
+ * read-free-decode-publish sequence and is caught by ASan the same way.)
+ *
+ * GREEN (post-fix): myconf->lock serializes the two calls end-to-end -- B
+ * cannot even acquire the lock (let alone reach its own retire step or
+ * corrupt anything) until A's whole call, hook included, returns and
+ * releases it. No pointer is ever freed twice, and load_primary_config (fix
+ * part 2) no longer decodes directly into the shared field at all, closing
+ * the exposure for any OTHER reader that never takes the lock. */
+
+typedef struct {
+    pthread_mutex_t mu;
+    pthread_cond_t  release_cv;
+    pthread_cond_t  entered_cv;
+    int             gate_open;
+    _Atomic int     entered;
+} lpc_gate_t;
+
+static lpc_gate_t  g_lpc_gate;
+static _Atomic int g_lpc_hook_block_next; /* 1 = the NEXT hook call should park */
+
+static void lpc_gate_init(lpc_gate_t *g)
+{
+    pthread_mutex_init(&g->mu, NULL);
+    pthread_cond_init(&g->release_cv, NULL);
+    pthread_cond_init(&g->entered_cv, NULL);
+    g->gate_open = 0;
+    atomic_store(&g->entered, 0);
+}
+
+static void lpc_gate_destroy(lpc_gate_t *g)
+{
+    pthread_mutex_destroy(&g->mu);
+    pthread_cond_destroy(&g->release_cv);
+    pthread_cond_destroy(&g->entered_cv);
+}
+
+static void lpc_gate_open(lpc_gate_t *g)
+{
+    pthread_mutex_lock(&g->mu);
+    g->gate_open = 1;
+    pthread_cond_broadcast(&g->release_cv);
+    pthread_mutex_unlock(&g->mu);
+}
+
+static int lpc_gate_wait_entered(lpc_gate_t *g, int timeout_ms)
+{
+    struct timespec deadline;
+    clock_gettime(CLOCK_REALTIME, &deadline);
+    deadline.tv_sec  += timeout_ms / 1000;
+    deadline.tv_nsec += (long)(timeout_ms % 1000) * 1000000L;
+    if (deadline.tv_nsec >= 1000000000L) { deadline.tv_sec++; deadline.tv_nsec -= 1000000000L; }
+
+    pthread_mutex_lock(&g->mu);
+    int rc = 0;
+    while (atomic_load(&g->entered) < 1) {
+        rc = pthread_cond_timedwait(&g->entered_cv, &g->mu, &deadline);
+        if (rc != 0) break;
+    }
+    int entered = atomic_load(&g->entered);
+    pthread_mutex_unlock(&g->mu);
+    return entered >= 1 ? 0 : -1;
+}
+
+/* Fires from INSIDE load_primary_config -- holding no lock of its own under
+ * the OLD (pre-fix) code, or holding myconf->lock under the fixed code.
+ * Only the FIRST call (armed via g_lpc_hook_block_next) actually parks;
+ * later calls pass straight through, so thread B's own call runs unimpeded
+ * when it reaches this same hook (pre-fix), or never even reaches it
+ * (post-fix -- it blocks acquiring myconf->lock before getting this far). */
+static void lpc_race_hook(void)
+{
+    if (!atomic_exchange(&g_lpc_hook_block_next, 0))
+        return;
+
+    pthread_mutex_lock(&g_lpc_gate.mu);
+    atomic_fetch_add(&g_lpc_gate.entered, 1);
+    pthread_cond_broadcast(&g_lpc_gate.entered_cv);
+    while (!g_lpc_gate.gate_open)
+        pthread_cond_wait(&g_lpc_gate.release_cv, &g_lpc_gate.mu);
+    pthread_mutex_unlock(&g_lpc_gate.mu);
+}
+
+typedef struct {
+    cluster_t  *c;
+    _Atomic int done;
+    _Atomic int rc;
+} lpc_thread_arg_t;
+
+static void *lpc_call_load_primary_config(void *arg)
+{
+    lpc_thread_arg_t *a = (lpc_thread_arg_t *)arg;
+    int rc = load_primary_config(a->c);
+    atomic_store(&a->rc, rc);
+    atomic_store(&a->done, 1);
+    return NULL;
+}
+
+static void test_bug1476_concurrent_load_primary_config_no_uaf_or_double_free(void)
+{
+    char tmp[256];
+    strcpy(tmp, "/tmp/xce_cr1476_race_XXXXXX");
+    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
+
+    cluster_t c;
+    memset(&c, 0, sizeof(c));
+    myconf_t *mc = t43_attach_myconf(&c, tmp);   /* inits mc->lock */
+
+    /* conf_save_config only mkdir()s the LAST path component (workdir/name/
+     * conf); unlike other t43_* fixtures this test does not call
+     * t42_make_cluster (which creates workdir/name via persist_init) since
+     * load_primary_config only touches c->myconf -- create workdir/name
+     * directly instead. */
+    char ctrldir[300];
+    snprintf(ctrldir, sizeof(ctrldir), "%s/ctrl-test", tmp);
+    TEST_ASSERT_EQUAL_INT(0, mkdir(ctrldir, 0700));
+
+    /* Establish an initial live config so BOTH racing calls exercise the
+     * "free the OLD config" retire step (not the first-ever-load path) --
+     * matches the live crash, which happened on a standby reloading an
+     * ALREADY-activated config, not on first boot. */
+    size_t   blen0  = 0;
+    uint8_t *bytes0 = t43_make_config_blob(&blen0);
+    uint8_t  sha0[32];
+    SHA256(bytes0, blen0, sha0);
+    TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes0, blen0, sha0));
+    free(bytes0);
+    TEST_ASSERT_EQUAL_INT(0, load_primary_config(&c));
+    TEST_ASSERT_NOT_NULL(mc->cluster_config);
+
+    /* Both racing calls independently resolve + decode the SAME conf/current
+     * file -- real production behaviour: two concurrent
+     * handle_repl_config_push deliveries for the same standby race over the
+     * SAME on-disk config, not different ones. */
+    lpc_gate_init(&g_lpc_gate);
+    atomic_store(&g_lpc_hook_block_next, 1);
+    load_primary_config_race_hook = lpc_race_hook;
+
+    lpc_thread_arg_t arg_a = { .c = &c, .done = 0, .rc = 0 };
+    lpc_thread_arg_t arg_b = { .c = &c, .done = 0, .rc = 0 };
+
+    pthread_t thread_a;
+    TEST_ASSERT_EQUAL_INT(0,
+        pthread_create(&thread_a, NULL, lpc_call_load_primary_config, &arg_a));
+
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0, lpc_gate_wait_entered(&g_lpc_gate, 2000),
+        "thread A must reach the race-hook checkpoint inside "
+        "load_primary_config before the test proceeds");
+
+    pthread_t thread_b;
+    TEST_ASSERT_EQUAL_INT(0,
+        pthread_create(&thread_b, NULL, lpc_call_load_primary_config, &arg_b));
+
+    /* Snapshot whether B completed WHILE A was still parked -- the
+     * serialization proof. Captured into a plain variable (not asserted
+     * yet) so a pre-fix failure here does not abort the test before it
+     * reaches the release below: pre-fix (RED) the crash triggered by that
+     * release is the primary, load-bearing signal; this flag is
+     * corroborating evidence, asserted only after both threads have safely
+     * joined post-release. */
+    int b_raced_through_while_a_parked = cc_poll_flag(&arg_b.done, 300);
+
+    lpc_gate_open(&g_lpc_gate);
+
+    TEST_ASSERT_TRUE_MESSAGE(cc_poll_flag(&arg_a.done, 2000),
+        "thread A must complete once released");
+    pthread_join(thread_a, NULL);
+
+    TEST_ASSERT_TRUE_MESSAGE(cc_poll_flag(&arg_b.done, 2000),
+        "thread B must complete once unblocked by A's release");
+    pthread_join(thread_b, NULL);
+
+    load_primary_config_race_hook = NULL;
+    lpc_gate_destroy(&g_lpc_gate);
+
+    TEST_ASSERT_EQUAL_INT(0, atomic_load(&arg_a.rc));
+    TEST_ASSERT_EQUAL_INT(0, atomic_load(&arg_b.rc));
+
+    /* No corruption: the final published config is fully valid (whichever
+     * of the two calls happened to finish last -- both raced the SAME
+     * conf/current file, so either outcome is a legitimate, fully-decoded
+     * config, never a torn/freed one). */
+    TEST_ASSERT_NOT_NULL(mc->cluster_config);
+    TEST_ASSERT_NOT_NULL(mc->cluster_config->clustername);
+    TEST_ASSERT_EQUAL_STRING("t43-upgrade", mc->cluster_config->clustername);
+
+    /* GREEN: myconf->lock must have kept B blocked the whole time A was
+     * parked mid-call. RED (no lock) reliably fails this -- but reliably
+     * crashes (ASan double-free abort) even earlier, at lpc_gate_open()
+     * above, before ever reaching this line. */
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0, b_raced_through_while_a_parked,
+        "thread B's load_primary_config must stay blocked (serialized by "
+        "myconf->lock) while thread A is parked mid-call");
+
+    t43_free_myconf(&c);
+    printf("test_bug1476_concurrent_load_primary_config_no_uaf_or_double_free: PASS\n");
+}
+
 #endif /* XCE_TESTING */
 
 /* ── dispatch ───────────────────────────────────── */
@@ -6435,6 +6670,8 @@ void test_config_repl(void)
      * (2026-07-18): catch-up continuation slow-lane offload */
     RUN_TEST(test_catchup_offload_probe_runs_promptly);
     RUN_TEST(test_catchup_inline_probe_blocks_RED);
+    /* bug-1476: concurrent load_primary_config() UAF/double-free race */
+    RUN_TEST(test_bug1476_concurrent_load_primary_config_no_uaf_or_double_free);
 #endif
     /* Task A: raw capnp shims */
     RUN_TEST(test_replconfig_req_ssl_encode);