Merge branch 'working' into job_peek_lchen

job_peek_lchen: Add fetchfile feature (job output/error peek proxy)
working: HA/Raft infrastructure (consensus, replication, config mgmt)

Conflict resolution summary:
- controller.capnp: fetchfile @41, fetchfilereply @18, fetchfilemsg @30
- cluster.c: bifurcated OPCode_reply for fetchfile + config-push ACK
- job.c: kept full fetchfile pending-request subsystem (~183 lines)
- request.c: kept both xce_fetchfile_req() and XCE_RETRY_DEFAULTS
- helper.c: added missing preVoteRequest/preVoteReply mappings
- All 15 new test registrations (test_cbuf.c + 14 HA tests)

src/lib/xcectx.c

@@ -14,6 +14,14 @@ static int handle_get_reply(xce_ctx_t* ctx, xce_reply_t* reply)
     xce_data_t* getdata = reply->data.getdata;
     dataset_t*  dataset = ctx->reply.u.get;
 
+    /* No get is in flight (slot cleared by the consumer on every exit path):
+     * drop a late/duplicate reply rather than dereference a NULL or stale
+     * destination — bug-1077, the shared reply union must never be used as a
+     * stale typed pointer. */
+    if (dataset == NULL) {
+        return -1;
+    }
+
     return extract_data(dataset, getdata);
 }
 
@@ -27,10 +35,39 @@ static int handle_sub_reply(xce_ctx_t* ctx, xce_reply_t* reply)
     return 1;
 }
 
-static int handle_reply(xce_ctx_t* ctx, xce_reply_t* reply)
+void xce_ctx_capture_redirect_hint(xce_ctx_t* ctx, const char* host,
+                                   uint16_t port)
+{
+    if (ctx == NULL) {
+        return;
+    }
+
+    /* No usable redirect address → leave the hint untouched (empty). The
+     * capnp-owned host pointer may be NULL; an empty host or a zero port is
+     * likewise "no address". Advisory only — correctness never depends on it. */
+    if ((host == NULL) || (host[0] == '\0') || (port == 0)) {
+        return;
+    }
+
+    strncpy(ctx->redirect_hint_host, host, XCE_MAXHOSTNAME_LEN - 1);
+    ctx->redirect_hint_host[XCE_MAXHOSTNAME_LEN - 1] = '\0';
+    ctx->redirect_hint_port                          = port;
+}
+
+/* Exposed (non-static) so the libxce test suite can drive the reply path
+ * directly without a live endpoint (see test_handle_reply_* regression for the
+ * shared-reply-union type-confusion crash, bug-1077). */
+int xce_handle_reply(xce_ctx_t* ctx, xce_reply_t* reply)
 {
     int rc = 0;
 
+    /* Capture the redirect address (if any) as a best-effort failover hint
+     * BEFORE the capnp-owned reply — including reply->redirect_host — is freed
+     * by the caller (Req 2.2, 8.3). The status still flows through the existing
+     * cbuf_tryput(&ctx->wbuf,&status) path unchanged, so xceapi.c is untouched. */
+    xce_ctx_capture_redirect_hint(ctx, reply->redirect_host,
+                                  reply->redirect_port);
+
     if (reply->status < 0) {
         return reply->status;
     }
@@ -45,10 +82,27 @@ static int handle_reply(xce_ctx_t* ctx, xce_reply_t* reply)
     case OPCode_clusterconf:
         rc = -1;
         if (reply->data_which == RequestReply_data_clusterconf) {
+            /* MOVE_PTR overwrites the slot WITHOUT dereferencing the old value.
+             * Do NOT "free the prior occupant": ctx->reply.u is a single
+             * untagged union shared by every opcode, so a non-NULL slot does NOT
+             * imply a ClusterConf* — it may hold a dataset_t* (a dead stack
+             * `getreply` left by a failover-aborted get) or a jobid. Freeing it
+             * as a ClusterConf walked a foreign pointer and crashed (bug-1077).
+             * The consumer owns slot lifecycle and clears it on every path. */
             MOVE_PTR(ctx->reply.u.clusterconf, reply->data.clusterconf);
             rc = 1;
         }
         break;
+    case OPCode_ctrlstatus:
+        rc = -1;
+        if (reply->data_which == RequestReply_data_ctrlstatus) {
+            /* See the clusterconf note above: never free the shared union slot
+             * here as a CtrlStatus* — a foreign occupant would be type-confused
+             * and dereferenced (bug-1077, ASan stack-use-after-return). */
+            MOVE_PTR(ctx->reply.u.ctrlstatus, reply->data.ctrlstatus);
+            rc = 1;
+        }
+        break;
     case OPCode_fetchfile:
         rc = -1;
         if (reply->data_which == RequestReply_data_fetchfilereply) {
@@ -90,7 +144,7 @@ static int vtask_entry(vtask_t* task)
         int need_wake = 1;
         int status    = request.body.replymsg->status;
 
-        need_wake = handle_reply(ctx, request.body.replymsg);
+        need_wake = xce_handle_reply(ctx, request.body.replymsg);
         if (need_wake) {
             rc = xce_wakeup(ctx, (int)status);
         }
@@ -140,6 +194,29 @@ static int handle_msg(endpoint_t* ep)
                             (void*)ep);
 }
 
+/* Called by netio when the TCP connection drops (EPOLLHUP / EPOLLERR / read EOF).
+ * Unblocks any thread stuck in cbuf_get (e.g. xce_hello waiting for a reply)
+ * so that checking_connection can detect the failure and attempt to rejoin. */
+static int handle_disconnect(endpoint_t* ep)
+{
+    if (ep == NULL)
+        return 0;
+
+    xce_ctx_t* ctx = (xce_ctx_t*)ep->owner;
+    if (ctx == NULL)
+        return 0;
+
+    ctx->state = XCE_CTX_ERR;
+
+    /* Inject an error into wbuf so any blocked cbuf_get returns immediately.
+     * cbuf_tryput returns -EAGAIN if the buffer already has an item — that is
+     * fine: the existing item will wake the waiter. */
+    int err = -ECONNRESET;
+    cbuf_tryput(&ctx->wbuf, &err);
+
+    return 0;
+}
+
 int xce_ctx_init(xce_ctx_t* ctx, char* name, char* key)
 {
     int      rc;
@@ -244,6 +321,7 @@ int xce_ctx_start(xce_ctx_t* ctx)
 
     endpoint_set_owner(ctx->ep, ctx);
     endpoint_set_msg_handler(ctx->ep, (msg_handler_t)handle_msg);
+    endpoint_set_err_handler(ctx->ep, (err_handler_t)handle_disconnect);
     rc = iomgr_monitor(ctx->ep);
     if (rc < 0) {
         ctx->state = XCE_CTX_ERR;
@@ -258,6 +336,65 @@ int xce_ctx_start(xce_ctx_t* ctx)
     return 0;
 }
 
+/*
+ * xce_ctx_reconnect (libxce-failover) — re-establish ctx->ep on a fresh
+ * connection to host:port after the previous primary's connection dropped.
+ *
+ * This mirrors the connection-establishment part of xcecli_start/xce_ctx_start
+ * (the parts that DROP on endpoint_fini): tear down the dead fd/SSL, recreate
+ * the socket, re-bind the iomgr (endpoint_connect needs ep->iomgr->ssl_ctx),
+ * drive the SSL handshake, then re-arm the endpoint in the ALREADY-RUNNING
+ * iomgr with its owner + msg/err handlers and re-monitor the new fd. It does
+ * NOT restart iomgr/vtaskmgr (already running) and does NOT touch ctx->state
+ * (the failover loop's xcecli_ctx_recover owns the ERR→RUN reset).
+ *
+ * Without the iomgr re-bind, endpoint_connect dereferenced a NULL ep->iomgr
+ * (crash at netio.c endpoint_connect); without re-arming the handlers + monitor
+ * the reconnected endpoint would never route replies to ctx->wbuf.
+ */
+int xce_ctx_reconnect(xce_ctx_t* ctx, char* host, int port)
+{
+    int rc;
+
+    if ((ctx == NULL) || (ctx->ep == NULL) || (host == NULL)) {
+        return -EINVAL;
+    }
+
+    /* Tear down the dead connection (closes fd, frees SSL, clears handlers and
+     * resets the endpoint struct — including ep->iomgr). */
+    endpoint_fini(ctx->ep);
+
+    rc = endpoint_init(ctx->ep, AF_INET, SOCK_STREAM, 0, 0);
+    if (rc < 0) {
+        return rc;
+    }
+
+    endpoint_set_peer(ctx->ep, &(ctx->myinfo));
+
+    /* Re-bind the iomgr so endpoint_connect can reach ctx->iomgr->ssl_ctx
+     * (xce_ctx_set_endpoint does this on the initial start). */
+    ctx->ep->iomgr = &(ctx->iomgr);
+
+    rc = endpoint_connect(ctx->ep, host, port);
+    if (rc < 0) {
+        return rc;
+    }
+
+    /* Re-arm the endpoint in the running iomgr: owner + reply/error handlers,
+     * then monitor the new fd. handle_disconnect re-registers RDHUP-driven
+     * failover for the new primary. */
+    endpoint_set_owner(ctx->ep, ctx);
+    endpoint_set_msg_handler(ctx->ep, (msg_handler_t)handle_msg);
+    endpoint_set_err_handler(ctx->ep, (err_handler_t)handle_disconnect);
+
+    rc = iomgr_monitor(ctx->ep);
+    if (rc < 0) {
+        return rc;
+    }
+
+    return 0;
+}
+
 int xce_ctx_stop(xce_ctx_t* ctx)
 {
     if (ctx == NULL) {