feat(ha-convergence-recovery): self-restart primitive + demotion convergence (task 2.2)

src/controller/ha_convergence.c

@@ -3,34 +3,52 @@
  * (ha-convergence-recovery Requirement 1).
  *
  * ───────────────────────────────────────────────────────────────────────────
- * RED SCAFFOLD (task 2.1) — task 2.2 REPLACES the bodies below with the real
- * implementation (and moves controller_saved_argv's backing store into main.c).
+ * This TU owns the self-restart primitive and the deferred demotion-convergence
+ * task body.  The synchronous demotion BRANCH (hook disable + task scheduling)
+ * lives in raft_cb_become_follower (ha.c) because that callback is file-static;
+ * it calls ha_schedule_demotion_convergence() declared here.
  *
- * This file currently provides ONLY the link-time definitions the task-2.1 RED
- * tests need to compile and link:
- *   - controller_saved_argv()  — stub returning NULL (real argv capture is in
- *                                main.c, task 2.2)
- *   - ha_self_restart()        — RED stub: does NOT yet honor the ha_restart_hook
- *                                / ha_fatal_hook contract, so the fatal-hook test
- *                                FAILS until task 2.2 implements the real
- *                                flush → execv → abort sequence (and the
- *                                XCE_TESTING seam routing).
- *   - ha_restart_hook / ha_fatal_hook — the XCE_TESTING function-pointer seams,
- *                                defined =NULL in exactly one TU (this one),
- *                                mirroring the ha_*_hook seam pattern in ha.c.
+ * Split of responsibilities (design "Demotion Convergence + Self-Restart"):
+ *   - raft_cb_become_follower (ha.c):  read prior role → if was-PRIMARY, disable
+ *                                      the 3 persistence hooks INLINE (Req 1.1),
+ *                                      then schedule the one-shot convergence
+ *                                      task off the callback thread (D1).
+ *   - ha_demotion_convergence_fn (here, delaymgr task body): truncate above
+ *                                      committed_lsn (Req 1.2) → WARN range
+ *                                      (Req 1.5) → ha_self_restart (Req 1.3).
+ *   - ha_self_restart (here):          flush logs → execv(saved argv); exec
+ *                                      return → ERROR + abort (FATAL, Req 1.4);
+ *                                      under XCE_TESTING route to the restart /
+ *                                      fatal seams (Req 1.6, 1.4).
  *
- * The DEMOTION-SEQUENCE behavior (hook disable on become-follower-from-PRIMARY,
- * scheduled truncate-above-committed_lsn, restart) lives in raft_cb_become_
- * follower (ha.c) and is NOT implemented here or there yet — that is task 2.2.
- * Consequently every demotion-convergence test in test_demotion_convergence.c
- * is RED against the current tree.
+ * controller_saved_argv lives in main.c (the real argv capture); this TU only
+ * consumes it through the ha.h accessor.
+ *
+ * fd hygiene (Req 1.3 / design fd audit): every controller-to-controller socket
+ * — listener, accepted peer/replication connections, and outbound vote/peer
+ * connect sockets — is created FD_CLOEXEC in src/lib/netio.c (socket_create
+ * sets it at netio.c:219; socket_accept at netio.c:308).  The re-exec'd image
+ * therefore inherits no live sockets and re-creates all endpoints through the
+ * boot path; no extra close-before-exec sweep is required here.
  * ───────────────────────────────────────────────────────────────────────────
  */
 
+#define _GNU_SOURCE
+#include <errno.h>
+#include <inttypes.h>
+#include <pthread.h>
+#include <stdatomic.h>
 #include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <tlog.h>
 
 #include "cluster.h"
+#include "delay.h"
 #include "ha.h"
+#include "persist.h"
 #include "xce/cluster_def.h"
 
 #ifdef XCE_TESTING
@@ -41,30 +59,154 @@ void (*ha_fatal_hook)(const char *msg)  = NULL;
 #endif
 
 /*
- * controller_saved_argv — RED stub (task 2.2 moves the real, main()-populated
- * accessor into main.c).  Returns NULL: no argv has been captured in the
- * scaffold, so a production execv would have nothing to exec.
- */
-char **controller_saved_argv(void)
-{
-    return NULL;
-}
-
-/*
- * ha_self_restart — RED stub.
+ * ha_self_restart — the truncate-and-restart primitive's restart half.
  *
- * Task 2.2 replaces this with the real primitive:
- *   - production: flush logs, execv(controller_saved_argv()); on exec return,
- *     ERROR + abort() (FATAL — Req 1.4); never returns.
- *   - XCE_TESTING: invoke ha_restart_hook(c) instead of execv (Req 1.6); if it
- *     reports failure (or is unset), invoke ha_fatal_hook in place of abort.
+ * Production (Req 1.3, 1.4): flush logs, then execv() the saved argv so the
+ * controller rebuilds in-memory state purely from the surviving (committed)
+ * WAL prefix via boot recovery.  If exec cannot be performed, terminate
+ * FATALLY (abort) — a demoted ex-primary must NEVER continue running with
+ * potentially divergent state.  NEVER returns in production.
  *
- * Until then this stub deliberately does NOTHING (no hook routing), so the
- * task-2.1 fatal-hook test — which asserts a failing restart hook drives
- * ha_fatal_hook — FAILS (RED).
+ * Test (Req 1.6): under XCE_TESTING invoke ha_restart_hook(c) instead of
+ * execv and return its result on the success path; a non-zero hook return
+ * models an exec failure and routes to ha_fatal_hook in place of abort, so a
+ * test binary can assert the FATAL path was taken without dying (Req 1.4).
  */
 int ha_self_restart(cluster_t *c)
 {
+#ifdef XCE_TESTING
+    /* Test contract: the restart seam stands in for execv. */
+    int rc = 0;
+    if (ha_restart_hook != NULL)
+        rc = ha_restart_hook(c);
+
+    if (rc != 0 || ha_restart_hook == NULL) {
+        /* Restart could not be performed → FATAL contract (Req 1.4). */
+        if (ha_fatal_hook != NULL)
+            ha_fatal_hook("ha_self_restart: restart seam failed (FATAL)");
+        return rc != 0 ? rc : -1;
+    }
+    return 0;
+#else
     (void)c;
-    return 0; /* RED scaffold — task 2.2 implements exec/abort + seam routing */
+
+    char **argv = controller_saved_argv();
+    if (argv == NULL || argv[0] == NULL) {
+        ERROR("HA self-restart: no saved argv — cannot re-exec (FATAL)");
+        tlog_exit(); /* flush pending log buffer before dying */
+        abort();
+    }
+
+    INFO("HA self-restart: re-executing %s to rebuild from committed state",
+         argv[0]);
+    tlog_exit(); /* flush + drain the async log buffer before exec replaces us */
+
+    execv(argv[0], argv);
+
+    /* execv only returns on failure — FATAL (Req 1.4): never continue serving
+     * as a divergent ex-primary. */
+    ERROR("HA self-restart: execv(%s) failed: %d — aborting (FATAL)",
+          argv[0], errno);
+    abort();
+    return -1; /* unreachable */
+#endif
+}
+
+/*
+ * ha_demotion_convergence_ctx_t — heap context for the one-shot delaymgr task.
+ * Holds only the cluster pointer; committed_lsn is read at task time (not
+ * schedule time) so a late-advancing commit still truncates to the right
+ * boundary.
+ */
+typedef struct {
+    cluster_t *c;
+} ha_demotion_convergence_ctx_t;
+
+/*
+ * ha_demotion_convergence_fn — delaymgr task body (runs OFF the become-follower
+ * callback thread, D1).  Truncate above committed_lsn (Req 1.2), WARN with the
+ * truncated range (Req 1.5), then self-restart (Req 1.3).  On truncation failure
+ * (Req error strategy) ERROR then restart anyway — the rejoin reconciliation
+ * loop converges the untruncated case at the next join.
+ *
+ * In XCE_TESTING the test fixture drives this entry directly (un-started
+ * delaymgr); ha_self_restart routes to the restart seam there.  The ctx is
+ * freed before restart in production restart never returns, so only the test
+ * path reaches the free — kept on every non-restarting exit for safety.
+ */
+int ha_demotion_convergence_fn(void *arg)
+{
+    ha_demotion_convergence_ctx_t *ctx = (ha_demotion_convergence_ctx_t *)arg;
+    if (ctx == NULL || ctx->c == NULL) {
+        free(ctx);
+        return -EINVAL;
+    }
+    cluster_t *c = ctx->c;
+
+    uint64_t committed = atomic_load_explicit(&c->persist.committed_lsn,
+                                              memory_order_acquire);
+
+    /* Prior log tip for the WARN range (read under the persist lock). */
+    pthread_mutex_lock(&c->persist.lock);
+    uint64_t prior_tip = c->persist.current_lsn > 0
+                             ? c->persist.current_lsn - 1
+                             : 0;
+    pthread_mutex_unlock(&c->persist.lock);
+
+    int trc = persist_truncate_wal_above(&c->persist, committed);
+    if (trc < 0) {
+        ERROR("HA demotion convergence: truncate above committed_lsn=%" PRIu64
+              " failed: %d — restarting anyway (rejoin reconciliation will "
+              "converge)", committed, trc);
+    } else {
+        /* Req 1.5: log the discarded range — (committed_lsn, prior tip]. */
+        WARN("HA demotion convergence: truncated WAL above committed_lsn=%"
+             PRIu64 " (prior tip=%" PRIu64 "); discarding provisional suffix, "
+             "restarting to rebuild from committed state",
+             committed, prior_tip);
+    }
+
+    /* Free the task ctx before restart (production exec never returns). */
+    free(ctx);
+
+    /* Req 1.3: rebuild in-memory state through boot recovery. */
+    return ha_self_restart(c);
+}
+
+/*
+ * ha_schedule_demotion_convergence — called from raft_cb_become_follower's
+ * was-PRIMARY branch AFTER the persistence hooks have been disabled inline.
+ * Schedules exactly one one-shot delaymgr task running ha_demotion_convergence_fn
+ * so the file I/O (truncation) and exec never run on the callback thread (D1).
+ *
+ * Returns 0 on success, negative on allocation/scheduling failure.  On failure
+ * the node stays demoted with hooks disabled; the untruncated suffix is then
+ * reconciled by rejoin reconciliation at the next join (crash-safe by design).
+ */
+int ha_schedule_demotion_convergence(cluster_t *c)
+{
+    if (c == NULL)
+        return -EINVAL;
+
+    ha_demotion_convergence_ctx_t *ctx = calloc(1, sizeof(*ctx));
+    if (ctx == NULL) {
+        ERROR("HA demotion convergence: ctx alloc failed — suffix will be "
+              "reconciled at next rejoin");
+        return -ENOMEM;
+    }
+    ctx->c = c;
+
+    /* One-shot (runcount=1), minimal delay: the work must run promptly but off
+     * the iomgr callback thread.  delay_task_ms only enqueues (ordq_put); the
+     * delaymgr worker runs it.  In XCE_TESTING the fixture drives the queued
+     * task synchronously via its stored entry/args. */
+    int rc = delay_task_ms(&c->delaymgr, 1, 1,
+                           (delay_task_entry_fn_t)ha_demotion_convergence_fn,
+                           ctx);
+    if (rc < 0) {
+        ERROR("HA demotion convergence: delay_task_ms failed: %d", rc);
+        free(ctx);
+        return rc;
+    }
+    return 0;
 }