feat(ha-config-bootstrap): implement EPOLLRDHUP failover and heartbeat timeout vtask (task 9.4)

src/controller/ha.c

@@ -1524,3 +1524,165 @@ int ha_find_and_connect_to_primary(ha_state_t *ha,
                      ? HA_FIND_PRIMARY_T3_MAX_MS : backoff_ms * 2;
     }
 }
+
+/* ── Task 9.4: EPOLLRDHUP Failover and Heartbeat Timeout ───────────────────
+ *
+ * Requirements: 6.2 (RDHUP fast path), 6.3 (heartbeat timeout fallback),
+ *               6.4 (no heartbeat within timeout → elect), 6.8 (role flip)
+ *               10.5 (INFO logs on FSM transitions)
+ */
+
+void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c)
+{
+    if (!ha) return;
+
+    /* Close and null the primary control fd */
+    if (ha->primary_ctrl_fd >= 0) {
+        close(ha->primary_ctrl_fd);
+        ha->primary_ctrl_fd = -1;
+    }
+
+    /* Transition to CTRL_ELECTING (Req 6.2) */
+    atomic_store_explicit(&ha->role, CTRL_ELECTING, memory_order_release);
+    INFO("HA: primary_ctrl_fd RDHUP → CTRL_ELECTING (Req 10.5)");
+
+    if (!c) return; /* unit test mode: skip persist + election */
+
+    /* Truncate uncommitted WAL entries above committed_lsn (Req 6.2) */
+    uint64_t committed = atomic_load_explicit(&ha->last_known_committed_lsn,
+                                               memory_order_acquire);
+    if (committed > 0)
+        persist_truncate_wal_above(&c->persist, committed);
+
+    /* Fire election — am_i_primary_quorum handles the vote and role assignment */
+    if (c->myconf && c->myconf->peers_cfg.candidates_count > 0) {
+        const xce_cluster_config_t *cfg = &c->myconf->peers_cfg;
+        int n = cfg->candidates_count > 0 ? cfg->candidates_count - 1 : 0;
+        /* Build peer pointer array (exclude self) */
+        xce_candidate_t *peer_ptrs[MAX_HA_CANDIDATES];
+        int np = 0;
+        for (int i = 0; i < cfg->candidates_count && np < n; i++) {
+            if (strcmp(cfg->candidates[i].name, c->myhost ? c->myhost->name : "") != 0)
+                peer_ptrs[np++] = &cfg->candidates[i];
+        }
+        uint64_t my_tag = c->myconf->config_version_tag;
+        uint64_t my_lsn = atomic_load_explicit(&c->persist.committed_lsn,
+                                                memory_order_acquire);
+        ctrl_role_t new_role = am_i_primary_quorum(ha, np, peer_ptrs, my_tag, my_lsn);
+        if (new_role == CTRL_PRIMARY)
+            INFO("HA: RDHUP recovery → CTRL_PRIMARY (Req 6.8)");
+        else
+            INFO("HA: RDHUP recovery → CTRL_STANDBY, reconnecting (Req 6.8)");
+    }
+}
+
+/* ── Heartbeat timeout watchdog ─────────────────────────────────────────── */
+
+typedef struct {
+    ha_state_t   *ha;
+    cluster_t    *c;
+    sem_t         reset_sem;     /* ha_on_heartbeat_received posts here */
+    atomic_int    disarmed;      /* set to 1 by ha_disarm_heartbeat_timeout_vtask */
+    pthread_t     thread;
+    int           started;
+} hb_vtask_t;
+
+/* File-scope pointer to the active heartbeat watchdog.
+ * Safety: arm/disarm/heartbeat are called only from the single-threaded
+ * vtaskmgr dispatch context; no locking is needed in the current architecture.
+ * If concurrent access is added later, protect with ha->lock. */
+static hb_vtask_t *s_hb_vtask = NULL;
+
+static void *hb_timeout_thread(void *arg)
+{
+    hb_vtask_t *t = (hb_vtask_t *)arg;
+    ha_state_t *ha = t->ha;
+
+    while (!atomic_load(&t->disarmed)) {
+        /* Compute deadline: election_timeout_ms + rand_jitter */
+        uint32_t jitter_ms = 0;
+        if (ha->config.election_jitter_max_ms > 0) {
+            struct timespec now;
+            clock_gettime(CLOCK_REALTIME, &now);
+            unsigned int seed = (unsigned int)now.tv_nsec;
+            jitter_ms = (uint32_t)(rand_r(&seed) % ha->config.election_jitter_max_ms);
+        }
+        uint32_t total_ms = ha->config.election_timeout_ms + jitter_ms;
+
+        struct timespec deadline;
+        clock_gettime(CLOCK_REALTIME, &deadline);
+        deadline.tv_sec  += (long)(total_ms / 1000);
+        deadline.tv_nsec += (long)(total_ms % 1000) * 1000000L;
+        if (deadline.tv_nsec >= 1000000000L) {
+            deadline.tv_sec++;
+            deadline.tv_nsec -= 1000000000L;
+        }
+
+        /* Wait for heartbeat signal or deadline */
+        int rc = sem_timedwait(&t->reset_sem, &deadline);
+        if (rc == -1 && errno == ETIMEDOUT) {
+            /* Timeout with no heartbeat → failover (Req 6.3, 6.4) */
+            if (!atomic_load(&t->disarmed)) {
+                WARN("HA: heartbeat timeout (%ums) → EPOLLRDHUP failover path",
+                     total_ms);
+                ha_on_primary_ctrl_fd_rdhup(ha, t->c);
+                break;  /* fired; thread exits */
+            }
+        }
+        /* heartbeat received — loop and re-arm deadline */
+    }
+    return NULL;
+}
+
+void ha_arm_heartbeat_timeout_vtask(ha_state_t *ha, cluster_t *c)
+{
+    if (!ha) return;
+
+    /* Disarm any existing watchdog */
+    if (s_hb_vtask && s_hb_vtask->started) {
+        atomic_store(&s_hb_vtask->disarmed, 1);
+        sem_post(&s_hb_vtask->reset_sem);  /* wake sleeping thread */
+        pthread_join(s_hb_vtask->thread, NULL);
+        sem_destroy(&s_hb_vtask->reset_sem);
+        free(s_hb_vtask);
+        s_hb_vtask = NULL;
+    }
+
+    hb_vtask_t *t = calloc(1, sizeof(*t));
+    if (!t) return;
+
+    t->ha      = ha;
+    t->c       = c;
+    t->started = 0;
+    atomic_init(&t->disarmed, 0);
+    sem_init(&t->reset_sem, 0, 0);
+
+    if (pthread_create(&t->thread, NULL, hb_timeout_thread, t) != 0) {
+        sem_destroy(&t->reset_sem);
+        free(t);
+        return;
+    }
+    t->started = 1;
+    s_hb_vtask = t;
+}
+
+void ha_on_heartbeat_received(ha_state_t *ha)
+{
+    (void)ha;
+    if (s_hb_vtask && s_hb_vtask->started)
+        sem_post(&s_hb_vtask->reset_sem);
+}
+
+void ha_disarm_heartbeat_timeout_vtask(ha_state_t *ha)
+{
+    (void)ha;
+    if (!s_hb_vtask) return;
+    if (s_hb_vtask->started) {
+        atomic_store(&s_hb_vtask->disarmed, 1);
+        sem_post(&s_hb_vtask->reset_sem);
+        pthread_join(s_hb_vtask->thread, NULL);
+        sem_destroy(&s_hb_vtask->reset_sem);
+    }
+    free(s_hb_vtask);
+    s_hb_vtask = NULL;
+}