feat: P3-7 ctrl_metrics — 7 atomic counters/gauges + descriptor table + persist wiring

Add operational metrics infrastructure for the XCE controller persistence
and HA subsystems (design doc P3-7, review A8-007).

- src/include/ctrl_metrics.h: declares 7 _Atomic uint64_t metrics with
  ctrl_ prefix, CTRL_METRIC_INC / CTRL_METRIC_SET macros, ctrl_metric_desc_t
  descriptor struct, ctrl_metric_descriptors[] table, and ctrl_metrics_init()
- src/controller/ctrl_metrics.c: defines the 7 counters/gauges, the
  descriptor table (7 entries + NULL sentinel), and ctrl_metrics_init()
- src/controller/persist.c: #include ctrl_metrics.h; call ctrl_metrics_init()
  at persist_start() entry; CTRL_METRIC_INC(ctrl_repl_degraded_count) when
  repl_wait_ack returns < 0 (degraded / local-only durability mode)
- src/controller/meson.build: add ctrl_metrics.c to xce_controller and
  controller_unit_test source lists; add test_ctrl_metrics.c to test sources
- src/controller/testsuite/test_ctrl_metrics.c: 5 TDD-first Unity tests
  covering init-zeros, increment, set/overwrite gauge, descriptor sentinel
  count, and transient-stuck counter accumulation — all PASS

src/controller/ctrl_metrics.c

@@ -0,0 +1,68 @@
+/*
+ * ctrl_metrics.c — P3-7: Operational metrics for the XCE controller.
+ *
+ * Defines the 7 atomic counters/gauges and the descriptor table used
+ * for metric discovery.  All variables are initialised to 0 at program
+ * start (C guarantees this for file-scope variables), and are zeroed
+ * explicitly by ctrl_metrics_init() which is called from persist_start()
+ * so that tests can call it repeatedly without interference.
+ */
+#include "ctrl_metrics.h"
+
+#include <stdatomic.h>
+#include <stddef.h>
+#include <stdint.h>
+
+/* ── Counter/gauge definitions ──────────────────────────────────────── */
+_Atomic uint64_t ctrl_repl_degraded_count    = 0;
+_Atomic uint64_t ctrl_split_brain_detected   = 0;
+_Atomic uint64_t ctrl_async_events_lost      = 0;
+_Atomic uint64_t ctrl_transient_jobs_stuck   = 0;
+_Atomic uint64_t ctrl_wal_bytes              = 0;
+_Atomic uint64_t ctrl_checkpoint_duration_ms = 0;
+_Atomic uint64_t ctrl_repl_lag_lsn           = 0;
+
+/* ── Descriptor table ───────────────────────────────────────────────── */
+const ctrl_metric_desc_t ctrl_metric_descriptors[] = {
+    { "ctrl_repl_degraded_count",
+      &ctrl_repl_degraded_count,
+      "Sync replication fallbacks to local-only durability",
+      1 },
+    { "ctrl_split_brain_detected",
+      &ctrl_split_brain_detected,
+      "Term conflicts indicating split-brain risk",
+      1 },
+    { "ctrl_async_events_lost",
+      &ctrl_async_events_lost,
+      "Async WAL entries lost during failover",
+      1 },
+    { "ctrl_transient_jobs_stuck",
+      &ctrl_transient_jobs_stuck,
+      "TRANSIENT jobs auto-terminated after timeout",
+      1 },
+    { "ctrl_wal_bytes",
+      &ctrl_wal_bytes,
+      "Current WAL size in bytes",
+      0 },
+    { "ctrl_checkpoint_duration_ms",
+      &ctrl_checkpoint_duration_ms,
+      "Duration of last checkpoint in milliseconds",
+      0 },
+    { "ctrl_repl_lag_lsn",
+      &ctrl_repl_lag_lsn,
+      "LSN gap between primary and slowest standby",
+      0 },
+    { NULL, NULL, NULL, 0 }  /* sentinel */
+};
+
+/* ── ctrl_metrics_init ──────────────────────────────────────────────── */
+void ctrl_metrics_init(void)
+{
+    atomic_store_explicit(&ctrl_repl_degraded_count,    0, memory_order_relaxed);
+    atomic_store_explicit(&ctrl_split_brain_detected,   0, memory_order_relaxed);
+    atomic_store_explicit(&ctrl_async_events_lost,      0, memory_order_relaxed);
+    atomic_store_explicit(&ctrl_transient_jobs_stuck,   0, memory_order_relaxed);
+    atomic_store_explicit(&ctrl_wal_bytes,              0, memory_order_relaxed);
+    atomic_store_explicit(&ctrl_checkpoint_duration_ms, 0, memory_order_relaxed);
+    atomic_store_explicit(&ctrl_repl_lag_lsn,           0, memory_order_relaxed);
+}

src/controller/meson.build

@@ -19,6 +19,7 @@ controller_src = [
   'myconf.c',
   'persist.c',
   'repl.c',
+  'ctrl_metrics.c',
 ]
 
 incdir = include_directories('.', '../include', '../lib',
@@ -95,9 +96,11 @@ controller_unit_test_src = [
   'testsuite/test_p2_integration.c',
   'testsuite/test_p3_hardening.c',
   'testsuite/test_p3_manual_failover.c',
+  'testsuite/test_ctrl_metrics.c',
   'persist.c',
   'ha.c',
   'repl.c',
+  'ctrl_metrics.c',
 ]
 
 controller_unit_test_deps = [

src/controller/persist.c

@@ -1,6 +1,7 @@
 #define _GNU_SOURCE
 #include "persist.h"
 #include "config.h"
+#include "ctrl_metrics.h"
 #include "hook.h"
 #include "xce/cluster_def.h"
 #include "xce/types.h"
@@ -179,6 +180,8 @@ int persist_init(persist_t* p, persist_config_t* cfg, cluster_t* c)
 
 int persist_start(persist_t* p)
 {
+    ctrl_metrics_init();
+
     /* Register AFTER_EVENT persistence hooks if we have a cluster back-pointer */
     if (p->cluster != NULL) {
         xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_JOB,
@@ -869,6 +872,8 @@ int persist_wal_sync_append(persist_t* p, uint32_t type,
 
     int sync_err = repl_wait_ack(&p->repl, written_lsn,
                                  p->config.sync_timeout_ms);
+    if (sync_err < 0)
+        CTRL_METRIC_INC(ctrl_repl_degraded_count);
     if (sync_rc_out)
         *sync_rc_out = sync_err;
     /* -ETIMEDOUT = degraded mode (local-only durability); -ENODEV = no standbys.

src/controller/testsuite/controller_test.c

@@ -30,6 +30,7 @@ extern void test_repl_catchup(void);
 extern void test_p2_integration(void);
 extern void test_p3_hardening(void);
 extern void test_p3_manual_failover(void);
+extern void test_ctrl_metrics(void);
 
 void setUp(void) { return; }
 
@@ -129,6 +130,7 @@ int main(int argc, char* argv[])
     TEST(test_p2_integration);
     TEST(test_p3_hardening);
     TEST(test_p3_manual_failover);
+    TEST(test_ctrl_metrics);
 
     return UNITY_END();
 }

src/controller/testsuite/test_ctrl_metrics.c

@@ -0,0 +1,88 @@
+/* test_ctrl_metrics.c — P3-7: Metrics infrastructure unit tests
+ *
+ * TDD: these tests are written FIRST to drive the implementation.
+ * They verify:
+ *   - ctrl_metrics_init() zeroes all 7 counters/gauges
+ *   - CTRL_METRIC_INC increments atomically
+ *   - CTRL_METRIC_SET writes and overwrites a gauge
+ *   - The descriptor table has exactly 7 entries + NULL sentinel
+ *   - ctrl_transient_jobs_stuck counter accumulates correctly
+ */
+#define _GNU_SOURCE
+#include <stdatomic.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include <unity/unity.h>
+
+#include "ctrl_metrics.h"
+
+/* ------------------------------------------------------------------ */
+/* Tests                                                               */
+/* ------------------------------------------------------------------ */
+
+static void test_metrics_init_zeros_all_counters(void)
+{
+    ctrl_metrics_init();
+    TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&ctrl_repl_degraded_count));
+    TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&ctrl_split_brain_detected));
+    TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&ctrl_async_events_lost));
+    TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&ctrl_transient_jobs_stuck));
+    TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&ctrl_wal_bytes));
+    TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&ctrl_checkpoint_duration_ms));
+    TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&ctrl_repl_lag_lsn));
+    printf("test_metrics_init_zeros_all_counters: PASS\n");
+}
+
+static void test_metrics_increment_repl_degraded(void)
+{
+    ctrl_metrics_init();
+    CTRL_METRIC_INC(ctrl_repl_degraded_count);
+    CTRL_METRIC_INC(ctrl_repl_degraded_count);
+    TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&ctrl_repl_degraded_count));
+    printf("test_metrics_increment_repl_degraded: PASS\n");
+}
+
+static void test_metrics_set_gauge(void)
+{
+    ctrl_metrics_init();
+    CTRL_METRIC_SET(ctrl_wal_bytes, 1024 * 1024);
+    TEST_ASSERT_EQUAL_UINT64(1024 * 1024, atomic_load(&ctrl_wal_bytes));
+    CTRL_METRIC_SET(ctrl_wal_bytes, 0);
+    TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&ctrl_wal_bytes));
+    printf("test_metrics_set_gauge: PASS\n");
+}
+
+static void test_metrics_descriptor_table_has_sentinel(void)
+{
+    /* The descriptor table must end with a NULL-name sentinel
+     * and contain exactly 7 real entries. */
+    int i = 0;
+    while (ctrl_metric_descriptors[i].name != NULL)
+        i++;
+    TEST_ASSERT_EQUAL_INT(7, i);
+    printf("test_metrics_descriptor_table_has_sentinel: PASS\n");
+}
+
+static void test_transient_jobs_stuck_counter(void)
+{
+    ctrl_metrics_init();
+    /* Simulate 3 stuck transient jobs */
+    for (int j = 0; j < 3; j++)
+        CTRL_METRIC_INC(ctrl_transient_jobs_stuck);
+    TEST_ASSERT_EQUAL_UINT64(3, atomic_load(&ctrl_transient_jobs_stuck));
+    printf("test_transient_jobs_stuck_counter: PASS\n");
+}
+
+/* ------------------------------------------------------------------ */
+/* Suite entry point (called from controller_test.c)                   */
+/* ------------------------------------------------------------------ */
+
+void test_ctrl_metrics(void)
+{
+    RUN_TEST(test_metrics_init_zeros_all_counters);
+    RUN_TEST(test_metrics_increment_repl_degraded);
+    RUN_TEST(test_metrics_set_gauge);
+    RUN_TEST(test_metrics_descriptor_table_has_sentinel);
+    RUN_TEST(test_transient_jobs_stuck_counter);
+}

src/include/ctrl_metrics.h

@@ -0,0 +1,47 @@
+#pragma once
+/*
+ * ctrl_metrics.h — P3-7: Operational metrics for the XCE controller.
+ *
+ * All metrics use C11 _Atomic uint64_t for lock-free, thread-safe access.
+ * Metric names use the ctrl_ prefix.
+ *
+ * Design doc: Part P3-7, review A8-007.
+ */
+#include <stdatomic.h>
+#include <stdint.h>
+
+/* ── Counter/gauge globals ──────────────────────────────────────────── */
+extern _Atomic uint64_t ctrl_repl_degraded_count;    /* counter */
+extern _Atomic uint64_t ctrl_split_brain_detected;   /* counter */
+extern _Atomic uint64_t ctrl_async_events_lost;      /* counter */
+extern _Atomic uint64_t ctrl_transient_jobs_stuck;   /* counter */
+extern _Atomic uint64_t ctrl_wal_bytes;              /* gauge   */
+extern _Atomic uint64_t ctrl_checkpoint_duration_ms; /* gauge   */
+extern _Atomic uint64_t ctrl_repl_lag_lsn;           /* gauge   */
+
+/* ── Macros ─────────────────────────────────────────────────────────── */
+
+/* Increment a monotonic counter by 1 (relaxed ordering — counters are
+ * sampled, not used for synchronization). */
+#define CTRL_METRIC_INC(name) \
+    atomic_fetch_add_explicit(&(name), 1, memory_order_relaxed)
+
+/* Overwrite a gauge with a new absolute value. */
+#define CTRL_METRIC_SET(name, val) \
+    atomic_store_explicit(&(name), (uint64_t)(val), memory_order_relaxed)
+
+/* ── Descriptor table for metric discovery ──────────────────────────── */
+typedef struct {
+    const char*        name;
+    _Atomic uint64_t*  value;
+    const char*        description;
+    int                is_counter; /* 1 = counter (monotonic), 0 = gauge */
+} ctrl_metric_desc_t;
+
+/* Terminated by a { NULL, NULL, NULL, 0 } sentinel entry. */
+extern const ctrl_metric_desc_t ctrl_metric_descriptors[];
+
+/* ── Lifecycle ──────────────────────────────────────────────────────── */
+
+/* Reset all metrics to zero. Call from persist_start() and ha_init(). */
+void ctrl_metrics_init(void);