myxce/docs/superpowers/plans/2026-05-07-p3-hardening.md

P3: Integration & Hardening — TDD Implementation Plan

Branch: persistent_ha
Prerequisite: P2 complete (replication, sync ACK, catch-up, WAL trim guard).
Goal: Production-ready persistence + HA through comprehensive test coverage and a thin metrics layer. P3-1 through P3-6 are almost entirely test files; P3-7 adds the only new implementation code.

---

Overview

Step	Kind	New files	Modified files
P3-1	test-only	test_e2e_failover.c	—
P3-2	test-only	test_e2e_redirect.c	—
P3-3	test-only	test_e2e_wal_corrupt.c	—
P3-4	test-only	test_e2e_perf.c	—
P3-5	test-only	test_e2e_checkpoint_gc.c	—
P3-6	shim+test	test_e2e_manual_failover.c	ha.c, ha.h
P3-7	impl+test	metrics.h, metrics.c, test_metrics.c	persist.c, ha.c

Build integration: after each step add the new .c file to src/controller/testsuite/meson.build (same pattern as existing e2e tests).

---

P3-1: End-to-end failover test

What it covers

Kill-primary simulation: demote a persist+ha pair to standby, verify the standby's watchdog promotes, LSN continuity is maintained, and the promoted node's role is PRIMARY.

Files

• Create: src/controller/testsuite/test_e2e_failover.c

RED — write this test first

/* P3-1: End-to-end failover test
 *
 * Simulates primary failure without a live network:
 *   1. Init a cluster-like pair using bare ha_state_t + persist_t objects.
 *   2. "Primary" writes WAL entries and flushes.
 *   3. Simulate primary death: call ha_demote_to_standby() then stop persist.
 *   4. "Standby" calls ha_check_heartbeat_nocluster() after injecting a
 *      timed-out last_heartbeat — verifies failover_requested is set.
 *   5. Call ha_do_promote() and verify role == CTRL_PRIMARY, term incremented.
 *   6. Run persist_recover() on the standby's persist dir — verify LSN matches
 *      the last WAL entry the primary flushed.
 */
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unity/unity.h>

#include "config.h"
#include "ha.h"
#include "persist.h"

/* ---- helpers -------------------------------------------------------- */

static ha_config_t make_ha_cfg(void)
{
    ha_config_t c;
    memset(&c, 0, sizeof(c));
    c.heartbeat_interval_ms    = 2000;
    c.heartbeat_timeout_ms     = 4000;
    c.election_timeout_ms      = 500;
    c.probe_connect_timeout_ms = 100;
    c.max_promotion_retries    = 3;
    return c;
}

static persist_t *make_failover_persist(const char *tmpdir, const char *name)
{
    persist_t *p = calloc(1, sizeof(persist_t));
    if (!p) return NULL;
    persist_config_t cfg = persist_config_defaults();
    cfg.wal_sync_ms = 1;
    persist_set_workdir(p, tmpdir, name);
    if (persist_init(p, &cfg, NULL) != 0) { free(p); return NULL; }
    if (persist_start(p) != 0) { persist_fini(p); free(p); return NULL; }
    return p;
}

/* ---- test ----------------------------------------------------------- */

static void test_failover_standby_promotes_after_primary_death(void)
{
    /* ---- setup primary-side persist --------------------------------- */
    char primary_dir[256], standby_dir[256];
    strcpy(primary_dir, "/tmp/xce_p3_primary_XXXXXX");
    strcpy(standby_dir, "/tmp/xce_p3_standby_XXXXXX");
    TEST_ASSERT_NOT_NULL(mkdtemp(primary_dir));
    TEST_ASSERT_NOT_NULL(mkdtemp(standby_dir));

    /* Primary writes 10 WAL entries */
    persist_t *pri = make_failover_persist(primary_dir, "primary");
    TEST_ASSERT_NOT_NULL(pri);
    for (int i = 0; i < 10; i++)
        TEST_ASSERT_EQUAL_INT(0, persist_wal_async(pri, WAL_JOB_SUBMIT, "job", 3));
    TEST_ASSERT_EQUAL_INT(0, persist_flush(pri));
    uint64_t primary_lsn = pri->current_lsn;

    /* Primary "dies": stop without clean shutdown */
    persist_stop(pri);
    persist_fini(pri);
    free(pri);

    /* ---- standby ha_state_t ---------------------------------------- */
    ha_state_t standby_ha;
    ha_config_t ha_cfg = make_ha_cfg();
    TEST_ASSERT_EQUAL_INT(0, ha_init(&standby_ha, &ha_cfg, 1));

    /* Seed primary_host so heartbeat check considers a primary known */
    strncpy(standby_ha.primary_host, "dead-primary", XCE_MAXNAME_LEN - 1);
    standby_ha.primary_port = 7788;

    /* Inject an expired heartbeat: last_heartbeat = now - 10 s */
    clock_gettime(CLOCK_MONOTONIC, &standby_ha.last_heartbeat);
    standby_ha.last_heartbeat.tv_sec -= 10;

    /* Standby detects timeout */
    ha_check_heartbeat_nocluster(&standby_ha);
    TEST_ASSERT_EQUAL_INT(1,
        atomic_load_explicit(&standby_ha.failover_requested, memory_order_acquire));

    /* Promote */
    uint64_t term_before = atomic_load_explicit(&standby_ha.term, memory_order_acquire);
    ha_do_promote(&standby_ha);

    ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&standby_ha.role,
                                                          memory_order_acquire);
    TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, (int)role);
    TEST_ASSERT_EQUAL_UINT64(term_before + 1,
        atomic_load_explicit(&standby_ha.term, memory_order_acquire));

    /* ---- standby recover from primary WAL (copy-dir simulation) ----- */
    /* In a real cluster the standby has already applied entries via repl.
     * Here we point standby persist at the same primary dir (read-only sim). */
    persist_t *standby_p = make_failover_persist(primary_dir, "primary");
    TEST_ASSERT_NOT_NULL(standby_p);
    TEST_ASSERT_EQUAL_INT(0, persist_recover(standby_p));
    /* LSN after recovery must match primary's last flush */
    TEST_ASSERT_EQUAL_UINT64(primary_lsn, standby_p->current_lsn);

    persist_stop(standby_p);
    persist_fini(standby_p);
    free(standby_p);
    ha_fini(&standby_ha);

    printf("test_failover_standby_promotes_after_primary_death: PASS\n");
}

/* ---- suite ---------------------------------------------------------- */

void test_e2e_failover(void)
{
    printf("\n>> P3-1: End-to-end failover\n\n");
    RUN_TEST(test_failover_standby_promotes_after_primary_death);
}

GREEN

No new implementation required. All called functions (ha_init, ha_check_heartbeat_nocluster, ha_do_promote, persist_recover) are complete from P1/P2.

Commit

git add src/controller/testsuite/test_e2e_failover.c \
        src/controller/testsuite/meson.build
git commit -m "test(p3-1): end-to-end failover — standby promotes after primary death"

---

P3-2: Redirect test

What it covers

A standby controller's ha_should_redirect() returns 1 when role is STANDBY and primary is known, returns 0 when role is PRIMARY, and returns 0 when primary is unknown (client should retry). Also verifies ha_no_primary_known() complement.

Files

• Create: src/controller/testsuite/test_e2e_redirect.c

RED

/* P3-2: Redirect behaviour
 *
 * ha_should_redirect()  → 1 when standby + primary known
 *                       → 0 when primary (never redirect)
 *                       → 0 when standby + primary unknown (let client retry)
 * ha_no_primary_known() → 1 when standby + no primary info
 *                       → 0 when primary known
 */
#define _GNU_SOURCE
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <unity/unity.h>

#include "ha.h"
#include "config.h"

static ha_config_t make_min_cfg(void)
{
    ha_config_t c;
    memset(&c, 0, sizeof(c));
    c.heartbeat_interval_ms    = 2000;
    c.heartbeat_timeout_ms     = 4000;
    c.election_timeout_ms      = 500;
    c.probe_connect_timeout_ms = 100;
    c.max_promotion_retries    = 3;
    return c;
}

static void test_redirect_standby_with_known_primary(void)
{
    ha_state_t ha;
    ha_config_t cfg = make_min_cfg();
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1));

    strncpy(ha.primary_host, "ctrl-0", XCE_MAXNAME_LEN - 1);
    ha.primary_port = 7788;

    /* Role is STANDBY after init */
    TEST_ASSERT_EQUAL_INT(1, ha_should_redirect(&ha));
    TEST_ASSERT_EQUAL_INT(0, ha_no_primary_known(&ha));

    ha_fini(&ha);
    printf("test_redirect_standby_with_known_primary: PASS\n");
}

static void test_no_redirect_when_primary(void)
{
    ha_state_t ha;
    ha_config_t cfg = make_min_cfg();
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
    ha_do_promote(&ha);

    TEST_ASSERT_EQUAL_INT(0, ha_should_redirect(&ha));

    ha_fini(&ha);
    printf("test_no_redirect_when_primary: PASS\n");
}

static void test_no_redirect_no_primary_known(void)
{
    ha_state_t ha;
    ha_config_t cfg = make_min_cfg();
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1));
    /* primary_host is empty after init */

    TEST_ASSERT_EQUAL_INT(0, ha_should_redirect(&ha));
    TEST_ASSERT_EQUAL_INT(1, ha_no_primary_known(&ha));

    ha_fini(&ha);
    printf("test_no_redirect_no_primary_known: PASS\n");
}

void test_e2e_redirect(void)
{
    printf("\n>> P3-2: Redirect behaviour\n\n");
    RUN_TEST(test_redirect_standby_with_known_primary);
    RUN_TEST(test_no_redirect_when_primary);
    RUN_TEST(test_no_redirect_no_primary_known);
}

GREEN

No new implementation. ha_should_redirect and ha_no_primary_known are complete from P1.

Commit

git add src/controller/testsuite/test_e2e_redirect.c \
        src/controller/testsuite/meson.build
git commit -m "test(p3-2): redirect — standby redirects to known primary, no-primary returns retry signal"

---

P3-3: WAL corruption — additional scenarios

What it covers

Extends test_e2e_wal.c with two new scenarios not covered by the torn-write and chained-CRC tests from P0:

1. Partial segment header: file truncated mid-header (fewer bytes than sizeof(wal_entry_hdr_t)).
2. CRC mismatch on first entry: payload corrupted so the very first entry fails CRC check — recovery must return 0 (graceful) with LSN unchanged (0 entries replayed).

Files

• Modify: src/controller/testsuite/test_e2e_wal.c — append two new test functions and add them to the test_e2e_wal() suite entry point.

RED — append to existing file

/* ---- Test 4: partial header truncation ----------------------------- */

static void test_wal_partial_header_truncation(void)
{
    persist_t *p = make_e2e_wal_persist();
    TEST_ASSERT_NOT_NULL(p);

    const char payload[] = "hdr-trunc";
    TEST_ASSERT_EQUAL_INT(0,
        persist_wal_async(p, WAL_JOB_SUBMIT, payload, (uint32_t)strlen(payload)));
    TEST_ASSERT_EQUAL_INT(0, persist_flush(p));

    char seg_path[512];
    TEST_ASSERT_EQUAL_INT(0,
        find_first_wal_segment(p->wal_path, seg_path, sizeof(seg_path)));

    /* Truncate to 4 bytes — not enough for a full header */
    int fd = open(seg_path, O_RDWR);
    TEST_ASSERT_TRUE(fd >= 0);
    TEST_ASSERT_EQUAL_INT(0, ftruncate(fd, 4));
    close(fd);

    uint64_t lsn_before = p->current_lsn;
    teardown_e2e_wal(p);

    persist_t *p2 = calloc(1, sizeof(persist_t));
    TEST_ASSERT_NOT_NULL(p2);
    persist_config_t cfg = persist_config_defaults();
    cfg.wal_sync_ms = 5000;
    persist_set_workdir(p2, g_e2e_wal_tmpdir, "e2e-wal");
    TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL));
    TEST_ASSERT_EQUAL_INT(0, persist_start(p2));

    /* Must not crash; partial header is silently dropped */
    int rc = persist_recover(p2);
    TEST_ASSERT_EQUAL_INT(0, rc);
    /* No complete entry: LSN did not advance past start */
    TEST_ASSERT_TRUE(p2->current_lsn < lsn_before);

    teardown_e2e_wal(p2);
    printf("test_wal_partial_header_truncation: PASS\n");
}

/* ---- Test 5: first-entry CRC mismatch ------------------------------ */

static void test_wal_first_entry_crc_mismatch(void)
{
    persist_t *p = make_e2e_wal_persist();
    TEST_ASSERT_NOT_NULL(p);

    const char payload[] = "first-corrupt";
    TEST_ASSERT_EQUAL_INT(0,
        persist_wal_async(p, WAL_JOB_SUBMIT, payload, (uint32_t)strlen(payload)));
    TEST_ASSERT_EQUAL_INT(0, persist_flush(p));

    char seg_path[512];
    TEST_ASSERT_EQUAL_INT(0,
        find_first_wal_segment(p->wal_path, seg_path, sizeof(seg_path)));

    /* Corrupt the payload bytes of entry 0 */
    off_t payload_offset = (off_t)sizeof(wal_entry_hdr_t);
    int fd = open(seg_path, O_RDWR);
    TEST_ASSERT_TRUE(fd >= 0);
    const char garbage[] = "\xDE\xAD\xBE\xEF";
    pwrite(fd, garbage, sizeof(garbage) - 1, payload_offset);
    close(fd);

    teardown_e2e_wal(p);

    persist_t *p2 = calloc(1, sizeof(persist_t));
    TEST_ASSERT_NOT_NULL(p2);
    persist_config_t cfg = persist_config_defaults();
    cfg.wal_sync_ms = 5000;
    persist_set_workdir(p2, g_e2e_wal_tmpdir, "e2e-wal");
    TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL));
    TEST_ASSERT_EQUAL_INT(0, persist_start(p2));

    int rc = persist_recover(p2);
    TEST_ASSERT_EQUAL_INT(0, rc);
    /* Recovery stops at entry 0; LSN stays at its initial value */
    TEST_ASSERT_EQUAL_UINT64(0, p2->current_lsn);

    teardown_e2e_wal(p2);
    printf("test_wal_first_entry_crc_mismatch: PASS\n");
}

Also add these two RUN_TEST() calls to the existing test_e2e_wal() function.

GREEN

No new implementation. persist_recover already handles partial reads and CRC mismatches from P0/P2 work.

Commit

git add src/controller/testsuite/test_e2e_wal.c
git commit -m "test(p3-3): WAL corruption — partial header and first-entry CRC mismatch scenarios"

---

P3-4: Performance benchmark

What it covers

Measures WAL sync-append throughput: 1000 job submits with sync replication stub, confirms the entire batch completes within a 10-second wall-clock budget (generous for CI; real hardware easily hits 1000/s for local disk).

Files

• Create: src/controller/testsuite/test_e2e_perf.c

RED

/* P3-4: Performance — 1000 sync WAL appends within time budget
 *
 * Uses persist_wal_sync_append() (the sync replication path).
 * repl is empty (no standbys) so repl_wait_ack() returns 0 immediately —
 * this measures the local disk + fsync cost, which is the bottleneck.
 *
 * Budget: 10 seconds for 1000 entries on any CI machine.
 * A healthy local SSD sustains 5000+ synced appends/s.
 */
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unity/unity.h>

#include "config.h"
#include "persist.h"

#define PERF_ENTRY_COUNT 1000
#define PERF_BUDGET_S    10

static double elapsed_s(const struct timespec *start, const struct timespec *end)
{
    return (double)(end->tv_sec - start->tv_sec)
         + (double)(end->tv_nsec - start->tv_nsec) * 1e-9;
}

static void test_sync_wal_1000_submits_within_budget(void)
{
    char tmpdir[256];
    strcpy(tmpdir, "/tmp/xce_p3_perf_XXXXXX");
    TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir));

    persist_t *p = calloc(1, sizeof(persist_t));
    TEST_ASSERT_NOT_NULL(p);

    persist_config_t cfg = persist_config_defaults();
    cfg.wal_sync_ms    = 1;   /* flush quickly */
    cfg.sync_timeout_ms = 5;  /* 5 ms ACK timeout — no standbys, returns immediately */

    persist_set_workdir(p, tmpdir, "perf");
    TEST_ASSERT_EQUAL_INT(0, persist_init(p, &cfg, NULL));
    TEST_ASSERT_EQUAL_INT(0, persist_start(p));

    const char payload[] = "bench-submit";
    struct timespec t0, t1;
    clock_gettime(CLOCK_MONOTONIC, &t0);

    for (int i = 0; i < PERF_ENTRY_COUNT; i++) {
        int sync_rc = 0;
        int rc = persist_wal_sync_append(p, WAL_JOB_SUBMIT,
                                         payload, (uint32_t)strlen(payload),
                                         &sync_rc);
        /* rc==0 even if no standby; sync_rc may be -ETIMEDOUT which is fine */
        TEST_ASSERT_EQUAL_INT(0, rc);
    }

    clock_gettime(CLOCK_MONOTONIC, &t1);
    double secs = elapsed_s(&t0, &t1);

    printf("  %d sync appends in %.3f s (%.0f/s)\n",
           PERF_ENTRY_COUNT, secs, PERF_ENTRY_COUNT / secs);

    TEST_ASSERT_TRUE(secs < (double)PERF_BUDGET_S);
    TEST_ASSERT_EQUAL_UINT64(PERF_ENTRY_COUNT, p->current_lsn);

    persist_stop(p);
    persist_fini(p);
    free(p);
    printf("test_sync_wal_1000_submits_within_budget: PASS\n");
}

void test_e2e_perf(void)
{
    printf("\n>> P3-4: Performance — sync WAL throughput\n\n");
    RUN_TEST(test_sync_wal_1000_submits_within_budget);
}

GREEN

No new implementation needed.

Commit

git add src/controller/testsuite/test_e2e_perf.c \
        src/controller/testsuite/meson.build
git commit -m "test(p3-4): performance benchmark — 1000 sync WAL appends within 10 s budget"

---

P3-5: Checkpoint GC — dependents block cleanup

What it covers

Extends the existing GC tests (test_e2e_gc.c) with two new cases:

1. A finished job whose finish_time is within the retain window is not GC'd.
2. A worker that is UNAVAIL but dead_since is within worker_retain_s is not GC'd (dependents — i.e. jobs still referencing it — block worker GC).

Files

• Modify: src/controller/testsuite/test_e2e_gc.c — append two tests, register in test_e2e_gc().

RED — append to existing file

/* ---- Test 6: recently-finished job must be retained --------------- */

static void test_job_gc_retain_recent_finished(void)
{
    time_t now = time(NULL);
    persist_config_t cfg = persist_config_defaults();
    cfg.finished_retain_s = 3600;
    /* Finished 30 minutes ago — within retain window */
    job_t j = make_gc_job(JOB_STAT_EXIT, now - 1800, 0);
    TEST_ASSERT_TRUE(job_should_retain(&j, &cfg));
    printf("test_job_gc_retain_recent_finished: PASS\n");
}

/* ---- Test 7: recently-dead worker must be retained ---------------- */

static void test_worker_gc_retain_recent_dead(void)
{
    time_t now = time(NULL);
    persist_config_t cfg = persist_config_defaults();
    cfg.worker_retain_s = 86400; /* 24 h */
    /* Dead only 1 hour ago — within window */
    xce_worker_t w = make_gc_worker(XCE_WORKER_UNAVAIL, now - 3600);
    TEST_ASSERT_TRUE(worker_should_retain(&w, &cfg));
    printf("test_worker_gc_retain_recent_dead: PASS\n");
}

Add RUN_TEST() calls for both in test_e2e_gc().

GREEN

No new implementation. job_should_retain and worker_should_retain are complete.

Commit

git add src/controller/testsuite/test_e2e_gc.c
git commit -m "test(p3-5): GC checkpoint — recent-finished job and recent-dead worker are retained"

---

P3-6: Manual failover via test shim

What it covers

Exercises the full ha_promote_to_primary / ha_demote_to_standby state-machine path (including hook re-registration idempotency) from a test harness without requiring a live cluster. Needs a small test-only shim in ha.c + declaration in ha.h.

P3-6a: GREEN first — add shim to ha.c / ha.h

The shim wraps ha_demote_to_standby in a form testable without a real cluster_t.

Modify ha.h — add after the existing API declarations:

/*
 * ha_trigger_manual_failover_for_test: test-only entry point.
 * Calls ha_do_demote() directly (no cluster_t; hooks are not de-registered).
 * Use only from test code — production callers must use ha_demote_to_standby(c).
 */
void ha_trigger_manual_failover_for_test(ha_state_t *ha);

Modify ha.c — append at end of file:

/*
 * P3-6: Test-only shim — demote without cluster_t.
 * Validates role transitions + term reset without registering/removing hooks.
 */
void ha_trigger_manual_failover_for_test(ha_state_t *ha)
{
    ha_do_demote(ha);
}

Files

• Modify: src/controller/ha.h (add declaration)
• Modify: src/controller/ha.c (add shim implementation)
• Create: src/controller/testsuite/test_e2e_manual_failover.c

RED

/* P3-6: Manual failover state machine
 *
 * Verifies:
 *   1. Promote: role=PRIMARY, term incremented, hooks_registered NOT set
 *      (because we call ha_do_promote, not ha_promote_to_primary which needs cluster_t).
 *   2. Demote via shim: role=STANDBY, hooks_registered reset to 0.
 *   3. Second promote: term incremented again (idempotency: re-promotion works).
 *   4. promote() is idempotent for hooks_registered when called twice without demote.
 */
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>

#include "ha.h"
#include "config.h"

static ha_config_t make_min_ha_cfg(void)
{
    ha_config_t c;
    memset(&c, 0, sizeof(c));
    c.heartbeat_interval_ms    = 2000;
    c.heartbeat_timeout_ms     = 4000;
    c.election_timeout_ms      = 500;
    c.probe_connect_timeout_ms = 100;
    c.max_promotion_retries    = 3;
    return c;
}

static void test_manual_failover_role_transitions(void)
{
    ha_state_t ha;
    ha_config_t cfg = make_min_ha_cfg();
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1));

    /* Initial state: STANDBY, term 0 */
    TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire));
    TEST_ASSERT_EQUAL_UINT64(0,
        atomic_load_explicit(&ha.term, memory_order_acquire));

    /* Promote */
    ha_do_promote(&ha);
    TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire));
    TEST_ASSERT_EQUAL_UINT64(1,
        atomic_load_explicit(&ha.term, memory_order_acquire));

    /* Demote via test shim */
    ha_trigger_manual_failover_for_test(&ha);
    TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire));
    TEST_ASSERT_EQUAL_INT(0, ha.hooks_registered);

    /* Re-promote: term must increment again */
    uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
    ha_do_promote(&ha);
    TEST_ASSERT_EQUAL_UINT64(term_before + 1,
        atomic_load_explicit(&ha.term, memory_order_acquire));
    TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire));

    ha_fini(&ha);
    printf("test_manual_failover_role_transitions: PASS\n");
}

static void test_promote_idempotency_of_hooks_registered(void)
{
    /* ha_do_promote() itself does NOT set hooks_registered — that is
     * ha_promote_to_primary()'s job. Verify ha_do_promote never touches it. */
    ha_state_t ha;
    ha_config_t cfg = make_min_ha_cfg();
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));

    ha_do_promote(&ha);
    TEST_ASSERT_EQUAL_INT(0, ha.hooks_registered); /* untouched */

    ha_do_promote(&ha);
    TEST_ASSERT_EQUAL_INT(0, ha.hooks_registered); /* still untouched */

    ha_fini(&ha);
    printf("test_promote_idempotency_of_hooks_registered: PASS\n");
}

static void test_demote_resets_crc_chain(void)
{
    ha_state_t ha;
    ha_config_t cfg = make_min_ha_cfg();
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1));

    /* Seed a non-zero CRC chain then promote (resets it) */
    ha.last_received_crc = 0xDEADBEEF;
    ha_do_promote(&ha);
    TEST_ASSERT_EQUAL_UINT32(WAL_CRC_NONE, ha.last_received_crc);

    ha_fini(&ha);
    printf("test_demote_resets_crc_chain: PASS\n");
}

void test_e2e_manual_failover(void)
{
    printf("\n>> P3-6: Manual failover state machine\n\n");
    RUN_TEST(test_manual_failover_role_transitions);
    RUN_TEST(test_promote_idempotency_of_hooks_registered);
    RUN_TEST(test_demote_resets_crc_chain);
}

Commit

git add src/controller/ha.h src/controller/ha.c \
        src/controller/testsuite/test_e2e_manual_failover.c \
        src/controller/testsuite/meson.build
git commit -m "feat(p3-6): ha_trigger_manual_failover_for_test shim + role transition tests"

---

P3-7: Metrics infrastructure

This is the only step with meaningful new implementation code.

Design

src/include/ctrl_metrics.h      — metric name constants + metric_def_t table type
src/controller/ctrl_metrics.c   — global metrics table, counters, metrics_init()
src/controller/testsuite/test_metrics.c — all tests

Counters are _Atomic uint64_t global variables — no external dependencies, no lock contention. metrics_init() is called from persist_start() and ha_init(). The test file reads the globals directly (same translation unit via #include).

Metric naming

Metric name	Type	Where incremented
ctrl_split_brain_detected	counter	ha.c term-conflict path
ctrl_async_events_lost	gauge	promoted standby during ha_do_promote
ctrl_transient_jobs_stuck	counter	persist.c job_should_retain false path
ctrl_wal_bytes_written	counter	persist.c WAL write paths
ctrl_checkpoint_duration_ms	gauge	persist_checkpoint()
ctrl_repl_lag_lsn	gauge	repl.c ACK handler
ctrl_standbys_healthy	gauge	repl.c on add/remove

P3-7a: GREEN first (this step has implementation before tests)

Create src/include/ctrl_metrics.h:

#pragma once
/*
 * ctrl_metrics.h — P3-7: HA/persistence metrics with ctrl_ prefix.
 *
 * All metrics are _Atomic uint64_t globals. Call ctrl_metrics_init() once
 * from persist_start() / ha_init(). Read them directly for monitoring.
 */
#include <stdatomic.h>
#include <stdint.h>

/* ---- metric types ---- */
typedef enum {
    CTRL_METRIC_COUNTER = 0, /* monotonically increasing */
    CTRL_METRIC_GAUGE   = 1  /* can go up or down */
} ctrl_metric_type_t;

/* ---- metric descriptor (for documentation / introspection) ---- */
typedef struct {
    const char          *name;
    ctrl_metric_type_t   type;
    const char          *description;
} ctrl_metric_def_t;

/* ---- global counters / gauges ---- */
extern _Atomic uint64_t ctrl_split_brain_detected;
extern _Atomic uint64_t ctrl_async_events_lost;
extern _Atomic uint64_t ctrl_transient_jobs_stuck;
extern _Atomic uint64_t ctrl_wal_bytes_written;
extern _Atomic uint64_t ctrl_checkpoint_duration_ms;
extern _Atomic uint64_t ctrl_repl_lag_lsn;
extern _Atomic uint64_t ctrl_standbys_healthy;

/* ---- convenience macros ---- */
#define CTRL_METRIC_INC(m)    atomic_fetch_add_explicit(&(m), 1,   memory_order_relaxed)
#define CTRL_METRIC_ADD(m, v) atomic_fetch_add_explicit(&(m), (v), memory_order_relaxed)
#define CTRL_METRIC_SET(m, v) atomic_store_explicit(&(m), (v),     memory_order_relaxed)
#define CTRL_METRIC_GET(m)    atomic_load_explicit(&(m),            memory_order_relaxed)

/* Init: zero all metrics. Call once at startup. */
void ctrl_metrics_init(void);

/* Table: NULL-terminated array of all metric descriptors (for introspection). */
const ctrl_metric_def_t *ctrl_metrics_table(void);

Create src/controller/ctrl_metrics.c:

#include "ctrl_metrics.h"
#include <stddef.h>

/* ---- global definitions ---- */
_Atomic uint64_t ctrl_split_brain_detected   = 0;
_Atomic uint64_t ctrl_async_events_lost      = 0;
_Atomic uint64_t ctrl_transient_jobs_stuck   = 0;
_Atomic uint64_t ctrl_wal_bytes_written      = 0;
_Atomic uint64_t ctrl_checkpoint_duration_ms = 0;
_Atomic uint64_t ctrl_repl_lag_lsn           = 0;
_Atomic uint64_t ctrl_standbys_healthy       = 0;

/* ---- descriptor table ---- */
static const ctrl_metric_def_t s_table[] = {
    { "ctrl_split_brain_detected",   CTRL_METRIC_COUNTER,
      "Incremented each time a term conflict signals split-brain" },
    { "ctrl_async_events_lost",      CTRL_METRIC_GAUGE,
      "WAL entries in-flight on primary when standby took over" },
    { "ctrl_transient_jobs_stuck",   CTRL_METRIC_COUNTER,
      "Jobs auto-terminated after transient_job_timeout_s elapsed" },
    { "ctrl_wal_bytes_written",      CTRL_METRIC_COUNTER,
      "Total bytes appended to local WAL segments" },
    { "ctrl_checkpoint_duration_ms", CTRL_METRIC_GAUGE,
      "Wall-clock time of the most recent checkpoint in milliseconds" },
    { "ctrl_repl_lag_lsn",          CTRL_METRIC_GAUGE,
      "LSN lag between primary and slowest standby" },
    { "ctrl_standbys_healthy",       CTRL_METRIC_GAUGE,
      "Number of standbys with a live replication connection" },
    { NULL, 0, NULL } /* sentinel */
};

void ctrl_metrics_init(void)
{
    CTRL_METRIC_SET(ctrl_split_brain_detected,   0);
    CTRL_METRIC_SET(ctrl_async_events_lost,      0);
    CTRL_METRIC_SET(ctrl_transient_jobs_stuck,   0);
    CTRL_METRIC_SET(ctrl_wal_bytes_written,      0);
    CTRL_METRIC_SET(ctrl_checkpoint_duration_ms, 0);
    CTRL_METRIC_SET(ctrl_repl_lag_lsn,          0);
    CTRL_METRIC_SET(ctrl_standbys_healthy,       0);
}

const ctrl_metric_def_t *ctrl_metrics_table(void)
{
    return s_table;
}

Modify src/controller/persist.c — add #include "ctrl_metrics.h" and call ctrl_metrics_init() at the top of persist_start().

Also add metric increments at the right sites:

• persist_wal_async / persist_wal_sync_append: CTRL_METRIC_ADD(ctrl_wal_bytes_written, sizeof(hdr) + len);
• persist_checkpoint: before and after the operation, compute elapsed ms and CTRL_METRIC_SET(ctrl_checkpoint_duration_ms, elapsed).
• job_should_retain when returning false for JOB_STAT_TRANSIENT: CTRL_METRIC_INC(ctrl_transient_jobs_stuck);

Modify src/controller/ha.c — add #include "ctrl_metrics.h" and call ctrl_metrics_init() in ha_init(). In the term-conflict detection path (when a received message has a higher term than ha->term): CTRL_METRIC_INC(ctrl_split_brain_detected);.

RED

Create src/controller/testsuite/test_metrics.c:

/* P3-7: Metrics infrastructure tests
 *
 * All metrics are _Atomic uint64_t globals.
 * Tests verify init resets to 0, INC/ADD/SET/GET macros work,
 * descriptor table is well-formed and sentinel-terminated.
 */
#define _GNU_SOURCE
#include <string.h>
#include <stdio.h>
#include <unity/unity.h>

#include "ctrl_metrics.h"

/* ------------------------------------------------------------------ */
/* Test 1: ctrl_metrics_init resets all counters to 0                 */
/* ------------------------------------------------------------------ */

static void test_metrics_init_zeros_all(void)
{
    /* Dirty all metrics */
    CTRL_METRIC_SET(ctrl_split_brain_detected,   99);
    CTRL_METRIC_SET(ctrl_async_events_lost,      99);
    CTRL_METRIC_SET(ctrl_transient_jobs_stuck,   99);
    CTRL_METRIC_SET(ctrl_wal_bytes_written,      99);
    CTRL_METRIC_SET(ctrl_checkpoint_duration_ms, 99);
    CTRL_METRIC_SET(ctrl_repl_lag_lsn,          99);
    CTRL_METRIC_SET(ctrl_standbys_healthy,       99);

    ctrl_metrics_init();

    TEST_ASSERT_EQUAL_UINT64(0, CTRL_METRIC_GET(ctrl_split_brain_detected));
    TEST_ASSERT_EQUAL_UINT64(0, CTRL_METRIC_GET(ctrl_async_events_lost));
    TEST_ASSERT_EQUAL_UINT64(0, CTRL_METRIC_GET(ctrl_transient_jobs_stuck));
    TEST_ASSERT_EQUAL_UINT64(0, CTRL_METRIC_GET(ctrl_wal_bytes_written));
    TEST_ASSERT_EQUAL_UINT64(0, CTRL_METRIC_GET(ctrl_checkpoint_duration_ms));
    TEST_ASSERT_EQUAL_UINT64(0, CTRL_METRIC_GET(ctrl_repl_lag_lsn));
    TEST_ASSERT_EQUAL_UINT64(0, CTRL_METRIC_GET(ctrl_standbys_healthy));

    printf("test_metrics_init_zeros_all: PASS\n");
}

/* ------------------------------------------------------------------ */
/* Test 2: INC, ADD, SET, GET macros                                  */
/* ------------------------------------------------------------------ */

static void test_metrics_macros(void)
{
    ctrl_metrics_init();

    CTRL_METRIC_INC(ctrl_split_brain_detected);
    TEST_ASSERT_EQUAL_UINT64(1, CTRL_METRIC_GET(ctrl_split_brain_detected));

    CTRL_METRIC_INC(ctrl_split_brain_detected);
    TEST_ASSERT_EQUAL_UINT64(2, CTRL_METRIC_GET(ctrl_split_brain_detected));

    CTRL_METRIC_ADD(ctrl_wal_bytes_written, 512);
    TEST_ASSERT_EQUAL_UINT64(512, CTRL_METRIC_GET(ctrl_wal_bytes_written));

    CTRL_METRIC_ADD(ctrl_wal_bytes_written, 512);
    TEST_ASSERT_EQUAL_UINT64(1024, CTRL_METRIC_GET(ctrl_wal_bytes_written));

    CTRL_METRIC_SET(ctrl_repl_lag_lsn, 42);
    TEST_ASSERT_EQUAL_UINT64(42, CTRL_METRIC_GET(ctrl_repl_lag_lsn));

    CTRL_METRIC_SET(ctrl_repl_lag_lsn, 7);
    TEST_ASSERT_EQUAL_UINT64(7, CTRL_METRIC_GET(ctrl_repl_lag_lsn));

    printf("test_metrics_macros: PASS\n");
}

/* ------------------------------------------------------------------ */
/* Test 3: descriptor table is well-formed and sentinel-terminated    */
/* ------------------------------------------------------------------ */

static void test_metrics_table_sentinel_terminated(void)
{
    const ctrl_metric_def_t *tbl = ctrl_metrics_table();
    TEST_ASSERT_NOT_NULL(tbl);

    int count = 0;
    while (tbl[count].name != NULL) {
        /* Each entry must have a non-empty description and valid type */
        TEST_ASSERT_NOT_NULL(tbl[count].description);
        TEST_ASSERT_TRUE(strlen(tbl[count].description) > 0);
        TEST_ASSERT_TRUE(strlen(tbl[count].name) > 0);
        /* Name must start with "ctrl_" */
        TEST_ASSERT_EQUAL_INT(0, strncmp(tbl[count].name, "ctrl_", 5));
        TEST_ASSERT_TRUE(tbl[count].type == CTRL_METRIC_COUNTER
                      || tbl[count].type == CTRL_METRIC_GAUGE);
        count++;
    }

    /* We define exactly 7 metrics */
    TEST_ASSERT_EQUAL_INT(7, count);

    /* Sentinel: name is NULL, type is 0 */
    TEST_ASSERT_NULL(tbl[count].name);

    printf("test_metrics_table_sentinel_terminated: PASS (count=%d)\n", count);
}

/* ------------------------------------------------------------------ */
/* Test 4: transient_jobs_stuck incremented by job_should_retain      */
/* ------------------------------------------------------------------ */

static void test_transient_jobs_stuck_counter(void)
{
    ctrl_metrics_init();

    /* Include persist.h for job_should_retain + fixtures */
    /* The fixture: transient job that has timed out */
    extern bool job_should_retain(const job_t*, const persist_config_t*);
    #include "xce/types.h"

    persist_config_t cfg = persist_config_defaults();
    cfg.transient_job_timeout_s = 300;

    job_t j;
    memset(&j, 0, sizeof(j));
    j.job.state           = JOB_STAT_TRANSIENT;
    j.job.transient_since = (uint64_t)(time(NULL) - 400); /* 400 s ago */

    bool retain = job_should_retain(&j, &cfg);
    TEST_ASSERT_FALSE(retain);

    /* Counter must have been bumped */
    TEST_ASSERT_EQUAL_UINT64(1, CTRL_METRIC_GET(ctrl_transient_jobs_stuck));

    printf("test_transient_jobs_stuck_counter: PASS\n");
}

/* ------------------------------------------------------------------ */
/* Test 5: wal_bytes_written incremented by persist_wal_async         */
/* ------------------------------------------------------------------ */

static void test_wal_bytes_written_counter(void)
{
    ctrl_metrics_init();

    char tmpdir[256];
    strcpy(tmpdir, "/tmp/xce_p3_metrics_XXXXXX");
    TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir));

    persist_t *p = calloc(1, sizeof(persist_t));
    TEST_ASSERT_NOT_NULL(p);
    persist_config_t cfg = persist_config_defaults();
    cfg.wal_sync_ms = 5000;
    persist_set_workdir(p, tmpdir, "metrics");
    TEST_ASSERT_EQUAL_INT(0, persist_init(p, &cfg, NULL));
    TEST_ASSERT_EQUAL_INT(0, persist_start(p));

    const char payload[] = "metric-payload";
    uint32_t plen = (uint32_t)strlen(payload);

    uint64_t before = CTRL_METRIC_GET(ctrl_wal_bytes_written);
    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_SUBMIT, payload, plen));
    uint64_t after = CTRL_METRIC_GET(ctrl_wal_bytes_written);

    uint64_t expected_delta = sizeof(wal_entry_hdr_t) + plen;
    TEST_ASSERT_EQUAL_UINT64(expected_delta, after - before);

    persist_stop(p);
    persist_fini(p);
    free(p);
    printf("test_wal_bytes_written_counter: PASS\n");
}

/* ------------------------------------------------------------------ */
/* Suite entry point                                                    */
/* ------------------------------------------------------------------ */

void test_metrics(void)
{
    printf("\n>> P3-7: Metrics infrastructure\n\n");
    RUN_TEST(test_metrics_init_zeros_all);
    RUN_TEST(test_metrics_macros);
    RUN_TEST(test_metrics_table_sentinel_terminated);
    RUN_TEST(test_transient_jobs_stuck_counter);
    RUN_TEST(test_wal_bytes_written_counter);
}

Commit sequence

# Commit 1: headers + implementation
git add src/include/ctrl_metrics.h src/controller/ctrl_metrics.c
git commit -m "feat(p3-7): ctrl_metrics — atomic counters/gauges with ctrl_ prefix"

# Commit 2: wire into persist + ha
git add src/controller/persist.c src/controller/ha.c
git commit -m "feat(p3-7): wire ctrl_metrics_init into persist_start and ha_init; add increment sites"

# Commit 3: tests + meson
git add src/controller/testsuite/test_metrics.c src/controller/testsuite/meson.build
git commit -m "test(p3-7): metrics — init, macros, table validation, WAL counter, transient counter"

---

meson.build integration (cumulative)

After all steps, the additions to src/controller/testsuite/meson.build are:

# P3 integration & hardening tests
e2e_sources += files(
  'test_e2e_failover.c',
  'test_e2e_redirect.c',
  'test_e2e_perf.c',
  'test_e2e_manual_failover.c',
  'test_metrics.c',
)

test_e2e_wal.c and test_e2e_gc.c are already in the build; their new test functions are picked up automatically.

Add suite entry calls to src/controller/testsuite/test_main.c (or equivalent runner):

void test_e2e_failover(void);
void test_e2e_redirect(void);
void test_e2e_perf(void);
void test_e2e_manual_failover(void);
void test_metrics(void);

/* ... in main() ... */
test_e2e_failover();
test_e2e_redirect();
test_e2e_perf();
test_e2e_manual_failover();
test_metrics();

Also add ctrl_metrics.c to the controller's main meson.build source list so the object is linked into both the controller binary and the test runner.

---

Verification

cd /worktrees/persistent_ha
nix develop --command bash -c "
  cd src
  meson setup build --wipe -Db_sanitize=address,undefined 2>&1 | tail -5
  ninja -C build
  ninja -C build test
"

Expected: all existing tests pass plus all P3 tests pass. ASAN/UBSan clean.

---

What P3 does NOT do

• No live network tests (no real TCP connections between controllers)
• No Prometheus / alerting integration (out of scope per design doc)
• No xce hostcontrol --failover CLI surface (deferred to future work)
• No WAL compaction, multi-DC replication, or snapshot compression