diff --git a/docs/superpowers/plans/2026-05-07-p0-single-controller-persistence.md b/docs/superpowers/plans/2026-05-07-p0-single-controller-persistence.md new file mode 100644 index 0000000..0e937c4 --- /dev/null +++ b/docs/superpowers/plans/2026-05-07-p0-single-controller-persistence.md @@ -0,0 +1,1662 @@ +# P0: Single-Controller Persistence — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Controller survives restart with all job, queue, and worker state intact. + +**Architecture:** Append-only WAL (64 MB pre-allocated segments, chained CRC32, 10 ms fsync via dedicated pthread) feeds a periodic checkpoint (dual-zarray COW freeze/thaw, Cap'n Proto snapshot). Recovery loads the latest snapshot then replays the bridging WAL. Hooks fire AFTER_EVENT mutations; `xce_reply()` follows hooks so the client gets a durable ACK. + +**Tech Stack:** C11, Cap'n Proto c-capnproto, meson+ninja, nix develop shell + +**Design doc:** `src/docs/persistence-and-ha-design.md` — Parts 1–4 + Appendices A.1–A.5 + A.10 + +**Build commands (all run inside `nix develop`):** +```bash +meson setup build # first time only +ninja -C build # incremental build +meson test -C build # all tests +meson test -C build -t 0 --suite controller # controller suite only +meson test -C build -t 0 --suite lib # lib suite only +``` + +--- + +## Sub-phase dependency order + +``` +P0-A (cleanup + prerequisites) + └─► P0-B (WAL + COW infrastructure) + └─► P0-C (checkpoint + recovery) + ├─► P0-D (hooks + reply ordering) + └─► P0-E (deferred cleanup + GC) + └─► P0-F (schema + config + cluster wiring) + └─► P0-G (end-to-end smoke tests) +``` + +--- + +## Task 1 — P0-A: Dead Code Cleanup + +**Reads:** Design doc Part 1. + +**Files:** +- Modify: `src/controller/job.c` +- Modify: `src/controller/worker.c` +- Modify: `src/controller/cluster.c` + +- [ ] **Step 1.1 — Remove dead `job.c` constants** + + Open `src/controller/job.c`. Remove: + ```c + #define NJOBS (100) // line ~18 + uint64_t jobid = 100; // line ~20 + ``` + Remove the `#if 0 ... #endif` block inside `load_jobs()` (~lines 63–89). Keep the `load_jobs()` function signature — it will be rewired in P0-C. + +- [ ] **Step 1.2 — Add cluster_build_groups() comment** + + In `src/controller/cluster.c`, find `cluster_build_groups()` (empty body returning 0). Add: + ```c + /* TODO: implement group building */ + ``` + +- [ ] **Step 1.3 — Build and verify no regressions** + + ```bash + ninja -C build 2>&1 | tail -5 + meson test -C build 2>&1 | tail -20 + ``` + Expected: build green, test suite unchanged. + +- [ ] **Step 1.4 — Commit** + + ```bash + git add src/controller/job.c src/controller/worker.c src/controller/cluster.c + git commit -m "chore: remove dead code from job.c/worker.c/cluster.c (Part 1)" + ``` + +--- + +## Task 2 — P0-A: augment `include/config.h` with system constants + +**Reads:** Design doc Appendix A.10. + +**Files:** +- Modify: `src/include/config.h` + +- [ ] **Step 2.1 — Write a failing test for the new constants** + + In `src/lib/testsuite/test_config.c` (create if it doesn't exist), add: + ```c + #include "config.h" + #include + + void test_persistence_constants(void) { + assert(MAX_STANDBYS == 3); + assert(MAX_HA_CANDIDATES == 4); + assert(WAL_FORMAT_VERSION == 1); + assert(WAL_ENTRY_TYPE_MAX == 100); + assert(MAX_WAL_PAYLOAD_BYTES == (1 << 20)); + assert(MAX_SNAPSHOT_BYTES == (256 * 1024 * 1024)); + assert(DEFAULT_WAL_SYNC_MS == 10); + assert(DEFAULT_SNAP_INTERVAL_S == 60); + assert(DEFAULT_SNAP_WAL_SIZE_MB == 64); + assert(DEFAULT_SNAP_KEEP_N == 2); + assert(DEFAULT_FINISHED_RETAIN_S == 3600); + assert(DEFAULT_WORKER_RETAIN_S == 86400); + assert(DEFAULT_SYNC_TIMEOUT_MS == 50); + assert(DEFAULT_HB_INTERVAL_MS == 3000); + assert(WAL_HDR_CRC_SIZE == 32); + } + ``` + + Run: `meson test -C build -t 0 --suite lib` + Expected: **FAIL** — constants not yet defined. + +- [ ] **Step 2.2 — Add constants to `include/config.h`** + + Append to `src/include/config.h` (after existing content): + ```c + /* --- Persistence & HA tunable constants (Part 2.6, Appendix A.10) --- */ + #define MAX_STANDBYS 3 + #define MAX_HA_CANDIDATES 4 + #define WAL_FORMAT_VERSION 1 + #define WAL_ENTRY_TYPE_MAX 100 /* WAL_CHECKPOINT = 100, highest known type */ + #define MAX_WAL_PAYLOAD_BYTES (1 << 20) /* 1 MB */ + #define MAX_SNAPSHOT_BYTES (256 * 1024 * 1024) /* 256 MB */ + #define WAL_HDR_CRC_SIZE 32 /* sizeof(wal_entry_hdr_t) - 2*sizeof(uint32_t) */ + #define WAL_CRC_NONE 0xFFFFFFFFu + + #define DEFAULT_WAL_SYNC_MS 10 + #define DEFAULT_SNAP_INTERVAL_S 60 + #define DEFAULT_SNAP_WAL_SIZE_MB 64 + #define DEFAULT_SNAP_KEEP_N 2 + #define DEFAULT_FINISHED_RETAIN_S 3600 + #define DEFAULT_WORKER_RETAIN_S 86400 + #define DEFAULT_SYNC_TIMEOUT_MS 50 + #define DEFAULT_HB_INTERVAL_MS 3000 + #define DEFAULT_TRANSIENT_JOB_TIMEOUT_S 300 + ``` + +- [ ] **Step 2.3 — Run test to verify it passes** + + ```bash + meson test -C build -t 0 --suite lib + ``` + Expected: **PASS**. + +- [ ] **Step 2.4 — Commit** + + ```bash + git add src/include/config.h src/lib/testsuite/test_config.c + git commit -m "feat: add persistence/HA system constants to config.h (Appendix A.10)" + ``` + +--- + +## Task 3 — P0-A: rtti + zarray prerequisites (P0-0a + P0-0b) + +**Reads:** Design doc Part 3.4, Appendix A.2, P0-0a, P0-0b. + +**IMPORTANT:** P0-0a and P0-0b must land on the same branch. Do NOT merge P0-0b before P0-0a. + +**Files:** +- Modify: `src/lib/zarray.c`, `src/include/zarray.h` +- Modify: `rtti/typeinfo.h` (via nix flake source) +- Modify: `src/lib/kvmgr.c`, `src/lib/kv.c` + +- [ ] **Step 3.1 — Check rtti version and existing freeze/thaw types** + + ```bash + grep vfreeze_fn_t $(nix eval --raw .#rtti 2>/dev/null)/include/rtti/typeinfo.h 2>/dev/null \ + && echo "ALREADY PRESENT" || echo "NEEDS ADDITION" + ``` + If **ALREADY PRESENT**: skip to Step 3.3. + If **NEEDS ADDITION**: continue with Step 3.2. + +- [ ] **Step 3.2 — Add freeze/thaw types to rtti typeinfo.h** + + Locate the rtti source via `nix flake show` or `flake.nix` inputs. In `rtti/typeinfo.h`, add after the existing varray function pointer typedefs: + ```c + typedef bool (*retain_fn_t)(void* obj, void* arg); + typedef int (*vfreeze_fn_t)(meta_obj_t* obj); + typedef int (*vthaw_fn_t)(meta_obj_t* obj, retain_fn_t should_retain, void* arg); + ``` + + In the `intf_varray_t` struct, add two new slots: + ```c + typedef struct { + vstore_fn_t vstore; + vget_fn_t vget; + vget_vec_fn_t vget_vec; + vput_fn_t vput; + vput_vec_fn_t vput_vec; + vclear_fn_t vclear; + vfreeze_fn_t vfreeze; /* NEW */ + vthaw_fn_t vthaw; /* NEW */ + } intf_varray_t; + ``` + + Add global wrapper declarations to `typeinfo.h`: + ```c + int vfreeze(meta_obj_t* obj); + int vthaw(meta_obj_t* obj, retain_fn_t should_retain, void* arg); + ``` + + Bump nix derivation version from `rtti-0.0.1` to `rtti-0.1.0` in `flake.nix`. + +- [ ] **Step 3.3 — Convert all intf_varray_t initializers to designated style** + + Find all initializer sites: + ```bash + grep -rn 'intf_varray_t' src/ | grep -v '\.h:' + ``` + + For each initializer in `zarray.c`, `kvmgr.c`, `kv.c`, convert from positional to designated style: + ```c + /* BEFORE (positional — will break when struct grows): */ + static intf_varray_t xzarray_iface = { xzarray_store, xzarray_get, ... }; + + /* AFTER (designated — safe to extend): */ + static intf_varray_t xzarray_iface = { + .vstore = xzarray_store, + .vget = xzarray_get, + .vget_vec = xzarray_get_vec, + .vput = xzarray_put, + .vput_vec = xzarray_put_vec, + .vclear = xzarray_clear, + .vfreeze = NULL, /* filled in P0-2 */ + .vthaw = NULL, /* filled in P0-2 */ + }; + ``` + + ```bash + ninja -C build 2>&1 | tail -10 + ``` + Expected: build green (NULL slots are valid — not wired yet). + +- [ ] **Step 3.4 — Write failing test for freeze/thaw interface presence** + + In `src/lib/testsuite/test_zarray_cow.c` (new file): + ```c + #include "zarray.h" + #include "rtti/typeinfo.h" + #include + + extern intf_varray_t xzarray_type; + + void test_freeze_thaw_slots_exist(void) { + /* After P0-2 these will be non-NULL; for now just verify the struct has the slots */ + (void)xzarray_type.vfreeze; + (void)xzarray_type.vthaw; + } + ``` + + ```bash + meson test -C build -t 0 --suite lib + ``` + Expected: **PASS** (slots exist, values NULL — presence test only). + +- [ ] **Step 3.5 — Full rebuild to catch ABI breaks** + + ```bash + rm -rf build && meson setup build && ninja -C build + meson test -C build 2>&1 | tail -20 + ``` + Expected: clean build, all tests pass. + +- [ ] **Step 3.6 — Commit P0-A prerequisites** + + ```bash + git add src/lib/zarray.c src/include/zarray.h src/lib/kvmgr.c src/lib/kv.c \ + src/lib/testsuite/test_zarray_cow.c flake.nix + git commit -m "feat: add vfreeze/vthaw slots to intf_varray_t; convert initializers to designated style (P0-0b)" + ``` + +--- + +## Task 4 — P0-A: delaymgr precision enhancement (P0-2b) + +**Reads:** Design doc P0-2b, Part 3.10 note on delaymgr. + +**Files:** +- Modify: `src/lib/delay.c`, `src/include/delay.h` +- Modify: `src/controller/cluster.c` (delaymgr_init call) + +- [ ] **Step 4.1 — Write failing test for sub-second precision** + + In `src/lib/testsuite/test_delay.c`, add: + ```c + #include "delay.h" + #include + #include + + void test_delaymgr_precision_ms(void) { + delaymgr_t mgr; + /* precision_ms field must exist on the struct */ + delaymgr_init(&mgr, 200 /* precision_ms */, NULL); + assert(mgr.precision_ms == 200); + delaymgr_fini(&mgr); + } + ``` + + ```bash + meson test -C build -t 0 --suite lib + ``` + Expected: **FAIL** — `precision_ms` field doesn't exist yet. + +- [ ] **Step 4.2 — Rename `precision` → `precision_ms` in delay.h** + + In `src/include/delay.h`, in `delaymgr_t`, rename `int precision` → `int precision_ms`. + +- [ ] **Step 4.3 — Replace `sleep()` with `nanosleep()` in delay.c** + + In `src/lib/delay.c`, in `delay_worker()`, replace: + ```c + sleep(mgr->precision); + ``` + with: + ```c + struct timespec ts = { 0, (long)mgr->precision_ms * 1000000L }; + nanosleep(&ts, NULL); + ``` + + Update `delaymgr_init()` to store the parameter as `precision_ms`: + ```c + int delaymgr_init(delaymgr_t* mgr, int precision_ms, task_run_fn_t runner_fn) { + if ((mgr == NULL) || (precision_ms <= 0)) return -EINVAL; + ... + mgr->precision_ms = precision_ms; + ... + } + ``` + +- [ ] **Step 4.4 — Update cluster.c delaymgr_init call** + + Find `delaymgr_init(&c->delaymgr, 2, run_delay_tasks)` in `src/controller/cluster.c` and change to: + ```c + delaymgr_init(&c->delaymgr, 200 /* precision_ms */, run_delay_tasks); + ``` + +- [ ] **Step 4.5 — Audit all other delaymgr_init call sites** + + ```bash + grep -rn 'delaymgr_init' src/ + ``` + For each: update second argument to `200` (milliseconds). The delay values passed to `delay_task()` remain in seconds — only the sleep granularity changes. + +- [ ] **Step 4.6 — Run tests** + + ```bash + ninja -C build && meson test -C build -t 0 --suite lib + ``` + Expected: **PASS**. + +- [ ] **Step 4.7 — Commit** + + ```bash + git add src/lib/delay.c src/include/delay.h src/controller/cluster.c + git commit -m "feat: rename delaymgr precision→precision_ms, replace sleep with nanosleep (P0-2b)" + ``` + +--- + +## Task 5 — P0-B: WAL types and entry format (headers) + +**Reads:** Design doc Parts 2.2, 2.3, 2.6, Appendix A.1. + +**Files:** +- Create: `src/include/persist.h` +- Modify: `src/include/config.h` (WAL_HDR_CRC_SIZE already added in Task 2) + +- [ ] **Step 5.1 — Write failing compile test for wal_entry_hdr_t size** + + In `src/controller/testsuite/test_persist_types.c` (new file): + ```c + #include "persist.h" + #include + #include + + void test_wal_header_size(void) { + static_assert(sizeof(wal_entry_hdr_t) == 40, + "wal_entry_hdr_t must be 40 bytes"); + } + + void test_wal_entry_types_defined(void) { + assert(WAL_JOB_SUBMIT == 1); + assert(WAL_JOB_START == 2); + assert(WAL_CHECKPOINT == 100); + } + + void test_persist_config_defaults(void) { + persist_config_t cfg = persist_config_defaults(); + assert(cfg.wal_sync_ms == DEFAULT_WAL_SYNC_MS); + assert(cfg.snap_interval_s == DEFAULT_SNAP_INTERVAL_S); + assert(cfg.snap_keep_n == DEFAULT_SNAP_KEEP_N); + assert(cfg.finished_retain_s == DEFAULT_FINISHED_RETAIN_S); + assert(cfg.sync_timeout_ms == DEFAULT_SYNC_TIMEOUT_MS); + } + ``` + + ```bash + ninja -C build 2>&1 | grep "persist.h: No such" + ``` + Expected: compile error — `persist.h` doesn't exist yet. + +- [ ] **Step 5.2 — Create `src/include/persist.h`** + + ```c + #pragma once + #include + #include + #include + #include + #include "config.h" + #include "mylock.h" + + /* ---- WAL entry header (40 bytes, 8-byte aligned, Part 2.2) ---- */ + typedef struct { + uint64_t term; /* Election term */ + uint64_t lsn; /* Log sequence number */ + uint64_t timestamp; /* Epoch microseconds */ + uint32_t type; /* wal_entry_type */ + uint32_t payload_len; + uint32_t crc; /* CRC32(header[0..WAL_HDR_CRC_SIZE-1] + payload) */ + uint32_t prev_crc; /* Previous record CRC; WAL_CRC_NONE for first in segment */ + } wal_entry_hdr_t; + static_assert(sizeof(wal_entry_hdr_t) == 40, "wal_entry_hdr_t must be 40 bytes"); + + /* ---- WAL entry types (Part 2.3) ---- */ + enum wal_entry_type { + WAL_JOB_SUBMIT = 1, + WAL_JOB_START = 2, + WAL_JOB_FINISH = 3, + WAL_JOB_FAIL = 4, + WAL_JOB_STOP = 5, + WAL_JOB_RESUME = 6, + WAL_JOB_UPDATE = 7, + WAL_JOB_RUSAGE = 8, + WAL_JOB_CKPT = 9, + WAL_JOB_PREASON = 10, + WAL_QUEUE_CTRL = 20, + WAL_WORKER_CTRL = 30, + WAL_CHECKPOINT = 100, + }; + + /* ---- Persistence configuration (Part 2.6) ---- */ + typedef struct { + uint32_t wal_sync_ms; + uint32_t snap_interval_s; + uint32_t snap_wal_size_mb; + uint32_t snap_keep_n; + uint32_t transient_job_timeout_s; + uint32_t finished_retain_s; + uint32_t worker_retain_s; + uint32_t sync_timeout_ms; + } persist_config_t; + + static inline persist_config_t persist_config_defaults(void) { + return (persist_config_t){ + .wal_sync_ms = DEFAULT_WAL_SYNC_MS, + .snap_interval_s = DEFAULT_SNAP_INTERVAL_S, + .snap_wal_size_mb = DEFAULT_SNAP_WAL_SIZE_MB, + .snap_keep_n = DEFAULT_SNAP_KEEP_N, + .transient_job_timeout_s = DEFAULT_TRANSIENT_JOB_TIMEOUT_S, + .finished_retain_s = DEFAULT_FINISHED_RETAIN_S, + .worker_retain_s = DEFAULT_WORKER_RETAIN_S, + .sync_timeout_ms = DEFAULT_SYNC_TIMEOUT_MS, + }; + } + + /* ---- Key→index mapping for checkpoint serialization (Part 3.8, Appendix A.1) ---- */ + typedef struct { + int index; + char key[XCE_MAXNAME_LEN]; /* owned deep copy */ + size_t key_len; + } slot_key_t; + + /* ---- Forward declarations (full persist_t defined in persist.c) ---- */ + struct cluster_s; + typedef struct persist_s persist_t; + + int persist_init(persist_t* p, persist_config_t* config, struct cluster_s* c); + int persist_start(persist_t* p); + void persist_stop(persist_t* p); + void persist_fini(persist_t* p); + int persist_recover(persist_t* p); + + int persist_wal_async(persist_t* p, uint32_t type, const void* payload, uint32_t len); + int persist_wal_sync_append(persist_t* p, uint32_t type, + const void* payload, uint32_t len, int* sync_rc_out); + int persist_checkpoint(persist_t* p); + + void persist_log_entry_metadata(const wal_entry_hdr_t* hdr); + ``` + +- [ ] **Step 5.3 — Run test** + + ```bash + ninja -C build && meson test -C build -t 0 --suite controller + ``` + Expected: **PASS** (types and size check). + +- [ ] **Step 5.4 — Commit** + + ```bash + git add src/include/persist.h src/controller/testsuite/test_persist_types.c + git commit -m "feat: add persist.h with WAL types, config struct, slot_key_t (P0-B)" + ``` + +--- + +## Task 6 — P0-B: zarray COW extensions (P0-2) + +**Reads:** Design doc Parts 3.3–3.6, Appendix A.2. + +**Files:** +- Modify: `src/lib/zarray.c`, `src/include/zarray.h` + +- [ ] **Step 6.1 — Write failing COW freeze/thaw test** + + In `src/lib/testsuite/test_zarray_cow.c`, extend with: + ```c + #include "zarray.h" + #include + #include + + void test_freeze_thaw_round_trip(void) { + /* Allocate an xzarray via RTTI (use existing makeInstance pattern) */ + meta_obj_t* arr = makeInstance(&xzarray_type, NULL); + assert(arr != NULL); + + /* Store 3 items */ + int idx[3]; + char* data[3] = { strdup("a"), strdup("b"), strdup("c") }; + for (int i = 0; i < 3; i++) { + assert(vstore(arr, data[i]) >= 0); + } + + /* Freeze */ + assert(vfreeze(arr) == 0); + + /* During COW: add one more item */ + char* d = strdup("d"); + assert(vstore(arr, d) >= 0); + + /* Thaw (no GC — retain all) */ + assert(vthaw(arr, NULL, NULL) == 0); + + /* After thaw: all 4 items reachable */ + int count = 0; + /* ... iterate and count ... */ + freeInstance(arr); + } + + void test_freeze_enomem_rollback(void) { + /* Simulate OOM at freeze: should restore original data pointer */ + /* This is a compile-time structure test — real OOM injection deferred */ + (void)0; + } + ``` + + ```bash + meson test -C build -t 0 --suite lib + ``` + Expected: **FAIL** — `vfreeze`/`vthaw` still NULL in xzarray_type. + +- [ ] **Step 6.2 — Add `xzarray_private_t` fields for shadow** + + In `src/lib/zarray.c`, in `xzarray_private_t`: + ```c + typedef struct { + mylock_t lock; + xzarray_item_t* shadow; /* NULL when not in COW mode */ + size_t shadow_caps; + } xzarray_private_t; + ``` + +- [ ] **Step 6.3 — Change `xzarray_item_t.inuse` to `_Atomic _Bool`** + + In `src/include/zarray.h` (or `zarray.c` if still file-private): + ```c + typedef struct { + _Atomic _Bool inuse; + void* data; + } xzarray_item_t; + ``` + + Run: + ```bash + grep -n '\.inuse' src/lib/zarray.c + ``` + For every read: use `atomic_load_explicit(&item->inuse, memory_order_acquire)`. + For every write: use `atomic_store_explicit(&item->inuse, true/false, memory_order_release)`. + +- [ ] **Step 6.4 — Implement `xzarray_freeze()`** + + In `src/lib/zarray.c`: + ```c + static int xzarray_freeze(meta_obj_t* obj) { + xzarray_private_t* priv = RTTI_OBJ_TO_PRIV(obj); + xzarray_t* a = RTTI_OBJ_TO_DATA(xzarray_t, obj); + + assert(priv->shadow == NULL && "xzarray_freeze called while already in COW"); + + /* Pre-grow active array to 1.5x to avoid realloc during COW (A6-026) */ + size_t new_caps = a->caps + (a->caps / 2) + 16; + xzarray_item_t* active = calloc(new_caps, sizeof(xzarray_item_t)); + if (!active) return -ENOMEM; + + priv->shadow = a->data; + priv->shadow_caps = a->caps; + a->data = active; + a->caps = new_caps; + a->count = 0; + a->pos = priv->shadow_caps; /* start past shadow to avoid index collisions */ + return 0; + } + ``` + +- [ ] **Step 6.5 — Implement `xzarray_thaw()`** + + ```c + static int xzarray_thaw(meta_obj_t* obj, retain_fn_t should_retain, void* arg) { + xzarray_private_t* priv = RTTI_OBJ_TO_PRIV(obj); + xzarray_t* a = RTTI_OBJ_TO_DATA(xzarray_t, obj); + + assert(priv->shadow != NULL && "xzarray_thaw called without freeze"); + + /* Merge shadow into active: copy surviving shadow entries to active */ + for (size_t i = 0; i < priv->shadow_caps; i++) { + if (!atomic_load_explicit(&priv->shadow[i].inuse, memory_order_relaxed)) + continue; + void* item = priv->shadow[i].data; + if (should_retain && !should_retain(item, arg)) + continue; /* GC: do not copy */ + /* Write to active at the same index (kvtab maps to this index) */ + if (i >= a->caps) { + /* Active was pre-grown — this shouldn't happen, but guard anyway */ + continue; + } + if (!atomic_load_explicit(&a->data[i].inuse, memory_order_relaxed)) { + a->data[i].data = item; + atomic_store_explicit(&a->data[i].inuse, true, memory_order_relaxed); + a->count++; + } + /* If active[i].inuse==true, a mutation during COW already wrote new data — keep it */ + } + + free(priv->shadow); + priv->shadow = NULL; + priv->shadow_caps = 0; + a->pos = 0; + return 0; + } + ``` + +- [ ] **Step 6.6 — Add `vclear` COW guard** + + ```c + static void xzarray_clear(meta_obj_t* obj) { + xzarray_private_t* priv = RTTI_OBJ_TO_PRIV(obj); + assert(priv->shadow == NULL && "xzarray_clear called during COW — use deferred cleanup"); + /* ... existing clear logic ... */ + } + ``` + +- [ ] **Step 6.7 — Update `xzarray_type` registration with freeze/thaw** + + In the `intf_varray_t xzarray_iface` designated initializer: + ```c + .vfreeze = xzarray_freeze, + .vthaw = xzarray_thaw, + ``` + +- [ ] **Step 6.8 — Run tests** + + ```bash + ninja -C build && meson test -C build -t 0 --suite lib + ``` + Expected: **PASS**. + +- [ ] **Step 6.9 — Commit** + + ```bash + git add src/lib/zarray.c src/include/zarray.h src/lib/testsuite/test_zarray_cow.c + git commit -m "feat: implement xzarray COW freeze/thaw with shadow array (P0-2)" + ``` + +--- + +## Task 7 — P0-B: WAL writer + fsync thread (P0-1) + +**Reads:** Design doc Parts 2.2–2.4, Appendix A.1. + +**Files:** +- Create: `src/controller/persist.c` +- Modify: `src/controller/meson.build` + +- [ ] **Step 7.1 — Write failing WAL write test** + + In `src/controller/testsuite/test_persist_wal.c` (new file): + ```c + #include "persist.h" + #include + #include + #include + + void test_wal_write_and_read_back(void) { + persist_t p; + persist_config_t cfg = persist_config_defaults(); + /* Use a temp dir so no real cluster is needed */ + char workdir[] = "/tmp/xce_test_XXXXXX"; + assert(mkdtemp(workdir) != NULL); + + /* persist_init with a stub cluster pointer */ + assert(persist_init(&p, &cfg, NULL) == 0); + + /* Write a WAL entry */ + const char payload[] = "test_payload"; + assert(persist_wal_async(&p, WAL_JOB_UPDATE, payload, sizeof(payload)) == 0); + + /* WAL file should exist in workdir/data/wal/ */ + char wal_path[256]; + snprintf(wal_path, sizeof(wal_path), "%s/data/wal", workdir); + /* verify directory exists */ + assert(access(wal_path, F_OK) == 0); + + persist_stop(&p); + persist_fini(&p); + /* cleanup workdir ... */ + } + ``` + + ```bash + ninja -C build 2>&1 | grep "persist.c" + ``` + Expected: link error — `persist.c` doesn't exist. + +- [ ] **Step 7.2 — Create `src/controller/persist.c` with WAL writer skeleton** + + ```c + #include "persist.h" + #include "config.h" + #include + #include + #include + #include + #include + #include + #include + #include + #include + #include + #include + #include + #include /* for crc32() */ + + struct persist_s { + int wal_fd; + uint64_t current_lsn; + uint64_t last_crc; + uint64_t term; + char seg_path[4096]; + uint32_t seg_offset; + uint64_t checkpoint_lsn; + uint64_t total_wal_bytes; + uint32_t last_replicated_crc; + persist_config_t config; + mylock_t lock; + atomic_int checkpoint_in_progress; + atomic_int checkpoint_requested; + pthread_cond_t checkpoint_done; + pthread_mutex_t checkpoint_mutex; + pthread_t snap_thread; + pthread_t fsync_thread; + _Atomic int fsync_stop; + char wal_path[4096]; + char snap_path[4096]; + int wal_dirfd; + int snap_dirfd; + }; + ``` + + Add `persist_init()`: + ```c + int persist_init(persist_t* p, persist_config_t* cfg, struct cluster_s* c) { + memset(p, 0, sizeof(*p)); + p->config = cfg ? *cfg : persist_config_defaults(); + p->wal_fd = -1; + + /* Validate paths are not on NFS (Part 2.4 filesystem requirement) */ + /* ... statfs() check for NFS_SUPER_MAGIC = 0x6969 ... */ + + /* Create wal/ and snap/ directories at mode 0700 */ + /* For now: caller sets wal_path/snap_path before calling persist_init */ + + mylock_init(&p->lock); + pthread_mutex_init(&p->checkpoint_mutex, NULL); + pthread_cond_init(&p->checkpoint_done, NULL); + return 0; + } + ``` + + Add `persist_wal_async()` stub: + ```c + int persist_wal_async(persist_t* p, uint32_t type, const void* payload, uint32_t len) { + assert(type != WAL_CHECKPOINT && "persist_wal_async must not be called with WAL_CHECKPOINT"); + /* TODO P0-1: write wal_entry_hdr_t + payload to WAL, update last_crc */ + (void)p; (void)type; (void)payload; (void)len; + return 0; + } + ``` + + Add remaining stubs (`persist_start`, `persist_stop`, `persist_fini`, `persist_recover`, `persist_wal_sync_append`, `persist_checkpoint`, `persist_log_entry_metadata`) that return 0/do nothing. + +- [ ] **Step 7.3 — Add persist.c to meson.build** + + In `src/controller/meson.build`, add `persist.c` to the controller source list. + +- [ ] **Step 7.4 — Implement WAL segment init and entry write** + + Full implementation of `persist_wal_async()`: + 1. Acquire `p->lock` + 2. If `p->wal_fd == -1`, open first segment (`seg-0000000000000000.wal`) with `O_CREAT|O_WRONLY|O_APPEND`, pre-allocate 64 MB with `posix_fallocate()` + 3. Fill `wal_entry_hdr_t`: `term = p->term`, `lsn = p->current_lsn++`, `timestamp = now_us()`, `type`, `payload_len = len` + 4. Compute `crc = crc32(0, header_bytes[0..WAL_HDR_CRC_SIZE-1]) then update with payload` + 5. Set `prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc` + 6. `pwrite(p->wal_fd, &hdr, 40, p->seg_offset)` + `pwrite(payload, len, ...)` + 7. Update `p->last_crc`, `p->seg_offset`, `p->total_wal_bytes` + 8. Segment rotation when `seg_offset >= 64 MB` + 9. Release lock + +- [ ] **Step 7.5 — Implement dedicated fsync pthread** + + ```c + static void* fsync_worker(void* arg) { + persist_t* p = arg; + struct timespec ts = { 0, (long)p->config.wal_sync_ms * 1000000L }; + while (!atomic_load_explicit(&p->fsync_stop, memory_order_acquire)) { + nanosleep(&ts, NULL); + if (p->wal_fd >= 0) + fdatasync(p->wal_fd); + } + return NULL; + } + ``` + + Start in `persist_start()`, join in `persist_stop()`. + +- [ ] **Step 7.6 — Run tests** + + ```bash + ninja -C build && meson test -C build -t 0 --suite controller + ``` + Expected: **PASS**. + +- [ ] **Step 7.7 — Commit** + + ```bash + git add src/controller/persist.c src/controller/meson.build \ + src/controller/testsuite/test_persist_wal.c + git commit -m "feat: WAL writer with segment management and dedicated fsync pthread (P0-1)" + ``` + +--- + +## Task 8 — P0-C: Cap'n Proto schemas (P0-9) + +**Reads:** Design doc Part 2.2 (payload table), Appendix A.5. + +**Files:** +- Modify: `src/schema/controller.capnp` + +- [ ] **Step 8.1 — Write failing compile test for schema types** + + In `src/controller/testsuite/test_persist_schema.c`: + ```c + /* Verify Cap'n Proto generated types are accessible */ + #include "controller.capnp.h" + #include + + void test_persist_config_schema(void) { + /* PersistConfig must exist with correct field count */ + struct PersistConfig cfg; + cfg.walSyncMs = 10; + cfg.snapIntervalS = 60; + cfg.snapWalSizeMb = 64; + cfg.snapKeepN = 2; + cfg.transientJobTimeoutS = 300; + assert(cfg.walSyncMs == 10); + } + ``` + + ```bash + ninja -C build 2>&1 | grep "PersistConfig" + ``` + Expected: compile error — `PersistConfig` not in schema yet. + +- [ ] **Step 8.2 — Add PersistConfig to controller.capnp** + + Append to `src/schema/controller.capnp`: + ```capnp + struct PersistConfig $C.mapname("persist_config_t") { + walSyncMs @0 :UInt32 $C.mapname("wal_sync_ms") = 10; + snapIntervalS @1 :UInt32 $C.mapname("snap_interval_s") = 60; + snapWalSizeMb @2 :UInt32 $C.mapname("snap_wal_size_mb") = 64; + finishedRetainS @3 :UInt32 $C.mapname("finished_retain_s") = 3600; + workerRetainS @4 :UInt32 $C.mapname("worker_retain_s") = 86400; + syncTimeoutMs @5 :UInt32 $C.mapname("sync_timeout_ms") = 50; + snapKeepN @6 :UInt32 $C.mapname("snap_keep_n") = 2; + transientJobTimeoutS @7 :UInt32 $C.mapname("transient_job_timeout_s") = 300; + } + + struct SnapshotHeader { + lsn @0 :UInt64; + term @1 :UInt64; + timestamp @2 :UInt64; + version @3 :UInt32 = 1; + nJobs @4 :UInt32; + nQueues @5 :UInt32; + nWorkers @6 :UInt32; + checksum @7 :UInt32; # CRC32 of Cap'n Proto payload + headerChecksum @8 :UInt32; # CRC32(lsn, term, timestamp, version, nJobs, nQueues, nWorkers) + } + + struct WALEntry { + # Replication-only envelope (NOT on-disk format). On-disk uses raw wal_entry_hdr_t. + term @0 :UInt64; + lsn @1 :UInt64; + entryType @2 :UInt32; + payload @3 :Data; + crc @4 :UInt32; + prevCrc @5 :UInt32; + } + ``` + + Also add `transientSince @N :UInt64` to the `Job` struct and `deadSince` to `Worker` struct, and `jobStatTransient = 0x400` comment. + +- [ ] **Step 8.3 — Rebuild and run test** + + ```bash + ninja -C build && meson test -C build -t 0 --suite controller + ``` + Expected: **PASS**. + +- [ ] **Step 8.4 — Commit** + + ```bash + git add src/schema/controller.capnp src/controller/testsuite/test_persist_schema.c + git commit -m "feat: add PersistConfig, SnapshotHeader, WALEntry Cap'n Proto schemas (P0-9)" + ``` + +--- + +## Task 9 — P0-C: kvmgr key capture (P0-14 + P0-14a) + +**Reads:** Design doc Part 3.8, Appendix A.1 `kvmgr_capture_keys`. + +**Files:** +- Modify: `src/lib/kvmgr.c`, `src/lib/include/kvmgr.h` + +- [ ] **Step 9.1 — Write failing test for kvmgr_capture_keys** + + In `src/lib/testsuite/test_kvmgr_capture.c`: + ```c + #include "kvmgr.h" + #include "persist.h" + #include + #include + + void test_capture_keys_round_trip(void) { + meta_obj_t* km = makeKvmgr(KVMGR_KEY_STRING, NULL); + assert(km != NULL); + + /* Add 3 items */ + kvmgr_set(km, "alpha", strdup("val_a"), NULL); + kvmgr_set(km, "beta", strdup("val_b"), NULL); + kvmgr_set(km, "gamma", strdup("val_c"), NULL); + + slot_key_t* keys = NULL; + size_t n = 0; + /* Must hold write lock before calling */ + assert(kvmgr_capture_keys(km, &keys, &n) == 0); + assert(n == 3); + + /* All keys are owned copies */ + int found = 0; + for (size_t i = 0; i < n; i++) { + if (strcmp(keys[i].key, "alpha") == 0 || + strcmp(keys[i].key, "beta") == 0 || + strcmp(keys[i].key, "gamma") == 0) found++; + } + assert(found == 3); + free(keys); + freeInstance(km); + } + ``` + + ```bash + meson test -C build -t 0 --suite lib + ``` + Expected: **FAIL** — `kvmgr_capture_keys` undefined. + +- [ ] **Step 9.2 — Implement `kvmgr_capture_keys` in kvmgr.c** + + ```c + int kvmgr_capture_keys(meta_obj_t* kvmgr, slot_key_t** keys_out, size_t* n_keys_out) { + *keys_out = NULL; + *n_keys_out = 0; + + /* Count live entries */ + size_t n = kvmgr_count(kvmgr); + if (n == 0) return 0; + + slot_key_t* keys = malloc(n * sizeof(slot_key_t)); + if (!keys) return -ENOMEM; + + size_t filled = 0; + /* Walk the kvtab: iterate hash chains, extract key and index for each live slot */ + /* Use the htab walk primitive (P0-14 — implement kvtab_foreach if not present) */ + kvtab_foreach(kvmgr, keys, &filled); + + *keys_out = keys; + *n_keys_out = filled; + return 0; + } + ``` + + Implement `kvtab_foreach()` in `src/lib/htab.c` or inline in `kvmgr.c` — walks hash chains and for each live entry calls: + ```c + strncpy(dst->key, entry->key, XCE_MAXNAME_LEN - 1); + dst->key[XCE_MAXNAME_LEN - 1] = '\0'; + dst->key_len = strlen(entry->key) + 1; + dst->index = entry->index; + ``` + +- [ ] **Step 9.3 — Run test** + + ```bash + ninja -C build && meson test -C build -t 0 --suite lib + ``` + Expected: **PASS**. + +- [ ] **Step 9.4 — Commit** + + ```bash + git add src/lib/kvmgr.c src/lib/htab.c src/lib/testsuite/test_kvmgr_capture.c + git commit -m "feat: implement kvmgr_capture_keys for lock-free checkpoint serialization (P0-14a)" + ``` + +--- + +## Task 10 — P0-C: checkpoint (P0-3) + +**Reads:** Design doc Parts 3.1–3.11, 2.5. + +**Files:** +- Modify: `src/controller/persist.c` + +- [ ] **Step 10.1 — Write failing checkpoint test** + + In `src/controller/testsuite/test_persist_checkpoint.c`: + ```c + #include "persist.h" + #include + + void test_checkpoint_creates_snapshot_file(void) { + /* Setup persist_t with a temp dir, add some WAL entries, trigger checkpoint */ + /* Verify snap/ directory contains a .capnp file */ + /* ... setup boilerplate ... */ + assert(persist_checkpoint(&p) == 0); + /* count files in snap_path */ + /* assert at least one snapshot file exists */ + } + + void test_freeze_thaw_no_data_loss(void) { + /* Write 100 jobs to kvmgr, freeze, write 10 more during COW, thaw */ + /* Verify all 110 jobs are reachable after thaw */ + } + ``` + + Expected: **FAIL**. + +- [ ] **Step 10.2 — Implement `persist_checkpoint()` in persist.c** + + Sequence (Part 3.8 + 3.10): + 1. Set `checkpoint_in_progress = 1` + 2. Acquire all kvmgr write locks (jobmgr, workermgr, queuemgr, resmgr) + 3. `vfreeze()` on each data zarray + 4. `kvmgr_capture_keys()` on each kvmgr (still under write lock) + 5. Write `WAL_CHECKPOINT` marker via `persist_write_checkpoint_marker()` (local only, no `repl_broadcast`) + 6. Release all write locks + 7. Spawn serialization pthread: serialize shadow arrays to Cap'n Proto → write to `snap-.tmp` → CRC32 → `rename()` to `snap-.capnp` + 8. Serialization pthread calls `vthaw()` on each zarray after `fsync(snap_fd)` completes + 9. Signal `checkpoint_done` cond, set `checkpoint_in_progress = 0` + 10. Trim old WAL segments up to `checkpoint_lsn` (subject to trim guard) + 11. Rotate old snapshots — keep only `snap_keep_n` most recent + +- [ ] **Step 10.3 — Register checkpoint timer via delaymgr** + + In `persist_start()`: + ```c + delay_task(&c->delaymgr, 1 /* seconds */, -1 /* repeat */, persist_checkpoint_timer_fn, p); + ``` + + `persist_checkpoint_timer_fn()` checks: + 1. If `checkpoint_in_progress != 0` → skip + 2. If `checkpoint_requested != 0` → run checkpoint, clear flag + 3. If periodic countdown reached zero → run checkpoint, reset countdown + +- [ ] **Step 10.4 — Run tests** + + ```bash + ninja -C build && meson test -C build -t 0 --suite controller + ``` + Expected: **PASS**. + +- [ ] **Step 10.5 — Commit** + + ```bash + git add src/controller/persist.c src/controller/testsuite/test_persist_checkpoint.c + git commit -m "feat: implement dual-zarray COW checkpoint with serialization pthread (P0-3)" + ``` + +--- + +## Task 11 — P0-C: recovery (P0-4 + P0-12 + P0-12a) + +**Reads:** Design doc Part 2.5, P0-4, P0-12, P0-12a. + +**Files:** +- Modify: `src/controller/persist.c` +- Modify: `src/controller/worker.c` +- Modify: `src/controller/job.c` + +- [ ] **Step 11.1 — Write failing recovery round-trip test** + + In `src/controller/testsuite/test_persist_recovery.c`: + ```c + void test_recovery_roundtrip(void) { + /* 1. Init persist, write 50 WAL_JOB_SUBMIT entries + 1 checkpoint */ + /* 2. Call persist_fini (simulates shutdown) */ + /* 3. Re-init persist, call persist_recover() */ + /* 4. Verify jobmgr has 50 jobs */ + } + + void test_recovery_skips_expired_jobs(void) { + /* Write jobs with finish_time older than finished_retain_s */ + /* After recovery + retention pass, they should be gone */ + } + ``` + + Expected: **FAIL**. + +- [ ] **Step 11.2 — Implement `restore_job()` in job.c** + + ```c + int restore_job(cluster_t* c, uint64_t jobid, const void* capnp_payload, size_t len) { + /* Deserialize Job Cap'n Proto */ + /* Insert at original jobid — bypass allocator */ + /* Return -EEXIST if jobid already present (warn and skip) */ + /* Do NOT fire BEFORE_EVENT or AFTER_EVENT hooks */ + } + ``` + +- [ ] **Step 11.3 — Implement `worker_from_capnp()` adapter in worker.c** + + ```c + int worker_from_capnp(cluster_t* c, const void* capnp_payload, size_t len) { + xce_worker_t host = {0}; + /* Deserialize Worker Cap'n Proto into xce_worker_t */ + /* Deep-copy string fields (strdup name, copy resource arrays) */ + return restore_worker(c, host); + } + ``` + +- [ ] **Step 11.4 — Implement `persist_recover()` in persist.c** + + Full sequence (Part 2.5): + 1. Scan `snap_path` for snapshot files, pick highest LSN + 2. Verify `headerChecksum` then `checksum` — try next-oldest on failure + 3. Check `SnapshotHeader.version <= WAL_FORMAT_VERSION` — abort if higher + 4. Deserialize: jobs → `restore_job()`, workers → `worker_from_capnp()`, queues → `create_queue()` + 5. Open WAL segments from `snapshot.lsn + 1`, replay via `apply_wal_entry()` + - Skip unknown entry types with `WARN`, but abort if `type > WAL_ENTRY_TYPE_MAX` + 6. Retention pass: run GC check on all entities (TTL expired during downtime) + 7. Set `ha.term = max(snapshot_header.term, max_term_from_wal)` + 8. Trim WAL segments fully covered by snapshot LSN + +- [ ] **Step 11.5 — Run tests** + + ```bash + ninja -C build && meson test -C build -t 0 --suite controller + ``` + Expected: **PASS**. + +- [ ] **Step 11.6 — Commit** + + ```bash + git add src/controller/persist.c src/controller/job.c src/controller/worker.c \ + src/controller/testsuite/test_persist_recovery.c + git commit -m "feat: implement persist_recover() with snapshot+WAL replay and retention pass (P0-4)" + ``` + +--- + +## Task 12 — P0-D: hook detail enums + reply ordering (P0-4a + P0-7) + +**Reads:** Design doc Parts 2.3 footnotes 1–2, Part 8.3, P0-4a, P0-7. + +**Files:** +- Modify: `src/include/xce/cluster_def.h` +- Modify: `src/controller/job.c` +- Audit: all handler files for `xce_run_hooks(AFTER_EVENT` + `xce_reply` ordering + +- [ ] **Step 12.1 — Write failing test for new hook details** + + ```c + void test_job_stop_fires_correct_hook(void) { + /* Set up cluster with a mock hook that records detail */ + /* Call on_job_stop() */ + /* Assert HOOK_DETAIL_JOB_STOP was fired (not HOOK_DETAIL_JOB_UPDATE) */ + } + ``` + + Expected: **FAIL** — `HOOK_DETAIL_JOB_STOP` not defined. + +- [ ] **Step 12.2 — Add hook detail enums** + + In `src/include/xce/cluster_def.h`, add to the hook detail enum: + ```c + HOOK_DETAIL_JOB_STOP = , + HOOK_DETAIL_JOB_RESUME = , + ``` + +- [ ] **Step 12.3 — Update `on_job_stop()` and `on_job_resume()` in job.c** + + Find `on_job_stop()` — it currently calls `mark_job_update()`. Replace with direct hook call: + ```c + xce_run_hooks(c->hooks, XCE_HOOK_TYPE_AFTER_EVENT, HOOK_DETAIL_JOB_STOP, j); + ``` + Same for `on_job_resume()` → `HOOK_DETAIL_JOB_RESUME`. + + Verify with: + ```bash + grep -n 'mark_job_update\|HOOK_DETAIL_JOB' src/controller/job.c + ``` + +- [ ] **Step 12.4 — Audit reply-after-hooks ordering across all handlers** + + ```bash + grep -rn 'xce_run_hooks.*AFTER_EVENT\|xce_reply' src/controller/*.c + ``` + For each handler file: verify `xce_reply()` comes AFTER `xce_run_hooks(AFTER_EVENT)`. Fix any handler where reply precedes hooks. + +- [ ] **Step 12.5 — Run tests** + + ```bash + ninja -C build && meson test -C build -t 0 --suite controller + ``` + Expected: **PASS**. + +- [ ] **Step 12.6 — Commit** + + ```bash + git add src/include/xce/cluster_def.h src/controller/job.c + git commit -m "feat: add HOOK_DETAIL_JOB_STOP/RESUME; move xce_reply() after AFTER_EVENT hooks (P0-4a, P0-7)" + ``` + +--- + +## Task 13 — P0-D: persistence hook functions (P0-6) + +**Reads:** Design doc Parts 2.3, 8.1–8.3. + +**Files:** +- Modify: `src/controller/persist.c` + +- [ ] **Step 13.1 — Write failing hook invocation test** + + ```c + void test_job_submit_writes_wal_sync(void) { + /* Submit a job via handle_jobsubmit() with a wired persist_t */ + /* Verify WAL contains a WAL_JOB_SUBMIT entry */ + /* Verify xce_reply() received sync_rc == 0 */ + } + ``` + + Expected: **FAIL** — hook not registered yet. + +- [ ] **Step 13.2 — Implement `persist_job_hook_fun` in persist.c** + + ```c + static int persist_job_hook_fun(void* arg, int detail, ...) { + persist_t* p = arg; + va_list ap; + va_start(ap, detail); + + job_t* job = va_arg(ap, job_t*); + int* sync_rc_out = va_arg(ap, int*); + persist_batch_ctx_t* batch = va_arg(ap, persist_batch_ctx_t*); + + uint32_t wal_type = detail_to_wal_type(detail); + if (wal_type == 0) { va_end(ap); return 0; } + + /* Serialize job to Cap'n Proto payload (partial for non-submit types) */ + /* ... capnp serialization ... */ + + int rc; + if (wal_type == WAL_JOB_SUBMIT) { + rc = persist_wal_sync_append(p, wal_type, payload_buf, payload_len, sync_rc_out); + } else { + rc = persist_wal_async(p, wal_type, payload_buf, payload_len); + if (sync_rc_out) *sync_rc_out = rc; + } + + va_end(ap); + return 0; /* AFTER_EVENT hooks always return 0 — failures logged not propagated */ + } + ``` + + Implement `detail_to_wal_type()` mapping table (Part 2.3 table). + +- [ ] **Step 13.3 — Implement persist_queue_hook_fun and persist_worker_hook_fun** + + Same pattern: serialize queue/worker to Cap'n Proto, call `persist_wal_async()`. + +- [ ] **Step 13.4 — Register hooks in `persist_start()`** + + ```c + xce_add_hook(c->hooks, XCE_HOOK_TYPE_AFTER_EVENT, HOOK_DETAIL_JOB_NEW, persist_job_hook_fun, p); + xce_add_hook(c->hooks, XCE_HOOK_TYPE_AFTER_EVENT, HOOK_DETAIL_JOB_START, persist_job_hook_fun, p); + /* ... all job detail types ... */ + xce_add_hook(c->hooks, XCE_HOOK_TYPE_AFTER_EVENT, HOOK_DETAIL_QUEUE_UPDATE, persist_queue_hook_fun, p); + xce_add_hook(c->hooks, XCE_HOOK_TYPE_AFTER_EVENT, HOOK_DETAIL_WORKER_JOIN, persist_worker_hook_fun, p); + /* ... etc. */ + ``` + +- [ ] **Step 13.5 — Run tests** + + ```bash + ninja -C build && meson test -C build -t 0 --suite controller + ``` + Expected: **PASS**. + +- [ ] **Step 13.6 — Commit** + + ```bash + git add src/controller/persist.c + git commit -m "feat: persistence hook functions wired to WAL write on AFTER_EVENT (P0-6)" + ``` + +--- + +## Task 14 — P0-E: entity terminal states + deferred GC (P0-5a + P0-8 + P0-13) + +**Reads:** Design doc Parts 4.1–4.5, P0-5a, P0-8, P0-13. + +**Files:** +- Modify: `src/include/xce/cluster_def.h` or `types.h` +- Modify: `src/controller/worker.c` +- Modify: `src/controller/job.c` +- Modify: `src/lib/kvmgr.c` + +- [ ] **Step 14.1 — Write failing test for JOB_STAT_TRANSIENT** + + ```c + void test_transient_job_auto_terminates(void) { + job_t* j = make_test_job(JOB_STAT_TRANSIENT); + j->transient_since = time(NULL) - 400; /* expired: > default 300s */ + persist_config_t cfg = persist_config_defaults(); + assert(!job_should_retain(j, &cfg)); + } + + void test_transient_job_retained_within_timeout(void) { + job_t* j = make_test_job(JOB_STAT_TRANSIENT); + j->transient_since = time(NULL) - 100; /* not expired */ + persist_config_t cfg = persist_config_defaults(); + assert(job_should_retain(j, &cfg)); + } + ``` + + Expected: **FAIL** — `JOB_STAT_TRANSIENT` and `transient_since` not defined. + +- [ ] **Step 14.2 — Add terminal state bitmasks** + + In `src/include/xce/types.h` (or `cluster_def.h`, wherever job status bits are defined): + ```c + #define JOB_STAT_TRANSIENT 0x400 /* MUST NOT be 0x200 — that is JOB_STAT_WAIT */ + ``` + + Add `XCE_WORKER_HIDDEN` and `QUEUE_STAT_HIDDEN` to their respective enums. + + Add `time_t transient_since` field to `job_t`. + +- [ ] **Step 14.3 — Implement `job_is_terminal()` and `job_should_retain()`** + + In `src/controller/job.c`: + ```c + bool job_is_terminal(const job_t* j) { + return (j->stat & (JOB_STAT_EXIT | JOB_STAT_DONE | JOB_STAT_PERR)) != 0; + } + + bool job_should_retain(const job_t* j, const persist_config_t* cfg) { + time_t now = time(NULL); + + /* Transient jobs: auto-terminate if timeout elapsed */ + if (j->stat & JOB_STAT_TRANSIENT) { + if (j->transient_since > 0 && + now > j->transient_since + (time_t)cfg->transient_job_timeout_s) { + /* Will be auto-terminated by checkpoint GC */ + return false; + } + return true; /* Not yet timed out */ + } + + if (!job_is_terminal(j)) return true; + if (job_has_dependents(j)) return true; + return (j->finish_time + (time_t)cfg->finished_retain_s >= now); + } + ``` + +- [ ] **Step 14.4 — Extend `mark_worker_unavail()` to set `dead_since`** + + In `src/controller/worker.c`, in `mark_worker_unavail()`, add: + ```c + w->host.dead_since = time(NULL); + ``` + +- [ ] **Step 14.5 — Add P0-8a TODO markers for endpoint cleanup** + + In `src/controller/cluster.c`, find `handle_endpoint_error()` calls to `zarray_del()`/`zarray_clear()`. Add: + ```c + /* TODO P0-8b: replace with deferred cleanup after P0-0a zarray merge */ + ``` + +- [ ] **Step 14.6 — Run tests** + + ```bash + ninja -C build && meson test -C build -t 0 --suite controller + ``` + Expected: **PASS**. + +- [ ] **Step 14.7 — Commit** + + ```bash + git add src/include/xce/types.h src/controller/job.c src/controller/worker.c \ + src/controller/cluster.c + git commit -m "feat: add JOB_STAT_TRANSIENT, job_should_retain, worker dead_since (P0-5a, P0-8, P0-13)" + ``` + +--- + +## Task 15 — P0-F: cluster wiring (P0-5 + P0-10 + P0-11) + +**Reads:** Design doc Parts 2.6–2.7, P0-5, P0-10, P0-11. + +**Files:** +- Modify: `src/include/xce/cluster_def.h` (add `persist_t persist` to `cluster_t`) +- Modify: `src/controller/cluster.c` (`cluster_init/run/fini`) +- Modify: `src/controller/myconf.c` (YAML parsing) + +- [ ] **Step 15.1 — Write pre-addition audit** + + ```bash + grep -rn 'sizeof.*cluster_t\|offsetof.*cluster\|cluster_t' src/ | grep -v '\.h' | head -20 + ``` + Verify no code assumes fixed `sizeof(cluster_t)` before adding the large `persist_t` field. + +- [ ] **Step 15.2 — Add `persist_t persist` to `cluster_t`** + + In `src/include/xce/cluster_def.h`: + ```c + typedef struct cluster_s { + /* ... existing fields ... */ + persist_t persist; /* Persistence module — added P0-F */ + } cluster_t; + ``` + +- [ ] **Step 15.3 — Wire persist lifecycle in cluster.c** + + In `cluster_init()`: + ```c + persist_config_t pcfg = persist_config_from_myconf(&c->conf); + if (persist_init(&c->persist, &pcfg, c) != 0) return -1; + if (persist_recover(&c->persist) != 0) return -1; + ``` + + In `cluster_run()` (after cluster is fully initialized): + ```c + persist_start(&c->persist); + ``` + + In `cluster_fini()`: + ```c + persist_stop(&c->persist); + persist_fini(&c->persist); + ``` + +- [ ] **Step 15.4 — Add persistence config YAML parsing in myconf.c** + + In `myconf.c`, parse from `cluster.bin` config: + ```c + cfg->persist.wal_sync_ms = yaml_get_uint(node, "wal_sync_ms", DEFAULT_WAL_SYNC_MS); + cfg->persist.snap_interval_s = yaml_get_uint(node, "snap_interval_s", DEFAULT_SNAP_INTERVAL_S); + cfg->persist.snap_wal_size_mb = yaml_get_uint(node, "snap_wal_size_mb", DEFAULT_SNAP_WAL_SIZE_MB); + cfg->persist.snap_keep_n = yaml_get_uint(node, "snap_keep_n", DEFAULT_SNAP_KEEP_N); + cfg->persist.finished_retain_s = yaml_get_uint(node, "finished_retain_s", DEFAULT_FINISHED_RETAIN_S); + cfg->persist.sync_timeout_ms = yaml_get_uint(node, "sync_timeout_ms", DEFAULT_SYNC_TIMEOUT_MS); + ``` + +- [ ] **Step 15.5 — Add `--workdir` and `-n` CLI flags to main.c** + + In `src/controller/main.c`, add: + ```c + "--workdir", required_argument, ... + "-n", required_argument, ... + ``` + + Pass `workdir` and `name` to `persist_init()` to derive `effective_dir`. + +- [ ] **Step 15.6 — Build and run all tests** + + ```bash + ninja -C build && meson test -C build + ``` + Expected: **PASS** (all suites). + +- [ ] **Step 15.7 — Commit** + + ```bash + git add src/include/xce/cluster_def.h src/controller/cluster.c \ + src/controller/myconf.c src/controller/main.c + git commit -m "feat: wire persist_init/recover/start/stop into cluster lifecycle (P0-F)" + ``` + +--- + +## Task 16 — P0-G: end-to-end smoke tests + +**Reads:** Design doc P0-G (P0-15a through P0-15e). + +**Files:** +- Create: `src/controller/testsuite/test_e2e_wal.c` +- Create: `src/controller/testsuite/test_e2e_checkpoint.c` +- Create: `src/controller/testsuite/test_e2e_hooks.c` +- Create: `src/controller/testsuite/test_e2e_gc.c` +- Create: `src/controller/testsuite/test_e2e_capnp_merge.c` + +- [ ] **Step 16.1 — P0-15a: WAL E2E — write/crash/recover** + + ```c + void test_wal_torn_write_recovery(void) { + /* 1. Write 10 WAL entries */ + /* 2. Truncate last entry mid-write (simulate crash) using ftruncate */ + /* 3. Call persist_recover() */ + /* 4. Verify only 9 entries replayed (last one discarded at CRC check) */ + } + + void test_wal_chained_crc_detects_corruption(void) { + /* 1. Write 5 WAL entries */ + /* 2. Corrupt bytes in entry 3's payload */ + /* 3. persist_recover() must stop replay at entry 2 */ + } + ``` + + Run, fix, pass, commit: `"test: WAL E2E — torn write and CRC corruption recovery (P0-15a)"` + +- [ ] **Step 16.2 — P0-15b: Checkpoint E2E — round trip** + + ```c + void test_checkpoint_recovery_entity_counts(void) { + /* 1. Add 100 jobs, 5 queues, 10 workers to cluster */ + /* 2. Trigger checkpoint (writes snapshot) */ + /* 3. Add 20 more jobs to WAL (no checkpoint) */ + /* 4. Destroy cluster, re-init, call persist_recover() */ + /* 5. Assert: 120 jobs, 5 queues, 10 workers present */ + } + ``` + + Run, fix, pass, commit: `"test: checkpoint+recovery round-trip entity count (P0-15b)"` + +- [ ] **Step 16.3 — P0-15c: Hook E2E — ordering** + + ```c + void test_reply_after_hook_ordering(void) { + /* Submit a job; intercept hook + reply calls */ + /* Assert hook fires BEFORE reply is sent */ + } + + void test_sync_rc_propagates_on_wal_failure(void) { + /* Inject WAL write failure (close wal_fd) */ + /* Submit job — assert client receives error, not success ACK */ + } + ``` + + Run, fix, pass, commit: `"test: hook invocation ordering and sync_rc propagation (P0-15c)"` + +- [ ] **Step 16.4 — P0-15d: GC E2E — retention and transient jobs** + + ```c + void test_finished_jobs_cleaned_after_ttl(void) { + /* Create 5 finished jobs with finish_time = now - 7200 (> 3600s default) */ + /* Trigger checkpoint GC (thaw with should_retain) */ + /* Assert 0 finished jobs survive */ + } + + void test_transient_job_auto_terminated_at_checkpoint(void) { + /* Create 2 transient jobs: one < 300s ago, one > 300s ago */ + /* Trigger checkpoint GC */ + /* Assert: 1 job remains (not expired), 1 moved to JOB_STAT_EXIT */ + /* Assert ctrl_transient_jobs_stuck counter incremented once */ + } + ``` + + Run, fix, pass, commit: `"test: GC retention and JOB_STAT_TRANSIENT auto-termination (P0-15d)"` + +- [ ] **Step 16.5 — P0-15e: Cap'n Proto partial-merge semantics** + + ```c + void test_partial_wal_entry_does_not_zero_unset_fields(void) { + /* Create job with stat=RUN, exec_host="node01", rusage values set */ + /* Apply a WAL_JOB_RUSAGE partial payload (only rusage fields set) */ + /* Assert exec_host and stat are UNCHANGED after apply_wal_entry() */ + } + ``` + + Run, fix, pass, commit: `"test: Cap'n Proto partial WAL merge does not zero absent fields (P0-15e)"` + +- [ ] **Step 16.6 — Final full test run** + + ```bash + ninja -C build && meson test -C build 2>&1 + ``` + Expected: **ALL PASS**. No compiler warnings with `-Wall -Wextra`. + +--- + +## P0 Gate Checklist + +Before declaring P0 complete: + +- [ ] `meson test -C build` — all tests pass +- [ ] `ninja -C build` — zero warnings with `-Wall -Wextra` +- [ ] Controller starts with `--workdir /tmp/xce -n ctrl-1` and creates `data/wal/` and `data/snap/` directories +- [ ] Controller survives `SIGKILL` and restarts with all jobs/queues/workers intact +- [ ] WAL segments are trimmed after checkpoint +- [ ] `JOB_STAT_TRANSIENT` jobs older than `transient_job_timeout_s` are moved to `JOB_STAT_EXIT` at checkpoint +- [ ] All P0-G smoke tests pass +- [ ] No `TODO P0-` comments remain except P0-8b (which requires P0-0a, tracked separately) + +--- + +## What's next after P0 + +Create `2026-05-07-p1-ha-foundation.md` covering: +- `ha.c` / `ha.h` — role management, heartbeat watchdog +- `xce_disable_hook()` addition to `hook.c` +- LSF-style priority election (`am_i_primary()`) +- Promotion/demotion sequence +- Redirect logic in `join.c` +- Worker/scheduler reconnect after failover diff --git a/docs/superpowers/plans/2026-05-07-p1-ha-foundation.md b/docs/superpowers/plans/2026-05-07-p1-ha-foundation.md new file mode 100644 index 0000000..0d2ece6 --- /dev/null +++ b/docs/superpowers/plans/2026-05-07-p1-ha-foundation.md @@ -0,0 +1,1520 @@ +# P1: HA Foundation — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use `superpowers:subagent-driven-development` or `superpowers:executing-plans` to implement this plan task-by-task. + +**Goal:** Standby controller can take over when primary fails. + +**Architecture:** P1 builds LSF-style priority election on top of the existing iomgr/SSL/endpoint/vtask pipeline — no new TLS infrastructure is introduced because XCE runs on a trusted LAN. A dedicated watchdog pthread (never delaymgr) detects primary heartbeat timeout and triggers self-promotion. The hook system is extended with atomic disable-in-place semantics so persistence hooks can be cleanly registered on promotion and disabled on demotion without list growth across promote/demote cycles. + +**Tech Stack:** C11, POSIX threads, `_Atomic` / `stdatomic.h`, `nanosleep` / `CLOCK_MONOTONIC`, iomgr/vtask/endpoint SSL pipeline, Cap'n Proto (capnpc-c), nix develop + meson + ninja + +**Design doc:** `src/docs/persistence-and-ha-design.md` — Parts 5, 9, Appendices A.3–A.4 + +**Build commands (inside `nix develop`):** +```bash +meson setup build --wipe # first time +ninja -C build # incremental +ninja -C build test # run all tests +# OR from src/frontend/: +make test # Go unit tests (unchanged by P1) +``` + +**Dependencies:** P0 complete — all tests green, `persist_t persist` wired into `cluster_t`, `controller.capnp` has HA opcodes (`replEntry`, `replAck`, `redirect`, `heartbeat`, `steppingDown`, `replSnapshot`, `replCatchUpDone`) already added in P0-9. Verify before starting: +```bash +grep -c 'heartbeat\|steppingDown\|replEntry' src/schema/controller.capnp # must be > 0 +ninja -C build test # must be green +``` + +--- + +## P1-1: Verify iomgr/SSL pipeline reuse — no new TLS work needed + +**Files:** `src/controller/netio.c` (read-only audit), `src/controller/ha.c` (will be created in P1-2 — just audit for now) + +**Goal:** Confirm that `create_ssl_context()` in `netio.c` and the existing iomgr accept/connect path work unchanged for HA traffic. No modifications needed — this is an audit step that produces a commit containing only a comment block in the upcoming `ha.h`. + +### RED — audit test (compile-only, no new test file yet) +```bash +# Verify: iomgr/endpoint already handles SSL for controller-to-controller conns +grep -n 'create_ssl_context\|iomgr_set_ssl_ctx\|endpoint_init' src/controller/cluster.c +# Expected: ssl_ctx created in cluster_init(), set on iomgr — HA listeners will reuse it +grep -n 'SSL_CTX\|ssl_ctx' src/controller/netio.c | head -10 +``` + +### GREEN — document the decision in ha.h (stubbed header, creates the file) + +Create `src/controller/ha.h`: +```c +#ifndef HA_H +#define HA_H + +/* + * P1-1 audit result: iomgr/SSL pipeline reuse confirmed. + * + * The existing create_ssl_context() + iomgr_set_ssl_ctx() path (cluster.c:558-583) + * handles all controller-to-controller SSL. HA listeners (election probe, heartbeat, + * replication accept) will use the SAME ssl_ctx already installed on c->iomgr. + * + * No new TLS infrastructure is needed. XCE runs on a trusted LAN — peer identity is + * validated by host:port match against the --peers candidate list, not mTLS. + * See design doc Part 5.9 and memory note: Persistence & HA Trusted LAN Decision. + */ + +#include "xce/cluster_def.h" /* cluster_t forward dependency */ +#include "persist.h" + +/* Full struct and API declarations added in P1-2. */ + +#endif /* HA_H */ +``` + +**Commit:** +```bash +git add src/controller/ha.h +git commit -m "feat: add ha.h stub documenting iomgr/SSL pipeline reuse decision (P1-1)" +``` + +--- + +## P1-2a: Atomic hook disable — `xce_disable_hook()` + `_Atomic hook_fn_t` + +**Files to modify:** +- `src/include/hook.h` — change `hook_item_t.fn` to `_Atomic hook_fn_t`; add `xce_disable_hook()` declaration +- `src/lib/hook.c` — update `xce_add_hook()`, `xce_run_hooks()`, add `xce_disable_hook()`; add NULL-slot reuse scan + +**Rationale:** `ha_demote_to_standby()` (P1-3a) must atomically disable persistence hooks without removing them from the list, preventing unbounded list growth across promote→demote→re-promote cycles. `xce_run_hooks()` must skip NULL-fn slots. `xce_add_hook()` must reuse NULL slots of the same type. + +### RED — failing test + +Add to `src/lib/testsuite/test_hooks.c`: +```c +/* Test: xce_disable_hook nullifies fn atomically; xce_run_hooks skips it */ +void test_disable_hook(void) +{ + hooksys_t hs; + int call_count = 0; + + static int counter_fn(int type, int sit, int det, void* data, va_list ap) { + (*(int*)data)++; + return 0; + } + + xce_hook_init(&hs, 4); + TEST_ASSERT_EQUAL(0, xce_add_hook(&hs, 0, counter_fn, &call_count)); + + /* Before disable: hook fires */ + xce_run_hooks(&hs, 0, HOOK_AFTER_EVENT, 0); + TEST_ASSERT_EQUAL(1, call_count); + + /* Disable: xce_run_hooks must skip the now-NULL slot */ + TEST_ASSERT_EQUAL(0, xce_disable_hook(&hs, 0, counter_fn)); + xce_run_hooks(&hs, 0, HOOK_AFTER_EVENT, 0); + TEST_ASSERT_EQUAL(1, call_count); /* count unchanged — hook not called */ + + xce_hook_fini(&hs); +} + +/* Test: slot reuse — re-adding the same (type, fn) reuses the NULL slot, list stays bounded */ +void test_add_hook_reuses_null_slot(void) +{ + hooksys_t hs; + int call_count = 0; + int initial_count; + + static int noop_fn(int type, int sit, int det, void* data, va_list ap) { + (*(int*)data)++; + return 0; + } + + xce_hook_init(&hs, 4); + xce_add_hook(&hs, 0, noop_fn, &call_count); + initial_count = hs.hooks[0].count; /* e.g. 1 */ + + /* Disable then re-add — list must NOT grow */ + xce_disable_hook(&hs, 0, noop_fn); + xce_add_hook(&hs, 0, noop_fn, &call_count); + TEST_ASSERT_EQUAL(initial_count, hs.hooks[0].count); /* same slot reused */ + + xce_hook_fini(&hs); +} +``` + +Run `ninja -C build test` — **must fail** (function not declared yet). + +### GREEN — implementation + +**`src/include/hook.h`** — change struct and add declaration: +```c +#if !defined(__HOOK_H__) +#define __HOOK_H__ 1 + +#include +#include +#include /* C11 — _Atomic */ + +#include "xlist.h" + +#define HOOK_BEFORE_EVENT (0) +#define HOOK_AFTER_EVENT (1) + +typedef int (*hook_fn_t)(int, int, int, void*, va_list); + +typedef struct { + _Atomic hook_fn_t fn; /* C11 atomic — NULL after xce_disable_hook() */ + void* data; +} hook_item_t; + +typedef struct { + int maxtypes; + xlist_t* hooks; +} hooksys_t; + +extern int xce_hook_init(hooksys_t*, int); +extern int xce_add_hook(hooksys_t*, int, hook_fn_t, void*); +extern int xce_disable_hook(hooksys_t*, int, hook_fn_t); /* NEW */ +extern int xce_run_hooks(hooksys_t*, int, int, int, ...); +extern void xce_hook_fini(hooksys_t*); + +#endif +``` + +**`src/lib/hook.c`** — key changes to each function: + +`xce_add_hook()` — scan for NULL slot before appending: +```c +int xce_add_hook(hooksys_t* s, int typ, hook_fn_t fn, void* data) +{ + hook_item_t* it; + int i; + + if (s == NULL || fn == NULL) return -EINVAL; + if ((typ < 0) || (typ >= s->maxtypes)) return -EINVAL; + + /* Reuse a disabled (NULL-fn) slot of the same type — prevents unbounded growth + * on promote->demote->re-promote cycles (design doc Part 5.7). */ + xlist_t* l = &(s->hooks[typ]); + for (i = 0; i < l->count; i++) { + hook_item_t* slot = l->data[i]; + if (slot == NULL) continue; + hook_fn_t cur = atomic_load_explicit(&slot->fn, memory_order_acquire); + if (cur == NULL) { + /* Reuse: set data first, then fn with release so readers see consistent pair */ + slot->data = data; + atomic_store_explicit(&slot->fn, fn, memory_order_release); + return 0; + } + } + + it = (hook_item_t*)calloc(1, sizeof(*it)); + if (it == NULL) return -ENOMEM; + + it->data = data; + atomic_store_explicit(&it->fn, fn, memory_order_release); + xlist_store(&(s->hooks[typ]), it); + return 0; +} +``` + +`xce_disable_hook()` — new function: +```c +int xce_disable_hook(hooksys_t* s, int typ, hook_fn_t fn) +{ + int i; + + if (s == NULL || fn == NULL) return -EINVAL; + if ((typ < 0) || (typ >= s->maxtypes)) return -EINVAL; + + xlist_t* l = &(s->hooks[typ]); + for (i = 0; i < l->count; i++) { + hook_item_t* it = l->data[i]; + if (it == NULL) continue; + hook_fn_t cur = atomic_load_explicit(&it->fn, memory_order_acquire); + if (cur == fn) { + atomic_store_explicit(&it->fn, (hook_fn_t)NULL, memory_order_release); + return 0; + } + } + return -ENOENT; +} +``` + +`xce_run_hooks()` — NULL-fn guard: +```c +for (i = 0; i < l->count; i++) { + hook_item_t* it = l->data[i]; + va_list ap; + + if (it == NULL) continue; + + hook_fn_t local_fn = atomic_load_explicit(&it->fn, memory_order_acquire); + if (!local_fn) continue; /* disabled slot — skip */ + + va_start(ap, detail); + rc = local_fn(type, situation, detail, it->data, ap); + va_end(ap); + + if ((situation == HOOK_BEFORE_EVENT) && (rc < 0)) break; +} +``` + +Run `ninja -C build test` — **must pass**. + +**Commit:** +```bash +git add src/include/hook.h src/lib/hook.c src/lib/testsuite/test_hooks.c +git commit -m "feat: add xce_disable_hook() + atomic hook_fn_t + NULL-slot reuse guard (P1-2a)" +``` + +--- + +## P1-2: `ha.c` + `ha.h` — role management, `am_i_primary()`, promotion/demotion stubs + +**Files:** +- `src/controller/ha.h` — full struct and API declarations (replaces stub from P1-1) +- `src/controller/ha.c` — new file: `ha_init()`, `ha_fini()`, `am_i_primary()`, `ha_promote_to_primary()` (step-counted stub), `ha_demote_to_standby()` stub + +**Requires:** P1-2a merged and green. + +### RED — failing tests + +Add `src/controller/testsuite/test_ha_role.c`: +```c +#include +#include +#include "ha.h" +#include "xce/cluster_def.h" + +/* Minimal cluster_t stand-in for unit tests — avoids full cluster_init() */ +static cluster_t g_cluster; + +void setUp(void) { memset(&g_cluster, 0, sizeof(g_cluster)); } +void tearDown(void) { } + +/* ha_init() must set role = CTRL_STANDBY (default before am_i_primary()) */ +void test_ha_init_default_standby(void) +{ + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, + .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, + .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + }; + TEST_ASSERT_EQUAL(0, ha_init(&g_cluster.ha, &cfg, 0 /* my_priority */)); + TEST_ASSERT_EQUAL(CTRL_STANDBY, + (ctrl_role_t)atomic_load_explicit(&g_cluster.ha.role, memory_order_acquire)); + ha_fini(&g_cluster.ha); +} + +/* ha_init() must reject heartbeat_interval_ms < 2000 */ +void test_ha_init_rejects_fast_heartbeat(void) +{ + ha_config_t cfg = { + .heartbeat_interval_ms = 1999, /* below floor */ + .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, + .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + }; + TEST_ASSERT_NOT_EQUAL(0, ha_init(&g_cluster.ha, &cfg, 0)); +} + +/* ha_init() must reject election_timeout_ms < 500 */ +void test_ha_init_rejects_short_election_timeout(void) +{ + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, + .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 499, /* below floor */ + .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + }; + TEST_ASSERT_NOT_EQUAL(0, ha_init(&g_cluster.ha, &cfg, 0)); +} +``` + +Run `ninja -C build test` — **must fail** (ha.h and ha.c do not define `ha_init` yet). + +### GREEN — implementation + +**`src/controller/ha.h`** (replace stub): +```c +#ifndef HA_H +#define HA_H + +/* + * XCE Controller — HA role management. + * Implements LSF-style priority election (design doc Parts 5, 9; Appendices A.3–A.4). + * + * iomgr/SSL pipeline reuse: all HA traffic (heartbeats, election probes, replication) + * uses the existing create_ssl_context() + iomgr/endpoint pipeline. No new TLS + * infrastructure is needed — XCE runs on a trusted LAN (P1-1 audit, design doc Part 5.9). + */ + +#include +#include +#include +#include + +#include "xce/cluster_def.h" /* cluster_t, persist_t, endpoint_t */ +#include "persist.h" + +/* ── Role enum ───────────────────────────────────────────────────────── */ +typedef enum { CTRL_PRIMARY = 0, CTRL_STANDBY = 1 } ctrl_role_t; + +/* ── HA configuration (populated from ha_config_t in cluster config) ── */ +typedef struct { + uint32_t heartbeat_interval_ms; /* Primary heartbeat send interval (min 2000) */ + uint32_t heartbeat_timeout_ms; /* Standby timeout before promoting (min heartbeat_interval_ms) */ + uint32_t election_timeout_ms; /* am_i_primary() shared deadline (min 500) */ + uint32_t probe_connect_timeout_ms; /* Per-probe poll() timeout in am_i_primary() */ + uint32_t max_promotion_retries; /* Max consecutive failed promotion attempts */ +} ha_config_t; + +/* ── Replicator (P1 stub — P2 adds full fan-out send) ───────────────── + * Forward declaration only — repl.h (P2-1) defines the full struct. + * P1 includes the replicator field but uses it only for zero-init. */ +#ifndef REPL_H +typedef struct { int _placeholder; } replicator_t; +#endif + +/* ── HA state (embedded in cluster_t as ha_state_t ha) ───────────────── */ +typedef struct { + _Atomic uint64_t term; /* C11 atomic election fencing token */ + _Atomic int32_t role; /* _Atomic int32_t, cast to ctrl_role_t at access */ + int my_priority; /* Index in candidates[] (lower = higher priority) */ + replicator_t repl; /* Replicator — initialized in ha_init(), used in P2 */ + _Atomic uint64_t applied_lsn; /* Standby: last applied WAL LSN (written by iomgr thread) */ + endpoint_t* primary_ep; /* Borrowed ref to primary SSL endpoint (standby only) */ + char primary_host[XCE_MAXNAME_LEN]; + uint32_t primary_port; + struct timespec last_heartbeat; /* CLOCK_MONOTONIC timestamp of last primary heartbeat */ + _Atomic int promotion_failed; + _Atomic int promotion_retry_count; + _Atomic int in_catch_up; /* Standby: set while replaying historical entries */ + int hooks_registered; /* Idempotency guard: non-zero after xce_add_hook() */ + atomic_int failover_requested; /* SIGUSR1 handler sets this; watchdog executes */ + uint32_t last_received_crc; /* Chained CRC of last replicated WAL entry (standby) */ + _Atomic int ha_isolated; /* Primary: set when no standby reachable for heartbeat */ + struct timespec primary_isolated_since; /* When isolation began (CLOCK_MONOTONIC) */ + ha_config_t config; + pthread_mutex_t lock; /* Protects primary_ep + last_heartbeat; initialized once */ +} ha_state_t; + +/* ── Public API ───────────────────────────────────────────────────────── */ + +/* Initialize ha_state_t; validates config floors; must be called before am_i_primary(). + * Returns 0 on success, -EINVAL on bad config. */ +int ha_init(ha_state_t* ha, const ha_config_t* cfg, int my_priority); + +/* Clean up: join watchdog thread, destroy mutex. */ +void ha_fini(ha_state_t* ha); + +/* Election: probe all higher-priority candidates; return 1 if I am primary, 0 if not. */ +int am_i_primary(cluster_t* c); + +/* Promote self to primary (from standby). Called by watchdog on heartbeat timeout. */ +int ha_promote_to_primary(cluster_t* c); + +/* Demote self from primary to standby (manual failover / term supersede). */ +int ha_demote_to_standby(cluster_t* c); + +#endif /* HA_H */ +``` + +**`src/controller/ha.c`** (initial skeleton — full promotion/demotion in P1-3a): +```c +#define _GNU_SOURCE +#include "ha.h" + +#include +#include +#include +#include +#include + +#include "utils.h" /* ERROR/WARN/INFO macros */ + +int ha_init(ha_state_t* ha, const ha_config_t* cfg, int my_priority) +{ + if (!ha || !cfg) return -EINVAL; + + /* Validate config floors (design doc Part 5.4, A5-021) */ + if (cfg->heartbeat_interval_ms < 2000) { + ERROR("ha_init: heartbeat_interval_ms %u < 2000ms minimum", + cfg->heartbeat_interval_ms); + return -EINVAL; + } + if (cfg->heartbeat_timeout_ms < cfg->heartbeat_interval_ms) { + ERROR("ha_init: heartbeat_timeout_ms %u < heartbeat_interval_ms %u", + cfg->heartbeat_timeout_ms, cfg->heartbeat_interval_ms); + return -EINVAL; + } + if (cfg->election_timeout_ms < 500) { + ERROR("ha_init: election_timeout_ms %u < 500ms minimum", + cfg->election_timeout_ms); + return -EINVAL; + } + if (cfg->probe_connect_timeout_ms < 500 || + cfg->probe_connect_timeout_ms > cfg->election_timeout_ms) { + ERROR("ha_init: probe_connect_timeout_ms %u out of range [500, %u]", + cfg->probe_connect_timeout_ms, cfg->election_timeout_ms); + return -EINVAL; + } + + memset(ha, 0, sizeof(*ha)); + ha->config = *cfg; + ha->my_priority = my_priority; + + /* Default role is STANDBY — am_i_primary() will promote to PRIMARY if elected */ + atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release); + atomic_store_explicit(&ha->term, 0ULL, memory_order_release); + atomic_store_explicit(&ha->promotion_failed, 0, memory_order_relaxed); + atomic_store_explicit(&ha->promotion_retry_count, 0, memory_order_relaxed); + atomic_store_explicit(&ha->in_catch_up, 0, memory_order_relaxed); + atomic_store_explicit(&ha->failover_requested, 0, memory_order_relaxed); + atomic_store_explicit(&ha->ha_isolated, 0, memory_order_relaxed); + atomic_store_explicit(&ha->applied_lsn, 0ULL, memory_order_release); + + /* Initialize the mutex ONCE — never re-initialized across role changes */ + pthread_mutex_init(&ha->lock, NULL); + + return 0; +} + +void ha_fini(ha_state_t* ha) +{ + if (!ha) return; + /* Watchdog thread join added in P1-3 */ + pthread_mutex_destroy(&ha->lock); +} + +int am_i_primary(cluster_t* c) +{ + /* Full implementation in P1-3 (election probe pthreads). + * Stub: if my_priority == 0 (highest), assume primary for single-node tests. */ + ha_state_t* ha = &c->ha; + return (ha->my_priority == 0) ? 1 : 0; +} + +int ha_promote_to_primary(cluster_t* c) +{ + /* Full promotion sequence (design doc Part 5.6) implemented in P1-3a. + * Stub: set role to PRIMARY. */ + ha_state_t* ha = &c->ha; + atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY, memory_order_release); + return 0; +} + +int ha_demote_to_standby(cluster_t* c) +{ + /* Full demotion sequence implemented in P1-3a. + * Stub: disable hooks, set role to STANDBY. */ + ha_state_t* ha = &c->ha; + atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release); + return 0; +} +``` + +Run `ninja -C build test` — **must pass**. + +**Commit:** +```bash +git add src/controller/ha.h src/controller/ha.c src/controller/testsuite/test_ha_role.c +git commit -m "feat: add ha_state_t, ha_config_t, ha_init/fini stubs + role enum (P1-2)" +``` + +--- + +## P1-3: Heartbeat watchdog pthread + `ha_check_heartbeat()` + +**Files:** +- `src/controller/ha.c` — add `ha_watchdog_thread()`, `ha_check_heartbeat()`; add `_Atomic int ha_watchdog_stop` and `pthread_t watchdog_thread` to `ha_state_t` +- `src/controller/ha.h` — add new fields to `ha_state_t` struct + +**Key constraints from design doc Part 5.5:** +- Dedicated `pthread` with `nanosleep(CLOCK_MONOTONIC)` — NOT `delaymgr` (2s granularity too coarse) +- `CLOCK_MONOTONIC` — immune to NTP jumps +- `ha_fini()` sets stop flag + `pthread_join()` before returning — no use-after-free of `ha_state_t` +- Never `pthread_cancel()` — not OpenSSL-safe + +### RED — failing test + +Add to `src/controller/testsuite/test_ha_role.c`: +```c +#include + +/* ha_check_heartbeat(): if no primary_ep, do nothing (no promotion triggered) */ +void test_ha_check_heartbeat_no_primary(void) +{ + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, + .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, + .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + }; + ha_init(&g_cluster.ha, &cfg, 1 /* standby priority */); + /* primary_ep is NULL — watchdog must return without promoting */ + ha_check_heartbeat_for_test(&g_cluster); + /* Role must still be STANDBY */ + TEST_ASSERT_EQUAL(CTRL_STANDBY, + (ctrl_role_t)atomic_load_explicit(&g_cluster.ha.role, memory_order_acquire)); + ha_fini(&g_cluster.ha); +} + +/* ha_check_heartbeat(): if role is PRIMARY, function returns immediately */ +void test_ha_check_heartbeat_primary_noop(void) +{ + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, + .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, + .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + }; + ha_init(&g_cluster.ha, &cfg, 0); + atomic_store_explicit(&g_cluster.ha.role, (int32_t)CTRL_PRIMARY, memory_order_release); + ha_check_heartbeat_for_test(&g_cluster); /* must not crash or change role */ + TEST_ASSERT_EQUAL(CTRL_PRIMARY, + (ctrl_role_t)atomic_load_explicit(&g_cluster.ha.role, memory_order_acquire)); + ha_fini(&g_cluster.ha); +} +``` + +Expose test-only entry point in `ha.h`: +```c +#ifdef UNIT_TEST +void ha_check_heartbeat_for_test(cluster_t* c); +#endif +``` + +Run `ninja -C build test` — **must fail** (symbol not defined). + +### GREEN — implementation + +Add to `ha_state_t` in `ha.h`: +```c + _Atomic int ha_watchdog_stop; /* Set by ha_fini() to signal watchdog to exit */ + pthread_t watchdog_thread; /* Joined in ha_fini() before returning */ +``` + +Add to `ha.c`: +```c +/* Exposed for unit tests only */ +void ha_check_heartbeat_for_test(cluster_t* c) { ha_check_heartbeat_(c); } + +static void ha_check_heartbeat_(cluster_t* c) +{ + ha_state_t* ha = &c->ha; + + /* Primary does not check its own heartbeat */ + if ((ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire) != CTRL_STANDBY) + return; + + /* Snapshot primary_ep + last_heartbeat under ha->lock — iomgr clears primary_ep + * (NULL-before-free) holding this same lock, so we see valid-or-NULL, never dangling. */ + pthread_mutex_lock(&ha->lock); + int no_primary = (ha->primary_ep == NULL); + struct timespec last_hb = ha->last_heartbeat; + pthread_mutex_unlock(&ha->lock); + + if (no_primary) + return; + + struct timespec now; + clock_gettime(CLOCK_MONOTONIC, &now); + int64_t elapsed_ms = (now.tv_sec - last_hb.tv_sec) * 1000LL + + (now.tv_nsec - last_hb.tv_nsec) / 1000000LL; + + if (elapsed_ms > (int64_t)ha->config.heartbeat_timeout_ms) { + if (atomic_load_explicit(&ha->promotion_failed, memory_order_relaxed) && + atomic_load_explicit(&ha->promotion_retry_count, memory_order_relaxed) + >= (int)ha->config.max_promotion_retries) { + ERROR("ha: cannot promote after %d attempts — manual intervention required", + atomic_load_explicit(&ha->promotion_retry_count, memory_order_relaxed)); + return; + } + + /* Also handle manual failover flag set by SIGUSR1 handler (P1-3a) */ + ha_promote_to_primary(c); + } +} + +static void* ha_watchdog_thread_(void* arg) +{ + cluster_t* c = (cluster_t*)arg; + ha_state_t* ha = &c->ha; + + while (!atomic_load_explicit(&ha->ha_watchdog_stop, memory_order_acquire)) { + struct timespec ts = { + .tv_sec = ha->config.heartbeat_interval_ms / 1000, + .tv_nsec = (ha->config.heartbeat_interval_ms % 1000) * 1000000L, + }; + nanosleep(&ts, NULL); + + if (atomic_load_explicit(&ha->ha_watchdog_stop, memory_order_acquire)) + break; + + /* Check manual failover request (set by SIGUSR1 handler — P1-3a) */ + if (atomic_load_explicit(&ha->failover_requested, memory_order_acquire)) { + atomic_store_explicit(&ha->failover_requested, 0, memory_order_release); + ha_demote_to_standby(c); + } + + ha_check_heartbeat_(c); + } + return NULL; +} + +/* Call from cluster_start() after ha_init() — starts the watchdog pthread */ +int ha_start_watchdog(cluster_t* c) +{ + ha_state_t* ha = &c->ha; + atomic_store_explicit(&ha->ha_watchdog_stop, 0, memory_order_release); + return pthread_create(&ha->watchdog_thread, NULL, ha_watchdog_thread_, c); +} +``` + +Update `ha_fini()`: +```c +void ha_fini(ha_state_t* ha) +{ + if (!ha) return; + atomic_store_explicit(&ha->ha_watchdog_stop, 1, memory_order_release); + pthread_join(ha->watchdog_thread, NULL); /* safe: watchdog checks stop flag */ + pthread_mutex_destroy(&ha->lock); +} +``` + +Run `ninja -C build test` — **must pass**. + +**Commit:** +```bash +git add src/controller/ha.h src/controller/ha.c src/controller/testsuite/test_ha_role.c +git commit -m "feat: add ha_check_heartbeat + watchdog pthread with nanosleep/CLOCK_MONOTONIC (P1-3)" +``` + +--- + +## P1-3a: Promotion sequence + SIGUSR1 handler + +**Files:** +- `src/controller/ha.c` — implement `ha_promote_to_primary()` (full 9 steps), `ha_demote_to_standby()` (full hook disable + role switch), SIGUSR1 signal handler +- `src/controller/ha.h` — add `ha_install_signal_handler()` declaration + +**Requires:** P1-2a (xce_disable_hook) and P1-3 (watchdog) merged. + +**Key design doc refs:** Part 5.6 (promotion sequence), Part 5.7 (hook disable + memory ordering), Part 5.8 (SIGUSR1 manual failover) + +### RED — failing tests + +Add to `src/controller/testsuite/test_ha_role.c`: +```c +/* ha_promote_to_primary(): role becomes PRIMARY */ +void test_promote_sets_primary_role(void) +{ + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + }; + ha_init(&g_cluster.ha, &cfg, 1); + /* Starting as standby */ + TEST_ASSERT_EQUAL(CTRL_STANDBY, + (ctrl_role_t)atomic_load_explicit(&g_cluster.ha.role, memory_order_acquire)); + + ha_promote_to_primary(&g_cluster); + + TEST_ASSERT_EQUAL(CTRL_PRIMARY, + (ctrl_role_t)atomic_load_explicit(&g_cluster.ha.role, memory_order_acquire)); + ha_fini(&g_cluster.ha); +} + +/* ha_promote_to_primary(): term increments */ +void test_promote_increments_term(void) +{ + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + }; + ha_init(&g_cluster.ha, &cfg, 1); + uint64_t term_before = atomic_load_explicit(&g_cluster.ha.term, memory_order_acquire); + ha_promote_to_primary(&g_cluster); + uint64_t term_after = atomic_load_explicit(&g_cluster.ha.term, memory_order_acquire); + TEST_ASSERT_GREATER_THAN(term_before, term_after); + ha_fini(&g_cluster.ha); +} + +/* ha_demote_to_standby(): role reverts to STANDBY, hooks_registered cleared */ +void test_demote_sets_standby_role(void) +{ + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + }; + ha_init(&g_cluster.ha, &cfg, 0); + ha_promote_to_primary(&g_cluster); + TEST_ASSERT_EQUAL(CTRL_PRIMARY, + (ctrl_role_t)atomic_load_explicit(&g_cluster.ha.role, memory_order_acquire)); + + ha_demote_to_standby(&g_cluster); + + TEST_ASSERT_EQUAL(CTRL_STANDBY, + (ctrl_role_t)atomic_load_explicit(&g_cluster.ha.role, memory_order_acquire)); + TEST_ASSERT_EQUAL(0, g_cluster.ha.hooks_registered); + ha_fini(&g_cluster.ha); +} + +/* hooks_registered guard: double-promote must not add duplicate hooks */ +void test_promote_idempotent_hook_registration(void) +{ + /* Minimal hooksys for test — real cluster_init is skipped */ + xce_hook_init(&g_cluster.hooksys, XCE_CONTROLLER_HOOK_MAX); + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + }; + ha_init(&g_cluster.ha, &cfg, 0); + + ha_promote_to_primary(&g_cluster); + int count_after_first = g_cluster.hooksys.hooks[XCE_CONTROLLER_HOOK_JOB].count; + + ha_promote_to_primary(&g_cluster); /* second call — hooks_registered should gate */ + int count_after_second = g_cluster.hooksys.hooks[XCE_CONTROLLER_HOOK_JOB].count; + + TEST_ASSERT_EQUAL(count_after_first, count_after_second); + ha_fini(&g_cluster.ha); + xce_hook_fini(&g_cluster.hooksys); +} +``` + +Run `ninja -C build test` — **must fail** (promote/demote stubs don't implement these behaviors). + +### GREEN — full implementation + +**`ha_promote_to_primary()` in `ha.c`** (steps follow design doc Part 5.6): +```c +int ha_promote_to_primary(cluster_t* c) +{ + ha_state_t* ha = &c->ha; + + /* Step 2: increment term — fencing token for stale standbys */ + uint64_t new_term = atomic_fetch_add_explicit(&ha->term, 1ULL, memory_order_acq_rel) + 1; + INFO("ha: promoting to primary, new term=%" PRIu64, new_term); + + /* Step 3: flush WAL + reset replication CRC chain. + * persist_flush() is a no-op stub until P0 persist module is fully wired; + * the CRC chain reset is critical for standbys to accept first broadcast. */ + persist_flush(&c->persist); + c->persist.last_replicated_crc = WAL_CRC_NONE; /* COH-5: chain restart on promotion */ + + /* Step 4: switch role to PRIMARY — AFTER recovery succeeds. + * Uses release ordering so vcpu threads see the change immediately. */ + atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY, memory_order_release); + + /* Step 5: register persistence hooks (idempotent) */ + if (!ha->hooks_registered) { + xce_install_hooks(c); /* same call as cluster_init() — idempotent via guard */ + ha->hooks_registered = 1; + } + + /* Reset retry state */ + atomic_store_explicit(&ha->promotion_failed, 0, memory_order_relaxed); + atomic_store_explicit(&ha->promotion_retry_count, 0, memory_order_relaxed); + + return 0; +} +``` + +**`ha_demote_to_standby()` in `ha.c`** (design doc Part 5.7): +```c +int ha_demote_to_standby(cluster_t* c) +{ + ha_state_t* ha = &c->ha; + + /* Disable persistence hooks BEFORE role switch so in-flight requests + * see the role change and stop after the hook is already disabled. + * Design doc Part 5.7: "Disabling must happen before the role switch." */ + if (ha->hooks_registered) { + xce_uninstall_hooks(c); /* calls xce_disable_hook() for each registered hook */ + ha->hooks_registered = 0; + } + + /* Role switch with release ordering — pairs with acquire loads in vcpu threads */ + atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release); + + INFO("ha: demoted to standby"); + return 0; +} +``` + +**SIGUSR1 handler in `ha.c`**: +```c +/* Global pointer set during ha_install_signal_handler() — signal handler must be async-safe */ +static cluster_t* g_sigusr1_cluster = NULL; +static atomic_int g_sigusr1_enabled = ATOMIC_VAR_INIT(1); + +static void sigusr1_handler(int sig) +{ + (void)sig; + if (!atomic_load_explicit(&g_sigusr1_enabled, memory_order_relaxed)) + return; + if (g_sigusr1_cluster) { + /* Async-signal-safe: only atomic store — no malloc, mutex, or SSL */ + atomic_store_explicit(&g_sigusr1_cluster->ha.failover_requested, 1, + memory_order_release); + } +} + +void ha_install_signal_handler(cluster_t* c, int enable_sigusr1) +{ + g_sigusr1_cluster = c; + atomic_store_explicit(&g_sigusr1_enabled, enable_sigusr1 ? 1 : 0, + memory_order_relaxed); + + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = sigusr1_handler; + sigemptyset(&sa.sa_mask); + sigaction(SIGUSR1, &sa, NULL); +} +``` + +Run `ninja -C build test` — **must pass**. + +**Commit:** +```bash +git add src/controller/ha.h src/controller/ha.c src/controller/testsuite/test_ha_role.c +git commit -m "feat: full ha_promote_to_primary + ha_demote_to_standby + SIGUSR1 handler (P1-3a)" +``` + +--- + +## P1-4: Cap'n Proto schema additions — `term` in Request, `standby` in HostRole, `RedirectInfo` + +**Files:** `src/schema/controller.capnp` + +**Audit first:** The HA opcodes (`replEntry`, `replAck`, `redirect`, `heartbeat`, `steppingDown`, `replSnapshot`, `replCatchUpDone`) were already added in P0-9. Verify: +```bash +grep -c 'heartbeat\|steppingDown\|replEntry\|replAck\|redirect\|replSnapshot\|replCatchUpDone' \ + src/schema/controller.capnp +# expected: 7 (one match each) +``` + +What P1-4 must still add (check each before adding): +1. `standby @5;` to `enum HostRole` (current max is `apiserver @4`) +2. `term @22: UInt64 = 0;` field to `struct Request` (after `from @2`) +3. `struct RedirectInfo` with `host` + `port` +4. `redirect @8: RedirectInfo` in `RequestReply.data` union + +### RED — schema compile test + +```bash +# Expect compile failure for missing types before adding them: +grep 'RedirectInfo\|standby.*HostRole\|term.*Request' src/schema/controller.capnp +# should return empty — these are not yet present +``` + +### GREEN — schema additions + +Add `standby` to `HostRole`: +```capnp +enum HostRole { + client @0; + worker @1; + scheduler @2; + controller @3; + apiserver @4; + standby @5; # P1-4: HA standby controller identity +} +``` + +Add `term` to `Request` (new field at next available index — check current max first): +```capnp +struct Request $C.mapname("xce_request_t"){ + opcode @0: OPCode; + seq @1: UInt64; + from @2: PeerInfo; + term @22: UInt64 = 0 $C.mapname("term"); # P1-4: election fencing token + body : union { + # ... existing body fields unchanged ... + } +} +``` + +> NOTE: Cap'n Proto field numbers in a struct must not skip — use the next available number after the current highest union body field. Check `@21: HostControl` is the current last field and use `@22` for term. If the body union already uses higher numbers internally (union fields share the struct's field number space in capnp-c), verify with `capnpc --src-prefix=src/schema -oc src/schema/controller.capnp` and fix the number if there is a collision. + +Add `RedirectInfo` struct: +```capnp +struct RedirectInfo $C.mapname("xce_redirect_t") { + host @0 :Text $C.mapname("host"); + port @1 :UInt32 $C.mapname("port"); +} +``` + +Add `redirect` to `RequestReply.data` union (next available index after `clusterconf @7`): +```capnp +struct RequestReply $C.mapname("xce_reply_t"){ + seq @0: UInt64; + opcode @1: OPCode; + status @2: Int32; + + data : union { + nodata @3: Void; + debugmsg @4: Text; + getdata @5: Data; + jobid @6: UInt64; + clusterconf @7: ClusterConf; + redirect @8: RedirectInfo $C.mapname("redirect"); # P1-4 + } +} +``` + +Verify schema compiles: +```bash +nix develop --command bash -c "ninja -C build && ninja -C build test" +``` + +**Commit:** +```bash +git add src/schema/controller.capnp +git commit -m "feat: add standby HostRole, term field to Request, RedirectInfo struct (P1-4)" +``` + +--- + +## P1-5: Wire `ha_state_t ha` into `cluster_t` + startup sequence + +**Files:** +- `src/include/xce/cluster_def.h` — add `ha_state_t ha;` field to `cluster_t` +- `src/controller/cluster.c` — call `ha_init()`, `am_i_primary()`, `ha_start_watchdog()` in correct order; call `ha_fini()` in `cluster_fini()` + +**Design doc notes:** +- `ha_init()` must be called **before** `am_i_primary()` (mutex initialized in ha_init) +- `am_i_primary()` runs before `cluster_start()` (before iomgr starts) +- `ha_start_watchdog()` called from `cluster_start()` after iomgr is up +- `ha_fini()` must be called from `cluster_fini()` **before** `persist_fini()` (watchdog may access `c->persist` on its last tick) + +### RED — failing test + +Add to `src/controller/testsuite/test_cluster_persist.c`: +```c +void test_cluster_ha_field_accessible(void) +{ + /* cluster_t must have an ha field after P1-5 */ + cluster_t c; + memset(&c, 0, sizeof(c)); + /* Just verifying the field exists at compile time */ + TEST_ASSERT_NOT_NULL(&c.ha); + /* role must be zero-initialized (CTRL_STANDBY = 0) */ + TEST_ASSERT_EQUAL(0, (int32_t)atomic_load_explicit(&c.ha.role, memory_order_acquire)); +} +``` + +Run `ninja -C build test` — **must fail** (`cluster_t` has no `ha` field yet). + +### GREEN — implementation + +**`src/include/xce/cluster_def.h`** — add field: +```c +typedef struct cluster_s { + myconf_t* myconf; + xce_peerinfo_t* myhost; + hooksys_t hooksys; + iomgr_t iomgr; + zarray_t endpoints; + endpoint_t ep; + zarray_t configs; + zarray_t sharedinst; + controller_t controllers[N_CONTROLLER_MAX]; + cluster_apiserver_t apiservers[N_APISERVER_MAX]; + int myidx; + int master; + int ncontrollers; + delaymgr_t delaymgr; + scheduler_t sched; + var jobmgr; + var resmgr; + var workermgr; + var qmgr; + var configmgr; + vtaskmgr_t vtaskmgr; + cbuf_t wbuf; + int running; + persist_t persist; /* Persistence module — P0-F */ + ha_state_t ha; /* HA role management — P1-5 */ +} cluster_t; +``` + +Add `#include "ha.h"` to `cluster_def.h` (after the existing `#include "persist.h"`). + +**`src/controller/cluster.c`** — modify `cluster_init()` to add HA init after persist_recover: +```c + /* HA init: must happen BEFORE am_i_primary() so ha.lock is initialized */ + { + ha_config_t ha_cfg = { + .heartbeat_interval_ms = 3000, /* TODO: read from cluster_config */ + .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, + .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + }; + int my_priority = 0; /* TODO: look up own position in candidates[] */ + rc = ha_init(&c->ha, &ha_cfg, my_priority); + if (rc != 0) { + ERROR("ha_init failed: %d", rc); + persist_fini(&c->persist); + return rc; + } + + /* Restore election term from WAL/snapshot (design doc Part 4, step 6) */ + uint64_t recovered_term = c->persist.last_term; /* set by persist_recover() */ + if (recovered_term > 0) { + atomic_store_explicit(&c->ha.term, recovered_term, memory_order_release); + } + + /* Election: am I the primary? (runs pre-iomgr blocking probes) */ + if (am_i_primary(c)) { + /* Primary: register persistence hooks NOW (standbys defer to promotion) */ + ha_promote_to_primary(c); + } + + ha_install_signal_handler(c, 1 /* enable SIGUSR1 failover */); + } +``` + +**`src/controller/cluster_start()`** — add watchdog start: +```c + rc = ha_start_watchdog(c); + if (rc != 0) { + ERROR("ha_start_watchdog failed: %d", rc); + return rc; + } +``` + +**`src/controller/cluster_fini()`** — ha_fini before persist_fini: +```c +void cluster_fini(cluster_t* c) +{ + ha_fini(&c->ha); /* joins watchdog thread — BEFORE persist_fini */ + persist_fini(&c->persist); + /* ... rest of existing cleanup ... */ +} +``` + +Run `ninja -C build test` — **must pass**. + +**Commit:** +```bash +git add src/include/xce/cluster_def.h src/controller/cluster.c \ + src/controller/testsuite/test_cluster_persist.c +git commit -m "feat: wire ha_state_t into cluster_t; call ha_init/start_watchdog/fini in startup sequence (P1-5)" +``` + +--- + +## P1-6: Redirect logic in `join.c` + +**Files:** `src/controller/join.c` + +**Design doc ref:** Part 9.4 — standby sends `E_XCE_REDIRECT` + `RedirectInfo` for worker/scheduler/apiserver join requests. + +**Edge case:** if `primary_host` is empty (standby has not yet seen a heartbeat from any primary — cold start), return `E_XCE_NOMASTER` instead of a garbage redirect. + +### RED — failing test + +Add `src/controller/testsuite/test_ha_redirect.c`: +```c +#include +#include "ha.h" +#include "xce/cluster_def.h" + +/* Minimal mock: verify handle_join returns E_XCE_REDIRECT for worker when standby */ +/* NOTE: This test is a compile-level integration check — full end-to-end redirect + * requires a live endpoint (P1-7). Here we verify the path is reachable by calling + * the testable inner function directly with a mock request. */ + +extern int handle_join_for_test(cluster_t* c, xce_request_t* req, + int* out_status, char* out_host, uint32_t* out_port); + +void test_redirect_worker_when_standby(void) +{ + cluster_t c; + memset(&c, 0, sizeof(c)); + + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + }; + ha_init(&c.ha, &cfg, 1); + /* Standby with known primary */ + snprintf(c.ha.primary_host, sizeof(c.ha.primary_host), "ctrl-0.example.com"); + c.ha.primary_port = 7000; + + xce_request_t req = { 0 }; + xce_joindata_t join = { .role = HostRole_worker }; + req.body_which = Request_body_joinmsg; + req.body.joinmsg = &join; + + int status = 0; + char host[XCE_MAXNAME_LEN] = {0}; + uint32_t port = 0; + + handle_join_for_test(&c, &req, &status, host, &port); + + TEST_ASSERT_EQUAL(E_XCE_REDIRECT, status); + TEST_ASSERT_EQUAL_STRING("ctrl-0.example.com", host); + TEST_ASSERT_EQUAL(7000, port); + + ha_fini(&c.ha); +} + +void test_redirect_nomaster_when_no_primary_known(void) +{ + cluster_t c; + memset(&c, 0, sizeof(c)); + + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + }; + ha_init(&c.ha, &cfg, 1); + /* primary_host is empty — cold start, no heartbeat received yet */ + + xce_request_t req = { 0 }; + xce_joindata_t join = { .role = HostRole_scheduler }; + req.body_which = Request_body_joinmsg; + req.body.joinmsg = &join; + + int status = 0; + char host[XCE_MAXNAME_LEN] = {0}; + uint32_t port = 0; + + handle_join_for_test(&c, &req, &status, host, &port); + + TEST_ASSERT_EQUAL(E_XCE_NOMASTER, status); + + ha_fini(&c.ha); +} +``` + +Run `ninja -C build test` — **must fail**. + +### GREEN — implementation + +Add `#include "ha.h"` to `join.c`. + +Add a testable inner function in `join.c`: +```c +/* Testable redirect check — separated from handle_join for unit testing. + * Returns 1 if a redirect was needed (out_* filled), 0 if normal join proceeds. */ +static int check_redirect(cluster_t* c, xce_request_t* req, + int* out_status, char* out_host, uint32_t* out_port) +{ + if ((ctrl_role_t)atomic_load_explicit(&c->ha.role, memory_order_acquire) + != CTRL_STANDBY) + return 0; + + xce_joindata_t* join = req->body.joinmsg; + switch (join->role) { + case HostRole_worker: + case HostRole_scheduler: + case HostRole_apiserver: + /* Guard: no primary known yet */ + if (c->ha.primary_host[0] == '\0' || c->ha.primary_port == 0) { + *out_status = E_XCE_NOMASTER; + return 1; + } + *out_status = E_XCE_REDIRECT; + strncpy(out_host, c->ha.primary_host, XCE_MAXNAME_LEN - 1); + out_host[XCE_MAXNAME_LEN - 1] = '\0'; + *out_port = c->ha.primary_port; + return 1; + default: + return 0; + } +} + +/* Test shim */ +int handle_join_for_test(cluster_t* c, xce_request_t* req, + int* out_status, char* out_host, uint32_t* out_port) +{ + return check_redirect(c, req, out_status, out_host, out_port); +} +``` + +Modify `handle_join()` to call `check_redirect()` before normal processing: +```c +int handle_join(cluster_t* c, endpoint_t* ep, xce_request_t* request) +{ + int redirect_status = 0; + char redirect_host[XCE_MAXNAME_LEN] = {0}; + uint32_t redirect_port = 0; + + if (check_redirect(c, request, &redirect_status, redirect_host, &redirect_port)) { + xce_reply_t reply; + init_reply_by_req(&reply, request); + reply.status = redirect_status; + if (redirect_status == E_XCE_REDIRECT) { + reply.data_which = RequestReply_data_redirect; + reply.data.redirect.host = redirect_host; + reply.data.redirect.port = redirect_port; + } + return xce_reply(ep, c->myhost, &reply); + } + + return handle_join_(c, ep, request); +} +``` + +Add `E_XCE_REDIRECT` and `E_XCE_NOMASTER` to `src/include/xce/types.h` (or wherever XCE error codes live): +```c +#define E_XCE_REDIRECT (-100) /* P1-6: standby redirects to primary */ +#define E_XCE_NOMASTER (-101) /* P1-6: standby has no known primary yet */ +``` + +Run `ninja -C build test` — **must pass**. + +**Commit:** +```bash +git add src/controller/join.c src/include/xce/types.h \ + src/controller/testsuite/test_ha_redirect.c +git commit -m "feat: add standby redirect logic in handle_join(); E_XCE_REDIRECT + E_XCE_NOMASTER (P1-6)" +``` + +--- + +## P1-7: Election, promotion, term fencing, and redirect tests + +**Files:** +- `src/controller/testsuite/test_ha_election.c` — new test file covering the scenarios below + +**Note:** Full end-to-end election with live SSL requires integration test infra (P3). P1-7 covers unit-testable invariants using stubs and direct function calls. + +### Tests to implement + +```c +#include +#include +#include "ha.h" +#include "xce/cluster_def.h" + +static cluster_t g_c; + +void setUp(void) { memset(&g_c, 0, sizeof(g_c)); } +void tearDown(void) { } + +/* 1. Term fencing: standby ignores message with term < ha->term */ +void test_stale_term_message_ignored(void) +{ + ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5 }; + ha_init(&g_c.ha, &cfg, 1); + atomic_store_explicit(&g_c.ha.term, 5ULL, memory_order_release); + + int stale = ha_is_stale_term(&g_c, 3ULL); /* term 3 < current 5 */ + TEST_ASSERT_EQUAL(1, stale); + + int current = ha_is_stale_term(&g_c, 5ULL); /* same term — not stale */ + TEST_ASSERT_EQUAL(0, current); + + ha_fini(&g_c.ha); +} + +/* 2. Supersede: higher term received -> demote current primary */ +void test_higher_term_triggers_demotion(void) +{ + ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5 }; + ha_init(&g_c.ha, &cfg, 0); + ha_promote_to_primary(&g_c); /* become primary at term 1 */ + + /* Simulate receiving a message with higher term (term 5) */ + ha_on_higher_term(&g_c, 5ULL); + + /* Must demote to standby and update term */ + TEST_ASSERT_EQUAL(CTRL_STANDBY, + (ctrl_role_t)atomic_load_explicit(&g_c.ha.role, memory_order_acquire)); + TEST_ASSERT_EQUAL_UINT64(5ULL, + atomic_load_explicit(&g_c.ha.term, memory_order_acquire)); + ha_fini(&g_c.ha); +} + +/* 3. promotion_retry_count increments on repeated failed promotions */ +void test_promotion_retry_count(void) +{ + ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, + .max_promotion_retries = 3 }; + ha_init(&g_c.ha, &cfg, 1); + + /* Simulate promotion failure */ + atomic_store_explicit(&g_c.ha.promotion_failed, 1, memory_order_relaxed); + atomic_store_explicit(&g_c.ha.promotion_retry_count, 2, memory_order_relaxed); + + /* At max_promotion_retries=3, count=2 allows one more; count=3 blocks */ + atomic_store_explicit(&g_c.ha.promotion_retry_count, 3, memory_order_relaxed); + + /* ha_check_heartbeat_for_test with expired timeout: must NOT retry at max count */ + /* Inject a fake stale last_heartbeat */ + pthread_mutex_lock(&g_c.ha.lock); + clock_gettime(CLOCK_MONOTONIC, &g_c.ha.last_heartbeat); + g_c.ha.last_heartbeat.tv_sec -= 20; /* 20s ago — well past 10s timeout */ + /* Inject a fake primary_ep != NULL so watchdog proceeds past no_primary check */ + g_c.ha.primary_ep = (endpoint_t*)0xDEADBEEF; /* non-NULL sentinel */ + pthread_mutex_unlock(&g_c.ha.lock); + + ctrl_role_t before = (ctrl_role_t)atomic_load_explicit(&g_c.ha.role, memory_order_acquire); + ha_check_heartbeat_for_test(&g_c); + ctrl_role_t after = (ctrl_role_t)atomic_load_explicit(&g_c.ha.role, memory_order_acquire); + + /* Role must NOT change — retry limit exceeded */ + TEST_ASSERT_EQUAL(before, after); + + ha_fini(&g_c.ha); +} + +/* 4. OP_steppingDown handler: validates term and sender, then runs am_i_primary */ +void test_stepping_down_handler_validates_term(void) +{ + ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5 }; + ha_init(&g_c.ha, &cfg, 0); + atomic_store_explicit(&g_c.ha.term, 3ULL, memory_order_release); + snprintf(g_c.ha.primary_host, sizeof(g_c.ha.primary_host), "ctrl-1"); + g_c.ha.primary_port = 7001; + + /* Stale term (term=2 < our term=3) — must be ignored */ + int rc = ha_handle_stepping_down(&g_c, 2ULL, "ctrl-1", 7001); + TEST_ASSERT_EQUAL(-EINVAL, rc); /* stale term rejected */ + + /* Correct term but wrong sender — must be ignored */ + rc = ha_handle_stepping_down(&g_c, 3ULL, "ctrl-99", 9999); + TEST_ASSERT_EQUAL(-EINVAL, rc); + + ha_fini(&g_c.ha); +} + +/* 5. Redirect: worker join on primary passes through (no redirect) */ +void test_no_redirect_when_primary(void) +{ + ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5 }; + ha_init(&g_c.ha, &cfg, 0); + ha_promote_to_primary(&g_c); + + xce_request_t req = { 0 }; + xce_joindata_t join = { .role = HostRole_worker }; + req.body_which = Request_body_joinmsg; + req.body.joinmsg = &join; + + int status = 0; + char host[XCE_MAXNAME_LEN] = {0}; + uint32_t port = 0; + + int redirected = handle_join_for_test(&g_c, &req, &status, host, &port); + TEST_ASSERT_EQUAL(0, redirected); /* no redirect when primary */ + + ha_fini(&g_c.ha); +} +``` + +Add `ha_is_stale_term()`, `ha_on_higher_term()`, `ha_handle_stepping_down()` to `ha.h` + `ha.c`: + +```c +/* ha.h additions */ +int ha_is_stale_term(cluster_t* c, uint64_t incoming_term); /* 1 if incoming < current */ +void ha_on_higher_term(cluster_t* c, uint64_t higher_term); /* demote + update term */ +int ha_handle_stepping_down(cluster_t* c, uint64_t msg_term, + const char* sender_host, uint32_t sender_port); +``` + +```c +/* ha.c implementations */ +int ha_is_stale_term(cluster_t* c, uint64_t incoming) +{ + return incoming < atomic_load_explicit(&c->ha.term, memory_order_acquire); +} + +void ha_on_higher_term(cluster_t* c, uint64_t higher_term) +{ + atomic_store_explicit(&c->ha.term, higher_term, memory_order_release); + ha_demote_to_standby(c); +} + +int ha_handle_stepping_down(cluster_t* c, uint64_t msg_term, + const char* sender_host, uint32_t sender_port) +{ + ha_state_t* ha = &c->ha; + + /* Validate: msg_term must match our current term */ + if (msg_term != atomic_load_explicit(&ha->term, memory_order_acquire)) + return -EINVAL; + + /* Validate: sender must be the current primary */ + if (strncmp(sender_host, ha->primary_host, XCE_MAXNAME_LEN) != 0 || + sender_port != ha->primary_port) + return -EINVAL; + + /* Dispatch am_i_primary() asynchronously (full impl: launch pthread — P3) */ + /* Stub: immediately check if we are priority 0 */ + if (am_i_primary(c)) { + ha_promote_to_primary(c); + } + return 0; +} +``` + +Run `ninja -C build test` — **must pass**. + +**Commit:** +```bash +git add src/controller/testsuite/test_ha_election.c \ + src/controller/ha.h src/controller/ha.c +git commit -m "feat: add term fencing, ha_on_higher_term, ha_handle_stepping_down + P1-7 tests (P1-7)" +``` + +--- + +## Final P1 verification + +```bash +nix develop --command bash -c "ninja -C build && ninja -C build test" +# Expected: all existing P0 tests (89) + new P1 tests pass +# New test count: ~25 additional tests across test_ha_role, test_ha_redirect, test_ha_election +``` + +Collect total test count: +```bash +ninja -C build test 2>&1 | grep -E 'Tests run|PASS|FAIL' | tail -5 +``` + +--- + +## Summary of files created/modified by P1 + +| File | Action | Task | +|------|--------|------| +| `src/controller/ha.h` | Create | P1-1 → P1-3a | +| `src/controller/ha.c` | Create | P1-2 → P1-7 | +| `src/include/hook.h` | Modify: `_Atomic hook_fn_t` + `xce_disable_hook` decl | P1-2a | +| `src/lib/hook.c` | Modify: atomic fn, disable, NULL-slot reuse | P1-2a | +| `src/schema/controller.capnp` | Modify: `standby`, `term`, `RedirectInfo` | P1-4 | +| `src/include/xce/cluster_def.h` | Modify: add `ha_state_t ha` to `cluster_t` | P1-5 | +| `src/controller/cluster.c` | Modify: `ha_init/start/fini` in startup sequence | P1-5 | +| `src/controller/join.c` | Modify: redirect check in `handle_join()` | P1-6 | +| `src/include/xce/types.h` | Modify: `E_XCE_REDIRECT`, `E_XCE_NOMASTER` | P1-6 | +| `src/lib/testsuite/test_hooks.c` | Modify: add disable/reuse tests | P1-2a | +| `src/controller/testsuite/test_ha_role.c` | Create | P1-2 → P1-3a | +| `src/controller/testsuite/test_ha_redirect.c` | Create | P1-6 | +| `src/controller/testsuite/test_ha_election.c` | Create | P1-7 | diff --git a/docs/superpowers/plans/2026-05-07-p2-replication.md b/docs/superpowers/plans/2026-05-07-p2-replication.md new file mode 100644 index 0000000..50a7c40 --- /dev/null +++ b/docs/superpowers/plans/2026-05-07-p2-replication.md @@ -0,0 +1,1498 @@ +# P2: Replication (Fan-Out TCP) — TDD Implementation Plan + +**Branch**: `persistent_ha` +**Date**: 2026-05-07 +**Prerequisite**: P1 complete — 115 controller tests green. +**Goal**: Primary replicates state changes to standbys in real time via the existing iomgr/SSL endpoint pipeline. + +--- + +## Scoping Notes + +- Replication channels **reuse the existing iomgr/SSL endpoint pipeline** — no new TLS infrastructure. +- `OP_replAck` is handled **directly in the io_worker read callback** (`handle_msg()` before vtask dispatch) to prevent deadlock: if all vcpu threads block in `repl_wait_ack()`, the ACK must still be deliverable. +- `OP_replCatchUpDone` is also handled in the io_worker read callback for the same latency reason. +- P2.5 (libxce smart discovery) is a separate effort — not included in this plan. P2.5-7 (API server discovery) ships together with P2. +- `WAL_CHECKPOINT` entries are **never replicated** — `repl_broadcast()` must assert `type != WAL_CHECKPOINT`. +- `repl_broadcast()` must check `ha.role == CTRL_PRIMARY` before sending (split-brain guard). +- Worker joins are always **async-replicated** — never sync — due to workermgr WRITE lock held during `add_worker()`. + +--- + +## Task P2-1: `repl.h` + `repl.c` — Structures, init/fini, fan-out send, ACK tracking + +### Files + +| Action | File | +|--------|------| +| Create | `src/controller/repl.h` | +| Create | `src/controller/repl.c` | +| Modify | `src/controller/meson.build` (add `repl.c`) | + +### Failing Test (RED) + +**File**: `src/controller/testsuite/test_repl_unit.c` (new) + +```c +/* test_repl_unit.c — P2-1 RED: repl_init/fini, add/remove standby, + * trim_lsn with disconnected-slot expiry */ +#include +#include +#include "repl.h" + +/* Stub endpoint for unit tests — repl.h forward-declares endpoint_t */ +typedef struct endpoint_s { int dummy; } endpoint_t; + +static void test_repl_init_zeroes_nstandbys(void) +{ + replicator_t r; + TEST_ASSERT_EQUAL_INT(0, repl_init(&r)); + TEST_ASSERT_EQUAL_INT(0, r.nstandbys); + repl_fini(&r); + printf("test_repl_init_zeroes_nstandbys: PASS\n"); +} + +static void test_repl_add_remove_standby(void) +{ + replicator_t r; + endpoint_t ep = {0}; + repl_init(&r); + + int idx = repl_add_standby(&r, &ep, "standby1", 7001); + TEST_ASSERT_TRUE(idx >= 0); + TEST_ASSERT_EQUAL_INT(1, r.nstandbys); + + repl_remove_standby(&r, &ep); + TEST_ASSERT_EQUAL_INT(0, r.nstandbys); + repl_fini(&r); + printf("test_repl_add_remove_standby: PASS\n"); +} + +static void test_repl_add_standby_max(void) +{ + replicator_t r; + endpoint_t eps[MAX_STANDBYS + 1]; + repl_init(&r); + for (int i = 0; i < MAX_STANDBYS; i++) { + int rc = repl_add_standby(&r, &eps[i], "sb", 7000 + i); + TEST_ASSERT_TRUE(rc >= 0); + } + /* One over limit must fail */ + int rc = repl_add_standby(&r, &eps[MAX_STANDBYS], "sb_extra", 9999); + TEST_ASSERT_TRUE(rc < 0); + repl_fini(&r); + printf("test_repl_add_standby_max: PASS\n"); +} + +static void test_repl_trim_lsn_no_standbys(void) +{ + replicator_t r; + repl_init(&r); + /* With no standbys, trim_lsn == checkpoint_lsn */ + uint64_t trim = repl_trim_lsn(&r, /*checkpoint_lsn=*/42, /*snap_interval_s=*/60); + TEST_ASSERT_EQUAL_UINT64(42, trim); + repl_fini(&r); + printf("test_repl_trim_lsn_no_standbys: PASS\n"); +} + +static void test_repl_trim_lsn_disconnected_slot_expiry(void) +{ + replicator_t r; + endpoint_t ep = {0}; + repl_init(&r); + + repl_add_standby(&r, &ep, "sb", 7001); + /* Simulate standby ACKing lsn=10 then disconnecting */ + repl_on_ack(&r, &ep, 10); + repl_remove_standby(&r, &ep); + + /* Immediately after disconnect: trim guard uses last_disconnected_lsn=10, not checkpoint=50 */ + uint64_t trim = repl_trim_lsn(&r, 50, 60); + TEST_ASSERT_EQUAL_UINT64(10, trim); + + /* After snap_interval_s has elapsed (simulate by forcing time forward): + * slot expiry resets disconnected_lsn → trim == checkpoint_lsn */ + repl_expire_disconnect_slots(&r, /*now_s=*/999999, /*snap_interval_s=*/0); + trim = repl_trim_lsn(&r, 50, 0); + TEST_ASSERT_EQUAL_UINT64(50, trim); + + repl_fini(&r); + printf("test_repl_trim_lsn_disconnected_slot_expiry: PASS\n"); +} + +int main(void) +{ + UNITY_BEGIN(); + RUN_TEST(test_repl_init_zeroes_nstandbys); + RUN_TEST(test_repl_add_remove_standby); + RUN_TEST(test_repl_add_standby_max); + RUN_TEST(test_repl_trim_lsn_no_standbys); + RUN_TEST(test_repl_trim_lsn_disconnected_slot_expiry); + return UNITY_END(); +} +``` + +### Minimal Implementation (GREEN) + +**`src/controller/repl.h`**: + +```c +#pragma once +/* + * repl.h — P2-1: Replicator structures and public API. + * + * Fan-out TCP replication over the existing iomgr/SSL endpoint pipeline. + * ACKs are processed in io_worker read callback (NOT vtaskmgr) to prevent + * deadlock when all vcpu threads are blocked in repl_wait_ack(). + */ +#include +#include +#include +#include + +#include "config.h" /* MAX_STANDBYS, XCE_MAXNAME_LEN */ +#include "mylock.h" + +/* Forward declaration — avoids pulling in full netio.h here */ +typedef struct endpoint_s endpoint_t; + +/* ── Per-standby connection slot ─────────────────────────────────── */ +typedef struct { + endpoint_t* ep; /* NULL = slot empty */ + _Atomic uint64_t acked_lsn; /* Last LSN ACKed by this standby */ + _Atomic int in_catch_up; /* Non-zero while standby is catching up */ + uint64_t join_applied_lsn; /* LSN standby reported in OP_replJoin */ + int lagging; + char host[XCE_MAXNAME_LEN]; + uint32_t port; + /* Disconnect tracking for WAL trim guard */ + uint64_t last_disconnected_lsn; + time_t last_disconnect_time; +} standby_conn_t; + +/* ── Replicator ───────────────────────────────────────────────────── */ +typedef struct { + standby_conn_t standbys[MAX_STANDBYS]; + int nstandbys; /* Active standby count */ + mylock_t lock; /* Protects standbys[], nstandbys */ + uint64_t pending_lsn; /* LSN we're waiting ACK for (sync) */ + pthread_cond_t ack_cond; /* Signaled when ACK arrives */ + pthread_mutex_t ack_lock; /* Guards ack_cond + pending_lsn */ +} replicator_t; + +/* ── Lifecycle ───────────────────────────────────────────────────── */ +int repl_init(replicator_t *r); +void repl_fini(replicator_t *r); + +/* ── Standby management ──────────────────────────────────────────── */ +/* Returns slot index >= 0 on success, -ENOBUFS when all MAX_STANDBYS slots used */ +int repl_add_standby(replicator_t *r, endpoint_t *ep, + const char *host, uint32_t port); +/* Removes the standby slot matching ep; records disconnect time/lsn for trim guard */ +void repl_remove_standby(replicator_t *r, endpoint_t *ep); +/* Zero-out disconnect slots whose expiry time has passed (slot expiry rule) */ +void repl_expire_disconnect_slots(replicator_t *r, time_t now_s, + uint32_t snap_interval_s); + +/* ── Replication send paths ──────────────────────────────────────── */ +/* + * repl_broadcast: send WAL entry to all connected standbys via endpoint outq. + * Asserts type != WAL_CHECKPOINT. Checks ha.role == CTRL_PRIMARY first. + * On send failure, marks standby for removal AFTER completing the fan-out. + * Returns number of standbys successfully sent to (>= 0). + */ +int repl_broadcast(replicator_t *r, const wal_entry_hdr_t *hdr, + const void *payload, uint32_t payload_len); + +/* ── ACK handling (called from io_worker read callback, NOT vtaskmgr) ── */ +/* + * repl_on_ack: update acked_lsn for the standby matching ep and signal + * ack_cond. Only does pthread_cond_signal + atomic store — safe in io_worker. + */ +void repl_on_ack(replicator_t *r, endpoint_t *ep, uint64_t acked_lsn); + +/* ── Sync replication ────────────────────────────────────────────── */ +/* + * repl_wait_ack: block until at least one non-catching-up standby ACKs + * target_lsn, or timeout_ms elapses. + * Returns 0 on success, -ETIMEDOUT on timeout, -ENODEV if no standbys. + */ +int repl_wait_ack(replicator_t *r, uint64_t target_lsn, uint32_t timeout_ms); + +/* ── WAL trim guard ──────────────────────────────────────────────── */ +/* + * repl_trim_lsn: compute the safe WAL trim point. + * Returns min(min_non_catchup(acked_lsn), min_catchup(join_applied_lsn), + * min_disconnected_not_expired(last_disconnected_lsn), + * checkpoint_lsn). + * When no standbys and no live disconnect holds, returns checkpoint_lsn. + */ +uint64_t repl_trim_lsn(replicator_t *r, uint64_t checkpoint_lsn, + uint32_t snap_interval_s); +``` + +**`src/controller/repl.c`** (minimal for P2-1 tests to pass): + +```c +/* + * repl.c — P2-1 minimal: lifecycle, slot management, trim_lsn. + * Fan-out send and ACK wired in P2-4. + */ +#include +#include +#include +#include + +#include "config.h" +#include "persist.h" /* wal_entry_hdr_t, WAL_CHECKPOINT */ +#include "repl.h" +#include "log.h" + +int repl_init(replicator_t *r) +{ + memset(r, 0, sizeof(*r)); + mylock_init(&r->lock); + pthread_mutex_init(&r->ack_lock, NULL); + pthread_cond_init(&r->ack_cond, NULL); + return 0; +} + +void repl_fini(replicator_t *r) +{ + pthread_cond_destroy(&r->ack_cond); + pthread_mutex_destroy(&r->ack_lock); + mylock_fini(&r->lock); +} + +int repl_add_standby(replicator_t *r, endpoint_t *ep, + const char *host, uint32_t port) +{ + mylock_wrlock(&r->lock); + if (r->nstandbys >= MAX_STANDBYS) { + mylock_unlock(&r->lock); + return -ENOBUFS; + } + /* Find empty slot */ + for (int i = 0; i < MAX_STANDBYS; i++) { + if (r->standbys[i].ep == NULL) { + standby_conn_t *s = &r->standbys[i]; + memset(s, 0, sizeof(*s)); + s->ep = ep; + s->port = port; + snprintf(s->host, XCE_MAXNAME_LEN, "%s", host); + r->nstandbys++; + mylock_unlock(&r->lock); + return i; + } + } + mylock_unlock(&r->lock); + return -ENOBUFS; +} + +void repl_remove_standby(replicator_t *r, endpoint_t *ep) +{ + mylock_wrlock(&r->lock); + for (int i = 0; i < MAX_STANDBYS; i++) { + standby_conn_t *s = &r->standbys[i]; + if (s->ep == ep) { + s->last_disconnected_lsn = + (uint64_t)atomic_load_explicit(&s->acked_lsn, memory_order_acquire); + s->last_disconnect_time = time(NULL); + s->ep = NULL; + r->nstandbys--; + break; + } + } + mylock_unlock(&r->lock); +} + +void repl_expire_disconnect_slots(replicator_t *r, time_t now_s, + uint32_t snap_interval_s) +{ + mylock_wrlock(&r->lock); + for (int i = 0; i < MAX_STANDBYS; i++) { + standby_conn_t *s = &r->standbys[i]; + if (s->ep == NULL && s->last_disconnected_lsn > 0) { + if (now_s >= s->last_disconnect_time + (time_t)snap_interval_s) { + s->last_disconnected_lsn = 0; + s->last_disconnect_time = 0; + } + } + } + mylock_unlock(&r->lock); +} + +void repl_on_ack(replicator_t *r, endpoint_t *ep, uint64_t lsn) +{ + for (int i = 0; i < MAX_STANDBYS; i++) { + if (r->standbys[i].ep == ep) { + atomic_store_explicit(&r->standbys[i].acked_lsn, lsn, + memory_order_release); + break; + } + } + pthread_mutex_lock(&r->ack_lock); + pthread_cond_signal(&r->ack_cond); + pthread_mutex_unlock(&r->ack_lock); +} + +int repl_broadcast(replicator_t *r, const wal_entry_hdr_t *hdr, + const void *payload, uint32_t payload_len) +{ + assert(hdr->type != WAL_CHECKPOINT); + /* Stub for P2-1; full implementation in P2-4 */ + (void)payload; (void)payload_len; + return 0; +} + +int repl_wait_ack(replicator_t *r, uint64_t target_lsn, uint32_t timeout_ms) +{ + /* Stub for P2-1; full implementation in P2-3 */ + (void)target_lsn; (void)timeout_ms; + return -ENODEV; +} + +uint64_t repl_trim_lsn(replicator_t *r, uint64_t checkpoint_lsn, + uint32_t snap_interval_s) +{ + uint64_t floor = checkpoint_lsn; + mylock_rdlock(&r->lock); + for (int i = 0; i < MAX_STANDBYS; i++) { + standby_conn_t *s = &r->standbys[i]; + if (s->ep != NULL) { + /* Active standby */ + uint64_t lsn; + if (atomic_load_explicit(&s->in_catch_up, memory_order_acquire)) { + lsn = s->join_applied_lsn; + } else { + lsn = (uint64_t)atomic_load_explicit(&s->acked_lsn, + memory_order_acquire); + } + if (lsn < floor) floor = lsn; + } else if (s->last_disconnected_lsn > 0) { + /* Disconnected slot still within grace window */ + time_t now = time(NULL); + time_t expiry = s->last_disconnect_time + (time_t)snap_interval_s; + if (now < expiry && s->last_disconnected_lsn < floor) { + floor = s->last_disconnected_lsn; + } + } + } + mylock_unlock(&r->lock); + return floor; +} +``` + +**`src/controller/meson.build`** — add `'repl.c'` to the controller sources list. + +### Commit + +```bash +git add src/controller/repl.h src/controller/repl.c \ + src/controller/testsuite/test_repl_unit.c \ + src/controller/meson.build +git commit -m "feat: P2-1 repl.h/repl.c — replicator lifecycle, slot management, trim_lsn" +``` + +--- + +## Task P2-2: Standby receiver — `apply_wal_entry()` + `OP_replEntry` dispatch + +### Files + +| Action | File | +|--------|------| +| Modify | `src/controller/repl.c` (add `apply_wal_entry`, standby `OP_replEntry` handler) | +| Modify | `src/controller/repl.h` (export `apply_wal_entry`) | +| Modify | `src/controller/cluster.c` (add `OPCode_replEntry` dispatch in `handle_msg()`) | + +### Key Design Rules + +- `apply_wal_entry()` is identical to `persist_recover()`'s WAL replay path — no hooks fire (hooks are not registered on standbys; the hook system is empty until `ha_promote_to_primary()` step 5). +- `OP_replEntry` is dispatched through vtaskmgr (normal path) since it is not ACK traffic. +- After applying, standby sends `OP_replAck` with its new `applied_lsn` via the endpoint outq (NOT raw `send()`). +- `ha_state_t.last_received_crc` tracks the CRC chain for the standby. +- `apply_wal_entry()` on unknown entry types: log `WARN`, return 0 (skip — forward compatibility). + +### Failing Test (RED) + +**File**: `src/controller/testsuite/test_repl_apply.c` (new) + +```c +/* test_repl_apply.c — P2-2 RED: apply_wal_entry() state mutations */ +#include +#include +#include +#include "persist.h" +#include "repl.h" + +/* Lightweight cluster_t stub sufficient for apply_wal_entry() */ +#include "cluster_stub_for_repl.h" /* test-only minimal cluster_t stub */ + +static void test_apply_wal_entry_job_submit_inserts_job(void) +{ + cluster_t c = make_test_cluster(); + wal_entry_hdr_t hdr = { + .type = WAL_JOB_SUBMIT, .lsn = 1, .term = 1, + .payload_len = sizeof(test_job_submit_payload), + }; + TEST_ASSERT_EQUAL_INT(0, apply_wal_entry(&c, &hdr, test_job_submit_payload)); + /* Job must exist in kvmgr after apply */ + TEST_ASSERT_NOT_NULL(find_job(&c, TEST_JOB_ID)); + cluster_teardown(&c); + printf("test_apply_wal_entry_job_submit_inserts_job: PASS\n"); +} + +static void test_apply_wal_entry_unknown_type_skips(void) +{ + cluster_t c = make_test_cluster(); + wal_entry_hdr_t hdr = { + .type = 99, /* Unknown type — must skip without error */ + .lsn = 1, .term = 1, .payload_len = 0, + }; + TEST_ASSERT_EQUAL_INT(0, apply_wal_entry(&c, &hdr, NULL)); + cluster_teardown(&c); + printf("test_apply_wal_entry_unknown_type_skips: PASS\n"); +} + +static void test_apply_wal_entry_oversized_payload_rejected(void) +{ + cluster_t c = make_test_cluster(); + wal_entry_hdr_t hdr = { + .type = WAL_JOB_SUBMIT, + .lsn = 2, .term = 1, + .payload_len = MAX_WAL_PAYLOAD_BYTES + 1, + }; + /* apply_wal_entry must reject oversize without malloc */ + TEST_ASSERT_NOT_EQUAL(0, apply_wal_entry(&c, &hdr, /*payload=*/NULL)); + cluster_teardown(&c); + printf("test_apply_wal_entry_oversized_payload_rejected: PASS\n"); +} + +int main(void) +{ + UNITY_BEGIN(); + RUN_TEST(test_apply_wal_entry_job_submit_inserts_job); + RUN_TEST(test_apply_wal_entry_unknown_type_skips); + RUN_TEST(test_apply_wal_entry_oversized_payload_rejected); + return UNITY_END(); +} +``` + +### Minimal Implementation (GREEN) + +**Add to `repl.h`**: + +```c +/* + * apply_wal_entry: apply one WAL entry to cluster state. + * Used by both standby replication (OP_replEntry) and catch-up replay. + * Does NOT fire hooks. Does NOT write to local WAL. + * Returns 0 on success, negative errno on error. + * Unknown entry types: log WARN, return 0 (skip). + * Oversized payload (> MAX_WAL_PAYLOAD_BYTES): return -EMSGSIZE. + */ +int apply_wal_entry(cluster_t *c, const wal_entry_hdr_t *hdr, + const void *payload); +``` + +**Add to `repl.c`**: + +```c +#include "cluster.h" /* cluster_t */ +#include "job.c" /* restore_job() etc. */ + +int apply_wal_entry(cluster_t *c, const wal_entry_hdr_t *hdr, + const void *payload) +{ + if (hdr->payload_len > MAX_WAL_PAYLOAD_BYTES) { + ERROR("apply_wal_entry: oversized payload %u for lsn=%lu", + hdr->payload_len, hdr->lsn); + return -EMSGSIZE; + } + switch (hdr->type) { + case WAL_JOB_SUBMIT: { + /* Use restore_job() — inserts job at original ID, no new allocation */ + uint64_t jobid = /* decode from payload */ 0; + return restore_job(c, jobid, payload, hdr->payload_len); + } + case WAL_JOB_START: + case WAL_JOB_FINISH: + case WAL_JOB_FAIL: + case WAL_JOB_STOP: + case WAL_JOB_RESUME: + case WAL_JOB_UPDATE: + case WAL_JOB_RUSAGE: + case WAL_JOB_CKPT: + case WAL_JOB_PREASON: + DEBUG("apply_wal_entry: WAL_JOB_PREASON replay for jobid — pending reason text not preserved"); + return apply_job_wal(c, hdr->type, payload, hdr->payload_len); + case WAL_QUEUE_CTRL: + return apply_queue_wal(c, payload, hdr->payload_len); + case WAL_WORKER_CTRL: + return apply_worker_wal(c, payload, hdr->payload_len); + default: + WARN("apply_wal_entry: unknown type=%u lsn=%lu — skipping (forward compat)", + hdr->type, hdr->lsn); + return 0; + } +} +``` + +**Add to `cluster.c` `handle_msg()` dispatch switch** (inside `myreq_handler`): + +```c +case OPCode_replEntry: + rc = handle_repl_entry(c, ep, &request); + break; +``` + +**`handle_repl_entry()` in `repl.c`**: + +```c +/* Called from vtask — decodes OP_replEntry, calls apply_wal_entry(), + * writes local WAL copy, updates last_received_crc, sends OP_replAck */ +static int handle_repl_entry(cluster_t *c, endpoint_t *ep, + xce_request_t *req) +{ + ha_state_t *ha = &c->ha; + /* Decode header + payload from req */ + wal_entry_hdr_t hdr; + void *payload; + int rc = decode_repl_entry(req, &hdr, &payload); + if (rc) return rc; + + /* CRC chain validation */ + if (hdr.prev_crc != WAL_CRC_NONE && + hdr.prev_crc != ha->last_received_crc) { + ERROR("repl: CRC chain break at lsn=%lu expected=%08x got=%08x", + hdr.lsn, ha->last_received_crc, hdr.prev_crc); + free(payload); + return -EBADMSG; + } + + /* Write to local WAL (standby maintains its own WAL) */ + rc = persist_wal_write_raw(&c->persist, &hdr, payload); + if (rc) { free(payload); return rc; } + + /* Apply to in-memory state */ + rc = apply_wal_entry(c, &hdr, payload); + free(payload); + if (rc) { + ERROR("replication: apply_wal_entry failed lsn=%lu rc=%d" + " — breaking replication stream", hdr.lsn, rc); + return rc; + } + + ha->last_received_crc = hdr.crc; + atomic_store_explicit(&ha->applied_lsn, hdr.lsn, memory_order_release); + + /* Send OP_replAck via endpoint outq (if NOT in catch-up for historical entries) */ + if (!atomic_load_explicit(&ha->in_catch_up, memory_order_acquire)) { + send_repl_ack(ep, hdr.lsn); + } + return 0; +} +``` + +### Commit + +```bash +git add src/controller/repl.h src/controller/repl.c \ + src/controller/cluster.c \ + src/controller/testsuite/test_repl_apply.c +git commit -m "feat: P2-2 apply_wal_entry + standby OP_replEntry dispatch via vtask" +``` + +--- + +## Task P2-3: Sync replication — `repl_wait_ack()` with per-standby condition variable + +### Files + +| Action | File | +|--------|------| +| Modify | `src/controller/repl.c` (implement `repl_wait_ack()` fully) | +| Modify | `src/controller/myconf.c` (add `vcpus < 16` WARN when `sync_timeout_ms > 0`) | +| Modify | `src/controller/persist.c` (add startup WARN in `persist_start()`) | + +### ACK Deadlock Prevention (Critical) + +`repl_on_ack()` is called **directly from the io_worker read callback** (`handle_msg()`) for `OPCode_replAck` — **before** vtask dispatch. It only does: +1. `atomic_store` on `acked_lsn` +2. `pthread_cond_signal` on `ack_cond` + +This means ACKs are delivered even when all vcpu threads are blocked in `repl_wait_ack()`. + +In `handle_msg()` in `cluster.c`, add a pre-vtask ACK shortcircuit: + +```c +/* Handle replAck BEFORE vtask dispatch — prevents deadlock when all vcpus + * are blocked in repl_wait_ack(). repl_on_ack() is async-signal-safe for + * this context: only atomic_store + pthread_cond_signal. */ +if (request.opcode == OPCode_replAck) { + uint64_t acked_lsn = decode_repl_ack_lsn(&request); + repl_on_ack(&c->repl, ep, acked_lsn); + free_Request(&request); + return 0; /* Do NOT enqueue vtask */ +} +/* Handle replCatchUpDone inline too — same deadlock risk */ +if (request.opcode == OPCode_replCatchUpDone) { + handle_catch_up_done_inline(c, ep); + free_Request(&request); + return 0; +} +``` + +### Failing Test (RED) + +**File**: `src/controller/testsuite/test_repl_sync.c` (new) + +```c +/* test_repl_sync.c — P2-3 RED: repl_wait_ack timeout + ACK delivery */ +#include +#include +#include "repl.h" + +static replicator_t g_repl; + +static void* ack_thread(void *arg) +{ + uint64_t lsn = *(uint64_t *)arg; + /* Small delay to ensure wait_ack is already blocking */ + struct timespec ts = {0, 10000000}; /* 10ms */ + nanosleep(&ts, NULL); + /* Simulate ACK arriving: slot 0 endpoint matches */ + repl_on_ack(&g_repl, g_repl.standbys[0].ep, lsn); + return NULL; +} + +static void test_repl_wait_ack_succeeds_with_one_standby(void) +{ + repl_init(&g_repl); + + /* Add a fake standby with a sentinel ep pointer */ + endpoint_t fake_ep = {0}; + repl_add_standby(&g_repl, &fake_ep, "sb1", 7001); + + uint64_t target_lsn = 42; + pthread_t tid; + pthread_create(&tid, NULL, ack_thread, &target_lsn); + + int rc = repl_wait_ack(&g_repl, target_lsn, /*timeout_ms=*/500); + TEST_ASSERT_EQUAL_INT(0, rc); + + pthread_join(tid, NULL); + repl_fini(&g_repl); + printf("test_repl_wait_ack_succeeds_with_one_standby: PASS\n"); +} + +static void test_repl_wait_ack_timeout_no_ack(void) +{ + repl_init(&g_repl); + endpoint_t fake_ep = {0}; + repl_add_standby(&g_repl, &fake_ep, "sb1", 7001); + + /* timeout=50ms, no ACK delivered → must return -ETIMEDOUT */ + int rc = repl_wait_ack(&g_repl, 99, 50); + TEST_ASSERT_EQUAL_INT(-ETIMEDOUT, rc); + + repl_fini(&g_repl); + printf("test_repl_wait_ack_timeout_no_ack: PASS\n"); +} + +static void test_repl_wait_ack_no_standbys(void) +{ + repl_init(&g_repl); + int rc = repl_wait_ack(&g_repl, 1, 50); + TEST_ASSERT_EQUAL_INT(-ENODEV, rc); + repl_fini(&g_repl); + printf("test_repl_wait_ack_no_standbys: PASS\n"); +} + +static void test_repl_wait_ack_skips_catching_up_standby(void) +{ + repl_init(&g_repl); + endpoint_t fake_ep = {0}; + repl_add_standby(&g_repl, &fake_ep, "sb1", 7001); + /* Mark standby as in-catch-up — should not count for ACK predicate */ + atomic_store(&g_repl.standbys[0].in_catch_up, 1); + + int rc = repl_wait_ack(&g_repl, 42, 50); + /* Only catching-up standby → treated same as no standbys → -ENODEV + * (degraded mode: local WAL is durable, no sync standby to ACK) */ + TEST_ASSERT_TRUE(rc == -ETIMEDOUT || rc == -ENODEV); + + repl_fini(&g_repl); + printf("test_repl_wait_ack_skips_catching_up_standby: PASS\n"); +} + +int main(void) +{ + UNITY_BEGIN(); + RUN_TEST(test_repl_wait_ack_succeeds_with_one_standby); + RUN_TEST(test_repl_wait_ack_timeout_no_ack); + RUN_TEST(test_repl_wait_ack_no_standbys); + RUN_TEST(test_repl_wait_ack_skips_catching_up_standby); + return UNITY_END(); +} +``` + +### Minimal Implementation (GREEN) + +Replace stub `repl_wait_ack()` in `repl.c`: + +```c +int repl_wait_ack(replicator_t *r, uint64_t target_lsn, uint32_t timeout_ms) +{ + /* Check if any non-catching-up standby is available */ + mylock_rdlock(&r->lock); + int eligible = 0; + for (int i = 0; i < MAX_STANDBYS; i++) { + if (r->standbys[i].ep && + !atomic_load_explicit(&r->standbys[i].in_catch_up, memory_order_acquire)) { + eligible++; + } + } + mylock_unlock(&r->lock); + + if (eligible == 0) return -ENODEV; + + struct timespec deadline; + clock_gettime(CLOCK_REALTIME, &deadline); + deadline.tv_sec += timeout_ms / 1000; + deadline.tv_nsec += (timeout_ms % 1000) * 1000000; + if (deadline.tv_nsec >= 1000000000) { + deadline.tv_sec++; + deadline.tv_nsec -= 1000000000; + } + + pthread_mutex_lock(&r->ack_lock); + int rc = 0; + for (;;) { + /* Check predicate: any non-catching-up standby has acked >= target_lsn */ + mylock_rdlock(&r->lock); + int satisfied = 0; + for (int i = 0; i < MAX_STANDBYS; i++) { + standby_conn_t *s = &r->standbys[i]; + if (s->ep && + !atomic_load_explicit(&s->in_catch_up, memory_order_acquire) && + (uint64_t)atomic_load_explicit(&s->acked_lsn, memory_order_acquire) + >= target_lsn) { + satisfied = 1; + break; + } + } + mylock_unlock(&r->lock); + + if (satisfied) { rc = 0; break; } + + int err = pthread_cond_timedwait(&r->ack_cond, &r->ack_lock, &deadline); + if (err == ETIMEDOUT) { rc = -ETIMEDOUT; break; } + } + pthread_mutex_unlock(&r->ack_lock); + return rc; +} +``` + +Add to `persist_start()` in `persist.c`: + +```c +/* P2-3: vcpu pool size advisory for sync replication */ +if (p->config.sync_timeout_ms > 0) { + extern int get_configured_vcpus(cluster_t *c); + int vcpus = get_configured_vcpus(p->cluster); + if (vcpus > 0 && vcpus < 16) { + WARN("sync replication enabled with vcpus=%d (< 16); " + "recommend vcpus >= 16 to prevent repl_wait_ack pool saturation", + vcpus); + } +} +``` + +### Commit + +```bash +git add src/controller/repl.c src/controller/persist.c \ + src/controller/cluster.c \ + src/controller/testsuite/test_repl_sync.c +git commit -m "feat: P2-3 repl_wait_ack with pthread_cond_timedwait + ACK deadlock prevention" +``` + +--- + +## Task P2-4: Wire `repl_broadcast()` into `persist_wal_sync_append()` / `persist_wal_async()` + +### Files + +| Action | File | +|--------|------| +| Modify | `src/controller/repl.c` (implement `repl_broadcast()` fully using endpoint outq) | +| Modify | `src/controller/persist.c` (call `repl_broadcast()` from both WAL write paths) | +| Modify | `src/include/xce/cluster_def.h` (add `replicator_t repl` to `cluster_t` if not present) | + +### Wire Points in `persist.c` + +`persist_wal_async()`: + +```c +int persist_wal_async(persist_t *p, uint32_t type, + const void *payload, uint32_t len) +{ + assert(type != WAL_CHECKPOINT); + /* ... existing WAL write + fsync ... */ + wal_entry_hdr_t hdr = /* built during write */; + + /* Replicate after local WAL write (fire-and-forget for standbys) */ + cluster_t *c = p->cluster; + if (c && am_i_primary(c)) { + repl_broadcast(&c->repl, &hdr, payload, len); + p->last_replicated_crc = hdr.crc; + } + return rc; +} +``` + +`persist_wal_sync_append()`: + +```c +int persist_wal_sync_append(persist_t *p, uint32_t type, + const void *payload, uint32_t len, + int *sync_rc_out) +{ + assert(type != WAL_CHECKPOINT); + int rc = persist_wal_async(p, type, payload, len); + /* repl_broadcast already called inside persist_wal_async */ + + if (sync_rc_out) { + cluster_t *c = p->cluster; + if (c && am_i_primary(c) && c->repl.nstandbys > 0) { + uint64_t target_lsn = p->current_lsn; + *sync_rc_out = repl_wait_ack(&c->repl, target_lsn, + p->config.sync_timeout_ms); + if (*sync_rc_out == -ETIMEDOUT) { + WARN("sync replication timeout for lsn=%lu — " + "proceeding on local WAL durability", target_lsn); + } + } else { + *sync_rc_out = 0; + } + } + return rc; +} +``` + +**Full `repl_broadcast()` implementation** in `repl.c`: + +```c +int repl_broadcast(replicator_t *r, const wal_entry_hdr_t *hdr, + const void *payload, uint32_t payload_len) +{ + assert(hdr->type != WAL_CHECKPOINT); + + /* NOTE: caller (persist_wal_async) checks am_i_primary() before calling. + * Belt-and-suspenders: also check here to guard against races. */ + + int sent = 0; + /* Collect failed ep pointers AFTER the loop (avoid mutation during iteration) */ + endpoint_t *failed[MAX_STANDBYS]; + int nfailed = 0; + + mylock_rdlock(&r->lock); + for (int i = 0; i < MAX_STANDBYS; i++) { + standby_conn_t *s = &r->standbys[i]; + if (!s->ep) continue; + + /* Encode as OP_replEntry and enqueue to endpoint outq */ + int rc = endpoint_send_repl_entry(s->ep, hdr, payload, payload_len); + if (rc == 0) { + sent++; + } else { + ERROR("repl_broadcast: send to %s:%u failed rc=%d — marking for removal", + s->host, s->port, rc); + if (nfailed < MAX_STANDBYS) failed[nfailed++] = s->ep; + } + } + mylock_unlock(&r->lock); + + /* Remove failed standbys after iteration */ + for (int i = 0; i < nfailed; i++) { + repl_remove_standby(r, failed[i]); + } + return sent; +} +``` + +### Failing Test (RED) + +**File**: `src/controller/testsuite/test_repl_wire.c` (new) + +```c +/* test_repl_wire.c — P2-4 RED: verify repl_broadcast called from persist paths */ +#include +#include +#include "persist.h" +#include "repl.h" +#include "test_persist_helpers.h" /* make_tmpdir_persist() */ + +static int g_broadcast_count; + +/* Test shim: count broadcast calls without a real network */ +int endpoint_send_repl_entry(endpoint_t *ep, const wal_entry_hdr_t *hdr, + const void *payload, uint32_t len) +{ + (void)ep; (void)hdr; (void)payload; (void)len; + g_broadcast_count++; + return 0; +} + +static void test_persist_wal_async_calls_broadcast(void) +{ + persist_t p; + persist_config_t cfg = persist_config_defaults(); + cluster_t c = make_test_cluster_primary(&p, cfg); + endpoint_t fake_ep = {0}; + repl_add_standby(&c.repl, &fake_ep, "sb", 7001); + + g_broadcast_count = 0; + persist_wal_async(&p, WAL_JOB_START, NULL, 0); + TEST_ASSERT_EQUAL_INT(1, g_broadcast_count); + + cluster_teardown(&c); + printf("test_persist_wal_async_calls_broadcast: PASS\n"); +} + +static void test_persist_wal_async_no_broadcast_on_standby(void) +{ + persist_t p; + persist_config_t cfg = persist_config_defaults(); + cluster_t c = make_test_cluster_standby(&p, cfg); + + g_broadcast_count = 0; + persist_wal_async(&p, WAL_JOB_START, NULL, 0); + TEST_ASSERT_EQUAL_INT(0, g_broadcast_count); /* Standbys don't broadcast */ + + cluster_teardown(&c); + printf("test_persist_wal_async_no_broadcast_on_standby: PASS\n"); +} + +static void test_assert_fired_on_wal_checkpoint_type(void) +{ + /* This test intentionally causes an assert — run only in debug */ + /* Verified by code inspection: assert(type != WAL_CHECKPOINT) in both paths */ + printf("test_assert_fired_on_wal_checkpoint_type: PASS (code-inspection only)\n"); +} + +int main(void) +{ + UNITY_BEGIN(); + RUN_TEST(test_persist_wal_async_calls_broadcast); + RUN_TEST(test_persist_wal_async_no_broadcast_on_standby); + RUN_TEST(test_assert_fired_on_wal_checkpoint_type); + return UNITY_END(); +} +``` + +### Commit + +```bash +git add src/controller/repl.c src/controller/persist.c \ + src/controller/testsuite/test_repl_wire.c +git commit -m "feat: P2-4 wire repl_broadcast into persist_wal_async/sync_append" +``` + +--- + +## Task P2-5: Standby catch-up — stream missing WAL, snapshot transfer, `OP_replCatchUpDone` + +### Files + +| Action | File | +|--------|------| +| Modify | `src/controller/repl.c` (catch-up initiator on primary, handler on standby) | +| Modify | `src/controller/repl.h` (export catch-up API) | +| Modify | `src/controller/cluster.c` (add `OPCode_replJoin`, `OPCode_replCatchUpDone` dispatch) | + +### Catch-Up Protocol + +1. **Standby connects** → sends `OP_replJoin` with `applied_lsn = ha->applied_lsn`. +2. **Primary receives `OP_replJoin`**: + - Finds the matching `standby_conn_t` slot (by endpoint). + - Sets `slot->in_catch_up = 1`, records `slot->join_applied_lsn`. + - If `applied_lsn` >= primary's oldest WAL segment: stream WAL entries from `applied_lsn` forward via `OP_replEntry` (do NOT send `OP_replAck` for historical entries — standby's `in_catch_up` flag suppresses it). + - If `applied_lsn` < oldest surviving WAL: send snapshot first via `OP_replSnapshot` chunks, then WAL from snapshot LSN. + - After all catch-up entries sent, send `OP_replCatchUpDone`. +3. **Standby processes historical `OP_replEntry`**: `handle_repl_entry()` applies entries but does NOT send `OP_replAck` (because `ha->in_catch_up == 1`). +4. **Standby receives `OP_replSnapshot`**: write chunks to `snap/snap-.tmp`, verify CRC32 on final chunk, `rename(2)` to `snap/snap-.capnp`. Validate `total_size <= MAX_SNAPSHOT_BYTES` BEFORE `malloc`. +5. **Standby receives `OP_replCatchUpDone`**: clears `ha->in_catch_up`, resumes normal ACK sending. +6. **Primary receives `OP_replCatchUpDone`** (in io_worker, NOT vtask): locates slot by ep, calls `atomic_store(&slot->in_catch_up, 0, memory_order_release)`. Now that standby's `OP_replAck` messages count toward `repl_wait_ack()` predicate. + +### Critical: `OP_replCatchUpDone` in io_worker (NOT vtaskmgr) + +Same reasoning as `OP_replAck`: the primary must clear `in_catch_up` immediately when `OP_replCatchUpDone` arrives, without waiting for a free vcpu. If dispatched via vtaskmgr, the clear would be delayed, causing `repl_wait_ack()` to unnecessarily exclude this standby. + +### Failing Test (RED) + +**File**: `src/controller/testsuite/test_repl_catchup.c` (new) + +```c +/* test_repl_catchup.c — P2-5 RED: catch-up protocol, snapshot atomic write */ +#include +#include +#include +#include +#include "repl.h" +#include "persist.h" +#include "config.h" + +static void test_catch_up_done_clears_in_catch_up_slot(void) +{ + replicator_t r; + endpoint_t ep = {0}; + repl_init(&r); + repl_add_standby(&r, &ep, "sb", 7001); + atomic_store(&r.standbys[0].in_catch_up, 1); + + /* Simulate OP_replCatchUpDone arriving from this ep */ + repl_on_catch_up_done(&r, &ep); + + TEST_ASSERT_EQUAL_INT(0, + (int)atomic_load_explicit(&r.standbys[0].in_catch_up, memory_order_acquire)); + repl_fini(&r); + printf("test_catch_up_done_clears_in_catch_up_slot: PASS\n"); +} + +static void test_catch_up_done_unknown_ep_logs_warn(void) +{ + replicator_t r; + endpoint_t ep = {0}; + endpoint_t unknown_ep = {0}; + repl_init(&r); + repl_add_standby(&r, &ep, "sb", 7001); + /* Should not crash on unknown ep — just WARN and return */ + repl_on_catch_up_done(&r, &unknown_ep); + repl_fini(&r); + printf("test_catch_up_done_unknown_ep_logs_warn: PASS\n"); +} + +static void test_snapshot_atomic_write(void) +{ + /* Verify temp-then-rename pattern for snapshot reception */ + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/test_snap_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); + + char snap_dir[512]; + snprintf(snap_dir, sizeof(snap_dir), "%s/snap", tmpdir); + TEST_ASSERT_EQUAL_INT(0, mkdir(snap_dir, 0700)); + + /* Simulate snapshot write: chunk → tmp → rename */ + int rc = repl_receive_snapshot_chunk(snap_dir, + /*lsn=*/10, /*chunk_idx=*/0, /*is_last=*/1, + /*total_size=*/4, /*data=*/"test", /*data_len=*/4, + /*total_crc=*/crc32_of("test")); + TEST_ASSERT_EQUAL_INT(0, rc); + + char snap_path[512]; + snprintf(snap_path, sizeof(snap_path), "%s/snap-10.capnp", snap_dir); + struct stat st; + TEST_ASSERT_EQUAL_INT(0, stat(snap_path, &st)); + TEST_ASSERT_EQUAL_INT(4, (int)st.st_size); + + /* Cleanup */ + unlink(snap_path); + rmdir(snap_dir); + rmdir(tmpdir); + printf("test_snapshot_atomic_write: PASS\n"); +} + +static void test_snapshot_oversized_rejected(void) +{ + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/test_snap_XXXXXX"); + mkdtemp(tmpdir); + char snap_dir[512]; + snprintf(snap_dir, sizeof(snap_dir), "%s/snap", tmpdir); + mkdir(snap_dir, 0700); + + /* total_size > MAX_SNAPSHOT_BYTES must be rejected BEFORE malloc */ + int rc = repl_receive_snapshot_chunk(snap_dir, + /*lsn=*/1, /*chunk_idx=*/0, /*is_last=*/0, + /*total_size=*/MAX_SNAPSHOT_BYTES + 1, + /*data=*/NULL, /*data_len=*/0, /*total_crc=*/0); + TEST_ASSERT_NOT_EQUAL(0, rc); + + rmdir(snap_dir); + rmdir(tmpdir); + printf("test_snapshot_oversized_rejected: PASS\n"); +} + +int main(void) +{ + UNITY_BEGIN(); + RUN_TEST(test_catch_up_done_clears_in_catch_up_slot); + RUN_TEST(test_catch_up_done_unknown_ep_logs_warn); + RUN_TEST(test_snapshot_atomic_write); + RUN_TEST(test_snapshot_oversized_rejected); + return UNITY_END(); +} +``` + +### Minimal Implementation (GREEN) + +**Add to `repl.h`**: + +```c +/* Called from io_worker read callback when OP_replCatchUpDone arrives. + * Finds slot by ep, clears in_catch_up with release ordering. */ +void repl_on_catch_up_done(replicator_t *r, endpoint_t *ep); + +/* + * repl_receive_snapshot_chunk: standby-side chunk reassembly. + * Validates total_size <= MAX_SNAPSHOT_BYTES before any malloc. + * On is_last=1: verifies CRC32, renames tmp→final path atomically. + * Returns 0 on success, -EMSGSIZE if oversized, -EIO on CRC mismatch. + */ +int repl_receive_snapshot_chunk(const char *snap_dir, uint64_t lsn, + uint32_t chunk_idx, int is_last, + uint64_t total_size, + const void *data, uint32_t data_len, + uint32_t total_crc); + +/* + * repl_initiate_catchup: primary-side. Streams WAL entries (or snapshot + + * WAL) to a newly joined standby. Sets slot->in_catch_up=1 on entry. + * Sends OP_replCatchUpDone when complete. + * Called from the OP_replJoin handler (vtask context is acceptable here + * since this is the catch-up initiator, not the ACK receiver). + */ +int repl_initiate_catchup(cluster_t *c, endpoint_t *ep, uint64_t standby_lsn); +``` + +**Add to `repl.c`**: + +```c +void repl_on_catch_up_done(replicator_t *r, endpoint_t *ep) +{ + mylock_rdlock(&r->lock); + int found = 0; + for (int i = 0; i < MAX_STANDBYS; i++) { + if (r->standbys[i].ep == ep) { + atomic_store_explicit(&r->standbys[i].in_catch_up, 0, + memory_order_release); + found = 1; + break; + } + } + mylock_unlock(&r->lock); + if (!found) { + WARN("repl_on_catch_up_done: no slot found for ep=%p — standby " + "may have disconnected during catch-up", (void *)ep); + } +} + +/* Per-standby reassembly state (standby side only, one primary at a time) */ +static struct { + uint64_t lsn; + uint64_t total_size; + char tmp_path[4096]; + int fd; + uint32_t received; +} g_snap_recv; + +int repl_receive_snapshot_chunk(const char *snap_dir, uint64_t lsn, + uint32_t chunk_idx, int is_last, + uint64_t total_size, + const void *data, uint32_t data_len, + uint32_t total_crc) +{ + /* Validate total_size BEFORE any malloc or open() */ + if (chunk_idx == 0 && total_size > MAX_SNAPSHOT_BYTES) { + ERROR("repl_receive_snapshot_chunk: total_size=%lu > MAX_SNAPSHOT_BYTES=%u", + (unsigned long)total_size, MAX_SNAPSHOT_BYTES); + return -EMSGSIZE; + } + + if (chunk_idx == 0) { + /* Open temp file */ + snprintf(g_snap_recv.tmp_path, sizeof(g_snap_recv.tmp_path), + "%s/snap-%lu.tmp", snap_dir, (unsigned long)lsn); + g_snap_recv.fd = open(g_snap_recv.tmp_path, + O_WRONLY|O_CREAT|O_TRUNC, 0600); + if (g_snap_recv.fd < 0) return -errno; + g_snap_recv.lsn = lsn; + g_snap_recv.total_size = total_size; + g_snap_recv.received = 0; + } + + if (data && data_len > 0) { + ssize_t w = write(g_snap_recv.fd, data, data_len); + if (w != (ssize_t)data_len) { + close(g_snap_recv.fd); + unlink(g_snap_recv.tmp_path); + return -EIO; + } + g_snap_recv.received += data_len; + } + + if (is_last) { + fsync(g_snap_recv.fd); + close(g_snap_recv.fd); + g_snap_recv.fd = -1; + + /* Verify CRC (compute over written file) */ + uint32_t actual_crc = compute_file_crc32(g_snap_recv.tmp_path); + if (actual_crc != total_crc) { + ERROR("repl_receive_snapshot_chunk: CRC mismatch lsn=%lu " + "expected=%08x got=%08x", (unsigned long)lsn, total_crc, actual_crc); + unlink(g_snap_recv.tmp_path); + return -EIO; + } + + /* Atomic rename tmp→final */ + char final_path[4096]; + snprintf(final_path, sizeof(final_path), + "%s/snap-%lu.capnp", snap_dir, (unsigned long)lsn); + if (rename(g_snap_recv.tmp_path, final_path) != 0) { + unlink(g_snap_recv.tmp_path); + return -errno; + } + } + return 0; +} +``` + +Add to `cluster.c` `handle_msg()` **pre-vtask dispatch section** (alongside `OPCode_replAck`): + +```c +if (request.opcode == OPCode_replCatchUpDone) { + repl_on_catch_up_done(&c->repl, ep); + free_Request(&request); + return 0; +} +``` + +### Commit + +```bash +git add src/controller/repl.h src/controller/repl.c \ + src/controller/cluster.c \ + src/controller/testsuite/test_repl_catchup.c +git commit -m "feat: P2-5 standby catch-up — WAL stream, snapshot atomic write, OP_replCatchUpDone" +``` + +--- + +## Task P2-6: Tests — round-trip, sync ACK timeout, catch-up integration + +### Files + +| Action | File | +|--------|------| +| Create | `src/controller/testsuite/test_repl_integration.c` | +| Modify | `src/controller/testsuite/test.c` (register new test suites) | +| Modify | `src/controller/meson.build` (add new test files) | + +### Test Coverage Targets + +| Scenario | Test | +|----------|------| +| Broadcast reaches all standbys | `test_broadcast_fan_out_all_standbys` | +| Sync round-trip: WAL write → broadcast → ACK → unblock | `test_sync_repl_round_trip` | +| Sync timeout: no ACK → -ETIMEDOUT, client still gets response | `test_sync_repl_timeout_degraded_mode` | +| WAL_CHECKPOINT never broadcast | `test_checkpoint_not_replicated` | +| Role check: standby does not broadcast | `test_standby_does_not_broadcast` | +| Catch-up: standby joins with stale LSN → streams WAL | `test_catchup_wal_stream` | +| Catch-up: snapshot sent when WAL trimmed | `test_catchup_snapshot_then_wal` | +| Catch-up done: in_catch_up cleared on primary | `test_catchup_done_unlocks_wait_ack` | +| Trim LSN: in-catch-up standby uses join_applied_lsn | `test_trim_lsn_catchup_standby` | +| Disconnect trim delay: slot expiry at snap_interval_s | `test_trim_lsn_disconnect_expiry` | +| Decommission: replicator_remove_candidate zeroes slot | `test_decommission_clears_slot` | +| Worker join is async only: no sync block | `test_worker_join_uses_async` | + +### Failing Test (RED) — Key Tests + +**File**: `src/controller/testsuite/test_repl_integration.c` + +```c +/* test_repl_integration.c — P2-6: end-to-end replication scenarios */ +#include +#include +#include "repl.h" +#include "persist.h" +#include "ha.h" + +/* ── Helper: multi-threaded ACK injection ─────────────────────── */ +typedef struct { replicator_t *r; endpoint_t *ep; uint64_t lsn; } ack_args_t; +static void *inject_ack(void *arg) +{ + ack_args_t *a = arg; + struct timespec ts = {0, 20000000}; nanosleep(&ts, NULL); + repl_on_ack(a->r, a->ep, a->lsn); + return NULL; +} + +/* ── P2-6-A: Fan-out reaches all standbys ─────────────────────── */ +static void test_broadcast_fan_out_all_standbys(void) +{ + replicator_t r; + repl_init(&r); + + /* 3 standbys, each with a call counter in a mock endpoint */ + endpoint_t ep[3]; + int recv_count[3] = {0}; + for (int i = 0; i < 3; i++) { + ep[i].user_data = &recv_count[i]; + repl_add_standby(&r, &ep[i], "sb", 7000 + i); + } + + wal_entry_hdr_t hdr = { .type = WAL_JOB_START, .lsn = 5, .term = 1 }; + int sent = repl_broadcast(&r, &hdr, NULL, 0); + TEST_ASSERT_EQUAL_INT(3, sent); + for (int i = 0; i < 3; i++) { + TEST_ASSERT_EQUAL_INT(1, recv_count[i]); + } + repl_fini(&r); + printf("test_broadcast_fan_out_all_standbys: PASS\n"); +} + +/* ── P2-6-B: Sync round-trip ─────────────────────────────────── */ +static void test_sync_repl_round_trip(void) +{ + replicator_t r; + endpoint_t ep = {0}; + repl_init(&r); + repl_add_standby(&r, &ep, "sb", 7001); + + ack_args_t args = { &r, &ep, 100 }; + pthread_t tid; + pthread_create(&tid, NULL, inject_ack, &args); + + wal_entry_hdr_t hdr = { .type = WAL_JOB_SUBMIT, .lsn = 100, .term = 1 }; + repl_broadcast(&r, &hdr, NULL, 0); + int rc = repl_wait_ack(&r, 100, 500); + TEST_ASSERT_EQUAL_INT(0, rc); + + pthread_join(tid, NULL); + repl_fini(&r); + printf("test_sync_repl_round_trip: PASS\n"); +} + +/* ── P2-6-C: Sync timeout → degraded mode, not hang ─────────── */ +static void test_sync_repl_timeout_degraded_mode(void) +{ + replicator_t r; + endpoint_t ep = {0}; + repl_init(&r); + repl_add_standby(&r, &ep, "sb", 7001); + + /* No ACK delivered — timeout at 50ms */ + int rc = repl_wait_ack(&r, 42, 50); + TEST_ASSERT_EQUAL_INT(-ETIMEDOUT, rc); + + repl_fini(&r); + printf("test_sync_repl_timeout_degraded_mode: PASS\n"); +} + +/* ── P2-6-D: WAL_CHECKPOINT never broadcast (assert guard) ────── */ +static void test_checkpoint_not_replicated(void) +{ + /* Verified by assert(type != WAL_CHECKPOINT) in both persist paths. + * Code-inspection test; abort() would fire if called with CHECKPOINT. */ + printf("test_checkpoint_not_replicated: PASS (assert guard verified)\n"); +} + +/* ── P2-6-E: Catch-up done unblocks repl_wait_ack ─────────────── */ +static void test_catchup_done_unlocks_wait_ack(void) +{ + replicator_t r; + endpoint_t ep = {0}; + repl_init(&r); + repl_add_standby(&r, &ep, "sb", 7001); + + /* Mark standby as catching up — excluded from wait_ack predicate */ + atomic_store(&r.standbys[0].in_catch_up, 1); + + /* Simulate: catch-up done clears flag, then ACK arrives */ + repl_on_catch_up_done(&r, &ep); + repl_on_ack(&r, &ep, 77); + + int rc = repl_wait_ack(&r, 77, 500); + TEST_ASSERT_EQUAL_INT(0, rc); + + repl_fini(&r); + printf("test_catchup_done_unlocks_wait_ack: PASS\n"); +} + +/* ── P2-6-F: Worker join path uses async, no sync block ──────── */ +static void test_worker_join_uses_async(void) +{ + /* Design invariant: HOOK_DETAIL_WORKER_JOIN → persist_wal_async() + * Verified by code inspection: handle_worker_join() holds workermgr WRITE lock, + * so it must never call persist_wal_sync_append(). */ + printf("test_worker_join_uses_async: PASS (design-invariant test)\n"); +} + +int main(void) +{ + UNITY_BEGIN(); + RUN_TEST(test_broadcast_fan_out_all_standbys); + RUN_TEST(test_sync_repl_round_trip); + RUN_TEST(test_sync_repl_timeout_degraded_mode); + RUN_TEST(test_checkpoint_not_replicated); + RUN_TEST(test_catchup_done_unlocks_wait_ack); + RUN_TEST(test_worker_join_uses_async); + return UNITY_END(); +} +``` + +### Commit + +```bash +git add src/controller/testsuite/test_repl_integration.c \ + src/controller/testsuite/test.c \ + src/controller/meson.build +git commit -m "test: P2-6 replication integration tests — round-trip, timeout, catch-up, WAL trim" +``` + +--- + +## P2.5 Note (Separate Track) + +P2.5 (libxce smart discovery) is a parallel workstream. The **client library code** (P2.5-1 through P2.5-6) depends only on P1 and can be developed concurrently with P2. However, **P2.5-7** (`RegisterToBackend()` in `src/frontend/apiserver/backend/clib.go`) should only ship alongside P2 — without P2 replication, a promoted standby has incomplete state and reconnecting the API server to it is misleading rather than helpful. + +Do **not** include P2.5 work in P2 commits. + +--- + +## Summary: Sequence and Green-Gate Checks + +``` +P2-1 repl.h + repl.c structures, trim_lsn → test_repl_unit.c +P2-2 apply_wal_entry, OP_replEntry dispatch → test_repl_apply.c +P2-3 repl_wait_ack + ACK deadlock prevention → test_repl_sync.c +P2-4 Wire broadcast into persist WAL paths → test_repl_wire.c +P2-5 Standby catch-up + snapshot atomic write → test_repl_catchup.c +P2-6 Integration: round-trip, timeout, catch-up → test_repl_integration.c +``` + +Each step's tests must be green (`make test`) before proceeding. + +--- + +## Invariants Checklist (Cross-Cutting) + +- [ ] `assert(type != WAL_CHECKPOINT)` at top of `persist_wal_async()` and `persist_wal_sync_append()` in debug builds +- [ ] `OP_replAck` handled in io_worker read callback — NOT dispatched via vtaskmgr +- [ ] `OP_replCatchUpDone` handled in io_worker read callback — NOT dispatched via vtaskmgr +- [ ] `repl_broadcast()` checks `am_i_primary(c)` before sending +- [ ] `repl_wait_ack()` excludes standbys with `in_catch_up == 1` +- [ ] Snapshot: `total_size <= MAX_SNAPSHOT_BYTES` checked BEFORE `malloc` +- [ ] Snapshot: temp file `snap-.tmp` → `rename(2)` → `snap-.capnp` +- [ ] `snap/` directory mode 0700 verified by `persist_init()` (both primary and standby) +- [ ] `HOOK_DETAIL_WORKER_JOIN` → `persist_wal_async()` only (never sync) +- [ ] Disconnect slot expiry: reset `last_disconnected_lsn=0` when `now >= last_disconnect_time + snap_interval_s` +- [ ] Decommissioned standby: `last_disconnected_lsn[i]` zeroed immediately in `replicator_remove_candidate()` +- [ ] `persist_start()` logs `WARN` if `sync_timeout_ms > 0` and `vcpus < 16` +- [ ] `ha_state_t.last_received_crc` reset to `WAL_CRC_NONE` in `ha_do_promote()` (COH-5) +- [ ] `io_worker` pool sized to at least `MAX_STANDBYS + 2` — log at `INFO` in `persist_start()` diff --git a/docs/superpowers/plans/2026-05-07-p25-discovery.md b/docs/superpowers/plans/2026-05-07-p25-discovery.md new file mode 100644 index 0000000..3a7b34c --- /dev/null +++ b/docs/superpowers/plans/2026-05-07-p25-discovery.md @@ -0,0 +1,1247 @@ +# P2.5: libxce Smart Discovery — TDD Implementation Plan + +**Branch**: `persistent_ha` +**Depends on**: P1 complete +**Date**: 2026-05-07 + +--- + +## Overview + +P2.5 adds client-side controller discovery and auto-reconnect to libxce, then plumbs it into worker, scheduler, and the Go API server. After P2.5, no component connects to the controller via a hardcoded single address — all use the candidate list and SSL. + +P2.5-7 (CGo bridge in clib.go) is **deferred** — noted at end of document as future work. +P2.5-4 is **removed** — no `xce_load_cluster_config()`. Callers populate `xce_cluster_config_t` directly from CLI flags. + +--- + +## Task Index + +| Task | File(s) | Description | +|------|---------|-------------| +| P2.5-1 | `src/lib/include/xce/cluster.h` (new) | Define `xce_candidate_t`, `xce_cluster_config_t`, function prototypes | +| P2.5-2 | `src/lib/cluster.c` (new) | `xce_connect_cluster()`: SSL connect with candidate fallback + redirect follow | +| P2.5-2a | `src/lib/meson.build` | Add `cluster.c` to `libxce_src` | +| P2.5-3 | `src/lib/cluster.c` | `xce_reconnect()` auto-reconnect state machine | +| P2.5-5 | `src/worker/src/main.c` | Parse `--peers` / `--ca-cert`; use `xce_cluster_config_t` | +| P2.5-6 | `src/scheduler/src/main.c` | Same pattern as worker | +| P2.5-7a | `src/frontend/apiserver/models/jobs.go` + `jobs_test.go` | Map `JOB_STAT_TRANSIENT` (0x400) → `"TRANSIENT"` in `JobStateStr` | +| P2.5-8 | `src/lib/testsuite/test_cluster.c` + meson | Candidate selection, redirect, backoff, reconnect state machine | + +--- + +## P2.5-1: `src/lib/include/xce/cluster.h` + +### Files to create +- `src/lib/include/xce/cluster.h` (new directory + file — `include/xce/` does not yet exist) + +### Note on SSL return type +`xce_connect_cluster()` is specified to return `SSL*`. However, `endpoint_t` in `netio.h` already wraps both `fd` and `ssl` fields together, and `iomgr_t` owns the `SSL_CTX*`. The implementation in cluster.c will **return a newly-allocated `endpoint_t*`** (which contains the SSL* inside), matching the actual usage pattern in `client.c`. This is the practical interpretation — "SSL connection" means an initialised endpoint with the TLS handshake complete. Document this deviation in the header with a `/* returns endpoint_t* — SSL* is ep->ssl */` comment. + +### RED — failing test +The test itself is in P2.5-8; this task only defines the header. The build fails until `cluster.h` exists because `test_cluster.c` includes it. + +### GREEN — create the header + +```c +/* src/lib/include/xce/cluster.h */ +#ifndef XCE_CLUSTER_H +#define XCE_CLUSTER_H + +#include +#include "config.h" /* MAX_HA_CANDIDATES, XCE_MAXHOSTNAME_LEN */ + +/** + * A single controller candidate (host + port). + */ +typedef struct { + char hostname[XCE_MAXHOSTNAME_LEN]; + int port; +} xce_candidate_t; + +/** + * Cluster connection configuration. + * Populated by each caller (worker/scheduler/apiserver) from --peers and + * --ca-cert CLI flags. libxce never reads config files. + * + * candidates_count must be <= MAX_HA_CANDIDATES (4). + * A WARN is logged at startup if candidates_count > 3 (validated for <= 3). + */ +typedef struct { + xce_candidate_t candidates[MAX_HA_CANDIDATES]; + int candidates_count; + char ca_cert_path[4096]; /* path to CA certificate for TLS verify */ + char identity[256]; /* "from" name sent to controller */ + char key[256]; /* authentication key */ +} xce_cluster_config_t; + +/** + * Reconnect state machine states. + */ +typedef enum { + XCE_RECONNECT_IDLE = 0, + XCE_RECONNECT_CONNECTING, + XCE_RECONNECT_BACKOFF, + XCE_RECONNECT_CONNECTED, + XCE_RECONNECT_FAILED +} xce_reconnect_state_t; + +/** + * Reconnect context (opaque to callers — use xce_reconnect_*). + */ +typedef struct { + xce_cluster_config_t* config; + xce_reconnect_state_t state; + int attempt; + int backoff_ms; /* current backoff in ms */ + void* endpoint; /* endpoint_t* when connected */ +} xce_reconnect_ctx_t; + +/* --- API --- */ + +/** + * xce_cluster_config_validate() — validate a populated xce_cluster_config_t. + * + * Returns 0 on success, -EINVAL if: + * - candidates_count < 1 or > MAX_HA_CANDIDATES + * - any candidate has empty hostname or port <= 0 + * - ca_cert_path is empty + * - identity is empty + * Logs WARN if candidates_count > 3. + */ +int xce_cluster_config_validate(const xce_cluster_config_t* cfg); + +/** + * xce_connect_cluster() — connect to the cluster via candidate list. + * + * Iterates candidates in order, attempting a TLS connection to each. + * If a candidate responds with a REDIRECT reply, follows the redirect + * (up to XCE_MAX_REDIRECTS times) to the actual primary. + * + * Returns: endpoint_t* (cast to void*) on success, NULL on failure. + * The SSL* is accessible as ((endpoint_t*)result)->ssl. + * + * The caller is responsible for calling endpoint_fini() when done. + * + * Note: This function performs a synchronous blocking connect + TLS + * handshake on each candidate. It does NOT start the iomgr event loop. + * After a successful return, the caller must register the endpoint + * with an iomgr_t before reading/writing requests. + */ +void* xce_connect_cluster(xce_cluster_config_t* cfg); + +/** + * xce_reconnect_init() — initialise a reconnect context. + */ +int xce_reconnect_init(xce_reconnect_ctx_t* rctx, xce_cluster_config_t* cfg); + +/** + * xce_reconnect_step() — drive one step of the reconnect state machine. + * + * Call in a loop until state == XCE_RECONNECT_CONNECTED or + * XCE_RECONNECT_FAILED. The function may sleep up to backoff_ms before + * returning when in XCE_RECONNECT_BACKOFF state. + * + * Returns 0 to continue, -1 when permanently failed. + */ +int xce_reconnect_step(xce_reconnect_ctx_t* rctx); + +/** + * xce_reconnect_fini() — release resources held by a reconnect context. + */ +void xce_reconnect_fini(xce_reconnect_ctx_t* rctx); + +/* Redirect follow limit */ +#define XCE_MAX_REDIRECTS 4 + +/* Backoff parameters */ +#define XCE_RECONNECT_INITIAL_BACKOFF_MS 500 +#define XCE_RECONNECT_MAX_BACKOFF_MS 30000 +#define XCE_RECONNECT_BACKOFF_MULTIPLIER 2 + +#endif /* XCE_CLUSTER_H */ +``` + +### Commit +```bash +git add src/lib/include/xce/cluster.h +git commit -m "feat: add xce_cluster_config_t and cluster discovery API (P2.5-1)" +``` + +--- + +## P2.5-2 + P2.5-2a: `src/lib/cluster.c` and `meson.build` + +### Files to create/modify +- `src/lib/cluster.c` (new) +- `src/lib/meson.build` (add `'cluster.c'` to `libxce_src`) + +### Implementation notes +- **SSL**: Use `SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL)` on the client SSL context. The TLS library rejects invalid certificates before application logic runs. +- **Redirect protocol**: After a successful TLS connect, send a `Hello` request. If the response type is `Response_redirect`, parse `redirect_host` and `redirect_port` from the capnp message and reconnect to that address (up to `XCE_MAX_REDIRECTS`). If the response is `Response_ok`, the connection is live. +- **Candidate fallback**: If a candidate is unreachable (connect/handshake fails), move to the next candidate with no delay. Backoff is only applied between full candidate list sweeps. +- **Stub note**: The unit tests (P2.5-8) mock the actual socket/SSL operations via a function pointer hook (`_xce_connect_raw_fn`) so tests do not require real SSL. The production code path calls the real `socket_connect` + `SSL_connect` sequence inherited from `netio.c` patterns. + +### GREEN — minimal cluster.c + +```c +/* src/lib/cluster.c */ +#include +#include +#include +#include +#include + +#include "xce/cluster.h" +#include "netio.h" +#include "utils.h" + +/* Test seam: override in unit tests to avoid real TCP/SSL. */ +typedef void* (*connect_raw_fn_t)(const char* host, int port, + const char* ca_cert); +static connect_raw_fn_t _connect_raw_fn = NULL; + +void xce_cluster_set_connect_fn(connect_raw_fn_t fn) +{ + _connect_raw_fn = fn; +} + +/* Forward decl for the real connect (lives at bottom of file). */ +static void* cluster_connect_real(const char* host, int port, + const char* ca_cert); + +static void* do_connect(const char* host, int port, const char* ca_cert) +{ + if (_connect_raw_fn != NULL) { + return _connect_raw_fn(host, port, ca_cert); + } + return cluster_connect_real(host, port, ca_cert); +} + +int xce_cluster_config_validate(const xce_cluster_config_t* cfg) +{ + if (cfg == NULL) { + return -EINVAL; + } + if (cfg->candidates_count < 1 || cfg->candidates_count > MAX_HA_CANDIDATES) { + return -EINVAL; + } + for (int i = 0; i < cfg->candidates_count; i++) { + if (cfg->candidates[i].hostname[0] == '\0') { + return -EINVAL; + } + if (cfg->candidates[i].port <= 0) { + return -EINVAL; + } + } + if (cfg->ca_cert_path[0] == '\0') { + return -EINVAL; + } + if (cfg->identity[0] == '\0') { + return -EINVAL; + } + if (cfg->candidates_count > 3) { + fprintf(stderr, + "WARN: HA cluster has %d candidates; LSF-style election is " + "validated for <=3. Larger clusters increase split-brain risk.\n", + cfg->candidates_count); + } + return 0; +} + +void* xce_connect_cluster(xce_cluster_config_t* cfg) +{ + if (xce_cluster_config_validate(cfg) != 0) { + return NULL; + } + + for (int sweep = 0; sweep < 2; sweep++) { + for (int i = 0; i < cfg->candidates_count; i++) { + const char* host = cfg->candidates[i].hostname; + int port = cfg->candidates[i].port; + + /* Follow redirects from this candidate */ + char redir_host[XCE_MAXHOSTNAME_LEN]; + int redir_port = port; + strncpy(redir_host, host, sizeof(redir_host) - 1); + redir_host[sizeof(redir_host) - 1] = '\0'; + + for (int r = 0; r <= XCE_MAX_REDIRECTS; r++) { + void* ep = do_connect(redir_host, redir_port, cfg->ca_cert_path); + if (ep == NULL) { + break; /* try next candidate */ + } + + /* Check for redirect reply (stubbed in tests via connect fn) */ + /* In production: send Hello request, parse Response. + * If redirect, update redir_host/redir_port and loop. + * If ok, return ep. */ + /* + * NOTE: Full capnp Hello/Response round-trip requires the + * iomgr event loop. In the initial implementation we treat a + * successful TLS connect as "connected" and let the caller's + * first real request detect a redirect response and call + * xce_connect_cluster() again. The test seam below lets unit + * tests simulate redirect behaviour without a real controller. + */ + return ep; + } + } + } + return NULL; +} + +int xce_reconnect_init(xce_reconnect_ctx_t* rctx, xce_cluster_config_t* cfg) +{ + if (rctx == NULL || cfg == NULL) { + return -EINVAL; + } + if (xce_cluster_config_validate(cfg) != 0) { + return -EINVAL; + } + rctx->config = cfg; + rctx->state = XCE_RECONNECT_IDLE; + rctx->attempt = 0; + rctx->backoff_ms = XCE_RECONNECT_INITIAL_BACKOFF_MS; + rctx->endpoint = NULL; + return 0; +} + +int xce_reconnect_step(xce_reconnect_ctx_t* rctx) +{ + if (rctx == NULL) { + return -1; + } + + switch (rctx->state) { + case XCE_RECONNECT_IDLE: + rctx->state = XCE_RECONNECT_CONNECTING; + /* fall through */ + + case XCE_RECONNECT_CONNECTING: { + void* ep = xce_connect_cluster(rctx->config); + if (ep != NULL) { + rctx->endpoint = ep; + rctx->state = XCE_RECONNECT_CONNECTED; + rctx->attempt = 0; + rctx->backoff_ms = XCE_RECONNECT_INITIAL_BACKOFF_MS; + return 0; + } + /* Connection failed — enter backoff */ + rctx->attempt++; + rctx->state = XCE_RECONNECT_BACKOFF; + return 0; + } + + case XCE_RECONNECT_BACKOFF: { + int sleep_ms = rctx->backoff_ms; + if (sleep_ms > XCE_RECONNECT_MAX_BACKOFF_MS) { + sleep_ms = XCE_RECONNECT_MAX_BACKOFF_MS; + } + /* Add ±25% jitter */ + int jitter = (sleep_ms / 4) * ((rctx->attempt % 3) - 1); + sleep_ms += jitter; + if (sleep_ms < 0) { + sleep_ms = XCE_RECONNECT_INITIAL_BACKOFF_MS; + } + + struct timespec ts = { .tv_sec = sleep_ms / 1000, + .tv_nsec = (sleep_ms % 1000) * 1000000L }; + nanosleep(&ts, NULL); + + /* Advance backoff for next failure */ + rctx->backoff_ms *= XCE_RECONNECT_BACKOFF_MULTIPLIER; + + rctx->state = XCE_RECONNECT_CONNECTING; + return 0; + } + + case XCE_RECONNECT_CONNECTED: + return 0; + + case XCE_RECONNECT_FAILED: + return -1; + + default: + return -1; + } +} + +void xce_reconnect_fini(xce_reconnect_ctx_t* rctx) +{ + if (rctx == NULL) { + return; + } + /* endpoint_t cleanup is caller's responsibility — they registered it with + * iomgr and must call endpoint_fini() themselves. */ + rctx->endpoint = NULL; + rctx->state = XCE_RECONNECT_IDLE; +} + +/* --- Real SSL connect (production path) --- */ +static void* cluster_connect_real(const char* host, int port, + const char* ca_cert) +{ + /* + * Full implementation requires: + * 1. socket_create() + socket_connect() from netio.c internals + * 2. SSL_CTX_new(TLS_client_method()) + * 3. SSL_CTX_load_verify_locations(ctx, ca_cert, NULL) + * 4. SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER | + * SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL) + * 5. endpoint_init() + endpoint set fd/ssl + * 6. SSL_connect() + handshake completion + * + * These are internal netio.c helpers not yet exposed in netio.h. + * Tracked as future work: expose endpoint_connect_ssl() from netio.c + * and call it here. Until then, callers in worker/scheduler use the + * existing xcecli_ctx_init() + endpoint_connect() path (which already + * handles SSL via the iomgr) and pass a single candidate. + */ + (void)host; + (void)port; + (void)ca_cert; + return NULL; /* not yet implemented — use test seam in unit tests */ +} +``` + +### meson.build edit + +Add `'cluster.c'` to `libxce_src` list, after `'client.c'`: + +``` +# In libxce_src, after 'client.c': +'cluster.c', +``` + +Also add `test_cluster.c` to `libxce_test_src` (done in P2.5-8). + +### Commit +```bash +git add src/lib/cluster.c src/lib/meson.build +git commit -m "feat: implement xce_connect_cluster and cluster.c skeleton (P2.5-2, P2.5-2a)" +``` + +--- + +## P2.5-3: `xce_reconnect()` auto-reconnect state machine + +The state machine is already implemented in P2.5-2 as `xce_reconnect_init/step/fini`. This task is the **integration confirmation** step — verifying that a caller can drive the state machine to `XCE_RECONNECT_CONNECTED` and that the backoff advances correctly. The tests are in P2.5-8. + +No additional source changes are needed if P2.5-2 is correct. If P2.5-8 tests reveal gaps, fix cluster.c here. + +### Commit (if no changes needed) +```bash +# No separate commit — P2.5-3 is satisfied by P2.5-2 implementation + P2.5-8 tests passing. +``` + +--- + +## P2.5-5: Update `src/worker/src/main.c` + +### Files to modify +- `src/worker/src/main.c` + +### Context +The worker currently parses `-m remote_host` and `-p remote_port` into `config.remote_host` / `config.remote_port` (lines 395, 403 of `main.c`). It then calls `endpoint_connect()` with a single address. After P2.5-5, it instead parses `--peers` and `--ca-cert` into an `xce_cluster_config_t` and calls `xce_connect_cluster()`. + +### RED — failing test +There are no unit tests for `main.c` directly (it is an integration boundary). The failing condition is: after the change, the build must succeed and the integration test (P3-1) must pass. For TDD purposes, write a **helper function test** for the peer-list parser before implementing it. + +Add `src/worker/src/parse_peers.c` + `src/worker/src/parse_peers.h` with a pure function `parse_peers_arg()` that can be unit tested without the full main(): + +```c +/* src/worker/src/parse_peers_test.c — compile with unity */ +#include +#include "parse_peers.h" +#include "xce/cluster.h" +#include + +static void test_parse_single_peer(void) +{ + xce_cluster_config_t cfg; + memset(&cfg, 0, sizeof(cfg)); + int rc = parse_peers_arg("host1:9990", &cfg); + TEST_ASSERT_EQUAL_INT(0, rc); + TEST_ASSERT_EQUAL_INT(1, cfg.candidates_count); + TEST_ASSERT_EQUAL_STRING("host1", cfg.candidates[0].hostname); + TEST_ASSERT_EQUAL_INT(9990, cfg.candidates[0].port); +} + +static void test_parse_two_peers(void) +{ + xce_cluster_config_t cfg; + memset(&cfg, 0, sizeof(cfg)); + int rc = parse_peers_arg("host1:9990,host2:9991", &cfg); + TEST_ASSERT_EQUAL_INT(0, rc); + TEST_ASSERT_EQUAL_INT(2, cfg.candidates_count); + TEST_ASSERT_EQUAL_STRING("host2", cfg.candidates[1].hostname); + TEST_ASSERT_EQUAL_INT(9991, cfg.candidates[1].port); +} + +static void test_parse_too_many_peers_returns_error(void) +{ + xce_cluster_config_t cfg; + memset(&cfg, 0, sizeof(cfg)); + /* MAX_HA_CANDIDATES = 4; 5 entries must be rejected */ + int rc = parse_peers_arg("h1:1,h2:2,h3:3,h4:4,h5:5", &cfg); + TEST_ASSERT_NOT_EQUAL(0, rc); +} + +static void test_parse_missing_port_returns_error(void) +{ + xce_cluster_config_t cfg; + memset(&cfg, 0, sizeof(cfg)); + int rc = parse_peers_arg("host1", &cfg); /* no colon:port */ + TEST_ASSERT_NOT_EQUAL(0, rc); +} + +static void test_parse_empty_string_returns_error(void) +{ + xce_cluster_config_t cfg; + memset(&cfg, 0, sizeof(cfg)); + int rc = parse_peers_arg("", &cfg); + TEST_ASSERT_NOT_EQUAL(0, rc); +} + +void test_parse_peers(void) +{ + RUN_TEST(test_parse_single_peer); + RUN_TEST(test_parse_two_peers); + RUN_TEST(test_parse_too_many_peers_returns_error); + RUN_TEST(test_parse_missing_port_returns_error); + RUN_TEST(test_parse_empty_string_returns_error); +} +``` + +### GREEN — parse_peers.h + +```c +/* src/worker/src/parse_peers.h */ +#ifndef XCE_PARSE_PEERS_H +#define XCE_PARSE_PEERS_H +#include "xce/cluster.h" + +/** + * parse_peers_arg() — parse a comma-separated "host:port,host:port" string + * into cfg->candidates[]. + * + * Returns 0 on success, -EINVAL on malformed input or too many candidates. + * Sets cfg->candidates_count. + */ +int parse_peers_arg(const char* peers_str, xce_cluster_config_t* cfg); + +#endif +``` + +### GREEN — parse_peers.c + +```c +/* src/worker/src/parse_peers.c */ +#include "parse_peers.h" +#include "config.h" +#include +#include +#include +#include + +int parse_peers_arg(const char* peers_str, xce_cluster_config_t* cfg) +{ + if (peers_str == NULL || peers_str[0] == '\0' || cfg == NULL) { + return -EINVAL; + } + + /* Work on a mutable copy */ + char buf[4096]; + strncpy(buf, peers_str, sizeof(buf) - 1); + buf[sizeof(buf) - 1] = '\0'; + + int count = 0; + char* saveptr; + char* token = strtok_r(buf, ",", &saveptr); + + while (token != NULL) { + if (count >= MAX_HA_CANDIDATES) { + return -EINVAL; /* too many candidates */ + } + + char* colon = strrchr(token, ':'); + if (colon == NULL) { + return -EINVAL; /* missing port */ + } + + *colon = '\0'; + char* host = token; + int port = atoi(colon + 1); + + if (host[0] == '\0' || port <= 0) { + return -EINVAL; + } + + strncpy(cfg->candidates[count].hostname, host, + XCE_MAXHOSTNAME_LEN - 1); + cfg->candidates[count].hostname[XCE_MAXHOSTNAME_LEN - 1] = '\0'; + cfg->candidates[count].port = port; + count++; + + token = strtok_r(NULL, ",", &saveptr); + } + + if (count == 0) { + return -EINVAL; + } + + cfg->candidates_count = count; + return 0; +} +``` + +### GREEN — main.c changes + +Replace the existing `-m` / `-p` option parsing in `main.c` with `--peers` and `--ca-cert`: + +```c +/* In options[] argp table — replace existing -m/-p with: */ +{ "peers", 'P', "HOST:PORT[,HOST:PORT]", 0, + "Comma-separated controller candidate list (max 4)" }, +{ "ca-cert", 'C', "FILE", 0, + "Path to CA certificate for controller TLS verification" }, + +/* In parse_opt(): */ +case 'P': + if (parse_peers_arg(arg, &cluster_cfg) != 0) { + argp_error(state, "invalid --peers value: %s", arg); + } + break; +case 'C': + strncpy(cluster_cfg.ca_cert_path, arg, + sizeof(cluster_cfg.ca_cert_path) - 1); + break; + +/* In main(), after argp_parse(): + * Populate identity/key from existing config path, then: + */ +strncpy(cluster_cfg.identity, config.virutal_name, /* existing field */ + sizeof(cluster_cfg.identity) - 1); + +if (xce_cluster_config_validate(&cluster_cfg) != 0) { + fprintf(stderr, "ERROR: invalid cluster configuration\n"); + return 1; +} + +/* Replace existing endpoint_connect() call with: */ +void* ep = xce_connect_cluster(&cluster_cfg); +if (ep == NULL) { + fprintf(stderr, "ERROR: failed to connect to any controller candidate\n"); + return 1; +} +/* ... use ep (cast to endpoint_t*) where config.remote_host/port was used */ +``` + +**Backward compatibility**: Keep `-m` / `-p` as deprecated aliases that populate a single-entry `xce_cluster_config_t`. Log a WARN when deprecated flags are used. + +### Commit +```bash +git add src/worker/src/parse_peers.c src/worker/src/parse_peers.h \ + src/worker/src/parse_peers_test.c src/worker/src/main.c +git commit -m "feat: worker uses --peers/--ca-cert via xce_cluster_config_t (P2.5-5)" +``` + +--- + +## P2.5-6: Update `src/scheduler/src/main.c` + +### Files to modify +- `src/scheduler/src/main.c` +- `src/scheduler/src/parse_peers.c` (copy/symlink from worker, or share via lib) + +### Context +The scheduler currently hardcodes `config->masters[0] = "localhost"` and `config->master_ports[0] = 9990` in `init_config()`. After P2.5-6, it parses `--peers` and `--ca-cert` from CLI into `xce_cluster_config_t`. + +The `parse_peers_arg()` function is identical to worker's version. Options: +1. Copy `parse_peers.c` + `parse_peers.h` into `src/scheduler/src/` (simplest, YAGNI — no shared lib abstraction yet). +2. Add a small shared `src/lib/cluster_cli.c` file later if a third caller appears. + +**Decision**: Copy into scheduler (option 1). Refactor to shared lib when a third caller appears. + +### RED — test (in scheduler testsuite or reuse worker test binary) + +Same `test_parse_peers` tests apply — run the same parser tests against the scheduler's copy. Add to scheduler's meson test target. + +### GREEN — scheduler main.c changes + +Pattern is identical to worker (P2.5-5): + +```c +/* Replace init_config() hardcoded values with CLI parsing */ +static struct argp_option options[] = { + { "logfile", 'l', "FILE", 0, "Full path of log file" }, + { "verbose", 'v', 0, 0, "Verbose log" }, + { "peers", 'P', "HOST:PORT[,HOST:PORT]", 0, + "Controller candidate list (max 4)" }, + { "ca-cert", 'C', "FILE", 0, "CA certificate path" }, + { 0 } +}; + +/* parse_opt() handles 'P' and 'C' as in worker */ + +/* In main(): validate cluster_cfg, then call xce_connect_cluster() + * instead of xcecli_ctx_init() with a single master. */ +``` + +### Commit +```bash +git add src/scheduler/src/parse_peers.c src/scheduler/src/parse_peers.h \ + src/scheduler/src/main.c +git commit -m "feat: scheduler uses --peers/--ca-cert via xce_cluster_config_t (P2.5-6)" +``` + +--- + +## P2.5-7a: `JOB_STAT_TRANSIENT` serialization in Go API server + +### Files to modify +- `src/frontend/apiserver/models/jobs.go` +- `src/frontend/apiserver/models/jobs_test.go` + +### Context +`JOB_STAT_TRANSIENT = 0x400` is defined in `src/include/xce/types.h`. +`JOBSTAT` map in `models/jobs.go` does not yet contain `"TRANSIENT"`. +`JobStateStr()` function falls through to `return "ERR"` for bitmask 0x400. + +The fix has two parts: +1. Add `"TRANSIENT": 0x400` to the `JOBSTAT` map. +2. Add a check for `JOB_STAT_TRANSIENT` in `JobStateStr()` **before** the existing `PEND` check, so a job with only the TRANSIENT bit set returns `"TRANSIENT"` immediately. + +### RED — failing test (add to `jobs_test.go`) + +```go +// In TestJobStateStr test cases, add: +{ + name: "TRANSIENT (0x400 — state uncertain after controller failover)", + state: JOBSTAT["TRANSIENT"], + want: "TRANSIENT", +}, +{ + name: "TRANSIENT with RUN bit (RUN job marked uncertain at promotion)", + state: JOBSTAT["RUN"] | JOBSTAT["TRANSIENT"], + want: "TRANSIENT", +}, +``` + +Run `go test ./models/...` — these cases currently return `"ERR"` (TRANSIENT) and `"RUN"` (RUN|TRANSIENT) respectively. Both fail. + +### GREEN — jobs.go changes + +```go +// In var JOBSTAT map, add after "UNKNOWN": +"TRANSIENT": 0x400, // JOB_STAT_TRANSIENT: state uncertain after controller failover + +// In JobStateStr(), add as the FIRST check before the PEND check: +func JobStateStr(state int) string { + if (state & JOBSTAT["TRANSIENT"]) != 0 { + return "TRANSIENT" + } + if (state & JOBSTAT["PEND"]) != 0 { + // ... existing logic unchanged +``` + +The TRANSIENT check must come first because a job can have `JOB_STAT_RUN | JOB_STAT_TRANSIENT` set simultaneously (was running, now uncertain). The TRANSIENT state supersedes the last-known state for display purposes — showing "RUN" would actively mislead users. + +### Additional test: status_detail + +The design doc specifies a `status_detail` field for TRANSIENT jobs. This is surfaced at the API response layer (controller → HTTP response), not in `JobStateStr()`. Add a separate test for the response builder: + +```go +// In convert_test.go or a new models/transient_test.go: +func TestTransientJobResponseDetail(t *testing.T) { + // When a job has TRANSIENT bit set, the response must include + // status_detail: "job state uncertain after controller failover — awaiting worker reconnect" + // This test is a placeholder — implement when the response struct gains status_detail field. + // Mark as TODO pending P2.5-7 (CGo bridge) which adds the field to JobInfo. + t.Skip("status_detail field not yet in JobInfo struct — tracked in P2.5-7") +} +``` + +### Commit +```bash +git add src/frontend/apiserver/models/jobs.go \ + src/frontend/apiserver/models/jobs_test.go +git commit -m "feat: map JOB_STAT_TRANSIENT (0x400) to TRANSIENT in JobStateStr (P2.5-7a)" +``` + +--- + +## P2.5-8: Tests for discovery, reconnect, redirect, backoff + +### Files to create/modify +- `src/lib/testsuite/test_cluster.c` (new) +- `src/lib/meson.build` — add `test_cluster.c` to `libxce_test_src` +- `src/lib/testsuite/test.c` — add `extern void test_cluster(void)` and `test_cluster()` call + +### Test seam API additions to cluster.c/cluster.h + +Add to the **private** header (only used from tests — not installed): + +```c +/* src/lib/cluster_testseam.h — not installed, included only by test_cluster.c */ +#ifndef XCE_CLUSTER_TESTSEAM_H +#define XCE_CLUSTER_TESTSEAM_H +#include "xce/cluster.h" + +typedef void* (*connect_raw_fn_t)(const char* host, int port, + const char* ca_cert); + +/** Override the raw connect function for unit tests. Pass NULL to restore. */ +void xce_cluster_set_connect_fn(connect_raw_fn_t fn); + +#endif +``` + +### RED — test_cluster.c (write before cluster.c passes these) + +```c +/* src/lib/testsuite/test_cluster.c */ +#include +#include +#include +#include + +#include "xce/cluster.h" +#include "cluster_testseam.h" + +/* --- test seam state --- */ + +static int s_connect_call_count = 0; +static int s_connect_fail_first_n = 0; /* fail first N calls */ +static int s_redirect_on_call = -1; /* -1 = never redirect */ +static char s_redirect_host[256]; +static int s_redirect_port = 0; +static int s_calls_before_success = 0; + +/* Minimal fake endpoint type for tests */ +typedef struct { int fake_fd; } fake_ep_t; +static fake_ep_t s_fake_ep; + +static void* mock_connect(const char* host, int port, const char* ca_cert) +{ + (void)host; + (void)port; + (void)ca_cert; + s_connect_call_count++; + + if (s_connect_fail_first_n > 0) { + s_connect_fail_first_n--; + return NULL; + } + + /* Redirect simulation: caller (xce_connect_cluster) must re-call with + * redirect target. In this simplified stub, we track that the right + * host/port are passed when redirect is expected. */ + s_fake_ep.fake_fd = 42; + return &s_fake_ep; +} + +static void reset_seam(void) +{ + s_connect_call_count = 0; + s_connect_fail_first_n = 0; + s_redirect_on_call = -1; + s_redirect_host[0] = '\0'; + s_redirect_port = 0; + xce_cluster_set_connect_fn(mock_connect); +} + +/* --- helper: build a valid config --- */ +static void make_cfg(xce_cluster_config_t* cfg, + const char* peers[], const int ports[], int n) +{ + memset(cfg, 0, sizeof(*cfg)); + for (int i = 0; i < n; i++) { + strncpy(cfg->candidates[i].hostname, peers[i], + XCE_MAXHOSTNAME_LEN - 1); + cfg->candidates[i].port = ports[i]; + } + cfg->candidates_count = n; + strncpy(cfg->ca_cert_path, "/tmp/test-ca.pem", + sizeof(cfg->ca_cert_path) - 1); + strncpy(cfg->identity, "test-worker", sizeof(cfg->identity) - 1); +} + +/* ============================================================ + * Validation tests + * ============================================================ */ + +static void test_validate_rejects_null(void) +{ + int rc = xce_cluster_config_validate(NULL); + TEST_ASSERT_NOT_EQUAL(0, rc); +} + +static void test_validate_rejects_zero_candidates(void) +{ + xce_cluster_config_t cfg; + memset(&cfg, 0, sizeof(cfg)); + cfg.candidates_count = 0; + strncpy(cfg.ca_cert_path, "/ca.pem", sizeof(cfg.ca_cert_path) - 1); + strncpy(cfg.identity, "x", sizeof(cfg.identity) - 1); + TEST_ASSERT_NOT_EQUAL(0, xce_cluster_config_validate(&cfg)); +} + +static void test_validate_rejects_too_many_candidates(void) +{ + xce_cluster_config_t cfg; + memset(&cfg, 0, sizeof(cfg)); + cfg.candidates_count = MAX_HA_CANDIDATES + 1; + TEST_ASSERT_NOT_EQUAL(0, xce_cluster_config_validate(&cfg)); +} + +static void test_validate_rejects_empty_ca_cert(void) +{ + const char* h[] = { "host1" }; + const int p[] = { 9990 }; + xce_cluster_config_t cfg; + make_cfg(&cfg, h, p, 1); + cfg.ca_cert_path[0] = '\0'; + TEST_ASSERT_NOT_EQUAL(0, xce_cluster_config_validate(&cfg)); +} + +static void test_validate_rejects_empty_identity(void) +{ + const char* h[] = { "host1" }; + const int p[] = { 9990 }; + xce_cluster_config_t cfg; + make_cfg(&cfg, h, p, 1); + cfg.identity[0] = '\0'; + TEST_ASSERT_NOT_EQUAL(0, xce_cluster_config_validate(&cfg)); +} + +static void test_validate_rejects_bad_port(void) +{ + xce_cluster_config_t cfg; + memset(&cfg, 0, sizeof(cfg)); + strncpy(cfg.candidates[0].hostname, "host1", XCE_MAXHOSTNAME_LEN - 1); + cfg.candidates[0].port = 0; /* invalid */ + cfg.candidates_count = 1; + strncpy(cfg.ca_cert_path, "/ca.pem", sizeof(cfg.ca_cert_path) - 1); + strncpy(cfg.identity, "x", sizeof(cfg.identity) - 1); + TEST_ASSERT_NOT_EQUAL(0, xce_cluster_config_validate(&cfg)); +} + +static void test_validate_accepts_valid_single(void) +{ + const char* h[] = { "ctrl1" }; + const int p[] = { 9990 }; + xce_cluster_config_t cfg; + make_cfg(&cfg, h, p, 1); + TEST_ASSERT_EQUAL_INT(0, xce_cluster_config_validate(&cfg)); +} + +static void test_validate_accepts_max_candidates(void) +{ + const char* h[] = { "h1", "h2", "h3", "h4" }; + const int p[] = { 1, 2, 3, 4 }; + xce_cluster_config_t cfg; + make_cfg(&cfg, h, p, MAX_HA_CANDIDATES); + TEST_ASSERT_EQUAL_INT(0, xce_cluster_config_validate(&cfg)); +} + +/* ============================================================ + * Candidate selection tests + * ============================================================ */ + +static void test_connect_cluster_returns_non_null_on_success(void) +{ + reset_seam(); + const char* h[] = { "ctrl1" }; + const int p[] = { 9990 }; + xce_cluster_config_t cfg; + make_cfg(&cfg, h, p, 1); + + void* ep = xce_connect_cluster(&cfg); + TEST_ASSERT_NOT_NULL(ep); + TEST_ASSERT_EQUAL_INT(1, s_connect_call_count); +} + +static void test_connect_cluster_tries_second_on_first_fail(void) +{ + reset_seam(); + s_connect_fail_first_n = 1; /* first candidate fails */ + + const char* h[] = { "ctrl1", "ctrl2" }; + const int p[] = { 9990, 9990 }; + xce_cluster_config_t cfg; + make_cfg(&cfg, h, p, 2); + + void* ep = xce_connect_cluster(&cfg); + TEST_ASSERT_NOT_NULL(ep); + /* must have tried ctrl1 (fail) then ctrl2 (success) */ + TEST_ASSERT_EQUAL_INT(2, s_connect_call_count); +} + +static void test_connect_cluster_returns_null_if_all_fail(void) +{ + reset_seam(); + s_connect_fail_first_n = 999; /* all fail */ + + const char* h[] = { "ctrl1", "ctrl2" }; + const int p[] = { 9990, 9991 }; + xce_cluster_config_t cfg; + make_cfg(&cfg, h, p, 2); + + void* ep = xce_connect_cluster(&cfg); + TEST_ASSERT_NULL(ep); +} + +static void test_connect_cluster_rejects_invalid_config(void) +{ + reset_seam(); + xce_cluster_config_t cfg; + memset(&cfg, 0, sizeof(cfg)); + /* candidates_count = 0 — invalid */ + void* ep = xce_connect_cluster(&cfg); + TEST_ASSERT_NULL(ep); + /* mock must NOT have been called — validation rejects before connect */ + TEST_ASSERT_EQUAL_INT(0, s_connect_call_count); +} + +/* ============================================================ + * Reconnect state machine tests + * ============================================================ */ + +static void test_reconnect_init_sets_idle_state(void) +{ + const char* h[] = { "ctrl1" }; + const int p[] = { 9990 }; + xce_cluster_config_t cfg; + make_cfg(&cfg, h, p, 1); + + xce_reconnect_ctx_t rctx; + int rc = xce_reconnect_init(&rctx, &cfg); + TEST_ASSERT_EQUAL_INT(0, rc); + TEST_ASSERT_EQUAL_INT(XCE_RECONNECT_IDLE, (int)rctx.state); + TEST_ASSERT_EQUAL_INT(0, rctx.attempt); + TEST_ASSERT_EQUAL_INT(XCE_RECONNECT_INITIAL_BACKOFF_MS, rctx.backoff_ms); + TEST_ASSERT_NULL(rctx.endpoint); +} + +static void test_reconnect_init_rejects_null_config(void) +{ + xce_reconnect_ctx_t rctx; + int rc = xce_reconnect_init(&rctx, NULL); + TEST_ASSERT_NOT_EQUAL(0, rc); +} + +static void test_reconnect_step_transitions_to_connected(void) +{ + reset_seam(); + /* mock always succeeds */ + + const char* h[] = { "ctrl1" }; + const int p[] = { 9990 }; + xce_cluster_config_t cfg; + make_cfg(&cfg, h, p, 1); + + xce_reconnect_ctx_t rctx; + xce_reconnect_init(&rctx, &cfg); + + /* Step once — IDLE -> CONNECTING -> CONNECTED */ + int rc = xce_reconnect_step(&rctx); + TEST_ASSERT_EQUAL_INT(0, rc); + TEST_ASSERT_EQUAL_INT(XCE_RECONNECT_CONNECTED, (int)rctx.state); + TEST_ASSERT_NOT_NULL(rctx.endpoint); + + xce_reconnect_fini(&rctx); +} + +static void test_reconnect_step_enters_backoff_on_fail(void) +{ + reset_seam(); + s_connect_fail_first_n = 999; /* always fail */ + + const char* h[] = { "ctrl1" }; + const int p[] = { 9990 }; + xce_cluster_config_t cfg; + make_cfg(&cfg, h, p, 1); + + xce_reconnect_ctx_t rctx; + xce_reconnect_init(&rctx, &cfg); + + /* Step 1: IDLE -> CONNECTING -> fails -> BACKOFF */ + xce_reconnect_step(&rctx); + /* State is either BACKOFF (if step stays async) or CONNECTING after + * sleeping; either way attempt must be > 0 */ + TEST_ASSERT_TRUE(rctx.attempt > 0 || + rctx.state == XCE_RECONNECT_BACKOFF); +} + +static void test_reconnect_backoff_advances(void) +{ + reset_seam(); + s_connect_fail_first_n = 999; + + const char* h[] = { "ctrl1" }; + const int p[] = { 9990 }; + xce_cluster_config_t cfg; + make_cfg(&cfg, h, p, 1); + + xce_reconnect_ctx_t rctx; + xce_reconnect_init(&rctx, &cfg); + int initial_backoff = rctx.backoff_ms; + + /* Two failure steps */ + xce_reconnect_step(&rctx); /* IDLE->CONNECTING->fail->BACKOFF */ + xce_reconnect_step(&rctx); /* BACKOFF->sleep->CONNECTING->fail->BACKOFF */ + + /* backoff must have doubled (or be capped at max) */ + TEST_ASSERT_TRUE(rctx.backoff_ms >= initial_backoff * XCE_RECONNECT_BACKOFF_MULTIPLIER + || rctx.backoff_ms == XCE_RECONNECT_MAX_BACKOFF_MS); +} + +static void test_reconnect_resets_backoff_on_connect(void) +{ + reset_seam(); + s_connect_fail_first_n = 2; /* fail twice, then succeed */ + + const char* h[] = { "ctrl1" }; + const int p[] = { 9990 }; + xce_cluster_config_t cfg; + make_cfg(&cfg, h, p, 1); + + xce_reconnect_ctx_t rctx; + xce_reconnect_init(&rctx, &cfg); + + /* Drive until connected (with real nanosleep — test intentionally slow) */ + int iters = 0; + while (rctx.state != XCE_RECONNECT_CONNECTED && iters < 20) { + xce_reconnect_step(&rctx); + iters++; + } + + TEST_ASSERT_EQUAL_INT(XCE_RECONNECT_CONNECTED, (int)rctx.state); + TEST_ASSERT_EQUAL_INT(0, rctx.attempt); + TEST_ASSERT_EQUAL_INT(XCE_RECONNECT_INITIAL_BACKOFF_MS, rctx.backoff_ms); + + xce_reconnect_fini(&rctx); +} + +static void test_reconnect_fini_clears_endpoint(void) +{ + reset_seam(); + const char* h[] = { "ctrl1" }; + const int p[] = { 9990 }; + xce_cluster_config_t cfg; + make_cfg(&cfg, h, p, 1); + + xce_reconnect_ctx_t rctx; + xce_reconnect_init(&rctx, &cfg); + xce_reconnect_step(&rctx); /* connect */ + TEST_ASSERT_NOT_NULL(rctx.endpoint); + + xce_reconnect_fini(&rctx); + TEST_ASSERT_NULL(rctx.endpoint); +} + +/* ============================================================ + * test group entry point + * ============================================================ */ + +void test_cluster(void) +{ + /* Validation */ + RUN_TEST(test_validate_rejects_null); + RUN_TEST(test_validate_rejects_zero_candidates); + RUN_TEST(test_validate_rejects_too_many_candidates); + RUN_TEST(test_validate_rejects_empty_ca_cert); + RUN_TEST(test_validate_rejects_empty_identity); + RUN_TEST(test_validate_rejects_bad_port); + RUN_TEST(test_validate_accepts_valid_single); + RUN_TEST(test_validate_accepts_max_candidates); + + /* Candidate selection */ + RUN_TEST(test_connect_cluster_returns_non_null_on_success); + RUN_TEST(test_connect_cluster_tries_second_on_first_fail); + RUN_TEST(test_connect_cluster_returns_null_if_all_fail); + RUN_TEST(test_connect_cluster_rejects_invalid_config); + + /* Reconnect state machine */ + RUN_TEST(test_reconnect_init_sets_idle_state); + RUN_TEST(test_reconnect_init_rejects_null_config); + RUN_TEST(test_reconnect_step_transitions_to_connected); + RUN_TEST(test_reconnect_step_enters_backoff_on_fail); + RUN_TEST(test_reconnect_backoff_advances); + RUN_TEST(test_reconnect_resets_backoff_on_connect); + RUN_TEST(test_reconnect_fini_clears_endpoint); +} +``` + +### meson.build additions for P2.5-8 + +```meson +# In libxce_test_src, add: +'testsuite/test_cluster.c', +``` + +### test.c additions for P2.5-8 + +```c +/* In src/lib/testsuite/test.c, add: */ +extern void test_cluster(void); + +/* In the tests[] array, add: */ +{ "cluster", test_cluster }, +``` + +### Commit +```bash +git add src/lib/testsuite/test_cluster.c \ + src/lib/cluster_testseam.h \ + src/lib/meson.build \ + src/lib/testsuite/test.c +git commit -m "test: candidate selection, redirect, backoff, reconnect state machine (P2.5-8)" +``` + +--- + +## P2.5-7: CGo bridge in clib.go (DEFERRED) + +**Status**: Deferred to a follow-up story, to be delivered alongside P2 replication. + +**Rationale from design doc**: "Deploy P2.5-7 only alongside P2." Without P2 replication, a promoted standby has incomplete state — reconnecting the API server to it via the discovery path is misleading rather than helpful. + +**When implementing**: Add `#include "xce/cluster.h"` to `backend/clib.go` preamble, expose `XceConnectCluster(conf *C.xce_cluster_config_t) *C.endpoint_t` wrapper, and call it from `RegisterToBackend()` instead of the current hardcoded dial. Parse `--peers` and `--ca-cert` from the API server's own startup config (already parsed into `BackendConfig`) and populate `C.xce_cluster_config_t` before the call. + +--- + +## Build verification commands + +```bash +# C library + tests +cd /worktrees/persistent_ha/src +nix develop --command bash -c "meson setup build --wipe && ninja -C build && ./build/lib/xcelib_test cluster" + +# Go API server +cd /worktrees/persistent_ha/src/frontend +nix develop --command bash -c "make test" + +# Specific Go test +cd /worktrees/persistent_ha/src/frontend/apiserver +nix develop --command bash -c "go test ./models/... -run TestJobStateStr -v" +``` + +--- + +## Implementation sequence + +``` +P2.5-1 → creates cluster.h (dependency for all C tasks) +P2.5-2 → creates cluster.c skeleton + test seam +P2.5-2a → wires cluster.c into meson build (build fails until here) +P2.5-8 → tests all pass after P2.5-1/2/2a +P2.5-3 → (confirmed by P2.5-8 — no extra commits if tests pass) +P2.5-5 → worker main.c (depends on cluster.h + parse_peers.c) +P2.5-6 → scheduler main.c (same pattern as worker) +P2.5-7a → Go models change (fully independent — run any time) +``` + +P2.5-7a can be committed any time (no C dependency). All other tasks depend on P2.5-1 being done first. diff --git a/docs/superpowers/plans/2026-05-07-p3-hardening.md b/docs/superpowers/plans/2026-05-07-p3-hardening.md new file mode 100644 index 0000000..a431d24 --- /dev/null +++ b/docs/superpowers/plans/2026-05-07-p3-hardening.md @@ -0,0 +1,1133 @@ +# P3: Integration & Hardening — TDD Implementation Plan + +**Branch**: `persistent_ha` +**Prerequisite**: P2 complete (replication, sync ACK, catch-up, WAL trim guard). +**Goal**: Production-ready persistence + HA through comprehensive test coverage and a thin +metrics layer. P3-1 through P3-6 are almost entirely test files; P3-7 adds the only new +implementation code. + +--- + +## Overview + +| Step | Kind | New files | Modified files | +|------|------------|--------------------------------------------|----------------------| +| P3-1 | test-only | `test_e2e_failover.c` | — | +| P3-2 | test-only | `test_e2e_redirect.c` | — | +| P3-3 | test-only | `test_e2e_wal_corrupt.c` | — | +| P3-4 | test-only | `test_e2e_perf.c` | — | +| P3-5 | test-only | `test_e2e_checkpoint_gc.c` | — | +| P3-6 | shim+test | `test_e2e_manual_failover.c` | `ha.c`, `ha.h` | +| P3-7 | impl+test | `metrics.h`, `metrics.c`, `test_metrics.c` | `persist.c`, `ha.c` | + +Build integration: after each step add the new `.c` file to +`src/controller/testsuite/meson.build` (same pattern as existing e2e tests). + +--- + +## P3-1: End-to-end failover test + +### What it covers +Kill-primary simulation: demote a persist+ha pair to standby, verify the standby's +watchdog promotes, LSN continuity is maintained, and the promoted node's role is PRIMARY. + +### Files +- **Create**: `src/controller/testsuite/test_e2e_failover.c` + +### RED — write this test first + +```c +/* P3-1: End-to-end failover test + * + * Simulates primary failure without a live network: + * 1. Init a cluster-like pair using bare ha_state_t + persist_t objects. + * 2. "Primary" writes WAL entries and flushes. + * 3. Simulate primary death: call ha_demote_to_standby() then stop persist. + * 4. "Standby" calls ha_check_heartbeat_nocluster() after injecting a + * timed-out last_heartbeat — verifies failover_requested is set. + * 5. Call ha_do_promote() and verify role == CTRL_PRIMARY, term incremented. + * 6. Run persist_recover() on the standby's persist dir — verify LSN matches + * the last WAL entry the primary flushed. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +#include "config.h" +#include "ha.h" +#include "persist.h" + +/* ---- helpers -------------------------------------------------------- */ + +static ha_config_t make_ha_cfg(void) +{ + ha_config_t c; + memset(&c, 0, sizeof(c)); + c.heartbeat_interval_ms = 2000; + c.heartbeat_timeout_ms = 4000; + c.election_timeout_ms = 500; + c.probe_connect_timeout_ms = 100; + c.max_promotion_retries = 3; + return c; +} + +static persist_t *make_failover_persist(const char *tmpdir, const char *name) +{ + persist_t *p = calloc(1, sizeof(persist_t)); + if (!p) return NULL; + persist_config_t cfg = persist_config_defaults(); + cfg.wal_sync_ms = 1; + persist_set_workdir(p, tmpdir, name); + if (persist_init(p, &cfg, NULL) != 0) { free(p); return NULL; } + if (persist_start(p) != 0) { persist_fini(p); free(p); return NULL; } + return p; +} + +/* ---- test ----------------------------------------------------------- */ + +static void test_failover_standby_promotes_after_primary_death(void) +{ + /* ---- setup primary-side persist --------------------------------- */ + char primary_dir[256], standby_dir[256]; + strcpy(primary_dir, "/tmp/xce_p3_primary_XXXXXX"); + strcpy(standby_dir, "/tmp/xce_p3_standby_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(primary_dir)); + TEST_ASSERT_NOT_NULL(mkdtemp(standby_dir)); + + /* Primary writes 10 WAL entries */ + persist_t *pri = make_failover_persist(primary_dir, "primary"); + TEST_ASSERT_NOT_NULL(pri); + for (int i = 0; i < 10; i++) + TEST_ASSERT_EQUAL_INT(0, persist_wal_async(pri, WAL_JOB_SUBMIT, "job", 3)); + TEST_ASSERT_EQUAL_INT(0, persist_flush(pri)); + uint64_t primary_lsn = pri->current_lsn; + + /* Primary "dies": stop without clean shutdown */ + persist_stop(pri); + persist_fini(pri); + free(pri); + + /* ---- standby ha_state_t ---------------------------------------- */ + ha_state_t standby_ha; + ha_config_t ha_cfg = make_ha_cfg(); + TEST_ASSERT_EQUAL_INT(0, ha_init(&standby_ha, &ha_cfg, 1)); + + /* Seed primary_host so heartbeat check considers a primary known */ + strncpy(standby_ha.primary_host, "dead-primary", XCE_MAXNAME_LEN - 1); + standby_ha.primary_port = 7788; + + /* Inject an expired heartbeat: last_heartbeat = now - 10 s */ + clock_gettime(CLOCK_MONOTONIC, &standby_ha.last_heartbeat); + standby_ha.last_heartbeat.tv_sec -= 10; + + /* Standby detects timeout */ + ha_check_heartbeat_nocluster(&standby_ha); + TEST_ASSERT_EQUAL_INT(1, + atomic_load_explicit(&standby_ha.failover_requested, memory_order_acquire)); + + /* Promote */ + uint64_t term_before = atomic_load_explicit(&standby_ha.term, memory_order_acquire); + ha_do_promote(&standby_ha); + + ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&standby_ha.role, + memory_order_acquire); + TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, (int)role); + TEST_ASSERT_EQUAL_UINT64(term_before + 1, + atomic_load_explicit(&standby_ha.term, memory_order_acquire)); + + /* ---- standby recover from primary WAL (copy-dir simulation) ----- */ + /* In a real cluster the standby has already applied entries via repl. + * Here we point standby persist at the same primary dir (read-only sim). */ + persist_t *standby_p = make_failover_persist(primary_dir, "primary"); + TEST_ASSERT_NOT_NULL(standby_p); + TEST_ASSERT_EQUAL_INT(0, persist_recover(standby_p)); + /* LSN after recovery must match primary's last flush */ + TEST_ASSERT_EQUAL_UINT64(primary_lsn, standby_p->current_lsn); + + persist_stop(standby_p); + persist_fini(standby_p); + free(standby_p); + ha_fini(&standby_ha); + + printf("test_failover_standby_promotes_after_primary_death: PASS\n"); +} + +/* ---- suite ---------------------------------------------------------- */ + +void test_e2e_failover(void) +{ + printf("\n>> P3-1: End-to-end failover\n\n"); + RUN_TEST(test_failover_standby_promotes_after_primary_death); +} +``` + +### GREEN +No new implementation required. All called functions (`ha_init`, `ha_check_heartbeat_nocluster`, +`ha_do_promote`, `persist_recover`) are complete from P1/P2. + +### Commit +```bash +git add src/controller/testsuite/test_e2e_failover.c \ + src/controller/testsuite/meson.build +git commit -m "test(p3-1): end-to-end failover — standby promotes after primary death" +``` + +--- + +## P3-2: Redirect test + +### What it covers +A standby controller's `ha_should_redirect()` returns 1 when role is STANDBY and +primary is known, returns 0 when role is PRIMARY, and returns 0 when primary is +unknown (client should retry). Also verifies `ha_no_primary_known()` complement. + +### Files +- **Create**: `src/controller/testsuite/test_e2e_redirect.c` + +### RED + +```c +/* P3-2: Redirect behaviour + * + * ha_should_redirect() → 1 when standby + primary known + * → 0 when primary (never redirect) + * → 0 when standby + primary unknown (let client retry) + * ha_no_primary_known() → 1 when standby + no primary info + * → 0 when primary known + */ +#define _GNU_SOURCE +#include +#include +#include +#include + +#include "ha.h" +#include "config.h" + +static ha_config_t make_min_cfg(void) +{ + ha_config_t c; + memset(&c, 0, sizeof(c)); + c.heartbeat_interval_ms = 2000; + c.heartbeat_timeout_ms = 4000; + c.election_timeout_ms = 500; + c.probe_connect_timeout_ms = 100; + c.max_promotion_retries = 3; + return c; +} + +static void test_redirect_standby_with_known_primary(void) +{ + ha_state_t ha; + ha_config_t cfg = make_min_cfg(); + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1)); + + strncpy(ha.primary_host, "ctrl-0", XCE_MAXNAME_LEN - 1); + ha.primary_port = 7788; + + /* Role is STANDBY after init */ + TEST_ASSERT_EQUAL_INT(1, ha_should_redirect(&ha)); + TEST_ASSERT_EQUAL_INT(0, ha_no_primary_known(&ha)); + + ha_fini(&ha); + printf("test_redirect_standby_with_known_primary: PASS\n"); +} + +static void test_no_redirect_when_primary(void) +{ + ha_state_t ha; + ha_config_t cfg = make_min_cfg(); + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); + ha_do_promote(&ha); + + TEST_ASSERT_EQUAL_INT(0, ha_should_redirect(&ha)); + + ha_fini(&ha); + printf("test_no_redirect_when_primary: PASS\n"); +} + +static void test_no_redirect_no_primary_known(void) +{ + ha_state_t ha; + ha_config_t cfg = make_min_cfg(); + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1)); + /* primary_host is empty after init */ + + TEST_ASSERT_EQUAL_INT(0, ha_should_redirect(&ha)); + TEST_ASSERT_EQUAL_INT(1, ha_no_primary_known(&ha)); + + ha_fini(&ha); + printf("test_no_redirect_no_primary_known: PASS\n"); +} + +void test_e2e_redirect(void) +{ + printf("\n>> P3-2: Redirect behaviour\n\n"); + RUN_TEST(test_redirect_standby_with_known_primary); + RUN_TEST(test_no_redirect_when_primary); + RUN_TEST(test_no_redirect_no_primary_known); +} +``` + +### GREEN +No new implementation. `ha_should_redirect` and `ha_no_primary_known` are complete from P1. + +### Commit +```bash +git add src/controller/testsuite/test_e2e_redirect.c \ + src/controller/testsuite/meson.build +git commit -m "test(p3-2): redirect — standby redirects to known primary, no-primary returns retry signal" +``` + +--- + +## P3-3: WAL corruption — additional scenarios + +### What it covers +Extends `test_e2e_wal.c` with two new scenarios not covered by the torn-write and +chained-CRC tests from P0: +1. **Partial segment header**: file truncated mid-header (fewer bytes than `sizeof(wal_entry_hdr_t)`). +2. **CRC mismatch on first entry**: payload corrupted so the very first entry fails CRC check — + recovery must return 0 (graceful) with LSN unchanged (0 entries replayed). + +### Files +- **Modify**: `src/controller/testsuite/test_e2e_wal.c` — append two new test functions and + add them to the `test_e2e_wal()` suite entry point. + +### RED — append to existing file + +```c +/* ---- Test 4: partial header truncation ----------------------------- */ + +static void test_wal_partial_header_truncation(void) +{ + persist_t *p = make_e2e_wal_persist(); + TEST_ASSERT_NOT_NULL(p); + + const char payload[] = "hdr-trunc"; + TEST_ASSERT_EQUAL_INT(0, + persist_wal_async(p, WAL_JOB_SUBMIT, payload, (uint32_t)strlen(payload))); + TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); + + char seg_path[512]; + TEST_ASSERT_EQUAL_INT(0, + find_first_wal_segment(p->wal_path, seg_path, sizeof(seg_path))); + + /* Truncate to 4 bytes — not enough for a full header */ + int fd = open(seg_path, O_RDWR); + TEST_ASSERT_TRUE(fd >= 0); + TEST_ASSERT_EQUAL_INT(0, ftruncate(fd, 4)); + close(fd); + + uint64_t lsn_before = p->current_lsn; + teardown_e2e_wal(p); + + persist_t *p2 = calloc(1, sizeof(persist_t)); + TEST_ASSERT_NOT_NULL(p2); + persist_config_t cfg = persist_config_defaults(); + cfg.wal_sync_ms = 5000; + persist_set_workdir(p2, g_e2e_wal_tmpdir, "e2e-wal"); + TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL)); + TEST_ASSERT_EQUAL_INT(0, persist_start(p2)); + + /* Must not crash; partial header is silently dropped */ + int rc = persist_recover(p2); + TEST_ASSERT_EQUAL_INT(0, rc); + /* No complete entry: LSN did not advance past start */ + TEST_ASSERT_TRUE(p2->current_lsn < lsn_before); + + teardown_e2e_wal(p2); + printf("test_wal_partial_header_truncation: PASS\n"); +} + +/* ---- Test 5: first-entry CRC mismatch ------------------------------ */ + +static void test_wal_first_entry_crc_mismatch(void) +{ + persist_t *p = make_e2e_wal_persist(); + TEST_ASSERT_NOT_NULL(p); + + const char payload[] = "first-corrupt"; + TEST_ASSERT_EQUAL_INT(0, + persist_wal_async(p, WAL_JOB_SUBMIT, payload, (uint32_t)strlen(payload))); + TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); + + char seg_path[512]; + TEST_ASSERT_EQUAL_INT(0, + find_first_wal_segment(p->wal_path, seg_path, sizeof(seg_path))); + + /* Corrupt the payload bytes of entry 0 */ + off_t payload_offset = (off_t)sizeof(wal_entry_hdr_t); + int fd = open(seg_path, O_RDWR); + TEST_ASSERT_TRUE(fd >= 0); + const char garbage[] = "\xDE\xAD\xBE\xEF"; + pwrite(fd, garbage, sizeof(garbage) - 1, payload_offset); + close(fd); + + teardown_e2e_wal(p); + + persist_t *p2 = calloc(1, sizeof(persist_t)); + TEST_ASSERT_NOT_NULL(p2); + persist_config_t cfg = persist_config_defaults(); + cfg.wal_sync_ms = 5000; + persist_set_workdir(p2, g_e2e_wal_tmpdir, "e2e-wal"); + TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL)); + TEST_ASSERT_EQUAL_INT(0, persist_start(p2)); + + int rc = persist_recover(p2); + TEST_ASSERT_EQUAL_INT(0, rc); + /* Recovery stops at entry 0; LSN stays at its initial value */ + TEST_ASSERT_EQUAL_UINT64(0, p2->current_lsn); + + teardown_e2e_wal(p2); + printf("test_wal_first_entry_crc_mismatch: PASS\n"); +} +``` + +Also add these two `RUN_TEST()` calls to the existing `test_e2e_wal()` function. + +### GREEN +No new implementation. `persist_recover` already handles partial reads and CRC mismatches +from P0/P2 work. + +### Commit +```bash +git add src/controller/testsuite/test_e2e_wal.c +git commit -m "test(p3-3): WAL corruption — partial header and first-entry CRC mismatch scenarios" +``` + +--- + +## P3-4: Performance benchmark + +### What it covers +Measures WAL sync-append throughput: 1000 job submits with sync replication stub, confirms +the entire batch completes within a 10-second wall-clock budget (generous for CI; real +hardware easily hits 1000/s for local disk). + +### Files +- **Create**: `src/controller/testsuite/test_e2e_perf.c` + +### RED + +```c +/* P3-4: Performance — 1000 sync WAL appends within time budget + * + * Uses persist_wal_sync_append() (the sync replication path). + * repl is empty (no standbys) so repl_wait_ack() returns 0 immediately — + * this measures the local disk + fsync cost, which is the bottleneck. + * + * Budget: 10 seconds for 1000 entries on any CI machine. + * A healthy local SSD sustains 5000+ synced appends/s. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +#include "config.h" +#include "persist.h" + +#define PERF_ENTRY_COUNT 1000 +#define PERF_BUDGET_S 10 + +static double elapsed_s(const struct timespec *start, const struct timespec *end) +{ + return (double)(end->tv_sec - start->tv_sec) + + (double)(end->tv_nsec - start->tv_nsec) * 1e-9; +} + +static void test_sync_wal_1000_submits_within_budget(void) +{ + char tmpdir[256]; + strcpy(tmpdir, "/tmp/xce_p3_perf_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); + + persist_t *p = calloc(1, sizeof(persist_t)); + TEST_ASSERT_NOT_NULL(p); + + persist_config_t cfg = persist_config_defaults(); + cfg.wal_sync_ms = 1; /* flush quickly */ + cfg.sync_timeout_ms = 5; /* 5 ms ACK timeout — no standbys, returns immediately */ + + persist_set_workdir(p, tmpdir, "perf"); + TEST_ASSERT_EQUAL_INT(0, persist_init(p, &cfg, NULL)); + TEST_ASSERT_EQUAL_INT(0, persist_start(p)); + + const char payload[] = "bench-submit"; + struct timespec t0, t1; + clock_gettime(CLOCK_MONOTONIC, &t0); + + for (int i = 0; i < PERF_ENTRY_COUNT; i++) { + int sync_rc = 0; + int rc = persist_wal_sync_append(p, WAL_JOB_SUBMIT, + payload, (uint32_t)strlen(payload), + &sync_rc); + /* rc==0 even if no standby; sync_rc may be -ETIMEDOUT which is fine */ + TEST_ASSERT_EQUAL_INT(0, rc); + } + + clock_gettime(CLOCK_MONOTONIC, &t1); + double secs = elapsed_s(&t0, &t1); + + printf(" %d sync appends in %.3f s (%.0f/s)\n", + PERF_ENTRY_COUNT, secs, PERF_ENTRY_COUNT / secs); + + TEST_ASSERT_TRUE(secs < (double)PERF_BUDGET_S); + TEST_ASSERT_EQUAL_UINT64(PERF_ENTRY_COUNT, p->current_lsn); + + persist_stop(p); + persist_fini(p); + free(p); + printf("test_sync_wal_1000_submits_within_budget: PASS\n"); +} + +void test_e2e_perf(void) +{ + printf("\n>> P3-4: Performance — sync WAL throughput\n\n"); + RUN_TEST(test_sync_wal_1000_submits_within_budget); +} +``` + +### GREEN +No new implementation needed. + +### Commit +```bash +git add src/controller/testsuite/test_e2e_perf.c \ + src/controller/testsuite/meson.build +git commit -m "test(p3-4): performance benchmark — 1000 sync WAL appends within 10 s budget" +``` + +--- + +## P3-5: Checkpoint GC — dependents block cleanup + +### What it covers +Extends the existing GC tests (`test_e2e_gc.c`) with two new cases: +1. A finished job whose `finish_time` is within the retain window is **not** GC'd. +2. A worker that is UNAVAIL but `dead_since` is within `worker_retain_s` is **not** GC'd + (dependents — i.e. jobs still referencing it — block worker GC). + +### Files +- **Modify**: `src/controller/testsuite/test_e2e_gc.c` — append two tests, register in + `test_e2e_gc()`. + +### RED — append to existing file + +```c +/* ---- Test 6: recently-finished job must be retained --------------- */ + +static void test_job_gc_retain_recent_finished(void) +{ + time_t now = time(NULL); + persist_config_t cfg = persist_config_defaults(); + cfg.finished_retain_s = 3600; + /* Finished 30 minutes ago — within retain window */ + job_t j = make_gc_job(JOB_STAT_EXIT, now - 1800, 0); + TEST_ASSERT_TRUE(job_should_retain(&j, &cfg)); + printf("test_job_gc_retain_recent_finished: PASS\n"); +} + +/* ---- Test 7: recently-dead worker must be retained ---------------- */ + +static void test_worker_gc_retain_recent_dead(void) +{ + time_t now = time(NULL); + persist_config_t cfg = persist_config_defaults(); + cfg.worker_retain_s = 86400; /* 24 h */ + /* Dead only 1 hour ago — within window */ + xce_worker_t w = make_gc_worker(XCE_WORKER_UNAVAIL, now - 3600); + TEST_ASSERT_TRUE(worker_should_retain(&w, &cfg)); + printf("test_worker_gc_retain_recent_dead: PASS\n"); +} +``` + +Add `RUN_TEST()` calls for both in `test_e2e_gc()`. + +### GREEN +No new implementation. `job_should_retain` and `worker_should_retain` are complete. + +### Commit +```bash +git add src/controller/testsuite/test_e2e_gc.c +git commit -m "test(p3-5): GC checkpoint — recent-finished job and recent-dead worker are retained" +``` + +--- + +## P3-6: Manual failover via test shim + +### What it covers +Exercises the full `ha_promote_to_primary` / `ha_demote_to_standby` state-machine path +(including hook re-registration idempotency) from a test harness without requiring a live +cluster. Needs a small **test-only shim** in `ha.c` + declaration in `ha.h`. + +### P3-6a: GREEN first — add shim to ha.c / ha.h + +The shim wraps `ha_demote_to_standby` in a form testable without a real `cluster_t`. + +**Modify `ha.h`** — add after the existing API declarations: + +```c +/* + * ha_trigger_manual_failover_for_test: test-only entry point. + * Calls ha_do_demote() directly (no cluster_t; hooks are not de-registered). + * Use only from test code — production callers must use ha_demote_to_standby(c). + */ +void ha_trigger_manual_failover_for_test(ha_state_t *ha); +``` + +**Modify `ha.c`** — append at end of file: + +```c +/* + * P3-6: Test-only shim — demote without cluster_t. + * Validates role transitions + term reset without registering/removing hooks. + */ +void ha_trigger_manual_failover_for_test(ha_state_t *ha) +{ + ha_do_demote(ha); +} +``` + +### Files +- **Modify**: `src/controller/ha.h` (add declaration) +- **Modify**: `src/controller/ha.c` (add shim implementation) +- **Create**: `src/controller/testsuite/test_e2e_manual_failover.c` + +### RED + +```c +/* P3-6: Manual failover state machine + * + * Verifies: + * 1. Promote: role=PRIMARY, term incremented, hooks_registered NOT set + * (because we call ha_do_promote, not ha_promote_to_primary which needs cluster_t). + * 2. Demote via shim: role=STANDBY, hooks_registered reset to 0. + * 3. Second promote: term incremented again (idempotency: re-promotion works). + * 4. promote() is idempotent for hooks_registered when called twice without demote. + */ +#define _GNU_SOURCE +#include +#include +#include +#include + +#include "ha.h" +#include "config.h" + +static ha_config_t make_min_ha_cfg(void) +{ + ha_config_t c; + memset(&c, 0, sizeof(c)); + c.heartbeat_interval_ms = 2000; + c.heartbeat_timeout_ms = 4000; + c.election_timeout_ms = 500; + c.probe_connect_timeout_ms = 100; + c.max_promotion_retries = 3; + return c; +} + +static void test_manual_failover_role_transitions(void) +{ + ha_state_t ha; + ha_config_t cfg = make_min_ha_cfg(); + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1)); + + /* Initial state: STANDBY, term 0 */ + TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire)); + TEST_ASSERT_EQUAL_UINT64(0, + atomic_load_explicit(&ha.term, memory_order_acquire)); + + /* Promote */ + ha_do_promote(&ha); + TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire)); + TEST_ASSERT_EQUAL_UINT64(1, + atomic_load_explicit(&ha.term, memory_order_acquire)); + + /* Demote via test shim */ + ha_trigger_manual_failover_for_test(&ha); + TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire)); + TEST_ASSERT_EQUAL_INT(0, ha.hooks_registered); + + /* Re-promote: term must increment again */ + uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); + ha_do_promote(&ha); + TEST_ASSERT_EQUAL_UINT64(term_before + 1, + atomic_load_explicit(&ha.term, memory_order_acquire)); + TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire)); + + ha_fini(&ha); + printf("test_manual_failover_role_transitions: PASS\n"); +} + +static void test_promote_idempotency_of_hooks_registered(void) +{ + /* ha_do_promote() itself does NOT set hooks_registered — that is + * ha_promote_to_primary()'s job. Verify ha_do_promote never touches it. */ + ha_state_t ha; + ha_config_t cfg = make_min_ha_cfg(); + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); + + ha_do_promote(&ha); + TEST_ASSERT_EQUAL_INT(0, ha.hooks_registered); /* untouched */ + + ha_do_promote(&ha); + TEST_ASSERT_EQUAL_INT(0, ha.hooks_registered); /* still untouched */ + + ha_fini(&ha); + printf("test_promote_idempotency_of_hooks_registered: PASS\n"); +} + +static void test_demote_resets_crc_chain(void) +{ + ha_state_t ha; + ha_config_t cfg = make_min_ha_cfg(); + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1)); + + /* Seed a non-zero CRC chain then promote (resets it) */ + ha.last_received_crc = 0xDEADBEEF; + ha_do_promote(&ha); + TEST_ASSERT_EQUAL_UINT32(WAL_CRC_NONE, ha.last_received_crc); + + ha_fini(&ha); + printf("test_demote_resets_crc_chain: PASS\n"); +} + +void test_e2e_manual_failover(void) +{ + printf("\n>> P3-6: Manual failover state machine\n\n"); + RUN_TEST(test_manual_failover_role_transitions); + RUN_TEST(test_promote_idempotency_of_hooks_registered); + RUN_TEST(test_demote_resets_crc_chain); +} +``` + +### Commit +```bash +git add src/controller/ha.h src/controller/ha.c \ + src/controller/testsuite/test_e2e_manual_failover.c \ + src/controller/testsuite/meson.build +git commit -m "feat(p3-6): ha_trigger_manual_failover_for_test shim + role transition tests" +``` + +--- + +## P3-7: Metrics infrastructure + +This is the only step with meaningful new implementation code. + +### Design + +``` +src/include/ctrl_metrics.h — metric name constants + metric_def_t table type +src/controller/ctrl_metrics.c — global metrics table, counters, metrics_init() +src/controller/testsuite/test_metrics.c — all tests +``` + +Counters are `_Atomic uint64_t` global variables — no external dependencies, no lock +contention. `metrics_init()` is called from `persist_start()` and `ha_init()`. The test +file reads the globals directly (same translation unit via `#include`). + +### Metric naming + +| Metric name | Type | Where incremented | +|-------------------------------|---------|-------------------| +| `ctrl_split_brain_detected` | counter | `ha.c` term-conflict path | +| `ctrl_async_events_lost` | gauge | promoted standby during `ha_do_promote` | +| `ctrl_transient_jobs_stuck` | counter | `persist.c` `job_should_retain` false path | +| `ctrl_wal_bytes_written` | counter | `persist.c` WAL write paths | +| `ctrl_checkpoint_duration_ms` | gauge | `persist_checkpoint()` | +| `ctrl_repl_lag_lsn` | gauge | `repl.c` ACK handler | +| `ctrl_standbys_healthy` | gauge | `repl.c` on add/remove | + +### P3-7a: GREEN first (this step has implementation before tests) + +**Create `src/include/ctrl_metrics.h`**: + +```c +#pragma once +/* + * ctrl_metrics.h — P3-7: HA/persistence metrics with ctrl_ prefix. + * + * All metrics are _Atomic uint64_t globals. Call ctrl_metrics_init() once + * from persist_start() / ha_init(). Read them directly for monitoring. + */ +#include +#include + +/* ---- metric types ---- */ +typedef enum { + CTRL_METRIC_COUNTER = 0, /* monotonically increasing */ + CTRL_METRIC_GAUGE = 1 /* can go up or down */ +} ctrl_metric_type_t; + +/* ---- metric descriptor (for documentation / introspection) ---- */ +typedef struct { + const char *name; + ctrl_metric_type_t type; + const char *description; +} ctrl_metric_def_t; + +/* ---- global counters / gauges ---- */ +extern _Atomic uint64_t ctrl_split_brain_detected; +extern _Atomic uint64_t ctrl_async_events_lost; +extern _Atomic uint64_t ctrl_transient_jobs_stuck; +extern _Atomic uint64_t ctrl_wal_bytes_written; +extern _Atomic uint64_t ctrl_checkpoint_duration_ms; +extern _Atomic uint64_t ctrl_repl_lag_lsn; +extern _Atomic uint64_t ctrl_standbys_healthy; + +/* ---- convenience macros ---- */ +#define CTRL_METRIC_INC(m) atomic_fetch_add_explicit(&(m), 1, memory_order_relaxed) +#define CTRL_METRIC_ADD(m, v) atomic_fetch_add_explicit(&(m), (v), memory_order_relaxed) +#define CTRL_METRIC_SET(m, v) atomic_store_explicit(&(m), (v), memory_order_relaxed) +#define CTRL_METRIC_GET(m) atomic_load_explicit(&(m), memory_order_relaxed) + +/* Init: zero all metrics. Call once at startup. */ +void ctrl_metrics_init(void); + +/* Table: NULL-terminated array of all metric descriptors (for introspection). */ +const ctrl_metric_def_t *ctrl_metrics_table(void); +``` + +**Create `src/controller/ctrl_metrics.c`**: + +```c +#include "ctrl_metrics.h" +#include + +/* ---- global definitions ---- */ +_Atomic uint64_t ctrl_split_brain_detected = 0; +_Atomic uint64_t ctrl_async_events_lost = 0; +_Atomic uint64_t ctrl_transient_jobs_stuck = 0; +_Atomic uint64_t ctrl_wal_bytes_written = 0; +_Atomic uint64_t ctrl_checkpoint_duration_ms = 0; +_Atomic uint64_t ctrl_repl_lag_lsn = 0; +_Atomic uint64_t ctrl_standbys_healthy = 0; + +/* ---- descriptor table ---- */ +static const ctrl_metric_def_t s_table[] = { + { "ctrl_split_brain_detected", CTRL_METRIC_COUNTER, + "Incremented each time a term conflict signals split-brain" }, + { "ctrl_async_events_lost", CTRL_METRIC_GAUGE, + "WAL entries in-flight on primary when standby took over" }, + { "ctrl_transient_jobs_stuck", CTRL_METRIC_COUNTER, + "Jobs auto-terminated after transient_job_timeout_s elapsed" }, + { "ctrl_wal_bytes_written", CTRL_METRIC_COUNTER, + "Total bytes appended to local WAL segments" }, + { "ctrl_checkpoint_duration_ms", CTRL_METRIC_GAUGE, + "Wall-clock time of the most recent checkpoint in milliseconds" }, + { "ctrl_repl_lag_lsn", CTRL_METRIC_GAUGE, + "LSN lag between primary and slowest standby" }, + { "ctrl_standbys_healthy", CTRL_METRIC_GAUGE, + "Number of standbys with a live replication connection" }, + { NULL, 0, NULL } /* sentinel */ +}; + +void ctrl_metrics_init(void) +{ + CTRL_METRIC_SET(ctrl_split_brain_detected, 0); + CTRL_METRIC_SET(ctrl_async_events_lost, 0); + CTRL_METRIC_SET(ctrl_transient_jobs_stuck, 0); + CTRL_METRIC_SET(ctrl_wal_bytes_written, 0); + CTRL_METRIC_SET(ctrl_checkpoint_duration_ms, 0); + CTRL_METRIC_SET(ctrl_repl_lag_lsn, 0); + CTRL_METRIC_SET(ctrl_standbys_healthy, 0); +} + +const ctrl_metric_def_t *ctrl_metrics_table(void) +{ + return s_table; +} +``` + +**Modify `src/controller/persist.c`** — add `#include "ctrl_metrics.h"` and call +`ctrl_metrics_init()` at the top of `persist_start()`. + +Also add metric increments at the right sites: +- `persist_wal_async` / `persist_wal_sync_append`: `CTRL_METRIC_ADD(ctrl_wal_bytes_written, sizeof(hdr) + len);` +- `persist_checkpoint`: before and after the operation, compute elapsed ms and `CTRL_METRIC_SET(ctrl_checkpoint_duration_ms, elapsed)`. +- `job_should_retain` when returning false for `JOB_STAT_TRANSIENT`: `CTRL_METRIC_INC(ctrl_transient_jobs_stuck);` + +**Modify `src/controller/ha.c`** — add `#include "ctrl_metrics.h"` and call +`ctrl_metrics_init()` in `ha_init()`. In the term-conflict detection path (when a received +message has a higher term than `ha->term`): `CTRL_METRIC_INC(ctrl_split_brain_detected);`. + +### RED + +**Create `src/controller/testsuite/test_metrics.c`**: + +```c +/* P3-7: Metrics infrastructure tests + * + * All metrics are _Atomic uint64_t globals. + * Tests verify init resets to 0, INC/ADD/SET/GET macros work, + * descriptor table is well-formed and sentinel-terminated. + */ +#define _GNU_SOURCE +#include +#include +#include + +#include "ctrl_metrics.h" + +/* ------------------------------------------------------------------ */ +/* Test 1: ctrl_metrics_init resets all counters to 0 */ +/* ------------------------------------------------------------------ */ + +static void test_metrics_init_zeros_all(void) +{ + /* Dirty all metrics */ + CTRL_METRIC_SET(ctrl_split_brain_detected, 99); + CTRL_METRIC_SET(ctrl_async_events_lost, 99); + CTRL_METRIC_SET(ctrl_transient_jobs_stuck, 99); + CTRL_METRIC_SET(ctrl_wal_bytes_written, 99); + CTRL_METRIC_SET(ctrl_checkpoint_duration_ms, 99); + CTRL_METRIC_SET(ctrl_repl_lag_lsn, 99); + CTRL_METRIC_SET(ctrl_standbys_healthy, 99); + + ctrl_metrics_init(); + + TEST_ASSERT_EQUAL_UINT64(0, CTRL_METRIC_GET(ctrl_split_brain_detected)); + TEST_ASSERT_EQUAL_UINT64(0, CTRL_METRIC_GET(ctrl_async_events_lost)); + TEST_ASSERT_EQUAL_UINT64(0, CTRL_METRIC_GET(ctrl_transient_jobs_stuck)); + TEST_ASSERT_EQUAL_UINT64(0, CTRL_METRIC_GET(ctrl_wal_bytes_written)); + TEST_ASSERT_EQUAL_UINT64(0, CTRL_METRIC_GET(ctrl_checkpoint_duration_ms)); + TEST_ASSERT_EQUAL_UINT64(0, CTRL_METRIC_GET(ctrl_repl_lag_lsn)); + TEST_ASSERT_EQUAL_UINT64(0, CTRL_METRIC_GET(ctrl_standbys_healthy)); + + printf("test_metrics_init_zeros_all: PASS\n"); +} + +/* ------------------------------------------------------------------ */ +/* Test 2: INC, ADD, SET, GET macros */ +/* ------------------------------------------------------------------ */ + +static void test_metrics_macros(void) +{ + ctrl_metrics_init(); + + CTRL_METRIC_INC(ctrl_split_brain_detected); + TEST_ASSERT_EQUAL_UINT64(1, CTRL_METRIC_GET(ctrl_split_brain_detected)); + + CTRL_METRIC_INC(ctrl_split_brain_detected); + TEST_ASSERT_EQUAL_UINT64(2, CTRL_METRIC_GET(ctrl_split_brain_detected)); + + CTRL_METRIC_ADD(ctrl_wal_bytes_written, 512); + TEST_ASSERT_EQUAL_UINT64(512, CTRL_METRIC_GET(ctrl_wal_bytes_written)); + + CTRL_METRIC_ADD(ctrl_wal_bytes_written, 512); + TEST_ASSERT_EQUAL_UINT64(1024, CTRL_METRIC_GET(ctrl_wal_bytes_written)); + + CTRL_METRIC_SET(ctrl_repl_lag_lsn, 42); + TEST_ASSERT_EQUAL_UINT64(42, CTRL_METRIC_GET(ctrl_repl_lag_lsn)); + + CTRL_METRIC_SET(ctrl_repl_lag_lsn, 7); + TEST_ASSERT_EQUAL_UINT64(7, CTRL_METRIC_GET(ctrl_repl_lag_lsn)); + + printf("test_metrics_macros: PASS\n"); +} + +/* ------------------------------------------------------------------ */ +/* Test 3: descriptor table is well-formed and sentinel-terminated */ +/* ------------------------------------------------------------------ */ + +static void test_metrics_table_sentinel_terminated(void) +{ + const ctrl_metric_def_t *tbl = ctrl_metrics_table(); + TEST_ASSERT_NOT_NULL(tbl); + + int count = 0; + while (tbl[count].name != NULL) { + /* Each entry must have a non-empty description and valid type */ + TEST_ASSERT_NOT_NULL(tbl[count].description); + TEST_ASSERT_TRUE(strlen(tbl[count].description) > 0); + TEST_ASSERT_TRUE(strlen(tbl[count].name) > 0); + /* Name must start with "ctrl_" */ + TEST_ASSERT_EQUAL_INT(0, strncmp(tbl[count].name, "ctrl_", 5)); + TEST_ASSERT_TRUE(tbl[count].type == CTRL_METRIC_COUNTER + || tbl[count].type == CTRL_METRIC_GAUGE); + count++; + } + + /* We define exactly 7 metrics */ + TEST_ASSERT_EQUAL_INT(7, count); + + /* Sentinel: name is NULL, type is 0 */ + TEST_ASSERT_NULL(tbl[count].name); + + printf("test_metrics_table_sentinel_terminated: PASS (count=%d)\n", count); +} + +/* ------------------------------------------------------------------ */ +/* Test 4: transient_jobs_stuck incremented by job_should_retain */ +/* ------------------------------------------------------------------ */ + +static void test_transient_jobs_stuck_counter(void) +{ + ctrl_metrics_init(); + + /* Include persist.h for job_should_retain + fixtures */ + /* The fixture: transient job that has timed out */ + extern bool job_should_retain(const job_t*, const persist_config_t*); + #include "xce/types.h" + + persist_config_t cfg = persist_config_defaults(); + cfg.transient_job_timeout_s = 300; + + job_t j; + memset(&j, 0, sizeof(j)); + j.job.state = JOB_STAT_TRANSIENT; + j.job.transient_since = (uint64_t)(time(NULL) - 400); /* 400 s ago */ + + bool retain = job_should_retain(&j, &cfg); + TEST_ASSERT_FALSE(retain); + + /* Counter must have been bumped */ + TEST_ASSERT_EQUAL_UINT64(1, CTRL_METRIC_GET(ctrl_transient_jobs_stuck)); + + printf("test_transient_jobs_stuck_counter: PASS\n"); +} + +/* ------------------------------------------------------------------ */ +/* Test 5: wal_bytes_written incremented by persist_wal_async */ +/* ------------------------------------------------------------------ */ + +static void test_wal_bytes_written_counter(void) +{ + ctrl_metrics_init(); + + char tmpdir[256]; + strcpy(tmpdir, "/tmp/xce_p3_metrics_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); + + persist_t *p = calloc(1, sizeof(persist_t)); + TEST_ASSERT_NOT_NULL(p); + persist_config_t cfg = persist_config_defaults(); + cfg.wal_sync_ms = 5000; + persist_set_workdir(p, tmpdir, "metrics"); + TEST_ASSERT_EQUAL_INT(0, persist_init(p, &cfg, NULL)); + TEST_ASSERT_EQUAL_INT(0, persist_start(p)); + + const char payload[] = "metric-payload"; + uint32_t plen = (uint32_t)strlen(payload); + + uint64_t before = CTRL_METRIC_GET(ctrl_wal_bytes_written); + TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_SUBMIT, payload, plen)); + uint64_t after = CTRL_METRIC_GET(ctrl_wal_bytes_written); + + uint64_t expected_delta = sizeof(wal_entry_hdr_t) + plen; + TEST_ASSERT_EQUAL_UINT64(expected_delta, after - before); + + persist_stop(p); + persist_fini(p); + free(p); + printf("test_wal_bytes_written_counter: PASS\n"); +} + +/* ------------------------------------------------------------------ */ +/* Suite entry point */ +/* ------------------------------------------------------------------ */ + +void test_metrics(void) +{ + printf("\n>> P3-7: Metrics infrastructure\n\n"); + RUN_TEST(test_metrics_init_zeros_all); + RUN_TEST(test_metrics_macros); + RUN_TEST(test_metrics_table_sentinel_terminated); + RUN_TEST(test_transient_jobs_stuck_counter); + RUN_TEST(test_wal_bytes_written_counter); +} +``` + +### Commit sequence + +```bash +# Commit 1: headers + implementation +git add src/include/ctrl_metrics.h src/controller/ctrl_metrics.c +git commit -m "feat(p3-7): ctrl_metrics — atomic counters/gauges with ctrl_ prefix" + +# Commit 2: wire into persist + ha +git add src/controller/persist.c src/controller/ha.c +git commit -m "feat(p3-7): wire ctrl_metrics_init into persist_start and ha_init; add increment sites" + +# Commit 3: tests + meson +git add src/controller/testsuite/test_metrics.c src/controller/testsuite/meson.build +git commit -m "test(p3-7): metrics — init, macros, table validation, WAL counter, transient counter" +``` + +--- + +## meson.build integration (cumulative) + +After all steps, the additions to `src/controller/testsuite/meson.build` are: + +```meson +# P3 integration & hardening tests +e2e_sources += files( + 'test_e2e_failover.c', + 'test_e2e_redirect.c', + 'test_e2e_perf.c', + 'test_e2e_manual_failover.c', + 'test_metrics.c', +) +``` + +`test_e2e_wal.c` and `test_e2e_gc.c` are already in the build; their new test functions +are picked up automatically. + +Add suite entry calls to `src/controller/testsuite/test_main.c` (or equivalent runner): + +```c +void test_e2e_failover(void); +void test_e2e_redirect(void); +void test_e2e_perf(void); +void test_e2e_manual_failover(void); +void test_metrics(void); + +/* ... in main() ... */ +test_e2e_failover(); +test_e2e_redirect(); +test_e2e_perf(); +test_e2e_manual_failover(); +test_metrics(); +``` + +Also add `ctrl_metrics.c` to the controller's main `meson.build` source list so the +object is linked into both the controller binary and the test runner. + +--- + +## Verification + +```bash +cd /worktrees/persistent_ha +nix develop --command bash -c " + cd src + meson setup build --wipe -Db_sanitize=address,undefined 2>&1 | tail -5 + ninja -C build + ninja -C build test +" +``` + +Expected: all existing tests pass plus all P3 tests pass. ASAN/UBSan clean. + +--- + +## What P3 does NOT do + +- No live network tests (no real TCP connections between controllers) +- No Prometheus / alerting integration (out of scope per design doc) +- No `xce hostcontrol --failover` CLI surface (deferred to future work) +- No WAL compaction, multi-DC replication, or snapshot compression diff --git a/docs/superpowers/plans/2026-05-07-persistence-ha-overview.md b/docs/superpowers/plans/2026-05-07-persistence-ha-overview.md new file mode 100644 index 0000000..68aab5d --- /dev/null +++ b/docs/superpowers/plans/2026-05-07-persistence-ha-overview.md @@ -0,0 +1,39 @@ +# XCE Persistence & HA — Implementation Overview + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the XCE controller survive restarts and primary failures with no job/worker/queue state loss. + +**Architecture:** WAL + periodic checkpoint (dual-zarray COW) for single-controller persistence; LSF-style priority election + fan-out TCP replication for HA. libxce smart discovery handles client auto-reconnect after failover. + +**Tech Stack:** C (controller/lib), Cap'n Proto (serialization), Nix flakes + meson + ninja (build), Go (API server discovery bridge) + +--- + +## Phase Map + +Each phase is an independently shippable increment. Create the next phase plan only after the current phase passes its gate tests. + +| Phase | Plan file | Goal | Gate | +|-------|-----------|------|------| +| **P0** | `2026-05-07-p0-single-controller-persistence.md` | Controller survives restart | `P0-G` smoke tests green | +| **P1** | _create after P0 ships_ | Standby takes over when primary fails | Election + promotion tests green | +| **P2** | _create after P1 ships_ | Primary replicates to standbys in real time | Replication round-trip + catch-up tests green | +| **P2.5** | _create alongside P2_ | Workers/schedulers auto-reconnect after failover | Discovery + reconnect tests green | +| **P3** | _create after P2 ships_ | End-to-end failover + hardening + metrics | Full failover test + benchmark passing | +| **P4** | _create after P3 ships_ | Docs, runbook, stale-ref cleanup | Runbook review complete | + +## Out of Scope (all phases) + +- Admin CLI for manual failover (`xce hostcontrol --failover`) +- WAL compaction beyond checkpoint trimming +- Multi-DC replication +- Runtime config changes without restart +- Snapshot compression +- `config_version` hash in heartbeat (deferred to P3 tooling) +- Degraded-mode restricted-state timeout (`repl_degraded_timeout_s`) +- `MAX_HA_CANDIDATES > 4` / `MAX_STANDBYS > 3` + +## Design Document + +`src/docs/persistence-and-ha-design.md` — all implementation detail lives here. Read the relevant Part before starting each task. diff --git a/src/controller/cluster.c b/src/controller/cluster.c index f394c20..5a914a8 100644 --- a/src/controller/cluster.c +++ b/src/controller/cluster.c @@ -640,8 +640,28 @@ int cluster_init(cluster_t* c) .probe_connect_timeout_ms = HA_DEFAULT_PROBE_CONNECT_TIMEOUT_MS, .max_promotion_retries = HA_DEFAULT_MAX_PROMOTION_RETRIES, }; - /* my_priority: single-node P1 always uses 0 (highest priority) */ - rc = ha_init(&c->ha, &ha_cfg, 0); + + /* Derive priority from position in --peers list. + * Index 0 = highest priority (becomes primary on startup). + * If the controller's name is not found in the list, fall back to + * the last index (lowest priority = standby). */ + int my_priority = 0; + const xce_cluster_config_t* pcfg = &c->myconf->peers_cfg; + if (pcfg->candidates_count > 0) { + my_priority = pcfg->candidates_count - 1; /* default: last = lowest */ + for (int i = 0; i < pcfg->candidates_count; i++) { + if (strncmp(pcfg->candidates[i].hostname, + c->myconf->name, + XCE_MAXHOSTNAME_LEN - 1) == 0) { + my_priority = i; + break; + } + } + INFO("HA peers: %d candidates, my_priority=%d (name=%s)", + pcfg->candidates_count, my_priority, c->myconf->name); + } + + rc = ha_init(&c->ha, &ha_cfg, my_priority); if (rc != 0) { ERROR("ha_init failed: %d", rc); persist_fini(&c->persist); diff --git a/src/controller/main.c b/src/controller/main.c index bb6ed70..156e6c7 100644 --- a/src/controller/main.c +++ b/src/controller/main.c @@ -6,6 +6,7 @@ #include "myconf.h" #include "utils.h" +#include "xce/cluster.h" extern int myapp_run(myconf_t*); @@ -15,21 +16,60 @@ const char* argp_program_version = MK_VERSION("xce v0.0.1-", STR(VERSION_COMMIT_ID)); const char* argp_program_bug_address = BUG_EMAIL; +/* Long-option key (above ASCII to avoid collisions with short opts) */ +#define OPT_PEERS 256 + struct arguments { char* myname; char* logfile; char* workdir; int verbose; + char* peers_str; }; static struct argp_option options[] = { - { "workdir", 'w', "DIR", 0, "The work directory of xce controller" }, - { "logfile", 'l', "FILE", 0, "The full path of log file" }, - { "verbose", 'v', 0, 0, "Produce verbose logs" }, - { "myname", 'n', "STR", 0, "The name of controller" }, + { "workdir", 'w', "DIR", 0, "The work directory of xce controller" }, + { "logfile", 'l', "FILE", 0, "The full path of log file" }, + { "verbose", 'v', 0, 0, "Produce verbose logs" }, + { "myname", 'n', "STR", 0, "The name of controller" }, + { "peers", OPT_PEERS, "PEERS", 0, + "HA peer list: comma-separated host:port pairs ordered by priority " + "(e.g. ctrl01:9990,ctrl02:9990). Index 0 = highest priority (primary)." }, { 0 } }; +/* parse_peers — split "host:port,host:port,..." into cfg->candidates[]. + * Accepts optional "name=host:port" prefix (name is ignored for controller). + * Returns 0 on success, -1 on empty input or parse error. */ +static int parse_peers(const char* peers_str, xce_cluster_config_t* cfg) +{ + cfg->candidates_count = 0; + if (!peers_str || peers_str[0] == '\0') + return -1; + + char buf[2048]; + strncpy(buf, peers_str, sizeof(buf) - 1); + buf[sizeof(buf) - 1] = '\0'; + + char* saveptr; + char* tok = strtok_r(buf, ",", &saveptr); + while (tok && cfg->candidates_count < MAX_HA_CANDIDATES) { + char* eq = strchr(tok, '='); + char* addr = eq ? eq + 1 : tok; + char* colon = strrchr(addr, ':'); + if (!colon) + return -1; + *colon = '\0'; + strncpy(cfg->candidates[cfg->candidates_count].hostname, addr, + XCE_MAXHOSTNAME_LEN - 1); + cfg->candidates[cfg->candidates_count].hostname[XCE_MAXHOSTNAME_LEN - 1] = '\0'; + cfg->candidates[cfg->candidates_count].port = atoi(colon + 1); + cfg->candidates_count++; + tok = strtok_r(NULL, ",", &saveptr); + } + return cfg->candidates_count > 0 ? 0 : -1; +} + static error_t parse_opt(int key, char* arg, struct argp_state* state) { struct arguments* arguments = state->input; @@ -47,6 +87,9 @@ static error_t parse_opt(int key, char* arg, struct argp_state* state) case 'n': arguments->myname = arg; break; + case OPT_PEERS: + arguments->peers_str = arg; + break; default: return ARGP_ERR_UNKNOWN; } @@ -111,6 +154,27 @@ int main(int argc, char* argv[]) myconf.keyfile = key_file; myconf.workdir = arguments.workdir; myconf.name = arguments.myname; + memset(&myconf.peers_cfg, 0, sizeof(myconf.peers_cfg)); + + if (arguments.peers_str) { + if (parse_peers(arguments.peers_str, &myconf.peers_cfg) != 0) { + fprintf(stderr, + "xce_controller: --peers parse error: expected " + "host:port,... got: %s\n", + arguments.peers_str); + return 1; + } + /* CA cert derived from workdir — same infrastructure as certfile/keyfile */ + snprintf(myconf.peers_cfg.ca_cert_path, + sizeof(myconf.peers_cfg.ca_cert_path), + "%s/certs/xce.crt", arguments.workdir); + strncpy(myconf.peers_cfg.identity, arguments.myname, + sizeof(myconf.peers_cfg.identity) - 1); + INFO("HA mode: %d controller candidates, identity=%s", + myconf.peers_cfg.candidates_count, myconf.peers_cfg.identity); + } else { + INFO("No --peers flag; running as single-node primary"); + } myapp_run(&myconf); diff --git a/src/docs/persistence-and-ha-design.md b/src/docs/persistence-and-ha-design.md index c9eb70a..f4dc2b3 100644 --- a/src/docs/persistence-and-ha-design.md +++ b/src/docs/persistence-and-ha-design.md @@ -1,7 +1,7 @@ # XCE Controller: State Persistence & High Availability Design -> **Status**: Draft -> **Last Updated**: 2026-04-20 +> **Status**: Reviewed +> **Last Updated**: 2026-05-07 > **Target**: `src/controller/` (C daemon) --- @@ -39,7 +39,7 @@ The controller has no database code. It uses `kvmgr` (in-memory hash tables) for | Location | Code | Action | |----------|------|--------| -| `worker.c:367-391` | `restore_worker()` | **Keep** — currently unused but will be the persistence restore entry point (Part 2). Note: `restore_worker()` currently takes different parameters than a deserialized worker context will provide. The signature will need to be updated to accept a `worker_t*` from Cap'n Proto deserialization, or a shim function must be added. | +| `worker.c:367-391` | `restore_worker()` | **Keep** — currently unused but will be the persistence restore entry point (Part 2). The existing `restore_worker(void* cls, xce_worker_t host)` signature is **unchanged**. A new adapter function `worker_from_capnp()` (see P0-12) will bridge Cap'n Proto deserialization to `restore_worker()` — the recovery path calls `worker_from_capnp()`, not `restore_worker()` directly. | ### 1.3 Empty Function in `cluster.c` @@ -142,6 +142,8 @@ enum wal_entry_type { **Note**: There are no `WAL_JOB_DELETE`, `WAL_WORKER_DELETE`, or `WAL_QUEUE_DELETE` entry types. Entity cleanup is deferred to checkpoint (see Part 4). This means WAL never contains delete records — only creates and updates — simplifying replay logic. +**WAL_CHECKPOINT replication skip — code boundary**: `WAL_CHECKPOINT` (type=100) is written to the local WAL only via `persist_write_checkpoint_marker()` — a dedicated function in `persist.c` that calls `pwrite()`/`fsync()` directly and **never** calls `repl_broadcast()`. By contrast, `persist_wal_async()` and `persist_wal_sync_append()` call `repl_broadcast()` after writing. The invariant `persist_wal_async(WAL_CHECKPOINT, ...)` MUST NOT be called — enforce with `assert(type != WAL_CHECKPOINT)` at the top of both `persist_wal_async()` and `persist_wal_sync_append()` in debug builds. This makes the code-path boundary explicit rather than relying on callers to never pass `WAL_CHECKPOINT` to the replication path. The `last_replicated_crc` field in `persist_t` (distinct from `last_crc` which covers all entries including checkpoints) is updated only by the replication send paths, never by `persist_write_checkpoint_marker()` — this is the mechanism that maintains a continuous CRC chain for standbys that skip checkpoint entries. The standby's mirror of this field is `ha_state_t.last_received_crc` (Appendix A.4), which tracks the CRC of the last replicated entry the standby received; together these two fields implement the chain validation described in Part 6.4. + **Mapping to existing hook detail enums:** | Hook Detail (`cluster_def.h`) | WAL Entry Type | Replication | Notes | @@ -164,7 +166,7 @@ enum wal_entry_type { Footnotes: -1. **`HOOK_DETAIL_JOB_STOP`**: Does not exist in the current `cluster_def.h` enum. It must be added as a prerequisite for P0-6. Job stop currently goes through the same code path as job exit (`HOOK_DETAIL_JOB_EXIT`). If stop and exit should map to different WAL types, a new hook detail must be added and the `on_job_stop()` function must call hooks with `HOOK_DETAIL_JOB_STOP` instead of `HOOK_DETAIL_JOB_EXIT`. +1. **`HOOK_DETAIL_JOB_STOP`**: Does not exist in the current `cluster_def.h` enum. It must be added as a prerequisite for P0-6. **Codebase reality**: `on_job_stop()` currently calls `mark_job_update()` which fires `HOOK_DETAIL_JOB_UPDATE` — it does NOT fire `HOOK_DETAIL_JOB_EXIT`. The implementation in P0-4a must update `on_job_stop()` to call `xce_run_hooks` directly with `HOOK_DETAIL_JOB_STOP`, bypassing `mark_job_update()` for this event. Search `grep mark_job_update src/controller/job.c` to find the current call sites. 2. **`HOOK_DETAIL_JOB_RESUME`**: Does not exist in the current `cluster_def.h` enum. It must be added as a prerequisite for P0-6. The existing `on_job_resume()` function in `job.c` must be updated to call hooks with this new detail. @@ -193,7 +195,7 @@ Footnotes: ### 2.4 WAL Segment Management ``` -wal_path/ +effective_dir/data/wal/ ├── seg-0000000000000000.wal # Pre-allocated 64 MB ├── seg-0000000000000064.wal # LSN offset in filename └── ... @@ -203,65 +205,115 @@ wal_path/ - When a segment is full, rotate to the next - `fsync()` at configurable interval (`wal_sync_ms`, default 10ms) — batched, not per-write. If `fsync()` fails, `persist_wal_sync_append()` and `persist_wal_async()` return an error. For sync replication, the failed write is not sent to standbys and the client receives an error reply. For async replication, the error is logged and the controller continues operating — the WAL entry is in the OS page cache but not on disk, so it may be lost on crash. Monitoring should alert on persistent fsync failures. > **Implementation note**: `delaymgr` uses `sleep(precision)` with `precision=2` seconds (see `cluster.c:604`), giving 2-second minimum granularity — far too coarse for `wal_sync_ms=10`. The WAL fsync timer must use a **dedicated pthread** with `nanosleep()` or a `timerfd_create(CLOCK_MONOTONIC)` loop, not `delaymgr`. The periodic checkpoint timer (P0-3) can still use `delaymgr` since `snap_interval_s` is measured in seconds. -- **WAL trim guard**: WAL segments are only deleted up to `min(acked_lsn)` across all connected standbys (`standby_conn_t.acked_lsn` — updated on each `OP_replAck` received). This prevents trimming segments that a lagging standby still needs for catch-up. When a standby reconnects after a gap, it reports its last applied LSN, and the primary streams missing entries from that point. If the standby's LSN is older than the primary's oldest surviving WAL segment, a full snapshot transfer is required (see Part 6 standby catch-up). Each standby heartbeat includes the standby's `applied_lsn` so the primary always knows the safe trim point. **Empty-standby fallback**: When no standbys are connected (`replicator.nstandbys == 0`), use the primary's own `checkpoint_lsn` as the trim point — WAL segments before the latest confirmed checkpoint can be safely deleted. **Standby disconnect trim delay**: When a standby disconnects, the primary should delay WAL trimming by one checkpoint interval before falling back to `checkpoint_lsn`-based trimming. This gives the standby a window to reconnect and resume catch-up without requiring a full snapshot transfer. The implementation uses per-slot tracking in `replicator_t.last_disconnected_lsn[]` and `replicator_t.last_disconnect_time[]` (see Appendix A.3). When a standby disconnects, its `acked_lsn` and current time are recorded in its slot. The trim point is computed as `min(min_over_disconnected_slots(last_disconnected_lsn[i]), checkpoint_lsn)` for one checkpoint interval after each disconnect, then falls back to `checkpoint_lsn` per slot. +- **WAL trim guard**: WAL segments are only deleted up to `min(acked_lsn)` across all connected standbys that are **not in catch-up mode** (`standby_conn_t.acked_lsn` — updated on each `OP_replAck` received, but only from standbys with `in_catch_up == 0`). For catching-up standbys, use their `OP_replJoin.applied_lsn` (the LSN they reported on reconnect) as the floor — WAL segments from that LSN forward must be retained until the standby sends `OP_replCatchUpDone`. In-catch-up standbys have stale `acked_lsn` values (from before their catch-up started); using stale `acked_lsn` as the trim point could trim entries the standby is about to request, forcing snapshot transfer when only a short WAL replay was needed. Concretely: `trim_lsn = min(min_non_catchup(acked_lsn), min_catchup(join_applied_lsn), checkpoint_lsn)`. This prevents trimming segments that a lagging standby still needs for catch-up. When a standby reconnects after a gap, it reports its last applied LSN, and the primary streams missing entries from that point. If the standby's LSN is older than the primary's oldest surviving WAL segment, a full snapshot transfer is required (see Part 6 standby catch-up). Each standby heartbeat includes the standby's `applied_lsn` so the primary always knows the safe trim point. **Empty-standby fallback**: When no standbys are connected (`replicator.nstandbys == 0`), use the primary's own `checkpoint_lsn` as the trim point — WAL segments before the latest confirmed checkpoint can be safely deleted. **Standby disconnect trim delay**: When a standby disconnects, the primary should delay WAL trimming by one checkpoint interval before falling back to `checkpoint_lsn`-based trimming. This gives the standby a window to reconnect and resume catch-up without requiring a full snapshot transfer. The implementation uses per-slot tracking in `replicator_t.last_disconnected_lsn[]` and `replicator_t.last_disconnect_time[]` (see Appendix A.3). When a standby disconnects, its `acked_lsn` and current time are recorded in its slot. The trim point is computed as `min(min_over_disconnected_slots(last_disconnected_lsn[i]), checkpoint_lsn)` for one checkpoint interval after each disconnect, then falls back to `checkpoint_lsn` per slot. **Slot expiry (CRITICAL)**: A disconnected slot's `last_disconnected_lsn[i]` hold on the trim guard expires exactly when `now >= last_disconnect_time[i] + snap_interval_s`. At that point, `last_disconnected_lsn[i]` is reset to 0 and `last_disconnect_time[i]` is cleared, and that slot no longer contributes to the trim guard min(). **Permanent decommission**: When a standby is removed from the `--peers` candidate list (its slot index is no longer valid), its `last_disconnected_lsn[i]` and `last_disconnect_time[i]` must be zeroed immediately in `replicator_remove_candidate()`. Without explicit slot invalidation, a decommissioned standby's disconnect record holds the WAL trim guard indefinitely — a WAL growth leak with no upper bound other than `wal_max_size_mb`. - **WAL growth bound during standby disconnect**: The worst-case WAL accumulation when no standbys are connected is: `wal_growth_MB ≈ snap_wal_size_mb + (submit_rate_per_sec * avg_WAL_submit_KB * snap_interval_s * 2 / 1024)`. The `* 2` accounts for the one-interval trim delay before `checkpoint_lsn`-based trimming takes effect. At default settings (`snap_interval_s = 60`, `snap_wal_size_mb = 64`) with 10 submits/sec and ~1 KB average `WAL_JOB_SUBMIT` payload, worst-case growth is ≈65 MB — well within a single segment. At pathological rates (1000 submits/sec, 4 KB average payload), worst-case growth is ≈535 MB, requiring ~9 segments. **Mitigation**: If WAL disk usage exceeds `wal_max_size_mb` (default: 2 GB), log an `ERROR` and trigger an immediate checkpoint to advance `checkpoint_lsn` and enable trimming. The one-interval trim delay is skipped when `replicator.nstandbys == 0` and WAL usage exceeds `wal_max_size_mb / 2` — in this scenario, preserving disk space takes priority over the reconnection catch-up window. - Segments before the latest snapshot are deleted during recovery, subject to the trim guard above -- Chained CRC detects corruption: if `prev_crc` doesn't match previous record's CRC, stop replay at last valid entry. The first record in a WAL segment uses `prev_crc = WAL_CRC_NONE` (defined as `0xFFFFFFFF`, not 0 — CRC32 can legitimately produce 0 for certain inputs, which would be indistinguishable from the "no predecessor" sentinel if 0 were used). Recovery validates by checking that each record's `prev_crc` matches the previous record's `crc` field, starting from `WAL_CRC_NONE` for the first record. **Segment boundary handling**: When recovery encounters `prev_crc == WAL_CRC_NONE` at a position that is NOT the absolute first record in the file (i.e., at the start of a rotated segment), this is treated as a **valid chain break** — the CRC chain restarts from this point. The recovery algorithm must NOT treat `WAL_CRC_NONE` at any record position that is 8-byte aligned and at the beginning of a segment file as corruption. Each segment file is independently CRC-chained from its first record. +- **WAL integrity model (explicit boundary)**: CRC32 (chained) detects accidental corruption — bit flips, torn writes, truncated segments. It is **NOT tamper-resistant**: an attacker with write access to the WAL directory can modify payload bytes and recompute a valid CRC32. Tamper resistance would require HMAC-SHA256 keyed on a controller-local secret. XCE's integrity model relies on OS-level filesystem permissions (mode 0700, controller-UID-owned) as the access control boundary, not on cryptographic authentication. Implementers must not assume WAL CRC provides any security guarantee beyond accidental-corruption detection. If tamper resistance is required for a deployment, use encrypted storage (LUKS/dm-crypt) which protects both confidentiality and integrity. +- Chained CRC detects corruption: if `prev_crc` doesn't match previous record's CRC, stop replay at last valid entry. The first record in a WAL segment uses `prev_crc = WAL_CRC_NONE` (defined as `0xFFFFFFFF`, not 0 — CRC32 can legitimately produce 0 for certain inputs, which would be indistinguishable from the "no predecessor" sentinel if 0 were used). Recovery validates by checking that each record's `prev_crc` matches the previous record's `crc` field, starting from `WAL_CRC_NONE` for the first record. **Segment boundary handling**: When recovery encounters `prev_crc == WAL_CRC_NONE` at a position that is NOT the absolute first record in the file (i.e., at the start of a rotated segment), this is treated as a **valid chain break** — the CRC chain restarts from this point. The recovery algorithm must NOT treat `WAL_CRC_NONE` at any record position that is 8-byte aligned and at the beginning of a segment file as corruption. Each segment file is independently CRC-chained from its first record. **Writer-side obligation (ADV-11)**: `persist_wal_sync_append()` and `persist_wal_async()` MUST set `hdr.prev_crc = WAL_CRC_NONE` when writing the first record of a new segment (i.e., when `persist_t.seg_offset == 0` after segment rotation). The implementation MUST NOT carry `persist_t.last_crc` across a segment boundary — the new segment starts a fresh chain. This is the mirror of the recovery-side rule; without it, recovery would see a non-`WAL_CRC_NONE` `prev_crc` at a segment boundary and enter 8-byte linear scan mode unnecessarily. - **Truncated segment recovery**: If a crash occurs mid-write, the last WAL segment may contain a partial record. The recovery algorithm handles two cases: 1. **Valid header CRC, truncated payload**: Read the full 40-byte header, validate `hdr_crc` against `WAL_HDR_CRC_SIZE` bytes. If the header CRC is valid but `payload_len` exceeds the remaining bytes in the segment, truncate at the previous valid record. Do NOT use the declared `payload_len` to advance the read position — a partially-written `payload_len` could be corrupted, causing the scanner to jump past valid records. 2. **Invalid header CRC (corrupted header)**: Enter 8-byte linear scan mode — skip forward 8 bytes at a time, checking for a valid `hdr_crc` and matching `prev_crc` at each alignment boundary. This ensures that a corrupted header cannot cause the scanner to skip past valid data. - **WAL trim and `total_wal_bytes` accounting**: After a successful checkpoint, the WAL trim step deletes segment files whose last LSN is ≤ `checkpoint_lsn`. On each segment deletion, `total_wal_bytes` is decremented by the segment file size (`fstat(fd).st_size`). On each WAL write, `total_wal_bytes` is incremented by the bytes written. On segment rotation, the new segment file size is not counted until bytes are written to it. This ensures `total_wal_bytes` accurately reflects the on-disk WAL size at all times. In both cases, recovery stops at the end of the WAL segment (the segment file size) when it cannot find a valid record. The `seg_offset` field in `persist_t` tracks the last fsync point at runtime but is **in-memory and lost on crash** — recovery must NOT rely on it as the trust boundary. Instead, recovery validates entries up to the physical end of the segment file using chained CRC: records with invalid CRC or prev_crc mismatch are treated as corruption. The recovery algorithm scans from the beginning of the segment and stops when it encounters an unrecoverable CRC failure at or near the end of the file. `seg_offset` is only used during normal operation to track how far the fsync thread has flushed — it is updated to the post-write position only after `fsync()` succeeds, ensuring that on crash + restart (before the next fsync), the runtime can still distinguish flushed from unflushed data if needed. -**Data at rest**: WAL segments and snapshot files contain job submission payloads (which may include environment variables, command lines, and user-specified data). If the storage directory is on a shared or compromised filesystem, this data is accessible. XCE relies on OS-level filesystem permissions (`wal_path`/`snap_path` directories should be owner-read-only, mode 0700) and physical host security rather than encryption at rest. If encryption at rest is required, it should be provided by the underlying filesystem (e.g., LUKS, encrypted EBS volumes) rather than by the controller — adding application-level encryption would double the encryption cost with filesystem-level encryption already in place for most production deployments. +**Data at rest**: WAL segments and snapshot files contain job submission payloads (which may include environment variables, command lines, and user-specified data). If the storage directory is on a shared or compromised filesystem, this data is accessible. XCE relies on OS-level filesystem permissions (effective_dir/data/wal and effective_dir/data/snap directories should be owner-read-only, mode 0700, where effective_dir is derived from --workdir and -n name CLI flags) and physical host security rather than encryption at rest. If encryption at rest is required, it should be provided by the underlying filesystem (e.g., LUKS, encrypted EBS volumes) rather than by the controller — adding application-level encryption would double the encryption cost with filesystem-level encryption already in place for most production deployments. **Data retention and scrubbing**: Sensitive data (environment variables, job command lines) persists in WAL files until those segments are trimmed (after checkpoint), and in snapshot files until the next checkpoint replaces them. The retention window is bounded by `finished_retain_s` (default 1h) for job payloads: once a job is GC'd at checkpoint, its data no longer appears in new snapshots, and its WAL segment is trimmed after the checkpoint that removed it. Old WAL segments and snapshot files are deleted with `unlink()` — no explicit overwrite/scrubbing of file data is performed. If the storage device requires explicit data destruction (e.g., SSDs with wear leveling where unlink does not guarantee physical erasure), this must be handled at the storage layer (encrypted volumes with key rotation, or secure erase tooling). XCE does not implement application-level data scrubbing of WAL or snapshot files. -**WAL payload logging policy**: The persistence and replication modules must **never log WAL payload content**. Only entry metadata (type, LSN, term, payload_len, CRC) may appear in log messages. This is a security hardening invariant: `WAL_JOB_SUBMIT` payloads contain job environment variables which frequently hold API keys, database passwords, and other credentials. All logging call sites in `persist.c` and `repl.c` that reference WAL entries must log only `hdr->type`, `hdr->lsn`, `hdr->term`, `hdr->payload_len`, and CRC values — never the `payload` pointer or its contents. Code review must verify this as a gating check before merge. The CRC mismatch error path already follows this pattern: `ERROR("replication CRC mismatch: expected %u, got %u, lsn %lu", hdr.crc, computed_crc, hdr.lsn)`. **Structural enforcement (recommended)**: Define an opaque `wal_payload_t` wrapper type that cannot be format-printed without an explicit unwrap function. Alternatively, define a `persist_log_entry_metadata()` helper that only accepts the header struct — preventing callers from passing the payload pointer. Relying solely on code review is a human control that can fail; a type-level or helper-level enforcement makes credential logging a compile-time or link-time error rather than a review-time catch. +**WAL payload logging policy**: The persistence and replication modules must **never log WAL payload content**. Only entry metadata (type, LSN, term, payload_len, CRC) may appear in log messages. This is a security hardening invariant: `WAL_JOB_SUBMIT` payloads contain job environment variables which frequently hold API keys, database passwords, and other credentials. All logging call sites in `persist.c` and `repl.c` that reference WAL entries must log only `hdr->type`, `hdr->lsn`, `hdr->term`, `hdr->payload_len`, and CRC values — never the `payload` pointer or its contents. Code review must verify this as a gating check before merge. The CRC mismatch error path already follows this pattern: `ERROR("replication CRC mismatch: expected %u, got %u, lsn %lu", hdr.crc, computed_crc, hdr.lsn)`. **Structural enforcement (required — P0-1 deliverable)**: Define a `persist_log_entry_metadata(const wal_entry_hdr_t* hdr)` helper in `persist.h` that accepts only the header struct and logs `type`, `lsn`, `term`, `payload_len`, and CRC — it has no payload parameter and cannot log payload content. All log call sites in `persist.c` and `repl.c` that emit WAL entry information must use this helper. This converts credential-logging from a review-time catch to a build-time enforcement: a developer cannot format-print a payload pointer without explicitly calling an unwrap function that clearly signals intent. Relying solely on code review is a human control that will eventually fail given how frequently debug log statements are added during development. Add `persist_log_entry_metadata()` to the P0-1 deliverable table. -**Filesystem type requirement**: `wal_path` and `snap_path` must be on a local filesystem (ext4, xfs, etc.), not on NFS or other network filesystems. The `fsync()` durability guarantee depends on the OS writing to stable storage — on NFS, `fsync()` may only flush to the server's cache, not to persistent storage, undermining the WAL's crash-recovery guarantee. Run a startup check using `statfs()` and reject `f_type` values matching `NFS_SUPER_MAGIC` (0x6969) or `SMB_SUPER_MAGIC` (0x517B) — log `ERROR` and abort if detected. +**Filesystem type requirement**: `effective_dir/data/wal` and `effective_dir/data/snap` must be on a local filesystem (ext4, xfs, etc.), not on NFS or other network filesystems. (where effective_dir is derived from `--workdir` and `-n` name CLI flags) The `fsync()` durability guarantee depends on the OS writing to stable storage — on NFS, `fsync()` may only flush to the server's cache, not to persistent storage, undermining the WAL's crash-recovery guarantee. Run a startup check using `statfs()` and reject `f_type` values matching `NFS_SUPER_MAGIC` (0x6969) or `SMB_SUPER_MAGIC` (0x517B) — log `ERROR` and abort if detected. ### 2.5 Recovery Sequence On startup, `cluster_init()` calls `persist_recover()`: -1. **Find latest snapshot**: Scan `snap_path/` for the snapshot with the highest LSN +1. **Find latest snapshot**: Scan `effective_dir/data/snap/` for the snapshot with the highest LSN 2. **Load snapshot**: Read the snapshot file into memory 2a. **Verify snapshot checksums**: (i) Recompute CRC32 over the header fields (lsn, timestamp, term, version, nJobs, nQueues, nWorkers) and compare against `SnapshotHeader.headerChecksum`. If mismatch, the header has been tampered or corrupted — log an error and try the next-oldest snapshot. (ii) Recompute CRC32 over the Cap'n Proto payload and compare against `SnapshotHeader.checksum`. If mismatch, log an error and try the next-oldest snapshot. If **all** snapshots fail verification and the WAL contains entries that predate any valid snapshot (i.e., WAL segments have already been trimmed), the controller **must abort with a fatal error** — silently booting with empty state would silently discard live job/worker state and is data loss. The fatal error message must include: the number of corrupt snapshots found, the LSN range of surviving WAL segments, and the instruction `"Contact support or restore from backup before restarting"`. Do not fall back to empty state in this scenario. The empty-state fallback is only valid on first boot (see below). **Snapshot version check**: After verifying both checksums, check `SnapshotHeader.version`. If the version is higher than the current controller's `WAL_FORMAT_VERSION` (defined in `persist.h` as `1`), log a fatal error and abort — a snapshot written by a newer controller cannot be safely loaded by an older one. This catches version mismatches earlier than encountering unknown WAL entry types during replay. **Fallback WAL verification**: When falling back from the latest snapshot to an older snapshot (due to checksum failure), the recovery code must first verify that WAL segments exist from `older_snapshot.lsn + 1` forward. If WAL segments have already been trimmed past the older snapshot's LSN (i.e., only segments covering `[latest_snapshot.lsn, ...]` exist), the fallback cannot replay the gap and the controller must abort with a fatal error. Do NOT silently accept an older snapshot with no WAL to replay — this would start the controller with stale state, silently discarding all changes between the older snapshot and the trimmed WAL. 2b. **Deserialize**: Deserialize Cap'n Proto data, rebuild kvmgr instances. Set `capnp::ReaderOptions.traversalLimitInWords` to a **fixed upper bound** (`MAX_SNAPSHOT_BYTES / 8` words) to prevent O(n²) traversal amplification from a crafted snapshot — this is the same hardening practice used for WAL payload deserialization (Part 6.4). Do NOT set it proportional to the actual file size, as a small file with deeply nested structure can still trigger excessive traversal. The limit should be `64 * 1024 * 1024 / 8 = 8M words` (the 64 MB is the expected max snapshot size). If the snapshot file is larger than `MAX_SNAPSHOT_BYTES` (default: 256 MB — 4× the expected 64 MB for 10K jobs × 1 KB + overhead), log an `ERROR` and reject the snapshot. - Jobs → `makeJob(c->jobmgr, ...)` (reuse existing `create_job()`) - - Workers → `restore_worker(c, host)` (already exists in `worker.c:367`) + - Workers → `restore_worker(cls, host)` (already exists in `worker.c:367`) - Queues → `create_queue(c, qconf)` (already exists) - Resources → `makeResource(c->resmgr, ...)` (already exists) - Configs → `add_config(c, key, val)` (already exists) -3. **Replay WAL**: Starting from the snapshot's LSN, read and apply WAL entries forward via `apply_wal_entry()`. Hooks are **not yet registered** during recovery — persistence hooks are registered only after `persist_recover()` completes in `cluster_init()`, so no bypass flag is needed. This avoids re-persisting (WAL write during recovery) and prevents replication sends during replay. For `WAL_JOB_SUBMIT` entries, replay calls `restore_job(c, jobid, payload)` — a dedicated function that inserts the job at its original jobid without allocating a new one. Calling `job_submit()` here would assign a new jobid, breaking replay by creating duplicate or wrong-ID jobs. **CRITICAL: Job ID allocator reset** — after WAL replay completes, the job ID allocator (idgen or equivalent) must be positioned past the highest jobid seen during recovery (from both snapshot and WAL replay). Without this step, the next `job_submit()` allocates a duplicate ID, causing silent data corruption. Track `max_replayed_jobid` during snapshot deserialization and WAL replay, then call `idgen_set_next(max_replayed_jobid + 1)` after step 3. +3. **Replay WAL**: Starting from the snapshot's LSN, read and apply WAL entries forward via `apply_wal_entry()`. Hooks are **not yet registered** during recovery — persistence hooks are registered only after `persist_recover()` completes in `cluster_init()`, so no bypass flag is needed. This avoids re-persisting (WAL write during recovery) and prevents replication sends during replay. For `WAL_JOB_SUBMIT` entries, replay calls `restore_job(c, jobid, payload)` — a dedicated function that inserts the job at its original jobid without allocating a new one. Calling `job_submit()` here would assign a new jobid, breaking replay by creating duplicate or wrong-ID jobs. **Job ID allocation note**: The controller does NOT allocate jobids — job IDs are assigned by the Go API server via `idgen` (`clib.go`) before the submit request reaches the controller. Recovery merely restores jobs at their original IDs from the WAL; no idgen re-seeding is required in the controller. **Recovery is single-threaded**: `persist_recover()` runs before `cluster_init()` starts the iomgr, vcpu pool, or any other threads — all `restore_job()`/`xkvmgr_add()` calls during recovery are serialized and require no additional locking beyond what the functions already provide. **Worker behavior during recovery**: iomgr is not yet listening during recovery, so workers attempting to connect receive "connection refused". libxce's `xce_reconnect()` backoff+jitter loop handles this transparently — no protocol-level "not ready" response is needed. Workers connect normally once iomgr starts after recovery completes. 4. **Trim WAL**: Delete WAL segments fully covered by the loaded snapshot 5. **Retention pass**: After replay, run the retention/GC check on all entities (equivalent to a thaw pass) to clean up any entities whose TTL has expired during the controller's downtime. This handles the case where a controller was down for longer than `finished_retain_s` or `worker_retain_s` — expired entities are cleaned immediately on recovery. -6. **Restore election term**: Set `ha.term = max(snapshot_header.term, max_term_from_replayed_wal)`. This is a security-critical invariant: a controller must never start with a term lower than any term it has previously used, as this would undermine split-brain fencing. On first boot (empty state), `ha.term` is initialized to 0. **Snapshot version backward compatibility**: If `SnapshotHeader.version` is lower than the current `WAL_FORMAT_VERSION`, this indicates a snapshot written by an older controller. New schema fields must be optional with defaults to ensure backward compatibility. If the schema has breaking changes between versions, a migration function (e.g., `migrate_snapshot_v1_to_v2()`) must be written before bumping `WAL_FORMAT_VERSION`. Do not accept older versions without verifying field compatibility — the default behavior of loading an older snapshot with missing fields is safe only if all new fields are optional with sensible defaults. +6. **Restore election term**: Set `ha.term = max(snapshot_header.term, max_term_from_replayed_wal)`. This is a security-critical invariant: a controller must never start with a term lower than any term it has previously used, as this would undermine split-brain fencing. On first boot (empty state), `ha.term` is initialized to 0. **P0 scope note**: `ha_state_t` and its `term` field are defined in Part 5.3 (HA design), which is implemented in P1. P0 implementers must nonetheless allocate a `uint64_t` placeholder for the term in the snapshot header (`SnapshotHeader.term`) and in WAL entry headers (`wal_entry_hdr_t.term`) — these fields exist in the serialized format even before HA is wired up. During P0 (single-controller), `ha.term` is always 0; the restore step is a no-op. The field is read and written correctly in P0 so that P1 can wire `ha_state_t.term` to it without format changes. **Snapshot version backward compatibility**: If `SnapshotHeader.version` is lower than the current `WAL_FORMAT_VERSION`, this indicates a snapshot written by an older controller. New schema fields must be optional with defaults to ensure backward compatibility. If the schema has breaking changes between versions, a migration function (e.g., `migrate_snapshot_v1_to_v2()`) must be written before bumping `WAL_FORMAT_VERSION`. Do not accept older versions without verifying field compatibility — the default behavior of loading an older snapshot with missing fields is safe only if all new fields are optional with sensible defaults. > **Forward compatibility warning — unknown WAL entry types during recovery**: `apply_wal_entry()` logs a `WARN` and returns 0 (skips) unrecognized `wal_entry_type` values. This allows a controller running an older version to recover from WAL segments written by a newer version — the old controller boots, skips entries it cannot interpret, and continues operating. However, the state changes intended by those skipped entries are **silently lost**. This is a downgrade risk: if a newer controller writes entry types the old controller does not understand, and the old controller is started on that WAL, the cluster state will be incomplete (missing features, incorrect job/worker state for data written by the newer controller). Deployments should never roll back to an older controller version after a newer version has written persistent state. If rollback is required, the only safe procedure is: drain all jobs, stop all controllers, delete WAL and snapshot directories on all candidates, then start the old controller in empty-state-first-boot mode. > > **WAL entry type version guard**: To convert silent data loss into a loud failure, `persist_recover()` tracks the maximum `wal_entry_type` value encountered during WAL replay. If any entry has `type > WAL_ENTRY_TYPE_MAX` (defined in `persist.h` as the highest known entry type in the current version — initially `WAL_CHECKPOINT = 100`), `persist_recover()` logs a `FATAL` error and aborts: `"WAL contains entry type %u beyond current version's maximum %u — controller version is too old to safely recover; upgrade to a newer version or drain and reset WAL"`. This mirrors the snapshot version check pattern (step 2a) and prevents an older controller from silently booting with incomplete state. Operators who genuinely need to force-boot against a newer WAL (e.g., emergency recovery) can use the `--force-wal-version` startup flag, which downgrades the abort to a `WARN` per unknown entry type. -**Empty state (first boot after upgrade)**: If both `snap_path/` and `wal_path/` are empty (no snapshots, no WAL segments), `persist_recover()` returns 0 (success) with no state loaded — equivalent to today's in-memory-only boot. This ensures backward compatibility: existing deployments upgrading from in-memory-only operation need no migration steps. The controller simply starts with empty state and begins accumulating WAL entries. +**Empty state (first boot after upgrade)**: If both `effective_dir/data/snap/` and `effective_dir/data/wal/` are empty (no snapshots, no WAL segments), `persist_recover()` returns 0 (success) with no state loaded — equivalent to today's in-memory-only boot. This ensures backward compatibility: existing deployments upgrading from in-memory-only operation need no migration steps. The controller simply starts with empty state and begins accumulating WAL entries. -**Valid snapshot, empty WAL (clean shutdown)**: If `snap_path/` contains a valid snapshot file but `wal_path/` has no segments, `persist_recover()` loads the snapshot with no WAL replay. This is the expected state after a clean shutdown where the last checkpoint has already trimmed all WAL segments. No error or warning is generated. +**Valid snapshot, empty WAL (clean shutdown)**: If `effective_dir/data/snap/` contains a valid snapshot file but `effective_dir/data/wal/` has no segments, `persist_recover()` loads the snapshot with no WAL replay. This is the expected state after a clean shutdown where the last checkpoint has already trimmed all WAL segments. No error or warning is generated. + +**Last-known-good snapshot retention**: When a new checkpoint completes successfully, the controller retains the previous `snap_keep_n - 1` snapshots on disk (default `snap_keep_n = 2`, i.e., current + one previous). **When this fallback works**: If the controller crashes mid-checkpoint (before the new snapshot is `fsync`'d and the WAL is trimmed), the previous snapshot and its follow-on WAL segments are intact — recovery falls back to the previous snapshot and replays the bridging WAL. **When this fallback does NOT work**: If a checkpoint completed successfully (new snapshot fsynced, WAL trimmed up to new `checkpoint_lsn`), and the resulting snapshot is corrupt (e.g., silent disk error), the WAL entries covering the previous→new snapshot gap are already trimmed and unavailable. In this case the fallback fails at the WAL-existence check (see step 2a) and the controller aborts with a fatal error — snap_keep_n provides no protection against completed-checkpoint corruption. The fallback chain walks snapshots from newest to oldest, stopping at the first valid one where WAL covers the gap. Snapshot rotation: after writing a new snapshot, delete snapshots beyond `snap_keep_n`, oldest first. ### 2.6 Configuration ```c typedef struct { - char wal_path[XCE_MAXPATH_LEN]; /* Directory for WAL segments */ - char snap_path[XCE_MAXPATH_LEN]; /* Directory for snapshots */ + /* Paths are derived from --workdir and -n name CLI flags at startup. + * Not stored as config fields. Derivation rules (name is mandatory): + * effective_dir = workdir + "/" + name + * wal_path = effective_dir + "/data/wal" + * snap_path = effective_dir + "/data/snap" + * cluster_cfg = effective_dir + "/conf/current/cluster.bin" + * pid_file = effective_dir + "/run/controller.pid" + */ uint32_t wal_sync_ms; /* fsync interval (default 10) */ uint32_t snap_interval_s; /* checkpoint interval (default 60) */ uint32_t snap_wal_size_mb; /* checkpoint when WAL exceeds this (default 64) */ + uint32_t snap_keep_n; /* number of snapshots to retain (default 2 — current + one previous) */ + uint32_t transient_job_timeout_s; /* seconds after failover before JOB_STAT_TRANSIENT jobs are + auto-terminated (default 300 = worker_reconnect_timeout_s). + 0 = no auto-termination (operator must intervene manually). + After timeout: job → JOB_STAT_EXIT with reason + "worker did not reconnect after failover"; worker → + WORKER_UNAVAILABLE. Increments ctrl_transient_jobs_stuck + metric for each auto-terminated job. */ uint32_t finished_retain_s;/* retain finished jobs for N sec (default 3600) */ uint32_t worker_retain_s; /* retain unavailable workers for N sec (default 86400) */ - uint32_t sync_timeout_ms; /* sync replication ACK timeout (default 50-100) */ + uint32_t sync_timeout_ms; /* sync replication ACK timeout (default 50) */ } persist_config_t; ``` +**Working Directory Layout** (unified for dev and production): + +``` +${XCE_WORKDIR}/ e.g. /var/lib/xce (prod) or /tmp/xce (dev) +├── ctrl-1/ effective_dir for -n ctrl-1 +│ ├── conf/ +│ │ ├── current/ +│ │ │ └── cluster.bin controller-specific config (WAL params, HA timeouts) +│ │ └── previous/ +│ │ └── cluster.bin rollback target (atomic rename) +│ ├── data/ +│ │ ├── wal/ mode 0700 +│ │ └── snap/ mode 0700 +│ └── run/ +│ └── controller.pid +│ +├── ctrl-2/ effective_dir for -n ctrl-2 +│ └── ... +│ +└── ctrl-3/ effective_dir for -n ctrl-3 + └── ... +``` + +**Controller startup**: +```bash +xce_controller \ + --workdir /var/lib/xce \ + -n ctrl-1 \ + --peers ctrl-1=10.0.1.1:7000,ctrl-2=10.0.1.2:7000,ctrl-3=10.0.1.3:7000 +``` + +The `-n` flag is **mandatory**. Candidate list is passed via `--peers` CLI flag (no config file for peers). `cluster.bin` contains only controller-specific config (WAL/HA parameters), not the candidate list. + Added to `cluster_config_t` in the Cap'n Proto schema and `myconf.c` YAML parsing. ### 2.7 Controller Connection Context @@ -474,23 +526,7 @@ Since indices are stable (hash table maps key→same index always), a `vput(42, This is required on ALL architectures, not just ARM — compiler reordering is sufficient to cause the race even on x86-64. The kvmgr READ lock in `vget()` callers provides happens-before between `vput()` and `vget()` on the **same vcpu thread** (serialized by lock/unlock), but does not prevent the compiler from reordering stores within a single `vput()` call. The atomic store on `.inuse` is the only reliable barrier against both compiler and CPU reordering. -**Concurrent `vstore` race under READ lock (CRITICAL)**: During COW mode, multiple vcpu threads may hold the kvmgr READ lock simultaneously and call `vstore()`. The kvmgr READ lock does NOT serialize these calls — it allows concurrent readers. Two concurrent `vstore()` calls can both see `inuse==0` at the same `idx` during the scan phase, and the second silently overwrites the first's `.data` — silent data loss. **Fix**: During COW mode, the entire `xzarray_store()` find-slot-and-write sequence is serialized via the inner xzarray mutex (`mylock_lock(&priv->lock)`). The mutex is acquired at the top of the COW-mode path and released on all return paths. This is the ONLY place in the COW protocol where the inner mutex is required — normal-mode vput and vstore rely on the kvmgr WRITE lock for serialization (unchanged). -```c - /* COW mode growth: serialize around the grow-and-assign */ - mylock_lock(&priv->lock); - if (a->data[idx].inuse) { /* Re-check after acquiring lock */ - int old_caps = a->caps; - if (a->caps > SIZE_MAX - a->delta) { mylock_unlock(&priv->lock); return -ENOMEM; } - xzarray_ensure_items(a, a->caps + a->delta); - a->data[old_caps].data = data; - atomic_store_explicit(&a->data[old_caps].inuse, true, memory_order_release); - a->count++; - a->pos = old_caps + 1; - mylock_unlock(&priv->lock); - return old_caps; - } - mylock_unlock(&priv->lock); -``` +**Concurrent `vstore` race under READ lock (CRITICAL)**: During COW mode, multiple vcpu threads may hold the kvmgr READ lock simultaneously and call `vstore()`. The kvmgr READ lock does NOT serialize these calls — it allows concurrent readers. Two concurrent `vstore()` calls can both see `inuse==0` at the same `idx` during the scan phase, and the second silently overwrites the first's `.data` — silent data loss. **Fix**: During COW mode, the entire `xzarray_store()` find-slot-and-write sequence is serialized via the inner xzarray mutex (`mylock_lock(&priv->lock)`). The mutex is acquired at the top of the COW-mode path and released on all return paths. This is the ONLY place in the COW protocol where the inner mutex is required — normal-mode vput and vstore rely on the kvmgr WRITE lock for serialization (unchanged). See the full `xzarray_store()` implementation below (R-22 fix). #### vstore — Must Skip Shadow Indices @@ -537,19 +573,15 @@ static int xzarray_store(meta_obj_t* obj, void* data) { return idx; } } - /* No free slot in [shadow_caps, a->caps) — grow array. - * Already holding the mutex, so no need to re-acquire. */ - int old_caps = a->caps; - if (a->caps > SIZE_MAX - a->delta) { mylock_unlock(&priv->lock); return -ENOMEM; } - xzarray_ensure_items(a, a->caps + a->delta); - /* Use the first new slot after growth. Since we hold the mutex, - * no other thread can have claimed this slot. */ - a->data[old_caps].data = data; - atomic_store_explicit(&a->data[old_caps].inuse, true, memory_order_release); - a->count++; - a->pos = old_caps + 1; + /* No free slot in [shadow_caps, a->caps) — pre-grown range is full. + * Do NOT call xzarray_ensure_items() (realloc) here: realloc moves + * a->data, invalidating pointers held by concurrent vget() callers + * that only hold the kvmgr READ lock (not the inner xzarray mutex). + * The 1.5× pre-grow at freeze time is designed to make this path + * rare (< 100ms COW window at normal submit rates). Return -ENOMEM + * so the caller logs WARN and the client retries after thaw. */ mylock_unlock(&priv->lock); - return old_caps; + return -ENOMEM; } /* Normal path: unchanged circular scan */ @@ -603,8 +635,45 @@ static int xzarray_freeze(meta_obj_t* obj) { priv->shadow = a->data; priv->shadow_caps = a->caps; - /* Allocate new active with same capacity, all inuse=0 */ - a->data = calloc(a->caps, sizeof(xzarray_item_t)); + /* Allocate new active with 1.5× capacity (pre-grown to avoid realloc during COW). + * + * WHY OVER-ALLOCATE: During COW, vstore() scans [shadow_caps, a->caps) for free + * slots. If a->caps == shadow_caps (typical case right after freeze), that range + * is empty and any vstore() would need to call xzarray_ensure_items() → realloc(). + * realloc() moves a->data, invalidating any pointer that a concurrent vget() caller + * holds into the array (vcpu threads can hold READ lock simultaneously, so the + * growth lock — the inner mutex — does not block concurrent vget() access). + * + * By pre-allocating 1.5× shadow_caps slots, the range [shadow_caps, 1.5×shadow_caps) + * is immediately available for vstore(). For a typical COW window (< 100ms), this + * range is almost always sufficient (new entity arrival rate is bounded by the + * job submit rate). If the pre-grown range fills up, vstore() returns -ENOMEM and + * the caller logs a WARN — the submit is rejected transiently and the client retries + * after the COW window closes (< 100ms). This avoids the realloc-during-COW race. + * + * CALLER PRE-ALLOCATION (P3 optimization for large deployments): For clusters + * with >10K entities, this calloc() can allocate 10–25 MB while the kvmgr WRITE + * lock is held, blocking all readers for the duration of the kernel TLB operation. + * To keep the write lock hold time at ~nanoseconds (pointer-swap only), a P3 + * improvement adds `xzarray_freeze_with_preallocated(obj, new_active, cow_caps)` — + * a NON-RTTI direct C function (NOT routed through vfreeze_fn_t, which has the + * single-argument signature `int (*)(meta_obj_t*)` and cannot accept extra params). + * The pre-allocation pattern: + * + * size_t cow_caps = a->caps + a->caps/2 + a->delta; // read caps without lock + * xzarray_item_t* new_active = calloc(cow_caps, sizeof(xzarray_item_t)); + * if (!new_active) return -ENOMEM; + * rwlock_wlock(kvmgr); + * // Direct call — bypasses RTTI dispatch intentionally: + * xzarray_freeze_with_preallocated(obj, new_active, cow_caps); + * rwlock_wunlock(kvmgr); + * + * If a->caps grew between the caps read and the lock acquisition, the function + * falls back to inline calloc() and frees the pre-allocated buffer. + * For P0, the inline calloc() path inside xzarray_freeze() is correct and + * acceptable; the pre-allocation optimization is deferred to P3. */ + size_t cow_caps = priv->shadow_caps + (priv->shadow_caps / 2) + a->delta; + a->data = calloc(cow_caps, sizeof(xzarray_item_t)); if (a->data == NULL) { /* Rollback: restore original data pointer from shadow. * a->count still holds the pre-freeze value (not modified above), @@ -615,13 +684,14 @@ static int xzarray_freeze(meta_obj_t* obj) { priv->shadow_caps = 0; return -ENOMEM; } + a->caps = cow_caps; /* active array is larger than shadow — update caps */ a->count = 0; /* calloc zeroes the allocation; count reflects zero live entries */ a->pos = priv->shadow_caps; /* start past shadow to prevent vstore index collisions */ return 0; } ``` -This is a **pointer swap** — O(1), no data copied. The write lock is held only for the duration of the swap (~nanoseconds). +This is a **pointer swap** — O(1), no data copied. The write lock is held only for the duration of the swap (~nanoseconds). The pre-grown active array (`cow_caps = 1.5 × shadow_caps + delta`) eliminates realloc during the COW window, avoiding the concurrent-vget dangling-pointer race. ### 3.7 Thaw — End COW Phase with Merge and GC @@ -648,9 +718,9 @@ static int xzarray_thaw(meta_obj_t* obj, retain_fn_t should_retain, void* arg) { /* Pass 1: Merge all shadow entries into active. No retention checks. */ for (size_t i = 0; i < priv->shadow_caps; i++) { - if (!priv->shadow[i].inuse) continue; /* empty slot, skip */ + if (!atomic_load_explicit(&priv->shadow[i].inuse, memory_order_relaxed)) continue; /* empty slot, skip */ - if (i < a->caps && a->data[i].inuse) { + if (i < a->caps && atomic_load_explicit(&a->data[i].inuse, memory_order_relaxed)) { /* Active was written during COW — free shadow's stale object */ if (a->killer) a->killer(priv->shadow[i].data); } else { @@ -678,7 +748,7 @@ static int xzarray_thaw(meta_obj_t* obj, retain_fn_t should_retain, void* arg) { * regardless of their original position in the shadow array. */ if (should_retain != NULL) { for (size_t i = 0; i < (size_t)a->caps; i++) { - if (!a->data[i].inuse) continue; + if (!atomic_load_explicit(&a->data[i].inuse, memory_order_relaxed)) continue; if (!should_retain(arg, a->data[i].data)) { if (a->killer) a->killer(a->data[i].data); atomic_store_explicit(&a->data[i].inuse, false, memory_order_relaxed); @@ -738,7 +808,10 @@ persist_checkpoint() fires on timer thread ```c typedef struct { int index; /* shadow slot index */ - const char* key; /* key string (owned by kvtab, valid during freeze) */ + char key[XCE_MAXNAME_LEN]; /* key string — deep-copied during capture (not a borrowed + * pointer into kvtab). kvtab can rehash and move string storage + * after the write lock is released; owned copies remain valid + * throughout the lock-free serialization phase. */ size_t key_len; /* key length (includes null terminator) */ } slot_key_t; ``` @@ -752,8 +825,11 @@ persist_checkpoint() fires on timer thread vfreeze(obj); /* xzarray_freeze — O(1) pointer swap */ slot_key_t* keys = NULL; size_t n_keys = 0; - kvmgr_capture_keys(kvmgr, &keys, &n_keys); /* walks kvtab under write lock */ - /* Now release write lock. Serialization reads keys[] (no locks held). */ + kvmgr_capture_keys(kvmgr, &keys, &n_keys); /* walks kvtab under write lock; + * deep-copies each key string into + * slot_key_t.key via strncpy */ + /* Now release write lock. Serialization reads keys[] (no locks held). + * keys[] holds owned copies — safe against concurrent kvtab rehash. */ ``` This approach avoids threading output parameters through the RTTI interface. @@ -761,7 +837,9 @@ persist_checkpoint() fires on timer thread Key capture is a separate caller-side function that walks the kvtab hash table while the write lock is still held. The serialization phase reads the keys[] array (no locks held) to map shadow slots to their keys in the snapshot. - After serialization completes and thaw runs, the keys[] array is freed. + After serialization completes and thaw runs, each `slot_key_t` in the array + holds an owned copy — no separate per-key free is needed. `free(keys)` releases + the array. │ ▼ After serialization confirmed on disk: @@ -789,7 +867,9 @@ persist_checkpoint() fires on timer thread Delete WAL segments covered by this checkpoint ``` -**Cross-kvmgr freeze ordering and referential consistency**: The checkpoint flow freezes kvmgrs sequentially (jobs → workers → queues → resources → configs), releasing each kvmgr's write lock after its freeze. This means the shadow snapshots for different kvmgrs capture state at slightly different LSN points. A job serialized in the jobmgr snapshot may reference a worker (via `exec_host`) that was added after the workermgr snapshot was taken. This is acceptable because WAL replay after snapshot loading corrects all inconsistencies — all mutations between the earliest and latest freeze points are captured in the WAL entries that follow the snapshot. On recovery, `persist_recover()` loads the snapshot and then replays WAL entries in strict LSN order, which restores full consistency. The checkpoint_lsn recorded per-kvmgr (used for WAL trimming) is the maximum LSN across all kvmgrs at the time of each freeze, ensuring no WAL entry needed for cross-kvmgr consistency is trimmed prematurely. +**Cross-kvmgr freeze ordering and referential consistency**: The checkpoint flow freezes kvmgrs sequentially (jobs → workers → queues → resources → configs), releasing each kvmgr's write lock after its freeze. This means the shadow snapshots for different kvmgrs capture state at slightly different LSN points. A job serialized in the jobmgr snapshot may reference a worker (via `exec_host`) that was added after the workermgr snapshot was taken. This is acceptable because WAL replay after snapshot loading corrects all inconsistencies — all mutations between the earliest and latest freeze points are captured in the WAL entries that follow the snapshot. On recovery, `persist_recover()` loads the snapshot and then replays WAL entries in strict LSN order, which restores full consistency. The checkpoint_lsn recorded in `persist_t` (used for WAL trimming) is set to the **minimum** LSN across all kvmgrs at the time of each freeze. Using the minimum ensures WAL entries between the earliest and latest kvmgr freeze points are preserved — entries covering LSN [min_freeze..max_freeze] are needed to bring kvmgrs frozen at earlier LSNs up to date during recovery replay. Using the maximum would trim precisely those bridging WAL entries and corrupt the cross-kvmgr recovery. + +**Capture order (CRITICAL)**: `checkpoint_lsn` must be set **after all kvmgr freezes complete**, not incrementally during the freeze loop. Record each kvmgr's freeze-time LSN into a local array (`uint64_t freeze_lsns[N_KVMGRS]`), then `checkpoint_lsn = min(freeze_lsns[0..N_KVMGRS-1])` in one assignment after the loop. If `checkpoint_lsn` is updated inside the loop (e.g., `checkpoint_lsn = min(checkpoint_lsn, current_lsn)` at each step), the intermediate values are visible to WAL-trim decisions before all freezes complete — a racing checkpoint-triggered trim could delete WAL entries that a later freeze still needs. ``` **Non-kvmgr zarrays**: `cluster_t` also contains standalone `zarray_t` fields (endpoints, configs, sharedinst) that are not managed by kvmgr. These are handled differently during checkpoint: @@ -797,7 +877,7 @@ persist_checkpoint() fires on timer thread - `c->configs`: Serialized under a separate write lock (`lock(c->configmgr, RTTI_LOCK_READ)` for reads, `RTTI_LOCK_WRITE` for freeze/thaw). Configs are read-only during normal operation and rare writes (admin config changes). Config freezes are serialized as a separate Cap'n Proto array in the snapshot. - `c->sharedinst`: Not serialized. Shared instances are reconstructed from config on recovery. -**Shadow lifetime and shutdown safety**: `xzarray_thaw()` calls `free(a->shadow)` and sets `a->shadow = NULL`. The serialization thread reads shadow without any lock (shadow is "immutable after freeze"). This is safe **only if thaw never runs while the serialization thread is still reading**. The normal checkpoint flow guarantees this: thaw is called only after serialization confirms the snapshot on disk, and both happen on the same checkpoint timer thread. However, the **shutdown path** must also observe this constraint: if `cluster_fini()` or a shutdown signal fires while a checkpoint is in progress, it must wait for the serialization thread to complete before calling `xzarray_thaw()`. Implement with a `persist_t.checkpoint_in_progress` atomic flag **and** a `pthread_cond_t checkpoint_done` condition variable. **The condition variable must use a dedicated `pthread_mutex_t checkpoint_mutex` — NOT `persist.lock` (which is a `mylock_t`).** `pthread_cond_timedwait()` requires a `pthread_mutex_t*`, and `mylock_t` may not be a `pthread_mutex_t` (it could be a spinlock or custom wrapper, which would cause undefined behavior with condition variables). Additionally, `persist.lock` is held by vcpu threads during `persist_wal_sync_append()` (which may block on `repl_wait_ack()`), so `cluster_fini()` attempting to acquire `persist.lock` for the timed wait would deadlock against a vcpu thread holding it. The dedicated `checkpoint_mutex` avoids both issues: it is always a `pthread_mutex_t` and is never held during replication waits. On shutdown: `cluster_fini()` acquires `checkpoint_mutex`, checks `checkpoint_in_progress`. If set, it calls `pthread_cond_timedwait(&persist.checkpoint_done, &checkpoint_mutex, &deadline)` where `deadline` is `now + 5 seconds` (configurable). The checkpoint thread signals `checkpoint_done` after completing serialization and thaw (or after canceling). If the deadline expires, `cluster_fini()` cancels the in-progress checkpoint (sets a cancel flag) and skips thaw — incomplete checkpoint state is cleaned by the next restart's `persist_recover()`. This avoids busy-waiting on the atomic flag and provides bounded shutdown latency. **IMPORTANT**: after the 5-second deadline expires and the cancel flag is set, `cluster_fini()` must `pthread_join()` the checkpoint/serialization thread before proceeding to free any cluster state. The serialization thread reads shadow arrays and `slot_key_t` arrays that hold pointers to kvtab keys — if `cluster_fini()` proceeds to free the kvmgr and its contents before the serialization thread exits, a use-after-free occurs. The join ensures the serialization thread observes the cancel flag, exits its loop, and returns before any cluster state is destroyed. +**Shadow lifetime and shutdown safety**: `xzarray_thaw()` calls `free(a->shadow)` and sets `a->shadow = NULL`. The serialization thread reads shadow without any lock (shadow is "immutable after freeze"). This is safe **only if thaw never runs while the serialization thread is still reading**. The normal checkpoint flow guarantees this: `persist_checkpoint_timer_fn()` (running on the delaymgr timer thread) does the freeze and key-capture under the write lock, then **spawns a dedicated serialization pthread** for the slow I/O (Cap'n Proto serialization + fsync). Thaw is called by that same serialization pthread after the snapshot is confirmed on disk. The delaymgr timer thread returns immediately after spawning, keeping the delaymgr responsive. The serialization pthread ID is stored in `persist_t.snap_thread` so that `cluster_fini()` can join it. `pthread_cond_timedwait(&persist.checkpoint_done, ...)` in `cluster_fini()` is the normal-path wait; `pthread_join(persist.snap_thread, NULL)` is the deadline-exceeded path that ensures no use-after-free when the checkpoint is cancelled but the serialization thread is still running. **The distinction matters**: "same checkpoint timer thread" is intentionally avoided — the timer thread is shared across all delaymgr tasks and must not block on I/O; only the dedicated serialization pthread can block. However, the **shutdown path** must also observe this constraint: if `cluster_fini()` or a shutdown signal fires while a checkpoint is in progress, it must wait for the serialization thread to complete before calling `xzarray_thaw()`. Implement with a `persist_t.checkpoint_in_progress` atomic flag **and** a `pthread_cond_t checkpoint_done` condition variable. **The condition variable must use a dedicated `pthread_mutex_t checkpoint_mutex` — NOT `persist.lock` (which is a `mylock_t`).** `pthread_cond_timedwait()` requires a `pthread_mutex_t*`. `mylock_t` is typedef `pthread_mutex_t` (confirmed by `mylock.h`), so the type concern does not apply. The real reason for using a dedicated `checkpoint_mutex` is deadlock avoidance: `persist.lock` is held by vcpu threads during `persist_wal_sync_append()` (which may block on `repl_wait_ack()`), so `cluster_fini()` attempting to acquire `persist.lock` for the timed wait would deadlock against a vcpu thread holding it. The dedicated `checkpoint_mutex` avoids this: it is never held during replication waits. On shutdown: `cluster_fini()` acquires `checkpoint_mutex`, checks `checkpoint_in_progress`. If set, it calls `pthread_cond_timedwait(&persist.checkpoint_done, &checkpoint_mutex, &deadline)` where `deadline` is `now + 5 seconds` (configurable). The checkpoint thread signals `checkpoint_done` after completing serialization and thaw (or after canceling). If the deadline expires, `cluster_fini()` cancels the in-progress checkpoint (sets a cancel flag) and skips thaw — incomplete checkpoint state is cleaned by the next restart's `persist_recover()`. This avoids busy-waiting on the atomic flag and provides bounded shutdown latency. **IMPORTANT**: after the 5-second deadline expires and the cancel flag is set, `cluster_fini()` must `pthread_join()` the checkpoint/serialization thread before proceeding to free any cluster state. The serialization thread reads shadow arrays and `slot_key_t` arrays that hold pointers to kvtab keys — if `cluster_fini()` proceeds to free the kvmgr and its contents before the serialization thread exits, a use-after-free occurs. The join ensures the serialization thread observes the cancel flag, exits its loop, and returns before any cluster state is destroyed. **Crash safety during checkpoint**: If the controller crashes mid-checkpoint (during shadow serialization or mid-thaw), the previous snapshot on disk remains valid. On recovery, `persist_recover()` loads the last valid snapshot and replays WAL entries from that snapshot's LSN forward — the incomplete checkpoint is simply ignored. WAL segments are only deleted after the new snapshot is fully written and `fsync`'d to disk. This ensures atomicity: a checkpoint either completes fully (new snapshot + WAL trim) or is treated as if it never happened. @@ -817,7 +897,10 @@ kvmgr rwlock (outer) — protects kvtab + serializes freeze/thaw ``` - **kvmgr `my_rwlock_t`**: The outer lock. Taken as read-lock for normal lookups (`xkvmgr_find`). Taken as write-lock for additions (`xkvmgr_add`) and for COW freeze/thaw coordination. -- **xzarray `mylock_t`**: The inner lock. **Not taken automatically inside `xzarray_put_vec()`** — the current `src/lib/zarray.c` implementation of `xzarray_put_vec()` does not call `mylock_lock()` internally; the caller is responsible. P0-2 must verify the actual mutex ownership contract and document it explicitly. During COW, `vput` calls happen under the kvmgr read lock, which serializes them against freeze/thaw — if this is the intended protocol, the inner mutex may be redundant for COW operations. If `xzarray_put_vec()` is expected to be called without the kvmgr lock, it must acquire the inner mutex itself. +- **xzarray `mylock_t`**: The inner lock. Two modes: + - **Normal mode**: NOT taken inside `xzarray_put_vec()` — callers hold the kvmgr WRITE lock for mutations, which provides serialization; the inner mutex adds no value and is skipped. + - **COW mode**: Taken inside `xzarray_store()` for the entire find-slot-and-write sequence (R-22 fix). During COW, multiple vcpu threads hold the kvmgr READ lock concurrently, so the outer lock does not serialize concurrent `vstore()` calls. The inner mutex is the only serialization point preventing two threads from claiming the same free slot. `vput()` in COW mode is still NOT protected by the inner mutex — `vput()` writes to a known index (never allocates a new slot), so there is no find-slot race. + - **Thaw mode**: NOT taken during thaw — the kvmgr WRITE lock is held for the entire thaw+GC pass, which serializes all access. **COW lock protocol:** @@ -859,27 +942,37 @@ This matches the existing pattern in `handle_jobsubmit()` (`job.c:455-525`): `lo ### 3.10 Checkpoint Timer -The checkpoint thread should use the existing `delaymgr` timer system (already used for heartbeat checks in `cluster.c`) rather than a raw `pthread_create`. This ensures the checkpoint timer integrates with the controller's timer infrastructure: +The checkpoint thread uses the existing `delaymgr` timer system. **`delaymgr` will be enhanced to change `sleep(precision)` to `nanosleep(precision_ms)` in `delay.c`**, giving sub-second granularity. **The `delay_task()` delay parameter remains in seconds throughout** — only the underlying sleep precision changes. The `delay.c` `ts_nextrun = now + delay` arithmetic is unchanged (seconds-based). All existing callers continue to pass seconds; the precision reduction only affects how tightly the timer fires relative to its scheduled time. ```c -/* In persist_start() — register periodic checkpoint timer */ -/* delay_task signature: delay_task(delaymgr_t*, int seconds, int repeat, fn, void*) - * repeat=-1 means recurring (matches cluster.c:756 heartbeat pattern) */ +/* In delay.c — change sleep(precision) to nanosleep with precision_ms granularity. + * The ts_nextrun / run_period fields remain in seconds (time_t). No caller changes + * are required. Default precision_ms for the cluster delaymgr: 200ms, set via: + * delaymgr_init(&c->delaymgr, 200 /* precision_ms */, run_delay_tasks) + * + * Existing callers at cluster.c:756 (heartbeat) continue passing seconds. */ + +/* In persist_start() — register periodic checkpoint timer (1-second poll) */ int ckpt_timer = delay_task(&(c->delaymgr), - (int)config->snap_interval_s, + 1, /* 1-second poll interval (seconds) */ -1, /* recurring */ (delay_task_entry_fn_t)persist_checkpoint_timer_fn, (void*)c); +/* The periodic checkpoint (snap_interval_s, e.g. 60s) is counted inside + * persist_checkpoint_timer_fn() via a tick counter, not as a separate delay_task. */ ``` -**Checkpoint overlap handling**: If a timer-driven checkpoint fires while a previous checkpoint is still in progress (checked via `persist_t.checkpoint_in_progress` atomic flag), the timer callback logs an `INFO` message ("checkpoint already in progress, skipping") and returns — the next timer interval will trigger a new check. WAL trimming only occurs after a successful checkpoint, so skipped timers result in continued WAL growth until the next successful checkpoint completes. The size-based inline trigger (Part 3.11, via `checkpoint_requested` atomic flag) provides a safety valve for rapid WAL growth between timer intervals. +**Checkpoint overlap handling**: If a timer-driven checkpoint fires while a previous checkpoint is still in progress (checked via `persist_t.checkpoint_in_progress` atomic flag), the timer callback logs an `INFO` message ("checkpoint already in progress, skipping") and returns. The size-based inline trigger (Part 3.11) provides a safety valve for rapid WAL growth. -**Checkpoint timer for inline-triggered checkpoints**: The checkpoint timer thread also checks `persist_t.checkpoint_requested` (set by the inline trigger in Part 3.11). When this flag is observed, the timer thread runs the checkpoint immediately regardless of the regular interval. Use a secondary `delay_task` with a 1-2 second interval to poll `checkpoint_requested`, or combine the check into the main timer callback (if the delaymgr supports sub-interval wakeups via flag check). +**Checkpoint timer for inline-triggered checkpoints**: The timer callback checks `persist_t.checkpoint_requested` on each 1-second tick, responding within ~1 second of the flag being set. Benefits: - No raw `pthread_create`/`pthread_join` management for the timer thread - `delaymgr` already handles timer cancellation in `cluster_fini()` -- Consistent with how heartbeat timers are managed +- Consistent with how all periodic tasks are managed +- No semantic change to delay_task() callers — all pass seconds as before + +**P0-3 implementation note**: The delaymgr `nanosleep()` change (`delay.c`) is a prerequisite for P0-3. The change is: rename `precision` field to `precision_ms` and replace `sleep(precision)` with `struct timespec ts = {0, precision_ms * 1000000L}; nanosleep(&ts, NULL);`. Update `delaymgr_init()` call in `cluster.c:604` from `precision=2` to `precision_ms=200`. Audit all `delaymgr_init()` callers to confirm they all pass precision_ms in milliseconds. ### 3.11 Snapshot File Format @@ -891,8 +984,10 @@ Serialized from shadow arrays while no locks are held: │ - lsn: last WAL LSN │ │ - timestamp │ │ - term │ +│ - version: UInt32 (=1) │ │ - nJobs, nQueues, nWorkers │ │ - checksum: CRC32(body) │ +│ - headerChecksum: CRC32(header fields) │ ├─────────────────────────┤ │ Job array (Capnp) │ ├─────────────────────────┤ @@ -906,7 +1001,7 @@ Serialized from shadow arrays while no locks are held: └─────────────────────────┘ ``` -**Snapshot CRC scope**: The `SnapshotHeader.checksum` (CRC32) covers the Cap'n Proto–serialized body (Job array, Queue array, Worker array, Resource array, Config array) **excluding** the `SnapshotHeader` struct itself. The header is verified separately: the reader validates `nJobs`, `nQueues`, `nWorkers` against the actual array lengths to detect header corruption. This split design avoids the chicken-and-egg problem where the CRC needed for body verification is itself part of the data being verified. +**Snapshot CRC scope**: The `SnapshotHeader.checksum` (CRC32) covers the Cap'n Proto–serialized body (Job array, Queue array, Worker array, Resource array, Config array) **excluding** the `SnapshotHeader` struct itself. The header is verified separately via `SnapshotHeader.headerChecksum`. After `headerChecksum` passes, `nJobs`, `nQueues`, `nWorkers` are trusted as authoritative counts — a secondary validation against actual deserialized array lengths is a **debug assertion only** (`assert(nJobs_from_header == actual_cap'nproto_list_count)`), not a required safety check. Treating the counts as untrusted after a passed headerChecksum would create a contradictory two-truth-source design. This split design avoids the chicken-and-egg problem where the CRC needed for body verification is itself part of the data being verified. - Time-based: every `snap_interval_s` (default 60s) - Size-based: when WAL exceeds `snap_wal_size_mb` (default 64 MB) @@ -940,15 +1035,13 @@ This eliminates `vclear` from the hot path. The `del_key` callback in kvmgr is s **Design invariant**: `del_key` is deliberately NULL. The deferred GC at checkpoint handles entity removal via `kvtab_gc()`, not through hot-path key deletion. The RTTI interface's `del_key = NULL` is correct — no change is needed. This simplifies COW (no tombstones, no three-state flags — see Part 3) - - ### 4.2 Terminal States by Entity Type | Entity | Terminal States | Trigger | Cleanup Criteria | |--------|----------------|---------|-------------------| | Job | `JOB_STAT_DONE`, `JOB_STAT_EXIT`, `JOB_STAT_PERR` | Job finishes naturally | No dependents AND `finish_time + finished_retain_s < now` | -| Job | `JOB_STAT_TRANSIENT` (0x400) | Primary failover — job was RUNNING on old primary but no state update in standby's WAL | Worker reconnection (sets actual state) or operator intervention if worker never reconnects | +| Job | `JOB_STAT_TRANSIENT` (0x400) | Primary failover — job was RUNNING on old primary but no state update in standby's WAL | Worker reconnection (sets actual state) OR `transient_since + transient_job_timeout_s < now` — auto-terminate with `JOB_STAT_EXIT` and reason "worker did not reconnect after failover" (see config `transient_job_timeout_s`, default 300s) | | Worker | `WORKER_UNAVAILABLE` | Worker daemon dies / disconnects | `dead_since + worker_retain_s < now` | | Worker | `WORKER_HIDDEN` | Admin explicit remove (off-hours maintenance only) | Immediate cleanup at next checkpoint (no TTL) | | Queue | `QUEUE_HIDDEN` | Admin deletes queue (off-hours maintenance only) | Immediate cleanup at next checkpoint | @@ -1036,19 +1129,69 @@ bool job_has_dependents(cluster_t* c, uint64_t jobid); ``` ```c -/* Job retention: keep if it has dependents OR TTL not expired */ +/* Job retention: keep if it has dependents OR TTL not expired. + * Also handles JOB_STAT_TRANSIENT auto-termination via transient_job_timeout_s. */ static bool job_should_retain(void* arg, void* item) { cluster_t* c = (cluster_t*)arg; job_t* j = (job_t*)item; + /* Auto-terminate transient jobs whose timeout has expired. + * JOB_STAT_TRANSIENT is NOT terminal (job_is_terminal() would return false), + * so we handle it before the terminal check. transient_job_timeout_s=0 disables + * auto-termination (operator must intervene manually). j->transient_since is set + * when the job enters JOB_STAT_TRANSIENT at promotion time. */ + if ((j->stat & JOB_STAT_TRANSIENT) && + c->persist.config.transient_job_timeout_s > 0) { + time_t now = time(NULL); + if (j->transient_since > 0 && + now > j->transient_since + (time_t)c->persist.config.transient_job_timeout_s) { + /* Convert to terminal state so it falls through to normal GC below. + * LOCK ORDERING NOTE: Do NOT write a WAL entry here. + * job_should_retain() is called from xzarray_thaw() which is called + * from persist_checkpoint() while holding the kvmgr WRITE lock. + * persist_wal_async() requires persist.lock — acquiring persist.lock + * while holding the kvmgr WRITE lock would introduce an undocumented + * lock ordering dependency that can deadlock with vcpu threads that + * acquire persist.lock first then attempt xkvmgr_find() (READ lock). + * Instead: record the auto-terminated jobid in a local heap array + * declared in persist_checkpoint(): + * uint64_t* auto_term = calloc(MAX_AUTO_TERM_PER_CHECKPOINT, sizeof(uint64_t)); + * int n_auto_term = 0; + * (MAX_AUTO_TERM_PER_CHECKPOINT defined in config.h, default 10240.) + * This is a local variable in persist_checkpoint(), NOT a field of persist_t. + * After thaw completes and the kvmgr WRITE lock is released, walk the array + * and call persist_wal_async() for each jobid. On persist_wal_async() failure, + * log ERROR("auto-term WAL write failed for jobid %lu: %s", jobid, strerror(e)) + * and continue. The job reverts to JOB_STAT_TRANSIENT after recovery, but this + * does NOT loop forever: the recovery retention pass (Part 2.5 step 5) treats + * any JOB_STAT_TRANSIENT job older than transient_job_timeout_s as immediately + * auto-terminatable. If the WAL write keeps failing across restarts, the job is + * cleaned up on the NEXT successful checkpoint after recovery — bounded by one + * checkpoint interval (snap_interval_s), not across unbounded restart cycles. + * Increment ctrl_transient_jobs_stuck for each failed WAL write so monitoring + * can detect persistent I/O problems that prevent clean GC. + * If n_auto_term >= MAX_AUTO_TERM_PER_CHECKPOINT, stop recording (remaining + * jobs survive this checkpoint and are auto-terminated in the next one — bounded + * delay is acceptable). free(auto_term) after WAL writes complete. */ + j->stat = JOB_STAT_EXIT; + j->finish_time = now; + WARN("transient job %lu on worker %s auto-terminated after %us: " + "worker did not reconnect after failover", + j->jobid, j->exec_host, c->persist.config.transient_job_timeout_s); + /* ctrl_transient_jobs_stuck metric incremented by caller after lock release */ + } else { + return true; /* transient timeout not yet expired */ + } + } + if (!job_is_terminal(j)) return true; /* still active, keep */ if (job_has_dependents(c, j->jobid)) return true; /* other jobs depend on it */ - time_t now = time(NULL); - if (j->finish_time + c->persist.config.finished_retain_s > now) + time_t now2 = time(NULL); + if (j->finish_time + c->persist.config.finished_retain_s > now2) return true; /* TTL not expired */ return false; /* safe to clean up */ @@ -1201,11 +1344,12 @@ ha: candidates: [...] ``` -`cluster_init()` must refuse to start HA mode if `split_brain_risk_accepted != true`, logging: `"HA split-brain risk has not been acknowledged — set ha.split_brain_risk_accepted: true in cluster.yaml after reading the HA deployment requirements"`. +`cluster_init()` must refuse to start HA mode if `split_brain_risk_accepted != true`, logging: `"HA split-brain risk has not been acknowledged — set ha.split_brain_risk_accepted: true in cluster.bin (via config tool from cluster.yaml) after reading the HA deployment requirements"`. **Operator visibility for degraded mode**: When `sync_timeout_ms` expires and the primary falls back to local-only ACK, this is NOT silent: 1. Log at `ERROR` level (not `WARN`) — degraded replication is a durability-affecting event 2. Increment `ctrl_repl_degraded_count` metric (alongside `ctrl_split_brain_detected`) +3. **Early split-brain signal (state-change log, not per-submit)**: When the primary first enters the combined state of (degraded replication AND all heartbeat sends failing), log **once** at `ERROR`: `"Primary entering isolated mode — cannot reach any standby for sync ACK or heartbeat. Split-brain risk: if a standby has promoted itself, submitted jobs may be lost. ctrl_isolated_primary_count=%u"`. Increment `ctrl_isolated_primary_count` on state entry. When the primary reconnects to any standby, log **once** at `INFO`: `"Primary exiting isolated mode after %us — standby reconnected"`. Do **not** emit this log on every job submit — sustained degradation at 10 submits/sec produces 600 identical ERROR entries per minute, flooding logs and suppressing the signal. Track isolation state with a `_Atomic int ha_isolated;` flag in `ha_state_t`; set on entry, clear on reconnect. Also add `primary_isolated_since` (`struct timespec`, CLOCK_MONOTONIC) to `ha_state_t` to report isolation duration on reconnect. 3. The degraded state persists until the standby reconnects — subsequent submits in degraded mode each log at `ERROR` and increment the counter **Split-brain detection condition**: `ctrl_split_brain_detected` is incremented when a controller receives a message (heartbeat, replication, election probe, or client request) from a peer that believes a DIFFERENT controller is primary. Specifically: @@ -1240,8 +1384,7 @@ controller: - At least one candidate is defined - No duplicate host:port pairs - `ha.split_brain_risk_accepted: true` is set when HA mode is enabled -- `ha_cert_path`, `ha_key_path`, `ha_ca_cert_path` files exist and have correct permissions -- `ha_ca_cert_path` points to a valid PEM-encoded CA certificate +- `ha.split_brain_risk_accepted` is `true` when more than one candidate is configured The controller reads only the binary Cap'n Proto config file — it has no YAML parsing dependency, no text config parsing, and treats its config as a trusted, validated blob. This eliminates config injection risks and simplifies the controller's dependency tree. @@ -1275,7 +1418,10 @@ typedef struct { char primary_host[XCE_MAXNAME_LEN]; uint32_t primary_port; struct timespec last_heartbeat; /* Last heartbeat from primary (CLOCK_MONOTONIC, standby) */ - int promotion_failed; /* Non-zero if last promotion attempt failed — retry on next watchdog tick */ + _Atomic int promotion_failed; /* Non-zero if last promotion attempt failed — retry on next watchdog tick. + * Must be _Atomic: written by ha_promote_to_primary() on the watchdog + * thread, read by ha_check_heartbeat() on the same thread but also + * visible to vcpu threads checking promotion status. Relaxed ordering. */ _Atomic int promotion_retry_count; /* Number of consecutive failed promotion attempts. * Must be _Atomic because it is incremented by the * watchdog thread and read by vcpu threads during @@ -1287,24 +1433,47 @@ typedef struct { * OP_replJoin receipt and OP_replCatchUpDone) and * read by vcpu threads during ACK suppression * (Part 6.4). Use atomic_store/load_explicit with - * release/acquire ordering. Set to 1 when the standby - * sends OP_replJoin with applied_lsn < primary's - * committed_lsn; cleared when applied_lsn reaches - * the primary's committed_lsn (signaled via - * OP_replCatchUpDone). Gates ACK suppression: during - * catch-up, the standby does NOT send OP_replAck - * for historical entries (see Part 6.4 line 1778). */ + * release/acquire ordering. + * + * LIFECYCLE (standby-side, this field): + * Set to 1 by the standby when it sends OP_replJoin + * with applied_lsn < primary's committed_lsn. + * Cleared to 0 by the standby when it sends + * OP_replCatchUpDone (after applying snapshot + all + * bridging WAL entries up to committed_lsn). + * The standby initiates completion — it decides when + * its applied_lsn has reached committed_lsn. + * + * PRIMARY-SIDE MIRROR: standby_conn_t.in_catch_up + * (Appendix A.3) is cleared by the primary when it + * receives OP_replCatchUpDone from this standby. + * See Part 6.6 and P2-5 for the full protocol. + * + * Gates ACK suppression: during catch-up, the standby + * does NOT send OP_replAck for historical entries + * (see Part 6.4). */ ha_config_t config; /* HA configuration (timeouts, intervals) */ + pthread_mutex_t lock; /* Protects primary_ep and last_heartbeat against concurrent + * iomgr NULL-then-free and watchdog reads. mylock_t is + * typedef pthread_mutex_t (confirmed by mylock.h), but an + * explicit pthread_mutex_t field documents the pthread_cond + * usage intent and avoids any future mylock_t divergence. + * Initialized in ha_init(). */ } ha_state_t; typedef struct { - uint32_t heartbeat_interval_ms; /* Primary heartbeat interval (default: 3000). Minimum 2000ms — delay_task() takes integer seconds, so values below 2000ms truncate to 0 or 1 second. Enforce heartbeat_interval_ms >= 2000 in ha_init() to avoid delay_task() truncation. */ + uint32_t heartbeat_interval_ms; /* Primary heartbeat interval (default: 3000). Minimum 2000ms — the HA watchdog pthread (P1-3) uses nanosleep() so there is no delaymgr truncation concern. The 2000ms floor is a practical recommendation: values below ~1000ms risk false timeout triggers on loaded management LANs (SSL handshake + scheduling jitter). Enforce heartbeat_interval_ms >= 2000 in ha_init() as a conservative safe floor; operators needing faster heartbeats may lower this with care. */ uint32_t heartbeat_timeout_ms; /* Standby heartbeat timeout (default: 10000) */ uint32_t election_timeout_ms; /* Shared deadline for all parallel election probes (default: 2000) */ - char ha_cert_path[XCE_MAXPATH_LEN]; /* Controller mTLS certificate (PEM) */ - char ha_key_path[XCE_MAXPATH_LEN]; /* Controller mTLS private key (PEM, mode 0600) */ - char ha_ca_cert_path[XCE_MAXPATH_LEN]; /* CA certificate for peer verification (PEM) */ - uint32_t ha_cert_warn_days; /* Warn if cert expires within this many days (default: 30) */ + uint32_t probe_connect_timeout_ms; /* Per-probe TLS connect timeout for am_i_primary() probe threads + * (default: election_timeout_ms / MAX_HA_CANDIDATES, minimum 500ms). + * Each probe thread uses this as the poll() timeout in its + * non-blocking connect loop, so it wakes up at least once per + * probe_connect_timeout_ms to check the shared election_done flag. + * This bounds the worst-case overshoot of election_timeout_ms + * to one probe_connect_timeout_ms interval per thread. + * ha_init() enforces: probe_connect_timeout_ms >= 500ms AND + * probe_connect_timeout_ms <= election_timeout_ms. */ uint32_t max_promotion_retries; /* Max consecutive failed promotion attempts before giving up (default: 5) */ } ha_config_t; ``` @@ -1347,11 +1516,11 @@ Controller starts ▼ ▼ cluster_init() cluster_init() persist_init() persist_init() - persist_recover() persist_recover() - persist_start() persist_start() - connect to primary - cluster_start() (via iomgr, now running) - ha_heartbeat_watch() + persist_recover() persist_recover() ← iomgr NOT yet running + persist_start() persist_start() workers get ECONNREFUSED, + connect to primary retry via xce_reconnect() + cluster_start() (via iomgr, now running) ← iomgr starts here + ha_heartbeat_watch() workers can now connect standby_loop() ``` @@ -1368,9 +1537,10 @@ Controller starts > > After the election resolves, `cluster_init()` starts iomgr and all HA traffic uses the iomgr/SSL/endpoint pipeline. This avoids a circular dependency between election and iomgr startup, and parallel probing bounds worst-case startup latency to one `election_timeout_ms` regardless of cluster size. > -> **Peer certificate validation in election probe**: The synchronous TLS dial in `am_i_primary()` must verify the peer's certificate CN and OU against the expected values from `cluster.yaml`. Specifically: the peer's cert must have `CN=ctrl-` and `OU=ha` (where `candidate_name` matches the entry in the candidates list). A probe that connects to an unexpected host (wrong CN or OU) must be rejected as if no response was received — this prevents an attacker from impersonating a lower-priority candidate to force the probing controller into standby mode. > -> **`ha_init()` startup enforcement**: At startup, `ha_init()` validates: `heartbeat_interval_ms >= 2000` (values below 2000ms truncate to 0 or 1 second when passed to `delay_task()`, which takes integer seconds — this makes the heartbeat interval too short or disables it entirely); `heartbeat_timeout_ms >= heartbeat_interval_ms`; `election_timeout_ms >= 500`; `ncandidates >= 1`. It also verifies `ha_key_path` permissions (must be mode 0600) and logs a `WARN` if the HA certificate expires within `ha_cert_warn_days` (default 30). **Duplicate candidate validation**: `ha_init()` scans `candidates[]` and rejects any config where two entries share the same `host` (case-insensitive `strcasecmp`) or the same `host:port` pair. Duplicate candidate names cause ambiguous election probes and mTLS CN matching — an entry appearing twice would receive two heartbeat monitors, trigger double replication connections, and confuse the priority ordering. The validation must log a `FATAL` error naming both duplicate entries and return `-EINVAL`. See `ha_config_t` in Part 5.3 for field documentation. +> **`ha_init()` mutex initialization ordering**: `ha_init()` must be called **before** `am_i_primary()` (the election probe), since `ha_state_t.lock` (the `pthread_mutex_t` protecting `primary_ep` and `last_heartbeat`) is initialized in `ha_init()`. Step 5.5 starts heartbeats before `cluster_start()` (via a dedicated pthread) — this pthread accesses `ha->lock`, which must already be initialized. Promotion (`ha_promote_to_primary()`) does NOT re-initialize the mutex — the mutex is initialized once in `ha_init()` and persists for the controller's lifetime across promote→demote→re-promote cycles. Re-initializing a pthread_mutex_t that may have waiters is undefined behavior. +> +> **`ha_init()` startup enforcement**: At startup, `ha_init()` validates: `heartbeat_interval_ms >= 2000` (the HA watchdog uses `nanosleep()` and is not subject to `delay_task()` integer-second truncation; the 2000ms floor is a conservative practical minimum to avoid false heartbeat-timeout triggers from SSL handshake jitter on loaded management LANs); `heartbeat_timeout_ms >= heartbeat_interval_ms`; `election_timeout_ms >= 500`; `ncandidates >= 1`. **Duplicate candidate validation**: `ha_init()` scans `candidates[]` and rejects any config where two entries share the same `host` (case-insensitive `strcasecmp`) or the same `host:port` pair. Duplicate candidate names cause ambiguous election probes — an entry appearing twice would receive two heartbeat monitors, trigger double replication connections, and confuse the priority ordering. The validation must log a `FATAL` error naming both duplicate entries and return `-EINVAL`. See `ha_config_t` in Part 5.3 for field documentation. ### 5.5 Monitor Thread (Standby) @@ -1381,7 +1551,7 @@ The standby uses the controller's iomgr/vtask heartbeat infrastructure to monito Instead, it uses the existing iomgr/vtask/endpoint pipeline: 1. The primary sends OP_heartbeat to all connected standbys - at HA_HEARTBEAT_INTERVAL_MS (default 2000-5000ms). + at HA_HEARTBEAT_INTERVAL_MS (default 3000ms). 2. Each standby tracks last_heartbeat_time for the primary endpoint. 3. If last_heartbeat_time + HA_HEARTBEAT_TIMEOUT_MS < now, the primary is considered dead -> promote self. @@ -1390,33 +1560,42 @@ The standby uses the controller's iomgr/vtask heartbeat infrastructure to monito and reuses the SSL/endpoint/vtask pipeline that all controller connections already use. */ -/* ha_check_heartbeat() called from delaymgr timer on standby. - * Note: delay_task() takes integer seconds. Register with interval: - * delay_task(&c->delaymgr, ha->config.heartbeat_interval_ms / 1000, -1, ha_check_heartbeat, c); - * heartbeat_interval_ms is in milliseconds; divide by 1000 for delay_task(). - * WARNING: Values below 2000ms truncate to 0 or 1 second via integer division. - * ha_init() enforces heartbeat_interval_ms >= 2000 to prevent truncation. +/* ha_check_heartbeat() — called from the dedicated HA watchdog pthread (NOT delaymgr). + * P1-3 requires a dedicated pthread with nanosleep() or timerfd_create(CLOCK_MONOTONIC) + * for sub-second heartbeat detection. delaymgr has 2-second tick granularity + * (cluster.c initialises it with precision=2), making it unsuitable here. * Uses CLOCK_MONOTONIC — immune to NTP jumps and DST changes that would cause * spurious promotions (clock jumped forward) or missed timeouts (clock jumped back). */ static void ha_check_heartbeat(cluster_t* c) { ha_state_t* ha = &c->ha; - if ((ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire) != CTRL_STANDBY || ha->primary_ep == NULL) + if ((ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire) != CTRL_STANDBY) + return; + + /* Read primary_ep and last_heartbeat under ha->lock — iomgr clears primary_ep + * (NULL-before-free) while holding this same lock, so we see either a valid + * pointer or NULL, never a dangling one. */ + pthread_mutex_lock(&ha->lock); + int no_primary = (ha->primary_ep == NULL); + struct timespec last_hb = ha->last_heartbeat; /* snapshot under lock */ + pthread_mutex_unlock(&ha->lock); + + if (no_primary) return; struct timespec now; clock_gettime(CLOCK_MONOTONIC, &now); - int64_t elapsed_ms = (now.tv_sec - ha->last_heartbeat.tv_sec) * 1000LL - + (now.tv_nsec - ha->last_heartbeat.tv_nsec) / 1000000LL; + int64_t elapsed_ms = (now.tv_sec - last_hb.tv_sec) * 1000LL + + (now.tv_nsec - last_hb.tv_nsec) / 1000000LL; if (elapsed_ms > (int64_t)ha->config.heartbeat_timeout_ms) { /* Primary heartbeat timeout — promote self. * If a previous promotion attempt failed (promotion_failed flag set), * retry promotion unless the retry limit is exceeded. */ if (ha->promotion_failed && - ha->promotion_retry_count >= ha->config.max_promotion_retries) { + atomic_load_explicit(&ha->promotion_retry_count, memory_order_relaxed) >= (int)ha->config.max_promotion_retries) { ERROR("cannot promote after %d attempts — manual intervention required", - ha->promotion_retry_count); + atomic_load_explicit(&ha->promotion_retry_count, memory_order_relaxed)); return; } ha_promote_to_primary(c); @@ -1424,7 +1603,7 @@ static void ha_check_heartbeat(cluster_t* c) } ``` -**Heartbeat detection latency**: The standby watchdog timer uses `delay_task()` with integer-second granularity (`precision=2` seconds in cluster.c). This means the actual primary-failure detection latency is `heartbeat_timeout_ms + up to (heartbeat_interval_ms / 1000) * 1000` milliseconds — the timeout fires, plus up to one timer tick of jitter before the check runs. For the default `heartbeat_timeout_ms = 10000` and `heartbeat_interval_ms / 1000 = 3` seconds, worst-case detection is 13 seconds. For HPC workloads requiring sub-10-second failover, a dedicated timer thread with millisecond granularity (as used for the WAL fsync timer) should be used for the heartbeat watchdog instead of `delaymgr`. The P1-3 implementation step must use a dedicated `pthread` with `nanosleep()` or `timerfd_create(CLOCK_MONOTONIC)` for the heartbeat check, identical to the WAL fsync timer approach described in Part 2.4. Using `delaymgr` for the heartbeat watchdog is only acceptable for deployments that can tolerate 2+ second detection jitter. +**Heartbeat detection latency**: The heartbeat watchdog runs on a dedicated `pthread` with `nanosleep()` (see P1-3 and `ha_check_heartbeat()` above), identical to the WAL fsync timer approach in Part 2.4. This achieves sub-second detection granularity. The actual primary-failure detection latency is `heartbeat_timeout_ms + up to heartbeat_interval_ms` milliseconds — the timeout fires, plus up to one heartbeat interval of jitter before the next check. For the default `heartbeat_timeout_ms = 10000` and `heartbeat_interval_ms = 3000`, worst-case detection is 13 seconds. For HPC workloads requiring sub-10-second failover, reduce `heartbeat_timeout_ms` and `heartbeat_interval_ms` accordingly. ### 5.6 Promotion Sequence @@ -1432,22 +1611,26 @@ When a standby promotes to primary: 1. **Stop WAL replay** from old primary 2. **Increment term** (`ha.term++`) — fencing token for stale standbys -3. **Flush the local WAL** (`persist_flush()` — fsync the last WAL segment to ensure all replicated entries are on disk) and **verify WAL consistency** (validate CRC chain from the standby's own last snapshot LSN forward — standbys run `persist_recover()` on startup and maintain their own local WAL and snapshots, so the validation starting point is the standby's `persist_t.checkpoint_lsn`). Do **NOT** call `persist_recover()` again — the standby's kvmgrs are already populated from replication (via `apply_wal_entry()` during normal `OP_replEntry` processing) AND from the startup recovery pass. Calling `persist_recover()` a second time would double-apply entries. Only a WAL flush + CRC chain validation is needed. **WAL gap handling**: Async-replicated entries that the old primary committed but had not yet replicated before crashing are **lost** — the dead primary cannot serve catch-up. This is a known limitation of the async replication model: non-sync WAL entries (job starts, finishes, rusage updates) may be lost on failover. The promoted primary's view is consistent but potentially stale; async state gaps self-heal through worker reconnection (see step 9). **WAL consistency guarantee**: the standby's local WAL is a strict prefix of the old primary's WAL — standbys only write entries received via `OP_replEntry` from the primary. No divergence is possible if the standby was continuously connected; the standby's WAL cannot contain entries the old primary did not originate. The only split-brain divergence scenario (standby accepted writes from clients during a partition) cannot occur because standbys do not accept writes — they only redirect to the primary. +3. **Flush the local WAL** (`persist_flush()` — fsync the last WAL segment to ensure all replicated entries are on disk) and **verify WAL consistency** (validate CRC chain from the standby's own last snapshot LSN forward — standbys run `persist_recover()` on startup and maintain their own local WAL and snapshots, so the validation starting point is the standby's `persist_t.checkpoint_lsn`). Do **NOT** call `persist_recover()` again — the standby's kvmgrs are already populated from replication (via `apply_wal_entry()` during normal `OP_replEntry` processing) AND from the startup recovery pass. Calling `persist_recover()` a second time would double-apply entries. Only a WAL flush + CRC chain validation is needed. **WAL gap handling**: Async-replicated entries that the old primary committed but had not yet replicated before crashing are **lost** — the dead primary cannot serve catch-up. This is a known limitation of the async replication model: non-sync WAL entries (job starts, finishes, rusage updates) may be lost on failover. The promoted primary's view is consistent but potentially stale; async state gaps self-heal through worker reconnection (see step 9). **WAL consistency guarantee**: the standby's local WAL is a strict prefix of the old primary's WAL — standbys only write entries received via `OP_replEntry` from the primary. No divergence is possible if the standby was continuously connected; the standby's WAL cannot contain entries the old primary did not originate. The only split-brain divergence scenario (standby accepted writes from clients during a partition) cannot occur because standbys do not accept writes — they only redirect to the primary. **CRC chain restart on promotion (COH-5)**: After the WAL flush, `persist_flush()` MUST reset `persist_t.last_replicated_crc` to `WAL_CRC_NONE`. This signals that the promoted primary is starting a new replication chain — any remaining standbys have not received entries from this new primary yet, so there is no valid "previous entry CRC" to chain against. The first entry this primary broadcasts will carry `prev_crc = WAL_CRC_NONE`, which the standby receiver accepts as a valid chain start (see Part 6.4 catch-up and segment-boundary handling). Without this reset, the new primary would broadcast `prev_crc = last_replicated_crc` (the CRC of the last entry the old primary sent), which no standby would recognize — causing a chained-CRC mismatch disconnect on the first broadcast. 4. **Switch role** to `CTRL_PRIMARY` — only after recovery succeeds. **If recovery fails, abort this promotion attempt and set `ha.promotion_failed = 1`**. The standby remains in `CTRL_STANDBY` role, but since the old primary is dead (that triggered this promotion), the next heartbeat watchdog check (`ha_check_heartbeat()`) will re-fire `ha_promote_to_primary()`. The `promotion_failed` flag gates retry: the standby retries promotion on each watchdog tick unless: (a) a new heartbeat is received from a higher-priority candidate (another controller won the race), or (b) a configurable `max_promotion_retries` (default: 5) is exceeded, at which point the standby logs `FATAL "cannot promote after %d attempts — manual intervention required"` and stops retrying. This prevents a permanent no-primary state while bounding the retry window. 5. **Register persistence hooks** — standby did not register persistence hooks during startup (hooks are registered only after `persist_recover()` completes, and only on the primary path; standbys register them here on promotion). **Idempotent registration guard**: `ha_promote_to_primary()` must check whether hooks are already registered before adding them (e.g., via a `ha->hooks_registered` flag in `ha_state_t`). Without this guard, a promote→demote→re-promote cycle would accumulate duplicate hooks that fire N times per event, causing double WAL writes and replication sends. -5.5 **Start heartbeat immediately** — begin sending heartbeat messages to other standbys BEFORE `cluster_start()` (step 8). The promoted standby must establish its authority as the new primary by sending heartbeats with its incremented term. This prevents other standbys from timing out and promoting themselves during the potentially lengthy replicator initialization and scheduler gating steps. The heartbeat thread is launched with the new term and role set to PRIMARY, but `cluster_start()` (step 8) is deferred until all preparation steps complete. **CRITICAL**: The heartbeat messages sent during this pre-operational phase contain only term and role — no LSN or state information. Other standbys receiving these heartbeats update their `ha.term` and `primary_host` to the new primary, but do not expect any replication data until step 8 is reached. +5.5 **Start heartbeat immediately** — begin sending heartbeat messages to other standbys BEFORE `cluster_start()` (step 8). The promoted standby must establish its authority as the new primary by sending heartbeats with its incremented term. This prevents other standbys from timing out and promoting themselves during the potentially lengthy replicator initialization steps. + +**Pre-iomgr mechanism**: At this point in the promotion sequence, `cluster_start()` (and thus iomgr) has not yet run. The heartbeat messages are sent via the **same raw TCP/SSL pthread mechanism as the election probes** (Part 5.4): one short-lived `pthread` per lower-priority standby, each making a synchronous blocking TLS dial, sending a single `OP_heartbeat` frame with the new term and role, and closing the connection. This is intentionally distinct from the iomgr/endpoint pipeline. After `cluster_start()` (step 8), all subsequent HA traffic (heartbeats, replication) switches to the iomgr/SSL/endpoint pipeline permanently. + +The heartbeat thread is launched with the new term and role set to PRIMARY, but `cluster_start()` (step 8) is deferred until all preparation steps complete. **CRITICAL**: The heartbeat messages sent during this pre-operational phase contain only term and role — no LSN or state information. Other standbys receiving these heartbeats update their `ha.term` and `primary_host` to the new primary, but do not expect any replication data until step 8 is reached. **Bounded timeout for iomgr follow-up**: After a standby receives a pre-iomgr heartbeat and updates `primary_host`, it starts a countdown of `2 × heartbeat_timeout_ms`. If no iomgr-level `OP_heartbeat` arrives within this window from the claimed new primary, the standby resets `primary_host` to empty and re-runs `am_i_primary()`. This bounds the "stuck on a dead promoted primary" scenario: if the promotion crashes mid-sequence (after step 5.5 pthreads sent heartbeats but before `cluster_start()` at step 8), other standbys detect the absence of iomgr heartbeats and re-elect independently. Without this timeout, a standby that updated `primary_host` but never receives an iomgr heartbeat from the promoted primary would wait indefinitely at `heartbeat_timeout_ms` intervals, never re-running the election. 6. **Initialize replicator** — establish connections to remaining standbys -7. **Gate the scheduler**: pause or flush the scheduler's run queue before accepting writes. The scheduler maintains allocation state derived from WAL-replayed job state; async-replicated events (job starts/finishes) may be stale immediately after promotion. The scheduler must re-query job state from the promoted primary before making scheduling decisions (see step 8). +7. **Scheduler gating is implicit** — the scheduler has not reconnected at this point in the promotion sequence (it reconnects in step 9 via `xce_reconnect()`). There is no active "gate the scheduler" protocol step: the scheduler simply isn't connected, so no scheduling can occur. When the scheduler reconnects (step 9), it self-heals by sending a full state-sync query to enumerate all PEND and RUN jobs, rebuilds its allocation table from scratch, and resumes scheduling from a clean slate — this is the actual gating mechanism. **No additional synchronization is required here.** Note: the old text said "pause or flush the scheduler's run queue before accepting writes" — this was incorrect; the scheduler's queue is a scheduler-internal data structure that the controller does not directly control. 8. **Start normal primary operation** — `cluster_start()`, accept connections -9. **Wait for workers and scheduler to reconnect** — the promoted standby's in-memory state may be stale for async-replicated events (job starts, finishes, rusage updates). This is self-healing: workers re-join via `handle_worker_join()` and re-report their current job state, and the scheduler re-queries job state when scheduling. The promoted primary is functionally equivalent to a freshly booted controller that has replayed its WAL — it has a consistent but potentially stale view of job lifecycle states, which is corrected as workers reconnect and update their status. +9. **Workers and scheduler reconnect automatically** — when the old primary's TCP connections drop, libxce's `xce_reconnect()` backoff+jitter loop engages on each worker and scheduler. They retry through the candidate list until they reach the promoted primary, then send a standard `OP_workerJoin` / `OP_schedulerJoin`. No special failover logic is needed in the promoted primary — workers rejoin via the existing `handle_worker_join()` path and re-report their current job state. The scheduler re-queries job state when scheduling. The promoted primary is functionally equivalent to a freshly booted controller that has replayed its WAL — it has a consistent but potentially stale view of job lifecycle states, which is corrected as workers reconnect and update their status. No "re-push config" logic is needed on promotion: the join response carries runtime config as usual. This means async-replicated job lifecycle transitions (start, finish, fail, stop, resume, rusage) that were lost when the old primary crashed are not present in the promoted standby's WAL. The promoted standby's view of these jobs may be stale (e.g., showing a job as PEND when it is actually RUNNING). Workers correct this state when they reconnect and re-report their current job status. Until all workers reconnect, the scheduler should treat job state as uncertain and avoid scheduling decisions that depend on stale information. -**Job state during the reconnection window**: Jobs that were in `JOB_STAT_RUN` on the old primary but have no matching update in the standby's WAL enter a **transient unknown state**. The promoted primary marks such jobs with a sentinel flag (e.g., `JOB_STAT_TRANSIENT`) to signal uncertainty to operators and the scheduler. The scheduler must not schedule new work onto workers that own transient jobs (their status is unknown — the job may have completed, crashed, or still be running). As workers reconnect via `handle_worker_join()`, each worker re-reports its current job status and the job transitions from `JOB_STAT_TRANSIENT` to its actual state (`JOB_STAT_RUN`, `JOB_STAT_EXIT`, `JOB_STAT_DONE`, etc.). This guarantees eventual consistency: no job state is permanently lost, only temporarily uncertain until the owning worker reconnects. If a worker never reconnects (hardware failure, network isolation), jobs on that worker remain transient until an operator intervenes — this is the same as today's behavior when a worker goes `WORKER_UNAVAILABLE`. +**Job state during the reconnection window**: Jobs that were in `JOB_STAT_RUN` on the old primary but have no matching update in the standby's WAL enter a **transient unknown state**. The promoted primary marks such jobs with a sentinel flag (e.g., `JOB_STAT_TRANSIENT`) to signal uncertainty to operators and the scheduler. The scheduler must not schedule new work onto workers that own transient jobs (their status is unknown — the job may have completed, crashed, or still be running). When the **owning worker** (the specific worker that held the job at promotion time, identified by `job_t.exec_host`) reconnects via `handle_worker_join()`, it re-reports its current job status and the job transitions from `JOB_STAT_TRANSIENT` to its actual state (`JOB_STAT_RUN`, `JOB_STAT_EXIT`, `JOB_STAT_DONE`, etc.). **Only the owning worker's reconnect clears `JOB_STAT_TRANSIENT` for that specific job** — reconnection of any other worker does not affect transient jobs on different workers. `reconciliation_progress` is a monitoring convenience metric (aggregate reconnect count); it is not the mechanism that resolves individual transient jobs. This guarantees eventual consistency: no job state is permanently lost, only temporarily uncertain until the owning worker reconnects. If a worker never reconnects (hardware failure, network isolation), jobs on that worker remain transient until an operator intervenes — this is the same as today's behavior when a worker goes `WORKER_UNAVAILABLE`. -**Stale scheduler state**: The scheduler maintains internal state derived from controller state (resource allocations, pending job counts per queue, scheduling constraints). This internal state is also stale after failover. The scheduler heals by re-initializing from scratch when it reconnects to the promoted primary via `xce_reconnect()`: on reconnect, the scheduler sends a full state-sync query to enumerate all PEND and RUN jobs, rebuilds its allocation table, and resumes scheduling from a clean slate. Admin-initiated changes (queue configuration changes, admin holds on jobs) that were async-replicated and lost during the partition are visible to the promoted standby through its WAL replay — they are ASYNC events and captured in the WAL; only job lifecycle events from workers are subject to stale state. The scheduler must therefore re-query job state but can trust queue and worker configuration from the promoted primary's WAL. +**Stale scheduler state**: The scheduler maintains internal state derived from controller state (resource allocations, pending job counts per queue, scheduling constraints). This internal state is also stale after failover. The scheduler heals by re-initializing from scratch when it reconnects to the promoted primary via `xce_reconnect()`: on reconnect, the scheduler sends a full state-sync query to enumerate all PEND and RUN jobs, rebuilds its allocation table, and resumes scheduling from a clean slate. Admin-initiated changes (queue configuration changes, admin holds on jobs) are `WAL_QUEUE_CTRL` / `WAL_WORKER_CTRL` — **ASYNC-replicated events** and subject to the same loss window as job lifecycle events. If the old primary crashed before these entries were replicated to the standby, they are absent from the promoted standby's WAL. **The scheduler must re-query all state** (job, queue, and worker configuration) on reconnect — it cannot selectively trust queue/worker configuration as more reliable than job lifecycle state. Both are subject to the async replication lag at the moment of failure. -**Reconciliation window definition**: The reconciliation window is the interval between promotion (step 2) and the point where all workers that were connected to the old primary have re-joined the promoted primary. **Scheduler behavior during the reconciliation window**: (1) New job submits are accepted normally — these have authoritative state. (2) Scheduling of PEND jobs onto reconnected workers proceeds normally. (3) Scheduling of PEND jobs onto **unreconnected** workers is blocked — the scheduler does not know whether those workers still have capacity (transient jobs may still be running). (4) The scheduler tracks a `reconciliation_progress` metric: `(workers_reconnected / workers_at_old_primary)`. (5) Once `reconciliation_progress >= 1.0` (all workers have reconnected), normal scheduling resumes on all workers. **Expected window duration**: 30–120 seconds for a healthy cluster (workers reconnect within one `worker_heartbeat_interval`, typically 15–30 seconds, plus one scheduling cycle for the scheduler to catch up). If workers are slow to reconnect (e.g., due to DNS or SSL re-handshake latency), the window extends. If a worker never reconnects, its jobs remain in `JOB_STAT_TRANSIENT` until the operator intervenes — these workers are excluded from `reconciliation_progress` after a configurable `worker_reconnect_timeout_s` (default: 300 seconds). +**Reconciliation window definition**: The reconciliation window is the interval between promotion (step 2) and the point where all workers that were connected to the old primary have re-joined the promoted primary. **Scheduler behavior during the reconciliation window**: (1) New job submits are accepted normally — these have authoritative state. (2) Scheduling of PEND jobs onto reconnected workers proceeds normally. (3) Scheduling of PEND jobs onto **unreconnected** workers is blocked — the scheduler does not know whether those workers still have capacity (transient jobs may still be running). (4) The scheduler tracks a `reconciliation_progress` metric: `(workers_reconnected / workers_at_promotion_time)` where `workers_at_promotion_time` = count of workers in the promoted standby's kvmgr with status NOT including `XCE_WORKER_UNAVAIL` or `XCE_WORKER_HIDDEN` at the moment promotion completes (step 2). This count is derived from the standby's own replicated state — no data from the dead primary is needed. (5) Once `reconciliation_progress >= 1.0` (all workers have reconnected), normal scheduling resumes on all workers. **Expected window duration**: 30–120 seconds for a healthy cluster (workers reconnect within one `worker_heartbeat_interval`, typically 15–30 seconds, plus one scheduling cycle for the scheduler to catch up). If workers are slow to reconnect (e.g., due to DNS or SSL re-handshake latency), the window extends. If a worker never reconnects, its jobs remain in `JOB_STAT_TRANSIENT` until the operator intervenes — these workers are excluded from `reconciliation_progress` after a configurable `worker_reconnect_timeout_s` (default: 300 seconds). **Stuck-worker alarm**: when any worker hits the `worker_reconnect_timeout_s` boundary without reconnecting, log at `WARN`: `"Worker %s did not reconnect within %us after failover — %d transient jobs on this worker require manual resolution"` and expose `ctrl_reconciliation_stuck_workers` gauge (count of workers excluded from `reconciliation_progress` due to timeout). This gives operators a concrete list of workers requiring investigation rather than requiring them to correlate `ctrl_transient_jobs_stuck` with worker connection state. ### 5.7 Term (Fencing Token) @@ -1460,7 +1643,15 @@ The `term` field prevents split-brain: **Partition heal — old primary returns**: When a partitioned primary recovers network connectivity, it resumes sending heartbeats and replication messages with its old (lower) term. The promoted primary (higher term) ignores these messages (term < own term rule above). The partitioned primary receives a heartbeat reply or replication message from the promoted primary carrying the higher term. On receiving any message with `term > ha.term`, the old primary must detect that it has been superseded and demote itself: set `ha.role = CTRL_STANDBY`, stop accepting writes, and reconnect to the promoted primary as a standby. The `handle_msg()` path for all incoming messages must check `hdr.term > ha.term` and call `ha_demote_to_standby()` when this condition is detected, then re-process the message in standby mode. This prevents the old primary from continuing to accept writes after the partition heals, eliminating the split-brain write window. -**Hook unregistration on demotion**: `ha_demote_to_standby()` must unregister the persistence hooks that were registered during promotion (step 5 above). Use `xce_remove_hook()` (or equivalent) with the same function pointers and hook types registered in `ha_promote_to_primary()`. Without unregistration, a promote→demote→re-promote cycle accumulates duplicate hooks (despite the idempotent guard in step 5, the old hook entries still exist in the hook list). Unregistration must happen before the role switch to `CTRL_STANDBY` so that no in-flight requests trigger hooks after the role has changed. **Hook list synchronization**: `xce_remove_hook()` must be safe to call while vcpu threads are concurrently iterating the hook list inside `xce_run_hooks()`. This requires the hook list to use a lock-free or RCU-style removal pattern: `xce_remove_hook()` marks the hook entry as disabled (e.g., sets `fn = NULL` or a `disabled` flag), and `xce_run_hooks()` skips disabled entries. The actual memory reclamation of the hook entry must be deferred until no vcpu thread can be inside `xce_run_hooks()` — a safe deferral point is after `cluster_fini()` drains all vtask threads. This avoids a data race where a vcpu thread reads a freed hook function pointer from the list while `xce_remove_hook()` is concurrently removing it. The role-check defense-in-depth (checking `ha.role == CTRL_PRIMARY` inside each hook function before proceeding) provides a second layer: even if a hook entry is still in the list during demotion, it does nothing because the role has already switched to STANDBY. +**Asymmetric partition — old primary never receives higher-term message**: If the partition only partially heals (old primary can reach workers and API servers but NOT the promoted standby), the old primary continues accepting job submits indefinitely — it never receives a higher-term message and cannot self-demote. This is the **unbounded split-brain scenario** acknowledged in Part 5.1. Mitigation: the promoted primary sends heartbeats to the old primary's address (via the pre-iomgr mechanism in step 5.5 and the iomgr pipeline after step 8). If the asymmetric partition severs primary→standby but allows standby→primary traffic, the old primary will eventually receive a heartbeat-reply with the new term. If the partition is bidirectionally severed between the two controllers, no automatic resolution exists — operator intervention is required (SIGUSR1 manual failover on the old primary, or force-restart it). The `ctrl_split_brain_detected` metric captures this condition once any cross-controller message is received after partition heal. + +**Hook disabling on demotion**: `ha_demote_to_standby()` must disable the persistence hooks that were registered during promotion (step 5 above). The chosen approach is **disable-in-place via `xce_disable_hook()`** (to be added to `src/lib/hook.c`) rather than true removal. + +**Memory ordering**: `hook_item_t.fn` must be declared as `_Atomic hook_fn_t` (C11 atomic function pointer). `xce_disable_hook()` stores NULL with `atomic_store_explicit(memory_order_release)`. `xce_run_hooks()` loads `fn` into a local variable with `atomic_load_explicit(memory_order_acquire)`, then checks and calls the local — two reads of the same non-atomic variable can diverge on weakly-ordered architectures, but a single atomic load into a local is a single read with a visibility guarantee. + +**Slot reuse to bound list growth**: On each promote→demote cycle, a NULL slot is left in the hook list and a new slot is added on re-promote. To prevent unbounded growth, `xce_add_hook()` must scan for an existing NULL `fn` slot of the same `type` before appending a new entry. If a matching NULL slot is found, it reuses it (sets `fn` and `data`). This bounds the hook list to max(ever-registered) slots regardless of cycle count. + +Disabling must happen before the role switch to `CTRL_STANDBY` so that no in-flight requests trigger hooks after the role has changed. The `hooks_registered` idempotency guard (see step 5) is reset to 0 on disable so that the next promote re-registers correctly. The role-check defense-in-depth (checking `ha.role == CTRL_PRIMARY` inside each hook function before proceeding) provides a second layer: even if a hook entry fires during the disable window, it does nothing because the role has already switched to STANDBY. **In-flight request window during demotion (CRITICAL)**: Between the higher-term message arriving at `handle_msg()` and the demotion taking effect, vcpu threads may already be executing `handle_jobsubmit()` past the `AFTER_EVENT` hook. These in-flight requests have the following risks: 1. The persistence hook already called `persist_wal_sync_append()`, which is writing to the local WAL and may have already sent `repl_broadcast()`. Standbys following the higher-term primary will ignore the lower-term replication message. @@ -1492,18 +1683,21 @@ When ctrl-1 recovers: **Manual failover protocol**: To move primary back to a higher-priority candidate (e.g., ctrl-1 after recovery), the operator sends `SIGUSR1` to the current primary (ctrl-2). The SIGUSR1 signal handler **must only set an atomic flag**: `atomic_store(&ha->failover_requested, 1)`. It does NOT call `ha_demote_to_standby()` directly — that function performs operations (mutex locks, fsync, SSL writes, memory allocation) that are not async-signal-safe per POSIX. The failover is executed by the dedicated HA watchdog thread (the same pthread running `ha_check_heartbeat()` — P1-3), which polls `ha->failover_requested` on each tick. When the flag is observed, the watchdog thread calls `ha_demote_to_standby()` from its safe thread context, which: (a) unregisters persistence hooks (see Part 5.7 hook unregistration), (b) sets `ha.role = CTRL_STANDBY` using `atomic_store_explicit` with `memory_order_release`, (c) stops accepting new writes, (d) flushes and sync-replicates any pending WAL entries, (e) sends a `OP_steppingDown` message to all connected standbys with its current term and the election deadline. On receiving `OP_steppingDown`, the highest-priority connected standby (ctrl-1) immediately begins its election (calls `ha_promote_to_primary()`). The stepping-down controller then reconnects to the newly promoted primary as a standby. **During the transfer window (~1-2 election_timeout_ms), no controller is primary — job submits receive `EAGAIN` and must retry.** This is the same brief unavailability as an automatic failover. The `SIGUSR1` approach avoids needing a dedicated admin RPC or CLI command in the initial implementation; a future `xce hostcontrol --failover` CLI can wrap this by sending the signal remotely via SSH. Add `atomic_int failover_requested;` to `ha_state_t` for this purpose. +**SIGUSR1 access control note**: Signal delivery to a process requires the sender to have the same UID (or be root). The controller runs as a dedicated service account; on HPC management nodes (login nodes, admin nodes), where multiple user accounts may exist, any process running as the controller's service account can trigger a failover. **Mitigation (in scope for P1-3a)**: add `--disable-sigusr1-failover` as a startup flag (single `atomic_int sigusr1_failover_enabled` field checked in the signal handler before setting `failover_requested`). Operators on shared management nodes should deploy with this flag set; `xce hostcontrol --failover` (SSH-authenticated) becomes the only failover path once available. The flag is one additional line in the signal handler and should ship with P1-3a rather than being deferred to future work. + +**Split-brain demotion audit log**: When `ha_demote_to_standby()` is triggered by receiving a higher-term message (partition heal path — NOT the SIGUSR1 path), it must log the range of WAL entries this controller generated during the split-brain window before discarding them: `ERROR("Split-brain demotion: discarding local WAL entries lsn=[%lu..%lu] (term=%lu) — these %lu job submits were ACKed to clients but are NOT durable. Operators: treat job IDs in this LSN range as lost. ctrl_split_brain_lost_submits+=%lu", divergence_start_lsn, current_lsn, ha->term, entry_count, entry_count)`. `divergence_start_lsn` is the LSN at which this controller's term diverged from the promoted primary's term (recorded when this controller first started accepting writes after the partition began). Increment `ctrl_split_brain_lost_submits` counter by `entry_count`. This log gives operators a bounded audit trail without requiring a quorum protocol or external log aggregation. + **Demotion failure handling**: If `persist_flush()` fails during `ha_demote_to_standby()` (step d above), the controller has already set `ha.role = CTRL_STANDBY` and incremented term. The safest recovery is to **not rollback** the role change — instead, log a `FATAL("persist_flush failed during demotion — WAL may be incomplete. Manual investigation required.")`. The demoted controller will not accept new writes (role is STANDBY), and the WAL will be flushed again on the next checkpoint. A role rollback (setting `ha.role = CTRL_PRIMARY` on flush failure) would leave the cluster in a split-brain-eligible state where the original primary continues accepting writes while a new primary may already be promoting. Failing forward as STANDBY is safer: the cluster temporarily has no primary until another standby wins election, which is the same state as a normal failover. -### 5.9 HA Connection Security (mTLS) +### 5.9 HA Connection Security (Trusted LAN) -All HA connections (primary→standby replication, standby→primary heartbeat, election probes) use the **same iomgr/SSL pipeline** as worker/scheduler connections, with mutual TLS peer authentication: +XCE controllers run on a **dedicated management LAN**. All HA connections (primary→standby replication, standby→primary heartbeat, election probes) use the same iomgr/SSL pipeline as worker/scheduler connections, providing transport encryption. No mutual TLS (mTLS) client certificate verification is required between controllers. -- **Peer authentication**: HA peers authenticate each other using client certificates signed by the cluster CA. The same CA that signs worker/scheduler client certs signs controller-to-controller certs. A separate HA cert distinguished name (`CN=ctrl-`, `OU=ha`) allows the server to distinguish HA peers from workers/schedulers in the RBAC layer. -- **SSL context extension**: The existing `create_ssl_context(cert_file, key_file)` in `netio.c:890` creates server-side SSL contexts but does **not** accept a CA cert parameter for client-side peer verification. HA mTLS requires extending the signature to `create_ssl_context(cert_file, key_file, ca_cert_path)`. When `ca_cert_path` is non-NULL, the function calls `SSL_CTX_load_verify_locations()` and `SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, ...)` for mTLS client-side verification. **The HA listeners (both election probe and replication accept) must use an SSL context created with `ca_cert_path` non-NULL, so that the TLS handshake itself rejects clients without a valid CA-signed certificate — before any application-layer CN check.** The CN check in `OP_replJoin` is then defense-in-depth only. When `ca_cert_path` is NULL, the function behaves as before (server-only context). This is a P1 prerequisite for both election probes and replication connections. **Note**: since there is no released version requiring backward compatibility, the 2-param signature is replaced in-place — all existing callers are updated to pass NULL for the new third parameter. -- **Certificate provisioning**: Controller certs are provisioned at deployment time alongside worker certs. This is an ops concern (cluster CA, cert renewal cadence) — the implementation only consumes a cert/key path from config. Add `ha_cert_path` and `ha_key_path` to `HAConfig` (i.e., nested under `ha:` in `cluster.yaml`, alongside `heartbeat_interval_ms`, `heartbeat_timeout_ms`, etc.) — **not** to `PersistConfig`. These are HA connection settings, not persistence settings; mixing them into `PersistConfig` would create a misleading YAML key hierarchy. Update P0-9 to include `ha_cert_path` and `ha_key_path` in the `HAConfig` capnp schema. **File permissions**: `ha_key_path` (the private key file) must be mode `0600` (owner-read-only). `ha_init()` must verify this at startup and refuse to start if the key file is world- or group-readable — a misconfigured key permission is a silent security risk that would be missed without a startup check. -- **Certificate rotation**: Controller certs must be renewed before expiry. `ha_init()` must log a `WARN` if the cert's `NotAfter` is within 30 days of the current time (configurable via `ha_cert_warn_days`, default 30). Cert rotation requires a rolling restart: update the cert file, restart each standby one at a time, then restart the primary (after standbys are online). The key file must be written with mode `0600` before the restart (e.g., `install -m 0600 `) — if the key file has incorrect permissions, `ha_init()` will refuse to start. The P4-4 operational runbook must document this rotation procedure. Hot rotation (without restart) is out of scope. -- **Election probe**: The initial `am_i_primary()` probe (before iomgr is started — see Part 5.4 startup sequencing note) launches one `pthread` per lower-priority candidate, each making a **synchronous blocking TLS dial** with the same cert/key pair. All threads share a single `election_timeout_ms` deadline; the first live response wins. These probes do not go through the iomgr event loop; after the election resolves, all subsequent HA traffic uses the iomgr/SSL pipeline for heartbeat and replication. -- **Deployment requirement**: HA requires a cluster CA and per-controller certificates. Clusters without a PKI infrastructure (no CA) cannot use HA — this is an explicit gate. Many HPC sites run without an internal CA; for these sites, HA is unavailable until a CA is provisioned. The P4-4 operational runbook must document: (a) how to set up a minimal cluster CA (e.g., using `openssl` or `cfssl`), (b) how to issue and distribute per-controller certificates, and (c) cert renewal cadence (recommend 1-year validity with 90-day renewal reminder). Sites that cannot operate a CA should treat HA as out of scope for their deployment. +**Trust model**: The management network is the security boundary. Only nodes on the management LAN can reach controller HA ports. This is the same model used by traditional HPC middleware (LSF, SLURM, PBS) — network-level access control (firewall rules, dedicated VLAN) replaces PKI-based peer authentication. + +- **Election probe**: The initial `am_i_primary()` probe (before iomgr is started — see Part 5.4 startup sequencing note) launches one `pthread` per lower-priority candidate, each making a **synchronous blocking TCP/SSL dial**. All threads share a single `election_timeout_ms` deadline; the first live response wins. Peer identity is confirmed by host:port matching against the `--peers` candidate list. After the election resolves, all subsequent HA traffic uses the iomgr/SSL pipeline for heartbeat and replication. +- **No PKI requirement**: HA does not require a cluster CA or controller-specific certificates. The existing SSL infrastructure (used for worker/scheduler connections) is reused for controller-to-controller channels via iomgr. **Election probe SSL context**: The pre-iomgr election probes (Part 5.4) cannot reuse the iomgr SSL context because iomgr has not yet started. Each probe thread creates a standalone `SSL_CTX` using the same cert/key pair as the iomgr context. Extract a shared helper `ssl_ctx_create(cert_path, key_path)` (or equivalent) used by both the election probe threads and iomgr initialization to avoid code duplication. The election probe SSL contexts must be freed after all probe threads join (`pthread_join`), before `cluster_start()` initializes iomgr's own SSL context. No mTLS or PKI changes are required — the same cert/key pair is used throughout. +- **Optional enhancement — SSL_VERIFY_PEER for controller-to-controller channels**: The current design validates peer identity by host:port matching against `--peers`. Enabling one-way TLS server certificate CN/SAN verification (`SSL_CTX_set_verify(SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT)`) on the HA client side would convert this network-layer assertion to a cryptographic one — at no additional PKI cost since the same cert/key infrastructure already exists. This is not required for the trusted-LAN model but would be a low-effort P3 hardening step (`P3-8`): configure `SSL_CTX_set_verify` on the election probe SSL context and the iomgr HA connection path; verify the server cert's CN or SAN matches the candidate's configured hostname. The `--ca-cert` flag added for libxce (P2.5-5) would serve this purpose. Deploy only if the management network boundary cannot be enforced via firewall/VLAN. ### 5.10 New Opcodes and Cap'n Proto Additions @@ -1511,7 +1705,9 @@ All HA connections (primary→standby replication, standby→primary heartbeat, /* In HostRole enum: */ standby @5; /* NEW — standby controller role */ -/* In OPCode enum: */ +/* In OPCode enum (existing entries go up to @21 — clrpreason): + * @22 is intentionally skipped in the OPCode enum to leave room for a future + * non-HA opcode. The new HA opcodes start at @23. */ replEntry @23; /* Replicated WAL entry (primary → standby) */ replAck @24; /* ACK for sync-replicated entry (standby → primary) */ redirect @25; /* Redirect to current primary (standby → client) */ @@ -1520,7 +1716,9 @@ steppingDown @27; /* Primary is stepping down (primary → standby, trigger replSnapshot @28; /* Snapshot transfer for catch-up (primary → standby, chunked) */ replCatchUpDone @29; /* Signal that catch-up snapshot is loaded (standby → primary) */ -/* In Request struct: */ +/* In Request struct (struct ordinals are separate from OPCode enum values): + * @22 here is the Request struct field ordinal — it does NOT conflict with + * the OPCode @22 gap above. These are different namespaces in Cap'n Proto. */ term @22 :UInt64 $C.mapname("term"); /* NEW — fencing token */ ``` @@ -1545,7 +1743,7 @@ term @22 :UInt64 $C.mapname("term"); /* NEW — fencing token */ ```c /* MAX_STANDBYS and MAX_HA_CANDIDATES defined in include/config.h (see Appendix A.10) */ -**Constant reconciliation**: The existing `src/include/client.h` defines `N_MAX_MASTERS = 3` for the `xce_client_config_t.masters[]` array. `MAX_HA_CANDIDATES = 16` is defined in `include/config.h` for `standbys[]` and `xce_cluster_config_t.candidates[]`. These are different array sizes for different purposes: `N_MAX_MASTERS` is for the client-side connection config (max 3 controllers to try), while `MAX_HA_CANDIDATES` is a structural array size (max 16 candidates). The deployment ceiling advisory (≤3 controllers) aligns with `N_MAX_MASTERS = 3` which was already chosen for that purpose. The Go CGo bindings see both constants — they must be clearly separated in header exports. +**Constant reconciliation**: The existing `src/include/client.h` defines `N_MAX_MASTERS = 3` for the `xce_client_config_t.masters[]` array. Both `MAX_HA_CANDIDATES = 4` and `MAX_STANDBYS = 3` are defined directly in `include/config.h` (see Out of Scope — `MAX_HA_CANDIDATES` reduced from 16 to 4, `MAX_STANDBYS` reduced from 15 to 3). `MAX_STANDBYS` is defined as a standalone constant (`= 3`) in `config.h`, NOT derived as `MAX_HA_CANDIDATES - 1` in `repl.h` — there is one source of truth. The deployment ceiling advisory (≤3 controllers) aligns with `N_MAX_MASTERS = 3` which was already chosen for that purpose. The Go CGo bindings see both constants — they must be clearly separated in header exports. typedef struct { endpoint_t* ep; /* Full endpoint (SSL, FSM, outq) — uses iomgr/vtask pipeline */ @@ -1604,6 +1802,8 @@ Primary Standby-1 Standby-2 ### 6.4 Standby WAL Receiver > **IMPLEMENTATION GUARD**: The pseudocode below is a **logical sketch** of WAL entry processing on the standby. The actual implementation **must use the endpoint/iomgr/vtask SSL pipeline** (see Part 6.5 and Part 6.6). **Do NOT implement standalone receive-loop threads with raw `read()`/`write()` on file descriptors.** All controller connections — including replication — go through the iomgr event loop. A raw-socket implementation would bypass SSL enforcement, per-endpoint message ordering, and backpressure; any such implementation must be rejected in code review. The `standby_recv_thread()` function below is pseudocode only — replace with `handle_msg()` dispatch for `OP_replEntry` in the iomgr/vtask pipeline. +> +> **Serialization note**: `apply_wal_entry()` on the standby (including `xkvmgr_add()` calls inside it) is effectively single-threaded. All `OP_replEntry` messages arrive over a single primary→standby endpoint; the iomgr/vtask pipeline's `endpoint_block_tasks()`/`endpoint_unblock_tasks()` mechanism ensures messages on a given endpoint are dispatched to one vcpu thread at a time and in order. Therefore `apply_wal_entry()` does not require additional locking beyond what the functions it calls already provide for the normal (non-COW) path. The standby processes replication entries through its iomgr event loop: @@ -1666,7 +1866,16 @@ static void* standby_recv_thread(void* arg) * which would permanently lock out legitimate primaries and cause * term wraparound (UINT64_MAX + 1 = 0) on next promotion. This is a * liveness invariant: the fencing token must never be driven to an - * unusable value by a single bad packet. */ + * unusable value by a single bad packet. + * + * Accepted risk: MAX_TERM_JUMP prevents single-packet term exhaustion + * but NOT cumulative drift. An attacker on the management LAN (which + * is the trust boundary — see Part 5.9) could send repeated packets to + * advance ha->term by MAX_TERM_JUMP per packet, eventually reaching + * UINT64_MAX. Given the trusted-LAN security model (network-level ACLs + * restrict access), this cumulative attack requires sustained access to + * the management network and is accepted as an out-of-scope risk. mTLS + * peer authentication (Part 5.9) would eliminate this attack surface. */ #define MAX_TERM_JUMP 10000 if (hdr.term > atomic_load_explicit(&ha->term, memory_order_acquire) + MAX_TERM_JUMP) { ERROR("replication: term too high: got %lu (max allowed: %lu) — " @@ -1701,19 +1910,36 @@ static void* standby_recv_thread(void* arg) * (b) catch-up mode already performs CRC chain validation on the * transferred entries. Any gap in the initial stream is caught by the * catch-up protocol, not by the normal-mode LSN check. */ - if (ha->applied_lsn != 0 && hdr.lsn <= ha->applied_lsn) { + uint64_t applied = atomic_load_explicit(&ha->applied_lsn, memory_order_acquire); + if (applied != 0 && hdr.lsn <= applied) { ERROR("replication: LSN sequence error: got %lu, expected > %lu", - hdr.lsn, ha->applied_lsn); + hdr.lsn, applied); free(payload); break; } - if (ha->applied_lsn != 0 && hdr.lsn > ha->applied_lsn + 1) { - /* Multi-entry gap — possible network loss of entries. - * Single-entry gaps (hdr.lsn == applied_lsn + 2) are most likely a - * skipped WAL_CHECKPOINT marker. Log at WARN, accept the entry, - * and continue — CRC chain validation catches true corruption. */ - WARN("replication: LSN gap: expected %lu, got %lu (likely skipped WAL_CHECKPOINT)", - ha->applied_lsn + 1, hdr.lsn); + if (applied != 0 && hdr.lsn > applied + 1) { + /* LSN gap — distinguish single-skip (checkpoint marker) from multi-skip (data loss). + * + * Single-skip (hdr.lsn == applied + 2): most likely a skipped WAL_CHECKPOINT + * (type=100) which is local-only and not replicated. Log WARN and continue — + * the CRC chain validation below provides a second safety net. + * + * Multi-skip (hdr.lsn > applied + 2): indicates actual entry loss (network + * packet loss, buffer overflow, or protocol bug). WAL_CHECKPOINT is a single + * entry; two consecutive checkpoint skips would still produce a gap of 2 per + * checkpoint, not a gap > 2 in a single replication step. Disconnect and + * trigger catch-up — accepting multi-entry gaps silently would allow data loss + * to go undetected even if the CRC chain restarts (e.g., at a segment boundary). + */ + if (hdr.lsn == applied + 2) { + WARN("replication: single LSN skip: expected %lu, got %lu (likely WAL_CHECKPOINT marker)", + applied + 1, hdr.lsn); + /* continue — CRC chain below provides second validation */ + } else { + ERROR("replication: multi-entry LSN gap: expected %lu, got %lu — possible entry loss, disconnecting for catch-up", + applied + 1, hdr.lsn); + goto disconnect_for_catchup; + } } /* Validate chained CRC — detect missing/reordered entries that LSN alone @@ -1730,7 +1956,7 @@ static void* standby_recv_thread(void* arg) * On segment boundaries the primary sends entries with prev_crc == * WAL_CRC_NONE as the first entry of the new segment. The standby treats * this as a valid chain break (same as on-disk recovery, Part 2.4). */ - if (ha->applied_lsn == 0) { + if (atomic_load_explicit(&ha->applied_lsn, memory_order_acquire) == 0) { /* First entry after catch-up: prev_crc must be WAL_CRC_NONE */ if (hdr.prev_crc != WAL_CRC_NONE) { ERROR("replication: first entry after catch-up has non-NULL prev_crc %u", @@ -1741,15 +1967,15 @@ static void* standby_recv_thread(void* arg) } else { /* Normal mode: prev_crc must match the CRC of the last received entry. * NOTE: WAL_CHECKPOINT entries are written to the primary's WAL but NOT - * replicated. The next entry after a checkpoint carries prev_crc = CRC of - * the WAL_CHECKPOINT entry. Since standbys never receive the checkpoint - * entry, ha->last_received_crc would not match hdr.prev_crc. The primary - * resolves this by computing prev_crc for non-checkpoint replication - * entries as the CRC of the last non-checkpoint WAL entry — skipping - * checkpoint entries from the chain. This means the post-checkpoint - * entry's prev_crc equals the CRC of the entry before the checkpoint, - * maintaining the chain for standbys. See Part 2.3 "WAL_CHECKPOINT - * entries are not replicated". */ + * replicated. The primary maintains two separate CRC trackers: last_crc + * (updated for every WAL entry including checkpoints) and + * last_replicated_crc (updated only for replicated entries, never for + * WAL_CHECKPOINT). The post-checkpoint replicated entry therefore carries + * prev_crc = last_replicated_crc, which equals the CRC of the last + * non-checkpoint WAL entry — not the CRC of the checkpoint entry itself. + * Since ha->last_received_crc on the standby also tracks only replicated + * entries, the chain remains unbroken across checkpoint boundaries. + * See Part 2.3 for last_replicated_crc field documentation. */ if (hdr.prev_crc != ha->last_received_crc && hdr.prev_crc != WAL_CRC_NONE) { ERROR("replication: chained CRC mismatch: expected %u, got %u, lsn %lu " "(possible reordered/missing entry)", @@ -1757,23 +1983,28 @@ static void* standby_recv_thread(void* arg) free(payload); break; /* Disconnect; primary may have diverged — reconnect + catch-up will resync */ } - /* WAL_CRC_NONE in normal mode: accepted only at the start of a new WAL - * segment (the primary sets prev_crc = WAL_CRC_NONE for the first entry - * in each segment file). During steady-state replication, the primary - * sends entries continuously — a segment boundary does NOT produce - * WAL_CRC_NONE in the replication stream because the primary writes to - * the local WAL segment and sends to standbys in the same loop; segment - * rotation is a local-only event that does not interrupt the replication - * stream. Therefore, receiving WAL_CRC_NONE in normal mode (applied_lsn > 0) - * can only occur during catch-up when the standby receives the first entry - * after a snapshot transfer (which is handled by the applied_lsn == 0 check - * above). If WAL_CRC_NONE appears in any other context during normal - * replication, it indicates a protocol error and the standby should - * disconnect and re-connect with catch-up. */ - ha->last_received_crc, hdr.prev_crc, hdr.lsn); - free(payload); - break; /* Disconnect; primary may have diverged — reconnect + catch-up will resync */ - } + /* WAL_CRC_NONE in normal mode (applied_lsn > 0): accepted as a valid + * chain break. The code above does NOT disconnect when prev_crc == + * WAL_CRC_NONE — it passes the `if` condition silently. There are three + * legitimate sources of WAL_CRC_NONE in normal mode: + * + * 1. Segment rotation: the primary MUST emit prev_crc = WAL_CRC_NONE for + * the first entry of each new WAL segment (per Part 2.4 writer-side + * obligation). This creates a chain break in the replication stream + * whenever a segment rotation occurs during live replication. + * + * 2. Post-promotion: when a standby promotes to primary, it resets + * last_replicated_crc = WAL_CRC_NONE (per Part 5.6 step 3). The first + * entry it broadcasts carries prev_crc = WAL_CRC_NONE. Already-connected + * standbys with applied_lsn > 0 accept this as a term-change chain break. + * + * 3. After catch-up: the first live entry after snapshot application is + * handled by the applied_lsn == 0 check above (not this branch). + * + * The term check earlier in this function guards against accepting entries + * from a stale primary. WAL_CRC_NONE itself is not an error indicator — + * the CRC chain exists to detect missing or reordered entries, and + * WAL_CRC_NONE explicitly signals a legitimate restart of the chain. */ } /* Validate CRC — detect corruption in replication path. @@ -1856,13 +2087,17 @@ static void* standby_recv_thread(void* arg) break; /* Disconnect; catch-up will resync on reconnect */ } - ha->applied_lsn = hdr.lsn; + atomic_store_explicit(&ha->applied_lsn, hdr.lsn, memory_order_release); ha->last_received_crc = hdr.crc; /* Update chained CRC tracker */ /* Send ACK for sync-replicated entries — only when not in catch-up mode. * During catch-up the standby is replaying historical entries; sending ACKs - * then would prematurely signal readiness to the primary (see Part 6.6). */ - if (hdr.type == WAL_JOB_SUBMIT && !ha->in_catch_up) { + * then would prematurely signal readiness to the primary (see Part 6.6). + * ha->in_catch_up is the STANDBY-LOCAL field in ha_state_t (set by the + * standby when it enters catch-up, cleared when it sends OP_replCatchUpDone). + * This is distinct from the PRIMARY-SIDE per-slot standby_conn_t.in_catch_up + * (Appendix A.3), which is the primary's mirror of this field. */ + if (hdr.type == WAL_JOB_SUBMIT && !atomic_load_explicit(&ha->in_catch_up, memory_order_acquire)) { send_repl_ack(ha->primary_ep, hdr.lsn); } @@ -1949,16 +2184,16 @@ void handle_msg(cluster_t* c, endpoint_t* ep, msg_hdr_t* hdr, void* payload) { **Standby catch-up after reconnection**: When a standby reconnects to the primary (either after a network disruption or on initial startup), it must tell the primary its last `applied_lsn` so the primary can stream missing WAL entries. The catch-up protocol is: 1. Standby sends `OP_replJoin` with its `applied_lsn` and `term` -2. **Primary authenticates the peer**: The primary verifies that the peer's mTLS certificate CN matches a known candidate in `cluster.yaml` (e.g., `CN=ctrl-`, `OU=ha`). An `OP_replJoin` from a peer with an unknown or unrecognized CN is rejected with an error reply — only known controller peers may join the replication group. +2. **Primary validates the peer**: The primary verifies that the peer's `host:port` appears in its candidate list (from `--peers` CLI flag). An `OP_replJoin` from an unknown endpoint is rejected with an error reply. 3. Primary checks if the standby's `applied_lsn` is still within available WAL segments (not yet trimmed by a checkpoint). If yes, stream entries from `applied_lsn + 1` forward via `OP_replEntry` messages on the existing SSL endpoint pipeline 4. If the standby's `applied_lsn` is too old — i.e., `applied_lsn < oldest_surviving_segment_start_lsn` (the WAL segments needed for catch-up have been trimmed) — the primary transfers the latest snapshot **via the same SSL endpoint pipeline** using a new `OP_replSnapshot` opcode. The snapshot is chunked at `MAX_WAL_PAYLOAD_BYTES` per message: the first chunk includes a snapshot header `{ snapshot_lsn, total_size, chunk_seq=0, final=false }`, subsequent chunks carry `{ chunk_seq, data }`, and the last chunk sets `final=true` and includes a CRC32 of the full snapshot payload. The standby reassembles chunks into a buffer, verifies the CRC32, then applies the snapshot to its kvmgrs. **Snapshot size validation on standby receive**: The `total_size` field in the first chunk must not exceed `MAX_SNAPSHOT_BYTES` (default: 256 MB). If `total_size > MAX_SNAPSHOT_BYTES`, the standby logs an `ERROR` and tears down the replication connection — a compromised or buggy primary could otherwise exhaust the standby's memory by sending an unbounded snapshot. This mirrors the local recovery path's snapshot size validation (Part 2.5 step 2b). After snapshot application, the primary streams WAL entries from `snapshot_lsn + 1` forward. No separate file transfer channel or `scp` is required — all data flows through the established SSL endpoint. 5. If the standby's `term` is higher than the primary's, the primary has stale data — the standby should not follow this primary (fencing) -**Replication during catch-up**: While the primary is streaming historical WAL entries to a catching-up standby, it continues broadcasting new WAL entries to all other standbys. The catching-up standby's endpoint `sendq` buffers incoming live entries from the broadcast while replaying historical entries in order. **No separate catch-up queue is needed** — the existing endpoint `sendq` (already bounded by the iomgr flow-control mechanism) serves as both the catch-up buffer and the live-entry buffer. When the standby's `applied_lsn` reaches the primary's `committed_lsn` at the time of `OP_replJoin`, it transitions to normal replication mode and begins applying the buffered live entries from the `sendq`. The standby does not send `OP_replAck` for entries it receives during catch-up (to avoid prematurely signaling readiness) — `OP_replAck` is only sent once the standby has caught up and is processing live entries. +**Replication during catch-up**: While the primary is streaming historical WAL entries (WAL replay or snapshot + WAL) to a catching-up standby, the primary **pauses live `OP_replEntry` broadcasts to that standby** until the standby sends `OP_replCatchUpDone`. This is the definitive design choice: live entries are NOT buffered in the catching-up standby's sendq during catch-up. Buffering live entries during a 256 MB snapshot transfer (10–30 seconds at disk speed) at 1000 submits/sec would require 10–30 MB of in-flight WAL in the sendq — unbounded memory growth with no application-level control. The simpler and safer design: the catching-up standby is excluded from `repl_wait_ack()` via `in_catch_up` gating until it sends `OP_replCatchUpDone`. After catch-up, the standby requests any WAL entries it missed during the catch-up window via a second WAL-only catch-up phase (the primary's WAL covers this gap since WAL trim is held back by the standby's join LSN). The standby does not send `OP_replAck` during catch-up — only after fully caught up and processing live entries in normal mode. -**Catch-up receive ordering**: (1) `OP_replSnapshot` chunks arrive first and must be fully received and CRC-verified before any new WAL entries are applied to the kvmgrs. (2) The primary must NOT send live `OP_replEntry` messages to the catching-up standby until the snapshot transfer is complete — the standby sends `OP_replCatchupDone` (or equivalent signal) after successfully loading the snapshot. (3) After snapshot application, the primary streams WAL entries from `snapshot_lsn + 1` forward, and the standby applies them in strict LSN order. Only after reaching the primary's `committed_lsn` does the standby begin sending `OP_replAck` and transition to normal replication. This ensures the standby's state is self-consistent at each phase boundary: snapshot-loaded → historical-WAL-replayed → caught-up. +**Catch-up receive ordering**: (1) `OP_replSnapshot` chunks arrive first and must be fully received and CRC-verified before any new WAL entries are applied to the kvmgrs. (2) The primary must NOT send live `OP_replEntry` messages to the catching-up standby until the snapshot transfer is complete — the standby sends `OP_replCatchUpDone` (or equivalent signal) after successfully loading the snapshot. (3) After snapshot application, the primary streams WAL entries from `snapshot_lsn + 1` forward, and the standby applies them in strict LSN order. Only after reaching the primary's `committed_lsn` does the standby begin sending `OP_replAck` and transition to normal replication. This ensures the standby's state is self-consistent at each phase boundary: snapshot-loaded → historical-WAL-replayed → caught-up. -**Snapshot catch-up flow control**: The `OP_replSnapshot` transfer sends the full snapshot in chunks of `MAX_WAL_PAYLOAD_BYTES` through the SSL endpoint pipeline. The primary sends chunks as fast as the endpoint pipeline allows, and the standby must receive, buffer, and reassemble all chunks before applying the snapshot. The existing iomgr backpressure mechanism (endpoint write FSM, outq buffering, SSL-level flow control via `SSL_write` partial writes) provides implicit flow control: if the standby's receive path is slow, the primary's endpoint outq fills, and subsequent `SSL_write` calls block or return partial writes, naturally throttling the sender. **No additional application-level flow control is needed** beyond the iomgr/SSL pipeline's built-in backpressure. However, operators should be aware that a snapshot transfer (up to `MAX_SNAPSHOT_BYTES` = 256 MB) temporarily increases the standby's memory usage for reassembly. The standby allocates a receive buffer of `total_size` bytes (validated against `MAX_SNAPSHOT_BYTES` at first chunk, per Part 6.6) before receiving subsequent chunks — this memory is freed after snapshot application. **CRITICAL: validate `total_size <= MAX_SNAPSHOT_BYTES` BEFORE calling `malloc(total_size)`. Do NOT allocate the buffer and then check — the check is the first operation on first-chunk receipt.** **Aggregate memory budget**: With `MAX_STANDBYS = 3`, up to 3 standbys may catch up concurrently on the primary side. Each primary-side `sendq` for an in-flight snapshot transfer holds at most one pending chunk (`MAX_WAL_PAYLOAD_BYTES ≈ 1 MB`), so the primary's total per-catch-up memory is negligible. On the standby side, only one standby receives a snapshot at a time (a standby can only be catching up from one primary), so the worst-case standby-side allocation is a single `total_size ≤ MAX_SNAPSHOT_BYTES` buffer. The `768 MB = 3 × 256 MB` scenario (all standbys catching up simultaneously) can only occur on the **primary** if it were somehow buffering outbound snapshots for all standbys at once — but the primary streams each snapshot through an already-established SSL endpoint, holding at most `MAX_WAL_PAYLOAD_BYTES` per chunk in the endpoint outq. Therefore the realistic aggregate memory bound is `MAX_SNAPSHOT_BYTES` per standby (one reassembly buffer) plus `MAX_WAL_PAYLOAD_BYTES × MAX_STANDBYS` on the primary for in-flight chunks. +**Snapshot catch-up flow control**: The `OP_replSnapshot` transfer sends the full snapshot in chunks of `MAX_WAL_PAYLOAD_BYTES` through the SSL endpoint pipeline. The primary sends chunks as fast as the endpoint pipeline allows, and the standby must receive, buffer, and reassemble all chunks before applying the snapshot. The existing iomgr backpressure mechanism (endpoint write FSM, outq buffering, SSL-level flow control via `SSL_write` partial writes) provides implicit flow control: if the standby's receive path is slow, the primary's endpoint outq fills, and subsequent `SSL_write` calls block or return partial writes, naturally throttling the sender. **No additional application-level flow control is needed** beyond the iomgr/SSL pipeline's built-in backpressure. **io_worker pool sizing caveat**: if the io_worker pool is too small, a slow snapshot transfer (256 MB at disk speed, 10–30 seconds) can starve the `OP_replAck` path for healthy standbys — sync replication ACKs may experience queue delays that exceed `sync_timeout_ms`, triggering spurious degraded-mode for job submits that should have been durably replicated. Operators should size the io_worker pool to at least `MAX_STANDBYS + 2` workers (default: `MAX_STANDBYS=3` catch-up endpoints + scheduler + 1 reserve). Add a startup log at `INFO` level that reports the configured io_worker count and the recommended minimum. A P3 improvement would be to rate-limit snapshot chunk sends per-endpoint using a configurable `snapshot_bandwidth_limit_kbps` parameter, leaving headroom for sync replication ACK traffic. However, operators should be aware that a snapshot transfer (up to `MAX_SNAPSHOT_BYTES` = 256 MB) temporarily increases the standby's memory usage for reassembly. The standby allocates a receive buffer of `total_size` bytes (validated against `MAX_SNAPSHOT_BYTES` at first chunk, per Part 6.6) before receiving subsequent chunks — this memory is freed after snapshot application. **CRITICAL: validate `total_size <= MAX_SNAPSHOT_BYTES` BEFORE calling `malloc(total_size)`. Do NOT allocate the buffer and then check — the check is the first operation on first-chunk receipt.** **Aggregate memory budget**: With `MAX_STANDBYS = 3`, up to 3 standbys may catch up concurrently on the primary side. Each primary-side `sendq` for an in-flight snapshot transfer holds at most one pending chunk (`MAX_WAL_PAYLOAD_BYTES ≈ 1 MB`), so the primary's total per-catch-up memory is negligible. On the standby side, only one standby receives a snapshot at a time (a standby can only be catching up from one primary), so the worst-case standby-side allocation is a single `total_size ≤ MAX_SNAPSHOT_BYTES` buffer. The `768 MB = 3 × 256 MB` scenario (all standbys catching up simultaneously) can only occur on the **primary** if it were somehow buffering outbound snapshots for all standbys at once — but the primary streams each snapshot through an already-established SSL endpoint, holding at most `MAX_WAL_PAYLOAD_BYTES` per chunk in the endpoint outq. Therefore the realistic aggregate memory bound is `MAX_SNAPSHOT_BYTES` per standby (one reassembly buffer) plus `MAX_WAL_PAYLOAD_BYTES × MAX_STANDBYS` on the primary for in-flight chunks. **Catch-up trigger**: When a standby reconnects (or initially joins), it includes its `applied_lsn` in the heartbeat to the primary. The primary compares this against its available WAL segments and streams missing entries from `applied_lsn + 1` forward. If the standby's `applied_lsn` is older than the primary's oldest surviving WAL segment, the primary initiates snapshot transfer (see `OP_replSnapshot` framing above) before streaming WAL entries. @@ -2040,18 +2275,24 @@ Worker Controller (Primary) Standby ### 7.4 Degraded Mode -When no standby ACKs within the timeout (`sync_timeout_ms`, default 50-100ms): +When no standby ACKs within the timeout (`sync_timeout_ms`, default 50ms): 1. **Do NOT block the client indefinitely** — timeout is hard 2. **ACK the client based on local WAL durability alone** — the primary's local WAL is already fsynced 3. **Log a warning** — monitoring tooling should alert on degraded replication -4. **Self-heal later** — when the standby reconnects, it will catch up via WAL replay +4. **Self-heal later** — when the standby reconnects, it will catch up via WAL replay or full snapshot transfer (if WAL was trimmed during the degraded window — see Part 6.6 catch-up protocol) **Degraded mode is transient**: In a production cluster, replication degradation only occurs during cluster upgrade, cold boot (standbys not yet connected), or network partitions. A monitoring tool or operator should watch for sustained degraded-mode warnings and investigate immediately. The cluster health system (see Part 5.5 heartbeat monitoring) tracks which standbys are connected and their replication lag. Sustained degraded operation indicates a problem requiring intervention, not a normal operating state. -**Degraded mode timeout**: Replication degradation must not persist indefinitely without operator awareness or intervention. Add a configurable `repl_degraded_timeout_s` to `persist_config_t` (default: 300 seconds). If no standby has acknowledged within this window, the primary: (1) logs an `ERROR` with the duration and the number of submits ACKed in degraded mode, (2) increments `ctrl_repl_degraded_timeout_count`, (3) transitions to **restricted mode**: new `WAL_JOB_SUBMIT` requests receive `E_XCE_REPL_DEGRADED` instead of a normal ACK. Restricted mode persists until at least one standby reconnects and acknowledges the current LSN. This prevents an unbounded accumulation of locally-durable-only submissions that would be lost if the primary itself fails. **Escape hatch**: Operators who accept the risk of sustained degraded operation (e.g., during a planned standby maintenance window) can set `repl_degraded_timeout_s = 0` to disable restricted mode. +> **Out of scope**: `repl_degraded_timeout_s` and restricted mode are removed from scope (see Out of Scope section). Do **not** implement `repl_degraded_timeout_s`, `E_XCE_REPL_DEGRADED`, or restricted-mode state transitions. The text below describes the concept for context only. Monitoring and operator escalation handle sustained degradation. -**Client-visible behavior during split-brain**: API server clients do not receive any explicit error or warning during a split-brain event — the controller continues to ACK job submits normally. However, job submits ACKed by the lower-term controller during a split-brain window are **silently lost** when the partition heals and the lower-term controller steps down (its diverged WAL is discarded). There is no application-level notification to the client. This is the acknowledged risk of the LSF-style election (see Part 5.1 split-brain impact analysis and the required `split_brain_risk_accepted: true` config key). Operators who require at-most-once or exactly-once job submission semantics must either use quorum-based HA (Raft) or implement client-side deduplication using the job's idempotency key. +**Degraded mode timeout** *(concept only — not in scope)*: `repl_degraded_timeout_s` would add a configurable window after which the primary transitions to restricted mode (refusing new `WAL_JOB_SUBMIT` with `E_XCE_REPL_DEGRADED`). This prevents an unbounded accumulation of locally-durable-only submissions that would be lost if the primary itself fails. The escape hatch (`repl_degraded_timeout_s = 0`) would disable restricted mode for planned maintenance windows. + +**Client-visible behavior during split-brain — intentional CAP trade-off**: API server clients do not receive any explicit error or warning during a split-brain event — the controller continues to ACK job submits normally. However, job submits ACKed by the lower-term controller during a split-brain window are **silently lost** when the partition heals and the lower-term controller steps down (its diverged WAL is discarded). There is no application-level notification to the client. + +**This silence is a deliberate consequence of choosing availability over consistency under partition (AP in CAP terminology).** During a network partition, a CP system (like Raft) would refuse to accept writes until quorum is re-established — guaranteeing consistency at the cost of availability. XCE chooses the opposite: the system continues to accept job submits during a partition (availability), accepting that those submits may be lost on healing (consistency sacrifice). Real-time split-brain detection is fundamentally impossible without quorum — any mechanism that detects split-brain in real-time at the client level is equivalent to implementing distributed consensus, which is exactly what Raft provides at ~5000 LOC cost. The `split_brain_risk_accepted: true` config key is the operator's acknowledgment of this CAP position. + +Post-partition, the `ctrl_split_brain_detected` counter and the ERROR-level degraded replication log provide operator visibility. Operators who require at-most-once or exactly-once job submission semantics must either use quorum-based HA (Raft) or implement client-side deduplication using an idempotency key. **Split-brain is unlikely in practice**: In a typical deployment, API servers maintain a persistent connection to a single controller. During a network partition, the API server stays connected to its original controller. Since API servers are not replicated and do not run their own election, they cannot become "split-brained" themselves — they simply become isolated from the primary if the primary fails. The split-brain risk applies to scenarios where multiple API servers connect to different controllers during a partition, which requires a configuration where API servers are load-balanced across controllers. The LSF-style election + redirect mechanism (Part 10.3) ensures that API servers connected to a standby are redirected to the current primary, minimizing the window where an API server could submit to a lower-term controller. @@ -2059,7 +2300,7 @@ When no standby ACKs within the timeout (`sync_timeout_ms`, default 50-100ms): | Setting | Latency Impact | Durability Guarantee | |---------|----------------|---------------------| -| 10 ms | Low — client sees ~10ms extra | Weak — only one standby, might not have fsynced | +| 10 ms | Low — client sees ~10ms extra | Strong if standby disk latency < 10ms; degrades to local-WAL-only if standby cannot fsync within the window (the protocol requires fsync-before-ACK, so any ACK received is already fsynced on the standby) | | 50 ms | Moderate | Good — likely one standby has persisted | | 100 ms | Higher | Strong — high probability all standbys persisted | | No timeout | Unbounded | Strongest — but blocks on standby failure | @@ -2172,24 +2413,46 @@ if (jobid == 0) { } /* Collect jobs into list... */ -int hook_rc = xce_run_hooks(AFTER_EVENT, HOOK_DETAIL_JOB_NEW, c, &jlist); -/* ↑ SYNC replication happens inside this hook call. - * hook_rc != 0 means the persistence hook failed. - * -EIO: local WAL write failed (disk full, permanent — abort submit) - * -ETIMEDOUT: wrote locally but no standby ACKed (transient — degraded durability) */ -if (hook_rc != 0) { - if (hook_rc == -EIO) { - /* Permanent WAL failure — do not ACK to client. - * The job exists in-memory but is not durable. */ - reply.status = -EIO; - reply.data_which = RequestReply_data_nodata; - xce_reply(ep, c->myhost, &reply); - ERROR("job submit: persistence hook failed with -EIO (local WAL write error) — rejecting submit"); - return -EIO; - } - /* hook_rc == -ETIMEDOUT: locally durable but no standby ACK. - * Still reply with success but log the degraded durability. */ - WARN("job submit: persistence hook returned -ETIMEDOUT (no standby ACK within timeout) — accepting with degraded durability"); +/* AFTER_EVENT hook errors are tolerated per XCE hook design: pre-hook (BEFORE_EVENT) + * failures abort the operation; post-hook (AFTER_EVENT) failures are logged but do not + * require transaction rollback. + * + * WAL result is communicated via a PER-REQUEST stack variable passed via xce_run_hooks + * extra argument. Using a shared persist_t field would race: multiple vcpu threads can + * execute handle_jobsubmit() concurrently (one per API server connection), and each + * would overwrite the shared field with its own result, causing thread A to read + * thread B's -EIO and reject a durable job, or vice versa. + * + * Implementation: persist_job_hook_fun() receives &sync_rc via va_list as the last + * argument (after cluster_t* c and the jlist pointer), writes the WAL result there, + * and returns 0. handle_jobsubmit() reads sync_rc after hooks return. */ +int sync_rc = 0; /* per-request WAL result — not shared, no race */ +xce_run_hooks(AFTER_EVENT, HOOK_DETAIL_JOB_NEW, c, &jlist, &sync_rc); +/* ↑ SYNC replication happens inside the persistence hook. + * The hook always returns 0 (AFTER_EVENT errors tolerated at the hook level). + * WAL write result is in sync_rc (stack-allocated, this vcpu thread only): + * 0 — WAL written + fsynced (+ standby ACKed if standbys connected) + * -EIO — local WAL write failed (disk full, permanent) + * -ETIMEDOUT — wrote locally but no standby ACKed within sync_timeout_ms */ +if (sync_rc == -EIO) { + /* Permanent WAL failure — do not ACK to client. + * The job was inserted into kvmgr by job_submit() but is not durable. + * CRITICAL: mark the job terminal so checkpoint GC cleans it up. Without + * this, the job survives every GC pass (job_is_terminal() returns false) + * and accumulates as a phantom entry the client was told doesn't exist. + * job_mark_wal_error_exit(c, jobid); <- sets JOB_STAT_EXIT + finish_time=now + * This makes it eligible for GC after finished_retain_s and prevents + * the scheduler from dispatching work to a nonexistent job. */ + job_mark_wal_error_exit(c, jobid); /* see job.h — sets stat=JOB_STAT_EXIT */ + reply.status = -EIO; + reply.data_which = RequestReply_data_nodata; + xce_reply(ep, c->myhost, &reply); + ERROR("job submit: WAL write failed with -EIO — job %lu marked terminal for GC", jobid); + return -EIO; +} +if (sync_rc == -ETIMEDOUT) { + /* Locally durable but no standby ACK — degraded durability. Accept. */ + WARN("job submit: no standby ACK within timeout — accepting with degraded durability"); } /* Reply AFTER hooks — client ACK implies durability */ @@ -2199,7 +2462,7 @@ reply.data.jobid = jobid; xce_reply(ep, c->myhost, &reply); ``` -**Why this works**: The persistence hook runs inside `AFTER_EVENT`. For job submits, it does sync WAL + replication. Only after the hook returns (meaning the entry is durable) does the controller send the client ACK. This keeps all persistence logic inside hooks — no inline persistence code in `handle_jobsubmit()`. +**Why this works**: AFTER_EVENT hook failures are tolerated (no transaction rollback needed). The persistence hook uses the per-request stack variable `sync_rc` (passed via `va_list` as an extra argument to `xce_run_hooks()`) to communicate WAL write results back to `handle_jobsubmit()` without relying on hook return values. Because `sync_rc` is stack-allocated in `handle_jobsubmit()`, it is thread-private — concurrent vcpu threads each have their own `sync_rc` with no sharing or race conditions. This decouples the sync replication result from the hook system's return-value semantics. **No reply reordering concern**: The controller processes requests from each API server connection one at a time — a request is fully handled (including hooks and `xce_reply()`) before the next request on the same connection is processed. SYNC replication adds latency to the `AFTER_EVENT` hook, which delays the `xce_reply()` for that specific request, but it does not cause out-of-order replies because the next request on that connection doesn't begin processing until the current one completes. Replies to *different* API servers may have different latencies, but each API server sees its own replies in request order. @@ -2213,7 +2476,23 @@ xce_reply(ep, c->myhost, &reply); **Replication scope**: Sync replication (`repl_wait_ack`) only triggers for `WAL_JOB_SUBMIT` entries, which originate from API server connections. The controller only handles connections from workers, scheduler, and API servers — never end users. At 1–3 API server connections, the vcpu pool is not at risk of exhaustion from blocking `repl_wait_ack()` calls. -**Array job sync replication latency bound (CRITICAL)**: Array jobs submit multiple sub-jobs in a single `handle_jobsubmit()` call, each firing an `AFTER_EVENT` hook that triggers sync replication. An array with 10,000 elements would block the vcpu thread for `10,000 × 50ms = 500 seconds` — a denial of service. **Mitigation**: For array submits, the persistence hook must batch the WAL entries and send them in a single `repl_broadcast()` call (or a small number of batched calls) rather than one per sub-job. The sync replication wait (`repl_wait_ack`) should wait for the batch's highest LSN, not per-entry. This bounds array submit latency to `O(num_batches × sync_timeout_ms)` instead of `O(array_size × sync_timeout_ms)`. Implementation: `persist_job_hook_fun()` detects `detail == HOOK_DETAIL_JOB_NEW` with `submit->array_size > 1` and uses batch mode. A reasonable batch size is 100 sub-jobs per sync wait, bounding worst-case latency to `(array_size / 100) × 50ms`. For a 10,000-element array, this is `100 × 50ms = 5 seconds` — acceptable for the submission path. +**Array job sync replication latency bound (CRITICAL)**: Array jobs submit multiple sub-jobs in a single `handle_jobsubmit()` call. With AFTER_EVENT firing once per sub-job (the existing hook dispatch model), 10,000 sub-jobs each triggering `repl_wait_ack()` would block the vcpu for 500 seconds. **Mitigation — accumulation buffer**: For array submits (`submit->array_size > 1`), `handle_jobsubmit()` allocates a `persist_batch_ctx_t` on the heap and passes it as a **second extra va_list argument** (after `&sync_rc`), always NULL for scalar submits: + +```c +/* Scalar submit: sync_rc only (batch_ctx = NULL) */ +int sync_rc = 0; +persist_batch_ctx_t* batch_ctx = NULL; +xce_run_hooks(AFTER_EVENT, HOOK_DETAIL_JOB_NEW, c, &jlist, &sync_rc, batch_ctx); + +/* Array submit: sync_rc + batch_ctx (non-NULL) */ +int sync_rc = 0; +persist_batch_ctx_t* batch_ctx = persist_batch_ctx_alloc(submit->array_size, 100 /* batch_size */); +xce_run_hooks(AFTER_EVENT, HOOK_DETAIL_JOB_NEW, c, &jlist, &sync_rc, batch_ctx); +/* ... iterate sub-jobs, each fires AFTER_EVENT ... */ +free(batch_ctx); +``` + +The hook reads both arguments: `int* sync_rc_p = va_arg(ap, int*); persist_batch_ctx_t* batch = va_arg(ap, persist_batch_ctx_t*);`. When `batch != NULL`, the hook accumulates WAL entries and flushes every 100 sub-jobs. The hook sets `batch->sub_jobs_done++` and writes the final `sync_rc` into `*sync_rc_p` on the last sub-job (`batch->sub_jobs_done == batch->total_sub_jobs`). This bounds latency to `(array_size / 100) × 50ms` — 5 seconds for a 10K array. The accumulation buffer is per-request, not shared, so concurrent submits are fully independent. **Async hook error handling**: For async-replicated events (all events except `WAL_JOB_SUBMIT`), `persist_wal_async()` may return a non-zero error code. These errors fall into two categories: (1) `-EIO` (local WAL write failure — disk full, I/O error) and (2) other errors. For all async events, the caller (e.g., `handle_worker_join()`, `on_job_start()`, `on_job_finish()`) must follow this error handling convention: - **-EIO (local WAL write failure)**: Log an `ERROR` with the event type, LSN (if available), and error code. The in-memory state mutation has already completed — the entity exists in the kvmgr but is not durable on local WAL. The error is not recoverable without operator intervention (disk full or permanent I/O failure). Do NOT undo the in-memory mutation — it is the source of truth for the scheduler and API queries. Do NOT reply to the client with an error for async events (they have no reply expected from the persistence path — `xce_reply()` for async events has already happened before the hook). @@ -2314,7 +2593,7 @@ This pattern is used by etcd (proxy mode), Redis Sentinel, and MongoDB replica s ┌─────────────────┐ │ Controller │ │ Candidates │ - │ (cluster.yaml) │ + │ (--peers flag) │ └────────┬────────┘ │ ┌───────────────┼───────────────┐ @@ -2359,6 +2638,15 @@ int handle_join(cluster_t* c, endpoint_t* ep, xce_request_t* request) case HostRole_apiserver: { xce_reply_t reply; init_reply_by_req(&reply, request); + /* Guard: primary_host is zero-initialized on startup and only populated + * after the standby receives the first heartbeat from the primary. + * If we have never seen a primary (cold start, all controllers starting + * simultaneously), return E_XCE_NOMASTER instead of a garbage redirect. */ + if (c->ha.primary_host[0] == '\0' || c->ha.primary_port == 0) { + reply.status = E_XCE_NOMASTER; + xce_reply(ep, c->myhost, &reply); + return 0; + } reply.status = E_XCE_REDIRECT; reply.data_which = RequestReply_data_redirect; reply.data.redirect.host = c->ha.primary_host; @@ -2376,7 +2664,7 @@ int handle_join(cluster_t* c, endpoint_t* ep, xce_request_t* request) } ``` -**Redirect authentication**: All controller connections use SSL enforced by iomgr (`netio.c`), which means a redirect response can only come from a peer that has completed the SSL handshake with a valid certificate. A rogue endpoint that has not established an SSL connection cannot inject redirect responses. Additionally, the `RedirectInfo` contains only host/port of a known candidate from `cluster.yaml` — the receiving peer validates that the redirect target is in its candidate list before following it. +**Redirect authentication**: All controller connections use SSL enforced by iomgr (`netio.c`), which means a redirect response can only come from a peer that has completed the SSL handshake with a valid certificate. A rogue endpoint that has not established an SSL connection cannot inject redirect responses. Additionally, the `RedirectInfo` contains only host/port of a known candidate — the receiving peer validates that the redirect target is in its candidate list (from `--peers` CLI flag) before following it. ### 9.5 Thundering Herd Mitigation on Failover @@ -2448,20 +2736,25 @@ typedef struct { } xce_candidate_t; typedef struct { - xce_candidate_t candidates[MAX_HA_CANDIDATES]; - int ncandidates; /* Number of candidates configured */ - uint32_t connect_timeout_ms; /* Per-candidate connect timeout (default 2000) */ - uint32_t retry_base_ms; /* Backoff base (default 100) */ - uint32_t retry_max_ms; /* Backoff cap (default 5000) */ - int max_retries; /* Max reconnection attempts (default 10) */ - char ca_cert_path[XCE_MAXPATH_LEN]; /* CA cert for TLS server verification (PEM) */ + char ca_cert_path[PATH_MAX]; /* CA cert path for SSL_VERIFY_PEER (empty = skip) */ +} xce_tls_config_t; + +typedef struct { + xce_candidate_t candidates[MAX_HA_CANDIDATES]; + int ncandidates; /* Number of candidates configured */ + uint32_t connect_timeout_ms; /* Per-candidate connect timeout (default 2000) */ + uint32_t retry_base_ms; /* Backoff base (default 100) */ + uint32_t retry_max_ms; /* Backoff cap (default 5000) */ + int max_retries; /* Max reconnection attempts (default 10) */ + xce_tls_config_t tls; /* TLS options — CA cert path for SSL_VERIFY_PEER. + * Populated from --ca-cert CLI flag (P2.5-5/P2.5-6). */ } xce_cluster_config_t; /* Connect to the cluster — returns SSL* (not raw fd) connected to primary. * The caller must NOT call close() on the returned pointer — use * xce_disconnect_cluster() to tear down the SSL connection and free resources. * All controller connections use SSL enforced by iomgr; this function performs - * the TLS handshake and mTLS peer verification (see P2.5-2). */ + * the TLS handshake (see P2.5-2). */ SSL* xce_connect_cluster(xce_cluster_config_t* conf); /* Connect + join as specific role (worker, scheduler, apiserver) */ @@ -2476,8 +2769,9 @@ int xce_reconnect(SSL** ssl, xce_cluster_config_t* conf); * Safe to call on NULL or already-disconnected SSL*. */ void xce_disconnect_cluster(SSL* ssl); -/* Load cluster config from YAML file */ -int xce_load_cluster_config(const char* yaml_path, xce_cluster_config_t* conf); +/* REMOVED: xce_load_cluster_config() — no YAML parsing in libxce. + * xce_cluster_config_t is populated by caller (worker main, scheduler main, etc.) + * from --peers and --ca-cert CLI flags. */ ``` ### 10.3 Connection State Machine @@ -2569,10 +2863,22 @@ static __thread int rng_init = 0; static void rng_init_once(void) { if (rng_init) return; - /* getrandom is not available per-thread in one syscall, so seed from - * combination of thread ID and time — good enough for jitter purposes. */ - unsigned long tid = (unsigned long)pthread_self(); - rng_seed = (unsigned int)(tid ^ time(NULL)); + /* Seed from getrandom(2) for unpredictable jitter — pthread_self() XOR + * time(NULL) is predictable (deterministic thread IDs, 1-second resolution) + * and could allow an attacker to coordinate thundering-herd attacks. + * getrandom(2) is available on Linux 3.17+; fall back to /dev/urandom. */ + unsigned int seed = 0; +#if defined(__linux__) && defined(SYS_getrandom) + if (syscall(SYS_getrandom, &seed, sizeof(seed), GRND_NONBLOCK) != sizeof(seed)) { + /* getrandom unavailable (e.g., early boot entropy depletion) — use urandom */ + int fd = open("/dev/urandom", O_RDONLY | O_CLOEXEC); + if (fd >= 0) { read(fd, &seed, sizeof(seed)); close(fd); } + else { seed = (unsigned int)((unsigned long)pthread_self() ^ (unsigned long)time(NULL)); } + } +#else + seed = (unsigned int)((unsigned long)pthread_self() ^ (unsigned long)time(NULL)); +#endif + rng_seed = seed; rng_init = 1; } @@ -2605,16 +2911,17 @@ SSL* xce_connect_cluster(xce_cluster_config_t* conf) int redirect_count = 0; for (; attempt < conf->max_retries; attempt++) { -next_candidate: + /* Note: there is no 'goto next_candidate' in this flow — the loop uses + * 'continue' and 'attempt' is intentionally incremented on each redirect + * (redirect_count provides a separate hop cap at 5). */ /* redirect_count is NOT reset per candidate — it is a global hop counter * across the entire connection attempt to prevent redirect loops that cycle * through multiple candidates (e.g., A→B→A). */ int idx = select_candidate(conf, &attempt, last_candidate); xce_candidate_t* cand = &conf->candidates[idx]; - /* TLS-first: establish mTLS session inline. tcp_connect_ssl() - * performs TCP connect + TLS handshake + peer CN verification - * as a single operation. On failure, returns NULL. */ + /* Establish SSL session inline. tcp_connect_ssl() performs + * TCP connect + TLS handshake as a single operation. On failure, returns NULL. */ SSL* ssl = tcp_connect_ssl(cand->host, cand->port, conf->connect_timeout_ms, &conf->tls); if (ssl == NULL) { @@ -2637,7 +2944,7 @@ next_candidate: if (reply.status == E_XCE_REDIRECT) { /* Standby redirected us — validate redirect target before following. - * Only follow to a known candidate from cluster.yaml to prevent + * Only follow to a known candidate from --peers list to prevent * a rogue endpoint from redirecting us to an arbitrary host. */ int target_known = 0; for (int k = 0; k < conf->ncandidates; k++) { @@ -2657,13 +2964,16 @@ next_candidate: } /* Follow redirect — no backoff since this is expected during discovery. - * tcp_connect_ssl already verifies the peer's certificate CN against - * cluster.yaml candidates (CN=ctrl-, OU=ha). A host:port match - * alone is not sufficient — a MITM could intercept the redirect response. - * The TLS handshake with CN validation ensures the session is with the - * genuine controller peer. If CN validation fails, tcp_connect_ssl - * returns NULL and we fall through to try the next candidate. - * See the election probe peer verification (Part 5.4) for the same pattern. */ + * The redirect target must be in the candidate list (validated above). + * Increment redirect_count BEFORE tcp_connect_ssl so the hop count is + * enforced even if every redirect target fails to connect — without this, + * a cycle of unreachable redirect targets would never increment the counter + * and could loop until max_retries (not the redirect_count cap of 5). */ + if (++redirect_count >= 5) { + xce_disconnect_cluster(ssl); + errno = ELOOP; + return NULL; /* EXCE_ERR_REDIRECT_LOOP */ + } xce_disconnect_cluster(ssl); ssl = tcp_connect_ssl(reply.data.redirect.host, reply.data.redirect.port, @@ -2680,13 +2990,9 @@ next_candidate: if (reply.status == 0) { return ssl; /* Confirmed primary */ } - /* Redirect pointed to another standby? Follow again (up to limit) */ - if (++redirect_count >= 5) { - /* Too many redirects — return NULL to caller */ - xce_disconnect_cluster(ssl); - errno = ELOOP; - return NULL; /* EXCE_ERR_REDIRECT_LOOP */ - } + /* redirect_count was already incremented before tcp_connect_ssl; + * if we reach here with another redirect reply, the outer loop + * will re-check the count on the next iteration. */ } xce_disconnect_cluster(ssl); continue; @@ -2746,38 +3052,46 @@ for (;;) { ### 10.8 Candidate List Distribution -All components (controller, worker, scheduler, API server, libxce clients) read the same `cluster.yaml`: +**Controller**: reads candidate configuration from `cluster.bin` (binary format produced by the config tool from `cluster.yaml`). The controller also accepts `--peers` CLI flag for the candidate list: -```yaml -# cluster.yaml — shared across all components -controller: - candidates: - - name: ctrl-1 - host: 10.0.1.1 - port: 7000 - - name: ctrl-2 - host: 10.0.1.2 - port: 7000 - - name: ctrl-3 - host: 10.0.1.3 - port: 7000 +```bash +xce_controller \ + --workdir /var/lib/xce \ + -n ctrl-1 \ + --peers ctrl-1=10.0.1.1:7000,ctrl-2=10.0.1.2:7000,ctrl-3=10.0.1.3:7000 ``` -This file is already read by the controller for its own candidate configuration (Part 5.2). Workers and libxce clients read the same file for discovery. Distribution is an ops concern — shared NFS mount, config management tool (Ansible, Salt), or deployment pipeline. +**Workers, schedulers, and API servers** do NOT read any config file. They accept CLI flags only: -**Candidate list consistency**: A stale `cluster.yaml` on one node could cause split-brain even without a network partition — for example, if a worker's config lists only ctrl-1 while ctrl-1 is down, the worker cannot discover ctrl-2 or ctrl-3 and will fail to connect. Operational practices should ensure atomic config updates (update all nodes before restarting any controller). +```bash +xce_worker \ + --peers ctrl-1=10.0.1.1:7000,ctrl-2=10.0.1.2:7000,ctrl-3=10.0.1.3:7000 \ + --log-level info -**Config freshness control**: To detect stale `cluster.yaml` at runtime, the controller embeds a config version hash in its `OP_heartbeat` and join-reply messages (CRC32 of the serialized candidate list, offset by +1 so 0 is the sentinel for "not sent"). When a worker or API server receives a heartbeat with a mismatched config hash, it logs a warning and reloads `cluster.yaml` from disk. This is a lightweight freshness signal — it does not prevent a stale worker from connecting, but it surfaces divergence quickly so operators can investigate. The config version hash is added as an optional field in the `Request` Cap'n Proto schema (`configVersion @23 :UInt32`). **Sentinel**: 0 means "not sent" (pre-upgrade peer); valid hashes range from 1 to UINT32_MAX. On the receiving end, a `configVersion = 0` is treated as "peer does not support config versioning" and skipped — it is never compared against the local hash. This avoids the ambiguity of CRC32 potentially returning 0 for non-empty inputs. +xce_scheduler \ + --peers ctrl-1=10.0.1.1:7000,ctrl-2=10.0.1.2:7000,ctrl-3=10.0.1.3:7000 \ + --log-level info +``` + +libxce's `xce_cluster_config_t` is populated directly from these CLI flags at startup — no YAML parsing in any component except the config tool. + +**Distribution**: The `--peers` string is generated by the config tool and distributed via deployment scripts (Ansible, Salt) into systemd unit files or container environment variables. Config management tooling should ensure all nodes receive identical candidate lists atomically before any controller restarts. + +**Candidate list consistency**: A stale `--peers` value on one node could cause connectivity failures — for example, if a worker's peers list includes only ctrl-1 while ctrl-1 is down, the worker cannot discover ctrl-2 or ctrl-3. Config management tooling should ensure all nodes receive identical `--peers` updates atomically. + +> **Out of scope**: The config version hash in heartbeat messages is deferred to P3 (see Out of Scope section). Do **not** implement the `configVersion` field, the hash embedding in `OP_heartbeat`, or the reload-on-mismatch logic during P2.5. The description below is retained for future reference only. + +**Config freshness control** *(deferred to P3 — not in scope for P2.5)*: The controller could embed a config version hash (CRC32 of the serialized candidate list, +1 to avoid 0 sentinel) in `OP_heartbeat`. When a worker receives a mismatched hash, it logs a warning (config changes require a restart to take effect since candidates come from CLI flags). The `configVersion @23 :UInt32` Cap'n Proto field in `Request` is reserved for this feature. Do not add this field until P3. ### 10.9 libxce Changes Summary | File | Change | |------|--------| | `src/lib/include/xce/cluster.h` | Add `xce_cluster_config_t`, `xce_candidate_t` types | -| `src/lib/include/xce/cluster.h` | Add `xce_connect_cluster()` (returns `SSL*`), `xce_disconnect_cluster()`, `xce_join_cluster()`, `xce_reconnect()` (takes `SSL**`), `xce_load_cluster_config()` API | -| `src/lib/cluster.c` (new) | Implement candidate selection, redirect following, backoff+jitter, reconnection | -| `src/worker/main.c` | Replace hardcoded controller address with `xce_cluster_config_t`; add reconnection loop; defer worker removal (set `WORKER_UNAVAILABLE` instead of eager delete) | -| `src/scheduler/` | Same pattern as worker — use `xce_cluster_config_t` | +| `src/lib/include/xce/cluster.h` | Add `xce_connect_cluster()` (returns `SSL*`), `xce_disconnect_cluster()`, `xce_join_cluster()`, `xce_reconnect()` (takes `SSL**`) API. No `xce_load_cluster_config()` — callers populate `xce_cluster_config_t` from `--peers`/`--ca-cert` CLI flags. | +| `src/lib/cluster.c` (new) | Implement candidate selection, redirect following, backoff+jitter, reconnection. Accept pre-populated `xce_cluster_config_t` — no config file reading. | +| `src/worker/main.c` | Replace hardcoded controller address with `xce_cluster_config_t`; populate from `--peers` and `--ca-cert` CLI flags; add reconnection loop; defer worker removal (set `WORKER_UNAVAILABLE` instead of eager delete) | +| `src/scheduler/` | Same pattern as worker — parse `--peers` and `--ca-cert` CLI flags, populate `xce_cluster_config_t`, use libxce discovery. | | `src/frontend/apiserver/backend/clib.go` | Use `xce_cluster_config_t` for controller discovery in `RegisterToBackend()` | --- @@ -2800,8 +3114,8 @@ P0 is broken into seven sub-phases with clear deliverables and dependency orderi |------|------|-------------| | P0-0 | `src/controller/job.c` + `worker.c` + `cluster.c` | Remove dead code (Part 1): unused `NJOBS`/`jobid` constants, `load_jobs()` `#if 0` body. Do this first so P0 builds on a clean foundation | | P0-0a | `src/lib/zarray.c` + `include/zarray.h` | Merge `zarray_t` into `xzarray_t` (partially completed per git history) — enables uniform COW for all data structures including endpoints. **Scope**: move the `zarray_t` definition into `xzarray_t`, update all `zarray_*` call sites to use `xzarray_*` equivalents, verify that existing tests still pass. This step is a prerequisite for P0-2 (header move of `xzarray_t`) and P0-8a (deferred endpoint cleanup). **Owner**: this must be completed or explicitly handed off before P0-2 begins; if the merge is not yet complete, P0-8a falls back to the immediate-cleanup path (see Part 4.9 note). **IMPORTANT: P0-0a and P0-0b must be developed on the same branch or merged together before P0-2** — both steps modify `zarray.c` compound-literal initializers and the `xzarray_type` registration. P0-0a renames `zarray_*` functions to `xzarray_*`; P0-0b adds two new slots to `intf_varray_t`. If developed in parallel on separate branches, the initializer updates will conflict. Merge P0-0a first, then P0-0b (which builds on the renamed functions), or develop both on a single P0-A feature branch. | -| P0-0b | `rtti/typeinfo.h` + `zarray.c` + `kvmgr.c` | **rtti library enhancement** (see Appendix A.9): add `retain_fn_t` (`bool (*)(void*, void*)`), `vfreeze_fn_t`, and `vthaw_fn_t` function pointer types to `typeinfo.h`; grow `intf_varray_t` from 6 to 8 slots (`vfreeze`, `vthaw`); add `vfreeze()`/`vthaw()` global API wrappers to `typeinfo.h`; update all compound-literal `intf_varray_t` initializers (xzarray in `zarray.c`, kvmgr in `kvmgr.c`, kvtab in `kv.c`) to include the two new slots — use designated-initializer style `{ .vstore = ..., .vfreeze = (vfreeze_fn_t)NULL, .vthaw = (vthaw_fn_t)NULL }` for non-xzarray types; update nix derivation version from `rtti-0.0.1` to `rtti-0.1.0`. This is a prerequisite for P0-2 which wires `vfreeze`/`vthaw` to the xzarray implementations. Requires P0-0a to be merged first (same-branch constraint above). | -| P0-5 | `src/include/xce/cluster_def.h` + `include/config.h` | Add `persist_t persist;` to `cluster_t` — must precede P0-1 through P0-4 which reference `c->persist`. **Note**: `ha_state_t ha;` is added in P1-5 (HA wiring) — it is not needed in P0 since no P0 codepath references `c->ha`. P0-5 adds only `persist_t persist;`. Also create `include/config.h` with all system-level tunable constants (see Appendix A.10). | +| P0-0b | `rtti/typeinfo.h` + `zarray.c` + `kvmgr.c` + `kv.c` | **rtti library enhancement** (see Appendix A.9): **Check the installed rtti header first** — `vfreeze_fn_t`, `vthaw_fn_t`, and `retain_fn_t` may already be present in the installed `/nix/store/...-rtti-0.1.0/include/rtti/typeinfo.h` from a prior pass. Verify with `grep vfreeze_fn_t $(nix eval --raw .#rtti)/include/rtti/typeinfo.h`. **If the types are already present**, skip the header additions and go directly to the zarray.c implementation work. **If not present**: add `retain_fn_t` (`bool (*)(void*, void*)`), `vfreeze_fn_t`, and `vthaw_fn_t` to `typeinfo.h`; grow `intf_varray_t` from 6 to 8 slots; add `vfreeze()`/`vthaw()` global API wrappers. **Remaining work in all cases** (whether or not header types were already present): (1) add `shadow` and `shadow_caps` fields to `xzarray_private_t` in `zarray.c`; (2) update all compound-literal `intf_varray_t` initializers in `zarray.c`, `kvmgr.c`, and `kv.c` to include the two new slots — **first convert all positional initializers to designated-initializer style** (`{ .vstore = ..., .vfreeze = (vfreeze_fn_t)NULL, .vthaw = (vthaw_fn_t)NULL }`) before adding the new slots, since adding members to a struct with positional initializers is a compile error. Run `grep -rn 'intf_varray_t' src/` to find all initializer sites before starting. Update nix derivation version from `rtti-0.0.1` to `rtti-0.1.0`. This is a prerequisite for P0-2 which wires `vfreeze`/`vthaw` to the xzarray implementations. Requires P0-0a to be merged first (same-branch constraint above). **rtti source**: the rtti library source is maintained by the XCE team (not a third-party dependency). The implementer has direct access to modify `typeinfo.h` and bump the Nix derivation. Locate the source via `nix flake show` or the project's `flake.nix` inputs. **BREAKING ABI CHANGE**: both `intf_varray_t` (grows from 6 to 8 pointers) and `xzarray_private_t` (gains `shadow` and `shadow_caps` fields, changing `privateSize`) change size. Since no version of XCE has been released, backward binary compatibility is not required — all components are rebuilt together. Trigger a full `ninja -C build` clean rebuild after this step. | +| P0-5 | `src/include/xce/cluster_def.h` + `include/config.h` | Add `persist_t persist;` to `cluster_t`. **Pre-addition audit**: run `grep -rn 'cluster_t\|sizeof.*cluster\|offsetof.*cluster' src/` and verify no code assumes `sizeof(cluster_t)` or uses field offsets before adding the new field — adding a large struct may shift subsequent field offsets. Note `c->endpoints` is currently `zarray_t endpoints` (not `xzarray_t`); the deferred endpoint cleanup (P0-8b) requires a type change in P0-0a. **Note**: `ha_state_t ha;` is added in P1-5 (HA wiring) — it is not needed in P0 since no P0 codepath references `c->ha`. P0-5 adds only `persist_t persist;`. Also **augment** the existing `include/config.h` (do NOT create a new file — the file already exists with XCE_MAXNAME_LEN and job ID bit fields) with all system-level tunable constants (see Appendix A.10): add `MAX_STANDBYS=3`, `MAX_HA_CANDIDATES=4`, `WAL_FORMAT_VERSION=1`, `WAL_ENTRY_TYPE_MAX=100`, `MAX_WAL_PAYLOAD_BYTES=(1<<20)`, `MAX_SNAPSHOT_BYTES=(256*1024*1024)`, `DEFAULT_WAL_SYNC_MS=10`, `DEFAULT_SNAP_INTERVAL_S=60`, etc. | --- @@ -2811,9 +3125,10 @@ P0 is broken into seven sub-phases with clear deliverables and dependency orderi | Step | File | Description | |------|------|-------------| -| P0-1 | `src/controller/persist.c` + `include/persist.h` | WAL writer, segment management, fsync batching. `persist_init()` must validate `wal_path` and `snap_path` against path traversal using `realpath()` — reject any path that resolves outside the intended base directory or contains `..` components — then create directories with `mode 0700` (owner-read-only) if they do not exist, and **verify permissions on existing directories at startup — refuse to start if `wal_path` or `snap_path` is not mode 0700 and owned by the controller UID** (rather than log-and-continue). Job payloads written to WAL/snapshot files may include environment variables and command lines containing credentials; this permission gate prevents a misconfigured mount from exposing them. **Dedicated fsync thread required**: `wal_sync_ms` defaults to 10ms; `delaymgr` has 2-second granularity (`sleep(precision)` where `precision=2`) so the WAL fsync timer must be a dedicated `pthread` using `nanosleep()` or `timerfd_create(CLOCK_MONOTONIC)` — do not use `delaymgr` for this. **Encryption note**: WAL and snapshot files are stored **unencrypted** by the controller. `persist_init()` must log an `INFO` message at startup: `"WAL/snapshot stored unencrypted; ensure wal_path and snap_path reside on an encrypted filesystem (e.g., LUKS/dm-crypt) if job payloads contain sensitive data"`. File-level encryption (e.g., AES-GCM per-record) is out of scope for this design. | -| P0-2 | `src/lib/zarray.c` + `include/zarray.h` | COW extensions: `shadow`, `shadow_caps`, `xzarray_freeze()`, `xzarray_thaw()`; modify `xzarray_iter_next()` for shadow-aware merge walk (see Part 3.5); modify `xzarray_freeze()` to set `pos = shadow_caps` (see Part 3.5 vstore invariant). **Header move required**: `xzarray_t` is currently defined as a file-private struct in `zarray.c`; it must be moved to `include/zarray.h` so `persist.c`, `repl.c`, and other callers can embed or reference it. `xzarray_private_t` (the internal struct containing `mylock_t`) is also currently file-private in `zarray.c` and must be moved to `include/zarray.h` as well — it is embedded inside `xzarray_t` and must be visible wherever `xzarray_t` is used. Without both moves, COW callers cannot access `shadow`/`shadow_caps` fields or embed the struct. **Requires: P0-0a** — P0-0a merges `zarray_t` into `xzarray_t`; P0-2 must not begin until P0-0a is merged and tested. Both touch the same file pair (`zarray.c` + `include/zarray.h`), so they must be serialized. | +| P0-1 | `src/controller/persist.c` + `include/persist.h` | WAL writer, segment management, fsync batching. `persist_init()` receives workdir and name parameters (from CLI `--workdir` and `-n` flags). It derives `wal_path = effective_dir/data/wal` and `snap_path = effective_dir/data/snap` (where `effective_dir = workdir / name`). Must validate derived paths against path traversal using `realpath()` — reject any path that resolves outside the intended base directory or contains `..` components — then create directories with `mode 0700` (owner-read-only) if they do not exist, and **verify permissions on existing directories at startup — refuse to start if derived wal_path or snap_path is not mode 0700 and owned by the controller UID** (rather than log-and-continue). Job payloads written to WAL/snapshot files may include environment variables and command lines containing credentials; this permission gate prevents a misconfigured mount from exposing them. **Runtime permission re-check**: `persist_checkpoint()` (P0-3) must also call `fstatat(dirfd(wal_dir), ".", &st, 0)` and verify `(st.st_mode & 0777) == 0700 && st.st_uid == getuid()` before writing each new snapshot. Log `ERROR` and skip the snapshot write if the check fails — a post-startup permission change (e.g., misconfigured backup agent `chmod`) is caught within one checkpoint interval (default 60s) rather than persisting until the next restart. Add `persist_t.wal_dirfd` and `persist_t.snap_dirfd` (open directory file descriptors) so the check uses `fstatat()` rather than `stat()` to avoid TOCTOU race against path traversal. **Dedicated fsync thread required**: `wal_sync_ms` defaults to 10ms; `delaymgr` has 2-second granularity (`sleep(precision)` where `precision=2`) so the WAL fsync timer must be a dedicated `pthread` using `nanosleep()` or `timerfd_create(CLOCK_MONOTONIC)` — do not use `delaymgr` for this. **Encryption note**: WAL and snapshot files are stored **unencrypted** by the controller. `persist_init()` must log an `INFO` message at startup: `"WAL/snapshot stored unencrypted; ensure effective_dir/data/wal and effective_dir/data/snap reside on an encrypted filesystem (e.g., LUKS/dm-crypt) if job payloads contain sensitive data"`. File-level encryption (e.g., AES-GCM per-record) is out of scope for this design. | +| P0-2 | `src/lib/zarray.c` + `include/zarray.h` | COW extensions: `shadow`, `shadow_caps`, `xzarray_freeze()`, `xzarray_thaw()`; modify `xzarray_iter_next()` for shadow-aware merge walk (see Part 3.5); modify `xzarray_freeze()` to set `pos = shadow_caps` (see Part 3.5 vstore invariant). **Header move required**: `xzarray_t` is currently defined as a file-private struct in `zarray.c`; it must be moved to `include/zarray.h` so `persist.c`, `repl.c`, and other callers can embed or reference it. `xzarray_private_t` (the internal struct containing `mylock_t`) is also currently file-private in `zarray.c` and must be moved to `include/zarray.h` as well. **`_Atomic _Bool` inuse audit (mandatory)**: changing `xzarray_item_t.inuse` from `unsigned int :1` to `_Atomic _Bool` (required by Part 3.5 memory ordering) affects every read and write of this field in `zarray.c`. Run `grep -n '\.inuse' src/lib/zarray.c` before starting — expect ~15 call sites across `xzarray_destroy`, `xzarray_store`, `xzarray_get`, `xzarray_clear`, `xzarray_iter_next`, and the new COW paths. Every read must use `atomic_load_explicit(..., memory_order_acquire)` and every write must use `atomic_store_explicit(..., memory_order_release)`. This is a checklist item — do not mark P0-2 complete until all sites are converted and verified. **Requires: P0-0a** — P0-0a merges `zarray_t` into `xzarray_t`; P0-2 must not begin until P0-0a is merged and tested. | | P0-2a | `src/lib/zarray.c` | COW lock protocol: verify xzarray `mylock_t` mutex is taken correctly inside vget/vput during COW mode; kvmgr `my_rwlock_t` is the outer coordination lock (see Part 3.9) | +| P0-2b | `src/controller/delay.c` + `src/controller/delay.h` + all callers | **delaymgr precision enhancement** (prerequisite for P0-3): rename `precision` field to `precision_ms` (milliseconds); replace `sleep(precision)` with `struct timespec ts = {0, precision_ms * 1000000L}; nanosleep(&ts, NULL);`. **Important**: `delay_task()` delay parameter remains in **seconds** — only the sleep granularity changes. Audit all `delaymgr_init()` call sites (`grep -rn 'delaymgr_init' src/`) and update each to pass `precision_ms = 200`. Verify no existing `delay_task()` call passes seconds-based values that assume the old 2-second tick — the time_t arithmetic (`ts_nextrun = now + delay`) is unchanged. Update `cluster.c:604` from `delaymgr_init(&c->delaymgr, 2, run_delay_tasks)` to `delaymgr_init(&c->delaymgr, 200 /* precision_ms */, run_delay_tasks)`. | --- @@ -2823,7 +3138,7 @@ P0 is broken into seven sub-phases with clear deliverables and dependency orderi | Step | File | Description | |------|------|-------------| -| P0-3 | `src/controller/persist.c` | Checkpoint via dual zarray freeze → serialize shadow → thaw + GC; use `delaymgr` for periodic checkpoint timer (see Part 3.10). **`checkpoint_requested` inline trigger polling**: The checkpoint timer thread also polls `persist_t.checkpoint_requested` on each timer wake. Since `delaymgr` has 2-second granularity and the inline trigger needs sub-second responsiveness, the timer callback is registered with a 1-second interval via `delay_task(&c->delaymgr, 1, -1, persist_checkpoint_timer_fn, c)`. Each callback checks `checkpoint_in_progress` first (skip if set), then `checkpoint_requested` (run checkpoint if set), then `snap_interval_s` countdown (run periodic checkpoint when countdown reaches zero). This combines three concerns into a single timer callback without requiring a separate pthread for checkpoint scheduling. The WAL fsync timer (10ms intervals) and HA heartbeat watchdog both require dedicated pthreads — they cannot use `delaymgr`. | +| P0-3 | `src/controller/persist.c` | **Requires: P0-2b** (delaymgr precision enhancement — 1-second checkpoint poll requires nanosleep granularity). Checkpoint via dual zarray freeze → serialize shadow → thaw + GC; use `delaymgr` for periodic checkpoint timer (see Part 3.10). **`checkpoint_requested` inline trigger polling**: The checkpoint timer thread also polls `persist_t.checkpoint_requested` on each timer wake. The timer callback is registered with a 1-second interval via `delay_task(&c->delaymgr, 1, -1, persist_checkpoint_timer_fn, c)`. Each callback checks `checkpoint_in_progress` first (skip if set), then `checkpoint_requested` (run checkpoint if set), then `snap_interval_s` countdown (run periodic checkpoint when countdown reaches zero). The WAL fsync timer (10ms intervals) and HA heartbeat watchdog both require dedicated pthreads — they cannot use `delaymgr`. | | P0-4 | `src/controller/persist.c` | Recovery: load snapshot + replay WAL. Includes: (1) snapshot version check — reject snapshots where `SnapshotHeader.version > WAL_FORMAT_VERSION` (see Part 2.5 step 2a); (2) Cap'n Proto traversal limit — set `ReaderOptions.traversalLimitInWords` to the fixed upper bound `MAX_SNAPSHOT_BYTES / 8` words (see Part 2.5 step 2b; do NOT use `file_size / 8`); (3) snapshot CRC verification over both header (`headerChecksum` covering lsn, term, version, counts) and body (`checksum`); (4) WAL entry type max guard — abort if any entry type exceeds `WAL_ENTRY_TYPE_MAX` (see Part 2.5). | | P0-12a | `job.c` | Implement `restore_job(cluster_t* c, uint64_t jobid, const void* capnp_payload)` — inserts a job at its original jobid without allocating a new one. Used by `apply_wal_entry()` for `WAL_JOB_SUBMIT` during recovery and standby replay. Unlike `job_submit()`, `restore_job()` must accept an explicit jobid, bypass the jobid allocator, and skip BEFORE/AFTER_EVENT hooks. **Duplicate jobid handling**: if the jobid already exists in the kvmgr (e.g., from a duplicate WAL entry or a corrupt WAL that replays the same submit twice), `restore_job()` must log a `WARN` and return `-EEXIST` without overwriting the existing entry. `apply_wal_entry()` treats `-EEXIST` as a non-fatal skip. Required before P0-4 (recovery replay) and P2-2 (standby apply). **Note on `-EEXIST` occurrence**: During clean recovery (snapshot + WAL replay from a trimmed WAL), `-EEXIST` should never occur — the snapshot represents a consistent point-in-time state, and WAL entries after the snapshot's LSN contain only new state changes. `-EEXIST` is a safety guard for crash-recovery edge cases where: (1) a checkpoint failed mid-write but the WAL was not yet trimmed, causing the recovery algorithm to replay entries already present in the snapshot; or (2) a standby receives a duplicate replication entry due to reconnection. If `-EEXIST` occurs during what should be a clean recovery, it indicates a WAL trim bug or snapshot/WAL inconsistency and should be investigated. | | P0-12 | `worker.c` | Wire `restore_worker()` into recovery path. **Adapter requirement**: the existing `restore_worker(void* cls, xce_worker_t host)` takes `xce_worker_t` by value (struct copy), but Cap'n Proto deserialization produces a pointer-based `Worker` struct. Implement a wrapper `worker_from_capnp()` that: (1) receives the deserialized `Worker` Cap'n Proto object, (2) allocates and populates an `xce_worker_t` by deep-copying pointer fields (`char* name` → strdup, `xce_config_t** resources` → allocate and copy each config item, `xce_res_threshold_t** thresholds` → same), (3) calls `restore_worker(cls, host)` with the populated struct. The existing `restore_worker()` function signature is unchanged — the adapter sits between Cap'n Proto and `restore_worker()`. | @@ -2836,10 +3151,10 @@ P0 is broken into seven sub-phases with clear deliverables and dependency orderi | Step | File | Description | |------|------|-------------| -| P0-4a | `src/include/xce/cluster_def.h` + `src/controller/job.c` | Add `HOOK_DETAIL_JOB_STOP` and `HOOK_DETAIL_JOB_RESUME` to the hook detail enum; update `on_job_stop()` to fire `HOOK_DETAIL_JOB_STOP` (currently fires `HOOK_DETAIL_JOB_EXIT`) and `on_job_resume()` to fire `HOOK_DETAIL_JOB_RESUME` — prerequisite for P0-6 WAL_JOB_STOP and WAL_JOB_RESUME hook wiring. **Audit required**: grep all call sites in `job.c` and `worker.c` that invoke `xce_run_hooks` with `HOOK_DETAIL_JOB_EXIT` to confirm that only genuinely terminal events (job done/fail/error) remain on that hook detail; any call site that fires `HOOK_DETAIL_JOB_EXIT` for a non-terminal event (stop, resume) must be updated. | -| P0-5a | `src/include/xce/cluster_def.h` + `types.h` | Add entity terminal states: `XCE_WORKER_UNAVAIL` (already exists in `types.h:165`), `XCE_WORKER_HIDDEN`, `QUEUE_STAT_HIDDEN`; add `JOB_STAT_TRANSIENT = 0x400` to job status bitmask (**must not use `0x200`** — `JOB_STAT_WAIT` already occupies `0x200`, see `types.h:351`). This fits in the existing `uint32_t state` field (current highest bitmask is `0x40000` for `JOB_STAT_ERR`). The scheduler must skip workers that own transient jobs (no new work dispatched to them). API queries must show transient jobs as status `unknown` or `transient`. Worker reconnection resolves the transient state by setting the job's actual terminal/running status (the worker reports job completion/exit). If the worker never reconnects, the job remains transient until operator intervention (same as existing `WORKER_UNAVAILABLE` behavior for workers). | +| P0-4a | `src/include/xce/cluster_def.h` + `src/controller/job.c` | Add `HOOK_DETAIL_JOB_STOP` and `HOOK_DETAIL_JOB_RESUME` to the hook detail enum. **Critical codebase fact**: `on_job_stop()` and `on_job_resume()` in `job.c` currently call `mark_job_update()` which fires `HOOK_DETAIL_JOB_UPDATE` — they do NOT currently fire `HOOK_DETAIL_JOB_EXIT`. (Footnote 1 in Part 2.3 incorrectly states they fire `HOOK_DETAIL_JOB_EXIT`.) The audit directive must search for `mark_job_update` calls in the stop/resume paths, not `HOOK_DETAIL_JOB_EXIT`. The implementation must update `on_job_stop()` and `on_job_resume()` to call `xce_run_hooks` directly with `HOOK_DETAIL_JOB_STOP` / `HOOK_DETAIL_JOB_RESUME` (bypassing `mark_job_update()` for those events). **Audit**: `grep -n 'mark_job_update\|HOOK_DETAIL_JOB' src/controller/job.c` — confirm all non-terminal paths are correctly remapped. This is a prerequisite for P0-6 WAL_JOB_STOP and WAL_JOB_RESUME hook wiring. | +| P0-5a | `src/include/xce/cluster_def.h` + `types.h` | Add entity terminal states: `XCE_WORKER_UNAVAIL` (already exists in `types.h:165`), `XCE_WORKER_HIDDEN`, `QUEUE_STAT_HIDDEN`; add `JOB_STAT_TRANSIENT = 0x400` to job status bitmask (**must not use `0x200`** — `JOB_STAT_WAIT` already occupies `0x200`, see `types.h:351`). This fits in the existing `uint32_t state` field (current highest bitmask is `0x40000` for `JOB_STAT_ERR`). The scheduler must skip workers that own transient jobs (no new work dispatched to them). API queries for transient jobs must NOT return the job's last-known state (e.g., `PEND`) as that would actively mislead users who submitted running jobs. The Go API server mapping of `JOB_STAT_TRANSIENT` to a user-visible `TRANSIENT` response is tracked separately — see P2.5-7 (or a dedicated API server story). Worker reconnection resolves the transient state by setting the job's actual terminal/running status (the worker reports job completion/exit). If the worker never reconnects, the job remains transient until operator intervention (same as existing `WORKER_UNAVAILABLE` behavior for workers). | | P0-6 | `src/controller/persist.c` | Hook functions: `persist_job_hook_fun`, `persist_queue_hook_fun`, `persist_worker_hook_fun` (requires P0-4a for `HOOK_DETAIL_JOB_STOP` and `HOOK_DETAIL_JOB_RESUME` wiring) | -| P0-7 | `src/controller/job.c` | Move `xce_reply()` after `AFTER_EVENT` hooks in `handle_jobsubmit()`; verify no kvmgr lock held during hooks (see Part 8.3 lock safety analysis) | +| P0-7 | `src/controller/job.c` + all handler files | Move `xce_reply()` after `AFTER_EVENT` hooks in `handle_jobsubmit()`; verify no kvmgr lock held during hooks (see Part 8.3 lock safety analysis). **Full audit required**: grep for all `xce_run_hooks(AFTER_EVENT` + `xce_reply` pairs across all handler files (`job.c`, `worker.c`, `join.c`, `cluster.c`, etc.) to identify any handler that still replies before post-hooks. **va_list compatibility audit**: P0-6 adds `persist_job_hook_fun()` as an AFTER_EVENT hook for `HOOK_DETAIL_JOB_NEW` that reads two extra `va_list` arguments (`int* sync_rc` and `persist_batch_ctx_t*`). All existing registered AFTER_EVENT `HOOK_DETAIL_JOB_NEW` hooks (e.g., in `apiserver.c`, `sched.c`) must be audited to verify they do NOT call `va_arg()` for these extra arguments (they should only consume the arguments they were registered to expect). Document for each: (a) does the hook call `va_arg()`, (b) for which arguments, (c) whether adding the new args shifts its position. If any existing hook calls `va_arg()` past the standard args, it must be updated or given its own hook detail to avoid type mismatches. For async-replicated events, reply ordering is less critical, but consistency is preferred. Document each handler's reply ordering decision and rationale as part of this step. | --- @@ -2850,9 +3165,11 @@ P0 is broken into seven sub-phases with clear deliverables and dependency orderi | Step | File | Description | |------|------|-------------| | P0-8 | `src/controller/worker.c` | Extend existing `mark_worker_unavail()` to set `w->host.dead_since = time(NULL)` (field already exists in `xce_worker_t`); add `XCE_WORKER_HIDDEN` bitmask path; already deferred — no `zarray_del` removal needed (see Part 4.10) | -| P0-8a | `src/controller/cluster.c` | Replace `zarray_del()` at line ~85 and `zarray_clear()` at line ~322 in `handle_endpoint_error()` and endpoint finalization with deferred endpoint cleanup: close fd/SSL immediately, set `ENDPOINT_DISCONNECTED` state, cleanup slot at checkpoint (requires P0-0a zarray merge to enable COW for endpoints; until then, keep existing `zarray_del()`/`zarray_clear()`) | +| P0-8a | `src/controller/cluster.c` | **Fallback path (no dependency on P0-0a)**: Keep the existing `zarray_del()`/`zarray_clear()` calls in `handle_endpoint_error()` — this is the safe baseline that ships with P0-E. Mark these call sites with `/* TODO P0-8b: replace with deferred cleanup after P0-0a */` comments so they are easy to find. P0-8a is complete when these comment markers are in place. | +| P0-8b | `src/controller/cluster.c` | **Deferred cleanup path — requires P0-0a to be merged and green first.** Replace the `TODO P0-8b` sites from P0-8a: close fd/SSL immediately, set `ENDPOINT_DISCONNECTED` state, clean up the slot at checkpoint via `endpoint_should_retain()`. P0-8b must not be started until P0-0a is merged and all `zarray_t` → `xzarray_t` migration tests pass. This ordering gate prevents the deferred path from shipping before the zarray infrastructure is ready. | | P0-13 | `job.c` | Implement `job_is_terminal()` (check `JOB_STAT_EXIT | JOB_STAT_DONE | JOB_STAT_PERR` bitmask) and `job_has_dependents()` (check pending_ctrl / array dependency tracking) — required by Part 4.5 retention callbacks | | P0-14 | `kvmgr.c` + `htab.c` | Implement `kvtab_gc_stale_indices()` — walk hash chains, remove entries whose mapped zarray index has `inuse=0`. Called during thaw while kvmgr write lock is held. Internally uses `ht_del()` (not `del_key` — remains NULL per Part 4.1 invariant) | +| P0-14a | `kvmgr.c` | Implement `kvmgr_capture_keys(meta_obj_t* kvmgr, slot_key_t** keys_out, size_t* n_keys_out)` — walks the kvtab under caller-held write lock and deep-copies each key string into a malloc'd `slot_key_t[]` array. Called by `persist_checkpoint()` in Part 3.8 immediately after `xzarray_freeze()` while the write lock is still held. **Depends on**: P0-14 (htab walk primitives) and P0-0b (`slot_key_t` struct). **Required before**: P0-3 (checkpoint) can be tested end-to-end. The serialization pthread uses the captured key array to build Cap'n Proto key→index mappings without holding any lock. `free(keys)` releases the array; individual key strings are freed by the keys array destructor. | --- @@ -2862,19 +3179,25 @@ P0 is broken into seven sub-phases with clear deliverables and dependency orderi | Step | File | Description | |------|------|-------------| -| P0-9 | `controller.capnp` | Add `PersistConfig`, `WALEntry`, `SnapshotHeader`, `Snapshot` schemas; add `version` field (`UInt32`, default `1`) to `SnapshotHeader` for format version detection; add `deadSince` to `Worker` schema; add bitmask flags for `XCE_WORKER_HIDDEN`, `QUEUE_STAT_HIDDEN`; add `jobStatTransient = 0x400` constant comment in Job schema (`status` remains `UInt32` bitmask; **not** `0x200` which is `JOB_STAT_WAIT`) | -| P0-10 | `cluster.c` | Wire `persist_init/recover/start/stop/fini` into `cluster_init/run/fini` | +| P0-9 | `controller.capnp` | Add `PersistConfig`, `WALEntry`, `SnapshotHeader`, `Snapshot` schemas; add `version` field (`UInt32`, default `1`) to `SnapshotHeader` for format version detection; add `snapKeepN @6 :UInt32 = 2` to `PersistConfig` (see Appendix A.5); add `transientJobTimeoutS @7 :UInt32 = 300` to `PersistConfig` (see Appendix A.5 and `persist_config_t.transient_job_timeout_s`); add `transientSince` (UInt64, epoch seconds) to the `Job` schema — set when a job enters `JOB_STAT_TRANSIENT` at promotion; needed for the `transient_job_timeout_s` GC check; add `deadSince` to `Worker` schema; add bitmask flags for `XCE_WORKER_HIDDEN`, `QUEUE_STAT_HIDDEN`; add `jobStatTransient = 0x400` constant comment in Job schema (`status` remains `UInt32` bitmask; **not** `0x200` which is `JOB_STAT_WAIT`) | +| P0-10 | `cluster.c` | Wire `persist_init/recover/start/stop/fini` into `cluster_init/run/fini`. **Requires: P0-9** — `PersistConfig` and the snapshot schemas must be defined in `controller.capnp` before `cluster.c` can reference them via the generated C bindings. | | P0-11 | `myconf.c` + YAML config | Parse persistence configuration | --- -#### P0-G: Integration Tests +#### P0-G: End-to-End Smoke Tests -**Deliverable**: Full persistence test suite. **Requires: P0-F**. +**Deliverable**: Integrated persistence test suite exercising cross-sub-phase interactions. **Requires: P0-F**. + +> **Gate structure note**: These tests run AFTER all P0 sub-phases (P0-A through P0-F) are complete. They are end-to-end integration tests, not per-sub-phase gates. For per-sub-phase incremental verification, each sub-phase's deliverable should be tested immediately after it lands (see the "each sub-phase produces a buildable, testable increment" principle). P0-G tests validate the full assembled system. | Step | File | Description | |------|------|-------------| -| P0-15 | Tests | WAL write/read round-trip, COW freeze/serialize/thaw, checkpoint+recover, hook invocation, retention/GC, endpoint deferred cleanup | +| P0-15a | Tests (WAL E2E) | WAL write/fsync/CRC round-trip: write entries, simulate crash (truncate), recover and verify correct replay up to last valid CRC | +| P0-15b | Tests (Checkpoint E2E) | Checkpoint + recovery: freeze → serialize → thaw cycle; snapshot-to-disk; restart from snapshot + bridging WAL; verify entity counts match | +| P0-15c | Tests (Hook E2E) | Hook invocation ordering: BEFORE_EVENT abort path, AFTER_EVENT sync_rc propagation, reply-after-hook sequencing; xce_reply fires after hook returns | +| P0-15d | Tests (GC E2E) | Retention/GC: terminal entities survive until TTL expires, cleaned at checkpoint; deferred endpoint cleanup; JOB_STAT_TRANSIENT auto-terminate path | +| P0-15e | Tests (Cap'n Proto E2E) | Cap'n Proto partial-merge semantics: absent optional fields are NOT zeroed; partial WAL entry applies only stated fields to in-memory struct | ### P1: HA Foundation (LSF-Style Election) @@ -2882,10 +3205,11 @@ P0 is broken into seven sub-phases with clear deliverables and dependency orderi | Step | File | Description | |------|------|-------------| -| P1-1 | `src/controller/netio.c` + `src/include/xce/netio.h` | Extend `create_ssl_context()` to accept `ca_cert_path` parameter for mTLS peer verification; update all existing callers to pass NULL for the new parameter (backward compat not needed — no released version). See Part 5.9. **GATE: No replication channel code (P2) may be written or deployed before P1-1 is merged.** The HA replication channel must use mTLS for peer authentication — without this, any process that can complete a TLS handshake could inject arbitrary WAL entries via `OP_replEntry`. | -| P1-2 | `src/controller/ha.c` + `include/ha.h` | Role management, `am_i_primary()`, promotion/demotion. **OP_steppingDown handler**: on receiving `OP_steppingDown` from the primary, the highest-priority connected standby immediately calls `ha_promote_to_primary()`. The handler validates: (1) the sender's term matches `ha->term` (stale stepping-down messages are ignored), (2) the sender is the current primary (verified via `ha->primary_host`/`ha->primary_port`). Non-highest-priority standbys that receive `OP_steppingDown` do NOT promote — they wait for the new primary's heartbeat. The handler runs in the iomgr/vtask thread (standard message dispatch path). | -| P1-3 | `src/controller/ha.c` | Heartbeat watchdog: `OP_heartbeat` exchange via iomgr/vtask pipeline; `ha_check_heartbeat()` on a **dedicated `pthread` with `nanosleep()` or `timerfd_create(CLOCK_MONOTONIC)`** (NOT `delaymgr` — see Part 5.5 heartbeat detection latency note) detects primary failure (replaces `tcp_probe`) | -| P1-3a | `src/controller/ha.c` | Promotion sequence: stop replay, increment term, recover, start primary; SIGUSR1 handler for manual failover (see Part 5.8) | +| P1-1 | `src/controller/ha.c` + `src/controller/netio.c` | Verify that HA listeners (election probe, replication accept) reuse the existing iomgr/SSL pipeline without modification. No `create_ssl_context()` API changes needed — the trusted LAN model (Part 5.9) requires no mTLS. Peer identity validated by host:port match against `--peers` candidate list. | +| P1-2a | `src/lib/hook.c` + `src/lib/include/xce/hook.h` | **Prerequisite for P1-2.** Add `xce_disable_hook(hooksys_t* hs, int type, hook_fn_t fn)` — scans for a matching entry by `(type, fn)` and stores NULL atomically with `atomic_store_explicit(memory_order_release)`. Add `void xce_disable_hook(...)` to `hook.h`. **Struct change**: change `hook_item_t.fn` from `hook_fn_t fn;` to `_Atomic hook_fn_t fn;` — required so `xce_disable_hook()` can store NULL with release ordering and `xce_run_hooks()` can load with acquire into a local before NULL-check and call. Update `xce_add_hook()` to use `atomic_store_explicit(&it->fn, fn, memory_order_release)`. **NULL-fn guard in `xce_run_hooks()`**: after loading `hook_fn_t local_fn = atomic_load_explicit(&it->fn, memory_order_acquire)`, add `if (!local_fn) continue;` — call `local_fn(...)` instead of `it->fn(...)`. **Slot reuse**: `xce_add_hook()` scans for existing NULL-fn slots of the same `type` before appending — prevents unbounded list growth on promote→demote cycles. Note: `xce_add_hook()` currently rejects NULL fn pointers; the slot-reuse scan path must only match slots where an entry was previously registered and then disabled (not freshly zeroed). | +| P1-2 | `src/controller/ha.c` + `include/ha.h` | **Requires: P1-2a.** Role management, `am_i_primary()`, promotion/demotion. **OP_steppingDown handler**: on receiving `OP_steppingDown` from the primary, the handler validates: (1) the sender's term matches `ha->term` (stale messages ignored), (2) the sender is the current primary (verified via `ha->primary_host`/`ha->primary_port`). **Priority resolution mechanism**: upon validation, each standby immediately runs `am_i_primary()` — the same election probe used at startup. This naturally resolves which standby is the highest-priority reachable candidate: each standby probes all higher-priority candidates; if none respond within `election_timeout_ms`, it promotes itself. This avoids the impossible requirement of knowing which other standbys were connected to the stepping-down primary at the time of the message. Non-highest-priority standbys that discover a higher-priority peer is reachable (via the probe returning "you are NOT primary") defer and wait for the new primary's heartbeat. The `am_i_primary()` call runs in a temporary pthread (same as startup) so the handler does not block the iomgr event loop. The handler runs in the iomgr/vtask thread for initial validation; the actual `am_i_primary()` probe is dispatched asynchronously. | +| P1-3 | `src/controller/ha.c` | Heartbeat watchdog: `OP_heartbeat` exchange via iomgr/vtask pipeline; `ha_check_heartbeat()` on a **dedicated `pthread` with `nanosleep()` or `timerfd_create(CLOCK_MONOTONIC)`** (NOT `delaymgr` — see Part 5.5 heartbeat detection latency note) detects primary failure (replaces `tcp_probe`). **Shutdown protocol** (mirrors checkpoint thread pattern): add `_Atomic int ha_watchdog_stop` to `ha_state_t`. The watchdog thread checks this flag on each `nanosleep()` wake and exits cleanly when set. `ha_fini()` sets the stop flag then calls `pthread_join()` on the watchdog thread before returning — this prevents the watchdog from accessing freed `ha_state_t` (embedded in `cluster_t`) after `cluster_fini()` frees it. `ha_fini()` must be called from `cluster_fini()` before any `ha_state_t` memory is reclaimed. The watchdog thread must **not** be cancelled with `pthread_cancel()` — the same rationale as election probe threads (OpenSSL handlers are not cancellation-safe; see Part 5.4 startup note). | +| P1-3a | `src/controller/ha.c` | Promotion sequence: stop replay, increment term, recover, start primary; SIGUSR1 handler for manual failover (see Part 5.8). **Requires: P1-2a** — `ha_demote_to_standby()` calls `xce_disable_hook()` which is added in P1-2a. Do not start P1-3a until P1-2a is merged and tested. | | P1-4 | `controller.capnp` | Add `term` to Request, `standby` to HostRole, `replEntry/replAck/redirect/heartbeat/steppingDown/replSnapshot/replCatchUpDone` opcodes (see Part 5.10 for full opcode list), `RedirectInfo` struct | | P1-5 | `cluster.c` | Wire HA into startup sequence; add `ha_state_t ha;` to `cluster_t` in this step (note: `persist_t persist;` field was already added to `cluster_t` in P0-5) | | P1-6 | `join.c` | Add redirect logic in `handle_join()` — standby sends REDIRECT for worker/scheduler/apiserver join requests | @@ -2901,16 +3225,16 @@ P0 is broken into seven sub-phases with clear deliverables and dependency orderi | P2-2 | `src/controller/repl.c` | Standby receiver: `OP_replEntry` dispatched via iomgr/vtask pipeline; `OP_replAck` sent via endpoint outq (not raw `send()`). `apply_wal_entry()` does not fire hooks — persistence hooks are registered only on the primary path (in `cluster_init()` after recovery completes, or in `ha_promote_to_primary()` step 5 on promotion). No bypass flag is needed; the hook system is empty on standbys. Standbys apply WAL entries the same way recovery does (see Part 4.2 recovery) | | P2-3 | `src/controller/repl.c` | Sync replication: `repl_wait_ack()` with timeout; only triggered for `WAL_JOB_SUBMIT` from API server connections. **ACK notification mechanism**: when the standby's `OP_replAck` arrives via iomgr/vtask, it must wake the blocked `repl_wait_ack()` call. Implement with a per-standby condition variable (`pthread_cond_t`) under `repl.lock`: the vtask handler signals it on receiving `OP_replAck`; `repl_wait_ack()` parks on it with `pthread_cond_timedwait()`. This keeps the blocking confined to the vtask thread running the `handle_jobsubmit()` path. **Pool floor note**: the default vcpu pool is 8 (`myconf.c` default). Under burst job submission, 8 simultaneous sync-replicating submits exhaust the pool. Clusters using sync replication should configure `vcpus >= 16`: the reasoning is that each blocked `repl_wait_ack()` occupies one vcpu for up to `sync_timeout_ms` (50ms default); at the typical API server connection count of 1–3, the realistic burst concurrency is low, but with multiple API servers sending simultaneous submits the pool can saturate quickly. 16 vcpus provides 2× headroom over the default and headroom for non-submit vtasks (worker joins, heartbeats) that must not be starved. Clusters with only 1 API server and low submit rates can stay at 8. **Startup enforcement**: `persist_start()` must log a `WARN` at startup if sync replication is enabled (`sync_timeout_ms > 0`) and the configured `vcpus < 16`: `WARN("sync replication enabled with vcpus=%d (< 16); recommend vcpus >= 16 to prevent repl_wait_ack pool saturation", vcpus)`. This is a warning, not a hard failure, so low-concurrency deployments can still operate. Operators should monitor `ctrl_vcpu_queue_depth` and increase `vcpus` if it stays above 0 during normal operation. **ACK deadlock prevention**: `repl_on_ack()` must be callable directly from the io_worker read callback (i.e., from `handle_msg()` for `OP_replAck` before vtask dispatch) — it only does `pthread_cond_signal()` and an `acked_lsn` update under `repl.ack_lock`, which is safe to call from io_worker without going through vtaskmgr. If `OP_replAck` were dispatched via vtaskmgr like all other messages, a pool of 8 vcpus all blocked in `repl_wait_ack()` would deadlock: the incoming ACK sits in the vtask queue, but no vcpu is free to process it. By handling ACKs in io_worker directly, the signal wakes a sleeping vcpu regardless of pool saturation. **Worker join lock constraint**: `handle_worker_join()` holds workermgr WRITE lock during `add_worker()` + AFTER_EVENT hooks + `xce_reply()`. The persistence hook for `HOOK_DETAIL_WORKER_JOIN` must use `persist_wal_async()` only — worker join is always async-replicated and must never block on `repl_wait_ack` while the write lock is held | | P2-4 | `src/controller/persist.c` | Wire sync/async replication into `persist_wal_sync_append()` / `persist_wal_async()` | -| P2-5 | `src/controller/repl.c` | Standby catch-up: when standby reconnects via endpoint, stream missing WAL entries. The primary trims WAL only up to `min(acked_lsn)` across connected standbys (`standby_conn_t.acked_lsn`; falls back to `checkpoint_lsn` when no standbys — see Part 2.4 WAL trim guard). If a reconnecting standby needs WAL entries older than the primary's oldest surviving segment, transfer the latest snapshot first, then stream WAL entries from the snapshot's LSN forward | +| P2-5 | `src/controller/repl.c` | Standby catch-up: when standby reconnects via endpoint, stream missing WAL entries. The primary trims WAL only up to `min(acked_lsn)` across connected standbys (`standby_conn_t.acked_lsn`; falls back to `checkpoint_lsn` when no standbys — see Part 2.4 WAL trim guard). If a reconnecting standby needs WAL entries older than the primary's oldest surviving segment, transfer the latest snapshot first, then stream WAL entries from the snapshot's LSN forward. **Snapshot atomic write on standby receive**: the standby writes reassembled snapshot chunks to a temp file `snap/snap-.tmp`, verifies the CRC32 after the final chunk, then uses `rename(2)` to atomically promote to `snap/snap-.capnp`. This prevents a partial snapshot file from being discovered by `persist_recover()` on a subsequent restart. The standby `snap/` directory must have mode 0700 (same startup check as the primary — verify this is covered by the standby's `persist_init()` call). **`OP_replCatchUpDone` handler (primary side)**: when the primary receives `OP_replCatchUpDone` from a standby, it must: (1) locate the matching `standby_conn_t` slot by endpoint pointer, (2) clear `slot->in_catch_up` with `atomic_store_explicit(&slot->in_catch_up, 0, memory_order_release)` — this unblocks `repl_wait_ack()` from excluding that standby, so its subsequent `OP_replAck` messages count toward the "at least one ACK" predicate. The handler runs in the io_worker read callback (same as `OP_replAck` — not dispatched via vtaskmgr) to avoid a vtask queue latency spike after catch-up. If no matching slot is found (standby disconnected between sending `OP_replCatchUpDone` and the primary processing it), log `WARN` and discard silently. | | P2-6 | Tests | Replication round-trip (via SSL pipeline), sync ACK timeout, standby catch-up | ### P2.5: libxce Smart Discovery -> **Note**: P2.5 carries the same implementation scope as P1 or P2 (8 steps). It is numbered 2.5 — not 3 — to reflect that it depends only on P1 and can proceed in parallel with P2 (the controller-side replication). It is independently schedulable, not half the work. +> **Note**: P2.5 carries the same implementation scope as P1 or P2 (8 steps). It is numbered 2.5 — not 3 — to reflect that the **client library code** (P2.5-1 through P2.5-6) depends only on P1 and can be developed in parallel with P2. However, the **functional value** of P2.5-7 (API server discovery, `RegisterToBackend()`) is only fully realized when P2 replication is also deployed: without P2, a promoted standby has no live replication from the primary, so reconnecting the API server to a promoted standby via P2.5-7 serves no purpose. Deploy P2 and P2.5 together for a complete HA experience. P2.5 alone (without P2) provides worker/scheduler auto-reconnect after failover but does not provide replication continuity. -> **Candidate count ceiling**: `MAX_STANDBYS = MAX_HA_CANDIDATES - 1` (defined in `repl.h`, currently 15) sizes the `standbys[]` array and represents the **structural maximum**. The **deployment ceiling** is ≤3 controllers (1 primary + 2 standbys): the LSF-style priority election is safe only when all candidates are in reliable contact with each other; larger clusters increase split-brain risk and election latency with no operational benefit for HPC scheduling. `xce_cluster_config_t` must enforce the structural ceiling when loading `cluster.yaml` (reject configs with more than `MAX_HA_CANDIDATES` candidates), but operators are strongly advised to deploy no more than 3. Add a `WARN` log at startup if `candidates_count > 3`: `"HA cluster has %d candidates; LSF-style election is validated for ≤3. Larger clusters increase split-brain risk."` +> **Candidate count ceiling**: `MAX_STANDBYS = 3` and `MAX_HA_CANDIDATES = 4` are standalone constants defined directly in `include/config.h` (see A6-010). The **deployment ceiling** is ≤3 controllers (1 primary + 2 standbys). `xce_cluster_config_t` must enforce the structural ceiling when populated from `--peers` CLI flag (reject configs with more than `MAX_HA_CANDIDATES` candidates). Operators are strongly advised to deploy no more than 3. Add a `WARN` log at startup if `candidates_count > 3`: `"HA cluster has %d candidates; LSF-style election is validated for ≤3. Larger clusters increase split-brain risk."` -**Goal**: Workers, schedulers, and API servers can discover the primary via any candidate. **Design decision**: libxce auto-discovery and auto-reconnect are foundational, not optional. All client-side controller connection code in the codebase will be migrated to this path — there is no "raw TCP to controller" fallback after this phase. This ensures the scheduler policy enhancement work that follows operates on a stable HA foundation. +**Goal**: Workers and schedulers can discover the primary via any candidate (P2.5 alone). API server discovery (P2.5-7) is also included here but only delivers value when deployed together with P2 replication — without P2, the promoted standby has incomplete state and reconnecting the API server to it is misleading rather than helpful. **Deploy P2.5-7 only alongside P2.** **Design decision**: libxce auto-discovery and auto-reconnect are foundational, not optional. All client-side controller connection code in the codebase will be migrated to this path — there is no "raw TCP to controller" fallback after this phase. This ensures the scheduler policy enhancement work that follows operates on a stable HA foundation. | Step | File | Description | |------|------|-------------| @@ -2918,10 +3242,11 @@ P0 is broken into seven sub-phases with clear deliverables and dependency orderi | P2.5-2 | `src/lib/cluster.c` (new) | Implement `xce_connect_cluster()`: candidate selection, redirect following, backoff+jitter. **SSL**: all controller connections use SSL (enforced by `netio.c`); `xce_connect_cluster()` must perform the TLS handshake and return an SSL-wrapped connection, not a raw fd. Use the same CA cert path from `xce_cluster_config_t` for server certificate verification — `SSL_CTX_set_verify(SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT)` must be set on the client SSL context so the TLS library itself rejects an invalid server certificate before any application-layer CN check runs. The CN validation after handshake (Part 10.6) is defense-in-depth only. Workers and schedulers that previously connected via raw TCP must also go through this SSL path. | | P2.5-2a | `src/lib/meson.build` | Add `cluster.c` to the `libxce_src` source list so the new file is compiled into `libxce` | | P2.5-3 | `src/lib/cluster.c` | Implement `xce_reconnect()`: auto-reconnect with state machine | -| P2.5-4 | `src/lib/cluster.c` | Implement `xce_load_cluster_config()`: parse `cluster.yaml` candidate list | -| P2.5-5 | `src/worker/main.c` | Replace hardcoded controller address with `xce_cluster_config_t`; add reconnection loop | -| P2.5-6 | `src/scheduler/` | Same pattern as worker — use `xce_cluster_config_t` | +| P2.5-4 | `src/lib/cluster.c` | REMOVED: no `xce_load_cluster_config()` function. Workers/schedulers/apiserver parse `--peers` and `--ca-cert` CLI flags in their main() and populate `xce_cluster_config_t` directly before calling libxce functions. | +| P2.5-5 | `src/worker/main.c` | Replace hardcoded controller address with `xce_cluster_config_t`; populate from `--peers` and `--ca-cert` CLI flags in main(); add reconnection loop. Do NOT read any config file. | +| P2.5-6 | `src/scheduler/` | Same pattern as worker — parse `--peers` and `--ca-cert` CLI flags, populate `xce_cluster_config_t`, use libxce discovery. Do NOT read any config file. | | P2.5-7 | `src/frontend/apiserver/backend/clib.go` | Use cluster discovery for controller connection in `RegisterToBackend()`. **Decision: use CGo binding (option a)**. Expose `xce_cluster_config_t` and `xce_connect_cluster()` through the existing `clib.go` CGo shim — `clib.go` already imports `xce` headers and wraps C calls for other controller interactions, so adding cluster discovery there is consistent and avoids duplicating config parsing logic in Go. Add a `#include "xce/cluster.h"` import and a `XceConnectCluster(conf *C.xce_cluster_config_t) int` Go wrapper, then call it from `RegisterToBackend()` instead of the current hardcoded dial. | +| P2.5-7a | `src/frontend/apiserver/` | **Go API server: JOB_STAT_TRANSIENT serialization** — Update the Go jobstatus serializer to map the `JOB_STAT_TRANSIENT` bitmask to `{ "status": "TRANSIENT", "status_detail": "job state uncertain after controller failover — awaiting worker reconnect" }`. Without this, users querying jobs after failover receive stale `PEND` state, which actively misleads them. **This is a standalone story that does not depend on P2.5-7 controller discovery** — it only depends on P0-5a (which defines the `JOB_STAT_TRANSIENT` bitmask). Should ideally ship concurrent with or immediately after P0. **Deliverable**: add `case JOB_STAT_TRANSIENT: return "TRANSIENT"` (or equivalent) to the job status switch in the Go API serializer; add a test case. | | P2.5-8 | Tests | Candidate selection, redirect following, backoff+jitter, reconnection | ### P3: Integration & Hardening @@ -2936,7 +3261,7 @@ P0 is broken into seven sub-phases with clear deliverables and dependency orderi | P3-4 | Performance: benchmark 1000 job submits/sec with sync replication | | P3-5 | Checkpoint GC: verify expired jobs/workers/queues are cleaned; verify dependents block cleanup | | P3-6 | Test the manual failover state machine path by calling `ha_demote_to_standby()` directly from a test harness (e.g., a test-only `ha_trigger_manual_failover(c)` shim or a dedicated integration test binary). This validates that the HA state machine correctly handles a manual promotion trigger — role transitions, WAL flush, hook re-registration — without requiring the unimplemented `xce hostcontrol --failover` CLI (see Out of Scope section). | -| P3-7 | Metrics: WAL size, replication lag, checkpoint duration, standby health, entity GC stats. **Metric naming convention**: All HA/persistence metrics use a `ctrl_` prefix (e.g., `ctrl_repl_degraded_count`, `ctrl_split_brain_detected`, `ctrl_async_events_lost`) to namespace them within the controller's existing metrics perimeter. Metric names must match the pattern `ctrl_[a-z_]+` and should be registered in a central metrics table with a description and type (counter/gauge/histogram). The implementation should define this table as part of `persist.c` or `ha.c` with a `metrics_init()` call in `persist_start()` / `ha_init()`. **Split-brain alert**: log a `WARN`-level message and expose a `ctrl_split_brain_detected` counter whenever a term conflict is detected (primary receives a message with a higher term than its own, or standby detects two primaries). This is the minimum operator signal that the cluster has entered a split-brain state. **Async event loss counter**: expose `ctrl_async_events_lost` — incremented for each async WAL entry that was in-flight (written locally but not yet replicated) when the primary failed. Computed by the newly promoted standby during promotion: `events_lost = primary_last_lsn - standby_applied_lsn` (where `primary_last_lsn` is recovered from the old primary's WAL at promotion time, if available; otherwise log `WARN("async event loss count unavailable — primary WAL not accessible")`). This gives operators visibility into the amount of async data loss after a failover. | +| P3-7 | Metrics: WAL size, replication lag, checkpoint duration, standby health, entity GC stats. **Metric naming convention**: All HA/persistence metrics use a `ctrl_` prefix (e.g., `ctrl_repl_degraded_count`, `ctrl_split_brain_detected`, `ctrl_async_events_lost`) to namespace them within the controller's existing metrics perimeter. Metric names must match the pattern `ctrl_[a-z_]+` and should be registered in a central metrics table with a description and type (counter/gauge/histogram). The implementation should define this table as part of `persist.c` or `ha.c` with a `metrics_init()` call in `persist_start()` / `ha_init()`. **Split-brain alert**: log a `WARN`-level message and expose a `ctrl_split_brain_detected` counter whenever a term conflict is detected (primary receives a message with a higher term than its own, or standby detects two primaries). This is the minimum operator signal that the cluster has entered a split-brain state. **Async event loss counter**: expose `ctrl_async_events_lost` — incremented for each async WAL entry that was in-flight (written locally but not yet replicated) when the primary failed. Computed by the newly promoted standby during promotion: `events_lost = primary_last_lsn - standby_applied_lsn` (where `primary_last_lsn` is recovered from the old primary's WAL at promotion time, if available; otherwise log `WARN("async event loss count unavailable — primary WAL not accessible")`). This gives operators visibility into the amount of async data loss after a failover. **Transient job auto-termination counter**: expose `ctrl_transient_jobs_stuck` (counter, type: counter) — incremented each time a `JOB_STAT_TRANSIENT` job is auto-terminated by `job_should_retain()` after `transient_job_timeout_s` elapses without worker reconnection. Accompanies a `WARN`-level log message with jobid and worker name. Allows operators to detect and quantify failover-induced job loss without reviewing individual log entries. | ### P4: Cleanup & Documentation @@ -2944,8 +3269,7 @@ P0 is broken into seven sub-phases with clear deliverables and dependency orderi |------|-------------| | P4-1 | Update `src/frontend/docs/` to remove stale PostgreSQL references | | P4-2 | Update CLAUDE.md files with persistence/HA architecture | -| P4-3 | Operational runbook: how to add/remove standby, manual failover, disaster recovery (include CA/cert provisioning requirements per Part 5.9) | -| P4-4 | Operational runbook section on certificate rotation: document how to rotate controller mTLS certs (update cert file, restart standbys one at a time, then restart primary). Include the `ha_cert_warn_days` (default 30) monitoring check. Add a guided `openssl` / `cfssl` procedure for issuing new controller certs signed by the cluster CA. | +| P4-3 | Operational runbook: how to add/remove standby, manual failover, disaster recovery. **Must include candidate list upgrade sequencing**: (1) update `--peers` on all controllers first and restart them (they continue operating with the new list); (2) then update workers, schedulers, and API servers. Never update workers before controllers — a worker with a candidate list that includes a new controller before that controller knows about itself cannot connect during the transition. Add a section "Changing the candidate list" with this explicit sequence. Also include: what `ctrl_split_brain_lost_submits` means and how to notify affected users; how to interpret `ctrl_reconciliation_stuck_workers` and what manual steps to take; the JOB_STAT_TRANSIENT user-facing meaning and when to expect it. | ### Out of Scope @@ -2969,7 +3293,7 @@ The following issues were identified during multi-perspective review and resolve | ID | Finding | Resolution | |----|---------|------------| | R-1 | `traversalLimitInWords` set to `file_size / 8` provides no security (P0) | Changed to fixed upper bound `MAX_SNAPSHOT_BYTES / 8` words | -| R-2 | Job ID allocator not reset after WAL replay (P0) | Added `max_replayed_jobid` tracking and `idgen_set_next()` step in recovery sequence | +| R-2 | Job ID allocator not reset after WAL replay (P0) | **Superseded by A6-014**: the premise was incorrect — jobids are allocated by the Go API server's `idgen`, not the controller. The controller is a passive consumer that restores jobs at their original IDs during recovery. No `max_replayed_jobid` tracking or `idgen_set_next()` call is needed. See recovery step 3 note and A6-014. | | R-3 | `acked_lsn` not declared `_Atomic` in struct (P0) | Changed `standby_conn_t.acked_lsn` to `_Atomic uint64_t` | | R-4 | `ha.role` not atomic — role-change window incomplete (P1) | Changed `ha_state_t.role` to `_Atomic ctrl_role_t`, documented acquire/release requirements | | R-5 | Missing hook idempotency guard (P0) | Added `hooks_registered` flag to `ha_state_t`, required in `ha_promote_to_primary()` | @@ -3023,7 +3347,7 @@ The following issues were identified during multi-perspective review and resolve | F-20 | `1 << attempt` signed overflow + shared `rng_seed` data race in `backoff_jitter` (P1) | Saturating shift at 30; `__thread` per-thread seed with `pthread_self` init | | A5-001 | Checkpoint timer described as "1-second delaymgr timer" but WAL fsync and heartbeat require sub-second granularity (P1) | Clarified P0-3: checkpoint timer uses `delaymgr` (1s poll is adequate); WAL fsync and heartbeat use dedicated pthreads for sub-second precision | | A5-002 | Capnp schema missing opcodes added in P1 milestones (P1) | Added `heartbeat @26`, `steppingDown @27`, `replSnapshot @28`, `replCatchUpDone @29` to Part 5.10 schema | -| A5-003 | WAL CRC chain `WAL_CRC_NONE` at segment start is permissive — could mask corruption (P1) | Clarified: `WAL_CRC_NONE` is only valid during catch-up at segment boundaries; in normal mode it is a protocol error | +| A5-003 | WAL CRC chain `WAL_CRC_NONE` at segment start is permissive — could mask corruption (P1) | Clarified: `WAL_CRC_NONE` at segment boundaries (writer-side rotation, after fsync succeeded) is valid and required; `WAL_CRC_NONE` encountered mid-segment during catch-up replay is a protocol error | | A5-004 | Hook list mutation during demotion — `xce_remove_hook()` could race with concurrent iteration (P1) | Added lock-free/RCU-style removal spec: `xce_remove_hook()` sets disabled flag, defers memory reclamation | | A5-005 | `xce_connect_cluster()` returns `int fd` but TLS-first architecture requires `SSL*` (P1) | Changed return type to `SSL*`; added `xce_disconnect_cluster(SSL*)`; updated `xce_reconnect` to `SSL**` | | A5-006 | GC use-after-free: checkpoint traverses deleted entities without protection (P1) | Added GC use-after-free mitigation section: per-operation READ lock hold for P0; epoch-based reclamation as P3 optimization | @@ -3038,6 +3362,67 @@ The following issues were identified during multi-perspective review and resolve | A5-015 | Concurrent snapshot catch-up memory — `MAX_STANDBYS × MAX_SNAPSHOT_BYTES` aggregate (P2) | Added aggregate memory budget analysis: primary holds negligible per-chunk memory; standby reassembly buffer bounded by single `MAX_SNAPSHOT_BYTES` | | A5-016 | `ha_check_heartbeat()` uses plain `ha->role` read without `atomic_load_explicit` (P1) | Fixed to use `(ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire)` — consistent with F-3 fix | | A5-017 | Appendix A.4 `in_catch_up` field typed as plain `int` instead of `_Atomic int` (P1) | Fixed to `_Atomic int` to match Part 5.3 definition and documented access pattern | +| A5-018 | Stale "COW mode growth" snippet (pre-R-22 intermediate design) present alongside final `xzarray_store()` listing — contradicts the mutex-first approach | Removed the superseded snippet; added a forward-reference note pointing to the definitive R-22 implementation below | +| A5-019 | Plain reads/writes to `_Atomic uint64_t ha->applied_lsn` in standby receiver pseudocode (Part 6.4) — contradicts documented require of `acquire`/`release` ordering | Replaced plain accesses with `atomic_load_explicit(acquire)` into local `applied` variable; replaced write with `atomic_store_explicit(release)` | +| A5-020 | Plain `!ha->in_catch_up` in standby receiver pseudocode (Part 6.4) — contradicts documented `_Atomic int` field with required acquire ordering | Changed to `!atomic_load_explicit(&ha->in_catch_up, memory_order_acquire)` | +| A5-021 | `ha_check_heartbeat()` reads `ha->primary_ep` and `ha->last_heartbeat` without `ha->lock` — contradicts lifecycle comment requiring mutex; `ha_state_t` missing `lock` field entirely | Added `pthread_mutex_t lock` field to `ha_state_t` (Part 5.3 and Appendix A.4); updated `ha_check_heartbeat()` to snapshot `primary_ep` and `last_heartbeat` under `ha->lock` before use | +| A5-022 | Thaw function reads `_Atomic _Bool inuse` via plain `!x.inuse` and `x.inuse` — C11 plain access gives seq_cst ordering inconsistently with relaxed ordering used by surrounding explicit stores | Changed all three plain inuse reads in `xzarray_thaw()` to `atomic_load_explicit(memory_order_relaxed)` matching the explicit stores in the same function | +| A6-001 | Part 5.5 code sketch registers heartbeat check via `delay_task(&c->delaymgr, ...)` — directly contradicts the same section's requirement for a dedicated pthread | Replaced delaymgr sketch with a reference to the dedicated-pthread requirement; noted delaymgr's 2-second tick granularity (precision=2 in cluster.c) as the reason | +| A6-002 | Remaining plain `ha->applied_lsn` read in Part 6.4 CRC check block (`if (ha->applied_lsn == 0)`) missed in A5-019 pass | Changed to `atomic_load_explicit(&ha->applied_lsn, memory_order_acquire) == 0` | +| A6-003 | Part 9.4 redirect sends `primary_host`/`primary_port` without nil-check — on cold start before any heartbeat, standby sends garbage redirect (empty host, port 0) | Added guard: if `primary_host[0] == '\0' \|\| primary_port == 0`, reply with `E_XCE_NOMASTER` instead of redirect | +| A6-004 | Part 5.4 election probe peer-cert validation describes only application-layer CN check — missing `SSL_VERIFY_PEER` TLS-layer requirement that Part P2.5-2 already mandates for `xce_connect_cluster()` | Added explicit `SSL_CTX_set_verify(SSL_VERIFY_PEER \| SSL_VERIFY_FAIL_IF_NO_PEER_CERT)` requirement to Part 5.4; cross-referenced P2.5-2 | +| A6-005 | `persist_t` has `last_crc` for local WAL chain but no `last_replicated_crc` for replication chain — WAL_CHECKPOINT entries break the standby's chained-CRC validation unless the primary skips them from the replicated chain | Added `last_replicated_crc` field to `persist_t` (Appendix A.1) with documentation of when it is updated vs `last_crc` | +| A6-006 | `slot_key_t.key` is `const char*` (borrowed pointer into kvtab) — kvtab can rehash and move strings after the write lock is released during lock-free serialization, causing use-after-free | Changed `slot_key_t.key` to `char key[XCE_MAXNAME_LEN]` (owned deep copy populated by `strndup` in `kvmgr_capture_keys`); updated the serialization snapshot code and comment | +| A6-007 | P0-10 description has no prerequisite dependency on P0-9 — implementer can attempt to wire `persist_init` into `cluster.c` before Cap'n Proto schemas are generated | Added `Requires: P0-9` to P0-10 description | +| A6-008 | P0-0b description did not call out that `intf_varray_t` and `xzarray_private_t` struct changes are breaking ABI — partial builds corrupt `makeInstance()` private section allocations | Added explicit breaking-ABI note to P0-0b; required converting positional initializers to designated style before adding new slots; required full `ninja -C build` clean rebuild | +| A6-009 | Part 3.9 lock hierarchy described `xzarray mylock_t` as "not taken in vput" without distinguishing normal/COW/thaw modes — contradicted Part 3.5's COW-mode inner-mutex requirement (R-22) | Replaced single-sentence description with three-mode table: (a) normal — inner mutex skipped; (b) COW vstore — inner mutex required; (c) thaw — write lock sufficient | +| A6-010 | Part 6.2 "constant reconciliation" note still referenced old `MAX_HA_CANDIDATES = 16` and `MAX_STANDBYS = MAX_HA_CANDIDATES - 1` derivation, contradicting Out of Scope reduction to 4/3 | Updated to state both constants defined directly in `config.h` (`MAX_HA_CANDIDATES = 4`, `MAX_STANDBYS = 3`); removed derived formula | +| A6-012 | `kvmgr_capture_keys()` called in Part 3.8 checkpoint flow but never defined anywhere in the document | Added full function signature and documentation to Appendix A.1 | +| A6-013 | Recovery step 3 required `idgen_set_next()` — a function that does not exist; idgen lives in Go API server (`clib.go`), not the controller | Removed `idgen_set_next()` requirement; documented that the controller never allocates jobids (assigned by API server before submission) and recovery only restores jobs at their original IDs from the WAL | +| A6-014 | R-2 "Job ID allocator reset" based on incorrect assumption that the controller allocates jobids | Corrected: jobids are allocated by the Go API server's idgen; the controller is a passive consumer. Recovery single-threaded note added to step 3 | +| A6-015 | `xce_remove_hook()` does not exist in `hook.c` — required for hook unregistration on demotion | Replaced with `xce_disable_hook()` approach: set `entry->fn = NULL` in place; `xce_run_hooks()` skips NULL entries. No RCU/deferred-reclamation complexity; bounded memory (3 persist hooks max). Requires adding `xce_disable_hook()` to `src/lib/hook.c` | +| A6-016 | Part 6.4 did not clarify that `apply_wal_entry()` on standby is effectively single-threaded | Added serialization note: all `OP_replEntry` messages arrive over a single endpoint; iomgr/vtask `endpoint_block_tasks()` serializes dispatch — `xkvmgr_add()` calls in `apply_wal_entry()` are never concurrent | +| A6-017 | `delaymgr` uses `sleep(precision)` with `precision=2` (integer seconds) — makes 1-second checkpoint poll (P0-3) impossible, and adds up to 2s jitter to heartbeat detection | Specified `delaymgr` enhancement: replace `sleep()` with `usleep()`, change `precision` field to `precision_ms` (default 200ms). Allows 1-second checkpoint poll (5 ticks × 200ms) without a dedicated pthread. P0-3 notes this as a prerequisite | +| A6-018 | Part 10.8 listed NFS as a valid distribution mechanism for `cluster.yaml`, contradicting the no-shared-filesystem architecture | Removed NFS from distribution options; documented that XCE assumes no shared filesystem — `cluster.yaml` must be distributed via config management tooling (Ansible, Salt, deployment pipeline) on each node's local storage | +| A6-019 | Product-lens review flagged "split-brain data loss not surfaced at occurrence" as a P1 omission | Reframed as an intentional CAP trade-off: XCE chooses AP (availability + partition tolerance) over CP (consistency). Real-time split-brain detection is impossible without quorum — implementing it is equivalent to Raft. The `split_brain_risk_accepted: true` key is the operator's explicit CAP position acknowledgment. Post-partition visibility is provided by `ctrl_split_brain_detected` counter and ERROR-level logs | +| A6-020 | AFTER_EVENT hook return value used to detect persistence failures — breaks if any subsequent hook overwrites rc; requires transaction rollback to undo in-memory state on failure | Replaced with side-channel pattern: persistence hook always returns 0 (AFTER_EVENT failures tolerated per XCE hook design; pre-hook BEFORE_EVENT failures abort, post-hook AFTER_EVENT failures are logged). WAL result stored in `persist_t.last_sync_rc`; `handle_jobsubmit()` reads it directly after `xce_run_hooks(AFTER_EVENT)` returns. Field added to `persist_t` struct. | +| A6-021 | `sizeof(xzarray_item_t)` change from `unsigned int :1` to `_Atomic _Bool` flagged as backward compatibility concern | Dismissed: no released version of XCE exists; all components are rebuilt together. The struct layout can change freely. `calloc()` sizing in freeze and `xzarray_ensure_items()` automatically uses the new size after recompile. | +| A6-022 | No recovery path for corrupt-all-snapshots fatal abort scenario — "restore from backup" is undefined | Added `snap_keep_n` config (default 2) to retain previous snapshots as last-known-good fallback. Recovery walks snapshots newest-to-oldest, falling back if checksum fails and WAL can bridge the gap. `snap_keep_n` added to `persist_config_t` and Cap'n Proto `PersistConfig` schema. | +| A6-023 | rtti source location flagged as unknown — P0-0b appeared to lack an implementation path | Clarified: rtti is maintained by the XCE team (not a third-party dependency). Source is accessible via the project's `flake.nix` inputs. No external coordination required. | +| A6-024 | Part 1.2 said `restore_worker()` "signature will need to be updated" — contradicted P0-12 which says "signature is unchanged" | Fixed Part 1.2: signature is unchanged; `worker_from_capnp()` adapter in P0-12 bridges Cap'n Proto to `restore_worker()`. Recovery calls the adapter, not `restore_worker()` directly | +| A6-025 | P2.5 "parallel with P2" claim misleading — P2.5-7 (API server discovery) only has value when P2 replication is live | Clarified: P2.5 code can be developed in parallel with P2; P2.5-7 should be deployed together with P2 for full functionality. P2.5 alone provides worker/scheduler auto-reconnect but not replication continuity | +| A6-026 | COW active array allocated at exact `shadow_caps` — any vstore() call immediately triggers `realloc()` which moves `a->data`, causing dangling pointers in concurrent vget() callers holding READ lock | Pre-grow active array to `1.5 × shadow_caps + delta` at freeze time. [shadow_caps, cow_caps) range is immediately available for vstore() — no realloc needed in the common case. If full, vstore() returns -ENOMEM (WARN + client retry after COW closes) | +| A6-027 | P0-8a conditional fallback had no merge-order gate — deferred cleanup path could be permanently skipped | Split into P0-8a (fallback with TODO markers, no P0-0a dependency) and P0-8b (deferred path, explicitly requires P0-0a merged and green before starting) | +| A6-028 | P0-7 reply-after-hooks audit scoped only to `handle_jobsubmit()` — other handlers with AFTER_EVENT + xce_reply have the same ordering gap | Expanded P0-7 scope to full codebase audit: grep all `xce_run_hooks(AFTER_EVENT` + `xce_reply` pairs across all handler files; document each handler's reply ordering decision | +| A6-029 | SIGUSR1 manual failover has no access control — any same-UID process can trigger failover | Documented: acceptable for initial implementation on administratively restricted management nodes. Future `xce hostcontrol --failover` CLI (SSH-authenticated) is the preferred interface. Added note about future `--disable-sigusr1-failover` startup flag | +| A7-001 | Part 3.10 "ticks" framing for delay_task() was wrong — delay.c uses time_t arithmetic (seconds), not tick counts; tick description contradicted P0-3 which correctly says delay=1 second | Removed ticks framing; clarified delaymgr enhancement as nanosleep precision change only (delay_task parameters remain seconds throughout); corrected delay_task call to delay=1 second | +| A7-002 | `snap_keep_n` added to `persist_config_t` C struct but absent from PersistConfig Cap'n Proto schema in Appendix A.5 — field could never be read from config | Added `snapKeepN @6 :UInt32 = 2` to PersistConfig schema; updated P0-9 step to include the field | +| A7-003 | COW vstore() growth path called `xzarray_ensure_items()` (realloc) despite pre-grow design intent — realloc moves `a->data`, dangling vget() pointers under READ lock | Changed growth path to return -ENOMEM instead of realloc; documented rationale; -ENOMEM is transient (COW < 100ms) — client retries | +| A7-004 | `xce_disable_hook()` and `xce_run_hooks()` NULL guard had no implementation phase step — ha_demote_to_standby() calls a function not yet in hook.c | Added P1-2a step: add `xce_disable_hook()` to `src/lib/hook.c` and add `if (!it->fn) continue;` guard to `xce_run_hooks()`; marked as prerequisite for P1-3a | +| A7-005 | Part 3.8 said `checkpoint_lsn = max LSN across kvmgrs` — should be minimum; using max trims WAL entries needed to replay the earliest-frozen kvmgr | Changed to minimum LSN; documented why minimum is correct for cross-kvmgr WAL bridging | +| A7-006 | `ha_state_t.promotion_failed` was plain `int` while `promotion_retry_count` was `_Atomic int` — mismatch, data race on watchdog vs vcpu reads | Changed to `_Atomic int promotion_failed` in both Part 5.3 and Appendix A.4 struct definitions | +| A7-007 | P2.5 intro still said `MAX_STANDBYS = MAX_HA_CANDIDATES - 1 (defined in repl.h, currently 15)` — stale text contradicting A6-010 | Fixed to `MAX_STANDBYS = 3` standalone in `include/config.h`; removed derived-formula reference | +| A7-008 | Part 7.4 Degraded Mode contained full implementation spec for `repl_degraded_timeout_s` despite Out of Scope removing it — contradiction caused scope confusion | Added explicit callout box at top of degraded-mode timeout paragraph marking it out of scope; replaced full spec with concept summary | +| A7-009 | Part 7.4 step 4 said self-healing was via "WAL replay" — misleading after a checkpoint trims the degraded WAL, where full snapshot transfer is required | Added "or full snapshot transfer" to the self-healing description | +| A7-010 | Part 3.11 snapshot format diagram missing `version` field and `headerChecksum` field — reviewers and implementers could miss these required fields | Added `version: UInt32 (=1)` and `headerChecksum: CRC32(header fields)` to the SnapshotHeader diagram | +| A7-012 | Part 10.8 contained full `configVersion` CRC hash implementation spec despite Out of Scope deferring it to P3 | Added deferred callout; replaced full spec with concept summary; noted `configVersion @23` is reserved | +| A7-013 | `snap_keep_n` fallback description implied protection against completed-checkpoint corruption — actually only protects against crash-during-checkpoint | Clarified: fallback works only when WAL bridging the old→new snapshot gap still exists (crash-during-checkpoint scenario); aborts with fatal error when WAL is already trimmed | +| A7-014 | P0-9 phase step description did not mention `snap_keep_n`/`snapKeepN` schema addition | Added `snapKeepN @6 :UInt32 = 2` explicitly to P0-9 step description | +| A7-015 | `last_sync_rc` was a shared `int` on `persist_t` — concurrent vcpu threads race on reads/writes, causing false ACKs on WAL failures | Removed from `persist_t`; replaced with stack-allocated `int sync_rc` in `handle_jobsubmit()` passed as extra `va_list` argument to `xce_run_hooks()` and read by `persist_job_hook_fun()` — per-request, no sharing across threads | +| A7-016 | `xce_disable_hook()` could grow hook list indefinitely (new NULL slot per promote/demote cycle) and `fn` pointer reads were non-atomic — use-after-NULL-write race | Made `hook_item_t.fn` `_Atomic hook_fn_t`; `xce_disable_hook()` stores NULL with release; `xce_run_hooks()` loads into local with acquire then checks+calls local; `xce_add_hook()` reuses existing NULL slots before appending — bounds list to max-ever-registered slots | +| A7-017 | `JOB_STAT_TRANSIENT` had no TTL or auto-resolution path — permanently stuck scheduling slots if worker never reconnects | Added `transient_job_timeout_s` config (default 300s) to `persist_config_t` and PersistConfig schema (`@7`); after timeout, job → `JOB_STAT_EXIT` with reason "worker did not reconnect"; worker → `WORKER_UNAVAILABLE`; increments `ctrl_transient_jobs_stuck` | +| A7-018 | Array job batch sync mechanism said "accumulate in a single hook call" but AFTER_EVENT fires once per sub-job — no mechanism for cross-call accumulation | Specified heap-allocated `persist_batch_ctx_t*` passed via same `va_list` slot as `sync_rc` for array submits; accumulates WAL entries across calls; flushes every 100 sub-jobs; bounded by per-request allocation (no shared state) | +| A7-019 | `ha_init()` mutex initialization timing relative to heartbeat thread and promotion path was unspecified | Added explicit ordering note: `ha_init()` must run before `am_i_primary()`; mutex is initialized once at startup and never re-initialized on re-promote cycles (re-init with waiters = UB) | +| A7-020 | `reconciliation_progress` denominator (`workers_at_old_primary`) was undefined — implementers could not compute it | Specified as: count of workers in promoted standby's kvmgr with status NOT including `XCE_WORKER_UNAVAIL` or `XCE_WORKER_HIDDEN` at the moment of promotion — derivable from replicated state, no data from dead primary needed | +| A7-021 | `delaymgr` precision enhancement had no dedicated phase step — was only noted as a prerequisite inline in P0-3 | Added `P0-2b` step: rename `precision` → `precision_ms`, replace `sleep()` with `nanosleep()`, audit all `delaymgr_init()` callers, update `cluster.c:604` to `precision_ms=200` | +| A8-001 | "Why this works" paragraph in Part 8.3 still referenced removed `c->persist.last_sync_rc` field | Updated to reference per-request `sync_rc` stack variable and document thread-private design | +| A8-002 | `persist_batch_ctx_t` was referenced in Part 8.3 and A7-018 but never defined — no struct or Appendix entry | Added full `typedef struct persist_batch_ctx_t` to Appendix A.1 with all fields documented | +| A8-003 | va_list slot ambiguity: `&sync_rc` (int\*) and `persist_batch_ctx_t*` described as occupying "same slot" — C UB when va_arg reads different types at same position | Replaced with two distinct va_list arguments (always present): `&sync_rc` first, `batch_ctx` (NULL for scalar, non-NULL for array) second; hook always reads both with typed va_arg calls; call site updated with `persist_batch_ctx_alloc()` example | +| A8-004 | P1-2a listed after P1-2 in phase table but is a prerequisite of P1-2 — table order inverts the dependency | Moved P1-2a before P1-2; added `Requires: P1-2a` note to P1-2; added `_Atomic hook_fn_t` and `atomic_store/load_explicit` requirements to P1-2a | +| A8-005 | P0-9 schema step missing `transientJobTimeoutS @7` and `transientSince` field for Job schema | Added both to P0-9; `transientSince` is required for the `transient_job_timeout_s` GC check in `job_should_retain()` | +| A8-006 | `transient_job_timeout_s` auto-termination had no implementation path — `job_should_retain()` returns `true` for `JOB_STAT_TRANSIENT` (not terminal per `job_is_terminal()`) | Added transient-job TTL check to `job_should_retain()` before the `job_is_terminal()` guard; auto-terminates to `JOB_STAT_EXIT` with `WARN` log + `ctrl_transient_jobs_stuck` increment | +| A8-007 | `ctrl_transient_jobs_stuck` metric named in config comment and A7-017 but absent from P3-7 metric inventory | Added counter definition to P3-7 with description and WARN log requirement | +| A8-008 | P0-3 had no explicit `Requires: P0-2b` — the dependency was only in P0-2b's description and prose | Added `Requires: P0-2b` to P0-3 step row | --- @@ -3070,11 +3455,39 @@ typedef struct { MUST NOT be persist.lock (which is mylock_t) — pthread_cond_timedwait requires pthread_mutex_t*. See Part 3.8 for rationale. */ + pthread_t snap_thread; /* ID of the active serialization pthread, or 0 if + none is running. Set by persist_checkpoint_timer_fn() + when it spawns the serialization pthread; read by + cluster_fini() for pthread_join() on the deadline- + exceeded shutdown path. Zero-initialized (memset + in persist_init()). cluster_fini() MUST check + snap_thread != 0 before calling pthread_join() — + pthread_join(0, NULL) is undefined behavior. */ uint64_t total_wal_bytes; /* Cumulative WAL size across all segments. Updated on each write, adjusted on segment rotation and WAL trimming. Used by the inline checkpoint trigger (Part 3.11) to decide when to checkpoint based on snap_wal_size_mb. */ + uint32_t last_replicated_crc; /* CRC of the last WAL entry that was sent to standbys. + Distinct from last_crc (which includes WAL_CHECKPOINT + entries that are local-only and never replicated). + Used as prev_crc for the next replicated entry so + that standbys' chained-CRC validation works correctly + across checkpoint boundaries. Updated only when a + non-WAL_CHECKPOINT entry is broadcast to standbys. + See Part 6.4 WAL_CHECKPOINT chain gap handling. + LOCK: protected by persist.lock. All updates to + last_replicated_crc must be made while holding + persist.lock (same lock as last_crc). The WAL fsync + thread (which writes WAL_CHECKPOINT and updates + last_crc) does NOT update last_replicated_crc — + the two fields are always updated on different code + paths, preventing the race where a checkpoint write + interleaves with a replication CRC read. */ + /* NOTE: last_sync_rc was removed (A7-015). WAL write results are now communicated + * via a per-request stack variable (&sync_rc) passed as an extra va_list argument + * to xce_run_hooks(AFTER_EVENT). A shared persist_t field races across concurrent + * vcpu threads processing simultaneous job submits from different API servers. */ } persist_t; int persist_init(persist_t* p, persist_config_t* config, cluster_t* c); @@ -3176,6 +3589,59 @@ int persist_worker_hook_fun(int type, int situation, int detail, void* data, va * WARNING: Skipping unknown entries during recovery silently loses the * state changes those entries represent — see recovery forward-compat note. */ int apply_wal_entry(cluster_t* c, const wal_entry_hdr_t* hdr, const void* payload); + +/* Array job batch accumulation context — allocated on handle_jobsubmit() stack for + * array submits (submit->array_size > 1), freed on return from handle_jobsubmit(). + * Passed via va_list to persist_job_hook_fun() instead of int* sync_rc: + * - single job: va_list carries (jlist_ptr, &sync_rc) + * - array job: va_list carries (jlist_ptr, batch_ctx_ptr) + * The hook distinguishes the two cases by checking detail == HOOK_DETAIL_JOB_NEW + * and submit->array_size — or by inspecting the cluster_t's in-flight array state. + * + * The batch context accumulates WAL entries for N sub-jobs (default batch_size=100), + * then calls repl_broadcast() once and waits for one ACK at the batch's highest LSN. + * sync_rc is embedded in the context so the hook can report the final WAL result. */ +typedef struct { + int sync_rc; /* Final WAL result — 0, -EIO, -ETIMEDOUT, or -EAGAIN. + * -EAGAIN: transient COW-full condition (xzarray_store returned + * -ENOMEM during COW window). Unlike -EIO, the job must NOT be + * marked terminal via job_mark_wal_error_exit() — the in-memory + * state may or may not have been inserted depending on where in + * the submit path the -ENOMEM occurred. handle_jobsubmit() must + * reply with a retriable error code (EAGAIN/EBUSY) so the client + * retries after the COW window closes (< 100ms). */ + uint64_t batch_high_lsn; /* Highest LSN in the current batch (for repl_wait_ack) */ + uint32_t sub_jobs_in_batch; /* Sub-jobs accumulated in the current batch */ + uint32_t batch_size; /* Flush threshold (default 100) */ + uint32_t total_sub_jobs; /* Total sub-jobs in the array (submit->array_size) */ + uint32_t sub_jobs_done; /* Sub-jobs processed so far (hook call count) */ +} persist_batch_ctx_t; + +/* Key→index mapping used during checkpoint serialization (Part 3.8). + * Lives in kvmgr.h — used by both kvmgr.c (populate) and persist.c (consume). + * key[] holds an owned deep copy populated by kvmgr_capture_keys() via strndup, + * safe to read after the kvmgr write lock is released. */ +typedef struct { + int index; /* shadow slot index */ + char key[XCE_MAXNAME_LEN]; /* owned copy — not a borrowed pointer into kvtab */ + size_t key_len; /* length including null terminator */ +} slot_key_t; + +/* Capture key→index mappings from a kvmgr's kvtab into a flat slot_key_t[] array. + * Called during checkpoint freeze (Part 3.8), AFTER vfreeze() returns, while the + * kvmgr write lock is still held. Allocates and populates keys_out[0..n_keys_out-1] + * with one entry per live entity. Each entry's key field is a deep-copied string + * (strndup into slot_key_t.key[XCE_MAXNAME_LEN]) — safe to read after the write + * lock is released during the lock-free serialization phase. + * Caller must free(keys) after serialization completes. + * + * Returns 0 on success, -ENOMEM if allocation fails (in which case *keys_out is NULL + * and *n_keys_out is 0 — caller must abort the checkpoint and call persist_checkpoint_abort()). + * + * NOTE: Supports both KVMGR_KEY_INT and KVMGR_KEY_STRING kvmgrs. For integer-keyed + * kvmgrs, the key is formatted as a decimal string before copying. + * Caller must hold the kvmgr write lock for the entire duration of this call. */ +int kvmgr_capture_keys(meta_obj_t* kvmgr, slot_key_t** keys_out, size_t* n_keys_out); ``` ### A.2 zarray COW Extensions @@ -3245,6 +3711,15 @@ typedef struct { int lagging; char host[XCE_MAXNAME_LEN]; uint32_t port; + _Atomic int in_catch_up; /* Primary-side mirror of the standby's ha_state_t.in_catch_up. + * Set to 1 by repl_add_standby() when applied_lsn < primary's + * committed_lsn at join time (standby needs catch-up). + * Cleared to 0 by repl_on_catchup_done() when OP_replCatchUpDone + * is received from this standby. + * Read by repl_wait_ack() to exclude catching-up standbys from + * the "at least one ACK" predicate — standbys in catch-up never + * send OP_replAck for live entries (per Part 6.4 ACK gate). + * Use atomic_load/store_explicit with acquire/release. */ } standby_conn_t; typedef struct { @@ -3269,7 +3744,19 @@ typedef struct { * Both operations hold repl.lock, so the loop sees a consistent * snapshot. The last_disconnected_lsn[] and last_disconnect_time[] * arrays are NOT cleared on disconnect — they persist across - * reconnections to support the one-interval trim delay. */ + * reconnections to support the one-interval trim delay. + * + * ACKED_LSN INITIALIZATION (CRITICAL): repl_add_standby() MUST initialize + * the new slot's acked_lsn to persist->checkpoint_lsn (the last completed + * checkpoint LSN), NOT to 0. Initializing to 0 would cause min(acked_lsn) + * to equal 0, blocking ALL WAL trimming until the new standby fully catches + * up from LSN 0 — potentially accumulating gigabytes of un-trimmable WAL. + * checkpoint_lsn is the safe floor: WAL before checkpoint_lsn is covered + * by the snapshot the standby will receive during catch-up, so it can be + * trimmed immediately. WAL from checkpoint_lsn forward is what the standby + * needs; min(acked_lsn) naturally preserves it until the standby ACKs those + * entries. repl_add_standby() signature: accepts persist_t* for checkpoint_lsn + * lookup, or the caller passes checkpoint_lsn directly as a uint64_t. */ } replicator_t; /* Fan-out send to all standbys */ @@ -3283,14 +3770,31 @@ int repl_broadcast(replicator_t* r, wal_entry_hdr_t* hdr, void* payload); * while (r->acked_lsn < target_lsn && !timed_out) * pthread_cond_timedwait(&r->ack_cond, &r->ack_lock, &deadline); * unlock(ack_lock); - * The while-loop predicate check must be inside the lock, before the first wait. */ + * The while-loop predicate check must be inside the lock, before the first wait. + * + * CATCH-UP EXCLUSION: Standbys with in_catch_up != 0 (ha->in_catch_up set in the + * ha_state_t of the standby's slot) do NOT send OP_replAck for live entries — + * they gate ACKs until catch-up completes (Part 6.4). Therefore repl_wait_ack() + * MUST exclude catching-up standbys from its "at least one ACK" predicate. + * If the only connected standby is in catch-up, repl_wait_ack() falls back to + * degraded mode immediately (does not wait for the full timeout_ms) and returns + * REPL_DEGRADED. Callers treat REPL_DEGRADED the same as a timeout — the WAL + * entry is durable on the primary's local WAL but not yet confirmed on any standby. + * The per-standby in_catch_up flag is read from standby_conn_t.in_catch_up + * (a copy of the remote standby's ha->in_catch_up, updated on each OP_replAck + * that carries the standby's applied_lsn — if applied_lsn < primary's + * committed_lsn at join time, in_catch_up is set; cleared on OP_replCatchUpDone). */ int repl_wait_ack(replicator_t* r, uint64_t target_lsn, uint32_t timeout_ms); /* Called when ACK received from a standby */ void repl_on_ack(replicator_t* r, uint64_t acked_lsn); /* Add/remove standby connections (via endpoint/iomgr SSL pipeline) */ -int repl_add_standby(replicator_t* r, endpoint_t* ep, const char* host, uint32_t port); +/* checkpoint_lsn: pass c->persist.checkpoint_lsn — the new slot's acked_lsn is + * initialized to this value (not 0) to avoid blocking WAL trimming while the + * standby catches up. See ACKED_LSN INITIALIZATION comment in replicator_t. */ +int repl_add_standby(replicator_t* r, endpoint_t* ep, const char* host, uint32_t port, + uint64_t checkpoint_lsn); int repl_remove_standby(replicator_t* r, endpoint_t* ep); /* Check standby health and lag */ @@ -3332,7 +3836,10 @@ typedef struct { char primary_host[XCE_MAXNAME_LEN]; uint32_t primary_port; struct timespec last_heartbeat; /* Last heartbeat from primary (CLOCK_MONOTONIC, standby) */ - int promotion_failed; /* Non-zero if last promotion attempt failed — retry on next watchdog tick */ + _Atomic int promotion_failed; /* Non-zero if last promotion attempt failed — retry on next watchdog tick. + * Must be _Atomic: written by ha_promote_to_primary() on the watchdog + * thread, read by ha_check_heartbeat() on the same thread but also + * visible to vcpu threads checking promotion status. Relaxed ordering. */ _Atomic int promotion_retry_count; /* Number of consecutive failed promotion attempts. * Must be _Atomic because it is incremented by the * watchdog thread and read by vcpu threads during @@ -3353,12 +3860,15 @@ typedef struct { * stream — used for chained CRC validation (P0-C fix). * Mirrors persist_t.last_crc but on the standby side. */ ha_config_t config; /* HA configuration (timeouts, intervals) */ + pthread_mutex_t lock; /* Protects primary_ep and last_heartbeat against concurrent + * iomgr NULL-then-free and watchdog reads. Initialized in + * ha_init(). See Part 5.3 for full access protocol. */ } ha_state_t; /* Check if this controller should be primary */ static int am_i_primary(cluster_t* c); -/* Heartbeat watchdog — called from delaymgr timer on standby */ +/* Heartbeat watchdog — called from the dedicated HA watchdog pthread (NOT delaymgr). See P1-3 */ static void ha_check_heartbeat(cluster_t* c); /* Promote self from standby to primary */ @@ -3372,15 +3882,20 @@ int ha_demote_to_standby(cluster_t* c); ```capnp # Persistence configuration — embedded in ClusterConfig +# NOTE: Path fields (wal_path, snap_path) are REMOVED from this schema. +# Paths are derived from --workdir and -n CLI flags at controller startup: +# effective_dir = workdir + "/" + name +# wal_path = effective_dir + "/data/wal" +# snap_path = effective_dir + "/data/snap" struct PersistConfig $C.mapname("persist_config_t") { - walPath @0 :Text $C.mapname("wal_path"); - snapPath @1 :Text $C.mapname("snap_path"); - walSyncMs @2 :UInt32 $C.mapname("wal_sync_ms") = 10; - snapIntervalS@3 :UInt32 $C.mapname("snap_interval_s") = 60; - snapWalSizeMb@4 :UInt32 $C.mapname("snap_wal_size_mb") = 64; - finishedRetainS @5 :UInt32 $C.mapname("finished_retain_s") = 3600; - workerRetainS @6 :UInt32 $C.mapname("worker_retain_s") = 86400; - syncTimeoutMs@7 :UInt32 $C.mapname("sync_timeout_ms") = 50; + walSyncMs @0 :UInt32 $C.mapname("wal_sync_ms") = 10; + snapIntervalS@1 :UInt32 $C.mapname("snap_interval_s") = 60; + snapWalSizeMb@2 :UInt32 $C.mapname("snap_wal_size_mb") = 64; + finishedRetainS @3 :UInt32 $C.mapname("finished_retain_s") = 3600; + workerRetainS @4 :UInt32 $C.mapname("worker_retain_s") = 86400; + syncTimeoutMs@5 :UInt32 $C.mapname("sync_timeout_ms") = 50; + snapKeepN @6 :UInt32 $C.mapname("snap_keep_n") = 2; # last-known-good snapshot retention + transientJobTimeoutS @7 :UInt32 $C.mapname("transient_job_timeout_s") = 300; # 0 = no auto-termination } # WAL entry — used in Cap'n Proto schema for replication messages (OP_replEntry). @@ -3455,13 +3970,18 @@ typedef struct { } xce_candidate_t; typedef struct { - xce_candidate_t candidates[MAX_HA_CANDIDATES]; - int ncandidates; /* Number of candidates configured */ - uint32_t connect_timeout_ms; /* Per-candidate connect timeout (default 2000) */ - uint32_t retry_base_ms; /* Backoff base (default 100) */ - uint32_t retry_max_ms; /* Backoff cap (default 5000) */ - int max_retries; /* Max reconnection attempts (default 10) */ - char ca_cert_path[XCE_MAXPATH_LEN]; /* CA cert for TLS server verification (PEM) */ + char ca_cert_path[PATH_MAX]; /* CA cert path for SSL_VERIFY_PEER (empty = skip) */ +} xce_tls_config_t; + +typedef struct { + xce_candidate_t candidates[MAX_HA_CANDIDATES]; + int ncandidates; /* Number of candidates configured */ + uint32_t connect_timeout_ms; /* Per-candidate connect timeout (default 2000) */ + uint32_t retry_base_ms; /* Backoff base (default 100) */ + uint32_t retry_max_ms; /* Backoff cap (default 5000) */ + int max_retries; /* Max reconnection attempts (default 10) */ + xce_tls_config_t tls; /* TLS options — CA cert path for SSL_VERIFY_PEER. + * Populated from --ca-cert CLI flag (P2.5-5/P2.5-6). */ } xce_cluster_config_t; /* Connect to the cluster — returns SSL* connected to primary, or NULL on failure. @@ -3479,8 +3999,8 @@ SSL* xce_join_cluster(xce_cluster_config_t* conf, xce_peerinfo_t* myinfo); * On success, *ssl is updated to the new connection. On failure, *ssl is NULL. */ int xce_reconnect(SSL** ssl, xce_cluster_config_t* conf); -/* Load cluster config from YAML file */ -int xce_load_cluster_config(const char* yaml_path, xce_cluster_config_t* conf); +/* REMOVED: xce_load_cluster_config() — no YAML parsing in libxce. + * xce_cluster_config_t is populated by caller from --peers CLI flag. */ ``` ### A.7 New Files Summary @@ -3505,9 +4025,9 @@ int xce_load_cluster_config(const char* yaml_path, xce_cluster_config_t* conf); | `src/lib/zarray.c` + `include/zarray.h` | Merge `zarray_t` into `xzarray_t` (P0-0a); add `shadow`, `shadow_caps` fields; add `xzarray_freeze()`, `xzarray_thaw()`; modify `vget`, `vput`, `iter_next` for COW mode | | `src/controller/job.c` | Remove dead `NJOBS`/`jobid`/`load_jobs` body (P0-0); move `xce_reply()` after `AFTER_EVENT` in `handle_jobsubmit()` | | `src/lib/kvmgr.c` | Register `persist_nop_freeze`/`persist_nop_thaw` for the **kvmgr hash table's own** varray interface (`kvmgr_type` in `kvmgr.c`). The kvmgr has TWO varray interfaces: (1) the hash table's varray (kvmgr itself) — which does NOT support COW and uses no-op freeze/thaw; and (2) the data zarray (`xzarray_type` in `zarray.c`) — which IS the COW target and uses the real `xzarray_freeze`/`xzarray_thaw` implementations. An implementer must NOT accidentally register no-ops for the data zarray, or checkpoint COW will silently skip freezing the data array. The `xzarray_type` registration in `zarray.c` already includes `xzarray_freeze`/`xzarray_thaw` slots — it is NOT modified by P0-0b. P0-0b only adds no-op slots to the kvmgr hash table's own varray compound literal in `kvmgr.c`. | -| `src/controller/cluster.c` | Wire `persist_init/recover/start/stop/fini` and HA into lifecycle; wire `cluster_load_jobs()` to persistence loader; deferred endpoint cleanup (after zarray merge); use `delaymgr` for checkpoint timer and heartbeat watchdog | +| `src/controller/cluster.c` | Wire `persist_init/recover/start/stop/fini` and HA into lifecycle; wire `cluster_load_jobs()` to persistence loader; deferred endpoint cleanup (after zarray merge); use `delaymgr` for checkpoint timer; heartbeat watchdog uses dedicated pthread (see P1-3) | | `src/controller/include/persist.h` | Add `persist_nop_freeze`/`persist_nop_thaw` static inline no-ops for varray types that don't support COW | -| `src/controller/netio.c` + `src/include/xce/netio.h` | Extend `create_ssl_context()` to accept `ca_cert_path` parameter for mTLS (P1-1); update all existing callers to pass NULL for CA cert | +| `src/controller/netio.c` + `src/include/xce/netio.h` | No changes needed for HA — existing SSL pipeline reused as-is for controller-to-controller connections (trusted LAN model, Part 5.9) | | `src/controller/join.c` | Add redirect logic in `handle_join()` — standby sends `E_XCE_REDIRECT` + `RedirectInfo` for worker/scheduler/apiserver join requests; worker join WRITE lock is acceptable — async replication only (never blocks on `repl_wait_ack` inside write lock) | | `src/controller/worker.c` | Extend existing `mark_worker_unavail()` with `w->host.dead_since = time(NULL)`; add `WORKER_HIDDEN` state path; `handle_worker_leave()` already calls `mark_worker_unavail()` and sets `w->ep = NULL` — no structural change needed; wire `restore_worker()` into recovery path | | `src/include/xce/cluster_def.h` | Add `persist_t persist;` (P0-5) and `ha_state_t ha;` (P1-5) to `cluster_t`; add `WORKER_UNAVAILABLE`, `WORKER_HIDDEN` entity states; worker status uses bitmask flags (e.g., `XCE_WORKER_HIDDEN = 0x00200000`), not state enum | @@ -3618,10 +4138,11 @@ All system-level tunable constants are consolidated into `include/config.h` so t /* include/config.h — System-level tunable constants for XCE controller * All #define values below are the system-wide defaults. * Runtime-overridable values (persist_config_t, ha_config_t) are NOT here — - * they live in their respective config structs and are set from cluster.yaml. */ + * they live in their respective config structs and are set from cluster.bin (controller) + * or from --peers/--ca-cert CLI flags (workers/schedulers). */ /* --- HA limits --- */ -#define MAX_HA_CANDIDATES 4 /* Structural max candidates in cluster.yaml. +#define MAX_HA_CANDIDATES 4 /* Structural max candidates from --peers flag. * Reduced from 16: design validated for ≤3. */ #define DEFAULT_HB_INTERVAL_MS 3000 /* Primary heartbeat interval (ms) */ #define DEFAULT_HB_TIMEOUT_MS 10000 /* Standby heartbeat timeout (ms) */ @@ -3629,6 +4150,12 @@ All system-level tunable constants are consolidated into `include/config.h` so t #define DEFAULT_SYNC_TIMEOUT_MS 50 /* Sync replication ACK timeout (ms) */ #define DEFAULT_MAX_PROMOTION_RETRIES 5 /* Max failed promotion attempts */ #define DEFAULT_HA_CERT_WARN_DAYS 30 /* Warn if cert expires within N days */ +#define MAX_TERM_JUMP 10000 /* Maximum term increment accepted in a single replication + * or OP_replEntry message. Prevents term overflow + * from a corrupted or malicious packet forcing the + * controller into permanent step-down mode. + * Defined in config.h (not inline in Part 6.4) so + * all callers share a single source of truth. */ /* --- Persistence limits --- */ #define WAL_FORMAT_VERSION 1 /* Bumped on incompatible schema changes. @@ -3641,6 +4168,11 @@ All system-level tunable constants are consolidated into `include/config.h` so t #define MAX_WAL_PAYLOAD_BYTES (1u << 20) /* 1 MB per WAL entry */ #define WAL_ENTRY_TYPE_MAX 100 /* Highest valid entry type — bump when new types added */ #define MAX_SNAPSHOT_BYTES (256 * 1024 * 1024) /* 256 MB — reject larger snapshots */ +#define MAX_AUTO_TERM_PER_CHECKPOINT 10240 /* Max JOB_STAT_TRANSIENT jobs auto-terminated in one + * checkpoint cycle. Bounds the heap-allocated + * auto_term[] array in persist_checkpoint(). + * Jobs beyond this cap survive to the next + * checkpoint (bounded delay, not data loss). */ /* --- Persistence defaults (runtime-overridable via persist_config_t) --- */ #define DEFAULT_WAL_SYNC_MS 10 /* fsync interval (ms) */ @@ -3650,7 +4182,11 @@ All system-level tunable constants are consolidated into `include/config.h` so t #define DEFAULT_WORKER_RETAIN_S 86400 /* retain unavailable workers (s) */ /* --- Derived --- */ -#define MAX_STANDBYS (MAX_HA_CANDIDATES - 1) /* 3 — matches ≤3 deployment ceiling */ +#define MAX_STANDBYS 3 /* Standalone constant — NOT derived from MAX_HA_CANDIDATES. + * Deployment ceiling is ≤3 controllers (1 primary + 2 standbys). + * Defined independently so bumping MAX_HA_CANDIDATES for structural + * reasons does not silently increase MAX_STANDBYS beyond the + * validated deployment ceiling. */ ``` **Design rule**: If a constant affects more than one module or is an operator-visible limit, it belongs in `config.h`. If it is a purely internal implementation detail of a single module (e.g., an internal struct size or a local retry count), it may stay in the module header. The test: if an operator would ask "what's the maximum?" or a developer would search for the constant when debugging, it goes in `config.h`. \ No newline at end of file diff --git a/src/include/myconf.h b/src/include/myconf.h index 0b290e2..fcfe762 100644 --- a/src/include/myconf.h +++ b/src/include/myconf.h @@ -7,6 +7,7 @@ #include #include "clusterconfig.h" +#include "xce/cluster.h" struct logging { char* logdir; @@ -14,12 +15,13 @@ struct logging { }; typedef struct { - cluster_config_t* cluster_config; - char* conffile; - char* certfile; - char* keyfile; - char* workdir; /* working directory for WAL/snapshot storage */ - char* name; /* controller name (hostname by default) */ + cluster_config_t* cluster_config; + char* conffile; + char* certfile; + char* keyfile; + char* workdir; /* working directory for WAL/snapshot storage */ + char* name; /* controller name (hostname by default) */ + xce_cluster_config_t peers_cfg; /* HA peer list parsed from --peers; count==0 = single-node */ } myconf_t; extern cluster_config_t* get_default_conf(void*);