From c8e55947c23b756e7274823b6842963cd4b7e14d Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Sun, 14 Jun 2026 03:12:25 +0000 Subject: [PATCH] test(ha-convergence-recovery): RED demotion-convergence + self-restart tests + seams (task 2.1) --- .kiro/specs/ha-convergence-recovery/tasks.md | 4 +- src/controller/ha.c | 15 + src/controller/ha.h | 76 +++ src/controller/ha_convergence.c | 70 +++ src/controller/meson.build | 3 + src/controller/testsuite/controller_test.c | 2 + .../testsuite/test_demotion_convergence.c | 499 ++++++++++++++++++ 7 files changed, 668 insertions(+), 1 deletion(-) create mode 100644 src/controller/ha_convergence.c create mode 100644 src/controller/testsuite/test_demotion_convergence.c diff --git a/.kiro/specs/ha-convergence-recovery/tasks.md b/.kiro/specs/ha-convergence-recovery/tasks.md index 290f51c..76e4856 100644 --- a/.kiro/specs/ha-convergence-recovery/tasks.md +++ b/.kiro/specs/ha-convergence-recovery/tasks.md @@ -19,7 +19,7 @@ - _Requirements: 3.1, 3.2, 3.3, 3.4_ - [ ] 2. Demotion convergence: self-restart primitive and the truncate-on-demotion path -- [ ] 2.1 Write RED tests for the self-restart primitive and demotion sequence +- [x] 2.1 Write RED tests for the self-restart primitive and demotion sequence - Create `testsuite/test_demotion_convergence.c` and register it in meson; declare `controller_saved_argv`, `ha_self_restart`, and the `XCE_TESTING` seams (`ha_restart_hook`, `ha_fatal_hook`) in ha.h so tests compile - Tests: become-follower invoked while role is CTRL_PRIMARY → persistence hooks disabled (verify via `hooks_registered` reset and a hook-counter probe) BEFORE further events are persisted, `persist_truncate_wal_above` called with the `committed_lsn` cutoff, WARN with both LSNs emitted, `ha_restart_hook` invoked exactly once; `ha_self_restart` with a failing restart hook path → `ha_fatal_hook` invoked (FATAL contract); become-follower from STANDBY/ELECTING/INIT → no truncation, no restart (regression); integration: append a committed prefix + uncommitted suffix, run the demotion seam, then replay the surviving WAL into a fresh context → committed-only state (provisional entries absent) - Observable completion: new cases FAIL (red) — current `raft_cb_become_follower` (ha.c:627-690 anchor) neither truncates nor disables hooks @@ -98,3 +98,5 @@ - **Test-link rule**: a production symbol that the unit-test suite already stubs cannot have its owning `.c` added to `controller_unit_test_src`. Factor newly-testable logic into its own TU instead. - **`restore_job` gained an `entry_lsn` parameter** (replay tagging). Callers updated: cluster.h decl, job.c def, persist.c extern + apply_wal_entry caller, test_repl_apply.c stub. Any new caller must pass the WAL entry header LSN (`hdr->lsn`), or `0` if unknown (0 = exportable). - **Build note**: the full meson controller suite SIGTERMs at the 300s wall-clock cap on the slow commit-barrier timeout suites (pre-existing). For per-task GREEN confirmation, run `controller_unit_test -t ` directly with `ASAN_OPTIONS=detect_leaks=0` (harness teardown leaks are a pre-existing accepted pattern; the meson gate sets the same option). +- **Task 2.1 RED scaffold + anchor drift (post ha-single-leader-role)**: throwaway stubs live in new `src/controller/ha_convergence.c` (`controller_saved_argv`→NULL, `ha_self_restart`→return 0, seam pointers `ha_restart_hook`/`ha_fatal_hook`=NULL) — **task 2.2 moves the real `controller_saved_argv` to main.c and `ha_self_restart` + the demotion branch to ha.c** per design file-layout. A test-only seam `raft_cb_become_follower_with_cluster_for_test` (preserves `raft_c=c`) was added to ha.c. Verified anchors: `raft_cb_become_follower` ha.c:622; `ha_demote_to_standby` ha.c:1711 (hook-disable sequence to reuse); `persist_truncate_wal_above` persist.c:1545; role enum CTRL_PRIMARY=0/STANDBY=1/INIT=2/ELECTING=3 (ha.h:59-64); `committed_lsn`/`current_lsn` on `persist_t` in src/include/persist.h; MAX_STANDBYS=3. +- **Follow-up for task 2.2**: add a `CTRL_INIT` non-PRIMARY regression case to `test_demotion_convergence.c` (currently only STANDBY+ELECTING are exercised though the comment says "ELECTING/INIT") and correct/keep the comment accurate. 1.5 (WARN with both LSNs) is verified only by the truncation-cutoff proxy — no log-capture harness exists; covered directly only at E2E (task 8). diff --git a/src/controller/ha.c b/src/controller/ha.c index afe9983..67860d0 100644 --- a/src/controller/ha.c +++ b/src/controller/ha.c @@ -3063,6 +3063,21 @@ void raft_cb_become_follower_for_test(ha_state_t *ha, uint64_t term) ha->raft_c = NULL; raft_cb_become_follower((void *)ha, term, -1); } + +/* + * raft_cb_become_follower_with_cluster_for_test (ha-convergence-recovery + * task 2.1): drive the follower path with a LIVE cluster pointer so the + * cluster-dependent demotion-convergence branch (task 2.2) is exercised + * against a real persist/hooksys/delaymgr fixture. Sets ha->raft_c = c so + * the become-follower callback sees a real cluster (unlike the NULL-cluster + * shim above). raft_cb_become_follower is file-static, so this seam must + * live here in ha.c. + */ +void raft_cb_become_follower_with_cluster_for_test(cluster_t *c, uint64_t term) +{ + c->ha.raft_c = c; + raft_cb_become_follower((void *)&c->ha, term, -1); +} #endif /* XCE_TESTING */ /* ── Task 2.4: vote_ep_err_handler ───────────────────────────────────────── diff --git a/src/controller/ha.h b/src/controller/ha.h index 4aebe1f..60d4cc9 100644 --- a/src/controller/ha.h +++ b/src/controller/ha.h @@ -239,6 +239,41 @@ void ha_do_promote(ha_state_t *ha); void ha_do_demote(ha_state_t *ha); +/* ── Demotion convergence + self-restart primitive ───────────── + * (ha-convergence-recovery Requirement 1: a demoted ex-primary discards its + * uncommitted, potentially divergent state and rejoins from committed state + * only via truncation + process self-restart through boot recovery.) + */ + +/* + * controller_saved_argv: accessor for the NULL-terminated argv captured by + * main() before argp_parse, used by ha_self_restart to re-exec the controller + * (ha-convergence-recovery Req 1.3, design "Self-Restart Primitive"). + * Returns NULL if argv was never saved (e.g. unit-test binaries). + */ +char **controller_saved_argv(void); + +/* + * ha_self_restart: the truncate-and-restart primitive's restart half. + * + * Production contract (Req 1.3, 1.4): flush logs, execv(controller_saved_argv()) + * to rebuild in-memory state through boot recovery from the persisted committed + * state only; if exec cannot be performed, terminate FATALLY (abort) rather than + * continue running with potentially divergent state — NEVER returns in + * production. + * + * Test contract (Req 1.6): under XCE_TESTING, invoke ha_restart_hook instead of + * execv and return its result (0 on the success path); a failing ha_restart_hook + * routes to ha_fatal_hook in place of abort so tests can assert the FATAL path + * without dying. + * + * Preconditions: persistence hooks already disabled; truncation completed + * before this call. Invariant: no new WAL entry is appended by this node + * between hook-disable and exec. + */ +int ha_self_restart(cluster_t *c); + + /* ── Redirect helpers used by join.c ──────────────────────── */ /* @@ -768,6 +803,47 @@ extern void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c, * Requirements: task 7.3 (ha-peer-connection-reuse). */ void vote_ep_ctx_set_cluster_for_test(struct endpoint_t *ep, cluster_t *c); + +/* + * ha_restart_hook: test-only seam for ha_self_restart's process re-execution + * (ha-convergence-recovery Req 1.6). + * + * When non-NULL, ha_self_restart() invokes this INSTEAD OF execv() so unit + * tests can observe the eventual restart deterministically (the demotion + * convergence task and rejoin reconciliation both end in ha_self_restart). + * The hook's return value is ha_self_restart's return value on the + * test-success path; a NON-ZERO return models an exec failure and routes + * ha_self_restart to the FATAL contract via ha_fatal_hook (Req 1.4). + * + * Signature: int hook(cluster_t *c) + * Returns 0 to model a successful restart; non-zero to model exec failure. + */ +extern int (*ha_restart_hook)(cluster_t *c); + +/* + * ha_fatal_hook: test-only seam for the FATAL (abort) contract of + * ha_self_restart (ha-convergence-recovery Req 1.4). + * + * When non-NULL, ha_self_restart() invokes this INSTEAD OF abort() when the + * restart cannot be performed, so a test binary can assert the FATAL path was + * taken without the process dying. ha_self_restart returns after invoking it + * (test builds only — production aborts). + * + * Signature: void hook(const char *msg) + */ +extern void (*ha_fatal_hook)(const char *msg); + +/* + * raft_cb_become_follower_with_cluster_for_test: drive the follower path with + * a LIVE cluster pointer (ha-convergence-recovery task 2.1 demotion-sequence + * tests). Unlike raft_cb_become_follower_for_test (which forces raft_c = NULL + * to suppress production calls), this shim sets ha->raft_c = c first so the + * cluster-dependent demotion-convergence branch (task 2.2: hook disable + + * scheduled truncate/restart) is exercised against a real persist/hooksys/ + * delaymgr fixture. Mirrors the real-cluster seam ha_promote_to_primary(c) + * used by the promotion-side lifecycle tests. + */ +void raft_cb_become_follower_with_cluster_for_test(cluster_t *c, uint64_t term); #endif #endif /* HA_H */ diff --git a/src/controller/ha_convergence.c b/src/controller/ha_convergence.c new file mode 100644 index 0000000..886f8fa --- /dev/null +++ b/src/controller/ha_convergence.c @@ -0,0 +1,70 @@ +/* + * ha_convergence.c — demotion convergence + self-restart primitive + * (ha-convergence-recovery Requirement 1). + * + * ─────────────────────────────────────────────────────────────────────────── + * RED SCAFFOLD (task 2.1) — task 2.2 REPLACES the bodies below with the real + * implementation (and moves controller_saved_argv's backing store into main.c). + * + * This file currently provides ONLY the link-time definitions the task-2.1 RED + * tests need to compile and link: + * - controller_saved_argv() — stub returning NULL (real argv capture is in + * main.c, task 2.2) + * - ha_self_restart() — RED stub: does NOT yet honor the ha_restart_hook + * / ha_fatal_hook contract, so the fatal-hook test + * FAILS until task 2.2 implements the real + * flush → execv → abort sequence (and the + * XCE_TESTING seam routing). + * - ha_restart_hook / ha_fatal_hook — the XCE_TESTING function-pointer seams, + * defined =NULL in exactly one TU (this one), + * mirroring the ha_*_hook seam pattern in ha.c. + * + * The DEMOTION-SEQUENCE behavior (hook disable on become-follower-from-PRIMARY, + * scheduled truncate-above-committed_lsn, restart) lives in raft_cb_become_ + * follower (ha.c) and is NOT implemented here or there yet — that is task 2.2. + * Consequently every demotion-convergence test in test_demotion_convergence.c + * is RED against the current tree. + * ─────────────────────────────────────────────────────────────────────────── + */ + +#include + +#include "cluster.h" +#include "ha.h" +#include "xce/cluster_def.h" + +#ifdef XCE_TESTING +/* XCE_TESTING seams (Req 1.6, 1.4) — defined here, declared in ha.h. + * Tests set these before exercising the restart/fatal contract. */ +int (*ha_restart_hook)(cluster_t *c) = NULL; +void (*ha_fatal_hook)(const char *msg) = NULL; +#endif + +/* + * controller_saved_argv — RED stub (task 2.2 moves the real, main()-populated + * accessor into main.c). Returns NULL: no argv has been captured in the + * scaffold, so a production execv would have nothing to exec. + */ +char **controller_saved_argv(void) +{ + return NULL; +} + +/* + * ha_self_restart — RED stub. + * + * Task 2.2 replaces this with the real primitive: + * - production: flush logs, execv(controller_saved_argv()); on exec return, + * ERROR + abort() (FATAL — Req 1.4); never returns. + * - XCE_TESTING: invoke ha_restart_hook(c) instead of execv (Req 1.6); if it + * reports failure (or is unset), invoke ha_fatal_hook in place of abort. + * + * Until then this stub deliberately does NOTHING (no hook routing), so the + * task-2.1 fatal-hook test — which asserts a failing restart hook drives + * ha_fatal_hook — FAILS (RED). + */ +int ha_self_restart(cluster_t *c) +{ + (void)c; + return 0; /* RED scaffold — task 2.2 implements exec/abort + seam routing */ +} diff --git a/src/controller/meson.build b/src/controller/meson.build index ce08698..77491cb 100644 --- a/src/controller/meson.build +++ b/src/controller/meson.build @@ -5,6 +5,7 @@ controller_src = [ 'dataex.c', 'egress.c', 'ha.c', + 'ha_convergence.c', 'job.c', 'join.c', 'main.c', @@ -124,9 +125,11 @@ controller_unit_test_src = [ 'testsuite/test_commit_barrier_rollout.c', 'testsuite/test_commit_barrier_reply.c', 'testsuite/test_egress_filter.c', + 'testsuite/test_demotion_convergence.c', 'commit.c', 'job.c', 'egress.c', + 'ha_convergence.c', 'queue.c', 'join.c', 'opupd.c', diff --git a/src/controller/testsuite/controller_test.c b/src/controller/testsuite/controller_test.c index 3430003..f4a6ede 100644 --- a/src/controller/testsuite/controller_test.c +++ b/src/controller/testsuite/controller_test.c @@ -53,6 +53,7 @@ extern void test_commit_engine(void); extern void test_commit_barrier_rollout(void); extern void test_commit_barrier_reply(void); extern void test_egress_filter(void); +extern void test_demotion_convergence(void); void setUp(void) { return; } @@ -175,6 +176,7 @@ int main(int argc, char* argv[]) TEST(test_commit_barrier_rollout); TEST(test_commit_barrier_reply); TEST(test_egress_filter); + TEST(test_demotion_convergence); return UNITY_END(); } diff --git a/src/controller/testsuite/test_demotion_convergence.c b/src/controller/testsuite/test_demotion_convergence.c new file mode 100644 index 0000000..0a2b88d --- /dev/null +++ b/src/controller/testsuite/test_demotion_convergence.c @@ -0,0 +1,499 @@ +#define _GNU_SOURCE +/* + * test_demotion_convergence.c — RED-phase tests for the demotion-convergence + * sequence and the self-restart primitive (ha-convergence-recovery task 2.1). + * + * Requirements (this spec's IDs — see requirements.md): + * 1.1 — on demotion from PRIMARY the controller STOPS persisting new state + * BEFORE continuing demotion (persistence hooks disabled synchronously, + * observed via hooks_registered reset + a hook-counter probe). + * 1.2 — on demotion the controller removes all log entries ABOVE committed_lsn + * (truncate above the committed boundary). + * 1.3 — when truncation completes the controller self-restarts so in-memory + * state is rebuilt by boot recovery from the committed prefix only. + * 1.4 — if the self-restart cannot be performed, terminate FATALLY rather + * than continue (FATAL contract; XCE_TESTING routes to ha_fatal_hook). + * 1.5 — the demotion is logged with the truncation boundary (committed LSN and + * prior log tip) before restarting (observed here via the truncation + * cutoff: current_lsn collapses to committed_lsn + 1). + * 1.6 — under test mode the controller invokes the ha_restart_hook seam in + * place of process re-execution. + * 1.7 — in-memory state is reconstructible from the persisted committed state + * only (provisional entries above committed_lsn absent after replay). + * 2.4 — single-outcome convergence (in-process half): a committed prefix + + * uncommitted suffix, run through the demotion seam, replays to a + * committed-only state. + * + * RED status against the current tree: raft_cb_become_follower (ha.c) neither + * disables the persistence hooks nor schedules any truncate/restart work on the + * PRIMARY→follower edge, and ha_self_restart is a RED scaffold (ha_convergence.c) + * that does not yet honor the restart/fatal seams. Every case below therefore + * FAILS until task 2.2 lands the demotion branch + the real primitive. + * + * Harness: a partial-but-real cluster_t (real persist over a private tmpdir, + * wired commit engine, stub standby registry, un-started delaymgr) mirroring + * test_ha_role_lifecycle.c's lifecycle_cluster_setup, plus the live-cluster + * become-follower seam raft_cb_become_follower_with_cluster_for_test. + * + * AAA structure throughout. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "commit.h" /* commit_engine_t, commit_engine_init */ +#include "delay.h" /* delaymgr_t, delay_task_t */ +#include "ha.h" /* raft_cb_become_follower_with_cluster_for_test, + * ha_self_restart, controller_saved_argv, + * ha_restart_hook / ha_fatal_hook seams */ +#include "ordq.h" /* ordq_item_t (drive the scheduled task) */ +#include "persist.h" /* persist_t, persist_truncate_wal_above */ +#include "repl.h" /* repl_add_standby / repl_set_commit_engine */ +#include "xce/cluster_def.h" /* full cluster_t layout */ + +#ifdef XCE_TESTING + +/* ── Broadcast drop-hook (precedent: test_ha_role_lifecycle.c lc_drop_send_hook): + * the stub int endpoints registered via repl_add_standby are not real + * endpoint_t, so repl_broadcast must never dereference ep->peer. Installing + * this hook drops the wire send harmlessly during priming appends. */ +extern int (*repl_broadcast_send_hook)(void *ep, + wal_catchup_entry_t **entries, + int n_entries); +static int dc_drop_send_hook(void *ep, wal_catchup_entry_t **entries, + int n_entries) +{ + (void)ep; (void)entries; (void)n_entries; + return 0; +} + +/* ── Restart / fatal seam counters (Req 1.4, 1.6) ───────────────────────── */ +static atomic_int g_restart_calls; +static atomic_int g_fatal_calls; +static int g_restart_hook_rc; /* return code dc_restart_hook reports */ + +static int dc_restart_hook(cluster_t *c) +{ + (void)c; + atomic_fetch_add(&g_restart_calls, 1); + return g_restart_hook_rc; +} + +static void dc_fatal_hook(const char *msg) +{ + (void)msg; + atomic_fetch_add(&g_fatal_calls, 1); +} + +/* ── Cluster fixture (mirror of lifecycle_cluster_setup) ────────────────── */ +static cluster_t g_dc; /* large — must not live on the test stack */ +static int g_dc_eps[MAX_STANDBYS]; +static char g_dc_workdir[256]; /* the fixture tmpdir (workdir for replay) */ +#define DC_PERSIST_NAME "ctrl-test" + +static ha_config_t dc_ha_cfg(void) +{ + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, + .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, + .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + }; + return cfg; +} + +/* Build g_dc: real persist over a fresh tmpdir, n_standbys stub standbys, a + * wired commit engine, an initialised-but-UNSTARTED delaymgr (so any task in + * tasks.head was armed by the code under test, not by a background runner), + * role forced to CTRL_PRIMARY with the persist hooks registered (so a demotion + * has hooks to disable). Leaves committed_lsn at 0; callers set it. */ +static void dc_setup(const char *suffix, int n_peers, int n_standbys, + uint64_t appended_lsn, int n_vcpus) +{ + char name[32]; + char tmpdir[256]; + + memset(&g_dc, 0, sizeof(g_dc)); + atomic_store(&g_restart_calls, 0); + atomic_store(&g_fatal_calls, 0); + g_restart_hook_rc = 0; + + ha_config_t cfg = dc_ha_cfg(); + TEST_ASSERT_EQUAL_INT(0, ha_init(&g_dc.ha, &cfg, 0)); + g_dc.ha.phase2_done = 1; /* no phase2 wiring in this fixture */ + g_dc.ha.n_peers = n_peers; + + persist_config_t pcfg = persist_config_defaults(); + pcfg.wal_sync_ms = 5000; /* slow background fsync — no races */ + pcfg.sync_timeout_ms = 50; /* no real standby ACKs here */ + snprintf(tmpdir, sizeof(tmpdir), "/tmp/xce_dc_%s_XXXXXX", suffix); + TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); + snprintf(g_dc_workdir, sizeof(g_dc_workdir), "%s", tmpdir); + persist_set_workdir(&g_dc.persist, tmpdir, DC_PERSIST_NAME); + TEST_ASSERT_EQUAL_INT(0, persist_init(&g_dc.persist, &pcfg, &g_dc)); + TEST_ASSERT_EQUAL_INT(0, persist_start(&g_dc.persist)); + + /* Prime the appended watermark: current_lsn = appended_lsn + 1. */ + pthread_mutex_lock(&g_dc.persist.lock); + g_dc.persist.current_lsn = appended_lsn + 1; + pthread_mutex_unlock(&g_dc.persist.lock); + atomic_store(&g_dc.persist.committed_lsn, 0); + + for (int i = 0; i < n_standbys; i++) { + snprintf(name, sizeof(name), "dcsb%d", i); + TEST_ASSERT_TRUE_MESSAGE( + repl_add_standby(&g_dc.persist.repl, &g_dc_eps[i], name, + 7500 + (uint32_t)i) >= 0, + "repl_add_standby must return a slot index >= 0"); + } + TEST_ASSERT_EQUAL_INT(0, + commit_engine_init(&g_dc.commit, &g_dc.persist, n_peers, n_vcpus)); + repl_set_commit_engine(&g_dc.persist.repl, &g_dc.commit); + + repl_broadcast_send_hook = dc_drop_send_hook; + + /* delaymgr initialised but NOT started — the demotion-convergence task, + * once task 2.2 schedules it, lands in tasks.head and is driven manually + * by dc_run_pending_task (D1: truncation + restart run off the callback). */ + TEST_ASSERT_EQUAL_INT(0, delaymgr_init(&g_dc.delaymgr, 100, NULL)); + + /* Make this a PRIMARY with hooks registered, so a demotion has the three + * persistence hooks to disable (Req 1.1). ha_promote_to_primary runs the + * shared promotion sequence (ha_do_promote → register hooks → heartbeat → + * own-term gate); the un-started delaymgr keeps the heartbeat arm skipped. */ + TEST_ASSERT_EQUAL_INT(0, ha_promote_to_primary(&g_dc)); + TEST_ASSERT_EQUAL_INT_MESSAGE( + (int)CTRL_PRIMARY, + (int)atomic_load_explicit(&g_dc.ha.role, memory_order_acquire), + "fixture precondition: node must be CTRL_PRIMARY before demotion"); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_dc.ha.hooks_registered, + "fixture precondition: persist hooks must be registered before demotion"); +} + +static void dc_teardown(void) +{ + repl_broadcast_send_hook = NULL; + ha_restart_hook = NULL; + ha_fatal_hook = NULL; + commit_engine_fini(&g_dc.commit); + delaymgr_fini(&g_dc.delaymgr); + persist_stop(&g_dc.persist); + persist_fini(&g_dc.persist); + repl_fini(&g_dc.persist.repl); + ha_fini(&g_dc.ha); + /* tmpdir left on disk — /tmp is ephemeral */ +} + +/* Count pending delaymgr tasks by walking the ordq head chain (read-only; + * single-threaded test, delaymgr never started). Same seam the lifecycle + * suite uses to assert promotion arms no watchdog. */ +static int dc_task_count(const delaymgr_t *mgr) +{ + int count = 0; + const ordq_item_t *it = mgr->tasks.head; + while (it != NULL) { + count++; + it = it->next; + } + return count; +} + +/* Run the single pending delaymgr task synchronously by invoking its stored + * entry(args), then return the entry's rc. Returns INT_MIN sentinel if there + * is no pending task (RED: nothing was scheduled). The task is NOT removed + * from the queue (delaymgr_fini frees it at teardown) — we only need to run + * its body once to observe truncation + restart. */ +#define DC_NO_TASK 0x7fffffff +static int dc_run_pending_task(const delaymgr_t *mgr) +{ + const ordq_item_t *it = mgr->tasks.head; + if (it == NULL || it->data == NULL) + return DC_NO_TASK; + const delay_task_t *t = (const delay_task_t *)it->data; + if (t->entry == NULL) + return DC_NO_TASK; + return t->entry(t->args); +} + +/* ── Tests ──────────────────────────────────────────────────────────────── */ + +/* 1.1: demotion from PRIMARY disables the persistence hooks synchronously in + * the callback (before any further event could be persisted), observed via + * hooks_registered transitioning 1 → 0. + * RED: raft_cb_become_follower never disables hooks on the PRIMARY edge. */ +static void test_demotion_from_primary_disables_hooks(void) +{ + dc_setup("hooks", /*n_peers=*/3, /*n_standbys=*/2, + /*appended_lsn=*/5, /*n_vcpus=*/8); + atomic_store(&g_dc.persist.committed_lsn, 3); + + /* Act: drive become-follower from PRIMARY with a LIVE cluster. */ + raft_cb_become_follower_with_cluster_for_test(&g_dc, /*term=*/7); + + /* Assert: hooks disabled synchronously (Req 1.1). */ + TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_dc.ha.hooks_registered, + "demotion from PRIMARY must disable the persistence hooks " + "synchronously (hooks_registered 1→0, Req 1.1)"); + + dc_teardown(); + printf("test_demotion_from_primary_disables_hooks: checked\n"); +} + +/* 1.2/1.3/1.5/1.6: demotion from PRIMARY schedules exactly one convergence task + * (off the callback thread, D1); running it truncates above committed_lsn and + * invokes the restart seam exactly once. + * RED: no task is scheduled, so dc_run_pending_task returns DC_NO_TASK and the + * truncation/restart assertions fail. */ +static void test_demotion_schedules_convergence_truncate_and_restart(void) +{ + dc_setup("conv", /*n_peers=*/3, /*n_standbys=*/2, + /*appended_lsn=*/9, /*n_vcpus=*/8); + /* committed prefix ends at LSN 4; entries 5..9 are the uncommitted suffix. */ + atomic_store(&g_dc.persist.committed_lsn, 4); + + /* Install the test restart seam so the eventual self-restart is observable + * deterministically rather than exec'ing the test binary (Req 1.6). */ + ha_restart_hook = dc_restart_hook; + ha_fatal_hook = dc_fatal_hook; + g_restart_hook_rc = 0; /* model a successful restart */ + + int tasks_before = dc_task_count(&g_dc.delaymgr); + + /* Act: demotion edge. */ + raft_cb_become_follower_with_cluster_for_test(&g_dc, /*term=*/7); + + /* The convergence work runs OFF the callback (D1) — exactly one one-shot + * delaymgr task must have been scheduled (Req 1.2/1.3 deferred). */ + int tasks_after = dc_task_count(&g_dc.delaymgr); + TEST_ASSERT_EQUAL_INT_MESSAGE(tasks_before + 1, tasks_after, + "demotion from PRIMARY must schedule exactly one convergence task " + "(truncate+restart off the callback thread, D1)"); + + /* Run the scheduled convergence task synchronously. */ + int rc = dc_run_pending_task(&g_dc.delaymgr); + TEST_ASSERT_NOT_EQUAL_MESSAGE(DC_NO_TASK, rc, + "no convergence task was scheduled on the demotion edge (Req 1.2/1.3)"); + + /* Assert (Req 1.2/1.5): truncation removed everything above committed_lsn — + * current_lsn collapses to committed_lsn + 1 (the truncation boundary). */ + TEST_ASSERT_EQUAL_UINT64_MESSAGE( + atomic_load(&g_dc.persist.committed_lsn) + 1, g_dc.persist.current_lsn, + "demotion convergence must truncate above committed_lsn (Req 1.2): " + "current_lsn must collapse to committed_lsn + 1"); + + /* Assert (Req 1.3/1.6): the restart seam was invoked exactly once. */ + TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_restart_calls), + "demotion convergence must invoke the restart seam exactly once " + "after truncation (Req 1.3, 1.6)"); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_fatal_calls), + "a successful restart must NOT invoke the fatal seam"); + + dc_teardown(); + printf("test_demotion_schedules_convergence_truncate_and_restart: checked\n"); +} + +/* 1.4: ha_self_restart with a FAILING restart hook must invoke the fatal seam + * (the FATAL contract — a divergent ex-primary must never continue serving). + * RED: the ha_self_restart scaffold (ha_convergence.c) does not route to the + * seams yet, so neither hook fires. */ +static void test_self_restart_failure_invokes_fatal_hook(void) +{ + dc_setup("fatal", /*n_peers=*/3, /*n_standbys=*/2, + /*appended_lsn=*/5, /*n_vcpus=*/8); + + ha_restart_hook = dc_restart_hook; + ha_fatal_hook = dc_fatal_hook; + g_restart_hook_rc = -EIO; /* model an exec failure */ + + /* Act: invoke the primitive directly. */ + (void)ha_self_restart(&g_dc); + + /* Assert (Req 1.4/1.6): the restart seam was attempted, and because it + * failed, the fatal seam fired in place of abort(). */ + TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_restart_calls), + "ha_self_restart must invoke the restart seam under XCE_TESTING (Req 1.6)"); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_fatal_calls), + "a failing restart must invoke the fatal seam — the FATAL contract " + "(Req 1.4); the ex-primary must never continue with divergent state"); + + dc_teardown(); + printf("test_self_restart_failure_invokes_fatal_hook: checked\n"); +} + +/* Regression: demotion from STANDBY must NOT truncate, NOT restart, NOT disable + * hooks beyond today's behavior — byte-equivalent to the pre-spec path. A + * STANDBY has no registered hooks and no convergence work. */ +static void test_demotion_from_standby_no_truncation_no_restart(void) +{ + dc_setup("standby", /*n_peers=*/3, /*n_standbys=*/2, + /*appended_lsn=*/9, /*n_vcpus=*/8); + atomic_store(&g_dc.persist.committed_lsn, 4); + + /* Force role to STANDBY and disable hooks (the non-PRIMARY precondition). */ + TEST_ASSERT_EQUAL_INT(0, ha_demote_to_standby(&g_dc)); + TEST_ASSERT_EQUAL_INT((int)CTRL_STANDBY, + (int)atomic_load_explicit(&g_dc.ha.role, memory_order_acquire)); + + ha_restart_hook = dc_restart_hook; + ha_fatal_hook = dc_fatal_hook; + + uint64_t tip_before = g_dc.persist.current_lsn; + int tasks_before = dc_task_count(&g_dc.delaymgr); + + /* Act: a second become-follower from a non-PRIMARY role. */ + raft_cb_become_follower_with_cluster_for_test(&g_dc, /*term=*/8); + + /* Assert: no convergence task, no truncation, no restart. */ + TEST_ASSERT_EQUAL_INT_MESSAGE(tasks_before, + dc_task_count(&g_dc.delaymgr), + "demotion from STANDBY must schedule NO convergence task (regression)"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(tip_before, g_dc.persist.current_lsn, + "demotion from STANDBY must NOT truncate the WAL (regression)"); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_restart_calls), + "demotion from STANDBY must NOT self-restart (regression)"); + + dc_teardown(); + printf("test_demotion_from_standby_no_truncation_no_restart: checked\n"); +} + +/* Regression: demotion from ELECTING/INIT must NOT truncate or restart. */ +static void test_demotion_from_electing_no_truncation_no_restart(void) +{ + dc_setup("electing", /*n_peers=*/3, /*n_standbys=*/2, + /*appended_lsn=*/9, /*n_vcpus=*/8); + atomic_store(&g_dc.persist.committed_lsn, 4); + + /* No registered hooks on a candidate; force role to ELECTING. */ + TEST_ASSERT_EQUAL_INT(0, ha_demote_to_standby(&g_dc)); /* clears hooks */ + atomic_store_explicit(&g_dc.ha.role, (int32_t)CTRL_ELECTING, + memory_order_release); + + ha_restart_hook = dc_restart_hook; + ha_fatal_hook = dc_fatal_hook; + + uint64_t tip_before = g_dc.persist.current_lsn; + int tasks_before = dc_task_count(&g_dc.delaymgr); + + /* Act. */ + raft_cb_become_follower_with_cluster_for_test(&g_dc, /*term=*/8); + + /* Assert. */ + TEST_ASSERT_EQUAL_INT_MESSAGE(tasks_before, + dc_task_count(&g_dc.delaymgr), + "demotion from ELECTING must schedule NO convergence task (regression)"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(tip_before, g_dc.persist.current_lsn, + "demotion from ELECTING must NOT truncate the WAL (regression)"); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_restart_calls), + "demotion from ELECTING must NOT self-restart (regression)"); + + dc_teardown(); + printf("test_demotion_from_electing_no_truncation_no_restart: checked\n"); +} + +/* 1.7/2.4 (integration, in-process half): append a committed prefix + an + * uncommitted suffix, run the demotion seam, then replay the SURVIVING WAL into + * a FRESH persist context. The replayed state must contain the committed prefix + * only — provisional entries above committed_lsn must be absent. + * RED: the demotion seam does not truncate, so the suffix survives into the + * fresh replay (current_lsn after recovery exceeds committed_lsn + 1). */ +static void test_demotion_replay_yields_committed_only(void) +{ + dc_setup("replay", /*n_peers=*/1, /*n_standbys=*/0, + /*appended_lsn=*/0, /*n_vcpus=*/8); + + /* Build a real on-disk log: priming + a committed prefix + an uncommitted + * suffix. With n_standbys=0 the sync append returns without majority but + * the LOCAL WAL write lands (production keeps the entry on replication + * timeout); we only need the bytes on disk. */ + for (int i = 0; i < 4; i++) + (void)persist_wal_async(&g_dc.persist, WAL_JOB_UPDATE, "p", 1); + uint64_t committed_boundary = g_dc.persist.current_lsn - 1; /* committed tip */ + atomic_store(&g_dc.persist.committed_lsn, committed_boundary); + + /* Uncommitted suffix (entries above the committed boundary). */ + for (int i = 0; i < 3; i++) + (void)persist_wal_async(&g_dc.persist, WAL_JOB_UPDATE, "u", 1); + uint64_t tip_before = g_dc.persist.current_lsn - 1; + TEST_ASSERT_GREATER_THAN_UINT64_MESSAGE(committed_boundary, tip_before, + "fixture: the uncommitted suffix must sit above the committed boundary"); + + /* Persist the on-disk WAL so a fresh context can replay the same segments. */ + (void)persist_flush(&g_dc.persist); + + /* Capture the workdir BEFORE teardown (the demotion seam restarts/converges + * the live context; we then boot-recover a fresh persist over the surviving + * segments using the SAME workdir + name — the boot-recovery contract). */ + char workdir[256]; + snprintf(workdir, sizeof(workdir), "%s", g_dc_workdir); + + ha_restart_hook = dc_restart_hook; + ha_fatal_hook = dc_fatal_hook; + g_restart_hook_rc = 0; + + /* Act: run the demotion seam (schedules + we drive the convergence task). */ + raft_cb_become_follower_with_cluster_for_test(&g_dc, /*term=*/7); + int rc = dc_run_pending_task(&g_dc.delaymgr); + TEST_ASSERT_NOT_EQUAL_MESSAGE(DC_NO_TASK, rc, + "demotion must schedule a convergence task that truncates (Req 1.2)"); + + /* Boot-recover a FRESH persist context over the surviving WAL (same workdir + + * name → same //data/wal segments the demotion seam left). + * This is the boot-recovery path the real self-restart re-enters (Req 1.7). */ + persist_t *fresh = calloc(1, sizeof(persist_t)); + TEST_ASSERT_NOT_NULL(fresh); + persist_config_t fcfg = persist_config_defaults(); + fcfg.wal_sync_ms = 5000; + persist_set_workdir(fresh, workdir, DC_PERSIST_NAME); + TEST_ASSERT_EQUAL_INT(0, persist_init(fresh, &fcfg, NULL)); + + int recov = persist_recover(fresh); + TEST_ASSERT_TRUE_MESSAGE(recov >= 0, "boot recovery over surviving WAL"); + + /* Assert (Req 1.7/2.4): the replayed tip is the committed boundary only — + * provisional suffix entries are absent. */ + TEST_ASSERT_TRUE_MESSAGE( + fresh->current_lsn <= committed_boundary + 1, + "boot recovery over the surviving WAL must yield the committed prefix " + "ONLY (Req 1.7/2.4): the uncommitted suffix must have been truncated by " + "the demotion convergence seam before restart"); + + persist_stop(fresh); + persist_fini(fresh); + free(fresh); + + dc_teardown(); + printf("test_demotion_replay_yields_committed_only: checked\n"); +} + +#endif /* XCE_TESTING */ + +/* ── Suite entry point ──────────────────────────────────────────────────── */ + +void test_demotion_convergence(void) +{ +#ifdef XCE_TESTING + printf("\n>> demotion convergence + self-restart tests (RED, task 2.1)\n\n"); + + RUN_TEST(test_demotion_from_primary_disables_hooks); + RUN_TEST(test_demotion_schedules_convergence_truncate_and_restart); + RUN_TEST(test_self_restart_failure_invokes_fatal_hook); + RUN_TEST(test_demotion_from_standby_no_truncation_no_restart); + RUN_TEST(test_demotion_from_electing_no_truncation_no_restart); + RUN_TEST(test_demotion_replay_yields_committed_only); +#else + printf("test_demotion_convergence: SKIP (non-test build)\n"); +#endif +}