wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity

test(ha-convergence-recovery): RED demotion-convergence + self-restart tests + seams (task 2.1)

Browse source
This commit is contained in:
Rongsong Shen 2026-06-14 03:12:25 +00:00
parent b9ddfa60cb
commit c8e55947c2
7 changed files with 668 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

15
src/controller/ha.c
View file

@ -3063,6 +3063,21 @@ void raft_cb_become_follower_for_test(ha_state_t *ha, uint64_t term)
ha->raft_c = NULL;
raft_cb_become_follower((void *)ha, term, -1);
}
/*
* raft_cb_become_follower_with_cluster_for_test (ha-convergence-recovery
* task 2.1): drive the follower path with a LIVE cluster pointer so the
* cluster-dependent demotion-convergence branch (task 2.2) is exercised
* against a real persist/hooksys/delaymgr fixture. Sets ha->raft_c = c so
* the become-follower callback sees a real cluster (unlike the NULL-cluster
* shim above). raft_cb_become_follower is file-static, so this seam must
* live here in ha.c.
*/
void raft_cb_become_follower_with_cluster_for_test(cluster_t *c, uint64_t term)
{
c->ha.raft_c = c;
raft_cb_become_follower((void *)&c->ha, term, -1);
}
#endif /* XCE_TESTING */
/* ── Task 2.4: vote_ep_err_handler ─────────────────────────────────────────

76
src/controller/ha.h
View file

@ -239,6 +239,41 @@ void ha_do_promote(ha_state_t *ha);
void ha_do_demote(ha_state_t *ha);
/* ── Demotion convergence + self-restart primitive ─────────────
* (ha-convergence-recovery Requirement 1: a demoted ex-primary discards its
* uncommitted, potentially divergent state and rejoins from committed state
* only via truncation + process self-restart through boot recovery.)
*/
/*
* controller_saved_argv: accessor for the NULL-terminated argv captured by
* main() before argp_parse, used by ha_self_restart to re-exec the controller
* (ha-convergence-recovery Req 1.3, design "Self-Restart Primitive").
* Returns NULL if argv was never saved (e.g. unit-test binaries).
*/
char **controller_saved_argv(void);
/*
* ha_self_restart: the truncate-and-restart primitive's restart half.
*
* Production contract (Req 1.3, 1.4): flush logs, execv(controller_saved_argv())
* to rebuild in-memory state through boot recovery from the persisted committed
* state only; if exec cannot be performed, terminate FATALLY (abort) rather than
* continue running with potentially divergent state — NEVER returns in
* production.
*
* Test contract (Req 1.6): under XCE_TESTING, invoke ha_restart_hook instead of
* execv and return its result (0 on the success path); a failing ha_restart_hook
* routes to ha_fatal_hook in place of abort so tests can assert the FATAL path
* without dying.
*
* Preconditions: persistence hooks already disabled; truncation completed
* before this call. Invariant: no new WAL entry is appended by this node
* between hook-disable and exec.
*/
int ha_self_restart(cluster_t *c);
/* ── Redirect helpers used by join.c ──────────────────────── */
/*
@ -768,6 +803,47 @@ extern void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c,
* Requirements: task 7.3 (ha-peer-connection-reuse).
*/
void vote_ep_ctx_set_cluster_for_test(struct endpoint_t *ep, cluster_t *c);
/*
* ha_restart_hook: test-only seam for ha_self_restart's process re-execution
* (ha-convergence-recovery Req 1.6).
*
* When non-NULL, ha_self_restart() invokes this INSTEAD OF execv() so unit
* tests can observe the eventual restart deterministically (the demotion
* convergence task and rejoin reconciliation both end in ha_self_restart).
* The hook's return value is ha_self_restart's return value on the
* test-success path; a NON-ZERO return models an exec failure and routes
* ha_self_restart to the FATAL contract via ha_fatal_hook (Req 1.4).
*
* Signature: int hook(cluster_t *c)
* Returns 0 to model a successful restart; non-zero to model exec failure.
*/
extern int (*ha_restart_hook)(cluster_t *c);
/*
* ha_fatal_hook: test-only seam for the FATAL (abort) contract of
* ha_self_restart (ha-convergence-recovery Req 1.4).
*
* When non-NULL, ha_self_restart() invokes this INSTEAD OF abort() when the
* restart cannot be performed, so a test binary can assert the FATAL path was
* taken without the process dying. ha_self_restart returns after invoking it
* (test builds only — production aborts).
*
* Signature: void hook(const char *msg)
*/
extern void (*ha_fatal_hook)(const char *msg);
/*
* raft_cb_become_follower_with_cluster_for_test: drive the follower path with
* a LIVE cluster pointer (ha-convergence-recovery task 2.1 demotion-sequence
* tests). Unlike raft_cb_become_follower_for_test (which forces raft_c = NULL
* to suppress production calls), this shim sets ha->raft_c = c first so the
* cluster-dependent demotion-convergence branch (task 2.2: hook disable +
* scheduled truncate/restart) is exercised against a real persist/hooksys/
* delaymgr fixture. Mirrors the real-cluster seam ha_promote_to_primary(c)
* used by the promotion-side lifecycle tests.
*/
void raft_cb_become_follower_with_cluster_for_test(cluster_t *c, uint64_t term);
#endif
#endif /* HA_H */

70
src/controller/ha_convergence.c Normal file
View file

@ -0,0 +1,70 @@
/*
* ha_convergence.c — demotion convergence + self-restart primitive
* (ha-convergence-recovery Requirement 1).
*
* ───────────────────────────────────────────────────────────────────────────
* RED SCAFFOLD (task 2.1) — task 2.2 REPLACES the bodies below with the real
* implementation (and moves controller_saved_argv's backing store into main.c).
*
* This file currently provides ONLY the link-time definitions the task-2.1 RED
* tests need to compile and link:
* - controller_saved_argv() — stub returning NULL (real argv capture is in
* main.c, task 2.2)
* - ha_self_restart() — RED stub: does NOT yet honor the ha_restart_hook
* / ha_fatal_hook contract, so the fatal-hook test
* FAILS until task 2.2 implements the real
* flush → execv → abort sequence (and the
* XCE_TESTING seam routing).
* - ha_restart_hook / ha_fatal_hook — the XCE_TESTING function-pointer seams,
* defined =NULL in exactly one TU (this one),
* mirroring the ha_*_hook seam pattern in ha.c.
*
* The DEMOTION-SEQUENCE behavior (hook disable on become-follower-from-PRIMARY,
* scheduled truncate-above-committed_lsn, restart) lives in raft_cb_become_
* follower (ha.c) and is NOT implemented here or there yet — that is task 2.2.
* Consequently every demotion-convergence test in test_demotion_convergence.c
* is RED against the current tree.
* ───────────────────────────────────────────────────────────────────────────
*/
#include <stddef.h>
#include "cluster.h"
#include "ha.h"
#include "xce/cluster_def.h"
#ifdef XCE_TESTING
/* XCE_TESTING seams (Req 1.6, 1.4) — defined here, declared in ha.h.
* Tests set these before exercising the restart/fatal contract. */
int (*ha_restart_hook)(cluster_t *c) = NULL;
void (*ha_fatal_hook)(const char *msg) = NULL;
#endif
/*
* controller_saved_argv — RED stub (task 2.2 moves the real, main()-populated
* accessor into main.c). Returns NULL: no argv has been captured in the
* scaffold, so a production execv would have nothing to exec.
*/
char **controller_saved_argv(void)
{
return NULL;
}
/*
* ha_self_restart — RED stub.
*
* Task 2.2 replaces this with the real primitive:
* - production: flush logs, execv(controller_saved_argv()); on exec return,
* ERROR + abort() (FATAL — Req 1.4); never returns.
* - XCE_TESTING: invoke ha_restart_hook(c) instead of execv (Req 1.6); if it
* reports failure (or is unset), invoke ha_fatal_hook in place of abort.
*
* Until then this stub deliberately does NOTHING (no hook routing), so the
* task-2.1 fatal-hook test — which asserts a failing restart hook drives
* ha_fatal_hook — FAILS (RED).
*/
int ha_self_restart(cluster_t *c)
{
(void)c;
return 0; /* RED scaffold — task 2.2 implements exec/abort + seam routing */
}

3
src/controller/meson.build
View file

@ -5,6 +5,7 @@ controller_src = [
'dataex.c',
'egress.c',
'ha.c',
'ha_convergence.c',
'job.c',
'join.c',
'main.c',
@ -124,9 +125,11 @@ controller_unit_test_src = [
'testsuite/test_commit_barrier_rollout.c',
'testsuite/test_commit_barrier_reply.c',
'testsuite/test_egress_filter.c',
'testsuite/test_demotion_convergence.c',
'commit.c',
'job.c',
'egress.c',
'ha_convergence.c',
'queue.c',
'join.c',
'opupd.c',

2
src/controller/testsuite/controller_test.c
View file

@ -53,6 +53,7 @@ extern void test_commit_engine(void);
extern void test_commit_barrier_rollout(void);
extern void test_commit_barrier_reply(void);
extern void test_egress_filter(void);
extern void test_demotion_convergence(void);
void setUp(void) { return; }
@ -175,6 +176,7 @@ int main(int argc, char* argv[])
TEST(test_commit_barrier_rollout);
TEST(test_commit_barrier_reply);
TEST(test_egress_filter);
TEST(test_demotion_convergence);
return UNITY_END();
}

499
src/controller/testsuite/test_demotion_convergence.c Normal file
View file

@ -0,0 +1,499 @@
#define _GNU_SOURCE
/*
* test_demotion_convergence.c — RED-phase tests for the demotion-convergence
* sequence and the self-restart primitive (ha-convergence-recovery task 2.1).
*
* Requirements (this spec's IDs — see requirements.md):
* 1.1 — on demotion from PRIMARY the controller STOPS persisting new state
* BEFORE continuing demotion (persistence hooks disabled synchronously,
* observed via hooks_registered reset + a hook-counter probe).
* 1.2 — on demotion the controller removes all log entries ABOVE committed_lsn
* (truncate above the committed boundary).
* 1.3 — when truncation completes the controller self-restarts so in-memory
* state is rebuilt by boot recovery from the committed prefix only.
* 1.4 — if the self-restart cannot be performed, terminate FATALLY rather
* than continue (FATAL contract; XCE_TESTING routes to ha_fatal_hook).
* 1.5 — the demotion is logged with the truncation boundary (committed LSN and
* prior log tip) before restarting (observed here via the truncation
* cutoff: current_lsn collapses to committed_lsn + 1).
* 1.6 — under test mode the controller invokes the ha_restart_hook seam in
* place of process re-execution.
* 1.7 — in-memory state is reconstructible from the persisted committed state
* only (provisional entries above committed_lsn absent after replay).
* 2.4 — single-outcome convergence (in-process half): a committed prefix +
* uncommitted suffix, run through the demotion seam, replays to a
* committed-only state.
*
* RED status against the current tree: raft_cb_become_follower (ha.c) neither
* disables the persistence hooks nor schedules any truncate/restart work on the
* PRIMARY→follower edge, and ha_self_restart is a RED scaffold (ha_convergence.c)
* that does not yet honor the restart/fatal seams. Every case below therefore
* FAILS until task 2.2 lands the demotion branch + the real primitive.
*
* Harness: a partial-but-real cluster_t (real persist over a private tmpdir,
* wired commit engine, stub standby registry, un-started delaymgr) mirroring
* test_ha_role_lifecycle.c's lifecycle_cluster_setup, plus the live-cluster
* become-follower seam raft_cb_become_follower_with_cluster_for_test.
*
* AAA structure throughout.
*/
#include <assert.h>
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <pthread.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#include <unity/unity.h>
#include "commit.h" /* commit_engine_t, commit_engine_init */
#include "delay.h" /* delaymgr_t, delay_task_t */
#include "ha.h" /* raft_cb_become_follower_with_cluster_for_test,
* ha_self_restart, controller_saved_argv,
* ha_restart_hook / ha_fatal_hook seams */
#include "ordq.h" /* ordq_item_t (drive the scheduled task) */
#include "persist.h" /* persist_t, persist_truncate_wal_above */
#include "repl.h" /* repl_add_standby / repl_set_commit_engine */
#include "xce/cluster_def.h" /* full cluster_t layout */
#ifdef XCE_TESTING
/* ── Broadcast drop-hook (precedent: test_ha_role_lifecycle.c lc_drop_send_hook):
* the stub int endpoints registered via repl_add_standby are not real
* endpoint_t, so repl_broadcast must never dereference ep->peer. Installing
* this hook drops the wire send harmlessly during priming appends. */
extern int (*repl_broadcast_send_hook)(void *ep,
wal_catchup_entry_t **entries,
int n_entries);
static int dc_drop_send_hook(void *ep, wal_catchup_entry_t **entries,
int n_entries)
{
(void)ep; (void)entries; (void)n_entries;
return 0;
}
/* ── Restart / fatal seam counters (Req 1.4, 1.6) ───────────────────────── */
static atomic_int g_restart_calls;
static atomic_int g_fatal_calls;
static int g_restart_hook_rc; /* return code dc_restart_hook reports */
static int dc_restart_hook(cluster_t *c)
{
(void)c;
atomic_fetch_add(&g_restart_calls, 1);
return g_restart_hook_rc;
}
static void dc_fatal_hook(const char *msg)
{
(void)msg;
atomic_fetch_add(&g_fatal_calls, 1);
}
/* ── Cluster fixture (mirror of lifecycle_cluster_setup) ────────────────── */
static cluster_t g_dc; /* large — must not live on the test stack */
static int g_dc_eps[MAX_STANDBYS];
static char g_dc_workdir[256]; /* the fixture tmpdir (workdir for replay) */
#define DC_PERSIST_NAME "ctrl-test"
static ha_config_t dc_ha_cfg(void)
{
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
return cfg;
}
/* Build g_dc: real persist over a fresh tmpdir, n_standbys stub standbys, a
* wired commit engine, an initialised-but-UNSTARTED delaymgr (so any task in
* tasks.head was armed by the code under test, not by a background runner),
* role forced to CTRL_PRIMARY with the persist hooks registered (so a demotion
* has hooks to disable). Leaves committed_lsn at 0; callers set it. */
static void dc_setup(const char *suffix, int n_peers, int n_standbys,
uint64_t appended_lsn, int n_vcpus)
{
char name[32];
char tmpdir[256];
memset(&g_dc, 0, sizeof(g_dc));
atomic_store(&g_restart_calls, 0);
atomic_store(&g_fatal_calls, 0);
g_restart_hook_rc = 0;
ha_config_t cfg = dc_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&g_dc.ha, &cfg, 0));
g_dc.ha.phase2_done = 1; /* no phase2 wiring in this fixture */
g_dc.ha.n_peers = n_peers;
persist_config_t pcfg = persist_config_defaults();
pcfg.wal_sync_ms = 5000; /* slow background fsync — no races */
pcfg.sync_timeout_ms = 50; /* no real standby ACKs here */
snprintf(tmpdir, sizeof(tmpdir), "/tmp/xce_dc_%s_XXXXXX", suffix);
TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir));
snprintf(g_dc_workdir, sizeof(g_dc_workdir), "%s", tmpdir);
persist_set_workdir(&g_dc.persist, tmpdir, DC_PERSIST_NAME);
TEST_ASSERT_EQUAL_INT(0, persist_init(&g_dc.persist, &pcfg, &g_dc));
TEST_ASSERT_EQUAL_INT(0, persist_start(&g_dc.persist));
/* Prime the appended watermark: current_lsn = appended_lsn + 1. */
pthread_mutex_lock(&g_dc.persist.lock);
g_dc.persist.current_lsn = appended_lsn + 1;
pthread_mutex_unlock(&g_dc.persist.lock);
atomic_store(&g_dc.persist.committed_lsn, 0);
for (int i = 0; i < n_standbys; i++) {
snprintf(name, sizeof(name), "dcsb%d", i);
TEST_ASSERT_TRUE_MESSAGE(
repl_add_standby(&g_dc.persist.repl, &g_dc_eps[i], name,
7500 + (uint32_t)i) >= 0,
"repl_add_standby must return a slot index >= 0");
}
TEST_ASSERT_EQUAL_INT(0,
commit_engine_init(&g_dc.commit, &g_dc.persist, n_peers, n_vcpus));
repl_set_commit_engine(&g_dc.persist.repl, &g_dc.commit);
repl_broadcast_send_hook = dc_drop_send_hook;
/* delaymgr initialised but NOT started — the demotion-convergence task,
* once task 2.2 schedules it, lands in tasks.head and is driven manually
* by dc_run_pending_task (D1: truncation + restart run off the callback). */
TEST_ASSERT_EQUAL_INT(0, delaymgr_init(&g_dc.delaymgr, 100, NULL));
/* Make this a PRIMARY with hooks registered, so a demotion has the three
* persistence hooks to disable (Req 1.1). ha_promote_to_primary runs the
* shared promotion sequence (ha_do_promote → register hooks → heartbeat →
* own-term gate); the un-started delaymgr keeps the heartbeat arm skipped. */
TEST_ASSERT_EQUAL_INT(0, ha_promote_to_primary(&g_dc));
TEST_ASSERT_EQUAL_INT_MESSAGE(
(int)CTRL_PRIMARY,
(int)atomic_load_explicit(&g_dc.ha.role, memory_order_acquire),
"fixture precondition: node must be CTRL_PRIMARY before demotion");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_dc.ha.hooks_registered,
"fixture precondition: persist hooks must be registered before demotion");
}
static void dc_teardown(void)
{
repl_broadcast_send_hook = NULL;
ha_restart_hook = NULL;
ha_fatal_hook = NULL;
commit_engine_fini(&g_dc.commit);
delaymgr_fini(&g_dc.delaymgr);
persist_stop(&g_dc.persist);
persist_fini(&g_dc.persist);
repl_fini(&g_dc.persist.repl);
ha_fini(&g_dc.ha);
/* tmpdir left on disk — /tmp is ephemeral */
}
/* Count pending delaymgr tasks by walking the ordq head chain (read-only;
* single-threaded test, delaymgr never started). Same seam the lifecycle
* suite uses to assert promotion arms no watchdog. */
static int dc_task_count(const delaymgr_t *mgr)
{
int count = 0;
const ordq_item_t *it = mgr->tasks.head;
while (it != NULL) {
count++;
it = it->next;
}
return count;
}
/* Run the single pending delaymgr task synchronously by invoking its stored
* entry(args), then return the entry's rc. Returns INT_MIN sentinel if there
* is no pending task (RED: nothing was scheduled). The task is NOT removed
* from the queue (delaymgr_fini frees it at teardown) — we only need to run
* its body once to observe truncation + restart. */
#define DC_NO_TASK 0x7fffffff
static int dc_run_pending_task(const delaymgr_t *mgr)
{
const ordq_item_t *it = mgr->tasks.head;
if (it == NULL || it->data == NULL)
return DC_NO_TASK;
const delay_task_t *t = (const delay_task_t *)it->data;
if (t->entry == NULL)
return DC_NO_TASK;
return t->entry(t->args);
}
/* ── Tests ──────────────────────────────────────────────────────────────── */
/* 1.1: demotion from PRIMARY disables the persistence hooks synchronously in
* the callback (before any further event could be persisted), observed via
* hooks_registered transitioning 1 → 0.
* RED: raft_cb_become_follower never disables hooks on the PRIMARY edge. */
static void test_demotion_from_primary_disables_hooks(void)
{
dc_setup("hooks", /*n_peers=*/3, /*n_standbys=*/2,
/*appended_lsn=*/5, /*n_vcpus=*/8);
atomic_store(&g_dc.persist.committed_lsn, 3);
/* Act: drive become-follower from PRIMARY with a LIVE cluster. */
raft_cb_become_follower_with_cluster_for_test(&g_dc, /*term=*/7);
/* Assert: hooks disabled synchronously (Req 1.1). */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_dc.ha.hooks_registered,
"demotion from PRIMARY must disable the persistence hooks "
"synchronously (hooks_registered 1→0, Req 1.1)");
dc_teardown();
printf("test_demotion_from_primary_disables_hooks: checked\n");
}
/* 1.2/1.3/1.5/1.6: demotion from PRIMARY schedules exactly one convergence task
* (off the callback thread, D1); running it truncates above committed_lsn and
* invokes the restart seam exactly once.
* RED: no task is scheduled, so dc_run_pending_task returns DC_NO_TASK and the
* truncation/restart assertions fail. */
static void test_demotion_schedules_convergence_truncate_and_restart(void)
{
dc_setup("conv", /*n_peers=*/3, /*n_standbys=*/2,
/*appended_lsn=*/9, /*n_vcpus=*/8);
/* committed prefix ends at LSN 4; entries 5..9 are the uncommitted suffix. */
atomic_store(&g_dc.persist.committed_lsn, 4);
/* Install the test restart seam so the eventual self-restart is observable
* deterministically rather than exec'ing the test binary (Req 1.6). */
ha_restart_hook = dc_restart_hook;
ha_fatal_hook = dc_fatal_hook;
g_restart_hook_rc = 0; /* model a successful restart */
int tasks_before = dc_task_count(&g_dc.delaymgr);
/* Act: demotion edge. */
raft_cb_become_follower_with_cluster_for_test(&g_dc, /*term=*/7);
/* The convergence work runs OFF the callback (D1) — exactly one one-shot
* delaymgr task must have been scheduled (Req 1.2/1.3 deferred). */
int tasks_after = dc_task_count(&g_dc.delaymgr);
TEST_ASSERT_EQUAL_INT_MESSAGE(tasks_before + 1, tasks_after,
"demotion from PRIMARY must schedule exactly one convergence task "
"(truncate+restart off the callback thread, D1)");
/* Run the scheduled convergence task synchronously. */
int rc = dc_run_pending_task(&g_dc.delaymgr);
TEST_ASSERT_NOT_EQUAL_MESSAGE(DC_NO_TASK, rc,
"no convergence task was scheduled on the demotion edge (Req 1.2/1.3)");
/* Assert (Req 1.2/1.5): truncation removed everything above committed_lsn —
* current_lsn collapses to committed_lsn + 1 (the truncation boundary). */
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
atomic_load(&g_dc.persist.committed_lsn) + 1, g_dc.persist.current_lsn,
"demotion convergence must truncate above committed_lsn (Req 1.2): "
"current_lsn must collapse to committed_lsn + 1");
/* Assert (Req 1.3/1.6): the restart seam was invoked exactly once. */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_restart_calls),
"demotion convergence must invoke the restart seam exactly once "
"after truncation (Req 1.3, 1.6)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_fatal_calls),
"a successful restart must NOT invoke the fatal seam");
dc_teardown();
printf("test_demotion_schedules_convergence_truncate_and_restart: checked\n");
}
/* 1.4: ha_self_restart with a FAILING restart hook must invoke the fatal seam
* (the FATAL contract — a divergent ex-primary must never continue serving).
* RED: the ha_self_restart scaffold (ha_convergence.c) does not route to the
* seams yet, so neither hook fires. */
static void test_self_restart_failure_invokes_fatal_hook(void)
{
dc_setup("fatal", /*n_peers=*/3, /*n_standbys=*/2,
/*appended_lsn=*/5, /*n_vcpus=*/8);
ha_restart_hook = dc_restart_hook;
ha_fatal_hook = dc_fatal_hook;
g_restart_hook_rc = -EIO; /* model an exec failure */
/* Act: invoke the primitive directly. */
(void)ha_self_restart(&g_dc);
/* Assert (Req 1.4/1.6): the restart seam was attempted, and because it
* failed, the fatal seam fired in place of abort(). */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_restart_calls),
"ha_self_restart must invoke the restart seam under XCE_TESTING (Req 1.6)");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_fatal_calls),
"a failing restart must invoke the fatal seam — the FATAL contract "
"(Req 1.4); the ex-primary must never continue with divergent state");
dc_teardown();
printf("test_self_restart_failure_invokes_fatal_hook: checked\n");
}
/* Regression: demotion from STANDBY must NOT truncate, NOT restart, NOT disable
* hooks beyond today's behavior — byte-equivalent to the pre-spec path. A
* STANDBY has no registered hooks and no convergence work. */
static void test_demotion_from_standby_no_truncation_no_restart(void)
{
dc_setup("standby", /*n_peers=*/3, /*n_standbys=*/2,
/*appended_lsn=*/9, /*n_vcpus=*/8);
atomic_store(&g_dc.persist.committed_lsn, 4);
/* Force role to STANDBY and disable hooks (the non-PRIMARY precondition). */
TEST_ASSERT_EQUAL_INT(0, ha_demote_to_standby(&g_dc));
TEST_ASSERT_EQUAL_INT((int)CTRL_STANDBY,
(int)atomic_load_explicit(&g_dc.ha.role, memory_order_acquire));
ha_restart_hook = dc_restart_hook;
ha_fatal_hook = dc_fatal_hook;
uint64_t tip_before = g_dc.persist.current_lsn;
int tasks_before = dc_task_count(&g_dc.delaymgr);
/* Act: a second become-follower from a non-PRIMARY role. */
raft_cb_become_follower_with_cluster_for_test(&g_dc, /*term=*/8);
/* Assert: no convergence task, no truncation, no restart. */
TEST_ASSERT_EQUAL_INT_MESSAGE(tasks_before,
dc_task_count(&g_dc.delaymgr),
"demotion from STANDBY must schedule NO convergence task (regression)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(tip_before, g_dc.persist.current_lsn,
"demotion from STANDBY must NOT truncate the WAL (regression)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_restart_calls),
"demotion from STANDBY must NOT self-restart (regression)");
dc_teardown();
printf("test_demotion_from_standby_no_truncation_no_restart: checked\n");
}
/* Regression: demotion from ELECTING/INIT must NOT truncate or restart. */
static void test_demotion_from_electing_no_truncation_no_restart(void)
{
dc_setup("electing", /*n_peers=*/3, /*n_standbys=*/2,
/*appended_lsn=*/9, /*n_vcpus=*/8);
atomic_store(&g_dc.persist.committed_lsn, 4);
/* No registered hooks on a candidate; force role to ELECTING. */
TEST_ASSERT_EQUAL_INT(0, ha_demote_to_standby(&g_dc)); /* clears hooks */
atomic_store_explicit(&g_dc.ha.role, (int32_t)CTRL_ELECTING,
memory_order_release);
ha_restart_hook = dc_restart_hook;
ha_fatal_hook = dc_fatal_hook;
uint64_t tip_before = g_dc.persist.current_lsn;
int tasks_before = dc_task_count(&g_dc.delaymgr);
/* Act. */
raft_cb_become_follower_with_cluster_for_test(&g_dc, /*term=*/8);
/* Assert. */
TEST_ASSERT_EQUAL_INT_MESSAGE(tasks_before,
dc_task_count(&g_dc.delaymgr),
"demotion from ELECTING must schedule NO convergence task (regression)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(tip_before, g_dc.persist.current_lsn,
"demotion from ELECTING must NOT truncate the WAL (regression)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_restart_calls),
"demotion from ELECTING must NOT self-restart (regression)");
dc_teardown();
printf("test_demotion_from_electing_no_truncation_no_restart: checked\n");
}
/* 1.7/2.4 (integration, in-process half): append a committed prefix + an
* uncommitted suffix, run the demotion seam, then replay the SURVIVING WAL into
* a FRESH persist context. The replayed state must contain the committed prefix
* only — provisional entries above committed_lsn must be absent.
* RED: the demotion seam does not truncate, so the suffix survives into the
* fresh replay (current_lsn after recovery exceeds committed_lsn + 1). */
static void test_demotion_replay_yields_committed_only(void)
{
dc_setup("replay", /*n_peers=*/1, /*n_standbys=*/0,
/*appended_lsn=*/0, /*n_vcpus=*/8);
/* Build a real on-disk log: priming + a committed prefix + an uncommitted
* suffix. With n_standbys=0 the sync append returns without majority but
* the LOCAL WAL write lands (production keeps the entry on replication
* timeout); we only need the bytes on disk. */
for (int i = 0; i < 4; i++)
(void)persist_wal_async(&g_dc.persist, WAL_JOB_UPDATE, "p", 1);
uint64_t committed_boundary = g_dc.persist.current_lsn - 1; /* committed tip */
atomic_store(&g_dc.persist.committed_lsn, committed_boundary);
/* Uncommitted suffix (entries above the committed boundary). */
for (int i = 0; i < 3; i++)
(void)persist_wal_async(&g_dc.persist, WAL_JOB_UPDATE, "u", 1);
uint64_t tip_before = g_dc.persist.current_lsn - 1;
TEST_ASSERT_GREATER_THAN_UINT64_MESSAGE(committed_boundary, tip_before,
"fixture: the uncommitted suffix must sit above the committed boundary");
/* Persist the on-disk WAL so a fresh context can replay the same segments. */
(void)persist_flush(&g_dc.persist);
/* Capture the workdir BEFORE teardown (the demotion seam restarts/converges
* the live context; we then boot-recover a fresh persist over the surviving
* segments using the SAME workdir + name — the boot-recovery contract). */
char workdir[256];
snprintf(workdir, sizeof(workdir), "%s", g_dc_workdir);
ha_restart_hook = dc_restart_hook;
ha_fatal_hook = dc_fatal_hook;
g_restart_hook_rc = 0;
/* Act: run the demotion seam (schedules + we drive the convergence task). */
raft_cb_become_follower_with_cluster_for_test(&g_dc, /*term=*/7);
int rc = dc_run_pending_task(&g_dc.delaymgr);
TEST_ASSERT_NOT_EQUAL_MESSAGE(DC_NO_TASK, rc,
"demotion must schedule a convergence task that truncates (Req 1.2)");
/* Boot-recover a FRESH persist context over the surviving WAL (same workdir +
* name → same <workdir>/<name>/data/wal segments the demotion seam left).
* This is the boot-recovery path the real self-restart re-enters (Req 1.7). */
persist_t *fresh = calloc(1, sizeof(persist_t));
TEST_ASSERT_NOT_NULL(fresh);
persist_config_t fcfg = persist_config_defaults();
fcfg.wal_sync_ms = 5000;
persist_set_workdir(fresh, workdir, DC_PERSIST_NAME);
TEST_ASSERT_EQUAL_INT(0, persist_init(fresh, &fcfg, NULL));
int recov = persist_recover(fresh);
TEST_ASSERT_TRUE_MESSAGE(recov >= 0, "boot recovery over surviving WAL");
/* Assert (Req 1.7/2.4): the replayed tip is the committed boundary only —
* provisional suffix entries are absent. */
TEST_ASSERT_TRUE_MESSAGE(
fresh->current_lsn <= committed_boundary + 1,
"boot recovery over the surviving WAL must yield the committed prefix "
"ONLY (Req 1.7/2.4): the uncommitted suffix must have been truncated by "
"the demotion convergence seam before restart");
persist_stop(fresh);
persist_fini(fresh);
free(fresh);
dc_teardown();
printf("test_demotion_replay_yields_committed_only: checked\n");
}
#endif /* XCE_TESTING */
/* ── Suite entry point ──────────────────────────────────────────────────── */
void test_demotion_convergence(void)
{
#ifdef XCE_TESTING
printf("\n>> demotion convergence + self-restart tests (RED, task 2.1)\n\n");
RUN_TEST(test_demotion_from_primary_disables_hooks);
RUN_TEST(test_demotion_schedules_convergence_truncate_and_restart);
RUN_TEST(test_self_restart_failure_invokes_fatal_hook);
RUN_TEST(test_demotion_from_standby_no_truncation_no_restart);
RUN_TEST(test_demotion_from_electing_no_truncation_no_restart);
RUN_TEST(test_demotion_replay_yields_committed_only);
#else
printf("test_demotion_convergence: SKIP (non-test build)\n");
#endif
}