test(ha-convergence-recovery): RED demotion-convergence + self-restart tests + seams (task 2.1)
This commit is contained in:
parent
b9ddfa60cb
commit
c8e55947c2
7 changed files with 668 additions and 1 deletions
|
|
@ -53,6 +53,7 @@ extern void test_commit_engine(void);
|
|||
extern void test_commit_barrier_rollout(void);
|
||||
extern void test_commit_barrier_reply(void);
|
||||
extern void test_egress_filter(void);
|
||||
extern void test_demotion_convergence(void);
|
||||
|
||||
void setUp(void) { return; }
|
||||
|
||||
|
|
@ -175,6 +176,7 @@ int main(int argc, char* argv[])
|
|||
TEST(test_commit_barrier_rollout);
|
||||
TEST(test_commit_barrier_reply);
|
||||
TEST(test_egress_filter);
|
||||
TEST(test_demotion_convergence);
|
||||
|
||||
return UNITY_END();
|
||||
}
|
||||
|
|
|
|||
499
src/controller/testsuite/test_demotion_convergence.c
Normal file
499
src/controller/testsuite/test_demotion_convergence.c
Normal file
|
|
@ -0,0 +1,499 @@
|
|||
#define _GNU_SOURCE
|
||||
/*
|
||||
* test_demotion_convergence.c — RED-phase tests for the demotion-convergence
|
||||
* sequence and the self-restart primitive (ha-convergence-recovery task 2.1).
|
||||
*
|
||||
* Requirements (this spec's IDs — see requirements.md):
|
||||
* 1.1 — on demotion from PRIMARY the controller STOPS persisting new state
|
||||
* BEFORE continuing demotion (persistence hooks disabled synchronously,
|
||||
* observed via hooks_registered reset + a hook-counter probe).
|
||||
* 1.2 — on demotion the controller removes all log entries ABOVE committed_lsn
|
||||
* (truncate above the committed boundary).
|
||||
* 1.3 — when truncation completes the controller self-restarts so in-memory
|
||||
* state is rebuilt by boot recovery from the committed prefix only.
|
||||
* 1.4 — if the self-restart cannot be performed, terminate FATALLY rather
|
||||
* than continue (FATAL contract; XCE_TESTING routes to ha_fatal_hook).
|
||||
* 1.5 — the demotion is logged with the truncation boundary (committed LSN and
|
||||
* prior log tip) before restarting (observed here via the truncation
|
||||
* cutoff: current_lsn collapses to committed_lsn + 1).
|
||||
* 1.6 — under test mode the controller invokes the ha_restart_hook seam in
|
||||
* place of process re-execution.
|
||||
* 1.7 — in-memory state is reconstructible from the persisted committed state
|
||||
* only (provisional entries above committed_lsn absent after replay).
|
||||
* 2.4 — single-outcome convergence (in-process half): a committed prefix +
|
||||
* uncommitted suffix, run through the demotion seam, replays to a
|
||||
* committed-only state.
|
||||
*
|
||||
* RED status against the current tree: raft_cb_become_follower (ha.c) neither
|
||||
* disables the persistence hooks nor schedules any truncate/restart work on the
|
||||
* PRIMARY→follower edge, and ha_self_restart is a RED scaffold (ha_convergence.c)
|
||||
* that does not yet honor the restart/fatal seams. Every case below therefore
|
||||
* FAILS until task 2.2 lands the demotion branch + the real primitive.
|
||||
*
|
||||
* Harness: a partial-but-real cluster_t (real persist over a private tmpdir,
|
||||
* wired commit engine, stub standby registry, un-started delaymgr) mirroring
|
||||
* test_ha_role_lifecycle.c's lifecycle_cluster_setup, plus the live-cluster
|
||||
* become-follower seam raft_cb_become_follower_with_cluster_for_test.
|
||||
*
|
||||
* AAA structure throughout.
|
||||
*/
|
||||
#include <assert.h>
|
||||
#include <dirent.h>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <inttypes.h>
|
||||
#include <pthread.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <unity/unity.h>
|
||||
|
||||
#include "commit.h" /* commit_engine_t, commit_engine_init */
|
||||
#include "delay.h" /* delaymgr_t, delay_task_t */
|
||||
#include "ha.h" /* raft_cb_become_follower_with_cluster_for_test,
|
||||
* ha_self_restart, controller_saved_argv,
|
||||
* ha_restart_hook / ha_fatal_hook seams */
|
||||
#include "ordq.h" /* ordq_item_t (drive the scheduled task) */
|
||||
#include "persist.h" /* persist_t, persist_truncate_wal_above */
|
||||
#include "repl.h" /* repl_add_standby / repl_set_commit_engine */
|
||||
#include "xce/cluster_def.h" /* full cluster_t layout */
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
|
||||
/* ── Broadcast drop-hook (precedent: test_ha_role_lifecycle.c lc_drop_send_hook):
|
||||
* the stub int endpoints registered via repl_add_standby are not real
|
||||
* endpoint_t, so repl_broadcast must never dereference ep->peer. Installing
|
||||
* this hook drops the wire send harmlessly during priming appends. */
|
||||
extern int (*repl_broadcast_send_hook)(void *ep,
|
||||
wal_catchup_entry_t **entries,
|
||||
int n_entries);
|
||||
static int dc_drop_send_hook(void *ep, wal_catchup_entry_t **entries,
|
||||
int n_entries)
|
||||
{
|
||||
(void)ep; (void)entries; (void)n_entries;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ── Restart / fatal seam counters (Req 1.4, 1.6) ───────────────────────── */
|
||||
static atomic_int g_restart_calls;
|
||||
static atomic_int g_fatal_calls;
|
||||
static int g_restart_hook_rc; /* return code dc_restart_hook reports */
|
||||
|
||||
static int dc_restart_hook(cluster_t *c)
|
||||
{
|
||||
(void)c;
|
||||
atomic_fetch_add(&g_restart_calls, 1);
|
||||
return g_restart_hook_rc;
|
||||
}
|
||||
|
||||
static void dc_fatal_hook(const char *msg)
|
||||
{
|
||||
(void)msg;
|
||||
atomic_fetch_add(&g_fatal_calls, 1);
|
||||
}
|
||||
|
||||
/* ── Cluster fixture (mirror of lifecycle_cluster_setup) ────────────────── */
|
||||
static cluster_t g_dc; /* large — must not live on the test stack */
|
||||
static int g_dc_eps[MAX_STANDBYS];
|
||||
static char g_dc_workdir[256]; /* the fixture tmpdir (workdir for replay) */
|
||||
#define DC_PERSIST_NAME "ctrl-test"
|
||||
|
||||
static ha_config_t dc_ha_cfg(void)
|
||||
{
|
||||
ha_config_t cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 2000,
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
};
|
||||
return cfg;
|
||||
}
|
||||
|
||||
/* Build g_dc: real persist over a fresh tmpdir, n_standbys stub standbys, a
|
||||
* wired commit engine, an initialised-but-UNSTARTED delaymgr (so any task in
|
||||
* tasks.head was armed by the code under test, not by a background runner),
|
||||
* role forced to CTRL_PRIMARY with the persist hooks registered (so a demotion
|
||||
* has hooks to disable). Leaves committed_lsn at 0; callers set it. */
|
||||
static void dc_setup(const char *suffix, int n_peers, int n_standbys,
|
||||
uint64_t appended_lsn, int n_vcpus)
|
||||
{
|
||||
char name[32];
|
||||
char tmpdir[256];
|
||||
|
||||
memset(&g_dc, 0, sizeof(g_dc));
|
||||
atomic_store(&g_restart_calls, 0);
|
||||
atomic_store(&g_fatal_calls, 0);
|
||||
g_restart_hook_rc = 0;
|
||||
|
||||
ha_config_t cfg = dc_ha_cfg();
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&g_dc.ha, &cfg, 0));
|
||||
g_dc.ha.phase2_done = 1; /* no phase2 wiring in this fixture */
|
||||
g_dc.ha.n_peers = n_peers;
|
||||
|
||||
persist_config_t pcfg = persist_config_defaults();
|
||||
pcfg.wal_sync_ms = 5000; /* slow background fsync — no races */
|
||||
pcfg.sync_timeout_ms = 50; /* no real standby ACKs here */
|
||||
snprintf(tmpdir, sizeof(tmpdir), "/tmp/xce_dc_%s_XXXXXX", suffix);
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir));
|
||||
snprintf(g_dc_workdir, sizeof(g_dc_workdir), "%s", tmpdir);
|
||||
persist_set_workdir(&g_dc.persist, tmpdir, DC_PERSIST_NAME);
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_init(&g_dc.persist, &pcfg, &g_dc));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_start(&g_dc.persist));
|
||||
|
||||
/* Prime the appended watermark: current_lsn = appended_lsn + 1. */
|
||||
pthread_mutex_lock(&g_dc.persist.lock);
|
||||
g_dc.persist.current_lsn = appended_lsn + 1;
|
||||
pthread_mutex_unlock(&g_dc.persist.lock);
|
||||
atomic_store(&g_dc.persist.committed_lsn, 0);
|
||||
|
||||
for (int i = 0; i < n_standbys; i++) {
|
||||
snprintf(name, sizeof(name), "dcsb%d", i);
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
repl_add_standby(&g_dc.persist.repl, &g_dc_eps[i], name,
|
||||
7500 + (uint32_t)i) >= 0,
|
||||
"repl_add_standby must return a slot index >= 0");
|
||||
}
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_engine_init(&g_dc.commit, &g_dc.persist, n_peers, n_vcpus));
|
||||
repl_set_commit_engine(&g_dc.persist.repl, &g_dc.commit);
|
||||
|
||||
repl_broadcast_send_hook = dc_drop_send_hook;
|
||||
|
||||
/* delaymgr initialised but NOT started — the demotion-convergence task,
|
||||
* once task 2.2 schedules it, lands in tasks.head and is driven manually
|
||||
* by dc_run_pending_task (D1: truncation + restart run off the callback). */
|
||||
TEST_ASSERT_EQUAL_INT(0, delaymgr_init(&g_dc.delaymgr, 100, NULL));
|
||||
|
||||
/* Make this a PRIMARY with hooks registered, so a demotion has the three
|
||||
* persistence hooks to disable (Req 1.1). ha_promote_to_primary runs the
|
||||
* shared promotion sequence (ha_do_promote → register hooks → heartbeat →
|
||||
* own-term gate); the un-started delaymgr keeps the heartbeat arm skipped. */
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_promote_to_primary(&g_dc));
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
(int)CTRL_PRIMARY,
|
||||
(int)atomic_load_explicit(&g_dc.ha.role, memory_order_acquire),
|
||||
"fixture precondition: node must be CTRL_PRIMARY before demotion");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_dc.ha.hooks_registered,
|
||||
"fixture precondition: persist hooks must be registered before demotion");
|
||||
}
|
||||
|
||||
static void dc_teardown(void)
|
||||
{
|
||||
repl_broadcast_send_hook = NULL;
|
||||
ha_restart_hook = NULL;
|
||||
ha_fatal_hook = NULL;
|
||||
commit_engine_fini(&g_dc.commit);
|
||||
delaymgr_fini(&g_dc.delaymgr);
|
||||
persist_stop(&g_dc.persist);
|
||||
persist_fini(&g_dc.persist);
|
||||
repl_fini(&g_dc.persist.repl);
|
||||
ha_fini(&g_dc.ha);
|
||||
/* tmpdir left on disk — /tmp is ephemeral */
|
||||
}
|
||||
|
||||
/* Count pending delaymgr tasks by walking the ordq head chain (read-only;
|
||||
* single-threaded test, delaymgr never started). Same seam the lifecycle
|
||||
* suite uses to assert promotion arms no watchdog. */
|
||||
static int dc_task_count(const delaymgr_t *mgr)
|
||||
{
|
||||
int count = 0;
|
||||
const ordq_item_t *it = mgr->tasks.head;
|
||||
while (it != NULL) {
|
||||
count++;
|
||||
it = it->next;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
/* Run the single pending delaymgr task synchronously by invoking its stored
|
||||
* entry(args), then return the entry's rc. Returns INT_MIN sentinel if there
|
||||
* is no pending task (RED: nothing was scheduled). The task is NOT removed
|
||||
* from the queue (delaymgr_fini frees it at teardown) — we only need to run
|
||||
* its body once to observe truncation + restart. */
|
||||
#define DC_NO_TASK 0x7fffffff
|
||||
static int dc_run_pending_task(const delaymgr_t *mgr)
|
||||
{
|
||||
const ordq_item_t *it = mgr->tasks.head;
|
||||
if (it == NULL || it->data == NULL)
|
||||
return DC_NO_TASK;
|
||||
const delay_task_t *t = (const delay_task_t *)it->data;
|
||||
if (t->entry == NULL)
|
||||
return DC_NO_TASK;
|
||||
return t->entry(t->args);
|
||||
}
|
||||
|
||||
/* ── Tests ──────────────────────────────────────────────────────────────── */
|
||||
|
||||
/* 1.1: demotion from PRIMARY disables the persistence hooks synchronously in
|
||||
* the callback (before any further event could be persisted), observed via
|
||||
* hooks_registered transitioning 1 → 0.
|
||||
* RED: raft_cb_become_follower never disables hooks on the PRIMARY edge. */
|
||||
static void test_demotion_from_primary_disables_hooks(void)
|
||||
{
|
||||
dc_setup("hooks", /*n_peers=*/3, /*n_standbys=*/2,
|
||||
/*appended_lsn=*/5, /*n_vcpus=*/8);
|
||||
atomic_store(&g_dc.persist.committed_lsn, 3);
|
||||
|
||||
/* Act: drive become-follower from PRIMARY with a LIVE cluster. */
|
||||
raft_cb_become_follower_with_cluster_for_test(&g_dc, /*term=*/7);
|
||||
|
||||
/* Assert: hooks disabled synchronously (Req 1.1). */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_dc.ha.hooks_registered,
|
||||
"demotion from PRIMARY must disable the persistence hooks "
|
||||
"synchronously (hooks_registered 1→0, Req 1.1)");
|
||||
|
||||
dc_teardown();
|
||||
printf("test_demotion_from_primary_disables_hooks: checked\n");
|
||||
}
|
||||
|
||||
/* 1.2/1.3/1.5/1.6: demotion from PRIMARY schedules exactly one convergence task
|
||||
* (off the callback thread, D1); running it truncates above committed_lsn and
|
||||
* invokes the restart seam exactly once.
|
||||
* RED: no task is scheduled, so dc_run_pending_task returns DC_NO_TASK and the
|
||||
* truncation/restart assertions fail. */
|
||||
static void test_demotion_schedules_convergence_truncate_and_restart(void)
|
||||
{
|
||||
dc_setup("conv", /*n_peers=*/3, /*n_standbys=*/2,
|
||||
/*appended_lsn=*/9, /*n_vcpus=*/8);
|
||||
/* committed prefix ends at LSN 4; entries 5..9 are the uncommitted suffix. */
|
||||
atomic_store(&g_dc.persist.committed_lsn, 4);
|
||||
|
||||
/* Install the test restart seam so the eventual self-restart is observable
|
||||
* deterministically rather than exec'ing the test binary (Req 1.6). */
|
||||
ha_restart_hook = dc_restart_hook;
|
||||
ha_fatal_hook = dc_fatal_hook;
|
||||
g_restart_hook_rc = 0; /* model a successful restart */
|
||||
|
||||
int tasks_before = dc_task_count(&g_dc.delaymgr);
|
||||
|
||||
/* Act: demotion edge. */
|
||||
raft_cb_become_follower_with_cluster_for_test(&g_dc, /*term=*/7);
|
||||
|
||||
/* The convergence work runs OFF the callback (D1) — exactly one one-shot
|
||||
* delaymgr task must have been scheduled (Req 1.2/1.3 deferred). */
|
||||
int tasks_after = dc_task_count(&g_dc.delaymgr);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(tasks_before + 1, tasks_after,
|
||||
"demotion from PRIMARY must schedule exactly one convergence task "
|
||||
"(truncate+restart off the callback thread, D1)");
|
||||
|
||||
/* Run the scheduled convergence task synchronously. */
|
||||
int rc = dc_run_pending_task(&g_dc.delaymgr);
|
||||
TEST_ASSERT_NOT_EQUAL_MESSAGE(DC_NO_TASK, rc,
|
||||
"no convergence task was scheduled on the demotion edge (Req 1.2/1.3)");
|
||||
|
||||
/* Assert (Req 1.2/1.5): truncation removed everything above committed_lsn —
|
||||
* current_lsn collapses to committed_lsn + 1 (the truncation boundary). */
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
||||
atomic_load(&g_dc.persist.committed_lsn) + 1, g_dc.persist.current_lsn,
|
||||
"demotion convergence must truncate above committed_lsn (Req 1.2): "
|
||||
"current_lsn must collapse to committed_lsn + 1");
|
||||
|
||||
/* Assert (Req 1.3/1.6): the restart seam was invoked exactly once. */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_restart_calls),
|
||||
"demotion convergence must invoke the restart seam exactly once "
|
||||
"after truncation (Req 1.3, 1.6)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_fatal_calls),
|
||||
"a successful restart must NOT invoke the fatal seam");
|
||||
|
||||
dc_teardown();
|
||||
printf("test_demotion_schedules_convergence_truncate_and_restart: checked\n");
|
||||
}
|
||||
|
||||
/* 1.4: ha_self_restart with a FAILING restart hook must invoke the fatal seam
|
||||
* (the FATAL contract — a divergent ex-primary must never continue serving).
|
||||
* RED: the ha_self_restart scaffold (ha_convergence.c) does not route to the
|
||||
* seams yet, so neither hook fires. */
|
||||
static void test_self_restart_failure_invokes_fatal_hook(void)
|
||||
{
|
||||
dc_setup("fatal", /*n_peers=*/3, /*n_standbys=*/2,
|
||||
/*appended_lsn=*/5, /*n_vcpus=*/8);
|
||||
|
||||
ha_restart_hook = dc_restart_hook;
|
||||
ha_fatal_hook = dc_fatal_hook;
|
||||
g_restart_hook_rc = -EIO; /* model an exec failure */
|
||||
|
||||
/* Act: invoke the primitive directly. */
|
||||
(void)ha_self_restart(&g_dc);
|
||||
|
||||
/* Assert (Req 1.4/1.6): the restart seam was attempted, and because it
|
||||
* failed, the fatal seam fired in place of abort(). */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_restart_calls),
|
||||
"ha_self_restart must invoke the restart seam under XCE_TESTING (Req 1.6)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_fatal_calls),
|
||||
"a failing restart must invoke the fatal seam — the FATAL contract "
|
||||
"(Req 1.4); the ex-primary must never continue with divergent state");
|
||||
|
||||
dc_teardown();
|
||||
printf("test_self_restart_failure_invokes_fatal_hook: checked\n");
|
||||
}
|
||||
|
||||
/* Regression: demotion from STANDBY must NOT truncate, NOT restart, NOT disable
|
||||
* hooks beyond today's behavior — byte-equivalent to the pre-spec path. A
|
||||
* STANDBY has no registered hooks and no convergence work. */
|
||||
static void test_demotion_from_standby_no_truncation_no_restart(void)
|
||||
{
|
||||
dc_setup("standby", /*n_peers=*/3, /*n_standbys=*/2,
|
||||
/*appended_lsn=*/9, /*n_vcpus=*/8);
|
||||
atomic_store(&g_dc.persist.committed_lsn, 4);
|
||||
|
||||
/* Force role to STANDBY and disable hooks (the non-PRIMARY precondition). */
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_demote_to_standby(&g_dc));
|
||||
TEST_ASSERT_EQUAL_INT((int)CTRL_STANDBY,
|
||||
(int)atomic_load_explicit(&g_dc.ha.role, memory_order_acquire));
|
||||
|
||||
ha_restart_hook = dc_restart_hook;
|
||||
ha_fatal_hook = dc_fatal_hook;
|
||||
|
||||
uint64_t tip_before = g_dc.persist.current_lsn;
|
||||
int tasks_before = dc_task_count(&g_dc.delaymgr);
|
||||
|
||||
/* Act: a second become-follower from a non-PRIMARY role. */
|
||||
raft_cb_become_follower_with_cluster_for_test(&g_dc, /*term=*/8);
|
||||
|
||||
/* Assert: no convergence task, no truncation, no restart. */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(tasks_before,
|
||||
dc_task_count(&g_dc.delaymgr),
|
||||
"demotion from STANDBY must schedule NO convergence task (regression)");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(tip_before, g_dc.persist.current_lsn,
|
||||
"demotion from STANDBY must NOT truncate the WAL (regression)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_restart_calls),
|
||||
"demotion from STANDBY must NOT self-restart (regression)");
|
||||
|
||||
dc_teardown();
|
||||
printf("test_demotion_from_standby_no_truncation_no_restart: checked\n");
|
||||
}
|
||||
|
||||
/* Regression: demotion from ELECTING/INIT must NOT truncate or restart. */
|
||||
static void test_demotion_from_electing_no_truncation_no_restart(void)
|
||||
{
|
||||
dc_setup("electing", /*n_peers=*/3, /*n_standbys=*/2,
|
||||
/*appended_lsn=*/9, /*n_vcpus=*/8);
|
||||
atomic_store(&g_dc.persist.committed_lsn, 4);
|
||||
|
||||
/* No registered hooks on a candidate; force role to ELECTING. */
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_demote_to_standby(&g_dc)); /* clears hooks */
|
||||
atomic_store_explicit(&g_dc.ha.role, (int32_t)CTRL_ELECTING,
|
||||
memory_order_release);
|
||||
|
||||
ha_restart_hook = dc_restart_hook;
|
||||
ha_fatal_hook = dc_fatal_hook;
|
||||
|
||||
uint64_t tip_before = g_dc.persist.current_lsn;
|
||||
int tasks_before = dc_task_count(&g_dc.delaymgr);
|
||||
|
||||
/* Act. */
|
||||
raft_cb_become_follower_with_cluster_for_test(&g_dc, /*term=*/8);
|
||||
|
||||
/* Assert. */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(tasks_before,
|
||||
dc_task_count(&g_dc.delaymgr),
|
||||
"demotion from ELECTING must schedule NO convergence task (regression)");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(tip_before, g_dc.persist.current_lsn,
|
||||
"demotion from ELECTING must NOT truncate the WAL (regression)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_restart_calls),
|
||||
"demotion from ELECTING must NOT self-restart (regression)");
|
||||
|
||||
dc_teardown();
|
||||
printf("test_demotion_from_electing_no_truncation_no_restart: checked\n");
|
||||
}
|
||||
|
||||
/* 1.7/2.4 (integration, in-process half): append a committed prefix + an
|
||||
* uncommitted suffix, run the demotion seam, then replay the SURVIVING WAL into
|
||||
* a FRESH persist context. The replayed state must contain the committed prefix
|
||||
* only — provisional entries above committed_lsn must be absent.
|
||||
* RED: the demotion seam does not truncate, so the suffix survives into the
|
||||
* fresh replay (current_lsn after recovery exceeds committed_lsn + 1). */
|
||||
static void test_demotion_replay_yields_committed_only(void)
|
||||
{
|
||||
dc_setup("replay", /*n_peers=*/1, /*n_standbys=*/0,
|
||||
/*appended_lsn=*/0, /*n_vcpus=*/8);
|
||||
|
||||
/* Build a real on-disk log: priming + a committed prefix + an uncommitted
|
||||
* suffix. With n_standbys=0 the sync append returns without majority but
|
||||
* the LOCAL WAL write lands (production keeps the entry on replication
|
||||
* timeout); we only need the bytes on disk. */
|
||||
for (int i = 0; i < 4; i++)
|
||||
(void)persist_wal_async(&g_dc.persist, WAL_JOB_UPDATE, "p", 1);
|
||||
uint64_t committed_boundary = g_dc.persist.current_lsn - 1; /* committed tip */
|
||||
atomic_store(&g_dc.persist.committed_lsn, committed_boundary);
|
||||
|
||||
/* Uncommitted suffix (entries above the committed boundary). */
|
||||
for (int i = 0; i < 3; i++)
|
||||
(void)persist_wal_async(&g_dc.persist, WAL_JOB_UPDATE, "u", 1);
|
||||
uint64_t tip_before = g_dc.persist.current_lsn - 1;
|
||||
TEST_ASSERT_GREATER_THAN_UINT64_MESSAGE(committed_boundary, tip_before,
|
||||
"fixture: the uncommitted suffix must sit above the committed boundary");
|
||||
|
||||
/* Persist the on-disk WAL so a fresh context can replay the same segments. */
|
||||
(void)persist_flush(&g_dc.persist);
|
||||
|
||||
/* Capture the workdir BEFORE teardown (the demotion seam restarts/converges
|
||||
* the live context; we then boot-recover a fresh persist over the surviving
|
||||
* segments using the SAME workdir + name — the boot-recovery contract). */
|
||||
char workdir[256];
|
||||
snprintf(workdir, sizeof(workdir), "%s", g_dc_workdir);
|
||||
|
||||
ha_restart_hook = dc_restart_hook;
|
||||
ha_fatal_hook = dc_fatal_hook;
|
||||
g_restart_hook_rc = 0;
|
||||
|
||||
/* Act: run the demotion seam (schedules + we drive the convergence task). */
|
||||
raft_cb_become_follower_with_cluster_for_test(&g_dc, /*term=*/7);
|
||||
int rc = dc_run_pending_task(&g_dc.delaymgr);
|
||||
TEST_ASSERT_NOT_EQUAL_MESSAGE(DC_NO_TASK, rc,
|
||||
"demotion must schedule a convergence task that truncates (Req 1.2)");
|
||||
|
||||
/* Boot-recover a FRESH persist context over the surviving WAL (same workdir +
|
||||
* name → same <workdir>/<name>/data/wal segments the demotion seam left).
|
||||
* This is the boot-recovery path the real self-restart re-enters (Req 1.7). */
|
||||
persist_t *fresh = calloc(1, sizeof(persist_t));
|
||||
TEST_ASSERT_NOT_NULL(fresh);
|
||||
persist_config_t fcfg = persist_config_defaults();
|
||||
fcfg.wal_sync_ms = 5000;
|
||||
persist_set_workdir(fresh, workdir, DC_PERSIST_NAME);
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_init(fresh, &fcfg, NULL));
|
||||
|
||||
int recov = persist_recover(fresh);
|
||||
TEST_ASSERT_TRUE_MESSAGE(recov >= 0, "boot recovery over surviving WAL");
|
||||
|
||||
/* Assert (Req 1.7/2.4): the replayed tip is the committed boundary only —
|
||||
* provisional suffix entries are absent. */
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
fresh->current_lsn <= committed_boundary + 1,
|
||||
"boot recovery over the surviving WAL must yield the committed prefix "
|
||||
"ONLY (Req 1.7/2.4): the uncommitted suffix must have been truncated by "
|
||||
"the demotion convergence seam before restart");
|
||||
|
||||
persist_stop(fresh);
|
||||
persist_fini(fresh);
|
||||
free(fresh);
|
||||
|
||||
dc_teardown();
|
||||
printf("test_demotion_replay_yields_committed_only: checked\n");
|
||||
}
|
||||
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
/* ── Suite entry point ──────────────────────────────────────────────────── */
|
||||
|
||||
void test_demotion_convergence(void)
|
||||
{
|
||||
#ifdef XCE_TESTING
|
||||
printf("\n>> demotion convergence + self-restart tests (RED, task 2.1)\n\n");
|
||||
|
||||
RUN_TEST(test_demotion_from_primary_disables_hooks);
|
||||
RUN_TEST(test_demotion_schedules_convergence_truncate_and_restart);
|
||||
RUN_TEST(test_self_restart_failure_invokes_fatal_hook);
|
||||
RUN_TEST(test_demotion_from_standby_no_truncation_no_restart);
|
||||
RUN_TEST(test_demotion_from_electing_no_truncation_no_restart);
|
||||
RUN_TEST(test_demotion_replay_yields_committed_only);
|
||||
#else
|
||||
printf("test_demotion_convergence: SKIP (non-test build)\n");
|
||||
#endif
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue