lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
myxce/src/controller/testsuite/test_ha_role_lifecycle.c

1187 lines
56 KiB
C
Raw Blame History

#define _GNU_SOURCE
/*
* test_ha_role_lifecycle.c — ha-single-leader-role lifecycle contract.
*
* Task 1.1 (TDD RED): the WAL_NOOP no-op entry type. Later tasks (2.1, 3.1)
* add direct-promotion and own-term-gate lifecycle cases to this same file.
*
* RED-PHASE APPROACH (b): WAL_NOOP does not exist yet (lands in task 1.2),
* so referencing the symbol would fail to COMPILE. Instead these cases use
* the literal value 40 (the value WAL_NOOP will take in wal_entry_type, per
* design.md §"WAL_NOOP Entry") so the file COMPILES and LINKS into the
* controller test binary; the assertions then fail at RUNTIME because the
* type-40 behavior (explicit replay no-op, RETAIN classification) is not
* implemented. This satisfies task 1.1's "build succeeds + new cases FAIL".
*
* The classification case (wal_classify(40,...)) cannot run inline in the
* current tree: the totality guard does `assert(0)` for any unclassified
* type, which would ABORT the whole test binary. It is isolated via the
* fork() abort-test pattern (cerebrum: child silences stderr + _exit, parent
* inspects exit status) so the RED state is a deterministic, captured
* assertion in the parent rather than a binary crash.
*
* AAA structure throughout.
*/
#include <assert.h>
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <pthread.h>
#include <semaphore.h>
#include <signal.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <time.h>
#include <unistd.h>
#include <unity/unity.h>
#include "config.h"
#include "persist.h"
/* Task 2.1 (RED): direct-promotion + own-term-gate lifecycle contract.
* These headers pull in the promotion seam (ha_*), the commit engine + its
* own-term gate (commit_*), the stub standby registry (repl_*), and the
* full cluster_t layout so a partial-but-real cluster can be promoted. */
#include "commit.h" /* commit_engine_t, commit_submit, commit_set_writable_lsn */
#include "ha.h" /* ha_promote_to_primary, ha_is_primary, ha_should_redirect */
#include "repl.h" /* repl_init / repl_add_standby / repl_set_commit_engine */
#include "xce/cluster_def.h" /* full cluster_t layout (c.commit / c.persist / c.ha / ...) */
#include "xce/errnox.h" /* XCE_ERR_NOT_READY */
/* The value WAL_NOOP will take in wal_entry_type (task 1.2). Using the
* literal keeps THIS file compiling before the enum value exists; task 1.2
* replaces these with the WAL_NOOP symbol once it lands. */
#define WAL_NOOP_VALUE 40u
/* ------------------------------------------------------------------ */
/* Broadcast drop-hook (precedent: test_p2_integration.c:32/313, */
/* test_commit_engine.c:1133/1308; hook defined in repl.c:21). */
/* The lifecycle fixture registers fake int g_lc_eps[] endpoints via */
/* repl_add_standby; the own-term WAL_NOOP append now broadcasts via */
/* repl_broadcast → xce_wal_batch_req_raw(ep), which would dereference */
/* ep->peer past the 4-byte int (ASAN global-buffer-overflow at */
/* request.c). Installing this hook drops the wire send harmlessly. */
/* ------------------------------------------------------------------ */
#ifdef XCE_TESTING
extern int (*repl_broadcast_send_hook)(void *ep,
wal_catchup_entry_t **entries,
int n_entries);
static int lc_drop_send_hook(void *ep, wal_catchup_entry_t **entries,
int n_entries)
{
(void)ep; (void)entries; (void)n_entries;
return 0; /* pretend success, drop the data */
}
#endif
/* ------------------------------------------------------------------ */
/* Test-local fixtures */
/* ------------------------------------------------------------------ */
/* Spin up a real on-disk persist_t in a fresh tmpdir (no cluster wired —
* apply_wal_entry's entity-restore path is skipped when p->cluster == NULL,
* which is exactly the no-effect guarantee we assert). */
static persist_t* make_lifecycle_persist(const char* suffix)
{
persist_t* p = calloc(1, sizeof(persist_t));
if (!p)
return NULL;
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000; /* slow background fsync — no races in tests */
char tmpdir[256];
snprintf(tmpdir, sizeof(tmpdir), "/tmp/xce_noop_%s_XXXXXX", suffix);
if (!mkdtemp(tmpdir)) {
free(p);
return NULL;
}
persist_set_workdir(p, tmpdir, "ctrl-test");
if (persist_init(p, &cfg, NULL) != 0) {
free(p);
return NULL;
}
if (persist_start(p) != 0) {
persist_fini(p);
free(p);
return NULL;
}
return p;
}
static void teardown_lifecycle_persist(persist_t* p)
{
if (!p)
return;
persist_stop(p);
persist_fini(p);
free(p);
/* tmpdir left on disk — /tmp is ephemeral */
}
/* Read the most-recently-appended WAL entry header back from the active
* segment file. Returns 0 and fills *out on success, negative on error.
* Reads the header at (seg_offset - sizeof(hdr)) — the last entry written. */
static int read_last_wal_hdr(persist_t* p, wal_entry_hdr_t* out)
{
if (p->seg_offset < sizeof(wal_entry_hdr_t))
return -ENOENT;
int fd = open(p->seg_path, O_RDONLY);
if (fd < 0)
return -errno;
off_t at = (off_t)(p->seg_offset - sizeof(wal_entry_hdr_t));
ssize_t r = pread(fd, out, sizeof(*out), at);
close(fd);
if (r != (ssize_t)sizeof(*out))
return -EIO;
return 0;
}
/* Evaluate wal_classify(WAL_NOOP, 0) in a forked child and report whether it
* cleanly returned WAL_MODE_RETAIN. Returns 1 iff the child exited 0 (no
* abort, RETAIN), 0 otherwise (abort/SIGABRT today, since WAL_NOOP is
* unclassified → the totality fail-fast `assert(0)` fires). Isolating the
* call protects the test binary from the abort (cerebrum fork pattern). */
static int noop_classifies_retain(void)
{
pid_t pid = fork();
if (pid < 0)
return -1;
if (pid == 0) {
setenv("ASAN_OPTIONS", "detect_leaks=0", 1);
int devnull = open("/dev/null", O_WRONLY);
if (devnull >= 0) {
dup2(devnull, STDERR_FILENO);
dup2(devnull, STDOUT_FILENO);
}
wal_commit_mode_t mode = wal_classify(WAL_NOOP_VALUE, 0);
_exit(mode == WAL_MODE_RETAIN ? 0 : 1);
}
int status = 0;
if (waitpid(pid, &status, 0) != pid)
return -1;
return (WIFEXITED(status) && WEXITSTATUS(status) == 0) ? 1 : 0;
}
/* ------------------------------------------------------------------ */
/* Task 1.1 — WAL_NOOP no-op entry (Req 2.1) */
/* ------------------------------------------------------------------ */
/* Case 1: persist_wal_async(p, WAL_NOOP, NULL, 0) appends a header-only
* entry (payload_len == 0) and broadcasts it through the production path.
*
* Observables:
* - return 0 (append succeeded);
* - the on-disk last entry has type == WAL_NOOP and payload_len == 0;
* - last_replicated_crc advanced to this entry's CRC (the same field
* persist_wal_async sets immediately alongside repl_broadcast,
* persist.c:359 — proxy for "the entry was handed to the replicator").
*
* RED today: persist_wal_async will write a type-40 entry, but the spec
* requires WAL_NOOP to be appended with payload_len 0 ONLY via this path;
* the binding of the value-40 type to the WAL_NOOP no-op contract does not
* exist until task 1.2 wires the explicit replay/classify handling. The
* payload_len/type-round-trip assertion below pins the entry shape that 1.2
* must produce; it is asserted now so 1.2 has a concrete target. */
static void test_noop_append_is_header_only_and_broadcast(void)
{
/* Arrange */
persist_t* p = make_lifecycle_persist("append");
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
uint32_t crc_before = p->last_replicated_crc;
/* Act: append a no-op (header only, NULL payload, len 0) */
int rc = persist_wal_async(p, WAL_NOOP_VALUE, NULL, 0);
/* Assert: append succeeded and consumed exactly one LSN */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"WAL_NOOP append must succeed via persist_wal_async");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before + 1, p->current_lsn,
"WAL_NOOP append must advance current_lsn by exactly one");
/* Assert: the on-disk entry is header-only of the WAL_NOOP type */
wal_entry_hdr_t hdr;
memset(&hdr, 0, sizeof(hdr));
TEST_ASSERT_EQUAL_INT_MESSAGE(0, read_last_wal_hdr(p, &hdr),
"must be able to read the appended WAL_NOOP header back from disk");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(WAL_NOOP_VALUE, hdr.type,
"appended entry must be of type WAL_NOOP");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(0u, hdr.payload_len,
"WAL_NOOP must be header-only (payload_len == 0)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before, hdr.lsn,
"WAL_NOOP entry LSN must be the pre-append current_lsn");
/* Assert: the entry was broadcast (last_replicated_crc moved off its
* pre-append value to this entry's crc — the replicate-path observable) */
TEST_ASSERT_NOT_EQUAL_MESSAGE(crc_before, p->last_replicated_crc,
"WAL_NOOP append must broadcast (last_replicated_crc advances)");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(hdr.crc, p->last_replicated_crc,
"broadcast CRC must match the appended WAL_NOOP entry CRC");
/* Assert: WAL_NOOP is a CLASSIFIED type. An entry appended through the
* write path must be classifiable (the design lands the enum value and
* its wal_classify row in the SAME change, task 1.2). Today WAL_NOOP is
* unclassified, so this is RED: the classify call aborts under the
* totality guard (observed via the forked-child helper). */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, noop_classifies_retain(),
"an appended WAL_NOOP must be a classified type (wal_classify → RETAIN)");
teardown_lifecycle_persist(p);
printf("test_noop_append_is_header_only_and_broadcast: PASS\n");
}
/* Case 2: apply_wal_entry on a WAL_NOOP advances current_lsn past the
* entry's LSN and touches NO entity state.
*
* With p->cluster == NULL the entity-restore switch is skipped entirely, so
* "no entity state touched" is structurally guaranteed; the contract this
* pins is the LSN advance via the common path (persist.c:719-723) for a
* WAL_NOOP header-only entry.
*
* RED today: the entry applies (type 40 < WAL_ENTRY_TYPE_MAX, so the common
* path advances LSN), BUT task 1.2 must add an EXPLICIT `case WAL_NOOP:
* break;` so the no-op is a recognised type rather than falling through the
* `default:` arm. We additionally require the apply to return 0 AND not
* emit the unknown-type WARN path — pinned here as the apply contract 1.2
* realises. The assertion that the entry is recognised (return 0 with a
* header-only payload and exact LSN advance) is the RED carrier. */
static void test_noop_apply_advances_lsn_no_entity_effect(void)
{
/* Arrange: a persist with NO cluster — entity restore is skipped, so any
* observable effect would have to come from the WAL_NOOP arm itself */
persist_t* p = make_lifecycle_persist("apply");
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_NULL_MESSAGE(p->cluster,
"fixture must have no cluster wired (no-effect guarantee depends on it)");
/* Build a header-only WAL_NOOP entry at a known LSN beyond current */
uint64_t noop_lsn = p->current_lsn + 5;
wal_entry_hdr_t hdr;
memset(&hdr, 0, sizeof(hdr));
hdr.term = p->term;
hdr.lsn = noop_lsn;
hdr.timestamp = 0;
hdr.type = WAL_NOOP_VALUE;
hdr.payload_len = 0;
hdr.crc = 0;
hdr.prev_crc = 0;
/* Act: apply the no-op with a NULL payload (header-only) */
int rc = apply_wal_entry(p, &hdr, NULL);
/* Assert: clean apply, LSN advanced past the entry, nothing else */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"applying a WAL_NOOP must succeed");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn + 1, p->current_lsn,
"applying a WAL_NOOP must advance current_lsn to entry LSN + 1");
/* No cluster → no entity state to inspect; the absence of effect is the
* point. Re-applying must be idempotent on LSN (already advanced). */
int rc2 = apply_wal_entry(p, &hdr, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc2,
"re-applying a WAL_NOOP must remain a clean no-op");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn + 1, p->current_lsn,
"re-applying a lower/equal LSN WAL_NOOP must not regress current_lsn");
/* Assert: WAL_NOOP is a recognised, classified entry type — not an
* unknown type the apply/classify paths only tolerate by fallthrough.
* RED today: wal_classify(WAL_NOOP) aborts (unclassified) per the
* totality guard; task 1.2 adds the explicit case + RETAIN row. */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, noop_classifies_retain(),
"an applied WAL_NOOP must be a classified type (wal_classify → RETAIN)");
teardown_lifecycle_persist(p);
printf("test_noop_apply_advances_lsn_no_entity_effect: PASS\n");
}
/* Case 3: boot replay over a segment containing a WAL_NOOP replays cleanly
* with the same no-effect guarantee, and the WAL_NOOP is a RECOGNISED type.
*
* Fixture shape (deliberate): append ONE ordinary, already-classified WAL
* entry (WAL_JOB_START, header-only) FIRST so it takes lsn 0, THEN append the
* WAL_NOOP so it lands at lsn 1. This matters because replay_wal skips any
* entry with `hdr.lsn <= from_lsn` (persist.c:907) and a fresh persist recovers
* with from_lsn == checkpoint_lsn == 0. A WAL_NOOP written as the FIRST entry
* would sit at lsn 0 and be skipped as snapshot-covered — an lsn-0/checkpoint-
* boundary FIXTURE ARTIFACT unrelated to the WAL_NOOP contract (it would fail
* for ANY type at lsn 0 and would NOT be fixed by task 1.2). By seating the
* noop at lsn > 0 we pin the actual boot-replay contract instead.
*
* The leading WAL_JOB_START is side-effect-free here: apply_wal_entry returns
* early at the `if (!p->cluster)` guard (persist.c:727-728) before the entity-
* restore switch, so neither the ordinary entry nor the noop touches entity
* state during replay — preserving the no-effect guarantee.
*
* Observables that PIN THE CONTRACT (these hold both today and after task 1.2,
* because WAL_NOOP_VALUE 40 <= WAL_ENTRY_TYPE_MAX 100 already routes through
* apply_wal_entry, which advances current_lsn):
* - persist_recover returns 0 (replay clean);
* - current_lsn advances STRICTLY past the noop's lsn (>= noop_lsn + 1);
* - no entity side effects (NULL cluster — structurally guaranteed).
*
* The RED PIN is the SHARED classify carrier noop_classifies_retain(): boot
* replay of a WAL_NOOP is only correct when the type is a recognised, RETAIN-
* classified entry — not an unclassified type tolerated by routing alone.
* Today wal_classify(40) aborts under the totality guard (carrier returns 0);
* task 1.2 lands the enum + `case WAL_NOOP: break;` apply arm + the RETAIN
* classify row TOGETHER, flipping the carrier to 1 and this case to green. */
static void test_noop_boot_replay_clean_no_effect(void)
{
char tmpdir[256];
uint64_t noop_lsn;
/* Arrange: write [WAL_JOB_START (lsn 0), WAL_NOOP (lsn 1)] into a fresh
* persist so the noop sits at lsn > 0 (escaping the lsn<=from_lsn skip),
* capture the noop's LSN, flush, then tear down (keeping the on-disk WAL). */
{
persist_t* p = calloc(1, sizeof(persist_t));
TEST_ASSERT_NOT_NULL(p);
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
strcpy(tmpdir, "/tmp/xce_noop_replay_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir));
persist_set_workdir(p, tmpdir, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(p, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(p));
/* Leading ORDINARY entry — already classified, header-only, harmless
* on replay under a NULL cluster. Takes lsn 0 so the noop gets lsn>0. */
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
persist_wal_async(p, WAL_JOB_START, NULL, 0),
"leading ordinary WAL entry must append for the replay fixture");
noop_lsn = p->current_lsn; /* the LSN the noop is about to take (> 0) */
TEST_ASSERT_TRUE_MESSAGE(noop_lsn > 0,
"noop must sit at lsn > 0 so replay does not skip it as "
"snapshot-covered (hdr.lsn <= from_lsn guard, persist.c:907)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
persist_wal_async(p, WAL_NOOP_VALUE, NULL, 0),
"WAL_NOOP must append for the replay fixture");
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
persist_stop(p);
persist_fini(p);
free(p);
}
/* Act: recover a FRESH persist over the same workdir (boot replay).
* persist_init does NOT scan segments / set current_lsn — recovery is
* explicit via persist_recover, so current_lsn starts at the calloc'd 0. */
persist_t* p2 = calloc(1, sizeof(persist_t));
TEST_ASSERT_NOT_NULL(p2);
persist_config_t cfg2 = persist_config_defaults();
cfg2.wal_sync_ms = 5000;
persist_set_workdir(p2, tmpdir, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg2, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(p2));
TEST_ASSERT_NULL_MESSAGE(p2->cluster,
"replay fixture must have no cluster (no-effect guarantee)");
int rc = persist_recover(p2);
/* Assert: replay clean, LSN advanced STRICTLY past the no-op (contract). */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"boot replay over a WAL_NOOP must succeed");
TEST_ASSERT_TRUE_MESSAGE(p2->current_lsn > noop_lsn,
"boot replay over a WAL_NOOP must advance current_lsn past the no-op");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn + 1, p2->current_lsn,
"boot replay must leave current_lsn at one past the WAL_NOOP's LSN");
persist_stop(p2);
persist_fini(p2);
free(p2);
/* Assert (RED PIN): the replayed WAL_NOOP must be a recognised, RETAIN-
* classified type — not merely tolerated by apply/replay routing. Today
* wal_classify(WAL_NOOP) aborts (unclassified) per the totality guard, so
* the forked carrier returns 0 and this case is RED for the WAL_NOOP
* CONTRACT reason. Task 1.2's enum + apply arm + RETAIN classify row flips
* it to 1, turning this case green. */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, noop_classifies_retain(),
"a replayed WAL_NOOP must be a classified type (wal_classify → RETAIN)");
printf("test_noop_boot_replay_clean_no_effect: PASS\n");
}
/* Case 4: wal_classify(WAL_NOOP, 0) == WAL_MODE_RETAIN.
*
* This keeps barrier-rollout's classification-totality test meaningful: the
* enum value and its classification row must land together (task 1.2).
*
* RED today: WAL_NOOP (40) is unclassified, so wal_classify hits the
* totality fail-fast `assert(0)` under XCE_TESTING and ABORTS. Calling it
* inline would crash the whole test binary, so it is evaluated in a forked
* child (cerebrum fork abort-test pattern). The contract is RETAIN; the
* child computes wal_classify(40,0) and exits 0 ONLY if it returns RETAIN
* without aborting. Today the child aborts (SIGABRT), so the parent's
* "exited cleanly with RETAIN" assertion FAILS — the captured RED state.
*
* detect_leaks=0 set in the child so ASAN does not flag the abort path. */
static void test_noop_classify_is_retain(void)
{
pid_t pid;
int status = 0;
/* Act: classify WAL_NOOP in a child (it may abort in the current tree) */
pid = fork();
TEST_ASSERT_TRUE_MESSAGE(pid >= 0, "fork must succeed");
if (pid == 0) {
/* Child: silence the expected assert/abort noise; ASAN-friendly. */
setenv("ASAN_OPTIONS", "detect_leaks=0", 1);
int devnull = open("/dev/null", O_WRONLY);
if (devnull >= 0) {
dup2(devnull, STDERR_FILENO);
dup2(devnull, STDOUT_FILENO);
}
wal_commit_mode_t mode = wal_classify(WAL_NOOP_VALUE, 0);
/* Reaching here means no abort; encode the result in the exit code:
* exit 0 ⇔ RETAIN (the contract), exit 1 ⇔ classified-but-wrong. */
_exit(mode == WAL_MODE_RETAIN ? 0 : 1);
}
/* Assert: the child must exit cleanly with RETAIN. Today it aborts
* (SIGABRT) because WAL_NOOP is unclassified — this assertion is RED. */
TEST_ASSERT_EQUAL_INT_MESSAGE(pid, waitpid(pid, &status, 0),
"waitpid must reap the classifier child");
TEST_ASSERT_TRUE_MESSAGE(WIFEXITED(status),
"wal_classify(WAL_NOOP) must NOT abort — it must classify the type");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, WEXITSTATUS(status),
"wal_classify(WAL_NOOP, 0) must return WAL_MODE_RETAIN");
printf("test_noop_classify_is_retain: PASS\n");
}
/* ══════════════════════════════════════════════════════════════════════════
* Task 2.1 (TDD RED) — DIRECT promotion + the OWN-TERM GATE.
*
* Contract (design §Promotion Rewrite, Reqs 1.1, 1.3, 1.4, 1.5, 2.1, 2.2, 2.3):
* every road to PRIMARY lands the node DIRECTLY in CTRL_PRIMARY — no
* CTRL_PRIMARY_COMMITTING ever observable — registers the persist hooks,
* arms NO wall-clock watchdog, and (production, c != NULL) appends a WAL_NOOP
* of the current term then sets the commit engine's own-term gate
* (commit_set_writable_lsn) to that no-op's LSN so confirmation of any
* operation is withheld until committed_lsn >= noop_lsn.
*
* SEAMS (study notes; no new production seam is introduced — RED-only):
* - Cases 1-3 drive the raft become-leader callback via its existing
* XCE_TESTING shim `raft_cb_become_leader_for_test(ha, term)` (ha.c:3121),
* which calls `raft_cb_become_leader` with `ha->raft_c = NULL`. This is
* THE seam test_ha_evaluate_committing.c / test_ha_connection_reuse.c use.
* RED today: that path stores CTRL_PRIMARY_COMMITTING (ha.c:514), so the
* direct-PRIMARY / is_primary / no-redirect / hooks-registered assertions
* fail. (The `c == NULL` shim also means the watchdog-arm + noop/gate
* blocks — both under `if (c)` — are skipped, so they are NOT observable
* through THIS seam; see cases 4-7 for the real-cluster seam.)
* - Cases 4-7 drive the SHARED promotion sequence `ha_promote_to_primary(c)`
* (ha.c:1717, public in ha.h) against a partial-but-real cluster_t: a real
* tmpdir persist_t, a wired commit_engine_t (n_peers), a stub standby
* registry (repl_add_standby + repl_set_commit_engine), and a delaymgr
* left un-started. Per design §Promotion Rewrite, ha_promote_to_primary
* becomes the canonical "every road to PRIMARY arms the gate" entry point
* (task 2.2 adds the noop+gate tail there + in the raft callback). TODAY
* ha_promote_to_primary does ha_do_promote + hook-register + heartbeat and
* NEVER touches c->commit, so the own-term gate stays unset (writable_lsn
* == 0) — that is the RED carrier for cases 5-6.
*
* GATE OBSERVABILITY (cerebrum + commit.c:166-171): the engine's WAIT
* completion predicate is `committed_lsn >= lsn && (writable_lsn == 0 ||
* committed_lsn >= writable_lsn)`. writable_lsn is a public commit_engine_t
* field; commit_set_writable_lsn(ce, N) installs the gate. With the gate at N
* a WAIT on an entry M < N blocks past M's own commit, completing only when
* committed_lsn reaches N (test_commit_engine.c test 15 shape). TODAY
* promotion leaves writable_lsn == 0, so that same WAIT completes the instant
* M commits — the behavioral RED carrier.
*
* DELAYMGR TASK-COUNT SEAM (case 4): delay_task_ms enqueues into
* mgr->tasks (an ordq_t) regardless of running state (delay.c) — count the
* `tasks.head` singly-linked chain directly (ordq has no public count API).
* ════════════════════════════════════════════════════════════════════════ */
#ifdef XCE_TESTING
/* The raft become-leader callback shim (defined in ha.c under XCE_TESTING). */
extern void raft_cb_become_leader_for_test(ha_state_t *ha, uint64_t term);
#endif
/* Minimal ha_config_t accepted by ha_init (mirrors test_ha_role.c). */
static ha_config_t lifecycle_ha_cfg(void)
{
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
};
return cfg;
}
/* Count pending delaymgr tasks by walking the ordq head chain directly.
* Read-only: takes no lock (single-threaded test context, delaymgr never
* started). This is the "delaymgr task-count seam" the contract pins. */
static int delaymgr_task_count(const delaymgr_t *mgr)
{
int count = 0;
const ordq_item_t *it = mgr->tasks.head;
while (it != NULL) {
count++;
it = it->next;
}
return count;
}
/* ── Case 1: DIRECT PRIMARY — become-leader lands in CTRL_PRIMARY, not a
* transitional state; ha_is_primary() is true immediately on return (Req 1.1,
* 1.4). RED today: raft_cb_become_leader stores CTRL_PRIMARY_COMMITTING. */
static void test_promotion_lands_directly_in_primary(void)
{
#ifdef XCE_TESTING
ha_state_t ha;
ha_config_t cfg = lifecycle_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
ha.phase2_done = 1; /* skip phase2 (no real cluster behind the shim) */
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
/* Act: drive the raft become-leader path (shim sets ha.raft_c = NULL) */
raft_cb_become_leader_for_test(&ha, /*term=*/1);
/* Assert: role is CTRL_PRIMARY directly — no transitional value */
ctrl_role_t role =
(ctrl_role_t)atomic_load_explicit(&ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, (int)role,
"winning election must land DIRECTLY in CTRL_PRIMARY (Req 1.1); "
"current code stores CTRL_PRIMARY_COMMITTING — RED");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha_is_primary(&ha),
"ha_is_primary() must be true the instant promotion returns (Req 1.4)");
ha_fini(&ha);
printf("test_promotion_lands_directly_in_primary: PASS\n");
#else
printf("test_promotion_lands_directly_in_primary: SKIP (non-test build)\n");
#endif
}
/* ── Case 2: NO REDIRECT — ha_should_redirect() returns 0 the instant
* promotion returns, before any standby ACK (Req 1.3). RED today: in
* CTRL_PRIMARY_COMMITTING ha_should_redirect can return 1 (a live election
* winner pointing clients at a primary that "does not exist yet"). */
static void test_promotion_no_redirect_from_live_primary(void)
{
#ifdef XCE_TESTING
ha_state_t ha;
ha_config_t cfg = lifecycle_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
ha.phase2_done = 1;
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
/* A stale primary hint is present (would trigger redirect for any
* non-PRIMARY role): host+port set as if learned from a prior leader. */
strncpy(ha.primary_host, "10.0.0.9", sizeof(ha.primary_host) - 1);
ha.primary_port = 9991;
/* Act */
raft_cb_become_leader_for_test(&ha, /*term=*/1);
/* Assert: a live primary never redirects (Req 1.3). In CTRL_PRIMARY the
* stale hint is ignored; in CTRL_PRIMARY_COMMITTING (today) the hint makes
* ha_should_redirect return 1 → RED. */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha_should_redirect(&ha),
"a freshly promoted primary must NOT redirect clients (Req 1.3); "
"current COMMITTING role + a primary hint yields redirect=1 — RED");
ha_fini(&ha);
printf("test_promotion_no_redirect_from_live_primary: PASS\n");
#else
printf("test_promotion_no_redirect_from_live_primary: SKIP (non-test build)\n");
#endif
}
/* Case 3 (HOOKS REGISTERED BY THE SHARED PROMOTION SEQUENCE) is defined just
* below the real-cluster fixture (lifecycle_cluster_setup / g_lc) it shares
* with cases 4-7, since C requires those helpers to be declared before use.
* It is still RUN as case 3 (see test_ha_role_lifecycle below). */
/* ------------------------------------------------------------------ */
/* Real-cluster fixture for cases 3-7: a partial-but-real cluster_t */
/* whose persist (real tmpdir), commit engine (wired), stub standby */
/* registry and (un-started) delaymgr are the surfaces promotion drives.*/
/* ------------------------------------------------------------------ */
/* Statics survive Unity's longjmp on assertion failure (test_repl_majority
* pattern); cluster_t is large and must not live on the test stack. */
static cluster_t g_lc;
static int g_lc_eps[MAX_STANDBYS];
/* Build g_lc with: real persist over a fresh tmpdir, current_lsn primed so a
* promotion no-op would land at lsn = appended_lsn (= current_lsn - 1);
* n_standbys dummy standbys in the registry; commit engine wired to the
* registry (n_peers, n_vcpus); delaymgr initialised but NOT started; role
* forced to CTRL_ELECTING. Returns the lsn the next appended entry takes. */
static uint64_t lifecycle_cluster_setup(const char *suffix, int n_peers,
int n_standbys, uint64_t appended_lsn,
int n_vcpus)
{
int i;
char name[32];
char tmpdir[256];
memset(&g_lc, 0, sizeof(g_lc));
/* HA state */
ha_config_t cfg = lifecycle_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&g_lc.ha, &cfg, 0));
g_lc.ha.phase2_done = 1; /* no phase2 wiring in this fixture */
g_lc.ha.n_peers = n_peers;
atomic_store_explicit(&g_lc.ha.role, (int32_t)CTRL_ELECTING,
memory_order_release);
/* Real persist over a private tmpdir (the no-op append target). */
persist_config_t pcfg = persist_config_defaults();
pcfg.wal_sync_ms = 5000; /* slow background fsync — no test races */
snprintf(tmpdir, sizeof(tmpdir), "/tmp/xce_slr_%s_XXXXXX", suffix);
TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir));
persist_set_workdir(&g_lc.persist, tmpdir, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(&g_lc.persist, &pcfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(&g_lc.persist));
/* Prime the appended watermark: current_lsn = appended_lsn + 1 means the
* highest appended entry is `appended_lsn` and the next append takes it. */
pthread_mutex_lock(&g_lc.persist.lock);
g_lc.persist.current_lsn = appended_lsn + 1;
pthread_mutex_unlock(&g_lc.persist.lock);
atomic_store(&g_lc.persist.committed_lsn, 0);
/* Stub standby registry + wired engine (commit derives from watermarks). */
for (i = 0; i < n_standbys; i++) {
snprintf(name, sizeof(name), "lcsb%d", i);
TEST_ASSERT_TRUE_MESSAGE(
repl_add_standby(&g_lc.persist.repl, &g_lc_eps[i], name,
7400 + (uint32_t)i) >= 0,
"repl_add_standby must return a slot index >= 0");
}
TEST_ASSERT_EQUAL_INT(0,
commit_engine_init(&g_lc.commit, &g_lc.persist, n_peers, n_vcpus));
repl_set_commit_engine(&g_lc.persist.repl, &g_lc.commit);
/* Drop wire sends from the own-term WAL_NOOP broadcast — the fake int
* endpoints have no real ep->peer (see lc_drop_send_hook above). */
#ifdef XCE_TESTING
repl_broadcast_send_hook = lc_drop_send_hook;
#endif
/* delaymgr initialised but deliberately NOT started: promotion's heartbeat
* arm is skipped (ha_start_primary_heartbeat guards on delaymgr.running),
* so any task that appears in tasks.head was armed by promotion itself. */
TEST_ASSERT_EQUAL_INT(0, delaymgr_init(&g_lc.delaymgr, 100, NULL));
return appended_lsn; /* lsn a promotion no-op would take (current_lsn - 1) */
}
static void lifecycle_cluster_teardown(void)
{
#ifdef XCE_TESTING
repl_broadcast_send_hook = NULL;
#endif
commit_engine_fini(&g_lc.commit);
delaymgr_fini(&g_lc.delaymgr);
persist_stop(&g_lc.persist);
persist_fini(&g_lc.persist);
repl_fini(&g_lc.persist.repl);
ha_fini(&g_lc.ha);
/* tmpdir left on disk — /tmp is ephemeral */
}
/* Drive a standby ACK watermark into the registry + engine (production
* repl_on_ack shape: store acked_lsn, then feed commit_on_ack). */
static void lifecycle_standby_ack(int idx, uint64_t acked_lsn)
{
atomic_store(&g_lc.persist.repl.standbys[idx].acked_lsn, acked_lsn);
commit_on_ack(&g_lc.commit, idx, acked_lsn);
}
static uint64_t lifecycle_committed(void)
{
return atomic_load(&g_lc.persist.committed_lsn);
}
static uint64_t lifecycle_mono_ms(void)
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (uint64_t)ts.tv_sec * 1000u + (uint64_t)ts.tv_nsec / 1000000u;
}
static void lifecycle_sleep_ms(uint32_t ms)
{
struct timespec ts;
ts.tv_sec = ms / 1000u;
ts.tv_nsec = (long)(ms % 1000u) * 1000000L;
nanosleep(&ts, NULL);
}
/* WAIT-on-an-entry thread context + body (test_commit_engine.c pattern). */
typedef struct {
uint64_t lsn;
uint32_t timeout_ms;
int rc;
uint64_t elapsed_ms;
atomic_int done;
} lc_wait_arg_t;
static void *lifecycle_wait_thread(void *p)
{
lc_wait_arg_t *a = (lc_wait_arg_t *)p;
uint64_t t0 = lifecycle_mono_ms();
a->rc = commit_submit(&g_lc.commit, a->lsn, COMMIT_MODE_WAIT,
a->timeout_ms);
a->elapsed_ms = lifecycle_mono_ms() - t0;
atomic_store(&a->done, 1);
return NULL;
}
/* ── Case 3: HOOKS REGISTERED BY THE SHARED PROMOTION SEQUENCE — the
* canonical promotion entry point ha_promote_to_primary(c) (ha.c:1717) must
* register the three persist hooks so an idle cluster's new primary is
* writable; observe via c->ha.hooks_registered transitioning 0 → 1 (Req 2.1).
*
* SHARED-SEQUENCE CONTRACT PIN (GREEN now, MUST STAY GREEN — not a behavioral
* RED carrier). After task 2.2 BOTH roads to PRIMARY (the raft become-leader
* callback and ha_repl_ack_received's quorum path) route through this same
* ha_promote_to_primary sequence, which task 2.2 augments with the noop+gate
* tail while KEEPING hook registration. This case is therefore a CONTRACT/
* REGRESSION pin proving the shared promotion sequence registers the persist
* hooks; the behavioral RED signal for 2.2 is carried by cases 1, 2, 5, 6.
*
* It uses the SAME real-cluster fixture cases 4-7 use (lifecycle_cluster_setup
* / g_lc), which is why it is defined here, below that fixture. This is the
* fix for the prior c==NULL shim defect: the raft become-leader shim forces
* ha->raft_c == NULL (ha.c:3121), which routes AROUND the c-dependent
* registration block, so it could never set the flag on that seam — pinning
* it there left an un-turnable RED. ha_promote_to_primary is the real shared
* seam and registers hooks today (ha.c:1722-1730).
*
* Why this is GREEN today: ha_promote_to_primary already registers the three
* persist hooks and sets hooks_registered = 1 (ha.c:1729). The g_lc fixture's
* hooksys is zeroed (memset), so xce_add_hook returns -EINVAL harmlessly
* (typ >= maxtypes==0, hook.c:41) WITHOUT storing the fns — but the flag still
* transitions 0 → 1 (ha.c:1714-1715 documents exactly this). The
* hooks_registered 0 → 1 transition is therefore the clean, observable
* contract pin on the zeroed-hooksys fixture; the three fns are NOT observable
* in g_lc.hooksys here (the zeroed registry rejects the add), so we pin the
* flag transition per the task wording rather than the slot contents. */
static void test_promotion_registers_persist_hooks(void)
{
#ifdef XCE_TESTING
/* Arrange: a partial-but-real cluster (same fixture as cases 4-7).
* n_peers value is not material to the hook contract. */
(void)lifecycle_cluster_setup("hooks", /*n_peers=*/3, /*n_standbys=*/2,
/*appended_lsn=*/5, /*n_vcpus=*/8);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_lc.ha.hooks_registered,
"fixture precondition: hooks must be unregistered before promotion");
/* Act: drive the SHARED promotion sequence both roads route through (2.2). */
TEST_ASSERT_EQUAL_INT(0, ha_promote_to_primary(&g_lc));
/* Assert: the shared promotion sequence registered the persist hooks —
* hooks_registered transitioned 0 → 1 (Req 2.1). Contract/regression pin:
* green today (ha.c:1729) and must remain green after task 2.2 augments the
* sequence with the noop+gate tail. */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_lc.ha.hooks_registered,
"the shared promotion sequence (ha_promote_to_primary) must register "
"the persist hooks (hooks_registered 0→1, Req 2.1) — contract pin, "
"must stay green through task 2.2");
lifecycle_cluster_teardown();
printf("test_promotion_registers_persist_hooks: PASS\n");
#else
printf("test_promotion_registers_persist_hooks: SKIP (non-test build)\n");
#endif
}
/* ── Case 4: NO WATCHDOG — promotion arms no delaymgr task (Req 1.5, 2.3).
*
* The shared promotion sequence must never arm a wall-clock watchdog (the
* deleted committing_timeout_fn). Asserted via the delaymgr task-count seam:
* the count is unchanged across ha_promote_to_primary.
*
* Status note (RED vs already-GREEN): the force-promotion watchdog is armed
* ONLY inside raft_cb_become_leader's `if (c)` block (ha.c:611-619), and that
* block has NO real-cluster unit seam (raft_cb_become_leader is static; its
* only shim forces c == NULL). ha_promote_to_primary — the road this case
* exercises — never armed a watchdog, so this case is ALREADY GREEN today and
* stands as a regression-pin guaranteeing task 2.2's shared sequence does not
* introduce one. The watchdog REMOVAL from raft_cb_become_leader is covered
* at integration scope (task 4.x deletes committing_timeout_fn and the
* test_ha_evaluate_committing committing-timeout cases). */
static void test_promotion_arms_no_delaymgr_watchdog(void)
{
#ifdef XCE_TESTING
/* Arrange: 3-member real cluster, delaymgr started so an armed watchdog
* (if any) would enqueue; heartbeat stays skipped because the static
* one-shot arm + the role no-op keep it from stacking a task here. */
(void)lifecycle_cluster_setup("nowd", /*n_peers=*/3, /*n_standbys=*/2,
/*appended_lsn=*/5, /*n_vcpus=*/8);
int before = delaymgr_task_count(&g_lc.delaymgr);
/* Act: drive the shared promotion sequence */
TEST_ASSERT_EQUAL_INT(0, ha_promote_to_primary(&g_lc));
/* Assert: no wall-clock task was armed by promotion (Req 1.5, 2.3) */
int after = delaymgr_task_count(&g_lc.delaymgr);
TEST_ASSERT_EQUAL_INT_MESSAGE(before, after,
"promotion must arm NO delaymgr watchdog (Req 1.5, 2.3) — "
"the delaymgr task count must be unchanged across promotion");
/* And the road still landed in CTRL_PRIMARY directly (Req 1.1) */
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY,
(int)atomic_load_explicit(&g_lc.ha.role, memory_order_acquire),
"shared promotion sequence must reach CTRL_PRIMARY directly");
lifecycle_cluster_teardown();
printf("test_promotion_arms_no_delaymgr_watchdog: PASS "
"(delaymgr tasks before=%d after=%d)\n", before, after);
#else
printf("test_promotion_arms_no_delaymgr_watchdog: SKIP (non-test build)\n");
#endif
}
/* ── Case 5: OWN-TERM GATE (real engine, n_peers=3, stub registry) —
* promotion must append a WAL_NOOP at lsn=N, register it RETAIN, and set the
* commit engine gate to N (commit_set_writable_lsn(N)). The behavioral
* consequence: a WAIT for an entry M < N does NOT complete while ACKs cover M
* but not N, and completes only once committed_lsn >= N (Req 2.1).
*
* RED today: ha_promote_to_primary never touches c->commit, so writable_lsn
* stays 0 (no gate) and the WAIT on M completes the instant M commits. Two
* RED carriers are pinned: (a) the gate field writable_lsn == noop_lsn after
* promotion (today it is 0); (b) the behavioral block-until-gate. */
static void test_promotion_appends_noop_and_arms_own_term_gate(void)
{
#ifdef XCE_TESTING
/* Arrange: 3-member cluster. Prime so the promotion no-op lands at lsn
* N=3 (current_lsn-1). Entries 1..3 are appended into the engine so a
* waiter on entry M=1 has a target below the gate. */
const uint64_t noop_lsn = 3;
/* persist_wal_async post-increments current_lsn (persist.c:318), and
* production derives noop_lsn = current_lsn - 1. The fixture primes
* current_lsn = appended_lsn + 1, so passing appended_lsn = noop_lsn - 1
* lands the no-op at exactly lsn = noop_lsn. */
(void)lifecycle_cluster_setup("gate", /*n_peers=*/3, /*n_standbys=*/2,
/*appended_lsn=*/noop_lsn - 1, /*n_vcpus=*/8);
/* Register pre-existing entries strictly BELOW the gate (1..noop_lsn-1);
* the no-op itself occupies noop_lsn. A waiter on M=1 still has a
* target below the gate. */
for (uint64_t lsn = 1; lsn < noop_lsn; lsn++)
(void)commit_submit(&g_lc.commit, lsn, COMMIT_MODE_RETAIN, 0);
/* Act: drive the shared promotion sequence (appends WAL_NOOP + gate). */
TEST_ASSERT_EQUAL_INT(0, ha_promote_to_primary(&g_lc));
/* Assert (gate field): writable_lsn must be the no-op's LSN (Req 2.1).
* RED today: promotion does not touch the engine → writable_lsn == 0. */
TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn, g_lc.commit.writable_lsn,
"promotion must set the own-term gate writable_lsn = noop_lsn (Req 2.1); "
"current promotion never appends a no-op nor sets the gate — RED");
/* Assert (WAL): a header-only WAL_NOOP must have been appended. RED
* today: nothing is appended, so current_lsn never advanced past the
* primed watermark. */
TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn + 1, g_lc.persist.current_lsn,
"promotion must append exactly one WAL_NOOP (current_lsn advances to "
"noop_lsn+1, Req 2.1) — current promotion appends nothing — RED");
/* Assert (behavioral gate): a WAIT on entry M=1 < gate must NOT complete
* once entry 1 commits — it must block until committed_lsn >= noop_lsn. */
static lc_wait_arg_t warg;
memset(&warg, 0, sizeof(warg));
warg.lsn = 1; warg.timeout_ms = 3000; warg.rc = -9999;
atomic_store(&warg.done, 0);
pthread_t wtid;
TEST_ASSERT_EQUAL_INT(0,
pthread_create(&wtid, NULL, lifecycle_wait_thread, &warg));
/* Entry 1 gains majority (self + sb0); gate (noop_lsn) still unmet. */
lifecycle_sleep_ms(50);
lifecycle_standby_ack(0, 1);
lifecycle_sleep_ms(150);
int done_after_entry1 = atomic_load(&warg.done);
/* Gate opens: a covering ACK advances committed_lsn to noop_lsn. */
lifecycle_standby_ack(0, noop_lsn);
pthread_join(wtid, NULL);
/* RED today: with writable_lsn == 0 the waiter completed at the first
* ACK (done_after_entry1 == 1). Contract: it must still be blocked. */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, done_after_entry1,
"WAIT on entry 1 must NOT complete while committed_lsn < noop_lsn "
"(own-term gate, Req 2.1); unset gate completes it early — RED");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, warg.rc,
"WAIT must succeed once committed_lsn reaches the own-term gate");
TEST_ASSERT_TRUE_MESSAGE(warg.elapsed_ms >= 180,
"WAIT must block past its own entry's commit until the gate opens");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn, lifecycle_committed(),
"committed_lsn must reach noop_lsn once the gate-covering ACK arrives");
lifecycle_cluster_teardown();
printf("test_promotion_appends_noop_and_arms_own_term_gate: PASS\n");
#else
printf("test_promotion_appends_noop_and_arms_own_term_gate: SKIP (non-test build)\n");
#endif
}
/* ── Case 6: NO THIRD PATH (gate unmet) — with the own-term gate unmet, a
* state-changing WAIT submission resolves ONLY as barrier success, -ETIMEDOUT,
* or -XCE_ERR_NOT_READY — never via any promotion-side completion (Req 2.2).
*
* Here the gate (= noop_lsn) is never satisfied (no covering ACK), so the only
* legitimate outcome is -ETIMEDOUT (the entry cannot reach majority within the
* deadline; the gate is a blocking predicate, not an admission fast-reject).
*
* RED today: promotion sets no gate, so a WAIT on the no-op's own entry can be
* "confirmed" by an ACK covering only that entry — a confirmation the
* own-term rule must withhold until majority of the new-term no-op. The
* carrier: with the contract gate at noop_lsn, an ACK covering ONLY a lower
* entry must leave the WAIT pending → -ETIMEDOUT; today (gate 0) the same ACK
* confirms early (rc == 0). */
static void test_pre_own_term_wait_has_no_third_path(void)
{
#ifdef XCE_TESTING
const uint64_t noop_lsn = 3;
/* appended_lsn = noop_lsn - 1 lands the no-op at lsn = noop_lsn (see
* case 5 comment); entries 1..noop_lsn-1 sit strictly below the gate. */
(void)lifecycle_cluster_setup("nothird", /*n_peers=*/3, /*n_standbys=*/2,
/*appended_lsn=*/noop_lsn - 1, /*n_vcpus=*/8);
for (uint64_t lsn = 1; lsn < noop_lsn; lsn++)
(void)commit_submit(&g_lc.commit, lsn, COMMIT_MODE_RETAIN, 0);
/* Act: promotion (must arm gate = noop_lsn). */
TEST_ASSERT_EQUAL_INT(0, ha_promote_to_primary(&g_lc));
/* A WAIT on entry 1 with a short deadline; only entry 1 ever gets a
* covering ACK — the gate (noop_lsn=3) is never satisfied. */
static lc_wait_arg_t warg;
memset(&warg, 0, sizeof(warg));
warg.lsn = 1; warg.timeout_ms = 250; warg.rc = -9999;
atomic_store(&warg.done, 0);
pthread_t wtid;
TEST_ASSERT_EQUAL_INT(0,
pthread_create(&wtid, NULL, lifecycle_wait_thread, &warg));
lifecycle_standby_ack(0, 1); /* entry 1 committed; gate (3) unmet */
pthread_join(wtid, NULL);
/* Assert: the ONLY admissible outcomes are barrier success (0) on a real
* gate-covering majority [not provided here], -ETIMEDOUT, or NOT_READY.
* With the gate unmet and no covering ACK, the contract outcome is
* -ETIMEDOUT. RED today: writable_lsn==0 admits early success (rc==0),
* which is the forbidden "third path" — a confirmation without the
* own-term entry's majority. */
TEST_ASSERT_TRUE_MESSAGE(
warg.rc == -ETIMEDOUT || warg.rc == -XCE_ERR_NOT_READY,
"pre-own-term WAIT must resolve only as -ETIMEDOUT or -XCE_ERR_NOT_READY "
"while the own-term gate is unmet (Req 2.2); an unset gate confirms it "
"early (rc==0) — the forbidden third path — RED");
TEST_ASSERT_EQUAL_INT_MESSAGE(-ETIMEDOUT, warg.rc,
"gate-unmet WAIT with no covering majority must time out (Req 2.2, 2.3)");
TEST_ASSERT_TRUE_MESSAGE(warg.elapsed_ms >= 200,
"an unmet own-term gate must BLOCK to the deadline, not fast-reject");
lifecycle_cluster_teardown();
printf("test_pre_own_term_wait_has_no_third_path: PASS\n");
#else
printf("test_pre_own_term_wait_has_no_third_path: SKIP (non-test build)\n");
#endif
}
/* ── Case 7: STANDALONE (n_peers=1) — promotion's no-op self-commits at
* registration, so the own-term gate opens immediately and a WAIT returns 0
* as before (Req 2.1, umbrella 6.3 / 7.5: standalone latency unchanged).
*
* Status note: REGRESSION-PIN. On the current tree there is no gate
* (writable_lsn stays 0), and a single-member WAIT already returns 0
* immediately (self is its own majority). So this case is EXPECTED GREEN
* today; it guards the contract that arming the gate in the n_peers=1 path
* (task 2.2) must not regress standalone promotion into a blocking wait.
* The writable_lsn-field assertion below WILL flip from already-satisfied
* behavior (WAIT==0) to additionally pinning writable_lsn==noop_lsn once 2.2
* lands; today writable_lsn==0 makes that one field assertion RED while the
* behavioral WAIT==0 stays GREEN — labelled inline. */
static void test_standalone_promotion_gate_opens_immediately(void)
{
#ifdef XCE_TESTING
const uint64_t noop_lsn = 1; /* standalone: noop is the first entry */
/* appended_lsn = 0 → current_lsn primed to 1 → the no-op lands at lsn=1
* (= noop_lsn). No entries exist below the no-op, so there is no
* pre-existing RETAIN submit. */
(void)lifecycle_cluster_setup("solo", /*n_peers=*/1, /*n_standbys=*/0,
/*appended_lsn=*/0, /*n_vcpus=*/8);
/* Act */
TEST_ASSERT_EQUAL_INT(0, ha_promote_to_primary(&g_lc));
/* Behavioral (EXPECTED GREEN today and after 2.2): single-member WAIT on
* the no-op entry returns 0 without blocking — self is the majority and
* the gate (if set) is self-satisfied at registration. */
uint64_t t0 = lifecycle_mono_ms();
int rc = commit_submit(&g_lc.commit, noop_lsn, COMMIT_MODE_WAIT, 5000);
uint64_t elapsed = lifecycle_mono_ms() - t0;
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"standalone WAIT on the own-term entry must return 0 (self-quorum)");
TEST_ASSERT_TRUE_MESSAGE(elapsed < 1000,
"standalone promotion must not block on the own-term gate (6.3/7.5)");
/* Contract field (RED today: writable_lsn==0; GREEN after task 2.2 sets
* the gate to the self-committed no-op LSN). */
TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn, g_lc.commit.writable_lsn,
"standalone promotion must still arm the own-term gate at noop_lsn "
"(Req 2.1); current promotion sets no gate — RED for this field");
lifecycle_cluster_teardown();
printf("test_standalone_promotion_gate_opens_immediately: PASS\n");
#else
printf("test_standalone_promotion_gate_opens_immediately: SKIP (non-test build)\n");
#endif
}
/* ── Case 8: LEGACY/RAFT PROMOTION SYMMETRY (ha-single-leader-role task 3.1,
* Req 1.1, 1.5; design §Legacy Path Symmetry). Both entry points to PRIMARY
* must produce IDENTICAL observable state so test-driven and raft-driven
* promotion cannot diverge:
*
* (1) raft road — raft_cb_become_leader_for_test (the XCE_TESTING shim, which
* forces ha->raft_c == NULL): only the c-independent state is observable
* on this seam (role == CTRL_PRIMARY). The hook/gate tail is c!=NULL-
* guarded and skipped here — that half of the contract is pinned on the
* real-cluster road below (matching cases 3/4/5/7, which drive the SAME
* shared helpers ha_register_persist_hooks / ha_arm_own_term_gate).
*
* (2) legacy road — ha_elect_evaluate_and_signal on the real g_lc cluster
* (ctx->c = &g_lc, single-node quorum win): drives the full shared
* sequence. Must land role == CTRL_PRIMARY, hooks_registered == 1,
* writable_lsn == noop_lsn (own-term gate armed), and arm NO delaymgr
* watchdog — byte-for-byte the state ha_promote_to_primary produces.
*
* Pin: the legacy real-cluster path produces role/hook/gate/no-watchdog state
* identical to the raft road's role and to ha_promote_to_primary's full state.
* RED before task 3.1: the legacy branch stored CTRL_PRIMARY_COMMITTING, reset
* repl_ack_count, and armed the committing watchdog — diverging from the raft
* road on every axis. */
static void test_legacy_and_raft_promotion_are_symmetric(void)
{
#ifdef XCE_TESTING
/* ── Road 1: raft shim (c == NULL) — role contract only ──────────────── */
ha_state_t ha_raft;
ha_config_t cfg = lifecycle_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha_raft, &cfg, 0));
ha_raft.phase2_done = 1;
atomic_store_explicit(&ha_raft.role, (int32_t)CTRL_ELECTING,
memory_order_release);
raft_cb_become_leader_for_test(&ha_raft, /*term=*/7);
ctrl_role_t raft_role =
(ctrl_role_t)atomic_load_explicit(&ha_raft.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, (int)raft_role,
"raft road must land DIRECTLY in CTRL_PRIMARY (Req 1.1)");
ha_fini(&ha_raft);
/* ── Road 2: legacy ha_elect_evaluate_and_signal on a real cluster ───── */
/* persist_wal_async appends at current_lsn then post-increments, and
* ha_arm_own_term_gate derives noop_lsn = current_lsn - 1; the fixture
* primes current_lsn = appended_lsn + 1, so passing appended_lsn =
* noop_lsn - 1 lands the own-term no-op at exactly lsn = noop_lsn
* (same idiom as case 5). */
const uint64_t noop_lsn = 9;
(void)lifecycle_cluster_setup("sym", /*n_peers=*/1,
/*n_standbys=*/0,
/*appended_lsn=*/noop_lsn - 1,
/*n_vcpus=*/8);
int before = delaymgr_task_count(&g_lc.delaymgr);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_lc.ha.hooks_registered,
"fixture precondition: hooks must be unregistered before promotion");
/* Single-node quorum win: n_peers==0 shortcircuit (votes[] empty),
* votes_granted=1 satisfies quorum=1. ctx->c = &g_lc so the shared
* promotion sequence runs against the real cluster. */
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx));
TEST_ASSERT_NOT_NULL_MESSAGE(ctx, "ctx calloc failed");
ctx->n_peers = 0;
ctx->votes_granted = 1;
ctx->n_replied = 0;
ctx->result = -1;
ctx->done_posted = 0;
ctx->c = &g_lc;
pthread_mutex_init(&ctx->lock, NULL);
sem_init(&ctx->done, 0, 0);
ctrl_role_t legacy_result = ha_elect_evaluate_and_signal(&g_lc.ha, ctx);
/* Symmetry axis 1: the legacy road reaches PRIMARY (matches the raft road). */
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, (int)legacy_result,
"legacy evaluate must RESULT in CTRL_PRIMARY (Req 1.1)");
ctrl_role_t legacy_role =
(ctrl_role_t)atomic_load_explicit(&g_lc.ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)raft_role, (int)legacy_role,
"legacy and raft promotion must land in the SAME role (CTRL_PRIMARY)");
/* Symmetry axis 2: persist hooks registered by the shared sequence. */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_lc.ha.hooks_registered,
"legacy road must register persist hooks via the shared sequence (Req 2.1)");
/* Symmetry axis 3: own-term gate armed at the no-op LSN. */
TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn, g_lc.commit.writable_lsn,
"legacy road must arm the own-term gate at noop_lsn (Req 1.4, 2.1) — "
"identical to the raft road's shared tail");
/* Symmetry axis 4: NO committing-timeout watchdog armed (Req 1.5). */
int after = delaymgr_task_count(&g_lc.delaymgr);
TEST_ASSERT_EQUAL_INT_MESSAGE(before, after,
"legacy road must arm NO delaymgr watchdog (Req 1.5) — symmetric with "
"the raft road; the committing-timeout arm is deleted from this path");
pthread_mutex_destroy(&ctx->lock);
sem_destroy(&ctx->done);
free(ctx);
lifecycle_cluster_teardown();
printf("test_legacy_and_raft_promotion_are_symmetric: PASS "
"(role=%d both roads; hooks=1; writable_lsn=%" PRIu64
"; watchdog tasks before=%d after=%d)\n",
(int)legacy_role, g_lc.commit.writable_lsn, before, after);
#else
printf("test_legacy_and_raft_promotion_are_symmetric: SKIP (non-test build)\n");
#endif
}
/* ------------------------------------------------------------------ */
/* Entry point (registered in controller_test.c / meson.build) */
/* ------------------------------------------------------------------ */
void test_ha_role_lifecycle(void)
{
/* Task 1.1 — WAL_NOOP no-op entry (Req 2.1) */
RUN_TEST(test_noop_append_is_header_only_and_broadcast);
RUN_TEST(test_noop_apply_advances_lsn_no_entity_effect);
RUN_TEST(test_noop_boot_replay_clean_no_effect);
RUN_TEST(test_noop_classify_is_retain);
/* Task 2.1 — DIRECT promotion + own-term gate (Reqs 1.1,1.3,1.4,1.5,2.1,2.2,2.3) */
RUN_TEST(test_promotion_lands_directly_in_primary); /* case 1 */
RUN_TEST(test_promotion_no_redirect_from_live_primary); /* case 2 */
RUN_TEST(test_promotion_registers_persist_hooks); /* case 3 */
RUN_TEST(test_promotion_arms_no_delaymgr_watchdog); /* case 4 */
RUN_TEST(test_promotion_appends_noop_and_arms_own_term_gate); /* case 5 */
RUN_TEST(test_pre_own_term_wait_has_no_third_path); /* case 6 */
RUN_TEST(test_standalone_promotion_gate_opens_immediately); /* case 7 */
/* Task 3.1 — legacy/raft promotion symmetry (Req 1.1, 1.5) */
RUN_TEST(test_legacy_and_raft_promotion_are_symmetric); /* case 8 */
}
Reference in a new issue View git blame Copy permalink