From b7a69afa0b90268429fe482c3b1e1f9f13d6f38c Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Sat, 13 Jun 2026 08:31:42 +0000 Subject: [PATCH] test(ha-single-leader-role): RED lifecycle tests for direct promotion + own-term gate (task 2.1) --- .kiro/specs/ha-single-leader-role/tasks.md | 2 +- .../testsuite/test_ha_role_lifecycle.c | 597 ++++++++++++++++++ 2 files changed, 598 insertions(+), 1 deletion(-) diff --git a/.kiro/specs/ha-single-leader-role/tasks.md b/.kiro/specs/ha-single-leader-role/tasks.md index 2717f970..72c23c63 100644 --- a/.kiro/specs/ha-single-leader-role/tasks.md +++ b/.kiro/specs/ha-single-leader-role/tasks.md @@ -18,7 +18,7 @@ - _Requirements: 2.1_ - [ ] 2. Promotion rewrite: direct PRIMARY + own-term gate -- [ ] 2.1 Write RED lifecycle tests for direct promotion and the own-term gate +- [x] 2.1 Write RED lifecycle tests for direct promotion and the own-term gate - Tests in `test_ha_role_lifecycle.c`: driving the become-leader path yields `ha->role == CTRL_PRIMARY` directly (no intermediate value observable; `ha_is_primary()` true immediately); `ha_should_redirect()` returns 0 the instant promotion returns, before any standby ACK; persistence hooks are registered by promotion (hooksys observes the three persist hook fns; `hooks_registered == 1`); promotion arms NO delaymgr watchdog (delaymgr task-count seam unchanged across promotion); with a real engine (n_peers=3, stub registry): promotion appends WAL_NOOP at lsn=N, registers it RETAIN, and `commit_set_writable_lsn(N)` is in effect — a `commit_submit(WAIT)` for entry N+1 does NOT complete while ACKs cover N+1 but not N, and completes once `committed_lsn >= N`; with the gate unmet a WAIT submission resolves only as barrier success, `-ETIMEDOUT`, or `-XCE_ERR_NOT_READY` — never via any promotion-side path; n_peers=1: gate opens immediately at promotion (noop self-committed), WAIT returns 0 as before - Observable completion: new cases FAIL (red) against the current COMMITTING-entering `raft_cb_become_leader` - _Requirements: 1.1, 1.3, 1.4, 1.5, 2.1, 2.2, 2.3_ diff --git a/src/controller/testsuite/test_ha_role_lifecycle.c b/src/controller/testsuite/test_ha_role_lifecycle.c index 049c1d5c..57ba8f81 100644 --- a/src/controller/testsuite/test_ha_role_lifecycle.c +++ b/src/controller/testsuite/test_ha_role_lifecycle.c @@ -27,12 +27,16 @@ #include #include #include +#include #include +#include +#include #include #include #include #include #include +#include #include #include @@ -40,6 +44,16 @@ #include "config.h" #include "persist.h" +/* Task 2.1 (RED): direct-promotion + own-term-gate lifecycle contract. + * These headers pull in the promotion seam (ha_*), the commit engine + its + * own-term gate (commit_*), the stub standby registry (repl_*), and the + * full cluster_t layout so a partial-but-real cluster can be promoted. */ +#include "commit.h" /* commit_engine_t, commit_submit, commit_set_writable_lsn */ +#include "ha.h" /* ha_promote_to_primary, ha_is_primary, ha_should_redirect */ +#include "repl.h" /* repl_init / repl_add_standby / repl_set_commit_engine */ +#include "xce/cluster_def.h" /* full cluster_t layout (c.commit / c.persist / c.ha / ...) */ +#include "xce/errnox.h" /* XCE_ERR_NOT_READY */ + /* The value WAL_NOOP will take in wal_entry_type (task 1.2). Using the * literal keeps THIS file compiling before the enum value exists; task 1.2 * replaces these with the WAL_NOOP symbol once it lands. */ @@ -426,6 +440,580 @@ static void test_noop_classify_is_retain(void) printf("test_noop_classify_is_retain: PASS\n"); } +/* ══════════════════════════════════════════════════════════════════════════ + * Task 2.1 (TDD RED) — DIRECT promotion + the OWN-TERM GATE. + * + * Contract (design §Promotion Rewrite, Reqs 1.1, 1.3, 1.4, 1.5, 2.1, 2.2, 2.3): + * every road to PRIMARY lands the node DIRECTLY in CTRL_PRIMARY — no + * CTRL_PRIMARY_COMMITTING ever observable — registers the persist hooks, + * arms NO wall-clock watchdog, and (production, c != NULL) appends a WAL_NOOP + * of the current term then sets the commit engine's own-term gate + * (commit_set_writable_lsn) to that no-op's LSN so confirmation of any + * operation is withheld until committed_lsn >= noop_lsn. + * + * SEAMS (study notes; no new production seam is introduced — RED-only): + * - Cases 1-3 drive the raft become-leader callback via its existing + * XCE_TESTING shim `raft_cb_become_leader_for_test(ha, term)` (ha.c:3121), + * which calls `raft_cb_become_leader` with `ha->raft_c = NULL`. This is + * THE seam test_ha_evaluate_committing.c / test_ha_connection_reuse.c use. + * RED today: that path stores CTRL_PRIMARY_COMMITTING (ha.c:514), so the + * direct-PRIMARY / is_primary / no-redirect / hooks-registered assertions + * fail. (The `c == NULL` shim also means the watchdog-arm + noop/gate + * blocks — both under `if (c)` — are skipped, so they are NOT observable + * through THIS seam; see cases 4-7 for the real-cluster seam.) + * - Cases 4-7 drive the SHARED promotion sequence `ha_promote_to_primary(c)` + * (ha.c:1717, public in ha.h) against a partial-but-real cluster_t: a real + * tmpdir persist_t, a wired commit_engine_t (n_peers), a stub standby + * registry (repl_add_standby + repl_set_commit_engine), and a delaymgr + * left un-started. Per design §Promotion Rewrite, ha_promote_to_primary + * becomes the canonical "every road to PRIMARY arms the gate" entry point + * (task 2.2 adds the noop+gate tail there + in the raft callback). TODAY + * ha_promote_to_primary does ha_do_promote + hook-register + heartbeat and + * NEVER touches c->commit, so the own-term gate stays unset (writable_lsn + * == 0) — that is the RED carrier for cases 5-6. + * + * GATE OBSERVABILITY (cerebrum + commit.c:166-171): the engine's WAIT + * completion predicate is `committed_lsn >= lsn && (writable_lsn == 0 || + * committed_lsn >= writable_lsn)`. writable_lsn is a public commit_engine_t + * field; commit_set_writable_lsn(ce, N) installs the gate. With the gate at N + * a WAIT on an entry M < N blocks past M's own commit, completing only when + * committed_lsn reaches N (test_commit_engine.c test 15 shape). TODAY + * promotion leaves writable_lsn == 0, so that same WAIT completes the instant + * M commits — the behavioral RED carrier. + * + * DELAYMGR TASK-COUNT SEAM (case 4): delay_task_ms enqueues into + * mgr->tasks (an ordq_t) regardless of running state (delay.c) — count the + * `tasks.head` singly-linked chain directly (ordq has no public count API). + * ════════════════════════════════════════════════════════════════════════ */ + +#ifdef XCE_TESTING +/* The raft become-leader callback shim (defined in ha.c under XCE_TESTING). */ +extern void raft_cb_become_leader_for_test(ha_state_t *ha, uint64_t term); +#endif + +/* Minimal ha_config_t accepted by ha_init (mirrors test_ha_role.c). */ +static ha_config_t lifecycle_ha_cfg(void) +{ + ha_config_t cfg = { + .heartbeat_interval_ms = 3000, + .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, + .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + .committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS, + }; + return cfg; +} + +/* Count pending delaymgr tasks by walking the ordq head chain directly. + * Read-only: takes no lock (single-threaded test context, delaymgr never + * started). This is the "delaymgr task-count seam" the contract pins. */ +static int delaymgr_task_count(const delaymgr_t *mgr) +{ + int count = 0; + const ordq_item_t *it = mgr->tasks.head; + while (it != NULL) { + count++; + it = it->next; + } + return count; +} + +/* ── Case 1: DIRECT PRIMARY — become-leader lands in CTRL_PRIMARY, not a + * transitional state; ha_is_primary() is true immediately on return (Req 1.1, + * 1.4). RED today: raft_cb_become_leader stores CTRL_PRIMARY_COMMITTING. */ +static void test_promotion_lands_directly_in_primary(void) +{ +#ifdef XCE_TESTING + ha_state_t ha; + ha_config_t cfg = lifecycle_ha_cfg(); + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); + ha.phase2_done = 1; /* skip phase2 (no real cluster behind the shim) */ + atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release); + + /* Act: drive the raft become-leader path (shim sets ha.raft_c = NULL) */ + raft_cb_become_leader_for_test(&ha, /*term=*/1); + + /* Assert: role is CTRL_PRIMARY directly — no transitional value */ + ctrl_role_t role = + (ctrl_role_t)atomic_load_explicit(&ha.role, memory_order_acquire); + TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, (int)role, + "winning election must land DIRECTLY in CTRL_PRIMARY (Req 1.1); " + "current code stores CTRL_PRIMARY_COMMITTING — RED"); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha_is_primary(&ha), + "ha_is_primary() must be true the instant promotion returns (Req 1.4)"); + + ha_fini(&ha); + printf("test_promotion_lands_directly_in_primary: PASS\n"); +#else + printf("test_promotion_lands_directly_in_primary: SKIP (non-test build)\n"); +#endif +} + +/* ── Case 2: NO REDIRECT — ha_should_redirect() returns 0 the instant + * promotion returns, before any standby ACK (Req 1.3). RED today: in + * CTRL_PRIMARY_COMMITTING ha_should_redirect can return 1 (a live election + * winner pointing clients at a primary that "does not exist yet"). */ +static void test_promotion_no_redirect_from_live_primary(void) +{ +#ifdef XCE_TESTING + ha_state_t ha; + ha_config_t cfg = lifecycle_ha_cfg(); + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); + ha.phase2_done = 1; + atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release); + + /* A stale primary hint is present (would trigger redirect for any + * non-PRIMARY role): host+port set as if learned from a prior leader. */ + strncpy(ha.primary_host, "10.0.0.9", sizeof(ha.primary_host) - 1); + ha.primary_port = 9991; + + /* Act */ + raft_cb_become_leader_for_test(&ha, /*term=*/1); + + /* Assert: a live primary never redirects (Req 1.3). In CTRL_PRIMARY the + * stale hint is ignored; in CTRL_PRIMARY_COMMITTING (today) the hint makes + * ha_should_redirect return 1 → RED. */ + TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha_should_redirect(&ha), + "a freshly promoted primary must NOT redirect clients (Req 1.3); " + "current COMMITTING role + a primary hint yields redirect=1 — RED"); + + ha_fini(&ha); + printf("test_promotion_no_redirect_from_live_primary: PASS\n"); +#else + printf("test_promotion_no_redirect_from_live_primary: SKIP (non-test build)\n"); +#endif +} + +/* Case 3 (HOOKS REGISTERED BY THE SHARED PROMOTION SEQUENCE) is defined just + * below the real-cluster fixture (lifecycle_cluster_setup / g_lc) it shares + * with cases 4-7, since C requires those helpers to be declared before use. + * It is still RUN as case 3 (see test_ha_role_lifecycle below). */ + +/* ------------------------------------------------------------------ */ +/* Real-cluster fixture for cases 3-7: a partial-but-real cluster_t */ +/* whose persist (real tmpdir), commit engine (wired), stub standby */ +/* registry and (un-started) delaymgr are the surfaces promotion drives.*/ +/* ------------------------------------------------------------------ */ + +/* Statics survive Unity's longjmp on assertion failure (test_repl_majority + * pattern); cluster_t is large and must not live on the test stack. */ +static cluster_t g_lc; +static int g_lc_eps[MAX_STANDBYS]; + +/* Build g_lc with: real persist over a fresh tmpdir, current_lsn primed so a + * promotion no-op would land at lsn = appended_lsn (= current_lsn - 1); + * n_standbys dummy standbys in the registry; commit engine wired to the + * registry (n_peers, n_vcpus); delaymgr initialised but NOT started; role + * forced to CTRL_ELECTING. Returns the lsn the next appended entry takes. */ +static uint64_t lifecycle_cluster_setup(const char *suffix, int n_peers, + int n_standbys, uint64_t appended_lsn, + int n_vcpus) +{ + int i; + char name[32]; + char tmpdir[256]; + + memset(&g_lc, 0, sizeof(g_lc)); + + /* HA state */ + ha_config_t cfg = lifecycle_ha_cfg(); + TEST_ASSERT_EQUAL_INT(0, ha_init(&g_lc.ha, &cfg, 0)); + g_lc.ha.phase2_done = 1; /* no phase2 wiring in this fixture */ + g_lc.ha.n_peers = n_peers; + atomic_store_explicit(&g_lc.ha.role, (int32_t)CTRL_ELECTING, + memory_order_release); + + /* Real persist over a private tmpdir (the no-op append target). */ + persist_config_t pcfg = persist_config_defaults(); + pcfg.wal_sync_ms = 5000; /* slow background fsync — no test races */ + snprintf(tmpdir, sizeof(tmpdir), "/tmp/xce_slr_%s_XXXXXX", suffix); + TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); + persist_set_workdir(&g_lc.persist, tmpdir, "ctrl-test"); + TEST_ASSERT_EQUAL_INT(0, persist_init(&g_lc.persist, &pcfg, NULL)); + TEST_ASSERT_EQUAL_INT(0, persist_start(&g_lc.persist)); + + /* Prime the appended watermark: current_lsn = appended_lsn + 1 means the + * highest appended entry is `appended_lsn` and the next append takes it. */ + pthread_mutex_lock(&g_lc.persist.lock); + g_lc.persist.current_lsn = appended_lsn + 1; + pthread_mutex_unlock(&g_lc.persist.lock); + atomic_store(&g_lc.persist.committed_lsn, 0); + + /* Stub standby registry + wired engine (commit derives from watermarks). */ + for (i = 0; i < n_standbys; i++) { + snprintf(name, sizeof(name), "lcsb%d", i); + TEST_ASSERT_TRUE_MESSAGE( + repl_add_standby(&g_lc.persist.repl, &g_lc_eps[i], name, + 7400 + (uint32_t)i) >= 0, + "repl_add_standby must return a slot index >= 0"); + } + TEST_ASSERT_EQUAL_INT(0, + commit_engine_init(&g_lc.commit, &g_lc.persist, n_peers, n_vcpus)); + repl_set_commit_engine(&g_lc.persist.repl, &g_lc.commit); + + /* delaymgr initialised but deliberately NOT started: promotion's heartbeat + * arm is skipped (ha_start_primary_heartbeat guards on delaymgr.running), + * so any task that appears in tasks.head was armed by promotion itself. */ + TEST_ASSERT_EQUAL_INT(0, delaymgr_init(&g_lc.delaymgr, 100, NULL)); + + return appended_lsn; /* lsn a promotion no-op would take (current_lsn - 1) */ +} + +static void lifecycle_cluster_teardown(void) +{ + commit_engine_fini(&g_lc.commit); + delaymgr_fini(&g_lc.delaymgr); + persist_stop(&g_lc.persist); + persist_fini(&g_lc.persist); + repl_fini(&g_lc.persist.repl); + ha_fini(&g_lc.ha); + /* tmpdir left on disk — /tmp is ephemeral */ +} + +/* Drive a standby ACK watermark into the registry + engine (production + * repl_on_ack shape: store acked_lsn, then feed commit_on_ack). */ +static void lifecycle_standby_ack(int idx, uint64_t acked_lsn) +{ + atomic_store(&g_lc.persist.repl.standbys[idx].acked_lsn, acked_lsn); + commit_on_ack(&g_lc.commit, idx, acked_lsn); +} + +static uint64_t lifecycle_committed(void) +{ + return atomic_load(&g_lc.persist.committed_lsn); +} + +static uint64_t lifecycle_mono_ms(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_sec * 1000u + (uint64_t)ts.tv_nsec / 1000000u; +} + +static void lifecycle_sleep_ms(uint32_t ms) +{ + struct timespec ts; + ts.tv_sec = ms / 1000u; + ts.tv_nsec = (long)(ms % 1000u) * 1000000L; + nanosleep(&ts, NULL); +} + +/* WAIT-on-an-entry thread context + body (test_commit_engine.c pattern). */ +typedef struct { + uint64_t lsn; + uint32_t timeout_ms; + int rc; + uint64_t elapsed_ms; + atomic_int done; +} lc_wait_arg_t; + +static void *lifecycle_wait_thread(void *p) +{ + lc_wait_arg_t *a = (lc_wait_arg_t *)p; + uint64_t t0 = lifecycle_mono_ms(); + a->rc = commit_submit(&g_lc.commit, a->lsn, COMMIT_MODE_WAIT, + a->timeout_ms); + a->elapsed_ms = lifecycle_mono_ms() - t0; + atomic_store(&a->done, 1); + return NULL; +} + +/* ── Case 3: HOOKS REGISTERED BY THE SHARED PROMOTION SEQUENCE — the + * canonical promotion entry point ha_promote_to_primary(c) (ha.c:1717) must + * register the three persist hooks so an idle cluster's new primary is + * writable; observe via c->ha.hooks_registered transitioning 0 → 1 (Req 2.1). + * + * SHARED-SEQUENCE CONTRACT PIN (GREEN now, MUST STAY GREEN — not a behavioral + * RED carrier). After task 2.2 BOTH roads to PRIMARY (the raft become-leader + * callback and ha_repl_ack_received's quorum path) route through this same + * ha_promote_to_primary sequence, which task 2.2 augments with the noop+gate + * tail while KEEPING hook registration. This case is therefore a CONTRACT/ + * REGRESSION pin proving the shared promotion sequence registers the persist + * hooks; the behavioral RED signal for 2.2 is carried by cases 1, 2, 5, 6. + * + * It uses the SAME real-cluster fixture cases 4-7 use (lifecycle_cluster_setup + * / g_lc), which is why it is defined here, below that fixture. This is the + * fix for the prior c==NULL shim defect: the raft become-leader shim forces + * ha->raft_c == NULL (ha.c:3121), which routes AROUND the c-dependent + * registration block, so it could never set the flag on that seam — pinning + * it there left an un-turnable RED. ha_promote_to_primary is the real shared + * seam and registers hooks today (ha.c:1722-1730). + * + * Why this is GREEN today: ha_promote_to_primary already registers the three + * persist hooks and sets hooks_registered = 1 (ha.c:1729). The g_lc fixture's + * hooksys is zeroed (memset), so xce_add_hook returns -EINVAL harmlessly + * (typ >= maxtypes==0, hook.c:41) WITHOUT storing the fns — but the flag still + * transitions 0 → 1 (ha.c:1714-1715 documents exactly this). The + * hooks_registered 0 → 1 transition is therefore the clean, observable + * contract pin on the zeroed-hooksys fixture; the three fns are NOT observable + * in g_lc.hooksys here (the zeroed registry rejects the add), so we pin the + * flag transition per the task wording rather than the slot contents. */ +static void test_promotion_registers_persist_hooks(void) +{ +#ifdef XCE_TESTING + /* Arrange: a partial-but-real cluster (same fixture as cases 4-7). + * n_peers value is not material to the hook contract. */ + (void)lifecycle_cluster_setup("hooks", /*n_peers=*/3, /*n_standbys=*/2, + /*appended_lsn=*/5, /*n_vcpus=*/8); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_lc.ha.hooks_registered, + "fixture precondition: hooks must be unregistered before promotion"); + + /* Act: drive the SHARED promotion sequence both roads route through (2.2). */ + TEST_ASSERT_EQUAL_INT(0, ha_promote_to_primary(&g_lc)); + + /* Assert: the shared promotion sequence registered the persist hooks — + * hooks_registered transitioned 0 → 1 (Req 2.1). Contract/regression pin: + * green today (ha.c:1729) and must remain green after task 2.2 augments the + * sequence with the noop+gate tail. */ + TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_lc.ha.hooks_registered, + "the shared promotion sequence (ha_promote_to_primary) must register " + "the persist hooks (hooks_registered 0→1, Req 2.1) — contract pin, " + "must stay green through task 2.2"); + + lifecycle_cluster_teardown(); + printf("test_promotion_registers_persist_hooks: PASS\n"); +#else + printf("test_promotion_registers_persist_hooks: SKIP (non-test build)\n"); +#endif +} + +/* ── Case 4: NO WATCHDOG — promotion arms no delaymgr task (Req 1.5, 2.3). + * + * The shared promotion sequence must never arm a wall-clock watchdog (the + * deleted committing_timeout_fn). Asserted via the delaymgr task-count seam: + * the count is unchanged across ha_promote_to_primary. + * + * Status note (RED vs already-GREEN): the force-promotion watchdog is armed + * ONLY inside raft_cb_become_leader's `if (c)` block (ha.c:611-619), and that + * block has NO real-cluster unit seam (raft_cb_become_leader is static; its + * only shim forces c == NULL). ha_promote_to_primary — the road this case + * exercises — never armed a watchdog, so this case is ALREADY GREEN today and + * stands as a regression-pin guaranteeing task 2.2's shared sequence does not + * introduce one. The watchdog REMOVAL from raft_cb_become_leader is covered + * at integration scope (task 4.x deletes committing_timeout_fn and the + * test_ha_evaluate_committing committing-timeout cases). */ +static void test_promotion_arms_no_delaymgr_watchdog(void) +{ +#ifdef XCE_TESTING + /* Arrange: 3-member real cluster, delaymgr started so an armed watchdog + * (if any) would enqueue; heartbeat stays skipped because the static + * one-shot arm + the role no-op keep it from stacking a task here. */ + (void)lifecycle_cluster_setup("nowd", /*n_peers=*/3, /*n_standbys=*/2, + /*appended_lsn=*/5, /*n_vcpus=*/8); + int before = delaymgr_task_count(&g_lc.delaymgr); + + /* Act: drive the shared promotion sequence */ + TEST_ASSERT_EQUAL_INT(0, ha_promote_to_primary(&g_lc)); + + /* Assert: no wall-clock task was armed by promotion (Req 1.5, 2.3) */ + int after = delaymgr_task_count(&g_lc.delaymgr); + TEST_ASSERT_EQUAL_INT_MESSAGE(before, after, + "promotion must arm NO delaymgr watchdog (Req 1.5, 2.3) — " + "the delaymgr task count must be unchanged across promotion"); + /* And the road still landed in CTRL_PRIMARY directly (Req 1.1) */ + TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, + (int)atomic_load_explicit(&g_lc.ha.role, memory_order_acquire), + "shared promotion sequence must reach CTRL_PRIMARY directly"); + + lifecycle_cluster_teardown(); + printf("test_promotion_arms_no_delaymgr_watchdog: PASS " + "(delaymgr tasks before=%d after=%d)\n", before, after); +#else + printf("test_promotion_arms_no_delaymgr_watchdog: SKIP (non-test build)\n"); +#endif +} + +/* ── Case 5: OWN-TERM GATE (real engine, n_peers=3, stub registry) — + * promotion must append a WAL_NOOP at lsn=N, register it RETAIN, and set the + * commit engine gate to N (commit_set_writable_lsn(N)). The behavioral + * consequence: a WAIT for an entry M < N does NOT complete while ACKs cover M + * but not N, and completes only once committed_lsn >= N (Req 2.1). + * + * RED today: ha_promote_to_primary never touches c->commit, so writable_lsn + * stays 0 (no gate) and the WAIT on M completes the instant M commits. Two + * RED carriers are pinned: (a) the gate field writable_lsn == noop_lsn after + * promotion (today it is 0); (b) the behavioral block-until-gate. */ +static void test_promotion_appends_noop_and_arms_own_term_gate(void) +{ +#ifdef XCE_TESTING + /* Arrange: 3-member cluster. Prime so the promotion no-op lands at lsn + * N=3 (current_lsn-1). Entries 1..3 are appended into the engine so a + * waiter on entry M=1 has a target below the gate. */ + const uint64_t noop_lsn = 3; + (void)lifecycle_cluster_setup("gate", /*n_peers=*/3, /*n_standbys=*/2, + /*appended_lsn=*/noop_lsn, /*n_vcpus=*/8); + /* Register appended entries 1..3 (RETAIN = lock-free bookkeeping). */ + for (uint64_t lsn = 1; lsn <= noop_lsn; lsn++) + (void)commit_submit(&g_lc.commit, lsn, COMMIT_MODE_RETAIN, 0); + + /* Act: drive the shared promotion sequence (appends WAL_NOOP + gate). */ + TEST_ASSERT_EQUAL_INT(0, ha_promote_to_primary(&g_lc)); + + /* Assert (gate field): writable_lsn must be the no-op's LSN (Req 2.1). + * RED today: promotion does not touch the engine → writable_lsn == 0. */ + TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn, g_lc.commit.writable_lsn, + "promotion must set the own-term gate writable_lsn = noop_lsn (Req 2.1); " + "current promotion never appends a no-op nor sets the gate — RED"); + + /* Assert (WAL): a header-only WAL_NOOP must have been appended. RED + * today: nothing is appended, so current_lsn never advanced past the + * primed watermark. */ + TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn + 1, g_lc.persist.current_lsn, + "promotion must append exactly one WAL_NOOP (current_lsn advances to " + "noop_lsn+1, Req 2.1) — current promotion appends nothing — RED"); + + /* Assert (behavioral gate): a WAIT on entry M=1 < gate must NOT complete + * once entry 1 commits — it must block until committed_lsn >= noop_lsn. */ + static lc_wait_arg_t warg; + memset(&warg, 0, sizeof(warg)); + warg.lsn = 1; warg.timeout_ms = 3000; warg.rc = -9999; + atomic_store(&warg.done, 0); + + pthread_t wtid; + TEST_ASSERT_EQUAL_INT(0, + pthread_create(&wtid, NULL, lifecycle_wait_thread, &warg)); + + /* Entry 1 gains majority (self + sb0); gate (noop_lsn) still unmet. */ + lifecycle_sleep_ms(50); + lifecycle_standby_ack(0, 1); + lifecycle_sleep_ms(150); + int done_after_entry1 = atomic_load(&warg.done); + + /* Gate opens: a covering ACK advances committed_lsn to noop_lsn. */ + lifecycle_standby_ack(0, noop_lsn); + pthread_join(wtid, NULL); + + /* RED today: with writable_lsn == 0 the waiter completed at the first + * ACK (done_after_entry1 == 1). Contract: it must still be blocked. */ + TEST_ASSERT_EQUAL_INT_MESSAGE(0, done_after_entry1, + "WAIT on entry 1 must NOT complete while committed_lsn < noop_lsn " + "(own-term gate, Req 2.1); unset gate completes it early — RED"); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, warg.rc, + "WAIT must succeed once committed_lsn reaches the own-term gate"); + TEST_ASSERT_TRUE_MESSAGE(warg.elapsed_ms >= 180, + "WAIT must block past its own entry's commit until the gate opens"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn, lifecycle_committed(), + "committed_lsn must reach noop_lsn once the gate-covering ACK arrives"); + + lifecycle_cluster_teardown(); + printf("test_promotion_appends_noop_and_arms_own_term_gate: PASS\n"); +#else + printf("test_promotion_appends_noop_and_arms_own_term_gate: SKIP (non-test build)\n"); +#endif +} + +/* ── Case 6: NO THIRD PATH (gate unmet) — with the own-term gate unmet, a + * state-changing WAIT submission resolves ONLY as barrier success, -ETIMEDOUT, + * or -XCE_ERR_NOT_READY — never via any promotion-side completion (Req 2.2). + * + * Here the gate (= noop_lsn) is never satisfied (no covering ACK), so the only + * legitimate outcome is -ETIMEDOUT (the entry cannot reach majority within the + * deadline; the gate is a blocking predicate, not an admission fast-reject). + * + * RED today: promotion sets no gate, so a WAIT on the no-op's own entry can be + * "confirmed" by an ACK covering only that entry — a confirmation the + * own-term rule must withhold until majority of the new-term no-op. The + * carrier: with the contract gate at noop_lsn, an ACK covering ONLY a lower + * entry must leave the WAIT pending → -ETIMEDOUT; today (gate 0) the same ACK + * confirms early (rc == 0). */ +static void test_pre_own_term_wait_has_no_third_path(void) +{ +#ifdef XCE_TESTING + const uint64_t noop_lsn = 3; + (void)lifecycle_cluster_setup("nothird", /*n_peers=*/3, /*n_standbys=*/2, + /*appended_lsn=*/noop_lsn, /*n_vcpus=*/8); + for (uint64_t lsn = 1; lsn <= noop_lsn; lsn++) + (void)commit_submit(&g_lc.commit, lsn, COMMIT_MODE_RETAIN, 0); + + /* Act: promotion (must arm gate = noop_lsn). */ + TEST_ASSERT_EQUAL_INT(0, ha_promote_to_primary(&g_lc)); + + /* A WAIT on entry 1 with a short deadline; only entry 1 ever gets a + * covering ACK — the gate (noop_lsn=3) is never satisfied. */ + static lc_wait_arg_t warg; + memset(&warg, 0, sizeof(warg)); + warg.lsn = 1; warg.timeout_ms = 250; warg.rc = -9999; + atomic_store(&warg.done, 0); + + pthread_t wtid; + TEST_ASSERT_EQUAL_INT(0, + pthread_create(&wtid, NULL, lifecycle_wait_thread, &warg)); + lifecycle_standby_ack(0, 1); /* entry 1 committed; gate (3) unmet */ + pthread_join(wtid, NULL); + + /* Assert: the ONLY admissible outcomes are barrier success (0) on a real + * gate-covering majority [not provided here], -ETIMEDOUT, or NOT_READY. + * With the gate unmet and no covering ACK, the contract outcome is + * -ETIMEDOUT. RED today: writable_lsn==0 admits early success (rc==0), + * which is the forbidden "third path" — a confirmation without the + * own-term entry's majority. */ + TEST_ASSERT_TRUE_MESSAGE( + warg.rc == -ETIMEDOUT || warg.rc == -XCE_ERR_NOT_READY, + "pre-own-term WAIT must resolve only as -ETIMEDOUT or -XCE_ERR_NOT_READY " + "while the own-term gate is unmet (Req 2.2); an unset gate confirms it " + "early (rc==0) — the forbidden third path — RED"); + TEST_ASSERT_EQUAL_INT_MESSAGE(-ETIMEDOUT, warg.rc, + "gate-unmet WAIT with no covering majority must time out (Req 2.2, 2.3)"); + TEST_ASSERT_TRUE_MESSAGE(warg.elapsed_ms >= 200, + "an unmet own-term gate must BLOCK to the deadline, not fast-reject"); + + lifecycle_cluster_teardown(); + printf("test_pre_own_term_wait_has_no_third_path: PASS\n"); +#else + printf("test_pre_own_term_wait_has_no_third_path: SKIP (non-test build)\n"); +#endif +} + +/* ── Case 7: STANDALONE (n_peers=1) — promotion's no-op self-commits at + * registration, so the own-term gate opens immediately and a WAIT returns 0 + * as before (Req 2.1, umbrella 6.3 / 7.5: standalone latency unchanged). + * + * Status note: REGRESSION-PIN. On the current tree there is no gate + * (writable_lsn stays 0), and a single-member WAIT already returns 0 + * immediately (self is its own majority). So this case is EXPECTED GREEN + * today; it guards the contract that arming the gate in the n_peers=1 path + * (task 2.2) must not regress standalone promotion into a blocking wait. + * The writable_lsn-field assertion below WILL flip from already-satisfied + * behavior (WAIT==0) to additionally pinning writable_lsn==noop_lsn once 2.2 + * lands; today writable_lsn==0 makes that one field assertion RED while the + * behavioral WAIT==0 stays GREEN — labelled inline. */ +static void test_standalone_promotion_gate_opens_immediately(void) +{ +#ifdef XCE_TESTING + const uint64_t noop_lsn = 1; /* standalone: noop is the first entry */ + (void)lifecycle_cluster_setup("solo", /*n_peers=*/1, /*n_standbys=*/0, + /*appended_lsn=*/noop_lsn, /*n_vcpus=*/8); + /* Register the appended entry the no-op occupies. */ + (void)commit_submit(&g_lc.commit, noop_lsn, COMMIT_MODE_RETAIN, 0); + + /* Act */ + TEST_ASSERT_EQUAL_INT(0, ha_promote_to_primary(&g_lc)); + + /* Behavioral (EXPECTED GREEN today and after 2.2): single-member WAIT on + * the no-op entry returns 0 without blocking — self is the majority and + * the gate (if set) is self-satisfied at registration. */ + uint64_t t0 = lifecycle_mono_ms(); + int rc = commit_submit(&g_lc.commit, noop_lsn, COMMIT_MODE_WAIT, 5000); + uint64_t elapsed = lifecycle_mono_ms() - t0; + TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, + "standalone WAIT on the own-term entry must return 0 (self-quorum)"); + TEST_ASSERT_TRUE_MESSAGE(elapsed < 1000, + "standalone promotion must not block on the own-term gate (6.3/7.5)"); + + /* Contract field (RED today: writable_lsn==0; GREEN after task 2.2 sets + * the gate to the self-committed no-op LSN). */ + TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn, g_lc.commit.writable_lsn, + "standalone promotion must still arm the own-term gate at noop_lsn " + "(Req 2.1); current promotion sets no gate — RED for this field"); + + lifecycle_cluster_teardown(); + printf("test_standalone_promotion_gate_opens_immediately: PASS\n"); +#else + printf("test_standalone_promotion_gate_opens_immediately: SKIP (non-test build)\n"); +#endif +} + /* ------------------------------------------------------------------ */ /* Entry point (registered in controller_test.c / meson.build) */ /* ------------------------------------------------------------------ */ @@ -437,4 +1025,13 @@ void test_ha_role_lifecycle(void) RUN_TEST(test_noop_apply_advances_lsn_no_entity_effect); RUN_TEST(test_noop_boot_replay_clean_no_effect); RUN_TEST(test_noop_classify_is_retain); + + /* Task 2.1 — DIRECT promotion + own-term gate (Reqs 1.1,1.3,1.4,1.5,2.1,2.2,2.3) */ + RUN_TEST(test_promotion_lands_directly_in_primary); /* case 1 */ + RUN_TEST(test_promotion_no_redirect_from_live_primary); /* case 2 */ + RUN_TEST(test_promotion_registers_persist_hooks); /* case 3 */ + RUN_TEST(test_promotion_arms_no_delaymgr_watchdog); /* case 4 */ + RUN_TEST(test_promotion_appends_noop_and_arms_own_term_gate); /* case 5 */ + RUN_TEST(test_pre_own_term_wait_has_no_third_path); /* case 6 */ + RUN_TEST(test_standalone_promotion_gate_opens_immediately); /* case 7 */ }