#define _GNU_SOURCE /* * test_ha_role_lifecycle.c — ha-single-leader-role lifecycle contract. * * Task 1.1 (TDD RED): the WAL_NOOP no-op entry type. Later tasks (2.1, 3.1) * add direct-promotion and own-term-gate lifecycle cases to this same file. * * RED-PHASE APPROACH (b): WAL_NOOP does not exist yet (lands in task 1.2), * so referencing the symbol would fail to COMPILE. Instead these cases use * the literal value 40 (the value WAL_NOOP will take in wal_entry_type, per * design.md §"WAL_NOOP Entry") so the file COMPILES and LINKS into the * controller test binary; the assertions then fail at RUNTIME because the * type-40 behavior (explicit replay no-op, RETAIN classification) is not * implemented. This satisfies task 1.1's "build succeeds + new cases FAIL". * * The classification case (wal_classify(40,...)) cannot run inline in the * current tree: the totality guard does `assert(0)` for any unclassified * type, which would ABORT the whole test binary. It is isolated via the * fork() abort-test pattern (cerebrum: child silences stderr + _exit, parent * inspects exit status) so the RED state is a deterministic, captured * assertion in the parent rather than a binary crash. * * AAA structure throughout. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "config.h" #include "persist.h" /* Task 2.1 (RED): direct-promotion + own-term-gate lifecycle contract. * These headers pull in the promotion seam (ha_*), the commit engine + its * own-term gate (commit_*), the stub standby registry (repl_*), and the * full cluster_t layout so a partial-but-real cluster can be promoted. */ #include "commit.h" /* commit_engine_t, commit_submit, commit_set_writable_lsn */ #include "ha.h" /* ha_promote_to_primary, ha_is_primary, ha_should_redirect */ #include "repl.h" /* repl_init / repl_add_standby / repl_set_commit_engine */ #include "xce/cluster_def.h" /* full cluster_t layout (c.commit / c.persist / c.ha / ...) */ #include "xce/errnox.h" /* XCE_ERR_NOT_READY */ /* The value WAL_NOOP will take in wal_entry_type (task 1.2). Using the * literal keeps THIS file compiling before the enum value exists; task 1.2 * replaces these with the WAL_NOOP symbol once it lands. */ #define WAL_NOOP_VALUE 40u /* ------------------------------------------------------------------ */ /* Broadcast drop-hook (precedent: test_p2_integration.c:32/313, */ /* test_commit_engine.c:1133/1308; hook defined in repl.c:21). */ /* The lifecycle fixture registers fake int g_lc_eps[] endpoints via */ /* repl_add_standby; the own-term WAL_NOOP append now broadcasts via */ /* repl_broadcast → xce_wal_batch_req_raw(ep), which would dereference */ /* ep->peer past the 4-byte int (ASAN global-buffer-overflow at */ /* request.c). Installing this hook drops the wire send harmlessly. */ /* ------------------------------------------------------------------ */ #ifdef XCE_TESTING extern int (*repl_broadcast_send_hook)(void *ep, wal_catchup_entry_t **entries, int n_entries); static int lc_drop_send_hook(void *ep, wal_catchup_entry_t **entries, int n_entries) { (void)ep; (void)entries; (void)n_entries; return 0; /* pretend success, drop the data */ } #endif /* ------------------------------------------------------------------ */ /* Test-local fixtures */ /* ------------------------------------------------------------------ */ /* Spin up a real on-disk persist_t in a fresh tmpdir (no cluster wired — * apply_wal_entry's entity-restore path is skipped when p->cluster == NULL, * which is exactly the no-effect guarantee we assert). */ static persist_t* make_lifecycle_persist(const char* suffix) { persist_t* p = calloc(1, sizeof(persist_t)); if (!p) return NULL; persist_config_t cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; /* slow background fsync — no races in tests */ char tmpdir[256]; snprintf(tmpdir, sizeof(tmpdir), "/tmp/xce_noop_%s_XXXXXX", suffix); if (!mkdtemp(tmpdir)) { free(p); return NULL; } persist_set_workdir(p, tmpdir, "ctrl-test"); if (persist_init(p, &cfg, NULL) != 0) { free(p); return NULL; } if (persist_start(p) != 0) { persist_fini(p); free(p); return NULL; } return p; } static void teardown_lifecycle_persist(persist_t* p) { if (!p) return; persist_stop(p); persist_fini(p); free(p); /* tmpdir left on disk — /tmp is ephemeral */ } /* Read the most-recently-appended WAL entry header back from the active * segment file. Returns 0 and fills *out on success, negative on error. * Reads the header at (seg_offset - sizeof(hdr)) — the last entry written. */ static int read_last_wal_hdr(persist_t* p, wal_entry_hdr_t* out) { if (p->seg_offset < sizeof(wal_entry_hdr_t)) return -ENOENT; int fd = open(p->seg_path, O_RDONLY); if (fd < 0) return -errno; off_t at = (off_t)(p->seg_offset - sizeof(wal_entry_hdr_t)); ssize_t r = pread(fd, out, sizeof(*out), at); close(fd); if (r != (ssize_t)sizeof(*out)) return -EIO; return 0; } /* Evaluate wal_classify(WAL_NOOP, 0) in a forked child and report whether it * cleanly returned WAL_MODE_RETAIN. Returns 1 iff the child exited 0 (no * abort, RETAIN), 0 otherwise (abort/SIGABRT today, since WAL_NOOP is * unclassified → the totality fail-fast `assert(0)` fires). Isolating the * call protects the test binary from the abort (cerebrum fork pattern). */ static int noop_classifies_retain(void) { pid_t pid = fork(); if (pid < 0) return -1; if (pid == 0) { setenv("ASAN_OPTIONS", "detect_leaks=0", 1); int devnull = open("/dev/null", O_WRONLY); if (devnull >= 0) { dup2(devnull, STDERR_FILENO); dup2(devnull, STDOUT_FILENO); } wal_commit_mode_t mode = wal_classify(WAL_NOOP_VALUE, 0); _exit(mode == WAL_MODE_RETAIN ? 0 : 1); } int status = 0; if (waitpid(pid, &status, 0) != pid) return -1; return (WIFEXITED(status) && WEXITSTATUS(status) == 0) ? 1 : 0; } /* ------------------------------------------------------------------ */ /* Task 1.1 — WAL_NOOP no-op entry (Req 2.1) */ /* ------------------------------------------------------------------ */ /* Case 1: persist_wal_async(p, WAL_NOOP, NULL, 0) appends a header-only * entry (payload_len == 0) and broadcasts it through the production path. * * Observables: * - return 0 (append succeeded); * - the on-disk last entry has type == WAL_NOOP and payload_len == 0; * - last_replicated_crc advanced to this entry's CRC (the same field * persist_wal_async sets immediately alongside repl_broadcast, * persist.c:359 — proxy for "the entry was handed to the replicator"). * * RED today: persist_wal_async will write a type-40 entry, but the spec * requires WAL_NOOP to be appended with payload_len 0 ONLY via this path; * the binding of the value-40 type to the WAL_NOOP no-op contract does not * exist until task 1.2 wires the explicit replay/classify handling. The * payload_len/type-round-trip assertion below pins the entry shape that 1.2 * must produce; it is asserted now so 1.2 has a concrete target. */ static void test_noop_append_is_header_only_and_broadcast(void) { /* Arrange */ persist_t* p = make_lifecycle_persist("append"); TEST_ASSERT_NOT_NULL(p); uint64_t lsn_before = p->current_lsn; uint32_t crc_before = p->last_replicated_crc; /* Act: append a no-op (header only, NULL payload, len 0) */ int rc = persist_wal_async(p, WAL_NOOP_VALUE, NULL, 0); /* Assert: append succeeded and consumed exactly one LSN */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "WAL_NOOP append must succeed via persist_wal_async"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before + 1, p->current_lsn, "WAL_NOOP append must advance current_lsn by exactly one"); /* Assert: the on-disk entry is header-only of the WAL_NOOP type */ wal_entry_hdr_t hdr; memset(&hdr, 0, sizeof(hdr)); TEST_ASSERT_EQUAL_INT_MESSAGE(0, read_last_wal_hdr(p, &hdr), "must be able to read the appended WAL_NOOP header back from disk"); TEST_ASSERT_EQUAL_UINT32_MESSAGE(WAL_NOOP_VALUE, hdr.type, "appended entry must be of type WAL_NOOP"); TEST_ASSERT_EQUAL_UINT32_MESSAGE(0u, hdr.payload_len, "WAL_NOOP must be header-only (payload_len == 0)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before, hdr.lsn, "WAL_NOOP entry LSN must be the pre-append current_lsn"); /* Assert: the entry was broadcast (last_replicated_crc moved off its * pre-append value to this entry's crc — the replicate-path observable) */ TEST_ASSERT_NOT_EQUAL_MESSAGE(crc_before, p->last_replicated_crc, "WAL_NOOP append must broadcast (last_replicated_crc advances)"); TEST_ASSERT_EQUAL_UINT32_MESSAGE(hdr.crc, p->last_replicated_crc, "broadcast CRC must match the appended WAL_NOOP entry CRC"); /* Assert: WAL_NOOP is a CLASSIFIED type. An entry appended through the * write path must be classifiable (the design lands the enum value and * its wal_classify row in the SAME change, task 1.2). Today WAL_NOOP is * unclassified, so this is RED: the classify call aborts under the * totality guard (observed via the forked-child helper). */ TEST_ASSERT_EQUAL_INT_MESSAGE(1, noop_classifies_retain(), "an appended WAL_NOOP must be a classified type (wal_classify → RETAIN)"); teardown_lifecycle_persist(p); printf("test_noop_append_is_header_only_and_broadcast: PASS\n"); } /* Case 2: apply_wal_entry on a WAL_NOOP advances current_lsn past the * entry's LSN and touches NO entity state. * * With p->cluster == NULL the entity-restore switch is skipped entirely, so * "no entity state touched" is structurally guaranteed; the contract this * pins is the LSN advance via the common path (persist.c:719-723) for a * WAL_NOOP header-only entry. * * RED today: the entry applies (type 40 < WAL_ENTRY_TYPE_MAX, so the common * path advances LSN), BUT task 1.2 must add an EXPLICIT `case WAL_NOOP: * break;` so the no-op is a recognised type rather than falling through the * `default:` arm. We additionally require the apply to return 0 AND not * emit the unknown-type WARN path — pinned here as the apply contract 1.2 * realises. The assertion that the entry is recognised (return 0 with a * header-only payload and exact LSN advance) is the RED carrier. */ static void test_noop_apply_advances_lsn_no_entity_effect(void) { /* Arrange: a persist with NO cluster — entity restore is skipped, so any * observable effect would have to come from the WAL_NOOP arm itself */ persist_t* p = make_lifecycle_persist("apply"); TEST_ASSERT_NOT_NULL(p); TEST_ASSERT_NULL_MESSAGE(p->cluster, "fixture must have no cluster wired (no-effect guarantee depends on it)"); /* Build a header-only WAL_NOOP entry at a known LSN beyond current */ uint64_t noop_lsn = p->current_lsn + 5; wal_entry_hdr_t hdr; memset(&hdr, 0, sizeof(hdr)); hdr.term = p->term; hdr.lsn = noop_lsn; hdr.timestamp = 0; hdr.type = WAL_NOOP_VALUE; hdr.payload_len = 0; hdr.crc = 0; hdr.prev_crc = 0; /* Act: apply the no-op with a NULL payload (header-only) */ int rc = apply_wal_entry(p, &hdr, NULL); /* Assert: clean apply, LSN advanced past the entry, nothing else */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "applying a WAL_NOOP must succeed"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn + 1, p->current_lsn, "applying a WAL_NOOP must advance current_lsn to entry LSN + 1"); /* No cluster → no entity state to inspect; the absence of effect is the * point. Re-applying must be idempotent on LSN (already advanced). */ int rc2 = apply_wal_entry(p, &hdr, NULL); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc2, "re-applying a WAL_NOOP must remain a clean no-op"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn + 1, p->current_lsn, "re-applying a lower/equal LSN WAL_NOOP must not regress current_lsn"); /* Assert: WAL_NOOP is a recognised, classified entry type — not an * unknown type the apply/classify paths only tolerate by fallthrough. * RED today: wal_classify(WAL_NOOP) aborts (unclassified) per the * totality guard; task 1.2 adds the explicit case + RETAIN row. */ TEST_ASSERT_EQUAL_INT_MESSAGE(1, noop_classifies_retain(), "an applied WAL_NOOP must be a classified type (wal_classify → RETAIN)"); teardown_lifecycle_persist(p); printf("test_noop_apply_advances_lsn_no_entity_effect: PASS\n"); } /* Case 3: boot replay over a segment containing a WAL_NOOP replays cleanly * with the same no-effect guarantee, and the WAL_NOOP is a RECOGNISED type. * * Fixture shape (deliberate): append ONE ordinary, already-classified WAL * entry (WAL_JOB_START, header-only) FIRST so it takes lsn 0, THEN append the * WAL_NOOP so it lands at lsn 1. This matters because replay_wal skips any * entry with `hdr.lsn <= from_lsn` (persist.c:907) and a fresh persist recovers * with from_lsn == checkpoint_lsn == 0. A WAL_NOOP written as the FIRST entry * would sit at lsn 0 and be skipped as snapshot-covered — an lsn-0/checkpoint- * boundary FIXTURE ARTIFACT unrelated to the WAL_NOOP contract (it would fail * for ANY type at lsn 0 and would NOT be fixed by task 1.2). By seating the * noop at lsn > 0 we pin the actual boot-replay contract instead. * * The leading WAL_JOB_START is side-effect-free here: apply_wal_entry returns * early at the `if (!p->cluster)` guard (persist.c:727-728) before the entity- * restore switch, so neither the ordinary entry nor the noop touches entity * state during replay — preserving the no-effect guarantee. * * Observables that PIN THE CONTRACT (these hold both today and after task 1.2, * because WAL_NOOP_VALUE 40 <= WAL_ENTRY_TYPE_MAX 100 already routes through * apply_wal_entry, which advances current_lsn): * - persist_recover returns 0 (replay clean); * - current_lsn advances STRICTLY past the noop's lsn (>= noop_lsn + 1); * - no entity side effects (NULL cluster — structurally guaranteed). * * The RED PIN is the SHARED classify carrier noop_classifies_retain(): boot * replay of a WAL_NOOP is only correct when the type is a recognised, RETAIN- * classified entry — not an unclassified type tolerated by routing alone. * Today wal_classify(40) aborts under the totality guard (carrier returns 0); * task 1.2 lands the enum + `case WAL_NOOP: break;` apply arm + the RETAIN * classify row TOGETHER, flipping the carrier to 1 and this case to green. */ static void test_noop_boot_replay_clean_no_effect(void) { char tmpdir[256]; uint64_t noop_lsn; /* Arrange: write [WAL_JOB_START (lsn 0), WAL_NOOP (lsn 1)] into a fresh * persist so the noop sits at lsn > 0 (escaping the lsn<=from_lsn skip), * capture the noop's LSN, flush, then tear down (keeping the on-disk WAL). */ { persist_t* p = calloc(1, sizeof(persist_t)); TEST_ASSERT_NOT_NULL(p); persist_config_t cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; strcpy(tmpdir, "/tmp/xce_noop_replay_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); persist_set_workdir(p, tmpdir, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(p, &cfg, NULL)); TEST_ASSERT_EQUAL_INT(0, persist_start(p)); /* Leading ORDINARY entry — already classified, header-only, harmless * on replay under a NULL cluster. Takes lsn 0 so the noop gets lsn>0. */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, persist_wal_async(p, WAL_JOB_START, NULL, 0), "leading ordinary WAL entry must append for the replay fixture"); noop_lsn = p->current_lsn; /* the LSN the noop is about to take (> 0) */ TEST_ASSERT_TRUE_MESSAGE(noop_lsn > 0, "noop must sit at lsn > 0 so replay does not skip it as " "snapshot-covered (hdr.lsn <= from_lsn guard, persist.c:907)"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, persist_wal_async(p, WAL_NOOP_VALUE, NULL, 0), "WAL_NOOP must append for the replay fixture"); TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); persist_stop(p); persist_fini(p); free(p); } /* Act: recover a FRESH persist over the same workdir (boot replay). * persist_init does NOT scan segments / set current_lsn — recovery is * explicit via persist_recover, so current_lsn starts at the calloc'd 0. */ persist_t* p2 = calloc(1, sizeof(persist_t)); TEST_ASSERT_NOT_NULL(p2); persist_config_t cfg2 = persist_config_defaults(); cfg2.wal_sync_ms = 5000; persist_set_workdir(p2, tmpdir, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg2, NULL)); TEST_ASSERT_EQUAL_INT(0, persist_start(p2)); TEST_ASSERT_NULL_MESSAGE(p2->cluster, "replay fixture must have no cluster (no-effect guarantee)"); int rc = persist_recover(p2); /* Assert: replay clean, LSN advanced STRICTLY past the no-op (contract). */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "boot replay over a WAL_NOOP must succeed"); TEST_ASSERT_TRUE_MESSAGE(p2->current_lsn > noop_lsn, "boot replay over a WAL_NOOP must advance current_lsn past the no-op"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn + 1, p2->current_lsn, "boot replay must leave current_lsn at one past the WAL_NOOP's LSN"); persist_stop(p2); persist_fini(p2); free(p2); /* Assert (RED PIN): the replayed WAL_NOOP must be a recognised, RETAIN- * classified type — not merely tolerated by apply/replay routing. Today * wal_classify(WAL_NOOP) aborts (unclassified) per the totality guard, so * the forked carrier returns 0 and this case is RED for the WAL_NOOP * CONTRACT reason. Task 1.2's enum + apply arm + RETAIN classify row flips * it to 1, turning this case green. */ TEST_ASSERT_EQUAL_INT_MESSAGE(1, noop_classifies_retain(), "a replayed WAL_NOOP must be a classified type (wal_classify → RETAIN)"); printf("test_noop_boot_replay_clean_no_effect: PASS\n"); } /* Case 4: wal_classify(WAL_NOOP, 0) == WAL_MODE_RETAIN. * * This keeps barrier-rollout's classification-totality test meaningful: the * enum value and its classification row must land together (task 1.2). * * RED today: WAL_NOOP (40) is unclassified, so wal_classify hits the * totality fail-fast `assert(0)` under XCE_TESTING and ABORTS. Calling it * inline would crash the whole test binary, so it is evaluated in a forked * child (cerebrum fork abort-test pattern). The contract is RETAIN; the * child computes wal_classify(40,0) and exits 0 ONLY if it returns RETAIN * without aborting. Today the child aborts (SIGABRT), so the parent's * "exited cleanly with RETAIN" assertion FAILS — the captured RED state. * * detect_leaks=0 set in the child so ASAN does not flag the abort path. */ static void test_noop_classify_is_retain(void) { pid_t pid; int status = 0; /* Act: classify WAL_NOOP in a child (it may abort in the current tree) */ pid = fork(); TEST_ASSERT_TRUE_MESSAGE(pid >= 0, "fork must succeed"); if (pid == 0) { /* Child: silence the expected assert/abort noise; ASAN-friendly. */ setenv("ASAN_OPTIONS", "detect_leaks=0", 1); int devnull = open("/dev/null", O_WRONLY); if (devnull >= 0) { dup2(devnull, STDERR_FILENO); dup2(devnull, STDOUT_FILENO); } wal_commit_mode_t mode = wal_classify(WAL_NOOP_VALUE, 0); /* Reaching here means no abort; encode the result in the exit code: * exit 0 ⇔ RETAIN (the contract), exit 1 ⇔ classified-but-wrong. */ _exit(mode == WAL_MODE_RETAIN ? 0 : 1); } /* Assert: the child must exit cleanly with RETAIN. Today it aborts * (SIGABRT) because WAL_NOOP is unclassified — this assertion is RED. */ TEST_ASSERT_EQUAL_INT_MESSAGE(pid, waitpid(pid, &status, 0), "waitpid must reap the classifier child"); TEST_ASSERT_TRUE_MESSAGE(WIFEXITED(status), "wal_classify(WAL_NOOP) must NOT abort — it must classify the type"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, WEXITSTATUS(status), "wal_classify(WAL_NOOP, 0) must return WAL_MODE_RETAIN"); printf("test_noop_classify_is_retain: PASS\n"); } /* ══════════════════════════════════════════════════════════════════════════ * Task 2.1 (TDD RED) — DIRECT promotion + the OWN-TERM GATE. * * Contract (design §Promotion Rewrite, Reqs 1.1, 1.3, 1.4, 1.5, 2.1, 2.2, 2.3): * every road to PRIMARY lands the node DIRECTLY in CTRL_PRIMARY — no * CTRL_PRIMARY_COMMITTING ever observable — registers the persist hooks, * arms NO wall-clock watchdog, and (production, c != NULL) appends a WAL_NOOP * of the current term then sets the commit engine's own-term gate * (commit_set_writable_lsn) to that no-op's LSN so confirmation of any * operation is withheld until committed_lsn >= noop_lsn. * * SEAMS (study notes; no new production seam is introduced — RED-only): * - Cases 1-3 drive the raft become-leader callback via its existing * XCE_TESTING shim `raft_cb_become_leader_for_test(ha, term)` (ha.c:3121), * which calls `raft_cb_become_leader` with `ha->raft_c = NULL`. This is * THE seam test_ha_evaluate_committing.c / test_ha_connection_reuse.c use. * RED today: that path stores CTRL_PRIMARY_COMMITTING (ha.c:514), so the * direct-PRIMARY / is_primary / no-redirect / hooks-registered assertions * fail. (The `c == NULL` shim also means the watchdog-arm + noop/gate * blocks — both under `if (c)` — are skipped, so they are NOT observable * through THIS seam; see cases 4-7 for the real-cluster seam.) * - Cases 4-7 drive the SHARED promotion sequence `ha_promote_to_primary(c)` * (ha.c:1717, public in ha.h) against a partial-but-real cluster_t: a real * tmpdir persist_t, a wired commit_engine_t (n_peers), a stub standby * registry (repl_add_standby + repl_set_commit_engine), and a delaymgr * left un-started. Per design §Promotion Rewrite, ha_promote_to_primary * becomes the canonical "every road to PRIMARY arms the gate" entry point * (task 2.2 adds the noop+gate tail there + in the raft callback). TODAY * ha_promote_to_primary does ha_do_promote + hook-register + heartbeat and * NEVER touches c->commit, so the own-term gate stays unset (writable_lsn * == 0) — that is the RED carrier for cases 5-6. * * GATE OBSERVABILITY (cerebrum + commit.c:166-171): the engine's WAIT * completion predicate is `committed_lsn >= lsn && (writable_lsn == 0 || * committed_lsn >= writable_lsn)`. writable_lsn is a public commit_engine_t * field; commit_set_writable_lsn(ce, N) installs the gate. With the gate at N * a WAIT on an entry M < N blocks past M's own commit, completing only when * committed_lsn reaches N (test_commit_engine.c test 15 shape). TODAY * promotion leaves writable_lsn == 0, so that same WAIT completes the instant * M commits — the behavioral RED carrier. * * DELAYMGR TASK-COUNT SEAM (case 4): delay_task_ms enqueues into * mgr->tasks (an ordq_t) regardless of running state (delay.c) — count the * `tasks.head` singly-linked chain directly (ordq has no public count API). * ════════════════════════════════════════════════════════════════════════ */ #ifdef XCE_TESTING /* The raft become-leader callback shim (defined in ha.c under XCE_TESTING). */ extern void raft_cb_become_leader_for_test(ha_state_t *ha, uint64_t term); #endif /* Minimal ha_config_t accepted by ha_init (mirrors test_ha_role.c). */ static ha_config_t lifecycle_ha_cfg(void) { ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, .committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS, }; return cfg; } /* Count pending delaymgr tasks by walking the ordq head chain directly. * Read-only: takes no lock (single-threaded test context, delaymgr never * started). This is the "delaymgr task-count seam" the contract pins. */ static int delaymgr_task_count(const delaymgr_t *mgr) { int count = 0; const ordq_item_t *it = mgr->tasks.head; while (it != NULL) { count++; it = it->next; } return count; } /* ── Case 1: DIRECT PRIMARY — become-leader lands in CTRL_PRIMARY, not a * transitional state; ha_is_primary() is true immediately on return (Req 1.1, * 1.4). RED today: raft_cb_become_leader stores CTRL_PRIMARY_COMMITTING. */ static void test_promotion_lands_directly_in_primary(void) { #ifdef XCE_TESTING ha_state_t ha; ha_config_t cfg = lifecycle_ha_cfg(); TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); ha.phase2_done = 1; /* skip phase2 (no real cluster behind the shim) */ atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release); /* Act: drive the raft become-leader path (shim sets ha.raft_c = NULL) */ raft_cb_become_leader_for_test(&ha, /*term=*/1); /* Assert: role is CTRL_PRIMARY directly — no transitional value */ ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha.role, memory_order_acquire); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, (int)role, "winning election must land DIRECTLY in CTRL_PRIMARY (Req 1.1); " "current code stores CTRL_PRIMARY_COMMITTING — RED"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha_is_primary(&ha), "ha_is_primary() must be true the instant promotion returns (Req 1.4)"); ha_fini(&ha); printf("test_promotion_lands_directly_in_primary: PASS\n"); #else printf("test_promotion_lands_directly_in_primary: SKIP (non-test build)\n"); #endif } /* ── Case 2: NO REDIRECT — ha_should_redirect() returns 0 the instant * promotion returns, before any standby ACK (Req 1.3). RED today: in * CTRL_PRIMARY_COMMITTING ha_should_redirect can return 1 (a live election * winner pointing clients at a primary that "does not exist yet"). */ static void test_promotion_no_redirect_from_live_primary(void) { #ifdef XCE_TESTING ha_state_t ha; ha_config_t cfg = lifecycle_ha_cfg(); TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); ha.phase2_done = 1; atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release); /* A stale primary hint is present (would trigger redirect for any * non-PRIMARY role): host+port set as if learned from a prior leader. */ strncpy(ha.primary_host, "10.0.0.9", sizeof(ha.primary_host) - 1); ha.primary_port = 9991; /* Act */ raft_cb_become_leader_for_test(&ha, /*term=*/1); /* Assert: a live primary never redirects (Req 1.3). In CTRL_PRIMARY the * stale hint is ignored; in CTRL_PRIMARY_COMMITTING (today) the hint makes * ha_should_redirect return 1 → RED. */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha_should_redirect(&ha), "a freshly promoted primary must NOT redirect clients (Req 1.3); " "current COMMITTING role + a primary hint yields redirect=1 — RED"); ha_fini(&ha); printf("test_promotion_no_redirect_from_live_primary: PASS\n"); #else printf("test_promotion_no_redirect_from_live_primary: SKIP (non-test build)\n"); #endif } /* Case 3 (HOOKS REGISTERED BY THE SHARED PROMOTION SEQUENCE) is defined just * below the real-cluster fixture (lifecycle_cluster_setup / g_lc) it shares * with cases 4-7, since C requires those helpers to be declared before use. * It is still RUN as case 3 (see test_ha_role_lifecycle below). */ /* ------------------------------------------------------------------ */ /* Real-cluster fixture for cases 3-7: a partial-but-real cluster_t */ /* whose persist (real tmpdir), commit engine (wired), stub standby */ /* registry and (un-started) delaymgr are the surfaces promotion drives.*/ /* ------------------------------------------------------------------ */ /* Statics survive Unity's longjmp on assertion failure (test_repl_majority * pattern); cluster_t is large and must not live on the test stack. */ static cluster_t g_lc; static int g_lc_eps[MAX_STANDBYS]; /* Build g_lc with: real persist over a fresh tmpdir, current_lsn primed so a * promotion no-op would land at lsn = appended_lsn (= current_lsn - 1); * n_standbys dummy standbys in the registry; commit engine wired to the * registry (n_peers, n_vcpus); delaymgr initialised but NOT started; role * forced to CTRL_ELECTING. Returns the lsn the next appended entry takes. */ static uint64_t lifecycle_cluster_setup(const char *suffix, int n_peers, int n_standbys, uint64_t appended_lsn, int n_vcpus) { int i; char name[32]; char tmpdir[256]; memset(&g_lc, 0, sizeof(g_lc)); /* HA state */ ha_config_t cfg = lifecycle_ha_cfg(); TEST_ASSERT_EQUAL_INT(0, ha_init(&g_lc.ha, &cfg, 0)); g_lc.ha.phase2_done = 1; /* no phase2 wiring in this fixture */ g_lc.ha.n_peers = n_peers; atomic_store_explicit(&g_lc.ha.role, (int32_t)CTRL_ELECTING, memory_order_release); /* Real persist over a private tmpdir (the no-op append target). */ persist_config_t pcfg = persist_config_defaults(); pcfg.wal_sync_ms = 5000; /* slow background fsync — no test races */ snprintf(tmpdir, sizeof(tmpdir), "/tmp/xce_slr_%s_XXXXXX", suffix); TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); persist_set_workdir(&g_lc.persist, tmpdir, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(&g_lc.persist, &pcfg, NULL)); TEST_ASSERT_EQUAL_INT(0, persist_start(&g_lc.persist)); /* Prime the appended watermark: current_lsn = appended_lsn + 1 means the * highest appended entry is `appended_lsn` and the next append takes it. */ pthread_mutex_lock(&g_lc.persist.lock); g_lc.persist.current_lsn = appended_lsn + 1; pthread_mutex_unlock(&g_lc.persist.lock); atomic_store(&g_lc.persist.committed_lsn, 0); /* Stub standby registry + wired engine (commit derives from watermarks). */ for (i = 0; i < n_standbys; i++) { snprintf(name, sizeof(name), "lcsb%d", i); TEST_ASSERT_TRUE_MESSAGE( repl_add_standby(&g_lc.persist.repl, &g_lc_eps[i], name, 7400 + (uint32_t)i) >= 0, "repl_add_standby must return a slot index >= 0"); } TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&g_lc.commit, &g_lc.persist, n_peers, n_vcpus)); repl_set_commit_engine(&g_lc.persist.repl, &g_lc.commit); /* Drop wire sends from the own-term WAL_NOOP broadcast — the fake int * endpoints have no real ep->peer (see lc_drop_send_hook above). */ #ifdef XCE_TESTING repl_broadcast_send_hook = lc_drop_send_hook; #endif /* delaymgr initialised but deliberately NOT started: promotion's heartbeat * arm is skipped (ha_start_primary_heartbeat guards on delaymgr.running), * so any task that appears in tasks.head was armed by promotion itself. */ TEST_ASSERT_EQUAL_INT(0, delaymgr_init(&g_lc.delaymgr, 100, NULL)); return appended_lsn; /* lsn a promotion no-op would take (current_lsn - 1) */ } static void lifecycle_cluster_teardown(void) { #ifdef XCE_TESTING repl_broadcast_send_hook = NULL; #endif commit_engine_fini(&g_lc.commit); delaymgr_fini(&g_lc.delaymgr); persist_stop(&g_lc.persist); persist_fini(&g_lc.persist); repl_fini(&g_lc.persist.repl); ha_fini(&g_lc.ha); /* tmpdir left on disk — /tmp is ephemeral */ } /* Drive a standby ACK watermark into the registry + engine (production * repl_on_ack shape: store acked_lsn, then feed commit_on_ack). */ static void lifecycle_standby_ack(int idx, uint64_t acked_lsn) { atomic_store(&g_lc.persist.repl.standbys[idx].acked_lsn, acked_lsn); commit_on_ack(&g_lc.commit, idx, acked_lsn); } static uint64_t lifecycle_committed(void) { return atomic_load(&g_lc.persist.committed_lsn); } static uint64_t lifecycle_mono_ms(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (uint64_t)ts.tv_sec * 1000u + (uint64_t)ts.tv_nsec / 1000000u; } static void lifecycle_sleep_ms(uint32_t ms) { struct timespec ts; ts.tv_sec = ms / 1000u; ts.tv_nsec = (long)(ms % 1000u) * 1000000L; nanosleep(&ts, NULL); } /* WAIT-on-an-entry thread context + body (test_commit_engine.c pattern). */ typedef struct { uint64_t lsn; uint32_t timeout_ms; int rc; uint64_t elapsed_ms; atomic_int done; } lc_wait_arg_t; static void *lifecycle_wait_thread(void *p) { lc_wait_arg_t *a = (lc_wait_arg_t *)p; uint64_t t0 = lifecycle_mono_ms(); a->rc = commit_submit(&g_lc.commit, a->lsn, COMMIT_MODE_WAIT, a->timeout_ms); a->elapsed_ms = lifecycle_mono_ms() - t0; atomic_store(&a->done, 1); return NULL; } /* ── Case 3: HOOKS REGISTERED BY THE SHARED PROMOTION SEQUENCE — the * canonical promotion entry point ha_promote_to_primary(c) (ha.c:1717) must * register the three persist hooks so an idle cluster's new primary is * writable; observe via c->ha.hooks_registered transitioning 0 → 1 (Req 2.1). * * SHARED-SEQUENCE CONTRACT PIN (GREEN now, MUST STAY GREEN — not a behavioral * RED carrier). After task 2.2 BOTH roads to PRIMARY (the raft become-leader * callback and ha_repl_ack_received's quorum path) route through this same * ha_promote_to_primary sequence, which task 2.2 augments with the noop+gate * tail while KEEPING hook registration. This case is therefore a CONTRACT/ * REGRESSION pin proving the shared promotion sequence registers the persist * hooks; the behavioral RED signal for 2.2 is carried by cases 1, 2, 5, 6. * * It uses the SAME real-cluster fixture cases 4-7 use (lifecycle_cluster_setup * / g_lc), which is why it is defined here, below that fixture. This is the * fix for the prior c==NULL shim defect: the raft become-leader shim forces * ha->raft_c == NULL (ha.c:3121), which routes AROUND the c-dependent * registration block, so it could never set the flag on that seam — pinning * it there left an un-turnable RED. ha_promote_to_primary is the real shared * seam and registers hooks today (ha.c:1722-1730). * * Why this is GREEN today: ha_promote_to_primary already registers the three * persist hooks and sets hooks_registered = 1 (ha.c:1729). The g_lc fixture's * hooksys is zeroed (memset), so xce_add_hook returns -EINVAL harmlessly * (typ >= maxtypes==0, hook.c:41) WITHOUT storing the fns — but the flag still * transitions 0 → 1 (ha.c:1714-1715 documents exactly this). The * hooks_registered 0 → 1 transition is therefore the clean, observable * contract pin on the zeroed-hooksys fixture; the three fns are NOT observable * in g_lc.hooksys here (the zeroed registry rejects the add), so we pin the * flag transition per the task wording rather than the slot contents. */ static void test_promotion_registers_persist_hooks(void) { #ifdef XCE_TESTING /* Arrange: a partial-but-real cluster (same fixture as cases 4-7). * n_peers value is not material to the hook contract. */ (void)lifecycle_cluster_setup("hooks", /*n_peers=*/3, /*n_standbys=*/2, /*appended_lsn=*/5, /*n_vcpus=*/8); TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_lc.ha.hooks_registered, "fixture precondition: hooks must be unregistered before promotion"); /* Act: drive the SHARED promotion sequence both roads route through (2.2). */ TEST_ASSERT_EQUAL_INT(0, ha_promote_to_primary(&g_lc)); /* Assert: the shared promotion sequence registered the persist hooks — * hooks_registered transitioned 0 → 1 (Req 2.1). Contract/regression pin: * green today (ha.c:1729) and must remain green after task 2.2 augments the * sequence with the noop+gate tail. */ TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_lc.ha.hooks_registered, "the shared promotion sequence (ha_promote_to_primary) must register " "the persist hooks (hooks_registered 0→1, Req 2.1) — contract pin, " "must stay green through task 2.2"); lifecycle_cluster_teardown(); printf("test_promotion_registers_persist_hooks: PASS\n"); #else printf("test_promotion_registers_persist_hooks: SKIP (non-test build)\n"); #endif } /* ── Case 4: NO WATCHDOG — promotion arms no delaymgr task (Req 1.5, 2.3). * * The shared promotion sequence must never arm a wall-clock watchdog (the * deleted committing_timeout_fn). Asserted via the delaymgr task-count seam: * the count is unchanged across ha_promote_to_primary. * * Status note (RED vs already-GREEN): the force-promotion watchdog is armed * ONLY inside raft_cb_become_leader's `if (c)` block (ha.c:611-619), and that * block has NO real-cluster unit seam (raft_cb_become_leader is static; its * only shim forces c == NULL). ha_promote_to_primary — the road this case * exercises — never armed a watchdog, so this case is ALREADY GREEN today and * stands as a regression-pin guaranteeing task 2.2's shared sequence does not * introduce one. The watchdog REMOVAL from raft_cb_become_leader is covered * at integration scope (task 4.x deletes committing_timeout_fn and the * test_ha_evaluate_committing committing-timeout cases). */ static void test_promotion_arms_no_delaymgr_watchdog(void) { #ifdef XCE_TESTING /* Arrange: 3-member real cluster, delaymgr started so an armed watchdog * (if any) would enqueue; heartbeat stays skipped because the static * one-shot arm + the role no-op keep it from stacking a task here. */ (void)lifecycle_cluster_setup("nowd", /*n_peers=*/3, /*n_standbys=*/2, /*appended_lsn=*/5, /*n_vcpus=*/8); int before = delaymgr_task_count(&g_lc.delaymgr); /* Act: drive the shared promotion sequence */ TEST_ASSERT_EQUAL_INT(0, ha_promote_to_primary(&g_lc)); /* Assert: no wall-clock task was armed by promotion (Req 1.5, 2.3) */ int after = delaymgr_task_count(&g_lc.delaymgr); TEST_ASSERT_EQUAL_INT_MESSAGE(before, after, "promotion must arm NO delaymgr watchdog (Req 1.5, 2.3) — " "the delaymgr task count must be unchanged across promotion"); /* And the road still landed in CTRL_PRIMARY directly (Req 1.1) */ TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, (int)atomic_load_explicit(&g_lc.ha.role, memory_order_acquire), "shared promotion sequence must reach CTRL_PRIMARY directly"); lifecycle_cluster_teardown(); printf("test_promotion_arms_no_delaymgr_watchdog: PASS " "(delaymgr tasks before=%d after=%d)\n", before, after); #else printf("test_promotion_arms_no_delaymgr_watchdog: SKIP (non-test build)\n"); #endif } /* ── Case 5: OWN-TERM GATE (real engine, n_peers=3, stub registry) — * promotion must append a WAL_NOOP at lsn=N, register it RETAIN, and set the * commit engine gate to N (commit_set_writable_lsn(N)). The behavioral * consequence: a WAIT for an entry M < N does NOT complete while ACKs cover M * but not N, and completes only once committed_lsn >= N (Req 2.1). * * RED today: ha_promote_to_primary never touches c->commit, so writable_lsn * stays 0 (no gate) and the WAIT on M completes the instant M commits. Two * RED carriers are pinned: (a) the gate field writable_lsn == noop_lsn after * promotion (today it is 0); (b) the behavioral block-until-gate. */ static void test_promotion_appends_noop_and_arms_own_term_gate(void) { #ifdef XCE_TESTING /* Arrange: 3-member cluster. Prime so the promotion no-op lands at lsn * N=3 (current_lsn-1). Entries 1..3 are appended into the engine so a * waiter on entry M=1 has a target below the gate. */ const uint64_t noop_lsn = 3; /* persist_wal_async post-increments current_lsn (persist.c:318), and * production derives noop_lsn = current_lsn - 1. The fixture primes * current_lsn = appended_lsn + 1, so passing appended_lsn = noop_lsn - 1 * lands the no-op at exactly lsn = noop_lsn. */ (void)lifecycle_cluster_setup("gate", /*n_peers=*/3, /*n_standbys=*/2, /*appended_lsn=*/noop_lsn - 1, /*n_vcpus=*/8); /* Register pre-existing entries strictly BELOW the gate (1..noop_lsn-1); * the no-op itself occupies noop_lsn. A waiter on M=1 still has a * target below the gate. */ for (uint64_t lsn = 1; lsn < noop_lsn; lsn++) (void)commit_submit(&g_lc.commit, lsn, COMMIT_MODE_RETAIN, 0); /* Act: drive the shared promotion sequence (appends WAL_NOOP + gate). */ TEST_ASSERT_EQUAL_INT(0, ha_promote_to_primary(&g_lc)); /* Assert (gate field): writable_lsn must be the no-op's LSN (Req 2.1). * RED today: promotion does not touch the engine → writable_lsn == 0. */ TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn, g_lc.commit.writable_lsn, "promotion must set the own-term gate writable_lsn = noop_lsn (Req 2.1); " "current promotion never appends a no-op nor sets the gate — RED"); /* Assert (WAL): a header-only WAL_NOOP must have been appended. RED * today: nothing is appended, so current_lsn never advanced past the * primed watermark. */ TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn + 1, g_lc.persist.current_lsn, "promotion must append exactly one WAL_NOOP (current_lsn advances to " "noop_lsn+1, Req 2.1) — current promotion appends nothing — RED"); /* Assert (behavioral gate): a WAIT on entry M=1 < gate must NOT complete * once entry 1 commits — it must block until committed_lsn >= noop_lsn. */ static lc_wait_arg_t warg; memset(&warg, 0, sizeof(warg)); warg.lsn = 1; warg.timeout_ms = 3000; warg.rc = -9999; atomic_store(&warg.done, 0); pthread_t wtid; TEST_ASSERT_EQUAL_INT(0, pthread_create(&wtid, NULL, lifecycle_wait_thread, &warg)); /* Entry 1 gains majority (self + sb0); gate (noop_lsn) still unmet. */ lifecycle_sleep_ms(50); lifecycle_standby_ack(0, 1); lifecycle_sleep_ms(150); int done_after_entry1 = atomic_load(&warg.done); /* Gate opens: a covering ACK advances committed_lsn to noop_lsn. */ lifecycle_standby_ack(0, noop_lsn); pthread_join(wtid, NULL); /* RED today: with writable_lsn == 0 the waiter completed at the first * ACK (done_after_entry1 == 1). Contract: it must still be blocked. */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, done_after_entry1, "WAIT on entry 1 must NOT complete while committed_lsn < noop_lsn " "(own-term gate, Req 2.1); unset gate completes it early — RED"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, warg.rc, "WAIT must succeed once committed_lsn reaches the own-term gate"); TEST_ASSERT_TRUE_MESSAGE(warg.elapsed_ms >= 180, "WAIT must block past its own entry's commit until the gate opens"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn, lifecycle_committed(), "committed_lsn must reach noop_lsn once the gate-covering ACK arrives"); lifecycle_cluster_teardown(); printf("test_promotion_appends_noop_and_arms_own_term_gate: PASS\n"); #else printf("test_promotion_appends_noop_and_arms_own_term_gate: SKIP (non-test build)\n"); #endif } /* ── Case 6: NO THIRD PATH (gate unmet) — with the own-term gate unmet, a * state-changing WAIT submission resolves ONLY as barrier success, -ETIMEDOUT, * or -XCE_ERR_NOT_READY — never via any promotion-side completion (Req 2.2). * * Here the gate (= noop_lsn) is never satisfied (no covering ACK), so the only * legitimate outcome is -ETIMEDOUT (the entry cannot reach majority within the * deadline; the gate is a blocking predicate, not an admission fast-reject). * * RED today: promotion sets no gate, so a WAIT on the no-op's own entry can be * "confirmed" by an ACK covering only that entry — a confirmation the * own-term rule must withhold until majority of the new-term no-op. The * carrier: with the contract gate at noop_lsn, an ACK covering ONLY a lower * entry must leave the WAIT pending → -ETIMEDOUT; today (gate 0) the same ACK * confirms early (rc == 0). */ static void test_pre_own_term_wait_has_no_third_path(void) { #ifdef XCE_TESTING const uint64_t noop_lsn = 3; /* appended_lsn = noop_lsn - 1 lands the no-op at lsn = noop_lsn (see * case 5 comment); entries 1..noop_lsn-1 sit strictly below the gate. */ (void)lifecycle_cluster_setup("nothird", /*n_peers=*/3, /*n_standbys=*/2, /*appended_lsn=*/noop_lsn - 1, /*n_vcpus=*/8); for (uint64_t lsn = 1; lsn < noop_lsn; lsn++) (void)commit_submit(&g_lc.commit, lsn, COMMIT_MODE_RETAIN, 0); /* Act: promotion (must arm gate = noop_lsn). */ TEST_ASSERT_EQUAL_INT(0, ha_promote_to_primary(&g_lc)); /* A WAIT on entry 1 with a short deadline; only entry 1 ever gets a * covering ACK — the gate (noop_lsn=3) is never satisfied. */ static lc_wait_arg_t warg; memset(&warg, 0, sizeof(warg)); warg.lsn = 1; warg.timeout_ms = 250; warg.rc = -9999; atomic_store(&warg.done, 0); pthread_t wtid; TEST_ASSERT_EQUAL_INT(0, pthread_create(&wtid, NULL, lifecycle_wait_thread, &warg)); lifecycle_standby_ack(0, 1); /* entry 1 committed; gate (3) unmet */ pthread_join(wtid, NULL); /* Assert: the ONLY admissible outcomes are barrier success (0) on a real * gate-covering majority [not provided here], -ETIMEDOUT, or NOT_READY. * With the gate unmet and no covering ACK, the contract outcome is * -ETIMEDOUT. RED today: writable_lsn==0 admits early success (rc==0), * which is the forbidden "third path" — a confirmation without the * own-term entry's majority. */ TEST_ASSERT_TRUE_MESSAGE( warg.rc == -ETIMEDOUT || warg.rc == -XCE_ERR_NOT_READY, "pre-own-term WAIT must resolve only as -ETIMEDOUT or -XCE_ERR_NOT_READY " "while the own-term gate is unmet (Req 2.2); an unset gate confirms it " "early (rc==0) — the forbidden third path — RED"); TEST_ASSERT_EQUAL_INT_MESSAGE(-ETIMEDOUT, warg.rc, "gate-unmet WAIT with no covering majority must time out (Req 2.2, 2.3)"); TEST_ASSERT_TRUE_MESSAGE(warg.elapsed_ms >= 200, "an unmet own-term gate must BLOCK to the deadline, not fast-reject"); lifecycle_cluster_teardown(); printf("test_pre_own_term_wait_has_no_third_path: PASS\n"); #else printf("test_pre_own_term_wait_has_no_third_path: SKIP (non-test build)\n"); #endif } /* ── Case 7: STANDALONE (n_peers=1) — promotion's no-op self-commits at * registration, so the own-term gate opens immediately and a WAIT returns 0 * as before (Req 2.1, umbrella 6.3 / 7.5: standalone latency unchanged). * * Status note: REGRESSION-PIN. On the current tree there is no gate * (writable_lsn stays 0), and a single-member WAIT already returns 0 * immediately (self is its own majority). So this case is EXPECTED GREEN * today; it guards the contract that arming the gate in the n_peers=1 path * (task 2.2) must not regress standalone promotion into a blocking wait. * The writable_lsn-field assertion below WILL flip from already-satisfied * behavior (WAIT==0) to additionally pinning writable_lsn==noop_lsn once 2.2 * lands; today writable_lsn==0 makes that one field assertion RED while the * behavioral WAIT==0 stays GREEN — labelled inline. */ static void test_standalone_promotion_gate_opens_immediately(void) { #ifdef XCE_TESTING const uint64_t noop_lsn = 1; /* standalone: noop is the first entry */ /* appended_lsn = 0 → current_lsn primed to 1 → the no-op lands at lsn=1 * (= noop_lsn). No entries exist below the no-op, so there is no * pre-existing RETAIN submit. */ (void)lifecycle_cluster_setup("solo", /*n_peers=*/1, /*n_standbys=*/0, /*appended_lsn=*/0, /*n_vcpus=*/8); /* Act */ TEST_ASSERT_EQUAL_INT(0, ha_promote_to_primary(&g_lc)); /* Behavioral (EXPECTED GREEN today and after 2.2): single-member WAIT on * the no-op entry returns 0 without blocking — self is the majority and * the gate (if set) is self-satisfied at registration. */ uint64_t t0 = lifecycle_mono_ms(); int rc = commit_submit(&g_lc.commit, noop_lsn, COMMIT_MODE_WAIT, 5000); uint64_t elapsed = lifecycle_mono_ms() - t0; TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "standalone WAIT on the own-term entry must return 0 (self-quorum)"); TEST_ASSERT_TRUE_MESSAGE(elapsed < 1000, "standalone promotion must not block on the own-term gate (6.3/7.5)"); /* Contract field (RED today: writable_lsn==0; GREEN after task 2.2 sets * the gate to the self-committed no-op LSN). */ TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn, g_lc.commit.writable_lsn, "standalone promotion must still arm the own-term gate at noop_lsn " "(Req 2.1); current promotion sets no gate — RED for this field"); lifecycle_cluster_teardown(); printf("test_standalone_promotion_gate_opens_immediately: PASS\n"); #else printf("test_standalone_promotion_gate_opens_immediately: SKIP (non-test build)\n"); #endif } /* ── Case 8: LEGACY/RAFT PROMOTION SYMMETRY (ha-single-leader-role task 3.1, * Req 1.1, 1.5; design §Legacy Path Symmetry). Both entry points to PRIMARY * must produce IDENTICAL observable state so test-driven and raft-driven * promotion cannot diverge: * * (1) raft road — raft_cb_become_leader_for_test (the XCE_TESTING shim, which * forces ha->raft_c == NULL): only the c-independent state is observable * on this seam (role == CTRL_PRIMARY). The hook/gate tail is c!=NULL- * guarded and skipped here — that half of the contract is pinned on the * real-cluster road below (matching cases 3/4/5/7, which drive the SAME * shared helpers ha_register_persist_hooks / ha_arm_own_term_gate). * * (2) legacy road — ha_elect_evaluate_and_signal on the real g_lc cluster * (ctx->c = &g_lc, single-node quorum win): drives the full shared * sequence. Must land role == CTRL_PRIMARY, hooks_registered == 1, * writable_lsn == noop_lsn (own-term gate armed), and arm NO delaymgr * watchdog — byte-for-byte the state ha_promote_to_primary produces. * * Pin: the legacy real-cluster path produces role/hook/gate/no-watchdog state * identical to the raft road's role and to ha_promote_to_primary's full state. * RED before task 3.1: the legacy branch stored CTRL_PRIMARY_COMMITTING, reset * repl_ack_count, and armed the committing watchdog — diverging from the raft * road on every axis. */ static void test_legacy_and_raft_promotion_are_symmetric(void) { #ifdef XCE_TESTING /* ── Road 1: raft shim (c == NULL) — role contract only ──────────────── */ ha_state_t ha_raft; ha_config_t cfg = lifecycle_ha_cfg(); TEST_ASSERT_EQUAL_INT(0, ha_init(&ha_raft, &cfg, 0)); ha_raft.phase2_done = 1; atomic_store_explicit(&ha_raft.role, (int32_t)CTRL_ELECTING, memory_order_release); raft_cb_become_leader_for_test(&ha_raft, /*term=*/7); ctrl_role_t raft_role = (ctrl_role_t)atomic_load_explicit(&ha_raft.role, memory_order_acquire); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, (int)raft_role, "raft road must land DIRECTLY in CTRL_PRIMARY (Req 1.1)"); ha_fini(&ha_raft); /* ── Road 2: legacy ha_elect_evaluate_and_signal on a real cluster ───── */ /* persist_wal_async appends at current_lsn then post-increments, and * ha_arm_own_term_gate derives noop_lsn = current_lsn - 1; the fixture * primes current_lsn = appended_lsn + 1, so passing appended_lsn = * noop_lsn - 1 lands the own-term no-op at exactly lsn = noop_lsn * (same idiom as case 5). */ const uint64_t noop_lsn = 9; (void)lifecycle_cluster_setup("sym", /*n_peers=*/1, /*n_standbys=*/0, /*appended_lsn=*/noop_lsn - 1, /*n_vcpus=*/8); int before = delaymgr_task_count(&g_lc.delaymgr); TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_lc.ha.hooks_registered, "fixture precondition: hooks must be unregistered before promotion"); /* Single-node quorum win: n_peers==0 shortcircuit (votes[] empty), * votes_granted=1 satisfies quorum=1. ctx->c = &g_lc so the shared * promotion sequence runs against the real cluster. */ ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx)); TEST_ASSERT_NOT_NULL_MESSAGE(ctx, "ctx calloc failed"); ctx->n_peers = 0; ctx->votes_granted = 1; ctx->n_replied = 0; ctx->result = -1; ctx->done_posted = 0; ctx->c = &g_lc; pthread_mutex_init(&ctx->lock, NULL); sem_init(&ctx->done, 0, 0); ctrl_role_t legacy_result = ha_elect_evaluate_and_signal(&g_lc.ha, ctx); /* Symmetry axis 1: the legacy road reaches PRIMARY (matches the raft road). */ TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, (int)legacy_result, "legacy evaluate must RESULT in CTRL_PRIMARY (Req 1.1)"); ctrl_role_t legacy_role = (ctrl_role_t)atomic_load_explicit(&g_lc.ha.role, memory_order_acquire); TEST_ASSERT_EQUAL_INT_MESSAGE((int)raft_role, (int)legacy_role, "legacy and raft promotion must land in the SAME role (CTRL_PRIMARY)"); /* Symmetry axis 2: persist hooks registered by the shared sequence. */ TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_lc.ha.hooks_registered, "legacy road must register persist hooks via the shared sequence (Req 2.1)"); /* Symmetry axis 3: own-term gate armed at the no-op LSN. */ TEST_ASSERT_EQUAL_UINT64_MESSAGE(noop_lsn, g_lc.commit.writable_lsn, "legacy road must arm the own-term gate at noop_lsn (Req 1.4, 2.1) — " "identical to the raft road's shared tail"); /* Symmetry axis 4: NO committing-timeout watchdog armed (Req 1.5). */ int after = delaymgr_task_count(&g_lc.delaymgr); TEST_ASSERT_EQUAL_INT_MESSAGE(before, after, "legacy road must arm NO delaymgr watchdog (Req 1.5) — symmetric with " "the raft road; the committing-timeout arm is deleted from this path"); pthread_mutex_destroy(&ctx->lock); sem_destroy(&ctx->done); free(ctx); lifecycle_cluster_teardown(); printf("test_legacy_and_raft_promotion_are_symmetric: PASS " "(role=%d both roads; hooks=1; writable_lsn=%" PRIu64 "; watchdog tasks before=%d after=%d)\n", (int)legacy_role, g_lc.commit.writable_lsn, before, after); #else printf("test_legacy_and_raft_promotion_are_symmetric: SKIP (non-test build)\n"); #endif } /* ------------------------------------------------------------------ */ /* Entry point (registered in controller_test.c / meson.build) */ /* ------------------------------------------------------------------ */ void test_ha_role_lifecycle(void) { /* Task 1.1 — WAL_NOOP no-op entry (Req 2.1) */ RUN_TEST(test_noop_append_is_header_only_and_broadcast); RUN_TEST(test_noop_apply_advances_lsn_no_entity_effect); RUN_TEST(test_noop_boot_replay_clean_no_effect); RUN_TEST(test_noop_classify_is_retain); /* Task 2.1 — DIRECT promotion + own-term gate (Reqs 1.1,1.3,1.4,1.5,2.1,2.2,2.3) */ RUN_TEST(test_promotion_lands_directly_in_primary); /* case 1 */ RUN_TEST(test_promotion_no_redirect_from_live_primary); /* case 2 */ RUN_TEST(test_promotion_registers_persist_hooks); /* case 3 */ RUN_TEST(test_promotion_arms_no_delaymgr_watchdog); /* case 4 */ RUN_TEST(test_promotion_appends_noop_and_arms_own_term_gate); /* case 5 */ RUN_TEST(test_pre_own_term_wait_has_no_third_path); /* case 6 */ RUN_TEST(test_standalone_promotion_gate_opens_immediately); /* case 7 */ /* Task 3.1 — legacy/raft promotion symmetry (Req 1.1, 1.5) */ RUN_TEST(test_legacy_and_raft_promotion_are_symmetric); /* case 8 */ }