/* test_job_array_replay_fence.c — job-array-single-wal-record task 7.2 * * Closes task 4.2's self-flagged residual gap: persist_apply_pump_run() (the * LIVE apply path, task 4.1/4.2) halts-and-fences on a negative * apply_committed_entry() return, but replay_wal() (the BOOT/RESTART REPLAY * path) previously only captured that return value for the job-count tally * (task 6.0a) — it never halted or engaged the Live Apply Resource Fence. * A node that crashed while fenced and restarted into a boot replay hitting * the IDENTICAL resource-exhaustion condition would silently pass it and * look fully promotable while actually divergent/behind. * * ── Scenario ("crash while fenced, restart into the same condition") ────── * Phase 1 ("boot 1"): a bare persist_t (no cluster) appends ONE real, * physical WAL_JOB_ARRAY_SUBMIT entry via persist_wal_async, then tears * down WITHOUT ever applying it — simulating a crash before the live apply * pump got a chance to run (e.g. the process died right after append, * before any apply kick). * * Phase 2 ("boot 2" / restart): a fresh cluster_t + its embedded persist_t * are pointed at the SAME on-disk workdir. The construction-phase * fault-injection seam (job_array_apply_test_set_force_fail_at — the SAME * seam task 4.4's capstone test uses, chosen for the identical reason: it * is the one seam Req 9.6's "no partial/divergent set of elements" claim * is provably true for, unlike the bulk-insert-phase seam) is armed BEFORE * persist_recover() runs, so replay_wal() hits the identical * resource-exhaustion condition during REPLAY, not live apply. * * ── Seam choice ───────────────────────────────────────────────────────── * See test_job_array_fence.c's header comment for the full rationale — * this test reuses the exact same construction-phase seam for the exact * same reason (the bulk-insert-phase seam's documented residual atomicity * gap would make a "no partial set" assertion dishonest here too). * * ── Fixture ───────────────────────────────────────────────────────────── * Mirrors test_replay_apply_seam.c's persist_recover()-driven fixture * (raft-apply-decoupling task 2.3) for phase 1/2, and test_job_array_fence.c's * tf_setup ha_init+delaymgr shape (task 4.4) for the promotion-gate * confirmation — reusing established patterns rather than inventing a new * fixture style. Deliberately OMITS the commit engine / repl standbys * test_job_array_fence.c wires: this test drives the physical WAL directly * via persist_wal_async (like test_replay_apply_seam.c), never through the * commit engine, so those collaborators are unnecessary here. * * IMPORTANT fixture detail: the persist_t used for phase 2 MUST be the one * EMBEDDED in the cluster_t (&g_trf_c.persist), not a standalone heap * persist_t — both raft_cb_become_leader_with_cluster_for_test(&g_trf_c, ...) * and ha_arm_promotion_fence_timer_for_test(&g_trf_c, ...) (and the * promotion_fence_timeout_fn the latter arms) read the fence state off * c->persist directly, so the replay under test and the promotion-gate * checks must observe the exact same persist_t instance. * * ── Remediation round 1 (task 7.2 adversarial review) ─────────────────── * The refusal proof below now drives raft_cb_become_leader_with_cluster_ * for_test() — the REAL election-time eligibility gate (ha.c:1303-1408) — * instead of ha_arm_promotion_fence_timer_for_test() (the periodic re-alert * timer, ha.c ~876), which was the WRONG gate for this test's actual claim * ("a node re-hits the identical condition on restart and is refused * promotion"). The periodic-timer seam is still used, correctly, for the * lighter-weight post-self-heal re-eligibility confirmation below. This * test still does NOT (and cannot, from persist.c alone) prove the node's * very FIRST-EVER promotion evaluation in a process's life is protected — * see the promotion-gate-confirmation block's comment for the precise * scope boundary and file:line citations. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include "cluster.h" #include "controller.capnp.h" #include "delay.h" #include "ha.h" #include "hook.h" #include "jobj.h" #include "persist.h" #include "wal_types.h" #include "xce/cluster_def.h" #include "xce/types.h" #include "xstring.h" #ifdef XCE_TESTING /* job.c's XCE_TESTING-only fault-injection seam (task 3.1) — reused * verbatim, not reimplemented. */ extern void job_array_apply_test_set_force_fail_at(int index); extern void job_array_apply_test_clear_force_fail(void); /* ha.c's XCE_TESTING-only seam (ha-commit-durability task 4.4) — drives the * exact bounded-window promotion-gate re-check without staging a full * election. Used below ONLY for the post-self-heal re-eligibility * confirmation (the periodic re-alert timer path, ha.c ~876) — NOT for the * refusal proof, which must use the real election-time gate below (task 7.2 * remediation round 1: this file previously misused this seam for the * refusal proof itself). */ extern void ha_arm_promotion_fence_timer_for_test(cluster_t *, bool, uint64_t, uint64_t); /* ha.c's XCE_TESTING-only seam (fix-promotion-standby-registration) — drives * raft_cb_become_leader() (the REAL election-time promotion path) with a * live cluster pointer, without staging a full election. This is the SAME * seam test_convergence_verification.c's * test_promotion_refused_when_apply_resource_fenced_even_if_eligible (task * 4.3) already uses to prove the election-time gate refuses promotion for a * fence engaged by the LIVE apply pump. This test drives it for a fence * engaged by BOOT REPLAY instead (task 7.2's actual deliverable) — task 7.2 * remediation round 1: the original version of this test proved refusal via * the WRONG gate (ha_arm_promotion_fence_timer_for_test's periodic re-alert * timer, ha.c ~876) instead of this one. */ extern void raft_cb_become_leader_with_cluster_for_test(cluster_t *, uint64_t); /* jobj.c's jobsub table / the process-global xstring table are shared * across every test file in the controller_unit_test binary — guard * idempotently, mirroring every sibling fixture in this suite. */ static int g_trf_jobobj_ready; static cluster_t g_trf_c; /* large — must not live on the test stack */ static int trf_run_delay_task(delay_task_entry_fn_t entry, void *arg) { return entry(arg); } static void trf_build_cluster(void) { kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16 }; kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING, .nslots = 32, .delta = 16 }; if (makeString((char *)"trf-probe") == NULL) xstring_table_init(64); if (!g_trf_jobobj_ready) { TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32)); g_trf_jobobj_ready = 1; } memset(&g_trf_c, 0, sizeof(g_trf_c)); g_trf_c.myhost = &g_trf_c.myhost_info; g_trf_c.myhost_info.name = (char *)"ctrl-test"; g_trf_c.myhost_info.key = (char *)""; TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&g_trf_c.hooksys, XCE_CONTROLLER_HOOK_MAX)); g_trf_c.jobmgr = makeInstance(&xkvmgr_type, &int_params); g_trf_c.qmgr = makeInstance(&xkvmgr_type, &str_params); g_trf_c.workermgr = makeInstance(&xkvmgr_type, &str_params); g_trf_c.resmgr = makeInstance(&xkvmgr_type, &str_params); TEST_ASSERT_NOT_NULL(g_trf_c.jobmgr); TEST_ASSERT_NOT_NULL(g_trf_c.qmgr); TEST_ASSERT_NOT_NULL(g_trf_c.workermgr); TEST_ASSERT_NOT_NULL(g_trf_c.resmgr); zarray_init(&g_trf_c.sharedinst, 16, NULL); } static void trf_teardown_cluster(void) { zarray_fini(&g_trf_c.sharedinst); if (g_trf_c.resmgr) dereference(g_trf_c.resmgr); if (g_trf_c.workermgr) dereference(g_trf_c.workermgr); if (g_trf_c.qmgr) dereference(g_trf_c.qmgr); if (g_trf_c.jobmgr) dereference(g_trf_c.jobmgr); xce_hook_fini(&g_trf_c.hooksys); } static job_t *trf_find_job(uint64_t jobid) { job_t *j = NULL; lock(g_trf_c.jobmgr, RTTI_LOCK_READ); int found = key_find(g_trf_c.jobmgr, (void *)jobid, (void **)&j); unlock(g_trf_c.jobmgr); return (found > 0) ? j : NULL; } /* Encodes a (base_jobid, array_size, spec) JobSubmit payload — the exact * WAL_JOB_ARRAY_SUBMIT payload shape (mirrors test_job_array_fence.c's * tf_encode_job_submit). Caller frees *out. */ static int trf_encode_job_submit(uint64_t base_jobid, uint64_t array_size, job_submit_data_t *spec, uint8_t **out, size_t *out_len) { struct capn c; struct capn_segment *cs; struct JobSubmit cjs; JobSubmit_ptr jsp; xce_job_submit_t s; memset(&s, 0, sizeof(s)); s.jobid = base_jobid; s.array_size = array_size; s.submit = spec; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_JobSubmit(cs, &cjs, &s); jsp = new_JobSubmit(cs); write_JobSubmit(&cjs, jsp); capn_setp(capn_root(&c), 0, jsp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } /* bug-1445 test-only stub: reproduces the REAL production call chain * (cluster_init_phase2 -> persist_recover -> replay_wal) at the REAL call * site inside raft_cb_become_leader's is_first_boot block, wired via * ha_phase2_hook (ha.h:1619) since XCE_TESTING builds skip the real * cluster_init_phase2() otherwise (ha.c:1431-1433). Deliberately calls ONLY * persist_recover -- the narrow slice bug-1445 is about -- not the rest of * cluster_init_phase2's other work (config load etc.), which this test does * not need. */ #ifdef XCE_TESTING static int trf_phase2_hook_calls_persist_recover(void *c) { cluster_t *cc = (cluster_t *)c; return persist_recover(&cc->persist); } #endif /* ── The test: crash-while-would-be-fenced, restart into a boot replay that * hits the SAME resource-exhaustion condition ────────────────────────────── */ static void test_replay_wal_engages_fence_on_resource_exhaustion_failure(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_trf_replay_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t base_jobid = 980000; const uint64_t array_size = 5; const int fail_index = 2; /* mid-array: proves a genuine halt, not a * lucky last-element edge case */ job_submit_data_t spec; memset(&spec, 0, sizeof(spec)); spec.queue = (char *)"normal"; spec.user = (char *)"tester"; spec.mintasks = 1; spec.maxtasks = 1; spec.job_name = (char *)"replay-fence-e2e"; uint8_t *payload = NULL; size_t len = 0; int enc_rc = trf_encode_job_submit(base_jobid, array_size, &spec, &payload, &len); /* ── Phase 1 ("boot 1"): append the real committed-shape array-submit * WAL entry physically to disk, then tear down WITHOUT ever applying * it -- simulating a crash before the live apply pump ran (e.g. the * process died moments after append, before any apply kick fired). No * cluster_t is needed: persist_wal_async is a physical-write-only path * (mirrors test_replay_apply_seam.c's phase 1). ─────────────────────── */ { persist_config_t cfg1 = persist_config_defaults(); cfg1.wal_sync_ms = 5000; persist_t *p1 = calloc(1, sizeof(persist_t)); TEST_ASSERT_NOT_NULL(p1); persist_set_workdir(p1, tmp, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(p1, &cfg1, NULL)); TEST_ASSERT_EQUAL_INT(0, persist_start(p1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p1, WAL_JOB_ARRAY_SUBMIT, payload, (uint32_t)len)); TEST_ASSERT_EQUAL_INT(0, persist_flush(p1)); persist_stop(p1); persist_fini(p1); free(p1); } /* ── Phase 2 ("boot 2" / restart): fresh cluster_t whose EMBEDDED * persist_t points at the SAME on-disk workdir. Arm the identical * construction-phase resource-exhaustion condition BEFORE * persist_recover() runs, so replay_wal() hits it during REPLAY. ────── */ trf_build_cluster(); persist_config_t cfg2 = persist_config_defaults(); cfg2.wal_sync_ms = 5000; persist_set_workdir(&g_trf_c.persist, tmp, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(&g_trf_c.persist, &cfg2, &g_trf_c)); TEST_ASSERT_EQUAL_INT(0, persist_start(&g_trf_c.persist)); job_array_apply_test_clear_force_fail(); job_array_apply_test_set_force_fail_at(fail_index); bool fenced_precondition = persist_apply_resource_fenced(&g_trf_c.persist); int recover_rc = persist_recover(&g_trf_c.persist); uint64_t applied_after_replay = persist_last_applied_lsn(&g_trf_c.persist); bool fenced_after_replay = persist_apply_resource_fenced(&g_trf_c.persist); bool any_element_visible_after_replay = false; for (uint64_t i = 0; i < array_size; i++) { if (trf_find_job(base_jobid + i) != NULL) any_element_visible_after_replay = true; } /* ── Promotion-gate confirmation ────────────────────────────────────── * task 7.2 remediation (round 1): drive the REAL election-time * eligibility gate (raft_cb_become_leader, ha.c:1303-1408 — reads * persist_apply_resource_fenced() at ha.c:1337) directly via * raft_cb_become_leader_with_cluster_for_test(), the SAME seam * test_convergence_verification.c's task-4.3 sibling test already uses * for a fence engaged by the LIVE apply pump. This is the gate an actual * RESTART's re-election evaluates, NOT the periodic re-alert timer used * below only for the post-self-heal confirmation. * * SCOPE NOTE (adversarial review finding, not fixed here — see ha.c * comment near the fence-engage branch in persist.c's replay_wal() for * the full citation): this only proves a promotion evaluation that runs * AFTER a boot replay has ALREADY completed and ALREADY engaged the * fence in this process — exactly what "boot 2" above is. It does NOT * prove the node's very FIRST-EVER promotion evaluation in a process's * life is protected: that check (ha.c:1303-1408) runs strictly BEFORE * the is_first_boot block (ha.c:1427-1439) that triggers * cluster_init_phase2()/persist_recover()/replay_wal() (cluster.c:1482) * for the very first time, with no re-check afterward — a fence engaged * by THAT SAME first replay is invisible to the one promotion check * gating that boot. Closing that gap needs an ha.c change, out of this * task's persist.c-only boundary. */ ha_config_t hcfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_EQUAL_INT(0, ha_init(&g_trf_c.ha, &hcfg, 0)); g_trf_c.ha.raft_c = &g_trf_c; g_trf_c.ha.repl = NULL; /* no healthy peer reachable at this level */ TEST_ASSERT_EQUAL_INT(0, delaymgr_init(&g_trf_c.delaymgr, 5, trf_run_delay_task)); TEST_ASSERT_EQUAL_INT(0, delaymgr_start(&g_trf_c.delaymgr)); /* Structurally healthy in every OTHER respect (not truncated, * durable_floor >= promotion_safe_lsn — no elect_ctx is set here so * promotion_safe_lsn defaults to 0 inside raft_cb_become_leader) so the * ONLY thing that can keep the member refused is the live apply * resource fence boot replay just engaged — same isolation rationale as * test_job_array_fence.c's step 3 and test_convergence_verification.c's * task 4.3 sibling test. */ g_trf_c.persist.recovery_verdict = RECOVER_CLEAN; ha_set_promotion_fence_window_ms(60); raft_cb_become_leader_with_cluster_for_test(&g_trf_c, /*term=*/1); int32_t role_after_election_check = atomic_load_explicit(&g_trf_c.ha.role, memory_order_acquire); int in_catch_up_after_election_check = atomic_load_explicit(&g_trf_c.ha.in_catch_up, memory_order_acquire); bool fenced_after_promotion_check = persist_apply_resource_fenced(&g_trf_c.persist); /* ── Self-heal: clear the forced condition and retry via the live apply * pump directly -- the SAME mechanism persist_recover()'s own * post-recovery kick would eventually invoke; driving it here directly * keeps this test deterministic (no vtask pool / async dispatch * needed). White-box "mark committed" mirrors test_job_array_fence.c's * own established convention for this exact fence area. ─────────────── */ job_array_apply_test_clear_force_fail(); atomic_store(&g_trf_c.persist.committed_lsn, 1); /* the one array entry */ int retry_rc = persist_apply_pump_run(&g_trf_c.persist); uint64_t applied_after_retry = persist_last_applied_lsn(&g_trf_c.persist); bool fenced_after_retry = persist_apply_resource_fenced(&g_trf_c.persist); bool all_elements_visible_after_retry = true; for (uint64_t i = 0; i < array_size; i++) { if (trf_find_job(base_jobid + i) == NULL) all_elements_visible_after_retry = false; } /* Post-self-heal re-eligibility confirmation uses the SIBLING periodic * re-alert timer gate (ha.c ~876) rather than re-driving * raft_cb_become_leader_with_cluster_for_test() a second time: both * gates read the exact same persist_apply_resource_fenced() flag this * fixture cleared above, so this is a legitimate, lighter-weight proof * that clearing the fence (regardless of which engaged it) restores * eligibility everywhere it is checked -- not a re-use of the "wrong" * gate this remediation round fixed above (that fix was specifically * about which gate proves the REFUSAL, not this self-heal check). */ ha_arm_promotion_fence_timer_for_test(&g_trf_c, /*truncated=*/false, /*durable_floor=*/0, /*hwm=*/0); usleep(250 * 1000); int in_catch_up_after_self_heal = atomic_load_explicit(&g_trf_c.ha.in_catch_up, memory_order_acquire); ha_set_promotion_fence_window_ms(0); /* restore default */ ha_cancel_promotion_fence_timer(&g_trf_c.ha); delaymgr_stop(&g_trf_c.delaymgr); delaymgr_fini(&g_trf_c.delaymgr); ha_fini(&g_trf_c.ha); job_array_apply_test_clear_force_fail(); free(payload); persist_stop(&g_trf_c.persist); persist_fini(&g_trf_c.persist); trf_teardown_cluster(); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp); if (system(cmd) != 0) fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp); /* ── Assertions ───────────────────────────────────────────────────── */ TEST_ASSERT_EQUAL_INT(0, enc_rc); TEST_ASSERT_FALSE_MESSAGE(fenced_precondition, "a fresh persist_t must start with the Live Apply Resource Fence " "CLEAR"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, recover_rc, "a resource-exhaustion apply failure during boot replay must NOT " "fail the boot itself (Req 9.4: never treat this as a reason to " "fail the commit)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, applied_after_replay, "last_applied_lsn must NOT advance past the failed array LSN during " "boot replay -- mirrors persist_apply_pump_run's live-path halt"); TEST_ASSERT_TRUE_MESSAGE(fenced_after_replay, "a node that crashes and restarts into a boot replay hitting the " "IDENTICAL resource-exhaustion condition must re-engage the Live " "Apply Resource Fence DURING REPLAY (task 4.2's self-flagged gap, " "closed by task 7.2) -- not silently boot clean and look promotable " "while actually divergent/behind"); TEST_ASSERT_FALSE_MESSAGE(any_element_visible_after_replay, "no partial/divergent set of array elements may be constructed " "during a fenced boot replay (Req 9.6)"); TEST_ASSERT_NOT_EQUAL_MESSAGE((int32_t)CTRL_PRIMARY, role_after_election_check, "a node fenced by BOOT REPLAY must be refused promotion by the REAL " "election-time eligibility gate (raft_cb_become_leader, ha.c:1303-" "1408) -- the same gate tasks 4.2/4.3 already wired for a live-apply-" "engaged fence, now proven for a boot-replay-engaged one too (task " "7.2 remediation round 1)"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, in_catch_up_after_election_check, "a node fenced by BOOT REPLAY must be refused promotion by the SAME " "existing ha.c gate (tasks 4.2/4.3) a live-apply fence already " "uses -- no separate mechanism, no drift"); TEST_ASSERT_TRUE_MESSAGE(fenced_after_promotion_check, "the fence must still read engaged after the promotion-gate check " "-- only the apply pump's own successful retry may clear it"); TEST_ASSERT_TRUE_MESSAGE(retry_rc >= 0, "once the forced condition clears, retrying the SAME stuck LSN via " "the live apply pump must succeed"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(1, applied_after_retry, "last_applied_lsn must advance to the array LSN once the retry " "succeeds"); TEST_ASSERT_TRUE_MESSAGE(all_elements_visible_after_retry, "every array element must be visible once the retry succeeds"); TEST_ASSERT_FALSE_MESSAGE(fenced_after_retry, "the Live Apply Resource Fence must self-clear once the retry " "succeeds (Req 9.5), regardless of whether it was engaged by boot " "replay or the live apply pump"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, in_catch_up_after_self_heal, "once the fence clears, the node must become eligible for " "promotion again with no other action"); printf("test_replay_wal_engages_fence_on_resource_exhaustion_failure: " "PASS\n"); } /* ── bug-1445: a node's very FIRST-EVER promotion evaluation races its OWN * first replay ───────────────────────────────────────────────────────────── * Unlike the sibling test above (which calls persist_recover() BEFORE the * election path, proving a LATER evaluation is protected), this test leaves * the persist_t completely un-recovered when raft_cb_become_leader_with_ * cluster_for_test() is invoked -- ha->phase2_done is fresh (0) and * ha_phase2_hook is wired to actually call persist_recover(), so * cluster_init_phase2()'s real call chain runs from INSIDE this SAME * election evaluation, exactly as it does in production * (raft_cb_become_leader, ha.c ~1427-1439). This is the exact scenario * bug-1445 names and the sibling test's own SCOPE NOTE explicitly says it * does not cover. */ static void test_first_boot_promotion_evaluates_own_first_replay_fence(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_trf_firstboot_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t base_jobid = 981000; const uint64_t array_size = 5; const int fail_index = 2; job_submit_data_t spec; memset(&spec, 0, sizeof(spec)); spec.queue = (char *)"normal"; spec.user = (char *)"tester"; spec.mintasks = 1; spec.maxtasks = 1; spec.job_name = (char *)"first-boot-fence"; uint8_t *payload = NULL; size_t len = 0; int enc_rc = trf_encode_job_submit(base_jobid, array_size, &spec, &payload, &len); /* Physically append ONE committed-shape array entry, then tear down * without ever applying it -- mirrors the sibling test's phase 1, * simulating a genuinely fresh, never-recovered on-disk WAL. */ { persist_config_t cfg1 = persist_config_defaults(); cfg1.wal_sync_ms = 5000; persist_t *p1 = calloc(1, sizeof(persist_t)); TEST_ASSERT_NOT_NULL(p1); persist_set_workdir(p1, tmp, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(p1, &cfg1, NULL)); TEST_ASSERT_EQUAL_INT(0, persist_start(p1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p1, WAL_JOB_ARRAY_SUBMIT, payload, (uint32_t)len)); TEST_ASSERT_EQUAL_INT(0, persist_flush(p1)); persist_stop(p1); persist_fini(p1); free(p1); } trf_build_cluster(); persist_config_t cfg2 = persist_config_defaults(); cfg2.wal_sync_ms = 5000; persist_set_workdir(&g_trf_c.persist, tmp, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(&g_trf_c.persist, &cfg2, &g_trf_c)); TEST_ASSERT_EQUAL_INT(0, persist_start(&g_trf_c.persist)); /* persist_init already defaults recovery_verdict=RECOVER_CLEAN and * apply_verdict=APPLY_CLEAN (persist.c:613,623) -- set explicitly anyway * to mirror the sibling test's defensive style, not because a fresh * persist_t needs it. */ g_trf_c.persist.recovery_verdict = RECOVER_CLEAN; job_array_apply_test_clear_force_fail(); job_array_apply_test_set_force_fail_at(fail_index); /* Precondition: this persist_t has NEVER been recovered -- the fence * must read clear, exactly the state the real eligibility gate * (ha.c:1337) sees at a process's very first election. This is the * blind spot bug-1445 exploits. */ bool fenced_precondition = persist_apply_resource_fenced(&g_trf_c.persist); ha_config_t hcfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_EQUAL_INT(0, ha_init(&g_trf_c.ha, &hcfg, 0)); g_trf_c.ha.raft_c = &g_trf_c; g_trf_c.ha.repl = NULL; /* no healthy peer reachable at this level */ TEST_ASSERT_EQUAL_INT(0, delaymgr_init(&g_trf_c.delaymgr, 5, trf_run_delay_task)); TEST_ASSERT_EQUAL_INT(0, delaymgr_start(&g_trf_c.delaymgr)); ha_set_promotion_fence_window_ms(60); /* The crux of the reproduction: ha->phase2_done is 0 (fresh ha_init * above), so raft_cb_become_leader's is_first_boot block fires and * calls this hook -- which calls persist_recover() for real, from * INSIDE this same election evaluation, racing the gate exactly like * production. */ ha_phase2_hook = trf_phase2_hook_calls_persist_recover; raft_cb_become_leader_with_cluster_for_test(&g_trf_c, /*term=*/1); ha_phase2_hook = NULL; /* restore -- shared process-global test seam */ int32_t role_after_election = atomic_load_explicit(&g_trf_c.ha.role, memory_order_acquire); int in_catch_up_after_election = atomic_load_explicit(&g_trf_c.ha.in_catch_up, memory_order_acquire); bool fenced_after_election = persist_apply_resource_fenced(&g_trf_c.persist); uint64_t applied_after_election = persist_last_applied_lsn(&g_trf_c.persist); bool any_element_visible = false; for (uint64_t i = 0; i < array_size; i++) { if (trf_find_job(base_jobid + i) != NULL) any_element_visible = true; } ha_set_promotion_fence_window_ms(0); /* restore default */ ha_cancel_promotion_fence_timer(&g_trf_c.ha); delaymgr_stop(&g_trf_c.delaymgr); delaymgr_fini(&g_trf_c.delaymgr); ha_fini(&g_trf_c.ha); job_array_apply_test_clear_force_fail(); free(payload); persist_stop(&g_trf_c.persist); persist_fini(&g_trf_c.persist); trf_teardown_cluster(); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp); if (system(cmd) != 0) fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp); /* ── Assertions ───────────────────────────────────────────────────── */ TEST_ASSERT_EQUAL_INT(0, enc_rc); TEST_ASSERT_FALSE_MESSAGE(fenced_precondition, "a never-recovered persist_t must start with the fence CLEAR -- " "this is the exact blind spot bug-1445 exploits"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, applied_after_election, "the first-ever replay (triggered from inside the election path) " "must halt at the failed LSN, same as any other halt-and-fence"); TEST_ASSERT_TRUE_MESSAGE(fenced_after_election, "the node's OWN first-ever replay, triggered from inside this SAME " "election evaluation, must have engaged the Live Apply Resource " "Fence by the time this call returns"); TEST_ASSERT_FALSE_MESSAGE(any_element_visible, "no partial/divergent set of array elements may be constructed " "(Req 9.6)"); TEST_ASSERT_NOT_EQUAL_MESSAGE((int32_t)CTRL_PRIMARY, role_after_election, "bug-1445: a node whose OWN first-ever replay (racing this SAME " "election evaluation) engages the Live Apply Resource Fence must " "be refused promotion -- not silently become CTRL_PRIMARY on a " "gate decision that was made before that first replay ever ran"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, in_catch_up_after_election, "a node refused for this reason must be fenced via the SAME " "existing in_catch_up mechanism every other refusal reason uses " "-- no separate mechanism, no drift"); printf("test_first_boot_promotion_evaluates_own_first_replay_fence: " "PASS\n"); } /* ── bug-1696 (positive-path sanity check, NOT the RED/GREEN discriminator * -- see test_first_boot_promotion_refuses_when_own_first_recovery_finds_ * corruption below for that): a node's very FIRST-EVER promotion evaluation * must still succeed and preserve its own genuinely durable WAL when that * WAL is intact ───────────────────────────────────────────────────────── * * IMPORTANT CORRECTION (found while proving this test's RED phase against * the unfixed ha.c -- do not re-introduce the original hypothesis this * comment used to state): the original bug-1696 write-up hypothesized that * the pre-recovery promotion-truncation-floor computation * (persist_durable_floor()==0 before recovery) would make * persist_truncate_wal_above() PHYSICALLY DESTROY real durable WAL data left * over from a prior life of this process. Source-tracing * truncate_wal_above_locked() (persist.c) disproves that specific mechanism: * its own no-op guard is `if (p->current_lsn == 0 || cutoff_lsn + 1 >= * p->current_lsn) return 0;`, and persist_init() unconditionally resets * current_lsn=1 on every fresh boot regardless of what is physically on * disk -- so a truncate call issued BEFORE persist_recover() has ever * re-derived current_lsn from the real WAL scan is ALWAYS a no-op (nothing * looks "written this boot" yet to truncate). This test proves that * empirically: it reproduces the exact fixture the original hypothesis * described (N real, fsynced entries from a prior life, then a truly fresh * persist_t on the same workdir, is_first_boot=true) and confirms the data * survives with or WITHOUT the bug-1696 reorder fix applied -- i.e. this * specific scenario was never actually destructive. It is kept as a * regression/sanity guard (the reorder must not newly break the healthy * first-boot-promotion path) but does NOT discriminate the bug. * * The REAL, mechanically-provable bug-1696 hazard is different: the * promotion-ELIGIBILITY GATE (persist_recovery_truncated()/persist_ * durable_floor()/persist_apply_verdict()) is ALSO evaluated before * cluster_init_phase2()/persist_recover() has run, so it can wrongly judge * a genuinely CORRUPT or DIVERGENT node ELIGIBLE (reading the fresh- * persist_init "clean" defaults) and let it become CTRL_PRIMARY -- only * discovering the real corruption a few lines later, too late to matter. * See test_first_boot_promotion_refuses_when_own_first_recovery_finds_ * corruption below for the actual RED/GREEN proof of that hazard, and * buglog.json's bug-1696 entry for the full corrected root-cause writeup. */ static void test_first_boot_promotion_never_truncates_own_durable_wal(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_trf_nodataloss_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t n_committed_entries = 5; /* ── "boot 1": physically append N real, fsynced WAL_NOOP entries, then * tear down -- no cluster needed (persist_wal_async is a physical-write * -only path when p->cluster is NULL, mirrors the sibling test's phase * 1 and test_replicated_apply_chain_survives_cold_restart's writer). ── */ { persist_config_t cfg1 = persist_config_defaults(); cfg1.wal_sync_ms = 5000; persist_t *p1 = calloc(1, sizeof(persist_t)); TEST_ASSERT_NOT_NULL(p1); persist_set_workdir(p1, tmp, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(p1, &cfg1, NULL)); TEST_ASSERT_EQUAL_INT(0, persist_start(p1)); for (uint64_t i = 0; i < n_committed_entries; i++) { TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p1, WAL_NOOP, NULL, 0)); } TEST_ASSERT_EQUAL_INT(0, persist_flush(p1)); persist_stop(p1); persist_fini(p1); free(p1); } /* ── "boot 2" / restart: fresh cluster_t + its EMBEDDED persist_t point * at the SAME on-disk workdir. Fresh persist_t -> durable_lsn=0 until * persist_recover() runs -- exactly a real restart. ─────────────────── */ trf_build_cluster(); persist_config_t cfg2 = persist_config_defaults(); cfg2.wal_sync_ms = 5000; persist_set_workdir(&g_trf_c.persist, tmp, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(&g_trf_c.persist, &cfg2, &g_trf_c)); TEST_ASSERT_EQUAL_INT(0, persist_start(&g_trf_c.persist)); g_trf_c.persist.recovery_verdict = RECOVER_CLEAN; /* Precondition: never recovered yet -- durable_floor reads 0 regardless * of the N real entries physically on disk. This is the exact blind * spot bug-1696 exploits. */ uint64_t durable_floor_precondition = persist_durable_floor(&g_trf_c.persist); ha_config_t hcfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_EQUAL_INT(0, ha_init(&g_trf_c.ha, &hcfg, 0)); g_trf_c.ha.raft_c = &g_trf_c; g_trf_c.ha.repl = NULL; /* no healthy peer reachable at this level */ TEST_ASSERT_EQUAL_INT(0, delaymgr_init(&g_trf_c.delaymgr, 5, trf_run_delay_task)); TEST_ASSERT_EQUAL_INT(0, delaymgr_start(&g_trf_c.delaymgr)); /* raft_cb_become_leader's tail (on a successful promotion) reaches * ha_arm_own_term_gate -> commit_submit/commit_set_writable_lsn, which * needs a wired commit engine -- unlike the sibling REFUSAL tests above, * THIS test drives promotion all the way through to CTRL_PRIMARY, so it * must be initialized (mirrors test_convergence_verification.c's * cv_setup, the only other fixture in this suite that reaches a * successful raft_cb_become_leader_with_cluster_for_test promotion). */ TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&g_trf_c.commit, &g_trf_c.persist, /*n_peers=*/0, /*n_vcpus=*/4)); /* Single-node-shaped elect_ctx: n_peers=0, self_lsn=0 -- required to even * exercise the truncation code path (gated on ha->elect_ctx != NULL) and * reproduces the bug narrative's "self also advertises ~0 before its own * recovery" condition. */ ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx)); TEST_ASSERT_NOT_NULL(ctx); ctx->n_peers = 0; ctx->self_lsn = 0; ctx->self_tag = 0; pthread_mutex_init(&ctx->lock, NULL); sem_init(&ctx->done, 0, 0); atomic_store_explicit(&g_trf_c.ha.elect_ctx, (void *)ctx, memory_order_release); /* The crux of the reproduction: ha->phase2_done is 0 (fresh ha_init * above), so raft_cb_become_leader's is_first_boot block fires and calls * this hook -- which calls persist_recover() for real, from INSIDE this * same election evaluation, racing the truncation/eligibility code * exactly like production. */ ha_phase2_hook = trf_phase2_hook_calls_persist_recover; raft_cb_become_leader_with_cluster_for_test(&g_trf_c, /*term=*/1); ha_phase2_hook = NULL; /* restore -- shared process-global test seam */ int32_t role_after_promotion = atomic_load_explicit(&g_trf_c.ha.role, memory_order_acquire); uint64_t durable_floor_after_promotion = persist_durable_floor(&g_trf_c.persist); bool truncated_after_promotion = persist_recovery_truncated(&g_trf_c.persist); atomic_store_explicit(&g_trf_c.ha.elect_ctx, NULL, memory_order_release); pthread_mutex_destroy(&ctx->lock); sem_destroy(&ctx->done); free(ctx); delaymgr_stop(&g_trf_c.delaymgr); delaymgr_fini(&g_trf_c.delaymgr); ha_fini(&g_trf_c.ha); commit_engine_fini(&g_trf_c.commit); persist_stop(&g_trf_c.persist); persist_fini(&g_trf_c.persist); trf_teardown_cluster(); /* ── Independent THIRD-boot verification: a brand fresh persist_t (no * cluster involved at all, so this reflects ONLY what is physically on * disk after everything above) recovering the SAME workdir -- the * `waldump`-equivalent proof the real bug was confirmed with. ────────── */ uint64_t third_boot_recovered_current_lsn = 0; { persist_t *p3 = calloc(1, sizeof(persist_t)); TEST_ASSERT_NOT_NULL(p3); persist_set_workdir(p3, tmp, "ctrl-test"); persist_config_t cfg3 = persist_config_defaults(); cfg3.wal_sync_ms = 5000; TEST_ASSERT_EQUAL_INT(0, persist_init(p3, &cfg3, NULL)); TEST_ASSERT_EQUAL_INT(0, persist_start(p3)); TEST_ASSERT_EQUAL_INT(0, persist_recover(p3)); third_boot_recovered_current_lsn = p3->current_lsn; persist_stop(p3); persist_fini(p3); free(p3); } char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp); if (system(cmd) != 0) fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp); /* ── Assertions ───────────────────────────────────────────────────── */ TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, durable_floor_precondition, "fixture precondition: a never-recovered persist_t must read " "durable_floor=0 regardless of the N real entries on disk -- this " "is the exact blind spot bug-1696 exploits"); TEST_ASSERT_EQUAL_INT_MESSAGE((int32_t)CTRL_PRIMARY, role_after_promotion, "this node is structurally healthy (clean recovery, no faults) and " "must actually become CTRL_PRIMARY on this election -- the bug is " "that it used to get there by first destroying its own WAL, not " "that it fails to promote"); TEST_ASSERT_FALSE_MESSAGE(truncated_after_promotion, "a clean recovery of genuinely-intact WAL data must never be " "reported as recovery-truncated"); TEST_ASSERT_GREATER_OR_EQUAL_UINT64_MESSAGE( n_committed_entries, durable_floor_after_promotion, "bug-1696: promotion must NEVER truncate this node's own durable " "WAL to a bogus pre-recovery floor -- durable_floor after promotion " "must still reflect the N entries committed before this restart, " "not collapse to ~0/1 because the truncation floor was computed " "before cluster_init_phase2()/persist_recover() ever ran"); TEST_ASSERT_GREATER_OR_EQUAL_UINT64_MESSAGE( n_committed_entries + 1, third_boot_recovered_current_lsn, "bug-1696 (independent physical proof): a THIRD boot recovering the " "SAME on-disk workdir after everything above must still find all N " "originally-committed entries physically present -- current_lsn == " "N+1 (one past the highest surviving LSN) proves the real on-disk " "WAL was never truncated away by the second boot's promotion"); printf("test_first_boot_promotion_never_truncates_own_durable_wal: " "PASS\n"); } /* ── bug-1696 (ACTUAL RED/GREEN proof): the promotion-eligibility gate must * NEVER judge a node ELIGIBLE using pre-recovery-blind defaults when its own * first-ever replay (racing this SAME election evaluation) would have * discovered the log is genuinely CORRUPT ─────────────────────────────────── * * raft_cb_become_leader()'s main eligibility gate reads persist_recovery_ * truncated()/persist_durable_floor()/persist_apply_verdict() to decide * ELIGIBLE vs REFUSE. Before the bug-1696 fix, that gate ran BEFORE cluster_ * init_phase2()/persist_recover() -- the ONLY call that can discover a real * on-disk WAL is structurally corrupt for THIS process's first-ever boot. * A freshly persist_init()'d persist_t defaults recovery_verdict= * RECOVER_CLEAN (persist.c ~1168) and durable_lsn=0 (persist.c ~1163) -- * "clean and empty", not "unknown" -- so the gate necessarily judged a node * whose WAL is ACTUALLY corrupt as ELIGIBLE, let it fall through to * CTRL_PRIMARY, and only discovered the real corruption a few lines later * (too late: the bug-1445 re-check immediately after phase2 only re-checks * persist_apply_resource_fenced(), never persist_recovery_truncated()). * This is the general form of the hazard bug-1445 already patched narrowly * for ONE flag; bug-1696 is the same hazard for the far more consequential * durable_floor/recovery_truncated gate inputs. * * Fixture: "boot 1" writes two real, fsynced WAL entries, then the FIRST * entry's on-disk CRC is corrupted (identical technique to test_commit_ * durability.c's test_recover_verdict_crc_break_first_entry_truncated) -- * persist_recover(), once it actually runs, will classify this as * RECOVER_TRUNCATED. "boot 2" points a fresh cluster_t + persist_t at the * SAME corrupted workdir with ha->phase2_done fresh (0, is_first_boot=true) * and ha_phase2_hook wired to call persist_recover() for real, from INSIDE * this same election evaluation -- exactly production's ordering. A single- * node-shaped ha_elect_ctx_t (n_peers=0, self_lsn=0, no reachable peer) * isolates the recovery_truncated conjunct: promotion_safe_lsn=0 means the * HWM comparison alone could never explain a refusal, so if this node is * refused, recovery_truncated is what did it. * * RED before the fix: role_after_election == CTRL_PRIMARY (the gate judged * this node ELIGIBLE while blind, and nothing after phase2 re-checked * recovery_truncated before falling through to CTRL_PRIMARY) despite * persist_recover() -- which DID run moments later, inside the very same * call -- correctly detecting RECOVER_TRUNCATED. * GREEN after the fix: phase2/persist_recover() runs BEFORE the gate, so the * gate sees the real RECOVER_TRUNCATED verdict and REFUSES promotion (stays * alive, fenced, in_catch_up=1, never self-exits). */ static void test_first_boot_promotion_refuses_when_own_first_recovery_finds_corruption(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_trf_corruptboot_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); /* ── "boot 1": two real, fsynced WAL_NOOP entries, then die. ─────────── */ { persist_config_t cfg1 = persist_config_defaults(); cfg1.wal_sync_ms = 5000; persist_t *p1 = calloc(1, sizeof(persist_t)); TEST_ASSERT_NOT_NULL(p1); persist_set_workdir(p1, tmp, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(p1, &cfg1, NULL)); TEST_ASSERT_EQUAL_INT(0, persist_start(p1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p1, WAL_NOOP, NULL, 0)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p1, WAL_NOOP, NULL, 0)); TEST_ASSERT_EQUAL_INT(0, persist_flush(p1)); persist_stop(p1); persist_fini(p1); free(p1); } /* Corrupt the FIRST entry's on-disk CRC -- breaks the chain at LSN 1 * (identical technique to test_commit_durability.c's * test_recover_verdict_crc_break_first_entry_truncated). A future * persist_recover() over this workdir will classify RECOVER_TRUNCATED. */ { char seg_path[512]; snprintf(seg_path, sizeof(seg_path), "%s/ctrl-test/data/wal/seg-%016" PRIx64 ".wal", tmp, (uint64_t)1); int fd = open(seg_path, O_RDWR); TEST_ASSERT_TRUE_MESSAGE(fd >= 0, "first WAL segment must exist"); wal_entry_hdr_t hdr; TEST_ASSERT_EQUAL_INT((int)sizeof(hdr), (int)pread(fd, &hdr, sizeof(hdr), 0)); hdr.crc ^= 0xDEADBEEFu; /* corrupt the entry CRC */ TEST_ASSERT_EQUAL_INT((int)sizeof(hdr), (int)pwrite(fd, &hdr, sizeof(hdr), 0)); fsync(fd); close(fd); } /* ── "boot 2" / restart: fresh cluster_t + persist_t on the SAME * corrupted workdir. Never recovered yet -- the pre-recovery defaults * (recovery_verdict=RECOVER_CLEAN, durable_lsn=0) read "clean and * empty" regardless of the real on-disk corruption. This is the exact * blind spot bug-1696 exploits. ──────────────────────────────────────── */ trf_build_cluster(); persist_config_t cfg2 = persist_config_defaults(); cfg2.wal_sync_ms = 5000; persist_set_workdir(&g_trf_c.persist, tmp, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(&g_trf_c.persist, &cfg2, &g_trf_c)); TEST_ASSERT_EQUAL_INT(0, persist_start(&g_trf_c.persist)); bool recovery_truncated_precondition = persist_recovery_truncated(&g_trf_c.persist); recover_verdict_t recovery_verdict_precondition = persist_recovery_verdict(&g_trf_c.persist); ha_config_t hcfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_EQUAL_INT(0, ha_init(&g_trf_c.ha, &hcfg, 0)); g_trf_c.ha.raft_c = &g_trf_c; g_trf_c.ha.repl = NULL; /* no healthy peer reachable at this level */ TEST_ASSERT_EQUAL_INT(0, delaymgr_init(&g_trf_c.delaymgr, 5, trf_run_delay_task)); TEST_ASSERT_EQUAL_INT(0, delaymgr_start(&g_trf_c.delaymgr)); ha_set_promotion_fence_window_ms(60); /* Single-node-shaped elect_ctx: n_peers=0, self_lsn=0, no reachable peer * -- isolates the recovery_truncated conjunct (promotion_safe_lsn=0 * means the HWM comparison alone can never explain a refusal here). */ ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx)); TEST_ASSERT_NOT_NULL(ctx); ctx->n_peers = 0; ctx->self_lsn = 0; ctx->self_tag = 0; pthread_mutex_init(&ctx->lock, NULL); sem_init(&ctx->done, 0, 0); atomic_store_explicit(&g_trf_c.ha.elect_ctx, (void *)ctx, memory_order_release); /* The crux of the reproduction: ha->phase2_done is 0 (fresh ha_init * above), so raft_cb_become_leader's is_first_boot block fires and calls * this hook -- which calls persist_recover() for real, from INSIDE this * same election evaluation, racing the eligibility gate exactly like * production. */ ha_phase2_hook = trf_phase2_hook_calls_persist_recover; raft_cb_become_leader_with_cluster_for_test(&g_trf_c, /*term=*/1); ha_phase2_hook = NULL; /* restore -- shared process-global test seam */ int32_t role_after_election = atomic_load_explicit(&g_trf_c.ha.role, memory_order_acquire); int in_catch_up_after_election = atomic_load_explicit(&g_trf_c.ha.in_catch_up, memory_order_acquire); bool recovery_truncated_after_election = persist_recovery_truncated(&g_trf_c.persist); recover_verdict_t recovery_verdict_after_election = persist_recovery_verdict(&g_trf_c.persist); atomic_store_explicit(&g_trf_c.ha.elect_ctx, NULL, memory_order_release); pthread_mutex_destroy(&ctx->lock); sem_destroy(&ctx->done); free(ctx); ha_set_promotion_fence_window_ms(0); /* restore default */ ha_cancel_promotion_fence_timer(&g_trf_c.ha); delaymgr_stop(&g_trf_c.delaymgr); delaymgr_fini(&g_trf_c.delaymgr); ha_fini(&g_trf_c.ha); persist_stop(&g_trf_c.persist); persist_fini(&g_trf_c.persist); trf_teardown_cluster(); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp); if (system(cmd) != 0) fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp); /* ── Assertions ───────────────────────────────────────────────────── */ TEST_ASSERT_FALSE_MESSAGE(recovery_truncated_precondition, "fixture precondition: a never-recovered persist_t must read " "recovery_truncated=false (the fresh-persist_init default) " "regardless of the real on-disk corruption -- this is the exact " "blind spot bug-1696 exploits"); TEST_ASSERT_EQUAL_INT_MESSAGE(RECOVER_CLEAN, recovery_verdict_precondition, "fixture precondition: the pre-recovery default verdict reads CLEAN, " "not the real corruption"); TEST_ASSERT_TRUE_MESSAGE(recovery_truncated_after_election, "this node's own first-ever replay (triggered from inside this SAME " "election evaluation) must have discovered and recorded the real " "on-disk corruption by the time this call returns"); TEST_ASSERT_EQUAL_INT_MESSAGE(RECOVER_TRUNCATED, recovery_verdict_after_election, "the real recovery verdict must be RECOVER_TRUNCATED once this " "node's own replay has actually run"); TEST_ASSERT_NOT_EQUAL_MESSAGE((int32_t)CTRL_PRIMARY, role_after_election, "bug-1696: a node whose OWN first-ever replay (racing this SAME " "election evaluation) discovers a genuinely CORRUPT/TRUNCATED WAL " "must be refused promotion -- not silently become CTRL_PRIMARY on " "an eligibility decision made blind, before that first replay ever " "ran"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, in_catch_up_after_election, "a node refused for this reason must be fenced via the SAME " "existing in_catch_up mechanism every other refusal reason uses, " "and stay ALIVE (never self-exit)"); printf("test_first_boot_promotion_refuses_when_own_first_recovery_finds_" "corruption: PASS\n"); } #endif /* XCE_TESTING */ void test_job_array_replay_fence(void) { UnitySetTestFile(__FILE__); #ifdef XCE_TESTING RUN_TEST(test_replay_wal_engages_fence_on_resource_exhaustion_failure); RUN_TEST(test_first_boot_promotion_evaluates_own_first_replay_fence); RUN_TEST(test_first_boot_promotion_never_truncates_own_durable_wal); RUN_TEST(test_first_boot_promotion_refuses_when_own_first_recovery_finds_corruption); #else printf("test_job_array_replay_fence: skipped (no XCE_TESTING)\n"); #endif }