/* test_job_array_replay_fence.c — job-array-single-wal-record task 7.2 * * Closes task 4.2's self-flagged residual gap: persist_apply_pump_run() (the * LIVE apply path, task 4.1/4.2) halts-and-fences on a negative * apply_committed_entry() return, but replay_wal() (the BOOT/RESTART REPLAY * path) previously only captured that return value for the job-count tally * (task 6.0a) — it never halted or engaged the Live Apply Resource Fence. * A node that crashed while fenced and restarted into a boot replay hitting * the IDENTICAL resource-exhaustion condition would silently pass it and * look fully promotable while actually divergent/behind. * * ── Scenario ("crash while fenced, restart into the same condition") ────── * Phase 1 ("boot 1"): a bare persist_t (no cluster) appends ONE real, * physical WAL_JOB_ARRAY_SUBMIT entry via persist_wal_async, then tears * down WITHOUT ever applying it — simulating a crash before the live apply * pump got a chance to run (e.g. the process died right after append, * before any apply kick). * * Phase 2 ("boot 2" / restart): a fresh cluster_t + its embedded persist_t * are pointed at the SAME on-disk workdir. The construction-phase * fault-injection seam (job_array_apply_test_set_force_fail_at — the SAME * seam task 4.4's capstone test uses, chosen for the identical reason: it * is the one seam Req 9.6's "no partial/divergent set of elements" claim * is provably true for, unlike the bulk-insert-phase seam) is armed BEFORE * persist_recover() runs, so replay_wal() hits the identical * resource-exhaustion condition during REPLAY, not live apply. * * ── Seam choice ───────────────────────────────────────────────────────── * See test_job_array_fence.c's header comment for the full rationale — * this test reuses the exact same construction-phase seam for the exact * same reason (the bulk-insert-phase seam's documented residual atomicity * gap would make a "no partial set" assertion dishonest here too). * * ── Fixture ───────────────────────────────────────────────────────────── * Mirrors test_replay_apply_seam.c's persist_recover()-driven fixture * (raft-apply-decoupling task 2.3) for phase 1/2, and test_job_array_fence.c's * tf_setup ha_init+delaymgr shape (task 4.4) for the promotion-gate * confirmation — reusing established patterns rather than inventing a new * fixture style. Deliberately OMITS the commit engine / repl standbys * test_job_array_fence.c wires: this test drives the physical WAL directly * via persist_wal_async (like test_replay_apply_seam.c), never through the * commit engine, so those collaborators are unnecessary here. * * IMPORTANT fixture detail: the persist_t used for phase 2 MUST be the one * EMBEDDED in the cluster_t (&g_trf_c.persist), not a standalone heap * persist_t — both raft_cb_become_leader_with_cluster_for_test(&g_trf_c, ...) * and ha_arm_promotion_fence_timer_for_test(&g_trf_c, ...) (and the * promotion_fence_timeout_fn the latter arms) read the fence state off * c->persist directly, so the replay under test and the promotion-gate * checks must observe the exact same persist_t instance. * * ── Remediation round 1 (task 7.2 adversarial review) ─────────────────── * The refusal proof below now drives raft_cb_become_leader_with_cluster_ * for_test() — the REAL election-time eligibility gate (ha.c:1303-1408) — * instead of ha_arm_promotion_fence_timer_for_test() (the periodic re-alert * timer, ha.c ~876), which was the WRONG gate for this test's actual claim * ("a node re-hits the identical condition on restart and is refused * promotion"). The periodic-timer seam is still used, correctly, for the * lighter-weight post-self-heal re-eligibility confirmation below. This * test still does NOT (and cannot, from persist.c alone) prove the node's * very FIRST-EVER promotion evaluation in a process's life is protected — * see the promotion-gate-confirmation block's comment for the precise * scope boundary and file:line citations. */ #include #include #include #include #include #include #include #include #include #include #include #include #include "cluster.h" #include "controller.capnp.h" #include "delay.h" #include "ha.h" #include "hook.h" #include "jobj.h" #include "persist.h" #include "wal_types.h" #include "xce/cluster_def.h" #include "xce/types.h" #include "xstring.h" #ifdef XCE_TESTING /* job.c's XCE_TESTING-only fault-injection seam (task 3.1) — reused * verbatim, not reimplemented. */ extern void job_array_apply_test_set_force_fail_at(int index); extern void job_array_apply_test_clear_force_fail(void); /* ha.c's XCE_TESTING-only seam (ha-commit-durability task 4.4) — drives the * exact bounded-window promotion-gate re-check without staging a full * election. Used below ONLY for the post-self-heal re-eligibility * confirmation (the periodic re-alert timer path, ha.c ~876) — NOT for the * refusal proof, which must use the real election-time gate below (task 7.2 * remediation round 1: this file previously misused this seam for the * refusal proof itself). */ extern void ha_arm_promotion_fence_timer_for_test(cluster_t *, bool, uint64_t, uint64_t); /* ha.c's XCE_TESTING-only seam (fix-promotion-standby-registration) — drives * raft_cb_become_leader() (the REAL election-time promotion path) with a * live cluster pointer, without staging a full election. This is the SAME * seam test_convergence_verification.c's * test_promotion_refused_when_apply_resource_fenced_even_if_eligible (task * 4.3) already uses to prove the election-time gate refuses promotion for a * fence engaged by the LIVE apply pump. This test drives it for a fence * engaged by BOOT REPLAY instead (task 7.2's actual deliverable) — task 7.2 * remediation round 1: the original version of this test proved refusal via * the WRONG gate (ha_arm_promotion_fence_timer_for_test's periodic re-alert * timer, ha.c ~876) instead of this one. */ extern void raft_cb_become_leader_with_cluster_for_test(cluster_t *, uint64_t); /* jobj.c's jobsub table / the process-global xstring table are shared * across every test file in the controller_unit_test binary — guard * idempotently, mirroring every sibling fixture in this suite. */ static int g_trf_jobobj_ready; static cluster_t g_trf_c; /* large — must not live on the test stack */ static int trf_run_delay_task(delay_task_entry_fn_t entry, void *arg) { return entry(arg); } static void trf_build_cluster(void) { kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16 }; kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING, .nslots = 32, .delta = 16 }; if (makeString((char *)"trf-probe") == NULL) xstring_table_init(64); if (!g_trf_jobobj_ready) { TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32)); g_trf_jobobj_ready = 1; } memset(&g_trf_c, 0, sizeof(g_trf_c)); g_trf_c.myhost = &g_trf_c.myhost_info; g_trf_c.myhost_info.name = (char *)"ctrl-test"; g_trf_c.myhost_info.key = (char *)""; TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&g_trf_c.hooksys, XCE_CONTROLLER_HOOK_MAX)); g_trf_c.jobmgr = makeInstance(&xkvmgr_type, &int_params); g_trf_c.qmgr = makeInstance(&xkvmgr_type, &str_params); g_trf_c.workermgr = makeInstance(&xkvmgr_type, &str_params); g_trf_c.resmgr = makeInstance(&xkvmgr_type, &str_params); TEST_ASSERT_NOT_NULL(g_trf_c.jobmgr); TEST_ASSERT_NOT_NULL(g_trf_c.qmgr); TEST_ASSERT_NOT_NULL(g_trf_c.workermgr); TEST_ASSERT_NOT_NULL(g_trf_c.resmgr); zarray_init(&g_trf_c.sharedinst, 16, NULL); } static void trf_teardown_cluster(void) { zarray_fini(&g_trf_c.sharedinst); if (g_trf_c.resmgr) dereference(g_trf_c.resmgr); if (g_trf_c.workermgr) dereference(g_trf_c.workermgr); if (g_trf_c.qmgr) dereference(g_trf_c.qmgr); if (g_trf_c.jobmgr) dereference(g_trf_c.jobmgr); xce_hook_fini(&g_trf_c.hooksys); } static job_t *trf_find_job(uint64_t jobid) { job_t *j = NULL; lock(g_trf_c.jobmgr, RTTI_LOCK_READ); int found = key_find(g_trf_c.jobmgr, (void *)jobid, (void **)&j); unlock(g_trf_c.jobmgr); return (found > 0) ? j : NULL; } /* Encodes a (base_jobid, array_size, spec) JobSubmit payload — the exact * WAL_JOB_ARRAY_SUBMIT payload shape (mirrors test_job_array_fence.c's * tf_encode_job_submit). Caller frees *out. */ static int trf_encode_job_submit(uint64_t base_jobid, uint64_t array_size, job_submit_data_t *spec, uint8_t **out, size_t *out_len) { struct capn c; struct capn_segment *cs; struct JobSubmit cjs; JobSubmit_ptr jsp; xce_job_submit_t s; memset(&s, 0, sizeof(s)); s.jobid = base_jobid; s.array_size = array_size; s.submit = spec; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_JobSubmit(cs, &cjs, &s); jsp = new_JobSubmit(cs); write_JobSubmit(&cjs, jsp); capn_setp(capn_root(&c), 0, jsp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } /* bug-1445 test-only stub: reproduces the REAL production call chain * (cluster_init_phase2 -> persist_recover -> replay_wal) at the REAL call * site inside raft_cb_become_leader's is_first_boot block, wired via * ha_phase2_hook (ha.h:1619) since XCE_TESTING builds skip the real * cluster_init_phase2() otherwise (ha.c:1431-1433). Deliberately calls ONLY * persist_recover -- the narrow slice bug-1445 is about -- not the rest of * cluster_init_phase2's other work (config load etc.), which this test does * not need. */ #ifdef XCE_TESTING static int trf_phase2_hook_calls_persist_recover(void *c) { cluster_t *cc = (cluster_t *)c; return persist_recover(&cc->persist); } #endif /* ── The test: crash-while-would-be-fenced, restart into a boot replay that * hits the SAME resource-exhaustion condition ────────────────────────────── */ static void test_replay_wal_engages_fence_on_resource_exhaustion_failure(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_trf_replay_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t base_jobid = 980000; const uint64_t array_size = 5; const int fail_index = 2; /* mid-array: proves a genuine halt, not a * lucky last-element edge case */ job_submit_data_t spec; memset(&spec, 0, sizeof(spec)); spec.queue = (char *)"normal"; spec.user = (char *)"tester"; spec.mintasks = 1; spec.maxtasks = 1; spec.job_name = (char *)"replay-fence-e2e"; uint8_t *payload = NULL; size_t len = 0; int enc_rc = trf_encode_job_submit(base_jobid, array_size, &spec, &payload, &len); /* ── Phase 1 ("boot 1"): append the real committed-shape array-submit * WAL entry physically to disk, then tear down WITHOUT ever applying * it -- simulating a crash before the live apply pump ran (e.g. the * process died moments after append, before any apply kick fired). No * cluster_t is needed: persist_wal_async is a physical-write-only path * (mirrors test_replay_apply_seam.c's phase 1). ─────────────────────── */ { persist_config_t cfg1 = persist_config_defaults(); cfg1.wal_sync_ms = 5000; persist_t *p1 = calloc(1, sizeof(persist_t)); TEST_ASSERT_NOT_NULL(p1); persist_set_workdir(p1, tmp, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(p1, &cfg1, NULL)); TEST_ASSERT_EQUAL_INT(0, persist_start(p1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p1, WAL_JOB_ARRAY_SUBMIT, payload, (uint32_t)len)); TEST_ASSERT_EQUAL_INT(0, persist_flush(p1)); persist_stop(p1); persist_fini(p1); free(p1); } /* ── Phase 2 ("boot 2" / restart): fresh cluster_t whose EMBEDDED * persist_t points at the SAME on-disk workdir. Arm the identical * construction-phase resource-exhaustion condition BEFORE * persist_recover() runs, so replay_wal() hits it during REPLAY. ────── */ trf_build_cluster(); persist_config_t cfg2 = persist_config_defaults(); cfg2.wal_sync_ms = 5000; persist_set_workdir(&g_trf_c.persist, tmp, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(&g_trf_c.persist, &cfg2, &g_trf_c)); TEST_ASSERT_EQUAL_INT(0, persist_start(&g_trf_c.persist)); job_array_apply_test_clear_force_fail(); job_array_apply_test_set_force_fail_at(fail_index); bool fenced_precondition = persist_apply_resource_fenced(&g_trf_c.persist); int recover_rc = persist_recover(&g_trf_c.persist); uint64_t applied_after_replay = persist_last_applied_lsn(&g_trf_c.persist); bool fenced_after_replay = persist_apply_resource_fenced(&g_trf_c.persist); bool any_element_visible_after_replay = false; for (uint64_t i = 0; i < array_size; i++) { if (trf_find_job(base_jobid + i) != NULL) any_element_visible_after_replay = true; } /* ── Promotion-gate confirmation ────────────────────────────────────── * task 7.2 remediation (round 1): drive the REAL election-time * eligibility gate (raft_cb_become_leader, ha.c:1303-1408 — reads * persist_apply_resource_fenced() at ha.c:1337) directly via * raft_cb_become_leader_with_cluster_for_test(), the SAME seam * test_convergence_verification.c's task-4.3 sibling test already uses * for a fence engaged by the LIVE apply pump. This is the gate an actual * RESTART's re-election evaluates, NOT the periodic re-alert timer used * below only for the post-self-heal confirmation. * * SCOPE NOTE (adversarial review finding, not fixed here — see ha.c * comment near the fence-engage branch in persist.c's replay_wal() for * the full citation): this only proves a promotion evaluation that runs * AFTER a boot replay has ALREADY completed and ALREADY engaged the * fence in this process — exactly what "boot 2" above is. It does NOT * prove the node's very FIRST-EVER promotion evaluation in a process's * life is protected: that check (ha.c:1303-1408) runs strictly BEFORE * the is_first_boot block (ha.c:1427-1439) that triggers * cluster_init_phase2()/persist_recover()/replay_wal() (cluster.c:1482) * for the very first time, with no re-check afterward — a fence engaged * by THAT SAME first replay is invisible to the one promotion check * gating that boot. Closing that gap needs an ha.c change, out of this * task's persist.c-only boundary. */ ha_config_t hcfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_EQUAL_INT(0, ha_init(&g_trf_c.ha, &hcfg, 0)); g_trf_c.ha.raft_c = &g_trf_c; g_trf_c.ha.repl = NULL; /* no healthy peer reachable at this level */ TEST_ASSERT_EQUAL_INT(0, delaymgr_init(&g_trf_c.delaymgr, 5, trf_run_delay_task)); TEST_ASSERT_EQUAL_INT(0, delaymgr_start(&g_trf_c.delaymgr)); /* Structurally healthy in every OTHER respect (not truncated, * durable_floor >= promotion_safe_lsn — no elect_ctx is set here so * promotion_safe_lsn defaults to 0 inside raft_cb_become_leader) so the * ONLY thing that can keep the member refused is the live apply * resource fence boot replay just engaged — same isolation rationale as * test_job_array_fence.c's step 3 and test_convergence_verification.c's * task 4.3 sibling test. */ g_trf_c.persist.recovery_verdict = RECOVER_CLEAN; ha_set_promotion_fence_window_ms(60); raft_cb_become_leader_with_cluster_for_test(&g_trf_c, /*term=*/1); int32_t role_after_election_check = atomic_load_explicit(&g_trf_c.ha.role, memory_order_acquire); int in_catch_up_after_election_check = atomic_load_explicit(&g_trf_c.ha.in_catch_up, memory_order_acquire); bool fenced_after_promotion_check = persist_apply_resource_fenced(&g_trf_c.persist); /* ── Self-heal: clear the forced condition and retry via the live apply * pump directly -- the SAME mechanism persist_recover()'s own * post-recovery kick would eventually invoke; driving it here directly * keeps this test deterministic (no vtask pool / async dispatch * needed). White-box "mark committed" mirrors test_job_array_fence.c's * own established convention for this exact fence area. ─────────────── */ job_array_apply_test_clear_force_fail(); atomic_store(&g_trf_c.persist.committed_lsn, 1); /* the one array entry */ int retry_rc = persist_apply_pump_run(&g_trf_c.persist); uint64_t applied_after_retry = persist_last_applied_lsn(&g_trf_c.persist); bool fenced_after_retry = persist_apply_resource_fenced(&g_trf_c.persist); bool all_elements_visible_after_retry = true; for (uint64_t i = 0; i < array_size; i++) { if (trf_find_job(base_jobid + i) == NULL) all_elements_visible_after_retry = false; } /* Post-self-heal re-eligibility confirmation uses the SIBLING periodic * re-alert timer gate (ha.c ~876) rather than re-driving * raft_cb_become_leader_with_cluster_for_test() a second time: both * gates read the exact same persist_apply_resource_fenced() flag this * fixture cleared above, so this is a legitimate, lighter-weight proof * that clearing the fence (regardless of which engaged it) restores * eligibility everywhere it is checked -- not a re-use of the "wrong" * gate this remediation round fixed above (that fix was specifically * about which gate proves the REFUSAL, not this self-heal check). */ ha_arm_promotion_fence_timer_for_test(&g_trf_c, /*truncated=*/false, /*durable_floor=*/0, /*hwm=*/0); usleep(250 * 1000); int in_catch_up_after_self_heal = atomic_load_explicit(&g_trf_c.ha.in_catch_up, memory_order_acquire); ha_set_promotion_fence_window_ms(0); /* restore default */ ha_cancel_promotion_fence_timer(&g_trf_c.ha); delaymgr_stop(&g_trf_c.delaymgr); delaymgr_fini(&g_trf_c.delaymgr); ha_fini(&g_trf_c.ha); job_array_apply_test_clear_force_fail(); free(payload); persist_stop(&g_trf_c.persist); persist_fini(&g_trf_c.persist); trf_teardown_cluster(); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp); if (system(cmd) != 0) fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp); /* ── Assertions ───────────────────────────────────────────────────── */ TEST_ASSERT_EQUAL_INT(0, enc_rc); TEST_ASSERT_FALSE_MESSAGE(fenced_precondition, "a fresh persist_t must start with the Live Apply Resource Fence " "CLEAR"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, recover_rc, "a resource-exhaustion apply failure during boot replay must NOT " "fail the boot itself (Req 9.4: never treat this as a reason to " "fail the commit)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, applied_after_replay, "last_applied_lsn must NOT advance past the failed array LSN during " "boot replay -- mirrors persist_apply_pump_run's live-path halt"); TEST_ASSERT_TRUE_MESSAGE(fenced_after_replay, "a node that crashes and restarts into a boot replay hitting the " "IDENTICAL resource-exhaustion condition must re-engage the Live " "Apply Resource Fence DURING REPLAY (task 4.2's self-flagged gap, " "closed by task 7.2) -- not silently boot clean and look promotable " "while actually divergent/behind"); TEST_ASSERT_FALSE_MESSAGE(any_element_visible_after_replay, "no partial/divergent set of array elements may be constructed " "during a fenced boot replay (Req 9.6)"); TEST_ASSERT_NOT_EQUAL_MESSAGE((int32_t)CTRL_PRIMARY, role_after_election_check, "a node fenced by BOOT REPLAY must be refused promotion by the REAL " "election-time eligibility gate (raft_cb_become_leader, ha.c:1303-" "1408) -- the same gate tasks 4.2/4.3 already wired for a live-apply-" "engaged fence, now proven for a boot-replay-engaged one too (task " "7.2 remediation round 1)"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, in_catch_up_after_election_check, "a node fenced by BOOT REPLAY must be refused promotion by the SAME " "existing ha.c gate (tasks 4.2/4.3) a live-apply fence already " "uses -- no separate mechanism, no drift"); TEST_ASSERT_TRUE_MESSAGE(fenced_after_promotion_check, "the fence must still read engaged after the promotion-gate check " "-- only the apply pump's own successful retry may clear it"); TEST_ASSERT_TRUE_MESSAGE(retry_rc >= 0, "once the forced condition clears, retrying the SAME stuck LSN via " "the live apply pump must succeed"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(1, applied_after_retry, "last_applied_lsn must advance to the array LSN once the retry " "succeeds"); TEST_ASSERT_TRUE_MESSAGE(all_elements_visible_after_retry, "every array element must be visible once the retry succeeds"); TEST_ASSERT_FALSE_MESSAGE(fenced_after_retry, "the Live Apply Resource Fence must self-clear once the retry " "succeeds (Req 9.5), regardless of whether it was engaged by boot " "replay or the live apply pump"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, in_catch_up_after_self_heal, "once the fence clears, the node must become eligible for " "promotion again with no other action"); printf("test_replay_wal_engages_fence_on_resource_exhaustion_failure: " "PASS\n"); } /* ── bug-1445: a node's very FIRST-EVER promotion evaluation races its OWN * first replay ───────────────────────────────────────────────────────────── * Unlike the sibling test above (which calls persist_recover() BEFORE the * election path, proving a LATER evaluation is protected), this test leaves * the persist_t completely un-recovered when raft_cb_become_leader_with_ * cluster_for_test() is invoked -- ha->phase2_done is fresh (0) and * ha_phase2_hook is wired to actually call persist_recover(), so * cluster_init_phase2()'s real call chain runs from INSIDE this SAME * election evaluation, exactly as it does in production * (raft_cb_become_leader, ha.c ~1427-1439). This is the exact scenario * bug-1445 names and the sibling test's own SCOPE NOTE explicitly says it * does not cover. */ static void test_first_boot_promotion_evaluates_own_first_replay_fence(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_trf_firstboot_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t base_jobid = 981000; const uint64_t array_size = 5; const int fail_index = 2; job_submit_data_t spec; memset(&spec, 0, sizeof(spec)); spec.queue = (char *)"normal"; spec.user = (char *)"tester"; spec.mintasks = 1; spec.maxtasks = 1; spec.job_name = (char *)"first-boot-fence"; uint8_t *payload = NULL; size_t len = 0; int enc_rc = trf_encode_job_submit(base_jobid, array_size, &spec, &payload, &len); /* Physically append ONE committed-shape array entry, then tear down * without ever applying it -- mirrors the sibling test's phase 1, * simulating a genuinely fresh, never-recovered on-disk WAL. */ { persist_config_t cfg1 = persist_config_defaults(); cfg1.wal_sync_ms = 5000; persist_t *p1 = calloc(1, sizeof(persist_t)); TEST_ASSERT_NOT_NULL(p1); persist_set_workdir(p1, tmp, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(p1, &cfg1, NULL)); TEST_ASSERT_EQUAL_INT(0, persist_start(p1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p1, WAL_JOB_ARRAY_SUBMIT, payload, (uint32_t)len)); TEST_ASSERT_EQUAL_INT(0, persist_flush(p1)); persist_stop(p1); persist_fini(p1); free(p1); } trf_build_cluster(); persist_config_t cfg2 = persist_config_defaults(); cfg2.wal_sync_ms = 5000; persist_set_workdir(&g_trf_c.persist, tmp, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(&g_trf_c.persist, &cfg2, &g_trf_c)); TEST_ASSERT_EQUAL_INT(0, persist_start(&g_trf_c.persist)); /* persist_init already defaults recovery_verdict=RECOVER_CLEAN and * apply_verdict=APPLY_CLEAN (persist.c:613,623) -- set explicitly anyway * to mirror the sibling test's defensive style, not because a fresh * persist_t needs it. */ g_trf_c.persist.recovery_verdict = RECOVER_CLEAN; job_array_apply_test_clear_force_fail(); job_array_apply_test_set_force_fail_at(fail_index); /* Precondition: this persist_t has NEVER been recovered -- the fence * must read clear, exactly the state the real eligibility gate * (ha.c:1337) sees at a process's very first election. This is the * blind spot bug-1445 exploits. */ bool fenced_precondition = persist_apply_resource_fenced(&g_trf_c.persist); ha_config_t hcfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_EQUAL_INT(0, ha_init(&g_trf_c.ha, &hcfg, 0)); g_trf_c.ha.raft_c = &g_trf_c; g_trf_c.ha.repl = NULL; /* no healthy peer reachable at this level */ TEST_ASSERT_EQUAL_INT(0, delaymgr_init(&g_trf_c.delaymgr, 5, trf_run_delay_task)); TEST_ASSERT_EQUAL_INT(0, delaymgr_start(&g_trf_c.delaymgr)); ha_set_promotion_fence_window_ms(60); /* The crux of the reproduction: ha->phase2_done is 0 (fresh ha_init * above), so raft_cb_become_leader's is_first_boot block fires and * calls this hook -- which calls persist_recover() for real, from * INSIDE this same election evaluation, racing the gate exactly like * production. */ ha_phase2_hook = trf_phase2_hook_calls_persist_recover; raft_cb_become_leader_with_cluster_for_test(&g_trf_c, /*term=*/1); ha_phase2_hook = NULL; /* restore -- shared process-global test seam */ int32_t role_after_election = atomic_load_explicit(&g_trf_c.ha.role, memory_order_acquire); int in_catch_up_after_election = atomic_load_explicit(&g_trf_c.ha.in_catch_up, memory_order_acquire); bool fenced_after_election = persist_apply_resource_fenced(&g_trf_c.persist); uint64_t applied_after_election = persist_last_applied_lsn(&g_trf_c.persist); bool any_element_visible = false; for (uint64_t i = 0; i < array_size; i++) { if (trf_find_job(base_jobid + i) != NULL) any_element_visible = true; } ha_set_promotion_fence_window_ms(0); /* restore default */ ha_cancel_promotion_fence_timer(&g_trf_c.ha); delaymgr_stop(&g_trf_c.delaymgr); delaymgr_fini(&g_trf_c.delaymgr); ha_fini(&g_trf_c.ha); job_array_apply_test_clear_force_fail(); free(payload); persist_stop(&g_trf_c.persist); persist_fini(&g_trf_c.persist); trf_teardown_cluster(); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp); if (system(cmd) != 0) fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp); /* ── Assertions ───────────────────────────────────────────────────── */ TEST_ASSERT_EQUAL_INT(0, enc_rc); TEST_ASSERT_FALSE_MESSAGE(fenced_precondition, "a never-recovered persist_t must start with the fence CLEAR -- " "this is the exact blind spot bug-1445 exploits"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, applied_after_election, "the first-ever replay (triggered from inside the election path) " "must halt at the failed LSN, same as any other halt-and-fence"); TEST_ASSERT_TRUE_MESSAGE(fenced_after_election, "the node's OWN first-ever replay, triggered from inside this SAME " "election evaluation, must have engaged the Live Apply Resource " "Fence by the time this call returns"); TEST_ASSERT_FALSE_MESSAGE(any_element_visible, "no partial/divergent set of array elements may be constructed " "(Req 9.6)"); TEST_ASSERT_NOT_EQUAL_MESSAGE((int32_t)CTRL_PRIMARY, role_after_election, "bug-1445: a node whose OWN first-ever replay (racing this SAME " "election evaluation) engages the Live Apply Resource Fence must " "be refused promotion -- not silently become CTRL_PRIMARY on a " "gate decision that was made before that first replay ever ran"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, in_catch_up_after_election, "a node refused for this reason must be fenced via the SAME " "existing in_catch_up mechanism every other refusal reason uses " "-- no separate mechanism, no drift"); printf("test_first_boot_promotion_evaluates_own_first_replay_fence: " "PASS\n"); } #endif /* XCE_TESTING */ void test_job_array_replay_fence(void) { UnitySetTestFile(__FILE__); #ifdef XCE_TESTING RUN_TEST(test_replay_wal_engages_fence_on_resource_exhaustion_failure); RUN_TEST(test_first_boot_promotion_evaluates_own_first_replay_fence); #else printf("test_job_array_replay_fence: skipped (no XCE_TESTING)\n"); #endif }