diff --git a/.kiro/specs/config-upgrade-snapshot/tasks.md b/.kiro/specs/config-upgrade-snapshot/tasks.md index dd5fef8d..02c455a3 100644 --- a/.kiro/specs/config-upgrade-snapshot/tasks.md +++ b/.kiro/specs/config-upgrade-snapshot/tasks.md @@ -61,7 +61,7 @@ - _Requirements: 1.8, 3.2, 3.4_ - _Boundary: Activation Fence (ha.c/ha.h)_ -- [ ] 6.2 Gate primary request-serving and promotion on the new fence +- [x] 6.2 Gate primary request-serving and promotion on the new fence - Extend the primary request-dispatch gate with this new fence check, composed alongside the existing configuration-byte fence rather than merged into it, admitting replication and read-only operations while fenced and refusing state-changing client requests - Extend the promotion gate the same way, so a member whose materialization is incomplete cannot become primary and immediately violate the serving gate the moment it wins an election - Introduce no new promotion-eligibility mechanism for WAL/log completeness — this fence is about materialization completeness only, layered on top of the existing Raft log-freshness election gate, not replacing or duplicating it @@ -149,6 +149,8 @@ ## Implementation Notes +- **Task 6.2 round-1 rejection (real bug, caught by review):** the first implementation wired the fence into both gates + a retry trampoline correctly, but `upgrade_materialization_fence` had no production path that could ever SET it from its zero-initialized default — every setter call site was gated on the fence already being set (closed loop). Fixed with two unconditional call sites: `persist.c`'s `apply_committed_entry` `WAL_CONFIG_MARKER` case now unconditionally calls `upgrade_materialization_verify(c)` right after `persist_checkpoint_marker_triggered` returns (regardless of its rc), and `myapp.c`'s boot sequence unconditionally calls it right after `config_activate_committed` (Req 5.2, crash/restart resume). New cross-module direction: `persist.c` now externs an `ha.c` function (mirrors the existing `persist.c → worker.c` extern pattern, no circular include). Lock-reentrancy re-verified safe (neither call chain touches `apply_pump_lock`). + - **Task 6.1 process note:** the Claude account's monthly API spend limit was hit mid-task (implementer subagent cut off after landing `ha.c`/`ha.h` but before tests/build). The orchestrator finished this task directly (no subagent implementer/reviewer pair) rather than stopping: verified the partial diff, wrote 4 tests into `test_config_repl.c` (reusing existing `t42_/t43_` fixtures rather than creating `test_upgrade_activation_fence.c` — that file is task 8.3's own boundary per design.md, not 6.1's), built, ran the full suite (1001 Tests 0 Failures 0 Ignored), and self-reviewed the diff in lieu of an independent reviewer subagent. Flagging this so a future session/reviewer knows this task's diff did not go through the normal two-subagent review gate. - **Task 5 correction to design.md:** design.md's "Marker Apply Trigger" section says "Call `persist_checkpoint()` directly" — this is unsafe as written. `persist_apply_pump_run` holds `apply_pump_lock` across the whole dispatch loop (including the `apply_committed_entry` call), and `persist_checkpoint()` itself re-acquires that same non-recursive mutex — a real self-deadlock on the live pump path (primary AND standby online-apply). Separately, `last_applied_lsn`/replay position only advance AFTER `apply_committed_entry` returns at both call sites, so pinning `checkpoint_lsn` from `last_applied_lsn` inside the marker case would mislabel the checkpoint with the previous entry's LSN, not the marker's own. Fixed via `persist_checkpoint_write_locked(p, checkpoint_lsn)` (no self-lock, explicit LSN param) + `persist_checkpoint_marker_triggered(p, lsn)`, called from `apply_committed_entry`'s new `WAL_CONFIG_MARKER` case with the marker's own `lsn` parameter. design.md should be updated to reflect this; not done as part of task 5 (docs-only, deferred). diff --git a/src/controller/ha.c b/src/controller/ha.c index 73444feb..27b2e58b 100644 --- a/src/controller/ha.c +++ b/src/controller/ha.c @@ -288,6 +288,124 @@ int upgrade_materialization_fenced(const cluster_t *c) memory_order_acquire) ? 1 : 0; } +/* ── config-upgrade-snapshot task 6.2: Activation Fence self-triggering retry ── + * Mirrors config_schedule_activate_retry's exact four-function shape + * (config_repl.c:409-507) for the upgrade_materialization_fence instead of + * config_serve_fence: a one-shot NON-BLOCKING delaymgr task offloads the + * actual blocking re-verify work (upgrade_materialization_verify's disk + * reads) onto a VTASK_LIVE_BLOCK vtask, so the delaymgr thread itself never + * blocks (ha-async-io.md thread-role contract). */ + +/* upgrade_materialization_retry_fn: re-runs upgrade_materialization_verify so + * a member whose materialization was incomplete or hash-mismatched re-checks + * and clears the fence once conditions allow. Does the actual blocking disk + * I/O; callers must invoke it off the delaymgr thread (see + * upgrade_materialization_retry_vtask_entry / + * schedule_upgrade_materialization_retry below). Public so tests can drive it + * directly without a vtask pool — mirrors config_activate_retry_fn. */ +int upgrade_materialization_retry_fn(void *arg) +{ + cluster_t *c = (cluster_t *)arg; + if (!c) + return -EINVAL; + if (!upgrade_materialization_fenced(c)) + return 0; /* already cleared by another path — nothing to retry */ + + int rc = upgrade_materialization_verify(c); + if (rc == 0) + INFO("upgrade materialization retry: fence CLEARED — this member's " + "own materialization for the committed config marker now " + "verified"); + else if (rc == -EAGAIN) + TRACE("upgrade materialization retry: still fenced (materialization " + "incomplete or hash-mismatched) — will retry on the next " + "commit advance"); + else + WARN("upgrade materialization retry: upgrade_materialization_verify " + "rc=%d", rc); + return rc; +} + +/* upgrade_materialization_retry_vtask_entry: vtask entry wrapping + * upgrade_materialization_retry_fn so its blocking disk I/O runs on a vtask + * worker, never inline on the single delaymgr thread. Mirrors + * config_activate_retry_vtask_entry. */ +static int upgrade_materialization_retry_vtask_entry(vtask_t *task) +{ + cluster_t *c = (cluster_t *)task->data->u.generic; + upgrade_materialization_retry_fn(c); + return 0; +} + +/* upgrade_materialization_retry_delaymgr_trampoline: the delaymgr task body. + * Delaymgr must only trigger + enqueue, never block — dispatch the actual + * retry work to a vtask (VTASK_LIVE_BLOCK, the same class + * config_activate_retry_delaymgr_trampoline uses) and return immediately. + * Fails loud and runs inline (stay alive, never silently drop a scheduled + * retry) if the pool is unavailable. Mirrors + * config_activate_retry_delaymgr_trampoline. */ +static int upgrade_materialization_retry_delaymgr_trampoline(void *arg) +{ + cluster_t *c = (cluster_t *)arg; + if (!c) + return -EINVAL; + + vtaskmgr_t *pool = &c->vtaskmgr; + vtask_data_t *data = calloc(1, sizeof(*data)); + vtask_t *task = vtask_alloc(); + + if (data == NULL || task == NULL) { + WARN("upgrade materialization retry: vtask alloc failed — running " + "inline (stay alive)"); + free(data); + free(task); + return upgrade_materialization_retry_fn(c); + } + + data->vtype = VTASK_GENERIC; + data->u.generic = c; + vtask_init(task, (vtask_func_t)upgrade_materialization_retry_vtask_entry, + data); + + if (vtask_enq_classified(pool, task, VTASK_LIVE_BLOCK) < 0) { + WARN("upgrade materialization retry: vtask enqueue failed — running " + "inline (stay alive)"); + free(data); + free(task); + return upgrade_materialization_retry_fn(c); + } + return 0; +} + +/* schedule_upgrade_materialization_retry: enqueue the retry trampoline on the + * delaymgr (NON-BLOCKING — delay_task_ms only does ordq_put; the trampoline + * itself offloads the blocking work to a vtask, see above). Called from the + * commit-advance path (iomgr ACK thread) so the retry's disk I/O runs OFF + * that thread (ha-async-io invariant). No-op when the member is NOT + * upgrade-materialization fenced (steady state schedules nothing) or when + * the delaymgr is not running (the unit-test fixture path, which never inits + * delaymgr — avoids leaking a task into an un-drained queue). Mirrors + * config_schedule_activate_retry exactly. Returns 0 on enqueue/no-op, + * negative on enqueue failure. */ +int schedule_upgrade_materialization_retry(cluster_t *c) +{ + if (!c) + return -EINVAL; + if (!upgrade_materialization_fenced(c)) + return 0; /* steady state: nothing to retry, schedule nothing */ + if (!c->delaymgr.running) + return 0; /* delaymgr not running (test fixture) — caller drives directly */ + + /* One-shot, minimal delay: run promptly but off the iomgr ACK thread. */ + int rc = delay_task_ms(&c->delaymgr, 1, 1, + (delay_task_entry_fn_t)upgrade_materialization_retry_delaymgr_trampoline, + c); + if (rc < 0) + WARN("upgrade materialization retry: delay_task_ms failed: %d — " + "fence will clear on a later commit advance", rc); + return rc; +} + static void on_vote_connected(endpoint_t *ep); static void on_pre_vote_connected(endpoint_t *ep); static int vote_ep_err_handler(void *ep_raw); @@ -1622,6 +1740,35 @@ static void raft_cb_become_leader(void *ctx, uint64_t term) return; /* do NOT become CTRL_PRIMARY; never self-exit */ } + /* ── config-upgrade-snapshot task 6.2 (Req 1.8, 3.2, 3.3, 4.3, 5.5): + * Activation Fence promotion gate ────────────────────────────────────── + * A member whose OWN upgrade-snapshot materialization for the current + * committed config marker has not completed and hash-verified must not + * become authoritative CTRL_PRIMARY — doing so would let it win an + * election (Raft's log-freshness gate only proves it has the committed + * MARKER, not that it has finished MATERIALIZING under it) and then + * immediately violate the serving gate above the moment it starts + * accepting requests. This is a SEPARATE, SEQUENTIAL check from the + * config serve-fence directly above (same rationale as the dispatch-gate + * composition: byte-mismatch and materialization-incomplete are distinct + * conditions) — never a new promotion-eligibility mechanism for WAL/log + * completeness (Req 3.1 is explicit that Raft's own freshness gate + * already owns that; this fence is layered on top, purely about + * materialization completeness). Refuse promotion and STAY ALIVE + * (controller-never-self-terminate): the member remains non-promotable + * and keeps retrying (schedule_upgrade_materialization_retry, driven off + * the commit-advance path); once its own materialization completes and + * re-verify clears the fence, a later promotion attempt passes this + * gate. Pure atomic read (no I/O) — safe on the election path. */ + if (upgrade_materialization_fenced(c)) { + WARN("HA: promotion REFUSED — upgrade materialization FENCED (this " + "member's own upgrade-snapshot materialization for the " + "committed config marker is incomplete or hash-mismatched); " + "member stays ALIVE, non-promotable, retrying materialization " + "(config-upgrade-snapshot Req 1.8/3.2/3.3/4.3/5.5)"); + return; /* do NOT become CTRL_PRIMARY; never self-exit */ + } + int is_first_boot = !ha->phase2_done; /* First boot: call cluster_init_phase2 */ @@ -2379,6 +2526,20 @@ static void ha_cb_commit_advanced_stub(void *ctx, uint64_t commit_index) * commit advance. */ if (config_serve_fenced(ha->raft_c)) config_schedule_activate_retry(ha->raft_c); + + /* config-upgrade-snapshot task 6.2 (Req 3.4, 5.5): symmetric commit- + * advance trigger for the Activation Fence. Materialization technically + * completes on APPLY (persist.c's WAL_CONFIG_MARKER case), which lags + * commit under raft-apply-decoupling — this hook does not need to fire + * at the PRECISE instant materialization becomes possible, only to + * eventually re-check and clear the fence once conditions allow (Req + * 3.4/5.5's "keep retrying" framing, not a precise event-driven + * guarantee). Commit advancing is a reasonable, precedented "something + * changed, worth re-checking" signal, and schedule_upgrade_materialization_ + * retry itself is a no-op when unfenced, so this adds no busy-spin in the + * steady state. */ + if (upgrade_materialization_fenced(ha->raft_c)) + schedule_upgrade_materialization_retry(ha->raft_c); } static const raft_callbacks_t s_real_cbs = { @@ -4031,6 +4192,48 @@ int ha_dispatch_gate(ha_state_t *ha, int opcode) return -XCE_ERR_NOT_READY; } } + /* config-upgrade-snapshot task 6.2 (Req 3.2, 3.3, 5.5): the Activation + * Fence is a SEPARATE, SEQUENTIAL check composed alongside the + * config-byte serve-fence above, not merged into it — the two + * conditions are logically distinct (byte-mismatch vs. this member's + * own materialization-incomplete) per design.md's Activation Fence + * component. Checked AFTER config_serve_fenced deliberately: byte + * delivery is the earlier step (the bytes must exist locally before + * this member's own checkpoint can even attempt to materialize + * against them), so checking byte-fence first surfaces the more + * fundamental blocker first in log/error ordering — but either fence + * alone is sufficient to refuse a state-changing op, so the final + * admit/refuse outcome does not depend on this ordering. Same + * allowlist as the byte-fence: replication + read-only/status ops + * stay admitted (they let materialization proceed and re-verify); + * everything else is refused with a loud non-serving error while the + * member STAYS ALIVE and keeps retrying (controller-never-self- + * terminate). */ + if (upgrade_materialization_fenced(ha->raft_c)) { + switch (opcode) { + /* Replication ops — let commit/apply advance + re-verify */ + case OPCode_heartbeat: + case OPCode_replConfig: + case OPCode_replConfigPush: + case OPCode_upgradeConfigAck: + case OPCode_hello: + case OPCode_walAck: + case OPCode_replEntry: + case OPCode_replAck: + case OPCode_replCatchUpDone: + case OPCode_walCatchupEntry: + case OPCode_snapshotTransfer: + /* Read-only queries / status — answerable while fenced */ + case OPCode_get: + case OPCode_probeConfigTag: + case OPCode_ctrlstatus: + break; /* fall through to the quorum check below */ + default: + /* State-changing op while this member's own upgrade-snapshot + * materialization is incomplete/unverified: refuse. */ + return -XCE_ERR_NOT_READY; + } + } /* State-changing operations require majority so committed entries * are durable across failover. Read-only queries and replication * ops are always allowed. */ diff --git a/src/controller/ha.h b/src/controller/ha.h index 88b218a2..7b98686e 100644 --- a/src/controller/ha.h +++ b/src/controller/ha.h @@ -828,6 +828,37 @@ int upgrade_materialization_verify(cluster_t *c); */ int upgrade_materialization_fenced(const cluster_t *c); +/* + * upgrade_materialization_retry_fn (config-upgrade-snapshot task 6.2, Req + * 3.4, 5.5): re-runs upgrade_materialization_verify so a member whose + * materialization was incomplete or hash-mismatched re-checks and clears the + * fence once conditions allow. Does the actual blocking disk I/O; production + * callers invoke it off the delaymgr thread via + * schedule_upgrade_materialization_retry's trampoline (below) — mirrors + * config_activate_retry_fn (config_repl.h). Public so tests can drive it + * directly without a vtask pool. `arg` is a `cluster_t *`. Returns 0 on + * success/already-unfenced, -EAGAIN while still fenced, negative errno on a + * genuine error. + */ +int upgrade_materialization_retry_fn(void *arg); + +/* + * schedule_upgrade_materialization_retry (config-upgrade-snapshot task 6.2, + * Req 3.4, 5.5): non-blocking, self-triggering retry for the Activation + * Fence — mirrors config_schedule_activate_retry's exact delaymgr- + * trampoline-then-vtask shape (config_repl.c:409-507): a one-shot delaymgr + * task (NON-BLOCKING — only enqueues) offloads the actual blocking re-verify + * work (upgrade_materialization_verify's disk reads) onto a VTASK_LIVE_BLOCK + * vtask so the delaymgr thread itself never blocks (ha-async-io.md + * thread-role contract). No-op when the member is NOT upgrade-materialization + * fenced (steady state schedules nothing — no busy-spin) or when the + * delaymgr is not running (unit-test fixture path; caller drives the retry + * directly instead). Falls back to running the verify inline (stay alive, + * never silently drop a scheduled retry) if the vtask pool is unavailable. + * Returns 0 on enqueue/no-op, negative on enqueue failure. + */ +int schedule_upgrade_materialization_retry(cluster_t *c); + /* * ha_apply_wal_catchup_entry: apply a single WAL catch-up entry to local * persistent storage and advance ha->applied_lsn on success. diff --git a/src/controller/myapp.c b/src/controller/myapp.c index cf214aed..712dc62d 100644 --- a/src/controller/myapp.c +++ b/src/controller/myapp.c @@ -251,6 +251,39 @@ int myapp_run(myconf_t *myconf) } /* Ignore other errors — node may have no config yet (first boot/standby); * it fetches config via the primary's push post-election (Req 7.3). */ + + /* config-upgrade-snapshot task 6.2 (Req 5.2): resume Activation-Fence + * verification here too, unconditionally, so a crash/restart resumes + * materialization-verification before serving rather than implicitly + * treating a fresh process's zero-initialized (unfenced) fence as + * "already verified". Sequenced AFTER config_activate_committed + * deliberately: that call is what establishes/repairs which config + * BYTES are locally active for the committed marker; only once that + * has run does the separate, sequential question + * upgrade_materialization_verify asks -- has THIS MEMBER'S OWN + * checkpoint machinery actually materialized under that marker? -- + * become meaningful to (re-)evaluate. persist_recover() (step 6, + * above) has already restored c.persist.cfgref_* from the latest + * loaded checkpoint by this point, so the materialization signal + * this reads is accurate here, at boot, same as it would be after + * any later checkpoint. Same "ignore other errors, log loud only on + * the fenced case" treatment as config_activate_committed above -- + * ha_dispatch_gate and the promotion gate (ha.c) are what actually + * enforce the fence from here on; this call's only job is to seed + * the correct fence value at boot instead of leaving a freshly- + * started process at its unfenced zero-default regardless of + * whether materialization genuinely completed before the crash. */ + int mat_rc = upgrade_materialization_verify(&c); + if (mat_rc == -EAGAIN) { + ERROR("Pre-election upgrade materialization FENCED: this " + "member's own upgrade-snapshot materialization for the " + "committed config marker is incomplete or hash-mismatched " + "— member NON-SERVING and NON-PROMOTABLE, staying alive " + "and retrying (config-upgrade-snapshot R1.8/3.2/3.4/5.2)"); + } + /* Ignore other errors/success silently — mirrors the "ignore other + * errors" handling above; ha_dispatch_gate / the promotion gate + * enforce the fence going forward. */ } /* ── Step 8: ha_start_election_async ─────────────────────────────────── */ diff --git a/src/controller/persist.c b/src/controller/persist.c index b2fb8aa1..5238aaa6 100644 --- a/src/controller/persist.c +++ b/src/controller/persist.c @@ -42,6 +42,19 @@ extern int restore_queue_state(cluster_t* c, * from load_snapshot_entities' host-frame loop below (task 4). */ extern int restore_host_state(cluster_t* c, const char* name, uint32_t admin_status); +/* upgrade_materialization_verify defined in ha.c (config-upgrade-snapshot + * task 6.1/6.2, Req 1.8/3.2/3.4/5.2/5.5) -- the Activation Fence's own + * verify function. Called unconditionally from apply_committed_entry's + * WAL_CONFIG_MARKER case below (task 6.2): checkpoint completion is the + * design-intended trigger that re-evaluates this fence (design.md's + * Activation Fence component, "naturally re-evaluates against the newer + * marker once that checkpoint's write_snapshot call runs"), since it is + * the ONLY event that can newly set OR clear it from a running process -- + * every other call path (upgrade_materialization_retry_fn and its + * callers) is itself gated on the fence already being set, so without + * this call site the fence could never transition from its zero- + * initialized default. */ +extern int upgrade_materialization_verify(cluster_t* c); /* raft-apply-decoupling task 3.2: pending command-apply hand-off list node * (producer side; the not-yet-built ordered apply pump, task 6.1/6.2, is @@ -3780,9 +3793,7 @@ int apply_committed_entry(persist_t* p, uint32_t wal_type, uint64_t lsn, TRACE("apply_committed_entry: WAL_CONFIG_MARKER lsn=%" PRIu64 " checkpoint already in progress -- skipping, " "self-resolving", lsn); - return 0; - } - if (rc != 0) { + } else if (rc != 0) { /* A genuine checkpoint failure (I/O error, etc.) -- propagate * so the caller's existing halt-and-retry-from-same-position * handling engages (persist_apply_pump_run / replay_wal), @@ -3790,12 +3801,47 @@ int apply_committed_entry(persist_t* p, uint32_t wal_type, uint64_t lsn, * dispatch. last_applied_lsn is NOT advanced past this marker, * so the next kick/retry re-attempts the checkpoint fresh * (Req 5.2 crash-safety: resume materialization from the - * marker). */ + * marker). upgrade_materialization_verify below still runs + * even on this path (it reads its own signals -- the + * committed marker + c->persist.cfgref_* -- independently of + * this rc; a failed checkpoint attempt here does not by + * itself change whether a PRIOR checkpoint already + * materialized under the current marker). */ ERROR("apply_committed_entry: WAL_CONFIG_MARKER lsn=%" PRIu64 " forced checkpoint failed (rc=%d)", lsn, rc); - return rc; } - return 0; + + /* config-upgrade-snapshot task 6.2 (Req 1.8, 3.2, 3.4, 5.5): + * unconditional Activation Fence re-evaluation right after every + * checkpoint attempt tied to this marker, regardless of whether + * persist_checkpoint_marker_triggered just above returned 0, + * -EBUSY, or a genuine error -- upgrade_materialization_verify + * itself already handles "materialization not done yet" correctly + * (it fences and returns -EAGAIN rather than erroring), so gating + * this call on rc would only reintroduce the closed loop this call + * site exists to break (every OTHER caller of + * upgrade_materialization_verify is itself gated on the fence + * already being set -- see upgrade_materialization_retry_fn and + * its callers in ha.c -- so without one unconditional trigger the + * fence could never transition from its zero-initialized default). + * Checkpoint completion is the design-intended trigger for this + * re-evaluation (design.md's Activation Fence component). Safe to + * call here: neither upgrade_materialization_verify nor anything + * it calls (config_committed_marker, conf_read_current_filename, + * conf_read_raw) acquires apply_pump_lock or p->apply_pump_lock -- + * they only do atomic reads of c->persist.committed_lsn / + * c->persist.cfgref_* and plain file I/O (opendir/pread-style WAL + * scan, conf_read_raw) -- so no reentrant-lock hazard even though + * this call runs while persist_apply_pump_run holds + * apply_pump_lock across this entire dispatch (same hazard class + * as task 5's own persist_checkpoint_marker_triggered, avoided the + * same way: never touch apply_pump_lock). Return value + * intentionally ignored here -- ha_dispatch_gate and the + * promotion gate (ha.c) are what actually consume the fence; + * this call's only job is to keep it current. */ + (void)upgrade_materialization_verify(c); + + return (rc == -EBUSY) ? 0 : rc; } default: diff --git a/src/controller/testsuite/test_config_repl.c b/src/controller/testsuite/test_config_repl.c index 542fa900..6cf4b274 100644 --- a/src/controller/testsuite/test_config_repl.c +++ b/src/controller/testsuite/test_config_repl.c @@ -1687,6 +1687,29 @@ static void test_serve_fence_enforced_by_dispatch_gate_and_promotion(void) TEST_ASSERT_EQUAL_INT(0, config_activate_committed(c)); TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c)); TEST_ASSERT_EQUAL_HEX64(ttag, mc->config_version_tag); + + /* config-upgrade-snapshot task 6.2 remediation: t43_commit_marker above + * drives the TARGET marker through the REAL apply path, which (as of + * the 6.2 fix) makes apply_committed_entry's WAL_CONFIG_MARKER case + * (persist.c) unconditionally re-evaluate the separate, independent + * upgrade_materialization_fence too -- not just config_serve_fence, + * which is all this test is otherwise about. At that moment the TARGET + * bytes had not arrived locally yet (this test's own deliberate setup, + * to exercise config_serve_fence), so that checkpoint's + * scan_latest_committed_config_ref already stamped cfgref_{tag,sha} + * from the marker itself (materialized-reference present), but the + * live-bytes hash-check genuinely failed (no local bytes yet) -- + * correctly setting upgrade_materialization_fence too, per Req 3.2. + * That is orthogonal, correct behavior, not a bug in either fence, but + * this test never previously accounted for it (the fence was + * unreachable before the 6.2 fix). Re-verify it here now that the + * correct bytes are locally present, mirroring config_activate_ + * committed's own re-verify just above for config_serve_fence, so the + * remaining assertions below exercise only the fence this test is + * actually about. */ + TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_verify(c)); + TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c)); + TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&c->ha, OPCode_jobsub)); TEST_ASSERT_EQUAL_INT(0, ha_config_promotion_fenced(&c->ha)); @@ -1857,6 +1880,455 @@ static void test_commit_advance_retry_clears_push_before_commit_fence(void) printf("test_commit_advance_retry_clears_push_before_commit_fence: PASS\n"); } +/* ── config-upgrade-snapshot task 6.2: Activation Fence gate composition + + * self-triggering retry (upgrade_materialization_fenced / ha_dispatch_gate / + * raft_cb_become_leader / schedule_upgrade_materialization_retry, ha.c) ──── + * + * These tests reuse this file's own t42_/t43_ fixture helpers and mirror + * the existing (4.3-C/D) config_serve_fence gate-composition/retry tests + * above as the direct template, for the new independent + * upgrade_materialization_fence instead. + */ + +/* raft_cb_become_leader is file-static; drive it through the LIVE-cluster + * XCE_TESTING seam (fix-promotion-standby-registration) so this test proves + * the REAL election-time promotion decision refuses while fenced, not just a + * proxy accessor read. */ +extern void raft_cb_become_leader_with_cluster_for_test(cluster_t *c, + uint64_t term); +/* ha-term-safety task 2.4: drive the WIRED on_commit_advanced callback + * (ha_cb_commit_advanced_stub) so the wiring test below exercises the real + * commit-advance trigger, not a hand-rolled call. */ +extern void ha_cb_commit_advanced_for_test(ha_state_t *ha, + uint64_t commit_index); + +/* (6.2-a) UPGRADE MATERIALIZATION FENCE IS ENFORCED by two production + * readers: (1) ha_dispatch_gate refuses a config-dependent state-changing + * operation on the PRIMARY while fenced (loud non-serving, member stays + * alive) but keeps replication/read-only ops flowing so materialization can + * proceed and re-verify; (2) a REAL election-time promotion attempt + * (raft_cb_become_leader_with_cluster_for_test — the actual gate, not a proxy + * accessor) refuses to become CTRL_PRIMARY while this member's own + * materialization for the current committed marker is incomplete. Regression + * for the dispatch-gate/promotion-gate composition (Req 3.2, 3.3, 4.3, 5.5). + */ +static void test_upgrade_materialization_fence_enforced_by_dispatch_gate_and_promotion(void) +{ + char tmp[256]; + strcpy(tmp, "/tmp/xce_cr62_fence_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); + + cluster_t *c = t42_make_cluster(tmp); + TEST_ASSERT_NOT_NULL(c); + t42_make_leader(c, /*term*/40, /*n_peers*/1); + TEST_ASSERT_EQUAL_INT(0, + commit_engine_init(&c->commit, &c->persist, 1, 8)); + repl_set_commit_engine(&c->persist.repl, &c->commit); + myconf_t *mc = t43_attach_myconf(c, tmp); + + /* Commit a marker but never checkpoint against it -- this member's own + * materialization is genuinely incomplete for it (identical setup to + * test_upgrade_materialization_verify_matching_materialization_clears_fence's + * sibling incomplete-materialization test, just without the -EBUSY + * choreography -- a fresh marker with no checkpoint yet run is already + * incomplete by construction). */ + size_t blen = 0; + uint8_t *bytes = t43_make_config_blob(&blen); + uint8_t sha[32]; + SHA256(bytes, blen, sha); + uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); } + TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha)); + free(bytes); + + /* Force materialization to stay incomplete for THIS marker: hold the + * shared checkpoint_in_progress CAS open across the commit (the same + * mechanism test_upgrade_materialization_verify_incomplete_materialization_ + * fences uses), so the marker-apply checkpoint hits -EBUSY and cfgref_* + * never advances to it. */ + TEST_ASSERT_EQUAL_INT( + 0, atomic_exchange_explicit(&c->persist.checkpoint_in_progress, 1, + memory_order_acq_rel)); + uint64_t mlsn = 0; + TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &mlsn)); + TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); + TEST_ASSERT_EQUAL_INT(0, + commit_submit(&c->commit, mlsn, COMMIT_MODE_WAIT, 1000)); + uint64_t deadline_iters = 200; + while (deadline_iters-- > 0 && + atomic_load_explicit(&c->persist.last_applied_lsn, + memory_order_acquire) < mlsn) { + struct timespec ts = { .tv_sec = 0, .tv_nsec = 10 * 1000 * 1000 }; + nanosleep(&ts, NULL); + } + atomic_store_explicit(&c->persist.checkpoint_in_progress, 0, + memory_order_release); + + TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_verify(c)); + TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); + + /* PRODUCTION READER 1: ha_dispatch_gate refuses a state-changing op on + * the fenced PRIMARY but stays alive (error, not crash/exit). */ + TEST_ASSERT_TRUE(ha_dispatch_gate(&c->ha, OPCode_jobsub) < 0); + /* Replication / read-only ops still pass. */ + TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&c->ha, OPCode_heartbeat)); + TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&c->ha, OPCode_get)); + + /* PRODUCTION READER 2: a real election-time promotion attempt refuses to + * become CTRL_PRIMARY while fenced. ha_primary_log_append above only + * required c->ha.raft.role == RAFT_LEADER (set by t42_make_leader), not + * ha->role == CTRL_PRIMARY, so resetting ha->role to CTRL_ELECTING here + * to observe a genuine promotion decision does not disturb the marker + * already committed above. */ + c->ha.phase2_done = 1; /* skip cluster_init_phase2 -- unrelated first-boot path */ + atomic_store_explicit(&c->ha.role, (int32_t)CTRL_ELECTING, + memory_order_release); + raft_cb_become_leader_with_cluster_for_test(c, /*term*/40); + TEST_ASSERT_NOT_EQUAL_MESSAGE( + (int32_t)CTRL_PRIMARY, + atomic_load_explicit(&c->ha.role, memory_order_acquire), + "config-upgrade-snapshot task 6.2 (Req 3.2/3.3/4.3/5.5): a member " + "whose own upgrade-snapshot materialization is incomplete must NOT " + "be promoted to CTRL_PRIMARY"); + TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); /* unaffected */ + + t43_free_myconf(c); + commit_engine_fini(&c->commit); + t42_teardown(c); + printf("test_upgrade_materialization_fence_enforced_by_dispatch_gate_and_promotion: PASS\n"); +} + +/* (6.2-b) schedule_upgrade_materialization_retry is a no-op when NOT fenced + * -- steady state schedules nothing (no busy-spin), mirroring + * config_schedule_activate_retry's own documented no-op behavior. */ +static void test_schedule_upgrade_materialization_retry_noop_when_not_fenced(void) +{ + char tmp[256]; + strcpy(tmp, "/tmp/xce_cr62_retry_noop_unfenced_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); + + cluster_t *c = t42_make_cluster(tmp); + TEST_ASSERT_NOT_NULL(c); + t42_make_leader(c, /*term*/41, /*n_peers*/1); + + TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c)); /* fixture default: unfenced */ + TEST_ASSERT_FALSE(c->delaymgr.running); /* fixture never starts delaymgr */ + + TEST_ASSERT_EQUAL_INT(0, schedule_upgrade_materialization_retry(c)); + + t42_teardown(c); + printf("test_schedule_upgrade_materialization_retry_noop_when_not_fenced: PASS\n"); +} + +/* (6.2-c) schedule_upgrade_materialization_retry is a no-op when the + * delaymgr is not running, even while fenced -- the unit-test fixture path + * (mirrors config_schedule_activate_retry's own documented delaymgr-not- + * running no-op; test_commit_advance_retry_clears_push_before_commit_fence + * relies on this same fixture property to drive its retry entry directly). */ +static void test_schedule_upgrade_materialization_retry_noop_when_delaymgr_not_running(void) +{ + char tmp[256]; + strcpy(tmp, "/tmp/xce_cr62_retry_noop_nodmgr_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); + + cluster_t *c = t42_make_cluster(tmp); + TEST_ASSERT_NOT_NULL(c); + t42_make_leader(c, /*term*/42, /*n_peers*/1); + TEST_ASSERT_EQUAL_INT(0, + commit_engine_init(&c->commit, &c->persist, 1, 8)); + repl_set_commit_engine(&c->persist.repl, &c->commit); + myconf_t *mc = t43_attach_myconf(c, tmp); + + size_t blen = 0; + uint8_t *bytes = t43_make_config_blob(&blen); + uint8_t sha[32]; + SHA256(bytes, blen, sha); + uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); } + TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha)); + free(bytes); + t43_commit_marker(c, tag, sha); + + /* Overwrite the local bytes without a new marker committing -- the same + * "stale local bytes" fence trigger + * test_upgrade_materialization_verify_stale_local_bytes_after_ + * materialization_fences uses -- so the fence is genuinely SET without + * needing the -EBUSY checkpoint choreography. */ + size_t blen2 = 0; + uint8_t *bytes2 = t43_make_config_blob(&blen2); + bytes2[blen2 - 1] ^= 0xFF; + uint8_t sha2[32]; + SHA256(bytes2, blen2, sha2); + TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes2, blen2, sha2)); + free(bytes2); + + TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_verify(c)); + TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); + TEST_ASSERT_FALSE(c->delaymgr.running); + + /* delaymgr not running -- no-op, returns 0, does not crash/leak a task. */ + TEST_ASSERT_EQUAL_INT(0, schedule_upgrade_materialization_retry(c)); + TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); /* still fenced -- untouched */ + + t43_free_myconf(c); + commit_engine_fini(&c->commit); + t42_teardown(c); + printf("test_schedule_upgrade_materialization_retry_noop_when_delaymgr_not_running: PASS\n"); +} + +/* (6.2-d) upgrade_materialization_retry_fn -- the same off-thread entry + * point schedule_upgrade_materialization_retry's trampoline would enqueue in + * production -- stays fenced (-EAGAIN) while the genuine mismatch persists, + * then CLEARS the fence once the live bytes are corrected to match the + * committed marker, without a new marker needing to commit. Mirrors + * config_activate_retry_fn's own direct-drive regression + * (test_commit_advance_retry_clears_push_before_commit_fence). */ +static void test_upgrade_materialization_retry_fn_recovers_from_stale_bytes_fence(void) +{ + char tmp[256]; + strcpy(tmp, "/tmp/xce_cr62_retry_fn_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); + + cluster_t *c = t42_make_cluster(tmp); + TEST_ASSERT_NOT_NULL(c); + t42_make_leader(c, /*term*/43, /*n_peers*/1); + TEST_ASSERT_EQUAL_INT(0, + commit_engine_init(&c->commit, &c->persist, 1, 8)); + repl_set_commit_engine(&c->persist.repl, &c->commit); + myconf_t *mc = t43_attach_myconf(c, tmp); + + size_t blen = 0; + uint8_t *bytes = t43_make_config_blob(&blen); + uint8_t sha[32]; + SHA256(bytes, blen, sha); + uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); } + TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha)); + t43_commit_marker(c, tag, sha); + TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_verify(c)); + TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c)); + + /* Local bytes are overwritten with DIFFERENT content, without a new + * marker committing -- cfgref_* still shows the old matching reference, + * but a fresh live re-read must fence (same trap 6.1-d proves for + * upgrade_materialization_verify itself). */ + size_t blenX = 0; + uint8_t *bytesX = t43_make_config_blob(&blenX); + bytesX[blenX - 1] ^= 0xA5; + uint8_t shaX[32]; + SHA256(bytesX, blenX, shaX); + TEST_ASSERT_FALSE(memcmp(sha, shaX, 32) == 0); + TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytesX, blenX, shaX)); + free(bytesX); + TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_verify(c)); + TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); + + /* Drive the retry entry directly while the mismatch is still genuine: + * stays fenced, does not crash/toggle incorrectly. */ + TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_retry_fn(c)); + TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); + + /* Restore the CORRECT bytes (matching the still-current committed + * marker) without any new marker commit -- models the out-of-band + * config-push bytes finally arriving/correcting. */ + TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha)); + free(bytes); + + /* Drive the retry entry again: re-verify now succeeds -- fence CLEARS. */ + TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_retry_fn(c)); + TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c)); + + t43_free_myconf(c); + commit_engine_fini(&c->commit); + t42_teardown(c); + printf("test_upgrade_materialization_retry_fn_recovers_from_stale_bytes_fence: PASS\n"); +} + +/* (6.2-e) WIRING: the commit-advance path (ha_cb_commit_advanced_stub, the + * WIRED on_commit_advanced callback) consults upgrade_materialization_fenced + * and calls schedule_upgrade_materialization_retry when fenced -- driven + * through the REAL wired callback (ha_cb_commit_advanced_for_test), not a + * hand-rolled call, so this proves the actual call-site wiring executes + * without crashing/corrupting state. schedule_upgrade_materialization_retry + * itself is a no-op in this fixture (delaymgr not running -- proven in + * isolation by 6.2-c above), so the fence is expected to remain SET + * afterward; this test's contribution is exercising the NEW branch at the + * real call site itself, not re-proving the no-op contract. */ +static void test_commit_advance_wiring_calls_upgrade_materialization_schedule_retry(void) +{ + char tmp[256]; + strcpy(tmp, "/tmp/xce_cr62_wiring_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); + + cluster_t *c = t42_make_cluster(tmp); + TEST_ASSERT_NOT_NULL(c); + t42_make_leader(c, /*term*/44, /*n_peers*/1); + TEST_ASSERT_EQUAL_INT(0, + commit_engine_init(&c->commit, &c->persist, 1, 8)); + repl_set_commit_engine(&c->persist.repl, &c->commit); + myconf_t *mc = t43_attach_myconf(c, tmp); + + size_t blen = 0; + uint8_t *bytes = t43_make_config_blob(&blen); + uint8_t sha[32]; + SHA256(bytes, blen, sha); + uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); } + TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha)); + free(bytes); + t43_commit_marker(c, tag, sha); + TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_verify(c)); + TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c)); + + /* Overwrite local bytes (stale-bytes trigger, same as 6.2-c) to fence + * without a new marker. */ + size_t blen2 = 0; + uint8_t *bytes2 = t43_make_config_blob(&blen2); + bytes2[blen2 - 1] ^= 0xFF; + uint8_t sha2[32]; + SHA256(bytes2, blen2, sha2); + TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes2, blen2, sha2)); + free(bytes2); + TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_verify(c)); + TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); + + uint64_t committed_lsn = + atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire); + + /* Drive the REAL wired commit-advance callback -- must not crash, must + * not corrupt the fence state (schedule_upgrade_materialization_retry's + * own no-op-when-delaymgr-not-running contract keeps it SET). */ + ha_cb_commit_advanced_for_test(&c->ha, committed_lsn); + TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); + + t43_free_myconf(c); + commit_engine_fini(&c->commit); + t42_teardown(c); + printf("test_commit_advance_wiring_calls_upgrade_materialization_schedule_retry: PASS\n"); +} + +/* ── config-upgrade-snapshot task 6.2 remediation: Activation Fence set by a + * REAL production trigger, not a direct test call ────────────────────────── + * + * Every test above (6.2-a..e, and 6.1's own 4 tests earlier in this file) + * reaches the fenced state by calling upgrade_materialization_verify() + * directly in the test body. That proves the fence's own logic is correct, + * but NOT that a running production controller can ever reach the fenced + * state in the first place: every PRODUCTION call path to + * upgrade_materialization_verify() -- upgrade_materialization_retry_fn, and + * both of ITS callers (schedule_upgrade_materialization_retry and the + * commit-advance hook in ha_cb_commit_advanced_stub) -- is itself gated on + * upgrade_materialization_fenced() already being true. Starting from the + * fence's zero-initialized (unfenced) default, that is a closed loop with + * no entry point: nothing in production could ever set the fence for the + * first time, which made the dispatch-gate/promotion-gate checks in 6.2-a + * permanently dead code against a real controller. + * + * The actual fix (persist.c's apply_committed_entry, WAL_CONFIG_MARKER + * case) calls upgrade_materialization_verify() UNCONDITIONALLY right after + * persist_checkpoint_marker_triggered() returns, regardless of that call's + * own rc -- checkpoint completion (or a self-resolving skip) is the + * design-intended re-evaluation trigger (design.md's Activation Fence + * component). This test reaches the fenced state purely through that real + * seam -- config_append_marker + commit_submit + wait-for-applied, with + * persist_checkpoint_marker_triggered() forced to miss via the exact + * checkpoint_in_progress technique test_upgrade_materialization_verify_ + * incomplete_materialization_fences (6.1-c) and test_upgrade_materialization + * _fence_enforced_by_dispatch_gate_and_promotion (6.2-a) use -- and NEVER + * calls upgrade_materialization_verify() itself. */ +static void test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss(void) +{ + char tmp[256]; + strcpy(tmp, "/tmp/xce_cr62_real_trigger_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); + + cluster_t *c = t42_make_cluster(tmp); + TEST_ASSERT_NOT_NULL(c); + t42_make_leader(c, /*term*/45, /*n_peers*/1); + TEST_ASSERT_EQUAL_INT(0, + commit_engine_init(&c->commit, &c->persist, 1, 8)); + repl_set_commit_engine(&c->persist.repl, &c->commit); + myconf_t *mc = t43_attach_myconf(c, tmp); + + /* Fixture default: unfenced (fresh cluster_t, zero-initialized atomic) + * -- the exact "closed loop, no entry point" starting condition the + * rejection identified. This test body never calls + * upgrade_materialization_verify() anywhere below -- the fence must be + * set purely by the real apply-time trigger. */ + TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c)); + + size_t blen = 0; + uint8_t *bytes = t43_make_config_blob(&blen); + uint8_t sha[32]; + SHA256(bytes, blen, sha); + uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); } + TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha)); + free(bytes); + + /* Force materialization to genuinely miss for THIS marker: hold the + * shared checkpoint_in_progress CAS open across the commit, so + * persist_checkpoint_marker_triggered() (called from + * apply_committed_entry's WAL_CONFIG_MARKER case) hits the real -EBUSY + * path (Req 4.4) -- self-resolving, not a failure, so apply itself + * still succeeds and last_applied_lsn still advances. */ + TEST_ASSERT_EQUAL_INT( + 0, atomic_exchange_explicit(&c->persist.checkpoint_in_progress, 1, + memory_order_acq_rel)); + uint64_t mlsn = 0; + TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &mlsn)); + TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); + TEST_ASSERT_EQUAL_INT(0, + commit_submit(&c->commit, mlsn, COMMIT_MODE_WAIT, 1000)); + + /* Wait for the REAL apply_committed_entry() dispatch to finish for this + * marker. persist_apply_pump_run advances last_applied_lsn only AFTER + * that dispatch call returns (see apply_committed_entry's own + * WAL_CONFIG_MARKER case doc comment), so by the time + * t42_wait_applied observes last_applied_lsn >= mlsn, the new + * unconditional upgrade_materialization_verify(c) call at the end of + * that case has already run synchronously on this same call stack. */ + t42_wait_applied(&c->persist, mlsn, 2000); + TEST_ASSERT_TRUE(persist_last_applied_lsn(&c->persist) >= mlsn); + + /* Release the simulated in-flight checkpoint now that the marker's own + * apply (and its -EBUSY'd checkpoint attempt) has already happened. */ + atomic_store_explicit(&c->persist.checkpoint_in_progress, 0, + memory_order_release); + + /* THE ASSERTION THAT PROVES THE FIX: the fence reads SET, and this test + * body never called upgrade_materialization_verify() itself -- the only + * possible source is persist.c's new unconditional post-checkpoint + * call. Before the fix, this read 0 forever: every production path to + * upgrade_materialization_verify() was gated on the fence already being + * 1, so nothing could ever set it for the first time. */ + TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); + /* Confirm this is a genuine incomplete-materialization fence (cfgref_* + * never advanced to this marker, because the checkpoint that would have + * stamped it was skipped) -- not a coincidental pass. */ + TEST_ASSERT_FALSE(c->persist.cfgref_present && + c->persist.cfgref_tag == tag && + memcmp(c->persist.cfgref_sha, sha, 32) == 0); + + /* Close the loop end-to-end: the two production READERS of this fence + * (ha_dispatch_gate and the promotion gate) both observe it too, using + * the same real seams 6.2-a exercises -- reusing existing test + * infrastructure, not re-deriving it. */ + TEST_ASSERT_TRUE(ha_dispatch_gate(&c->ha, OPCode_jobsub) < 0); + TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&c->ha, OPCode_heartbeat)); + + c->ha.phase2_done = 1; /* skip cluster_init_phase2 -- unrelated first-boot path */ + atomic_store_explicit(&c->ha.role, (int32_t)CTRL_ELECTING, + memory_order_release); + raft_cb_become_leader_with_cluster_for_test(c, /*term*/45); + TEST_ASSERT_NOT_EQUAL_MESSAGE( + (int32_t)CTRL_PRIMARY, + atomic_load_explicit(&c->ha.role, memory_order_acquire), + "config-upgrade-snapshot task 6.2 remediation: a fence set by the " + "REAL apply-time trigger must refuse promotion, same as a " + "directly-set fence does"); + + t43_free_myconf(c); + commit_engine_fini(&c->commit); + t42_teardown(c); + printf("test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss: PASS\n"); +} + /* ── wait_all_with_timeout (XCE_TESTING shim) ───── */ #ifdef XCE_TESTING @@ -4661,6 +5133,17 @@ void test_config_repl(void) /* Task 4.3 r2 remediation: commit-advance retry clears a push-before-commit * serve-fence (Finding D / R7.9). */ RUN_TEST(test_commit_advance_retry_clears_push_before_commit_fence); + /* config-upgrade-snapshot task 6.2: Activation Fence gate composition + + * self-triggering retry. */ + RUN_TEST(test_upgrade_materialization_fence_enforced_by_dispatch_gate_and_promotion); + RUN_TEST(test_schedule_upgrade_materialization_retry_noop_when_not_fenced); + RUN_TEST(test_schedule_upgrade_materialization_retry_noop_when_delaymgr_not_running); + RUN_TEST(test_upgrade_materialization_retry_fn_recovers_from_stale_bytes_fence); + RUN_TEST(test_commit_advance_wiring_calls_upgrade_materialization_schedule_retry); + /* config-upgrade-snapshot task 6.2 remediation: fence set by a REAL + * production trigger (apply_committed_entry's WAL_CONFIG_MARKER case), + * not a direct test call -- proves the fence is reachable at all. */ + RUN_TEST(test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss); #ifdef XCE_TESTING RUN_TEST(test_load_primary_config_missing_returns_error); RUN_TEST(test_wait_all_with_timeout_slow_threads);