/* * test_upgrade_snapshot_apply.c — config-upgrade-snapshot task 5 * * TDD tests for the new WAL_CONFIG_MARKER case in apply_committed_entry() * (persist.c), which forces a checkpoint at exactly the marker's own * (term, lsn) boundary (Requirements 1.1, 1.5, 1.8, 4.4, 5.2, 5.3). * * A naive "just call persist_checkpoint() from the new case" implementation * has two real hazards this file's tests are specifically shaped to catch: * * 1. Lock reentrancy: persist_apply_pump_run() (the live pump's call site * for BOTH primary and standby online-apply) holds apply_pump_lock * across its entire per-LSN dispatch loop, including the call into * apply_committed_entry() that would reach persist_checkpoint(). Since * persist_checkpoint() itself re-locks that same, non-recursive mutex, * a naive implementation self-deadlocks the instant a live marker * apply runs. A test that calls apply_committed_entry() directly and * in isolation (never through persist_apply_pump_run()) would NOT * catch this at all — no lock would ever be held. Every test below * that exercises the live-apply path therefore drives the marker * through persist_apply_pump_run() itself, the real production call * site, not a bare direct call. * * 2. Stale checkpoint_lsn: persist_checkpoint()'s existing "pin * checkpoint_lsn = last_applied_lsn" logic reads last_applied_lsn, * which is only advanced AFTER apply_committed_entry() returns (both * in persist_apply_pump_run() and in replay_wal()) — never before. A * naive implementation invoked synchronously from inside the marker's * own dispatch case would therefore capture the marker's PREDECESSOR's * lsn, not its own. Every test below appends at least one ordinary * entry BEFORE the marker so last_applied_lsn genuinely differs from * the marker's own lsn at the moment this hazard would bite, and * asserts the resulting checkpoint is labeled with the marker's own * lsn, not the preceding entry's. * * Fixture: a real disk-backed persist_t (persist_init/persist_start over a * private tmpdir, mirrors test_replay_apply_seam.c's trr_make_fresh_persist * / test_apply_pump.c's tap_setup) paired with a real but otherwise-empty * cluster_t (jobmgr/qmgr/workermgr/resmgr + hooksys, mirrors * test_replay_apply_seam.c's trr_build_cluster) — a disk-backed persist_t is * required here (unlike test_apply_committed_entry.c's bare in-memory * fixture) because this case's own effect IS a real checkpoint file write, * and driving persist_apply_pump_run() for real requires a real WAL on * disk to read committed entries back from. * * Every test below gathers every assertion-relevant observable into a plain * local variable FIRST, calls its fixture's teardown UNCONDITIONALLY, and * only THEN runs any TEST_ASSERT — mirroring test_apply_pump.c's own * documented discipline: a failed assertion's longjmp must never skip * teardown, or it leaks a live fsync_worker thread (still holding a pointer * into this test's about-to-be-reused stack frame) into later tests in this * same binary, corrupting them (the "known longjmp-past-teardown pathology", * confirmed the hard way while drafting this file — an early draft that * asserted before tearing down produced a stack-use-after-return crash in * flusher_fsync_forced_fail on a LATER test's frame the moment an earlier * test's assertion legitimately failed). */ #include #include #include #include #include #include #include #include #include #include #include #include "cluster.h" #include "config_repl.h" #include "hook.h" #include "jobj.h" #include "persist.h" #include "repl.h" /* repl_add_standby / repl_on_ack -- Gap-1 WAL-trim- * under-marker-trigger test (task 8.1, Req 7.4), * mirrors test_e2e_checkpoint.c's own include */ #include "snapverify.h" /* snapverify_path -- task 8.7 Scenario 1: verify the * restart-produced checkpoint is structurally * well-formed, not merely present */ #include "typemap.h" /* wal_catchup_entry_t -- repl_broadcast_send_hook's * signature */ #include "wal_types.h" #include "xce/cluster_def.h" #include "xce/types.h" #include "xstring.h" /* jobj.c's jobsub table and the process-global xstring table are shared * across every test file in the controller_unit_test binary — guard * idempotently, mirroring every other config-upgrade-snapshot/raft-apply- * decoupling test file. */ static int g_tusa_jobobj_ready; typedef struct { cluster_t c; persist_t p; char tmpdir[256]; } tusa_t; static void tusa_setup(tusa_t* t) { kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16 }; kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING, .nslots = 32, .delta = 16 }; if (makeString((char*)"tusa-probe") == NULL) xstring_table_init(64); if (!g_tusa_jobobj_ready) { TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32)); g_tusa_jobobj_ready = 1; } memset(t, 0, sizeof(*t)); t->c.myhost = &t->c.myhost_info; t->c.myhost_info.name = (char*)"ctrl-test"; t->c.myhost_info.key = (char*)""; TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&t->c.hooksys, XCE_CONTROLLER_HOOK_MAX)); t->c.jobmgr = makeInstance(&xkvmgr_type, &int_params); t->c.qmgr = makeInstance(&xkvmgr_type, &str_params); t->c.workermgr = makeInstance(&xkvmgr_type, &str_params); t->c.resmgr = makeInstance(&xkvmgr_type, &str_params); TEST_ASSERT_NOT_NULL(t->c.jobmgr); TEST_ASSERT_NOT_NULL(t->c.qmgr); TEST_ASSERT_NOT_NULL(t->c.workermgr); TEST_ASSERT_NOT_NULL(t->c.resmgr); zarray_init(&t->c.sharedinst, 16, NULL); strcpy(t->tmpdir, "/tmp/xce_tusa_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(t->tmpdir)); persist_config_t cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; /* slow fsync — no background races in tests */ cfg.snap_keep_n = 4; persist_set_workdir(&t->p, t->tmpdir, "tusa"); TEST_ASSERT_EQUAL_INT(0, persist_init(&t->p, &cfg, &t->c)); TEST_ASSERT_EQUAL_INT(0, persist_start(&t->p)); } /* Unconditional teardown — callers must gather every observable they need * to assert on BEFORE calling this (see file header). */ static void tusa_teardown(tusa_t* t) { persist_stop(&t->p); persist_fini(&t->p); zarray_fini(&t->c.sharedinst); if (t->c.resmgr) dereference(t->c.resmgr); if (t->c.workermgr) dereference(t->c.workermgr); if (t->c.qmgr) dereference(t->c.qmgr); if (t->c.jobmgr) dereference(t->c.jobmgr); xce_hook_fini(&t->c.hooksys); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", t->tmpdir); if (system(cmd) != 0) fprintf(stderr, "cleanup warning: rm -rf %s failed\n", t->tmpdir); } /* ── Helpers (none of these call TEST_ASSERT — every result is returned to * the caller, which asserts only after its fixture is torn down) ────── */ static bool tusa_snapshot_exists(persist_t* p, uint64_t lsn) { char path[4096]; snprintf(path, sizeof(path), "%s/snap-%016" PRIx64 ".capnp", p->snap_path, lsn); struct stat st; return stat(path, &st) == 0; } /* Counts every "snap-*.capnp" file in p->snap_path — used to prove a marker * apply produces exactly one checkpoint, not zero or two. */ static int tusa_count_snapshots(persist_t* p) { char cmd[4200]; snprintf(cmd, sizeof(cmd), "ls %s 2>/dev/null | grep -c '\\.capnp$'", p->snap_path); FILE* f = popen(cmd, "r"); if (!f) return -1; int n = -1; if (fscanf(f, "%d", &n) != 1) n = -1; pclose(f); return n; } /* Drop hook for repl_broadcast: a fixture that registers a standby with a * dummy (int) endpoint via repl_add_standby must install this so * persist_wal_async -> repl_broadcast does not crash trying to send to the * fake endpoint. Pattern from test_e2e_checkpoint.c (itself from * test_commit_engine.c / test_p2_integration.c) -- reused here, not * duplicated logic, for the Gap-1 WAL-trim-under-marker-trigger test * (task 8.1, Req 7.4). */ static int tusa_drop_broadcast(void* ep, wal_catchup_entry_t** entries, int count) { (void)ep; (void)entries; (void)count; return 0; } extern int (*repl_broadcast_send_hook)(void* ep, wal_catchup_entry_t** entries, int count); /* Checks for a WAL segment file "seg-.wal" at seg_start_lsn inside * wal_dir -- mirrors test_e2e_checkpoint.c's own seg_file_exists helper * exactly (same naming convention the production trim/rotate code uses). */ static bool tusa_seg_file_exists(const char* wal_dir, uint64_t seg_start_lsn) { char path[4096]; snprintf(path, sizeof(path), "%s/seg-%016" PRIx64 ".wal", wal_dir, seg_start_lsn); return access(path, F_OK) == 0; } static int tusa_count_entries(var mgr) { if (!mgr) return 0; int n = 0; iterator_t it; lock(mgr, RTTI_LOCK_READ); int irc = iter_make(&it, mgr); if (irc < 0) { unlock(mgr); return 0; } for (;;) { void* item = NULL; int nrc = iter_next(&it, &item); if (nrc < 0 || nrc == RTTI_ITER_NO_DATA) break; if (item) n++; } iter_destroy(&it); unlock(mgr); return n; } /* Result of appending one WAL_NOOP followed by one WAL_CONFIG_MARKER and * advancing committed_lsn to cover both. The leading WAL_NOOP entry is what * makes last_applied_lsn genuinely differ from the marker's own lsn at the * moment the marker's case executes — without it, Hazard 2 (stale * checkpoint_lsn) could not be distinguished from a coincidentally-correct * implementation. WAL_NOOP is a real, already-classified no-op type (mirrors * apply_wal_entry's own default treatment) that needs no cluster entity * setup. */ typedef struct { int rc_encode; int rc_noop; int rc_marker; uint64_t noop_lsn; uint64_t marker_lsn; uint64_t committed_after; } tusa_append_result_t; static tusa_append_result_t tusa_append_noop_then_marker(persist_t* p, uint8_t seed) { tusa_append_result_t r; memset(&r, 0, sizeof(r)); r.rc_noop = persist_wal_async(p, WAL_NOOP, NULL, 0); r.noop_lsn = p->current_lsn - 1; uint8_t sha[32]; memset(sha, seed, sizeof(sha)); uint8_t marker_payload[WAL_CONFIG_MARKER_PAYLOAD_LEN]; size_t out_len = 0; r.rc_encode = config_marker_encode(0x1122334455667788ULL ^ seed, sha, marker_payload, WAL_CONFIG_MARKER_PAYLOAD_LEN, &out_len); r.rc_marker = persist_wal_async(p, WAL_CONFIG_MARKER, marker_payload, WAL_CONFIG_MARKER_PAYLOAD_LEN); r.marker_lsn = p->current_lsn - 1; r.committed_after = persist_committed_lsn_advance_atomic(p, r.marker_lsn); return r; } /* ── 1: live pump path — proves both hazards are fixed ─────────────────── */ static void test_marker_apply_via_live_pump_checkpoints_at_marker_own_lsn(void) { tusa_t t; tusa_setup(&t); tusa_append_result_t ar = tusa_append_noop_then_marker(&t.p, 0xA1); /* The real production call site: persist_apply_pump_run() holds * apply_pump_lock across this entire call, exactly as it does for a * live primary/standby apply. If Hazard 1 (lock reentrancy) were not * fixed, this call would self-deadlock and never return -- the test * binary would hang here rather than fail cleanly, which is itself the * signal (see the CRITICAL note in this task's validation commands). */ int applied = persist_apply_pump_run(&t.p); uint64_t last_applied = atomic_load_explicit(&t.p.last_applied_lsn, memory_order_acquire); uint64_t checkpoint_lsn = t.p.checkpoint_lsn; bool snap_at_marker = tusa_snapshot_exists(&t.p, ar.marker_lsn); bool snap_at_predecessor = (ar.marker_lsn > 0) ? tusa_snapshot_exists(&t.p, ar.marker_lsn - 1) : false; tusa_teardown(&t); TEST_ASSERT_EQUAL_INT(0, ar.rc_encode); TEST_ASSERT_EQUAL_INT(0, ar.rc_noop); TEST_ASSERT_EQUAL_INT(0, ar.rc_marker); TEST_ASSERT_EQUAL_UINT64(ar.noop_lsn + 1, ar.marker_lsn); TEST_ASSERT_EQUAL_UINT64(ar.marker_lsn, ar.committed_after); TEST_ASSERT_EQUAL_INT(2, applied); TEST_ASSERT_EQUAL_UINT64(ar.marker_lsn, last_applied); /* Hazard 2 proof: checkpoint_lsn must equal the MARKER's own lsn, not * the preceding WAL_NOOP's (marker_lsn - 1) -- which is exactly what a * stale "checkpoint_lsn = last_applied_lsn read from inside the case" * implementation would have produced, since last_applied_lsn was still * pinned at marker_lsn - 1 at the moment the case ran. */ TEST_ASSERT_EQUAL_UINT64(ar.marker_lsn, checkpoint_lsn); TEST_ASSERT_TRUE_MESSAGE(snap_at_marker, "forced checkpoint must produce a snap file at the marker's own lsn"); TEST_ASSERT_FALSE_MESSAGE(snap_at_predecessor, "must NOT checkpoint at the preceding entry's lsn"); printf("test_marker_apply_via_live_pump_checkpoints_at_marker_own_lsn: PASS\n"); } /* ── 2: fires exactly once per marker apply ─────────────────────────────── */ static void test_marker_apply_fires_checkpoint_exactly_once(void) { tusa_t t; tusa_setup(&t); int before = tusa_count_snapshots(&t.p); tusa_append_result_t ar = tusa_append_noop_then_marker(&t.p, 0xB2); int applied = persist_apply_pump_run(&t.p); int after = tusa_count_snapshots(&t.p); int checkpoint_flag = atomic_load(&t.p.checkpoint_in_progress); tusa_teardown(&t); TEST_ASSERT_EQUAL_INT(0, ar.rc_encode); TEST_ASSERT_EQUAL_INT(0, ar.rc_noop); TEST_ASSERT_EQUAL_INT(0, ar.rc_marker); TEST_ASSERT_EQUAL_INT(2, applied); TEST_ASSERT_EQUAL_INT(0, before); TEST_ASSERT_EQUAL_INT_MESSAGE(1, after, "exactly one checkpoint file must exist after one marker apply"); /* The shared single-in-flight guard must be released again afterward, * not left stuck (which would silently wedge every future periodic * checkpoint too). */ TEST_ASSERT_EQUAL_INT(0, checkpoint_flag); printf("test_marker_apply_fires_checkpoint_exactly_once: PASS\n"); } /* ── 3: retry (re-apply of an already-applied marker) is idempotent ────── */ static void test_marker_apply_retry_is_idempotent(void) { tusa_t t; tusa_setup(&t); tusa_append_result_t ar = tusa_append_noop_then_marker(&t.p, 0xC3); int applied = persist_apply_pump_run(&t.p); bool snap_after_first = tusa_snapshot_exists(&t.p, ar.marker_lsn); /* Simulate a retried re-application of the SAME already-applied marker * (e.g. a second replay pass after a crash) by calling the dispatch * seam directly for the identical (type, lsn) a second time -- design.md's * own named observable ("safe under a forced retry"). Neither * persist_apply_pump_run() (already advanced past this lsn) nor * replay_wal() would normally re-dispatch it live, but apply_committed_ * entry() itself must not corrupt or crash on a direct re-drive. No * external lock is held here, mirroring the boot-replay calling * convention (persist_checkpoint_marker_triggered() needs none). */ uint8_t sha[32]; memset(sha, 0xC3, sizeof(sha)); uint8_t marker_payload[WAL_CONFIG_MARKER_PAYLOAD_LEN]; size_t out_len = 0; int rc_encode = config_marker_encode(0x1122334455667788ULL ^ 0xC3, sha, marker_payload, WAL_CONFIG_MARKER_PAYLOAD_LEN, &out_len); int retry_rc = apply_committed_entry(&t.p, WAL_CONFIG_MARKER, ar.marker_lsn, marker_payload, WAL_CONFIG_MARKER_PAYLOAD_LEN); bool snap_after_retry = tusa_snapshot_exists(&t.p, ar.marker_lsn); uint64_t checkpoint_lsn = t.p.checkpoint_lsn; int checkpoint_flag = atomic_load(&t.p.checkpoint_in_progress); tusa_teardown(&t); TEST_ASSERT_EQUAL_INT(0, ar.rc_encode); TEST_ASSERT_EQUAL_INT(0, ar.rc_noop); TEST_ASSERT_EQUAL_INT(0, ar.rc_marker); TEST_ASSERT_EQUAL_INT(2, applied); TEST_ASSERT_TRUE(snap_after_first); TEST_ASSERT_EQUAL_INT(0, rc_encode); TEST_ASSERT_EQUAL_INT_MESSAGE(0, retry_rc, "a retried marker apply at an already-checkpointed lsn must not error"); TEST_ASSERT_TRUE_MESSAGE(snap_after_retry, "the checkpoint file at the marker's lsn must still be present and valid"); TEST_ASSERT_EQUAL_UINT64(ar.marker_lsn, checkpoint_lsn); TEST_ASSERT_EQUAL_INT(0, checkpoint_flag); printf("test_marker_apply_retry_is_idempotent: PASS\n"); } /* ── 4: overlapping-marker -EBUSY is graceful, not an error ────────────── */ static void test_marker_apply_busy_checkpoint_handled_gracefully(void) { tusa_t t; tusa_setup(&t); tusa_append_result_t ar = tusa_append_noop_then_marker(&t.p, 0xD4); /* Simulate an earlier marker's (or a periodic WAL-threshold) checkpoint * already being in flight when this marker's own apply runs — the * shared checkpoint_in_progress CAS is already held by "someone else". */ int prev_flag = atomic_exchange(&t.p.checkpoint_in_progress, 1); /* The live pump must NOT halt or surface an error on -EBUSY (Req 4.4): * both entries still apply, last_applied_lsn still advances past the * marker, and no crash/retry-loop occurs. */ int applied = persist_apply_pump_run(&t.p); uint64_t last_applied = atomic_load_explicit(&t.p.last_applied_lsn, memory_order_acquire); bool snap_while_busy = tusa_snapshot_exists(&t.p, ar.marker_lsn); /* Self-resolving (Req 4.4): release the guard (as the "other" in-flight * checkpoint would on its own completion) and confirm a normal * checkpoint afterward still correctly picks up this marker's position. */ atomic_store(&t.p.checkpoint_in_progress, 0); int resolve_rc = persist_checkpoint(&t.p); uint64_t checkpoint_lsn = t.p.checkpoint_lsn; bool snap_after_resolve = tusa_snapshot_exists(&t.p, ar.marker_lsn); tusa_teardown(&t); TEST_ASSERT_EQUAL_INT(0, ar.rc_encode); TEST_ASSERT_EQUAL_INT(0, ar.rc_noop); TEST_ASSERT_EQUAL_INT(0, ar.rc_marker); TEST_ASSERT_EQUAL_INT(0, prev_flag); TEST_ASSERT_EQUAL_INT_MESSAGE(2, applied, "an in-flight concurrent checkpoint must not halt the apply pump"); TEST_ASSERT_EQUAL_UINT64(ar.marker_lsn, last_applied); /* Because the guard was already held, THIS marker's own checkpoint call * must have been skipped entirely -- no snapshot at its lsn yet. */ TEST_ASSERT_FALSE_MESSAGE(snap_while_busy, "a busy checkpoint must be skipped, not forced through"); TEST_ASSERT_EQUAL_INT(0, resolve_rc); TEST_ASSERT_EQUAL_UINT64(ar.marker_lsn, checkpoint_lsn); TEST_ASSERT_TRUE(snap_after_resolve); printf("test_marker_apply_busy_checkpoint_handled_gracefully: PASS\n"); } /* ── 5: apply must not itself reinterpret/re-home entities (Req 1.8) ───── */ static void test_marker_apply_does_not_reinterpret_entities(void) { tusa_t t; tusa_setup(&t); /* Baseline: nothing exists yet, in particular no lost_and_found queue * or host -- the concrete, observable side effect any accidental * re-homing trigger from this case would produce. */ int qmgr_before = tusa_count_entries(t.c.qmgr); int workermgr_before = tusa_count_entries(t.c.workermgr); int jobmgr_before = tusa_count_entries(t.c.jobmgr); tusa_append_result_t ar = tusa_append_noop_then_marker(&t.p, 0xE5); int applied = persist_apply_pump_run(&t.p); int qmgr_after = tusa_count_entries(t.c.qmgr); int workermgr_after = tusa_count_entries(t.c.workermgr); int jobmgr_after = tusa_count_entries(t.c.jobmgr); tusa_teardown(&t); TEST_ASSERT_EQUAL_INT(0, ar.rc_encode); TEST_ASSERT_EQUAL_INT(0, ar.rc_noop); TEST_ASSERT_EQUAL_INT(0, ar.rc_marker); TEST_ASSERT_EQUAL_INT(2, applied); TEST_ASSERT_EQUAL_INT(0, qmgr_before); TEST_ASSERT_EQUAL_INT(0, workermgr_before); TEST_ASSERT_EQUAL_INT(0, jobmgr_before); /* Applying the marker must ONLY materialize a checkpoint -- it must not * create, remove, or re-home any job/queue/host entity (that * interpretive switch is the later activation step's job, not apply's, * per Req 1.8). */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, qmgr_after, "marker apply must not create any queue (e.g. lost_and_found)"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, workermgr_after, "marker apply must not create any host (e.g. lost_and_found)"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, jobmgr_after, "marker apply must not create or remove any job"); printf("test_marker_apply_does_not_reinterpret_entities: PASS\n"); } /* ── 6: restart replay drives the SAME case (Req 1.1's "replay alike") ──── */ static void test_marker_apply_via_replay_checkpoints_at_marker_own_lsn(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_tusa_replay_XXXXXX"); char* mkd_rc = mkdtemp(tmp); uint64_t marker_lsn = 0; int rc_noop_p1 = -1; int rc_marker_p1 = -1; int rc_encode_p1 = -1; int rc_flush_p1 = -1; /* Phase 1: write a WAL_NOOP then a WAL_CONFIG_MARKER to a fresh, real * WAL -- no cluster needed for a pure physical write. */ { persist_t p; memset(&p, 0, sizeof(p)); persist_config_t cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; persist_set_workdir(&p, tmp, "tusa-replay"); int rc_init = persist_init(&p, &cfg, NULL); int rc_start = persist_start(&p); rc_noop_p1 = persist_wal_async(&p, WAL_NOOP, NULL, 0); uint8_t sha[32]; memset(sha, 0xF6, sizeof(sha)); uint8_t marker_payload[WAL_CONFIG_MARKER_PAYLOAD_LEN]; size_t out_len = 0; rc_encode_p1 = config_marker_encode(0x1122334455667788ULL ^ 0xF6, sha, marker_payload, WAL_CONFIG_MARKER_PAYLOAD_LEN, &out_len); rc_marker_p1 = persist_wal_async(&p, WAL_CONFIG_MARKER, marker_payload, WAL_CONFIG_MARKER_PAYLOAD_LEN); marker_lsn = p.current_lsn - 1; rc_flush_p1 = persist_flush(&p); persist_stop(&p); persist_fini(&p); TEST_ASSERT_EQUAL_INT(0, rc_init); TEST_ASSERT_EQUAL_INT(0, rc_start); } /* Phase 2: fresh persist_t + fresh cluster_t, recover from disk. Boot * replay (persist_recover -> replay_wal) is the SECOND production call * site of apply_committed_entry() -- unlike the live pump, it does NOT * hold apply_pump_lock, proving persist_checkpoint_marker_triggered() * is correct under both calling conventions, not just the locked one. */ tusa_t t2; memset(&t2, 0, sizeof(t2)); strncpy(t2.tmpdir, tmp, sizeof(t2.tmpdir) - 1); t2.c.myhost = &t2.c.myhost_info; t2.c.myhost_info.name = (char*)"ctrl-test-2"; t2.c.myhost_info.key = (char*)""; int rc_hook = xce_hook_init(&t2.c.hooksys, XCE_CONTROLLER_HOOK_MAX); kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16 }; kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING, .nslots = 32, .delta = 16 }; t2.c.jobmgr = makeInstance(&xkvmgr_type, &int_params); t2.c.qmgr = makeInstance(&xkvmgr_type, &str_params); t2.c.workermgr = makeInstance(&xkvmgr_type, &str_params); t2.c.resmgr = makeInstance(&xkvmgr_type, &str_params); zarray_init(&t2.c.sharedinst, 16, NULL); persist_config_t cfg2 = persist_config_defaults(); cfg2.wal_sync_ms = 5000; persist_set_workdir(&t2.p, tmp, "tusa-replay"); int rc_init2 = persist_init(&t2.p, &cfg2, &t2.c); int rc_start2 = persist_start(&t2.p); int rc_recover = persist_recover(&t2.p); uint64_t checkpoint_lsn = t2.p.checkpoint_lsn; bool snap_exists = tusa_snapshot_exists(&t2.p, marker_lsn); tusa_teardown(&t2); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp); if (system(cmd) != 0) fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp); TEST_ASSERT_NOT_NULL(mkd_rc); TEST_ASSERT_EQUAL_INT(0, rc_noop_p1); TEST_ASSERT_EQUAL_INT(0, rc_encode_p1); TEST_ASSERT_EQUAL_INT(0, rc_marker_p1); TEST_ASSERT_EQUAL_INT(0, rc_flush_p1); TEST_ASSERT_EQUAL_INT(0, rc_hook); TEST_ASSERT_NOT_NULL(t2.c.jobmgr); TEST_ASSERT_NOT_NULL(t2.c.qmgr); TEST_ASSERT_NOT_NULL(t2.c.workermgr); TEST_ASSERT_NOT_NULL(t2.c.resmgr); TEST_ASSERT_EQUAL_INT(0, rc_init2); TEST_ASSERT_EQUAL_INT(0, rc_start2); TEST_ASSERT_EQUAL_INT(0, rc_recover); TEST_ASSERT_EQUAL_UINT64(marker_lsn, checkpoint_lsn); TEST_ASSERT_TRUE_MESSAGE(snap_exists, "boot replay of a WAL_CONFIG_MARKER must also produce a " "checkpoint at the marker's own lsn"); printf("test_marker_apply_via_replay_checkpoints_at_marker_own_lsn: PASS\n"); } /* ── 7: WAL-trim safety under the marker-triggered path (Req 7.4) ─────── * * Six existing tests already prove trim_wal_segments()'s standby-cursor * retention guard (repl_trim_lsn(), bug-1147) holds under the OLD trigger * -- a direct/periodic persist_checkpoint() call: * test_e2e_checkpoint.c's test_trim_guard_preserves_standby_segments, plus * repl_trim_lsn()'s own unit coverage in test_repl_unit.c / * test_p2_integration.c. None of them drive that SAME guard through this * spec's NEW trigger -- a WAL_CONFIG_MARKER committed and applied via the * real apply_committed_entry()/persist_apply_pump_run() path, which * reaches the identical, unmodified trim_wal_segments()/repl_trim_lsn() * code via persist_checkpoint_marker_triggered() instead of a bare * persist_checkpoint() call. That is a real, previously-flagged coverage * gap (task 5's own implementation notes), not something covered by proxy * -- this test closes it. * * Uses this file's own real-cluster tusa_t fixture, not * test_e2e_checkpoint.c's cluster-less make_ckpt_e2e() -- apply_committed_ * entry() short-circuits to a no-op whenever p->cluster is NULL, so a * cluster-less fixture would never actually reach persist_checkpoint_ * marker_triggered() at all and could not exercise this path. WAL_NOOP is * used for every filler entry (rather than test_e2e_checkpoint.c's * WAL_JOB_UPDATE) because that file's fixture has no cluster to apply a * job update against; WAL_NOOP is safe against a real cluster (default: * no-op case), exactly as tusa_append_noop_then_marker() already relies on * elsewhere in this file. * * Layout mirrors test_trim_guard_preserves_standby_segments' own LSN * numbering exactly, for direct comparability: * noop @ lsn1=1 → force segment (seg-1 closes, seg-2 opens @ lsn2=2) * noop @ lsn2=2 → standby acks lsn2 (mid-catch-up: this is the LAST * position it has acked -- in particular it has NOT * acked the marker below) * → force segment (seg-2 closes, seg-3 opens @ lsn3=3, * stays active -- no further force call) * noop @ lsn3=3 → keeps last_applied_lsn genuinely behind the marker's * own lsn at dispatch time (same discipline this file's * other tests already document as Hazard 2) * WAL_CONFIG_MARKER @ marker_lsn=4 → committed → applied via * persist_apply_pump_run(), which dispatches into apply_committed_ * entry()'s WAL_CONFIG_MARKER case → persist_checkpoint_marker_ * triggered() → trim_wal_segments(). * * floor = repl_trim_lsn(active_seg_lsn=3, standby acked=2) = min(3,2) = 2: * seg-1 (start=1) < floor → MUST be trimmed * seg-2 (start=2) == floor → MUST be preserved (the standby still needs * it for catch-up -- exactly bug-1147's guard) * seg-3 (start=3, active) → MUST always be preserved */ static void test_marker_triggered_checkpoint_trim_preserves_standby_mid_catchup_segments(void) { tusa_t t; tusa_setup(&t); repl_broadcast_send_hook = tusa_drop_broadcast; int dummy_ep = 0; int standby_idx = repl_add_standby(&t.p.repl, (void*)&dummy_ep, "standby1", 7001); int rc_noop1 = persist_wal_async(&t.p, WAL_NOOP, NULL, 0); uint64_t lsn1 = t.p.current_lsn - 1; int rc_force1 = persist_test_force_new_segment(&t.p); int rc_noop2 = persist_wal_async(&t.p, WAL_NOOP, NULL, 0); uint64_t lsn2 = t.p.current_lsn - 1; repl_on_ack(&t.p.repl, (void*)&dummy_ep, lsn2); int rc_force2 = persist_test_force_new_segment(&t.p); uint64_t active_seg_lsn = t.p.current_lsn; /* seg-3 opens here */ int rc_noop3 = persist_wal_async(&t.p, WAL_NOOP, NULL, 0); uint8_t sha[32]; memset(sha, 0xA7, sizeof(sha)); uint8_t marker_payload[WAL_CONFIG_MARKER_PAYLOAD_LEN]; size_t out_len = 0; int rc_encode = config_marker_encode(0x1122334455667788ULL ^ 0xA7, sha, marker_payload, WAL_CONFIG_MARKER_PAYLOAD_LEN, &out_len); int rc_marker = persist_wal_async(&t.p, WAL_CONFIG_MARKER, marker_payload, WAL_CONFIG_MARKER_PAYLOAD_LEN); uint64_t marker_lsn = t.p.current_lsn - 1; uint64_t committed_after = persist_committed_lsn_advance_atomic(&t.p, marker_lsn); int applied = persist_apply_pump_run(&t.p); bool seg1_exists = tusa_seg_file_exists(t.p.wal_path, lsn1); bool seg2_exists = tusa_seg_file_exists(t.p.wal_path, lsn2); bool seg3_exists = tusa_seg_file_exists(t.p.wal_path, active_seg_lsn); bool snap_at_marker = tusa_snapshot_exists(&t.p, marker_lsn); repl_broadcast_send_hook = NULL; tusa_teardown(&t); TEST_ASSERT_TRUE(standby_idx >= 0); TEST_ASSERT_EQUAL_INT(0, rc_noop1); TEST_ASSERT_EQUAL_INT(0, rc_force1); TEST_ASSERT_EQUAL_INT(0, rc_noop2); TEST_ASSERT_EQUAL_INT(0, rc_force2); TEST_ASSERT_EQUAL_INT(0, rc_noop3); TEST_ASSERT_EQUAL_INT(0, rc_encode); TEST_ASSERT_EQUAL_INT(0, rc_marker); TEST_ASSERT_EQUAL_UINT64(1, lsn1); TEST_ASSERT_EQUAL_UINT64(2, lsn2); TEST_ASSERT_EQUAL_UINT64(3, active_seg_lsn); TEST_ASSERT_EQUAL_UINT64(4, marker_lsn); TEST_ASSERT_EQUAL_UINT64(marker_lsn, committed_after); TEST_ASSERT_EQUAL_INT_MESSAGE(4, applied, "all four entries (3 noops + the marker) must apply successfully"); TEST_ASSERT_TRUE_MESSAGE(snap_at_marker, "the marker-triggered checkpoint must still fire and produce a " "snapshot even with a standby mid-catch-up"); TEST_ASSERT_FALSE_MESSAGE(seg1_exists, "seg-1: below the standby's mid-catch-up acked_lsn=2, MUST be " "trimmed by the marker-triggered checkpoint"); TEST_ASSERT_TRUE_MESSAGE(seg2_exists, "seg-2: at the standby's mid-catch-up acked_lsn=2, MUST be " "preserved (bug-1147 guard) even when trim runs via the NEW " "marker-triggered path"); TEST_ASSERT_TRUE_MESSAGE(seg3_exists, "seg-3: the active segment, MUST always be preserved"); printf("test_marker_triggered_checkpoint_trim_preserves_standby_" "mid_catchup_segments: PASS\n"); } /* ── 8: fault-injection seam at the marker-apply→checkpoint-durable * boundary (Req 6.3) ─────────────────────────────────────────────── * * Proves the new persist_marker_checkpoint_fault_hook seam (persist.c, * added by this task) fires exactly once for the marker's own (p, lsn) and * correctly gates whether persist_checkpoint_marker_triggered() is called * at all. This is deliberately ONLY the seam itself, not the full * crash/restart regression -- that is task 8.7's job, which depends on * this seam landing first. */ typedef struct { int calls; persist_t* last_p; uint64_t last_lsn; } tusa_fault_obs_t; static tusa_fault_obs_t g_tusa_fault_obs; static int tusa_marker_fault_hook(persist_t* p, uint64_t marker_lsn) { g_tusa_fault_obs.calls++; g_tusa_fault_obs.last_p = p; g_tusa_fault_obs.last_lsn = marker_lsn; return 1; /* simulate "the process never got to checkpoint" */ } static void test_marker_checkpoint_fault_hook_gates_checkpoint_call(void) { tusa_t t; extern int (*persist_marker_checkpoint_fault_hook)(persist_t* p, uint64_t marker_lsn); tusa_setup(&t); memset(&g_tusa_fault_obs, 0, sizeof(g_tusa_fault_obs)); persist_marker_checkpoint_fault_hook = tusa_marker_fault_hook; tusa_append_result_t ar = tusa_append_noop_then_marker(&t.p, 0x17); int apply_rc = persist_apply_pump_run(&t.p); bool snap_at_marker = tusa_snapshot_exists(&t.p, ar.marker_lsn); uint64_t last_applied = atomic_load_explicit(&t.p.last_applied_lsn, memory_order_acquire); int fault_calls = g_tusa_fault_obs.calls; uint64_t fault_lsn = g_tusa_fault_obs.last_lsn; bool fault_p_matches = (g_tusa_fault_obs.last_p == &t.p); persist_marker_checkpoint_fault_hook = NULL; tusa_teardown(&t); TEST_ASSERT_EQUAL_INT(0, ar.rc_encode); TEST_ASSERT_EQUAL_INT(0, ar.rc_noop); TEST_ASSERT_EQUAL_INT(0, ar.rc_marker); TEST_ASSERT_EQUAL_INT_MESSAGE(1, fault_calls, "the fault hook must fire exactly once, for the marker's own " "checkpoint attempt"); TEST_ASSERT_EQUAL_UINT64(ar.marker_lsn, fault_lsn); TEST_ASSERT_TRUE_MESSAGE(fault_p_matches, "the fault hook must observe the SAME persist_t the marker is " "being applied to"); TEST_ASSERT_EQUAL_INT_MESSAGE(-ECANCELED, apply_rc, "a nonzero fault-hook return must propagate as the marker's apply " "failure, taking the SAME halt-and-retry path a genuine checkpoint " "I/O error already takes"); TEST_ASSERT_FALSE_MESSAGE(snap_at_marker, "a nonzero fault-hook return must skip the checkpoint call " "entirely -- no snapshot file at the marker's lsn"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(ar.noop_lsn, last_applied, "last_applied_lsn must NOT advance past the marker when the " "checkpoint is skipped by the fault hook -- materialization must " "resume from the marker on the next attempt"); printf("test_marker_checkpoint_fault_hook_gates_checkpoint_call: " "PASS\n"); } /* ── 9: crash before checkpoint durable, restart resumes materialization * (task 8.7 Scenario 1, Req 5.2, 5.3, 5.5) ───────────────────────── * * Builds directly on test 8's fault-hook seam above, whose own header * comment is explicit that it proves ONLY the seam itself -- "not the full * crash/restart regression -- that is task 8.7's job, which depends on * this seam landing first." This test is that job. * * Phase 1 ("crash"): drive the marker's apply through the live pump with * the fault hook forcing "the checkpoint write never completed" -- the * same mechanics test 8 already proves in isolation (apply_rc == * -ECANCELED, no snapshot at the marker's lsn, last_applied_lsn stuck * below it). This test re-confirms that baseline lightly (not exhaustively * -- test 8 already owns that) and then goes one step further: physically * tear down this persist_t/cluster_t WITHOUT deleting the on-disk tmpdir, * mirroring test 6's own phase-1/phase-2 split exactly (test 6 is this * same file's established pattern for simulating "the process exits and a * fresh one boots against the same on-disk state" -- persist_stop/ * persist_fini inline, no rm -rf until the very end). * * Phase 2 ("restart succeeds this time"): clear the fault hook (the * transient condition that caused the crash is gone), boot a genuinely * FRESH persist_t + cluster_t against the SAME tmpdir/name, and call * persist_recover() -- boot replay, the SECOND real production call site * of apply_committed_entry() (test 6 above already proves this is a valid, * lock-free calling convention for the marker case). A bare re-call of * persist_apply_pump_run() on the SAME in-memory instance was deliberately * rejected in favor of this: last_applied_lsn never advanced past the * marker in phase 1, so a same-process re-call would only prove "this * process retries," not "a genuinely restarted process resumes from * durable state" -- persist_recover() against a fresh instance is the * faithful simulation Requirement 5.2 actually describes. Because * checkpoint_lsn was never advanced past 0 in phase 1 (no checkpoint ever * wrote), replay restarts from lsn 1 and re-drives both the noop and the * marker through apply_committed_entry() fresh -- this time succeeding. * * Phase 3 (idempotent re-restart, Req 5.3 in this exact crash/retry * shape): call persist_recover() a SECOND time on the already-recovered * instance (test_persist_recovery.c's own test_recover_from_empty_state * documents this as valid: "persist_recover() is valid to call on a * running instance too") -- proves a second, redundant "restart" after * materialization already succeeded does not duplicate or corrupt the * checkpoint. test_marker_apply_retry_is_idempotent (test 3 above) already * covers a DIFFERENT retry shape -- a bare, same-process direct * re-dispatch of apply_committed_entry(), never a real boot replay -- so * this is not a duplicate of that coverage, it closes the boot-replay- * specific shape that test doesn't touch. * * Discipline note: per this file's own header, every assertion-relevant * observable across all three phases is gathered into a local variable * FIRST; no TEST_ASSERT runs until every phase's teardown has already * happened (avoids the documented longjmp-past-teardown pathology). */ static void test_marker_checkpoint_fault_hook_crash_then_restart_resumes_materialization(void) { extern int (*persist_marker_checkpoint_fault_hook)(persist_t* p, uint64_t marker_lsn); tusa_t t; tusa_setup(&t); memset(&g_tusa_fault_obs, 0, sizeof(g_tusa_fault_obs)); persist_marker_checkpoint_fault_hook = tusa_marker_fault_hook; tusa_append_result_t ar = tusa_append_noop_then_marker(&t.p, 0x8A); int crash_apply_rc = persist_apply_pump_run(&t.p); bool crash_snap_exists = tusa_snapshot_exists(&t.p, ar.marker_lsn); uint64_t crash_last_applied = atomic_load_explicit( &t.p.last_applied_lsn, memory_order_acquire); int crash_fault_calls = g_tusa_fault_obs.calls; /* "Crash": tear down this instance WITHOUT deleting the on-disk tmpdir * -- mirrors test 6's own phase-1 teardown exactly (persist_stop/ * persist_fini + mgr cleanup inline, no tusa_teardown/rm -rf until the * very end of this test, after phase 2/3 have had a chance to read the * same directory). */ persist_stop(&t.p); persist_fini(&t.p); zarray_fini(&t.c.sharedinst); if (t.c.resmgr) dereference(t.c.resmgr); if (t.c.workermgr) dereference(t.c.workermgr); if (t.c.qmgr) dereference(t.c.qmgr); if (t.c.jobmgr) dereference(t.c.jobmgr); xce_hook_fini(&t.c.hooksys); /* "Restart succeeds this time": the transient condition that caused * the crash is gone. */ persist_marker_checkpoint_fault_hook = NULL; /* Phase 2: fresh persist_t + fresh cluster_t, SAME tmpdir/name as phase * 1 -- boot replay (persist_recover -> replay_wal), mirrors test 6's * own phase-2 construction exactly. */ kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16 }; kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING, .nslots = 32, .delta = 16 }; tusa_t t2; memset(&t2, 0, sizeof(t2)); strncpy(t2.tmpdir, t.tmpdir, sizeof(t2.tmpdir) - 1); t2.c.myhost = &t2.c.myhost_info; t2.c.myhost_info.name = (char*)"ctrl-test-restart"; t2.c.myhost_info.key = (char*)""; int rc_hook2 = xce_hook_init(&t2.c.hooksys, XCE_CONTROLLER_HOOK_MAX); t2.c.jobmgr = makeInstance(&xkvmgr_type, &int_params); t2.c.qmgr = makeInstance(&xkvmgr_type, &str_params); t2.c.workermgr = makeInstance(&xkvmgr_type, &str_params); t2.c.resmgr = makeInstance(&xkvmgr_type, &str_params); zarray_init(&t2.c.sharedinst, 16, NULL); persist_config_t cfg2 = persist_config_defaults(); cfg2.wal_sync_ms = 5000; cfg2.snap_keep_n = 4; persist_set_workdir(&t2.p, t.tmpdir, "tusa"); int rc_init2 = persist_init(&t2.p, &cfg2, &t2.c); int rc_start2 = persist_start(&t2.p); int rc_recover = persist_recover(&t2.p); uint64_t restart_checkpoint_lsn = t2.p.checkpoint_lsn; bool restart_snap_exists = tusa_snapshot_exists(&t2.p, ar.marker_lsn); uint64_t restart_last_applied = atomic_load_explicit( &t2.p.last_applied_lsn, memory_order_acquire); int restart_fault_calls = g_tusa_fault_obs.calls; char snap_path[4096]; snprintf(snap_path, sizeof(snap_path), "%s/snap-%016" PRIx64 ".capnp", t2.p.snap_path, ar.marker_lsn); snapverify_report_t verify_rep; memset(&verify_rep, 0, sizeof(verify_rep)); int verify_rc = snapverify_path(snap_path, NULL, &verify_rep); /* Phase 3: a SECOND, redundant "restart" after materialization already * succeeded -- proves idempotency in this exact crash/retry shape. */ int rc_recover_again = persist_recover(&t2.p); uint64_t checkpoint_lsn_after_second_recover = t2.p.checkpoint_lsn; int snap_count_after_second_recover = tusa_count_snapshots(&t2.p); tusa_teardown(&t2); /* ── Assertions (only now, after every phase's teardown) ───────────── */ TEST_ASSERT_EQUAL_INT(0, ar.rc_encode); TEST_ASSERT_EQUAL_INT(0, ar.rc_noop); TEST_ASSERT_EQUAL_INT(0, ar.rc_marker); TEST_ASSERT_EQUAL_INT_MESSAGE(1, crash_fault_calls, "fault hook must have fired exactly once to force the simulated " "crash"); TEST_ASSERT_EQUAL_INT_MESSAGE(-ECANCELED, crash_apply_rc, "the simulated crash must surface as the marker's apply failure"); TEST_ASSERT_FALSE_MESSAGE(crash_snap_exists, "crash state: no checkpoint must exist at the marker's lsn yet"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(ar.noop_lsn, crash_last_applied, "crash state: last_applied_lsn must be stuck below the marker"); TEST_ASSERT_EQUAL_INT(0, rc_hook2); TEST_ASSERT_NOT_NULL(t2.c.jobmgr); TEST_ASSERT_NOT_NULL(t2.c.qmgr); TEST_ASSERT_NOT_NULL(t2.c.workermgr); TEST_ASSERT_NOT_NULL(t2.c.resmgr); TEST_ASSERT_EQUAL_INT(0, rc_init2); TEST_ASSERT_EQUAL_INT(0, rc_start2); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc_recover, "restart's boot replay must succeed now that the fault clears"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, restart_fault_calls, "the fault hook must NOT fire again during restart -- it was " "cleared, simulating the transient condition having resolved"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(ar.marker_lsn, restart_checkpoint_lsn, "Req 5.2: restart must resume materialization AT THE MARKER, not " "some other position"); TEST_ASSERT_TRUE_MESSAGE(restart_snap_exists, "Req 5.2/5.5: restart must produce a durable checkpoint at the " "marker's own lsn"); TEST_ASSERT_TRUE_MESSAGE(restart_last_applied >= ar.marker_lsn, "Req 5.2: restart must advance last_applied_lsn past the marker " "-- materialization resumed and completed, not re-stuck"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, verify_rc, "Req 5.5: the restart-produced checkpoint must be structurally " "well-formed, not a partial/corrupt write"); TEST_ASSERT_TRUE_MESSAGE(verify_rep.ok, verify_rep.fail_reason); TEST_ASSERT_EQUAL_UINT64(ar.marker_lsn, verify_rep.checkpoint_lsn); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc_recover_again, "Req 5.3 in the crash/retry shape: a second, redundant restart " "after materialization already succeeded must not error"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(ar.marker_lsn, checkpoint_lsn_after_second_recover, "a redundant restart must not relabel or move the checkpoint"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, snap_count_after_second_recover, "a redundant restart must not duplicate the checkpoint file"); printf("test_marker_checkpoint_fault_hook_crash_then_restart_resumes_" "materialization: PASS\n"); } /* ── Dispatch ────────────────────────────────────────────────────────── */ void test_upgrade_snapshot_apply(void) { UnitySetTestFile(__FILE__); printf("\n>> config-upgrade-snapshot marker-apply-trigger tests (task 5)\n\n"); RUN_TEST(test_marker_apply_via_live_pump_checkpoints_at_marker_own_lsn); RUN_TEST(test_marker_apply_fires_checkpoint_exactly_once); RUN_TEST(test_marker_apply_retry_is_idempotent); RUN_TEST(test_marker_apply_busy_checkpoint_handled_gracefully); RUN_TEST(test_marker_apply_does_not_reinterpret_entities); RUN_TEST(test_marker_apply_via_replay_checkpoints_at_marker_own_lsn); RUN_TEST(test_marker_triggered_checkpoint_trim_preserves_standby_mid_catchup_segments); RUN_TEST(test_marker_checkpoint_fault_hook_gates_checkpoint_call); RUN_TEST(test_marker_checkpoint_fault_hook_crash_then_restart_resumes_materialization); }