/* * test_bug857_regression.c — config-upgrade-snapshot task 8.5 * * Reproduces the bug-857 materialized-state divergence on the PRE-fix * mechanism and proves it converges on the CURRENT (post-task-5) one, both * at the controller_unit_test level, entirely within this one file. * * ── Investigation summary (why this file looks the way it does) ──────── * * Task 8.5's original brief hypothesized bug-857's divergence as "a job * whose queue was removed by a config upgrade replays into its original * queue on one member and gets re-homed to the synthetic `lost_and_found` * queue on another, depending on each member's own restart timing." * Investigation (this task) found that mechanism does not exist as * implemented code, on ANY commit: `create_lost_and_found` (config_repl.c) * only ever creates the sentinel queue/host objects — no code path in * job.c/queue.c/worker.c ever reassigns an EXISTING job's queue reference * onto them. `restore_job`/`restore_job_array` (job.c) never consult * `c->qmgr` at all; `restore_queue_state` (queue.c) explicitly SKIPS * (returns -ENOENT) rather than re-homes when the target queue is absent. * `create_lost_and_found`'s body is also unconditionally `#ifndef * XCE_TESTING`'d out. bug-857's buglog entry itself already flagged this * as "NOT empirically reproduced yet" — the re-homing detail was TLA+- * derived aspiration, not observed behavior. * * design.md's Requirements Traceability table (row 6.2) originally * assigned this requirement to "HA soak new scenario" — written under the * same since-corrected hypothesis. The real, code-backed divergence this * file reproduces instead is CHECKPOINT-CONTENT divergence: pre-task-5, * `apply_committed_entry`'s WAL_CONFIG_MARKER case was a no-op (the * generic `default:` case) and a checkpoint's LSN/content was determined * SOLELY by `schedule_checkpoint`'s independent per-member WAL-record-count * threshold (persist.c) — completely decoupled from the config-change * marker's own position in the shared committed log. Two members reaching * that threshold at different real-world moments (because other activity — * more job submissions — is interleaved around the upgrade) materialize * genuinely different job/queue content, with NEITHER checkpoint anchored * to the marker at all: exactly "no snapshot pinning an identical * post-upgrade state" (bug-857's own root-cause wording). Post-task-5, * `apply_committed_entry`'s WAL_CONFIG_MARKER case forces * `persist_checkpoint_marker_triggered()` synchronously, INLINE, the * instant the marker entry itself is dispatched — before the per-LSN * apply loop ever reaches any later entry — so every member's * marker-anchored checkpoint captures identical content regardless of its * own processing pace. * * Both scenarios below run entirely within controller_unit_test — no * cluster.c linkage (cluster_init/cluster_build_resources), no scratch * worktree, no live multi-process cluster needed. See this task's Status * Report / design.md's 6.2 traceability-row correction note for the full * decision trail. * * ── Scenario 1: test_bug857_threshold_checkpoint_diverges_across_members ── * Reproduces the PRE-task-5 mechanism for real: two members receive the * identical physical WAL (job1, config-change marker, job2), but their * entity state and checkpoint are driven manually via restore_job() + * persist_checkpoint() — the SAME production functions * apply_committed_entry()/schedule_checkpoint() would themselves call — * WITHOUT ever routing the marker's LSN through apply_committed_entry() * itself (which, on the CURRENT codebase, unconditionally forces the * marker-anchored checkpoint task 5 added, masking exactly the bug this * scenario needs to demonstrate the absence of). Member A's threshold * trips right after job1 (before job2 exists locally); member B's does not * trip until after job2 exists locally too — modeling two members whose * own independent checkpoint/restart timing straddles ordinary cluster * activity happening around the upgrade. Asserts the two members' checkpoint * files are NOT byte-identical (snapdiff_paths) and that their job SETS * genuinely differ (job2 present in B, absent in A) — real content * divergence, not a cosmetic LSN-header mismatch. * * ── Scenario 2: test_bug857_marker_checkpoint_converges_across_members ── * The identical shared sequence, this time routed through the REAL, * current apply_committed_entry() dispatch (persist_apply_pump_run) on * both members. Member A catches up in a single pump-run call over the * whole range; member B processes job1+marker in one call and only later * commits+applies job2 in a separate second call — genuinely different * processing pace. Asserts both members' checkpoint at the marker's own * LSN exists and is byte-identical (snapdiff_paths) regardless of that * pacing difference. */ #include #include #include #include #include #include #include #include #include #include #include #include #include "cluster.h" #include "config_repl.h" #include "controller.capnp.h" #include "hook.h" #include "jobj.h" #include "persist.h" #include "snapdiff.h" #include "wal_types.h" #include "xce/cluster_def.h" #include "xce/types.h" #include "xstring.h" /* restore_job (job.c) has no dedicated header -- every direct-call site in * this codebase declares it locally (mirrors persist.c's own extern at the * top of that file). This file calls it directly (bypassing * apply_committed_entry) in Scenario 1 for the reason explained above: the * marker's LSN must never be dispatched through apply_committed_entry * there, or the CURRENT codebase's marker-anchored checkpoint (task 5) * would fire and mask the exact pre-fix behavior this test reproduces. */ extern int restore_job(cluster_t* c, uint64_t jobid_unused, const void* capnp_payload, size_t len, uint64_t entry_lsn); /* jobj.c's jobsub table and the process-global xstring table are shared * across every test file in the controller_unit_test binary -- guard * idempotently, mirroring every other config-upgrade-snapshot test file * (job_obj_init itself is also independently idempotent via its own * process-global atomic flag, jobj.c -- this local guard just skips the * redundant call within this file's own repeated test invocations). */ static int g_b857_jobobj_ready; typedef struct { cluster_t c; persist_t p; char tmpdir[256]; } b857_t; /* Fixture mirrors test_upgrade_snapshot_apply.c's tusa_t/tusa_setup exactly * (real disk-backed persist_t + a real but otherwise-empty cluster_t with * jobmgr/qmgr/workermgr/resmgr as xkvmgr_type instances) -- not reused * directly since tusa_setup/tusa_t are file-static, matching this * codebase's established convention of each test file keeping its own * minimal local copy of such fixtures (e.g. test_replay_apply_seam.c's * trr_make_fresh_persist mirroring test_persist_recovery.c's * make_fresh_persist). `label` must be short and filesystem-safe -- used * as the hostname, the persist_set_workdir name, AND the tmpdir prefix. */ static void b857_setup(b857_t* t, const char* label) { kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16 }; kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING, .nslots = 32, .delta = 16 }; if (makeString((char*)"b857-probe") == NULL) xstring_table_init(64); if (!g_b857_jobobj_ready) { TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32)); g_b857_jobobj_ready = 1; } memset(t, 0, sizeof(*t)); t->c.myhost = &t->c.myhost_info; t->c.myhost_info.name = (char*)label; t->c.myhost_info.key = (char*)""; TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&t->c.hooksys, XCE_CONTROLLER_HOOK_MAX)); t->c.jobmgr = makeInstance(&xkvmgr_type, &int_params); t->c.qmgr = makeInstance(&xkvmgr_type, &str_params); t->c.workermgr = makeInstance(&xkvmgr_type, &str_params); t->c.resmgr = makeInstance(&xkvmgr_type, &str_params); TEST_ASSERT_NOT_NULL(t->c.jobmgr); TEST_ASSERT_NOT_NULL(t->c.qmgr); TEST_ASSERT_NOT_NULL(t->c.workermgr); TEST_ASSERT_NOT_NULL(t->c.resmgr); zarray_init(&t->c.sharedinst, 16, NULL); snprintf(t->tmpdir, sizeof(t->tmpdir), "/tmp/xce_%s_XXXXXX", label); TEST_ASSERT_NOT_NULL(mkdtemp(t->tmpdir)); persist_config_t cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; /* slow fsync -- no background races in tests */ cfg.snap_keep_n = 4; persist_set_workdir(&t->p, t->tmpdir, label); TEST_ASSERT_EQUAL_INT(0, persist_init(&t->p, &cfg, &t->c)); TEST_ASSERT_EQUAL_INT(0, persist_start(&t->p)); } /* Unconditional teardown -- callers must gather every observable they need * to assert on BEFORE calling this (mirrors test_upgrade_snapshot_apply.c's * documented longjmp-past-teardown discipline). */ static void b857_teardown(b857_t* t) { persist_stop(&t->p); persist_fini(&t->p); zarray_fini(&t->c.sharedinst); if (t->c.resmgr) dereference(t->c.resmgr); if (t->c.workermgr) dereference(t->c.workermgr); if (t->c.qmgr) dereference(t->c.qmgr); if (t->c.jobmgr) dereference(t->c.jobmgr); xce_hook_fini(&t->c.hooksys); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", t->tmpdir); if (system(cmd) != 0) fprintf(stderr, "cleanup warning: rm -rf %s failed\n", t->tmpdir); } /* Encode a minimal, real job into a WAL_JOB_SUBMIT payload -- mirrors * test_replay_apply_seam.c's trr_encode_job exactly (capnp_encode_job * itself is file-static in persist.c, not reusable across translation * units, hence this small local copy). */ static int b857_encode_job(uint64_t jobid, const char* queue, uint8_t** out, size_t* out_len) { job_submit_data_t submit; memset(&submit, 0, sizeof(submit)); submit.queue = (char*)queue; submit.user = (char*)"b857-tester"; xce_job_t j; memset(&j, 0, sizeof(j)); j.jobid = jobid; j.state = JOB_STAT_PEND; j.submit = &submit; struct capn c; struct capn_segment* cs; struct Job cj; Job_ptr jp; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_Job(cs, &cj, &j); jp = new_Job(cs); write_Job(&cj, jp); capn_setp(capn_root(&c), 0, jp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } static bool b857_job_exists(var jobmgr, uint64_t jobid) { void* existing = NULL; lock(jobmgr, RTTI_LOCK_READ); int found = key_find(jobmgr, (void*)jobid, &existing); unlock(jobmgr); return found > 0 && existing != NULL; } /* Mirrors test_upgrade_snapshot_apply.c's tusa_snapshot_exists path * construction exactly (same "snap-.capnp" naming convention the * production checkpoint code uses). */ static void b857_snapshot_path(persist_t* p, uint64_t lsn, char* buf, size_t bufsz) { snprintf(buf, bufsz, "%s/snap-%016" PRIx64 ".capnp", p->snap_path, lsn); } /* ── Scenario 1 (Req 6.2, "reproduce ... on the pre-fix code") ──────────── */ static void test_bug857_threshold_checkpoint_diverges_across_members(void) { b857_t A, B; b857_setup(&A, "b857oa"); b857_setup(&B, "b857ob"); uint8_t* job1_payload = NULL; size_t job1_len = 0; int rc_enc1 = b857_encode_job(857001, "b857q", &job1_payload, &job1_len); uint8_t marker_sha[32]; memset(marker_sha, 0xB8, sizeof(marker_sha)); uint8_t marker_payload[WAL_CONFIG_MARKER_PAYLOAD_LEN]; size_t marker_out_len = 0; int rc_marker_enc = config_marker_encode(0x8570000000000001ULL, marker_sha, marker_payload, WAL_CONFIG_MARKER_PAYLOAD_LEN, &marker_out_len); uint8_t* job2_payload = NULL; size_t job2_len = 0; int rc_enc2 = b857_encode_job(857002, "b857q", &job2_payload, &job2_len); /* Both members physically receive the IDENTICAL committed sequence -- * job1 @ lsn1, config-change marker @ lsn2, job2 @ lsn3 -- mirroring * what majority replication of a real cluster guarantees. */ int rc_wal_a1 = persist_wal_async(&A.p, WAL_JOB_SUBMIT, job1_payload, (uint32_t)job1_len); int rc_wal_a2 = persist_wal_async(&A.p, WAL_CONFIG_MARKER, marker_payload, WAL_CONFIG_MARKER_PAYLOAD_LEN); int rc_wal_a3 = persist_wal_async(&A.p, WAL_JOB_SUBMIT, job2_payload, (uint32_t)job2_len); int rc_wal_b1 = persist_wal_async(&B.p, WAL_JOB_SUBMIT, job1_payload, (uint32_t)job1_len); int rc_wal_b2 = persist_wal_async(&B.p, WAL_CONFIG_MARKER, marker_payload, WAL_CONFIG_MARKER_PAYLOAD_LEN); int rc_wal_b3 = persist_wal_async(&B.p, WAL_JOB_SUBMIT, job2_payload, (uint32_t)job2_len); /* Member A's own threshold trips EARLY -- right after job1, before job2 * exists locally. Entity effect driven directly via restore_job(), the * exact function apply_committed_entry()'s WAL_JOB_SUBMIT case itself * calls -- the marker's LSN is never dispatched through * apply_committed_entry() at all, so the CURRENT codebase's * marker-anchored checkpoint (task 5) never fires here, faithfully * reproducing the OLD, pre-task-5 decoupling: checkpoint content/timing * driven only by schedule_checkpoint's independent WAL-record * threshold (simulated by calling persist_checkpoint(), its real * production entry point, directly). */ int rc_restore_a1 = restore_job(&A.c, 0, job1_payload, job1_len, 1); atomic_store_explicit(&A.p.last_applied_lsn, 1, memory_order_relaxed); int rc_ckpt_a = persist_checkpoint(&A.p); uint64_t ckpt_lsn_a = A.p.checkpoint_lsn; /* Member B's own threshold does not trip until LATER -- by then job2 * was already locally visible too (ordinary cluster activity continued * around the upgrade before B happened to checkpoint/restart). */ int rc_restore_b1 = restore_job(&B.c, 0, job1_payload, job1_len, 1); int rc_restore_b2 = restore_job(&B.c, 0, job2_payload, job2_len, 3); atomic_store_explicit(&B.p.last_applied_lsn, 3, memory_order_relaxed); int rc_ckpt_b = persist_checkpoint(&B.p); uint64_t ckpt_lsn_b = B.p.checkpoint_lsn; bool job1_in_a = b857_job_exists(A.c.jobmgr, 857001); bool job1_in_b = b857_job_exists(B.c.jobmgr, 857001); bool job2_in_a = b857_job_exists(A.c.jobmgr, 857002); bool job2_in_b = b857_job_exists(B.c.jobmgr, 857002); char snap_a[4096], snap_b[4096]; b857_snapshot_path(&A.p, ckpt_lsn_a, snap_a, sizeof(snap_a)); b857_snapshot_path(&B.p, ckpt_lsn_b, snap_b, sizeof(snap_b)); snapdiff_report_t rep; memset(&rep, 0, sizeof(rep)); int diff_rc = snapdiff_paths(snap_a, snap_b, NULL, &rep); const uint64_t marker_lsn = 2; b857_teardown(&A); b857_teardown(&B); free(job1_payload); free(job2_payload); TEST_ASSERT_EQUAL_INT(0, rc_enc1); TEST_ASSERT_EQUAL_INT(0, rc_marker_enc); TEST_ASSERT_EQUAL_INT(0, rc_enc2); TEST_ASSERT_EQUAL_INT(0, rc_wal_a1); TEST_ASSERT_EQUAL_INT(0, rc_wal_a2); TEST_ASSERT_EQUAL_INT(0, rc_wal_a3); TEST_ASSERT_EQUAL_INT(0, rc_wal_b1); TEST_ASSERT_EQUAL_INT(0, rc_wal_b2); TEST_ASSERT_EQUAL_INT(0, rc_wal_b3); TEST_ASSERT_EQUAL_INT(0, rc_restore_a1); TEST_ASSERT_EQUAL_INT(0, rc_restore_b1); TEST_ASSERT_EQUAL_INT(0, rc_restore_b2); TEST_ASSERT_EQUAL_INT(0, rc_ckpt_a); TEST_ASSERT_EQUAL_INT(0, rc_ckpt_b); /* Neither member's checkpoint is anchored to the marker's own lsn (2) * -- this IS the bug: "no snapshot pinning an identical post-upgrade * state" (bug-857's own root-cause wording). */ TEST_ASSERT_EQUAL_UINT64(1, ckpt_lsn_a); TEST_ASSERT_EQUAL_UINT64(3, ckpt_lsn_b); TEST_ASSERT_NOT_EQUAL(marker_lsn, ckpt_lsn_a); TEST_ASSERT_NOT_EQUAL(marker_lsn, ckpt_lsn_b); /* Real, content-level divergence -- the JOB SET each member actually * materialized genuinely differs, not merely a cosmetic LSN-header * mismatch. */ TEST_ASSERT_TRUE(job1_in_a); TEST_ASSERT_TRUE(job1_in_b); TEST_ASSERT_FALSE_MESSAGE(job2_in_a, "member A's checkpoint predates job2 -- must NOT contain it"); TEST_ASSERT_TRUE_MESSAGE(job2_in_b, "member B's checkpoint postdates job2 -- must contain it"); /* And the resulting ON-DISK ARTIFACTS -- what a rejoining or * catching-up member would actually receive via snapshot transfer -- * genuinely diverge too. */ TEST_ASSERT_EQUAL_INT(1, diff_rc); TEST_ASSERT_FALSE_MESSAGE(rep.identical, "pre-fix: two members' checkpoints materialized around the SAME " "config-change marker must NOT be byte-identical -- this is the " "bug-857 divergence, reproduced"); printf("test_bug857_threshold_checkpoint_diverges_across_members: " "PASS\n"); } /* ── Scenario 2 (Req 6.2, "... and pass on the fixed code") ─────────────── */ static void test_bug857_marker_checkpoint_converges_across_members(void) { b857_t A, B; b857_setup(&A, "b857na"); b857_setup(&B, "b857nb"); uint8_t* job1_payload = NULL; size_t job1_len = 0; int rc_enc1 = b857_encode_job(857101, "b857q", &job1_payload, &job1_len); uint8_t marker_sha[32]; memset(marker_sha, 0xC9, sizeof(marker_sha)); uint8_t marker_payload[WAL_CONFIG_MARKER_PAYLOAD_LEN]; size_t marker_out_len = 0; int rc_marker_enc = config_marker_encode(0x8570000000000002ULL, marker_sha, marker_payload, WAL_CONFIG_MARKER_PAYLOAD_LEN, &marker_out_len); uint8_t* job2_payload = NULL; size_t job2_len = 0; int rc_enc2 = b857_encode_job(857102, "b857q", &job2_payload, &job2_len); /* Member A: catches up in a SINGLE pump-run call over the whole range * [job1, marker, job2] -- as if it processed everything in one * uninterrupted burst. */ int rc_wal_a1 = persist_wal_async(&A.p, WAL_JOB_SUBMIT, job1_payload, (uint32_t)job1_len); int rc_wal_a2 = persist_wal_async(&A.p, WAL_CONFIG_MARKER, marker_payload, WAL_CONFIG_MARKER_PAYLOAD_LEN); int rc_wal_a3 = persist_wal_async(&A.p, WAL_JOB_SUBMIT, job2_payload, (uint32_t)job2_len); uint64_t committed_a = persist_committed_lsn_advance_atomic(&A.p, 3); int applied_a = persist_apply_pump_run(&A.p); /* Member B: processes job1+marker FIRST (one pump-run call, simulating * a pause/restart boundary exactly at the marker), and only later * commits+applies job2 in a SEPARATE second pump-run call -- * genuinely different real-world processing pace than member A. */ int rc_wal_b1 = persist_wal_async(&B.p, WAL_JOB_SUBMIT, job1_payload, (uint32_t)job1_len); int rc_wal_b2 = persist_wal_async(&B.p, WAL_CONFIG_MARKER, marker_payload, WAL_CONFIG_MARKER_PAYLOAD_LEN); uint64_t committed_b1 = persist_committed_lsn_advance_atomic(&B.p, 2); int applied_b1 = persist_apply_pump_run(&B.p); int rc_wal_b3 = persist_wal_async(&B.p, WAL_JOB_SUBMIT, job2_payload, (uint32_t)job2_len); uint64_t committed_b2 = persist_committed_lsn_advance_atomic(&B.p, 3); int applied_b2 = persist_apply_pump_run(&B.p); const uint64_t marker_lsn = 2; char snap_a[4096], snap_b[4096]; b857_snapshot_path(&A.p, marker_lsn, snap_a, sizeof(snap_a)); b857_snapshot_path(&B.p, marker_lsn, snap_b, sizeof(snap_b)); bool snap_a_exists = (access(snap_a, F_OK) == 0); bool snap_b_exists = (access(snap_b, F_OK) == 0); snapdiff_report_t rep; memset(&rep, 0, sizeof(rep)); int diff_rc = snapdiff_paths(snap_a, snap_b, NULL, &rep); b857_teardown(&A); b857_teardown(&B); free(job1_payload); free(job2_payload); TEST_ASSERT_EQUAL_INT(0, rc_enc1); TEST_ASSERT_EQUAL_INT(0, rc_marker_enc); TEST_ASSERT_EQUAL_INT(0, rc_enc2); TEST_ASSERT_EQUAL_INT(0, rc_wal_a1); TEST_ASSERT_EQUAL_INT(0, rc_wal_a2); TEST_ASSERT_EQUAL_INT(0, rc_wal_a3); TEST_ASSERT_EQUAL_INT(0, rc_wal_b1); TEST_ASSERT_EQUAL_INT(0, rc_wal_b2); TEST_ASSERT_EQUAL_INT(0, rc_wal_b3); TEST_ASSERT_EQUAL_UINT64(3, committed_a); TEST_ASSERT_EQUAL_INT(3, applied_a); TEST_ASSERT_EQUAL_UINT64(2, committed_b1); TEST_ASSERT_EQUAL_INT(2, applied_b1); TEST_ASSERT_EQUAL_UINT64(3, committed_b2); TEST_ASSERT_EQUAL_INT(1, applied_b2); TEST_ASSERT_TRUE_MESSAGE(snap_a_exists, "member A must checkpoint at the marker's own lsn, even mid-batch " "within a single pump-run call"); TEST_ASSERT_TRUE_MESSAGE(snap_b_exists, "member B must checkpoint at the marker's own lsn, even though its " "own processing was split across two separate pump-run calls"); /* The decisive proof: regardless of each member's own processing pace * (A: one burst; B: paused exactly at the marker, job2 applied later), * both marker-anchored checkpoints are byte-identical -- convergence, * closing the bug-857 divergence Scenario 1 reproduced above. */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, diff_rc, "post-fix: both members' marker-anchored checkpoints must be " "byte-identical regardless of differing processing pace"); TEST_ASSERT_TRUE_MESSAGE(rep.identical, "post-fix: materialized state converges identically across " "members -- bug-857 no longer occurs"); printf("test_bug857_marker_checkpoint_converges_across_members: " "PASS\n"); } /* ── Dispatch ────────────────────────────────────────────────────────── */ void test_bug857_regression(void) { UnitySetTestFile(__FILE__); printf("\n>> bug-857 materialized-state divergence regression " "(task 8.5)\n\n"); RUN_TEST(test_bug857_threshold_checkpoint_diverges_across_members); RUN_TEST(test_bug857_marker_checkpoint_converges_across_members); }