/* test_wal_job_ckpt_fact.c — bug-1691 regression suite * * Proves WAL_JOB_CKPT's payload -- previously bare wal_jobstate_t{jobid, * state}, exactly the same silent-field-drop bug class as bug-1678 * (WAL_JOB_DISPATCH/job.allocs), bug-1687 (WAL_JOB_START/job.pidinfo+ * start_time), bug-1689 (WAL_JOB_FINISH/job.exit_info+finish_time), and * bug-1690 (WAL_JOB_RUSAGE/job.run_usage) -- now carries the worker/ * scheduler-reported checkpoint info string (job.ckptinfo, a plain char*) * through BOTH of persist.c's apply seams: * * - apply_committed_entry() (live-commit apply / boot replay dispatch) * - apply_wal_entry() (standby catch-up / replicated-apply dispatch) * * job.ckptinfo (xce/types.h) is a plain NUL-terminated char* -- NOT the * richer xce_ckptinfo_t/CheckpointInfo capnp struct used elsewhere for a * different purpose (that struct is on_job_ckpt()'s OWN parameter type, * used only to carry the incoming report; the FIELD this fix restores is * job.ckptinfo itself). Being a plain string, NO capnp struct is needed at * all -- the fix appends job.ckptinfo as a NUL-terminated string directly * after the wal_jobstate_t header, mirroring WAL_QUEUE_CTRL's own * [header][name NUL-terminated] wire shape exactly (persist_queue_hook_fun / * apply_wal_entry's WAL_QUEUE_CTRL case), rather than DISPATCH/START/RUSAGE's * capnp-wrapper approach or FINISH's fixed-size native trailer. * * IMPORTANT DIVERGENCE FROM WAL_JOB_START/FINISH/DISPATCH's regression * suites, SAME AS bug-1690's WAL_JOB_RUSAGE: on_job_ckpt() (job.c) never * changes job.state -- it only ever sets job.ckptinfo. persist_job_hook_fun's * new HOOK_DETAIL_JOB_CKPT case captures js.state verbatim from the job's * CURRENT (unchanged) state, so apply_job_update_transition() sees wire * state == current state and reports JOB_TRANSITION_NOOP_DUPLICATE on every * single real CKPT replay. Gating ckptinfo adoption on * result==JOB_TRANSITION_APPLIED (the way START/FINISH/DISPATCH do) would * therefore make it dead code for the realistic, only-ever-occurring case. * The fix decodes/adopts ckptinfo UNCONDITIONALLY, before that gate. * test_wjc_apply_committed_entry_sets_ckptinfo_no_state_change below is this * suite's most important test -- it is the scenario a naive copy of the * START/FINISH/DISPATCH pattern would have silently failed to cover. * * ALSO DIVERGES FROM bug-1690's WAL_JOB_RUSAGE fix in one respect: adoption * here is a plain free-and-replace (dereference() the old value, adopt a * fresh makeString()'d copy), NOT a merge. job.ckptinfo IS xstring-backed * (makeString(), xstring.c) exactly like RUSAGE's host_name field was, and * CAN be the job's own live value when the primary's own live apply pump * re-applies a fact moments after on_job_ckpt() already set it -- but * dereference() is the correct RTTI release verb for an xstring regardless * of whether the pointer is "old" or coincidentally "the current live one" * (it decrements a refcount rather than raw free()ing a non-owned pointer), * so no merge-vs-replace hazard exists here the way it did for RUSAGE's * capnp-generated free_JobRunusage_ptr() (a plain-malloc-oriented free * function that would have corrupted the heap on an xstring pointer). * * Also proves backward compatibility: a WAL_JOB_CKPT entry with the OLD * payload shape (exactly sizeof(wal_jobstate_t), no trailing bytes -- * simulating an entry written before this fix, or one fired while * job.ckptinfo was still NULL at write time) still decodes cleanly, with * ckptinfo left untouched exactly as before this fix -- no crash, no * misread. * * Fixtures mirror test_wal_job_finish_fact.c's own conventions exactly: * - wjc_setup/wjc_teardown: bare cluster_t (jobmgr + hooksys) + in-memory * persist_t. * - wjc_make_fresh_persist/wjc_teardown_persist: real disk-backed * persist_t + persist_recover(), for restart-replay tests. * * test_wjc_real_on_job_ckpt_survives_restart drives the REAL, unmodified * on_job_ckpt() (the actual worker/scheduler-CKPT receipt handler -- * cluster.c's NotifyData_data_ckptinfo dispatch's only callee) through a * real WAL-write + persist_recover() restart cycle -- unlike bug-1690's * WAL_JOB_RUSAGE suite, on_job_ckpt() has no misc.c-style stubbed dependency * in this test binary, so it can be driven directly (mirrors * test_wjf_real_on_job_finish_survives_restart's simpler on_job_finish() * pattern, not RUSAGE's persist_job_hook_fun trampoline workaround). */ #include #include #include #include #include #include #include #include #include #include #include "cluster.h" #include "controller.capnp.h" #include "hook.h" #include "jobj.h" #include "persist.h" #include "wal_types.h" #include "xce/cluster_def.h" #include "xce/types.h" #include "xcealloc.h" #include "xstring.h" /* jobj.c's jobsub table and the process-global xstring table are shared * across every test file in the controller_unit_test binary — guard * idempotently, mirroring every sibling fixture in this suite. */ static int g_wjc_jobobj_ready; static cluster_t g_wjc_c; static persist_t g_wjc_p; /* ── Spy hook: counts AFTER_EVENT firings, ignores all payload args ─────── */ static int g_wjc_hook_fires; static int wjc_spy_hook(int type, int situation, int detail, void* data, va_list ap) { (void)type; (void)detail; (void)data; (void)ap; if (situation == HOOK_AFTER_EVENT) g_wjc_hook_fires++; return 0; } static void wjc_setup(void) { kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16 }; if (makeString((char*)"wjc-probe") == NULL) xstring_table_init(64); if (!g_wjc_jobobj_ready) { TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32)); g_wjc_jobobj_ready = 1; } memset(&g_wjc_c, 0, sizeof(g_wjc_c)); TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&g_wjc_c.hooksys, XCE_CONTROLLER_HOOK_MAX)); g_wjc_c.jobmgr = makeInstance(&xkvmgr_type, &int_params); TEST_ASSERT_NOT_NULL(g_wjc_c.jobmgr); memset(&g_wjc_p, 0, sizeof(g_wjc_p)); g_wjc_p.cluster = &g_wjc_c; g_wjc_hook_fires = 0; } static void wjc_teardown(void) { xce_hook_fini(&g_wjc_c.hooksys); if (g_wjc_c.jobmgr) dereference(g_wjc_c.jobmgr); } /* Look up a job_t wrapper by jobid (NULL if absent). */ static job_t* wjc_find_job(uint64_t jobid) { job_t* j = NULL; lock(g_wjc_c.jobmgr, RTTI_LOCK_READ); int found = key_find(g_wjc_c.jobmgr, (void*)jobid, (void**)&j); unlock(g_wjc_c.jobmgr); return (found > 0) ? j : NULL; } /* Creates a job directly in the fixture's jobmgr, bypassing dispatch/replay * entirely — for seeding fact-transition tests, mirroring * test_wal_job_finish_fact.c's wjf_seed_job / test_wal_job_rusage_fact.c's * wru_seed_job. */ static job_submit_data_t g_wjc_seed_submit; static job_t* wjc_seed_job(uint64_t jobid, uint32_t state) { xce_job_t jdef; memset(&g_wjc_seed_submit, 0, sizeof(g_wjc_seed_submit)); g_wjc_seed_submit.queue = (char*)"q"; g_wjc_seed_submit.user = (char*)"tester"; memset(&jdef, 0, sizeof(jdef)); jdef.jobid = jobid; jdef.state = state; jdef.submit = &g_wjc_seed_submit; return makeJob(g_wjc_c.jobmgr, &jdef); } /* Build a wal_jobstate_t header + trailing NUL-terminated ckptinfo string -- * persist_job_hook_fun's WAL_JOB_CKPT payload shape (bug-1691), mirroring * WAL_QUEUE_CTRL's own [header][name NUL-terminated] wire shape. Caller owns * *out. */ static void wjc_build_ckpt_payload(uint64_t jobid, uint32_t state, const char* ckptinfo, uint8_t** out, size_t* out_len) { wal_jobstate_t js = { .jobid = jobid, .state = state, ._pad = 0 }; size_t info_len = strlen(ckptinfo) + 1; *out_len = sizeof(js) + info_len; *out = malloc(*out_len); TEST_ASSERT_NOT_NULL(*out); memcpy(*out, &js, sizeof(js)); memcpy(*out + sizeof(js), ckptinfo, info_len); } static void wjc_assert_ckptinfo_matches(job_t* j, const char* expected) { TEST_ASSERT_NOT_NULL_MESSAGE(j->job.ckptinfo, "a WAL_JOB_CKPT payload with a trailing ckptinfo string must set " "job.ckptinfo (bug-1691), not just state"); TEST_ASSERT_EQUAL_STRING(expected, j->job.ckptinfo); } /* ── apply_committed_entry(): live-commit apply / boot replay dispatch ──── */ static void test_wjc_apply_committed_entry_sets_ckptinfo_no_state_change(void) { wjc_setup(); job_t* j = wjc_seed_job(74001, JOB_STAT_RUN); TEST_ASSERT_NOT_NULL(j); uint8_t* payload = NULL; size_t plen = 0; wjc_build_ckpt_payload(74001, JOB_STAT_RUN, "ckpt-blob-1", &payload, &plen); xce_add_hook(&g_wjc_c.hooksys, XCE_CONTROLLER_HOOK_JOB, wjc_spy_hook, NULL); int rc = apply_committed_entry(&g_wjc_p, WAL_JOB_CKPT, 300, payload, (uint32_t)plen); free(payload); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state); wjc_assert_ckptinfo_matches(j, "ckpt-blob-1"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_wjc_hook_fires, "a same-state (duplicate) CKPT replay -- the realistic, only-ever-" "occurring case, since on_job_ckpt() never changes job.state -- " "must NOT fire a notification hook, unchanged from before this fix, " "even though ckptinfo WAS adopted"); wjc_teardown(); printf("test_wjc_apply_committed_entry_sets_ckptinfo_no_state_change: " "PASS\n"); } /* Secondary scenario: a WAL_JOB_CKPT fact that DOES carry a forward state * transition (mechanism-level test, mirroring how bug-1690's WAL_JOB_RUSAGE * suite covers this same divergence). ckptinfo must be adopted here too, and * the existing forward-transition hook-fire behavior must be unchanged. */ static void test_wjc_apply_committed_entry_sets_ckptinfo_forward_transition(void) { wjc_setup(); job_t* j = wjc_seed_job(74002, JOB_STAT_PEND); TEST_ASSERT_NOT_NULL(j); uint8_t* payload = NULL; size_t plen = 0; wjc_build_ckpt_payload(74002, JOB_STAT_RUN, "ckpt-blob-2", &payload, &plen); xce_add_hook(&g_wjc_c.hooksys, XCE_CONTROLLER_HOOK_JOB, wjc_spy_hook, NULL); int rc = apply_committed_entry(&g_wjc_p, WAL_JOB_CKPT, 301, payload, (uint32_t)plen); free(payload); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state); wjc_assert_ckptinfo_matches(j, "ckpt-blob-2"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_wjc_hook_fires, "a forward CKPT-carried transition must still fire exactly one " "notification, unchanged from before this fix"); wjc_teardown(); printf( "test_wjc_apply_committed_entry_sets_ckptinfo_forward_transition: " "PASS\n"); } static void test_wjc_apply_committed_entry_backward_compat_old_payload(void) { wjc_setup(); job_t* j = wjc_seed_job(74003, JOB_STAT_RUN); TEST_ASSERT_NOT_NULL(j); TEST_ASSERT_NULL_MESSAGE(j->job.ckptinfo, "fixture must seed a job with ckptinfo still unset"); wal_jobstate_t js = { .jobid = 74003, .state = JOB_STAT_RUN, ._pad = 0 }; int rc = apply_committed_entry(&g_wjc_p, WAL_JOB_CKPT, 302, &js, sizeof(js)); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state); TEST_ASSERT_NULL_MESSAGE(j->job.ckptinfo, "an old-format (exactly sizeof(wal_jobstate_t)) WAL_JOB_CKPT " "payload must leave ckptinfo NULL -- no crash, no misread of " "absent trailing bytes"); wjc_teardown(); printf("test_wjc_apply_committed_entry_backward_compat_old_payload: " "PASS\n"); } /* ── apply_wal_entry(): standby catch-up / replicated-apply dispatch ────── */ static void test_wjc_apply_wal_entry_sets_ckptinfo_no_hook(void) { wjc_setup(); job_t* j = wjc_seed_job(74004, JOB_STAT_RUN); TEST_ASSERT_NOT_NULL(j); uint8_t* payload = NULL; size_t plen = 0; wjc_build_ckpt_payload(74004, JOB_STAT_RUN, "ckpt-blob-4", &payload, &plen); xce_add_hook(&g_wjc_c.hooksys, XCE_CONTROLLER_HOOK_JOB, wjc_spy_hook, NULL); int rc = apply_wal_entry(&g_wjc_p, WAL_JOB_CKPT, 303, payload, (uint32_t)plen); free(payload); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state); wjc_assert_ckptinfo_matches(j, "ckpt-blob-4"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_wjc_hook_fires, "apply_wal_entry's catch-up dispatch must never fire a " "notification hook"); wjc_teardown(); printf("test_wjc_apply_wal_entry_sets_ckptinfo_no_hook: PASS\n"); } static void test_wjc_apply_wal_entry_backward_compat_old_payload(void) { wjc_setup(); job_t* j = wjc_seed_job(74005, JOB_STAT_RUN); TEST_ASSERT_NOT_NULL(j); TEST_ASSERT_NULL_MESSAGE(j->job.ckptinfo, "fixture must seed a job with ckptinfo still unset"); wal_jobstate_t js = { .jobid = 74005, .state = JOB_STAT_RUN, ._pad = 0 }; int rc = apply_wal_entry(&g_wjc_p, WAL_JOB_CKPT, 304, &js, sizeof(js)); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state); TEST_ASSERT_NULL_MESSAGE(j->job.ckptinfo, "an old-format WAL_JOB_CKPT payload must leave ckptinfo NULL on " "this seam too"); wjc_teardown(); printf("test_wjc_apply_wal_entry_backward_compat_old_payload: PASS\n"); } /* ── Restart-replay Observables ──────────────────────────────────────── */ static persist_t* wjc_make_fresh_persist(const char* tmpdir) { persist_t* p = calloc(1, sizeof(persist_t)); if (!p) return NULL; persist_config_t cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; /* slow fsync — no races in tests */ persist_set_workdir(p, tmpdir, "ctrl-test"); if (persist_init(p, &cfg, NULL) != 0) { free(p); return NULL; } if (persist_start(p) != 0) { persist_fini(p); free(p); return NULL; } return p; } static void wjc_teardown_persist(persist_t* p) { persist_stop(p); persist_fini(p); free(p); } /* Encode helper mirroring test_wal_job_finish_fact.c's wjf_encode_job * (capnp_encode_job is file-static in persist.c). */ static int wjc_encode_job(const xce_job_t* j, uint8_t** out, size_t* out_len) { struct capn c; struct capn_segment* cs; struct Job cj; Job_ptr jp; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_Job(cs, &cj, (xce_job_t*)j); jp = new_Job(cs); write_Job(&cj, jp); capn_setp(capn_root(&c), 0, jp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } /* Direct-payload restart replay: appends WAL_JOB_SUBMIT + a new-format * WAL_JOB_CKPT (real ckptinfo string) straight onto disk via * persist_wal_async, then replays via a fresh persist_t's persist_recover() * (a real restart). Isolates the WAL fact shape from the live caller, * mirroring test_wal_job_finish_fact.c's own restart-replay test. */ static void test_wjc_restart_replay_recovers_ckptinfo(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_wjc_replay_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const char* wire_ckptinfo = "ckpt-blob-replay"; /* Phase 1: write the log to real disk. */ { persist_t* p = wjc_make_fresh_persist(tmp); TEST_ASSERT_NOT_NULL(p); job_submit_data_t submit; memset(&submit, 0, sizeof(submit)); submit.queue = (char*)"q"; submit.user = (char*)"tester"; xce_job_t jdef; memset(&jdef, 0, sizeof(jdef)); jdef.jobid = 74006; jdef.state = JOB_STAT_RUN; jdef.submit = &submit; uint8_t* job_payload = NULL; size_t job_plen = 0; TEST_ASSERT_EQUAL_INT(0, wjc_encode_job(&jdef, &job_payload, &job_plen)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_SUBMIT, job_payload, (uint32_t)job_plen)); free(job_payload); uint8_t* ckpt_payload = NULL; size_t ckpt_plen = 0; wjc_build_ckpt_payload(74006, JOB_STAT_RUN, wire_ckptinfo, &ckpt_payload, &ckpt_plen); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_CKPT, ckpt_payload, (uint32_t)ckpt_plen)); free(ckpt_payload); TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); wjc_teardown_persist(p); } /* Phase 2: fresh persist_t + fresh cluster_t, recover from disk — * simulates a controller restart. */ { persist_t* p2 = wjc_make_fresh_persist(tmp); TEST_ASSERT_NOT_NULL(p2); wjc_setup(); p2->cluster = &g_wjc_c; int rc = persist_recover(p2); TEST_ASSERT_EQUAL_INT(0, rc); job_t* j = wjc_find_job(74006); TEST_ASSERT_NOT_NULL_MESSAGE(j, "restart replay must construct the job via WAL_JOB_SUBMIT"); TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state); wjc_assert_ckptinfo_matches(j, wire_ckptinfo); wjc_teardown(); wjc_teardown_persist(p2); } char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp); if (system(cmd) != 0) fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp); printf("test_wjc_restart_replay_recovers_ckptinfo: PASS\n"); } /* bug-1691's primary regression test: drives the REAL production * on_job_ckpt() path (the actual worker/scheduler-CKPT receipt handler -- * cluster.c's NotifyData_data_ckptinfo dispatch's only callee) with a real * xce_ckptinfo_t, through a real WAL-write + persist_recover() restart cycle * -- the exact technique test_wjf_real_on_job_finish_survives_restart * (bug-1689) already established. Asserts the job's ckptinfo survives the * restart intact, not just state -- the audit's exact confirmed root * cause. */ static void test_wjc_real_on_job_ckpt_survives_restart(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_wjc_onckpt_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t jobid = 74007; const char* real_ckptinfo_wire = "real-ckpt-info-from-on-job-ckpt"; /* Phase 1: real cluster + real disk persist. Seed WAL_JOB_SUBMIT * directly (so replay can reconstruct the job) AND a live in-memory job * object (so the real on_job_ckpt() call site has something to * find/lock) -- mirrors test_wjf_real_on_job_finish_survives_restart's * dual-seed pattern exactly. Then drive an actual on_job_ckpt() call * and let IT append WAL_JOB_CKPT via persist_job_hook_fun -- this is * the production code path under test, not a hand-built WAL entry. */ { persist_config_t cfg; if (makeString((char*)"wjc-onckpt-probe") == NULL) xstring_table_init(64); if (!g_wjc_jobobj_ready) { TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32)); g_wjc_jobobj_ready = 1; } cluster_t c; kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16 }; memset(&c, 0, sizeof(c)); TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX)); c.jobmgr = makeInstance(&xkvmgr_type, &int_params); TEST_ASSERT_NOT_NULL(c.jobmgr); cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; persist_set_workdir(&c.persist, tmp, "ctrl-test"); /* Unlike test_wjf_real_on_job_finish_survives_restart's identical- * looking fixture (bug-1689, a DIRECT persist_submit_job_finish_ * fact() call site -- no hook registration needed, so that sibling * test can pass NULL here), on_job_ckpt() (job.c) is hook-driven: * it only ever calls xce_run_hooks(..., HOOK_DETAIL_JOB_CKPT, ...) * and relies ENTIRELY on persist_job_hook_fun (this fix's actual * changed code) being registered as a c.hooksys subscriber to reach * the WAL at all. persist_start() only registers that hook `if * (p->cluster != NULL)` -- passing &c (not NULL) here is required * for the real production path to append anything. */ TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, &c)); TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist)); /* Durable WAL_JOB_SUBMIT seed. */ { job_submit_data_t wal_submit; xce_job_t wal_jdef; uint8_t* buf = NULL; size_t sz = 0; memset(&wal_submit, 0, sizeof(wal_submit)); wal_submit.queue = (char*)"oq"; wal_submit.user = (char*)"tester"; memset(&wal_jdef, 0, sizeof(wal_jdef)); wal_jdef.jobid = jobid; wal_jdef.state = JOB_STAT_RUN; wal_jdef.submit = &wal_submit; TEST_ASSERT_EQUAL_INT(0, wjc_encode_job(&wal_jdef, &buf, &sz)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf, (uint32_t)sz)); free(buf); } /* Live in-memory job object, already RUN, for on_job_ckpt() to * find/lock. */ { job_submit_data_t submit; xce_job_t jdef; memset(&submit, 0, sizeof(submit)); submit.queue = (char*)"oq"; submit.user = (char*)"tester"; memset(&jdef, 0, sizeof(jdef)); jdef.jobid = jobid; jdef.state = JOB_STAT_RUN; jdef.submit = &submit; TEST_ASSERT_NOT_NULL(makeJob(c.jobmgr, &jdef)); } /* The real worker/scheduler-reported checkpoint report. `ep` is * unused by on_job_ckpt()'s body -- NULL is safe, mirroring how * production call sites pass whatever endpoint_t* the transport * layer happened to receive it on, which this fact-recording path * never dereferences. */ xce_ckptinfo_t ckptinfo = { .jobid = jobid, .ckptinfo = (char*)real_ckptinfo_wire }; int rc = on_job_ckpt(&c, NULL, &ckptinfo); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist)); persist_stop(&c.persist); persist_fini(&c.persist); dereference(c.jobmgr); xce_hook_fini(&c.hooksys); } /* Phase 2: fresh persist_t + fresh cluster_t (jobmgr/hooksys only), * recover from disk — simulates a controller restart. */ { persist_t* p2 = wjc_make_fresh_persist(tmp); cluster_t c2; job_t* j2 = NULL; int rc; kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16 }; TEST_ASSERT_NOT_NULL(p2); memset(&c2, 0, sizeof(c2)); TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX)); c2.jobmgr = makeInstance(&xkvmgr_type, &int_params); TEST_ASSERT_NOT_NULL(c2.jobmgr); p2->cluster = &c2; rc = persist_recover(p2); TEST_ASSERT_EQUAL_INT(0, rc); lock(c2.jobmgr, RTTI_LOCK_READ); key_find(c2.jobmgr, (void*)jobid, (void**)&j2); unlock(c2.jobmgr); TEST_ASSERT_NOT_NULL_MESSAGE(j2, "restart replay must reconstruct the job via WAL_JOB_SUBMIT"); TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j2->job.state, "restart replay must recover the job's state, unchanged by the " "real on_job_ckpt() call"); /* bug-1691: restart replay must recover the worker/scheduler- * reported ckptinfo string -- the audit's exact confirmed root * cause (mirrors bug-1678/1687/1689/1690's own acceptance * criterion). */ wjc_assert_ckptinfo_matches(j2, real_ckptinfo_wire); xce_hook_fini(&c2.hooksys); if (c2.jobmgr) dereference(c2.jobmgr); wjc_teardown_persist(p2); } char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp); if (system(cmd) != 0) fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp); printf("test_wjc_real_on_job_ckpt_survives_restart: PASS\n"); } /* ── Test group entry point ───────────────────────────────────────────── */ void test_wal_job_ckpt_fact(void) { UnitySetTestFile(__FILE__); RUN_TEST(test_wjc_apply_committed_entry_sets_ckptinfo_no_state_change); RUN_TEST(test_wjc_apply_committed_entry_sets_ckptinfo_forward_transition); RUN_TEST(test_wjc_apply_committed_entry_backward_compat_old_payload); RUN_TEST(test_wjc_apply_wal_entry_sets_ckptinfo_no_hook); RUN_TEST(test_wjc_apply_wal_entry_backward_compat_old_payload); RUN_TEST(test_wjc_restart_replay_recovers_ckptinfo); RUN_TEST(test_wjc_real_on_job_ckpt_survives_restart); }