/* test_wal_job_fail_fact.c — bug-1754 regression suite * * A worker reports EVERY nonzero job exit as a FAIL fact, not a FINISH * (xcecd_job_finished -> xcecd_notify_job_fail(jobid, stage, exit_code), * worker/src/controller_connector.c), so job_failinfo_t.code is the job's * real exit code for a MAIN-stage failure. WAL_JOB_FAIL's deferred-apply * payload used to be a bare wal_jobstate_t{jobid, 0, STAGE_FLAG|stage} with * no code slot, and apply_job_fail_transition() never touched exit_info -- * so job.exit_info stayed NULL for every failed job, live and after replay. * * Proves WAL_JOB_FAIL now carries a trailing int32_t code and that BOTH of * persist.c's apply seams adopt it into job.exit_info: * * - apply_committed_entry() (live-commit apply / boot replay dispatch) * - apply_wal_entry() (standby catch-up / replicated-apply dispatch) * * that a POST_EXEC-stage failure (the post-exec SCRIPT's code, not the * job's) leaves the job's own exit_info untouched, that an old-format entry * (no trailer) still decodes with exit_info left NULL, and that a real * on_job_fail() call survives a real WAL write + persist_recover() restart * with job.exit_info intact. * * Fixtures mirror test_wal_job_finish_fact.c (bug-1689) exactly. */ #include #include #include #include #include #include #include #include #include #include #include #include "cluster.h" #include "controller.capnp.h" #include "hook.h" #include "jobj.h" #include "persist.h" #include "wal_types.h" #include "xce/cluster_def.h" #include "xce/types.h" #include "xcealloc.h" #include "xstring.h" static int g_wjfl_jobobj_ready; static cluster_t g_wjfl_c; static persist_t g_wjfl_p; /* Captures the job_failinfo_t HOOK_DETAIL_JOB_FAIL carries to its AFTER_EVENT * subscribers (sched_job_fail_fun / apiserver_job_fail_fun read the same * (cluster_t*, job_failinfo_t*, job_t*) shape). */ static int g_wjfl_fail_hook_fires; static int g_wjfl_captured_code; static int wjfl_fail_spy_hook(int type, int situation, int detail, void* data, va_list ap) { (void)type; (void)data; if (situation != HOOK_AFTER_EVENT || detail != HOOK_DETAIL_JOB_FAIL) return 0; (void)va_arg(ap, cluster_t*); job_failinfo_t* info = va_arg(ap, job_failinfo_t*); g_wjfl_fail_hook_fires++; if (info != NULL) g_wjfl_captured_code = info->code; return 0; } static void wjfl_setup(void) { kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16 }; if (makeString((char*)"wjfl-probe") == NULL) xstring_table_init(64); if (!g_wjfl_jobobj_ready) { TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32)); g_wjfl_jobobj_ready = 1; } memset(&g_wjfl_c, 0, sizeof(g_wjfl_c)); TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&g_wjfl_c.hooksys, XCE_CONTROLLER_HOOK_MAX)); g_wjfl_c.jobmgr = makeInstance(&xkvmgr_type, &int_params); TEST_ASSERT_NOT_NULL(g_wjfl_c.jobmgr); memset(&g_wjfl_p, 0, sizeof(g_wjfl_p)); g_wjfl_p.cluster = &g_wjfl_c; g_wjfl_fail_hook_fires = 0; g_wjfl_captured_code = 0; } static void wjfl_teardown(void) { xce_hook_fini(&g_wjfl_c.hooksys); if (g_wjfl_c.jobmgr) dereference(g_wjfl_c.jobmgr); } static job_submit_data_t g_wjfl_seed_submit; static job_t* wjfl_seed_job(uint64_t jobid, uint32_t state) { xce_job_t jdef; memset(&g_wjfl_seed_submit, 0, sizeof(g_wjfl_seed_submit)); g_wjfl_seed_submit.queue = (char*)"q"; g_wjfl_seed_submit.user = (char*)"tester"; memset(&jdef, 0, sizeof(jdef)); jdef.jobid = jobid; jdef.state = state; jdef.start_time = 1699999000; jdef.submit = &g_wjfl_seed_submit; return makeJob(g_wjfl_c.jobmgr, &jdef); } /* wal_jobstate_t{jobid, 0, STAGE_FLAG|stage} + int32_t code -- * persist_submit_job_fail_fact()'s payload shape. */ typedef struct { wal_jobstate_t js; int32_t code; } __attribute__((packed)) wjfl_fail_payload_t; static wjfl_fail_payload_t wjfl_fail_payload(uint64_t jobid, uint32_t stage, int32_t code) { wjfl_fail_payload_t pl; memset(&pl, 0, sizeof(pl)); pl.js.jobid = jobid; pl.js.state = 0; pl.js._pad = WAL_JOB_FAIL_STAGE_FLAG | stage; pl.code = code; return pl; } static void wjfl_assert_exit_code(job_exitinfo_t* got, int expected) { TEST_ASSERT_NOT_NULL_MESSAGE(got, "a MAIN-stage WAL_JOB_FAIL carrying a code must populate " "job.exit_info (bug-1754), not leave it NULL"); TEST_ASSERT_EQUAL_INT(expected, got->exitcode); } /* ── apply_committed_entry ────────────────────────────────────────────── */ static void test_wjfl_apply_committed_entry_sets_exit_info(void) { wjfl_setup(); job_t* j = wjfl_seed_job(74001, JOB_STAT_RUN); TEST_ASSERT_NOT_NULL(j); xce_add_hook(&g_wjfl_c.hooksys, XCE_CONTROLLER_HOOK_JOB, wjfl_fail_spy_hook, NULL); wjfl_fail_payload_t pl = wjfl_fail_payload(74001, JOB_EXEC_STAGE_MAIN, 42); int rc = apply_committed_entry(&g_wjfl_p, WAL_JOB_FAIL, 400, &pl, sizeof(pl)); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_UINT32(JOB_STAT_EXIT, j->job.state); wjfl_assert_exit_code(j->job.exit_info, 42); TEST_ASSERT_EQUAL_INT(1, g_wjfl_fail_hook_fires); TEST_ASSERT_EQUAL_INT_MESSAGE(42, g_wjfl_captured_code, "HOOK_DETAIL_JOB_FAIL subscribers must see the wire-carried code, " "not a fabricated 0"); wjfl_teardown(); } static void test_wjfl_apply_committed_entry_post_exec_keeps_job_exit_info(void) { wjfl_setup(); job_t* j = wjfl_seed_job(74002, JOB_STAT_DONE); TEST_ASSERT_NOT_NULL(j); j->job.exit_info = calloc(1, sizeof(job_exitinfo_t)); TEST_ASSERT_NOT_NULL(j->job.exit_info); j->job.exit_info->exitcode = 0; wjfl_fail_payload_t pl = wjfl_fail_payload(74002, JOB_EXEC_STAGE_POST_EXEC, 3); int rc = apply_committed_entry(&g_wjfl_p, WAL_JOB_FAIL, 401, &pl, sizeof(pl)); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_UINT32(JOB_STAT_DONE | JOB_STAT_PERR, j->job.state); TEST_ASSERT_NOT_NULL(j->job.exit_info); TEST_ASSERT_EQUAL_INT_MESSAGE(0, j->job.exit_info->exitcode, "a POST_EXEC failure's code is the post-exec script's, not the " "job's -- it must not overwrite the job's own exit code"); wjfl_teardown(); } static void test_wjfl_apply_committed_entry_backward_compat_old_payload(void) { wjfl_setup(); job_t* j = wjfl_seed_job(74003, JOB_STAT_RUN); TEST_ASSERT_NOT_NULL(j); wal_jobstate_t js = { .jobid = 74003, .state = 0, ._pad = WAL_JOB_FAIL_STAGE_FLAG | JOB_EXEC_STAGE_MAIN }; int rc = apply_committed_entry(&g_wjfl_p, WAL_JOB_FAIL, 402, &js, sizeof(js)); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_UINT32(JOB_STAT_EXIT, j->job.state); TEST_ASSERT_NULL_MESSAGE(j->job.exit_info, "an old-format WAL_JOB_FAIL (no code trailer) must leave exit_info " "NULL rather than fabricate an exit code"); wjfl_teardown(); } /* ── apply_wal_entry ──────────────────────────────────────────────────── */ static void test_wjfl_apply_wal_entry_sets_state_and_exit_info(void) { wjfl_setup(); job_t* j = wjfl_seed_job(74004, JOB_STAT_RUN); TEST_ASSERT_NOT_NULL(j); wjfl_fail_payload_t pl = wjfl_fail_payload(74004, JOB_EXEC_STAGE_MAIN, 9); int rc = apply_wal_entry(&g_wjfl_p, WAL_JOB_FAIL, 403, &pl, sizeof(pl)); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_EXIT, j->job.state, "a deferred (STAGE_FLAG) WAL_JOB_FAIL carries state 0 on the wire; " "the standby seam must recompute EXIT, not assign that 0"); wjfl_assert_exit_code(j->job.exit_info, 9); wjfl_teardown(); } /* ── Restart replay through the real on_job_fail() receipt path ───────── */ static int wjfl_encode_job(const xce_job_t* j, uint8_t** out, size_t* out_len) { struct capn c; struct capn_segment* cs; struct Job cj; Job_ptr jp; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_Job(cs, &cj, (xce_job_t*)j); jp = new_Job(cs); write_Job(&cj, jp); capn_setp(capn_root(&c), 0, jp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } static void test_wjfl_real_on_job_fail_survives_restart(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_wjfl_onfail_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t jobid = 74005; const int real_exit_code = 2; /* Phase 1: durable WAL_JOB_SUBMIT + WAL_JOB_START seeds, a live RUN job, * then the real worker-FAIL receipt handler appends WAL_JOB_FAIL. */ { persist_config_t cfg; cluster_t c; kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16 }; if (makeString((char*)"wjfl-onfail-probe") == NULL) xstring_table_init(64); if (!g_wjfl_jobobj_ready) { TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32)); g_wjfl_jobobj_ready = 1; } memset(&c, 0, sizeof(c)); TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX)); c.jobmgr = makeInstance(&xkvmgr_type, &int_params); TEST_ASSERT_NOT_NULL(c.jobmgr); cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; persist_set_workdir(&c.persist, tmp, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL)); TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist)); job_submit_data_t submit; xce_job_t jdef; uint8_t* buf = NULL; size_t sz = 0; memset(&submit, 0, sizeof(submit)); submit.queue = (char*)"oq"; submit.user = (char*)"tester"; memset(&jdef, 0, sizeof(jdef)); jdef.jobid = jobid; jdef.state = JOB_STAT_PEND; jdef.submit = &submit; TEST_ASSERT_EQUAL_INT(0, wjfl_encode_job(&jdef, &buf, &sz)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf, (uint32_t)sz)); free(buf); wal_jobstate_t start_js = { .jobid = jobid, .state = JOB_STAT_RUN, ._pad = 0 }; TEST_ASSERT_EQUAL_INT(0, persist_wal_async(&c.persist, WAL_JOB_START, &start_js, sizeof(start_js))); jdef.state = JOB_STAT_RUN; jdef.start_time = 1699999500; TEST_ASSERT_NOT_NULL(makeJob(c.jobmgr, &jdef)); job_failinfo_t fail = { .jobid = jobid, .stage = JOB_EXEC_STAGE_MAIN, .code = real_exit_code, .detail = (char*)"exit" }; TEST_ASSERT_EQUAL_INT(0, on_job_fail(&c, &fail)); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist)); persist_stop(&c.persist); persist_fini(&c.persist); dereference(c.jobmgr); xce_hook_fini(&c.hooksys); } /* Phase 2: fresh persist_t + cluster_t, recover from disk. */ { persist_t* p2 = calloc(1, sizeof(persist_t)); cluster_t c2; job_t* j2 = NULL; kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16 }; persist_config_t cfg = persist_config_defaults(); TEST_ASSERT_NOT_NULL(p2); cfg.wal_sync_ms = 5000; persist_set_workdir(p2, tmp, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL)); TEST_ASSERT_EQUAL_INT(0, persist_start(p2)); memset(&c2, 0, sizeof(c2)); TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX)); c2.jobmgr = makeInstance(&xkvmgr_type, &int_params); TEST_ASSERT_NOT_NULL(c2.jobmgr); p2->cluster = &c2; TEST_ASSERT_EQUAL_INT(0, persist_recover(p2)); lock(c2.jobmgr, RTTI_LOCK_READ); key_find(c2.jobmgr, (void*)jobid, (void**)&j2); unlock(c2.jobmgr); TEST_ASSERT_NOT_NULL(j2); TEST_ASSERT_EQUAL_UINT32(JOB_STAT_EXIT, j2->job.state); wjfl_assert_exit_code(j2->job.exit_info, real_exit_code); xce_hook_fini(&c2.hooksys); dereference(c2.jobmgr); persist_stop(p2); persist_fini(p2); free(p2); } char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp); if (system(cmd) != 0) fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp); } /* ── bug-1747: WAL_JOB_FAIL also carries the failure time ─────────────── */ /* persist_submit_job_fail_fact()'s full shape: header + int32_t code + * uint64_t finish_time (seconds). */ typedef struct { wal_jobstate_t js; int32_t code; uint64_t finish_time; } __attribute__((packed)) wjfl_fail_payload_t2; static wjfl_fail_payload_t2 wjfl_fail_payload2(uint64_t jobid, uint32_t stage, int32_t code, uint64_t ft) { wjfl_fail_payload_t2 pl; memset(&pl, 0, sizeof(pl)); pl.js.jobid = jobid; pl.js._pad = WAL_JOB_FAIL_STAGE_FLAG | stage; pl.code = code; pl.finish_time = ft; return pl; } static void test_wjfl_apply_committed_entry_adopts_wire_finish_time(void) { wjfl_setup(); job_t* j = wjfl_seed_job(74101, JOB_STAT_RUN); TEST_ASSERT_NOT_NULL(j); wjfl_fail_payload_t2 pl = wjfl_fail_payload2(74101, JOB_EXEC_STAGE_MAIN, 42, 1700000500ULL); TEST_ASSERT_EQUAL_INT(0, apply_committed_entry(&g_wjfl_p, WAL_JOB_FAIL, 410, &pl, sizeof(pl))); TEST_ASSERT_EQUAL_UINT32(JOB_STAT_EXIT, j->job.state); wjfl_assert_exit_code(j->job.exit_info, 42); TEST_ASSERT_EQUAL_UINT64_MESSAGE(1700000500ULL, (uint64_t)j->job.finish_time, "bug-1747: a failed job's finish_time must be the wire-carried " "failure time, not apply-time time(NULL)"); wjfl_teardown(); } static void test_wjfl_apply_committed_entry_code_only_payload_compat(void) { wjfl_setup(); job_t* j = wjfl_seed_job(74102, JOB_STAT_RUN); TEST_ASSERT_NOT_NULL(j); time_t before = time(NULL); wjfl_fail_payload_t pl = wjfl_fail_payload(74102, JOB_EXEC_STAGE_MAIN, 7); TEST_ASSERT_EQUAL_INT(0, apply_committed_entry(&g_wjfl_p, WAL_JOB_FAIL, 411, &pl, sizeof(pl))); wjfl_assert_exit_code(j->job.exit_info, 7); TEST_ASSERT_TRUE_MESSAGE(j->job.finish_time >= before, "a code-only (bug-1754-era) payload carries no time: finish_time " "stays apply-time, as before"); wjfl_teardown(); } static void test_wjfl_apply_wal_entry_adopts_wire_finish_time(void) { wjfl_setup(); job_t* j = wjfl_seed_job(74103, JOB_STAT_RUN); TEST_ASSERT_NOT_NULL(j); wjfl_fail_payload_t2 pl = wjfl_fail_payload2(74103, JOB_EXEC_STAGE_MAIN, 5, 1700000600ULL); TEST_ASSERT_EQUAL_INT(0, apply_wal_entry(&g_wjfl_p, WAL_JOB_FAIL, 412, &pl, sizeof(pl))); TEST_ASSERT_EQUAL_UINT32(JOB_STAT_EXIT, j->job.state); wjfl_assert_exit_code(j->job.exit_info, 5); TEST_ASSERT_EQUAL_UINT64(1700000600ULL, (uint64_t)j->job.finish_time); wjfl_teardown(); } /* Real record_job_fail_fact_at() + WAL write + persist_recover(): a failure * reported with an old time (as a rejoining worker reports one) keeps that * time across a restart; a 0 time falls back to receipt time. */ static void test_wjfl_record_fail_fact_at_survives_restart(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_wjfl_failat_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t jobid_old = 74104; const uint64_t jobid_zero = 74105; const uint64_t old_s = (uint64_t)time(NULL) - 86400; time_t before, after; { persist_config_t cfg; cluster_t c; kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16 }; const uint64_t ids[2] = { jobid_old, jobid_zero }; int k; if (makeString((char*)"wjfl-failat-probe") == NULL) xstring_table_init(64); if (!g_wjfl_jobobj_ready) { TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32)); g_wjfl_jobobj_ready = 1; } memset(&c, 0, sizeof(c)); TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX)); c.jobmgr = makeInstance(&xkvmgr_type, &int_params); TEST_ASSERT_NOT_NULL(c.jobmgr); cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; persist_set_workdir(&c.persist, tmp, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL)); TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist)); for (k = 0; k < 2; k++) { job_submit_data_t submit; xce_job_t jdef; uint8_t* buf = NULL; size_t sz = 0; job_t* j; memset(&submit, 0, sizeof(submit)); submit.queue = (char*)"oq"; submit.user = (char*)"tester"; memset(&jdef, 0, sizeof(jdef)); jdef.jobid = ids[k]; jdef.state = JOB_STAT_PEND; jdef.submit = &submit; TEST_ASSERT_EQUAL_INT(0, wjfl_encode_job(&jdef, &buf, &sz)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf, (uint32_t)sz)); free(buf); jdef.state = JOB_STAT_RUN; j = makeJob(c.jobmgr, &jdef); TEST_ASSERT_NOT_NULL(j); job_failinfo_t fail = { .jobid = ids[k], .stage = JOB_EXEC_STAGE_MAIN, .code = 3, .detail = (char*)"exit" }; before = time(NULL); lock(j, RTTI_LOCK_WRITE); TEST_ASSERT_EQUAL_INT(0, record_job_fail_fact_at(&c, j, &fail, (ids[k] == jobid_old) ? old_s * 1000ULL : 0)); unlock(j); after = time(NULL); } TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist)); persist_stop(&c.persist); persist_fini(&c.persist); dereference(c.jobmgr); xce_hook_fini(&c.hooksys); } { persist_t* p2 = calloc(1, sizeof(persist_t)); cluster_t c2; job_t* j_old = NULL; job_t* j_zero = NULL; kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16 }; persist_config_t cfg = persist_config_defaults(); TEST_ASSERT_NOT_NULL(p2); cfg.wal_sync_ms = 5000; persist_set_workdir(p2, tmp, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL)); TEST_ASSERT_EQUAL_INT(0, persist_start(p2)); memset(&c2, 0, sizeof(c2)); TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX)); c2.jobmgr = makeInstance(&xkvmgr_type, &int_params); TEST_ASSERT_NOT_NULL(c2.jobmgr); p2->cluster = &c2; TEST_ASSERT_EQUAL_INT(0, persist_recover(p2)); lock(c2.jobmgr, RTTI_LOCK_READ); key_find(c2.jobmgr, (void*)jobid_old, (void**)&j_old); key_find(c2.jobmgr, (void*)jobid_zero, (void**)&j_zero); unlock(c2.jobmgr); TEST_ASSERT_NOT_NULL(j_old); TEST_ASSERT_NOT_NULL(j_zero); TEST_ASSERT_EQUAL_UINT32(JOB_STAT_EXIT, j_old->job.state); wjfl_assert_exit_code(j_old->job.exit_info, 3); TEST_ASSERT_EQUAL_UINT64_MESSAGE(old_s, (uint64_t)j_old->job.finish_time, "bug-1747: the reported failure time must survive restart " "replay, not become replay-time time(NULL)"); TEST_ASSERT_TRUE_MESSAGE(j_zero->job.finish_time >= before && j_zero->job.finish_time <= after, "a 0 (unknown) failure time falls back to receipt time"); xce_hook_fini(&c2.hooksys); dereference(c2.jobmgr); persist_stop(p2); persist_fini(p2); free(p2); } char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp); if (system(cmd) != 0) fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp); } /* ── Test group entry point ───────────────────────────────────────────── */ void test_wal_job_fail_fact(void) { UnitySetTestFile(__FILE__); RUN_TEST(test_wjfl_apply_committed_entry_sets_exit_info); RUN_TEST(test_wjfl_apply_committed_entry_post_exec_keeps_job_exit_info); RUN_TEST(test_wjfl_apply_committed_entry_backward_compat_old_payload); RUN_TEST(test_wjfl_apply_wal_entry_sets_state_and_exit_info); RUN_TEST(test_wjfl_real_on_job_fail_survives_restart); RUN_TEST(test_wjfl_apply_committed_entry_adopts_wire_finish_time); RUN_TEST(test_wjfl_apply_committed_entry_code_only_payload_compat); RUN_TEST(test_wjfl_apply_wal_entry_adopts_wire_finish_time); RUN_TEST(test_wjfl_record_fail_fact_at_survives_restart); }