lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
myxce/src/controller/testsuite/test_wal_job_fail_fact.c
Rongsong Shen 1d6c04c28a
fix(controller): persist a failed job's finish_time through WAL_JOB_FAIL (bug-1747) 
The FAIL half of bug-1747: apply_job_fail_transition() stamps
finish_time = time(NULL) at APPLY time, and WAL_JOB_FAIL carried no time,
so a failed job's finish_time changed on every restart replay / standby
catch-up, and a worker-reported failure found on rejoin could never keep
the worker's own time.

- WAL_JOB_FAIL's trailer grows to int32_t code + uint64_t finish_time
  (seconds). Decoded by length: 16 B (pre-bug-1754), 20 B (code only,
  bug-1754) and 28 B (code + time) all apply as before for what they carry.
- record_job_fail_fact_at(c, job, failinfo, finish_time_ms), the fail-side
  sibling of record_job_finish_fact_at(): 0 = receipt time. on_job_fail()
  uses receipt time; resync's finishedFail branch passes the worker-
  reported finish_time_ms (Req 4.2 now holds for failures too).
- Both apply seams adopt the wire time after the transition.

4 new tests in test_wal_job_fail_fact.c (3 RED before the fix) and the
resync round-trip now asserts the worker-reported failure time (RED
before). Controller suite 1232/0. Factstream HA soak (2 rounds, 60 jobs)
PASS.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-26 09:33:55 +00:00

635 lines
22 KiB
C
Raw Blame History

/* test_wal_job_fail_fact.c — bug-1754 regression suite
*
* A worker reports EVERY nonzero job exit as a FAIL fact, not a FINISH
* (xcecd_job_finished -> xcecd_notify_job_fail(jobid, stage, exit_code),
* worker/src/controller_connector.c), so job_failinfo_t.code is the job's
* real exit code for a MAIN-stage failure. WAL_JOB_FAIL's deferred-apply
* payload used to be a bare wal_jobstate_t{jobid, 0, STAGE_FLAG|stage} with
* no code slot, and apply_job_fail_transition() never touched exit_info --
* so job.exit_info stayed NULL for every failed job, live and after replay.
*
* Proves WAL_JOB_FAIL now carries a trailing int32_t code and that BOTH of
* persist.c's apply seams adopt it into job.exit_info:
*
* - apply_committed_entry() (live-commit apply / boot replay dispatch)
* - apply_wal_entry() (standby catch-up / replicated-apply dispatch)
*
* that a POST_EXEC-stage failure (the post-exec SCRIPT's code, not the
* job's) leaves the job's own exit_info untouched, that an old-format entry
* (no trailer) still decodes with exit_info left NULL, and that a real
* on_job_fail() call survives a real WAL write + persist_recover() restart
* with job.exit_info intact.
*
* Fixtures mirror test_wal_job_finish_fact.c (bug-1689) exactly.
*/
#include <rtti.h>
#include <capnp_c.h>
#include <errno.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unity/unity.h>
#include "cluster.h"
#include "controller.capnp.h"
#include "hook.h"
#include "jobj.h"
#include "persist.h"
#include "wal_types.h"
#include "xce/cluster_def.h"
#include "xce/types.h"
#include "xcealloc.h"
#include "xstring.h"
static int g_wjfl_jobobj_ready;
static cluster_t g_wjfl_c;
static persist_t g_wjfl_p;
/* Captures the job_failinfo_t HOOK_DETAIL_JOB_FAIL carries to its AFTER_EVENT
* subscribers (sched_job_fail_fun / apiserver_job_fail_fun read the same
* (cluster_t*, job_failinfo_t*, job_t*) shape). */
static int g_wjfl_fail_hook_fires;
static int g_wjfl_captured_code;
static int wjfl_fail_spy_hook(int type, int situation, int detail, void* data,
va_list ap)
{
(void)type; (void)data;
if (situation != HOOK_AFTER_EVENT || detail != HOOK_DETAIL_JOB_FAIL)
return 0;
(void)va_arg(ap, cluster_t*);
job_failinfo_t* info = va_arg(ap, job_failinfo_t*);
g_wjfl_fail_hook_fires++;
if (info != NULL)
g_wjfl_captured_code = info->code;
return 0;
}
static void wjfl_setup(void)
{
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32,
.delta = 16 };
if (makeString((char*)"wjfl-probe") == NULL)
xstring_table_init(64);
if (!g_wjfl_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
g_wjfl_jobobj_ready = 1;
}
memset(&g_wjfl_c, 0, sizeof(g_wjfl_c));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&g_wjfl_c.hooksys, XCE_CONTROLLER_HOOK_MAX));
g_wjfl_c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(g_wjfl_c.jobmgr);
memset(&g_wjfl_p, 0, sizeof(g_wjfl_p));
g_wjfl_p.cluster = &g_wjfl_c;
g_wjfl_fail_hook_fires = 0;
g_wjfl_captured_code = 0;
}
static void wjfl_teardown(void)
{
xce_hook_fini(&g_wjfl_c.hooksys);
if (g_wjfl_c.jobmgr)
dereference(g_wjfl_c.jobmgr);
}
static job_submit_data_t g_wjfl_seed_submit;
static job_t* wjfl_seed_job(uint64_t jobid, uint32_t state)
{
xce_job_t jdef;
memset(&g_wjfl_seed_submit, 0, sizeof(g_wjfl_seed_submit));
g_wjfl_seed_submit.queue = (char*)"q";
g_wjfl_seed_submit.user = (char*)"tester";
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = jobid;
jdef.state = state;
jdef.start_time = 1699999000;
jdef.submit = &g_wjfl_seed_submit;
return makeJob(g_wjfl_c.jobmgr, &jdef);
}
/* wal_jobstate_t{jobid, 0, STAGE_FLAG|stage} + int32_t code --
* persist_submit_job_fail_fact()'s payload shape. */
typedef struct {
wal_jobstate_t js;
int32_t code;
} __attribute__((packed)) wjfl_fail_payload_t;
static wjfl_fail_payload_t wjfl_fail_payload(uint64_t jobid, uint32_t stage,
int32_t code)
{
wjfl_fail_payload_t pl;
memset(&pl, 0, sizeof(pl));
pl.js.jobid = jobid;
pl.js.state = 0;
pl.js._pad = WAL_JOB_FAIL_STAGE_FLAG | stage;
pl.code = code;
return pl;
}
static void wjfl_assert_exit_code(job_exitinfo_t* got, int expected)
{
TEST_ASSERT_NOT_NULL_MESSAGE(got,
"a MAIN-stage WAL_JOB_FAIL carrying a code must populate "
"job.exit_info (bug-1754), not leave it NULL");
TEST_ASSERT_EQUAL_INT(expected, got->exitcode);
}
/* ── apply_committed_entry ────────────────────────────────────────────── */
static void test_wjfl_apply_committed_entry_sets_exit_info(void)
{
wjfl_setup();
job_t* j = wjfl_seed_job(74001, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
xce_add_hook(&g_wjfl_c.hooksys, XCE_CONTROLLER_HOOK_JOB,
wjfl_fail_spy_hook, NULL);
wjfl_fail_payload_t pl = wjfl_fail_payload(74001, JOB_EXEC_STAGE_MAIN, 42);
int rc = apply_committed_entry(&g_wjfl_p, WAL_JOB_FAIL, 400, &pl,
sizeof(pl));
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_EXIT, j->job.state);
wjfl_assert_exit_code(j->job.exit_info, 42);
TEST_ASSERT_EQUAL_INT(1, g_wjfl_fail_hook_fires);
TEST_ASSERT_EQUAL_INT_MESSAGE(42, g_wjfl_captured_code,
"HOOK_DETAIL_JOB_FAIL subscribers must see the wire-carried code, "
"not a fabricated 0");
wjfl_teardown();
}
static void test_wjfl_apply_committed_entry_post_exec_keeps_job_exit_info(void)
{
wjfl_setup();
job_t* j = wjfl_seed_job(74002, JOB_STAT_DONE);
TEST_ASSERT_NOT_NULL(j);
j->job.exit_info = calloc(1, sizeof(job_exitinfo_t));
TEST_ASSERT_NOT_NULL(j->job.exit_info);
j->job.exit_info->exitcode = 0;
wjfl_fail_payload_t pl
= wjfl_fail_payload(74002, JOB_EXEC_STAGE_POST_EXEC, 3);
int rc = apply_committed_entry(&g_wjfl_p, WAL_JOB_FAIL, 401, &pl,
sizeof(pl));
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_DONE | JOB_STAT_PERR, j->job.state);
TEST_ASSERT_NOT_NULL(j->job.exit_info);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, j->job.exit_info->exitcode,
"a POST_EXEC failure's code is the post-exec script's, not the "
"job's -- it must not overwrite the job's own exit code");
wjfl_teardown();
}
static void test_wjfl_apply_committed_entry_backward_compat_old_payload(void)
{
wjfl_setup();
job_t* j = wjfl_seed_job(74003, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
wal_jobstate_t js = { .jobid = 74003, .state = 0,
._pad = WAL_JOB_FAIL_STAGE_FLAG
| JOB_EXEC_STAGE_MAIN };
int rc = apply_committed_entry(&g_wjfl_p, WAL_JOB_FAIL, 402, &js,
sizeof(js));
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_EXIT, j->job.state);
TEST_ASSERT_NULL_MESSAGE(j->job.exit_info,
"an old-format WAL_JOB_FAIL (no code trailer) must leave exit_info "
"NULL rather than fabricate an exit code");
wjfl_teardown();
}
/* ── apply_wal_entry ──────────────────────────────────────────────────── */
static void test_wjfl_apply_wal_entry_sets_state_and_exit_info(void)
{
wjfl_setup();
job_t* j = wjfl_seed_job(74004, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
wjfl_fail_payload_t pl = wjfl_fail_payload(74004, JOB_EXEC_STAGE_MAIN, 9);
int rc = apply_wal_entry(&g_wjfl_p, WAL_JOB_FAIL, 403, &pl, sizeof(pl));
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_EXIT, j->job.state,
"a deferred (STAGE_FLAG) WAL_JOB_FAIL carries state 0 on the wire; "
"the standby seam must recompute EXIT, not assign that 0");
wjfl_assert_exit_code(j->job.exit_info, 9);
wjfl_teardown();
}
/* ── Restart replay through the real on_job_fail() receipt path ───────── */
static int wjfl_encode_job(const xce_job_t* j, uint8_t** out, size_t* out_len)
{
struct capn c;
struct capn_segment* cs;
struct Job cj;
Job_ptr jp;
capn_init_malloc(&c);
cs = capn_root(&c).seg;
encode_Job(cs, &cj, (xce_job_t*)j);
jp = new_Job(cs);
write_Job(&cj, jp);
capn_setp(capn_root(&c), 0, jp.p);
*out_len = capn_size(&c);
*out = malloc(*out_len);
if (!*out) { capn_free(&c); return -ENOMEM; }
capn_write_mem(&c, *out, *out_len, 0);
capn_free(&c);
return 0;
}
static void test_wjfl_real_on_job_fail_survives_restart(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_wjfl_onfail_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
const uint64_t jobid = 74005;
const int real_exit_code = 2;
/* Phase 1: durable WAL_JOB_SUBMIT + WAL_JOB_START seeds, a live RUN job,
* then the real worker-FAIL receipt handler appends WAL_JOB_FAIL. */
{
persist_config_t cfg;
cluster_t c;
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
if (makeString((char*)"wjfl-onfail-probe") == NULL)
xstring_table_init(64);
if (!g_wjfl_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
g_wjfl_jobobj_ready = 1;
}
memset(&c, 0, sizeof(c));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));
c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(c.jobmgr);
cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
persist_set_workdir(&c.persist, tmp, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));
job_submit_data_t submit;
xce_job_t jdef;
uint8_t* buf = NULL;
size_t sz = 0;
memset(&submit, 0, sizeof(submit));
submit.queue = (char*)"oq";
submit.user = (char*)"tester";
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = jobid;
jdef.state = JOB_STAT_PEND;
jdef.submit = &submit;
TEST_ASSERT_EQUAL_INT(0, wjfl_encode_job(&jdef, &buf, &sz));
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf, (uint32_t)sz));
free(buf);
wal_jobstate_t start_js = { .jobid = jobid, .state = JOB_STAT_RUN,
._pad = 0 };
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(&c.persist, WAL_JOB_START, &start_js,
sizeof(start_js)));
jdef.state = JOB_STAT_RUN;
jdef.start_time = 1699999500;
TEST_ASSERT_NOT_NULL(makeJob(c.jobmgr, &jdef));
job_failinfo_t fail = { .jobid = jobid,
.stage = JOB_EXEC_STAGE_MAIN,
.code = real_exit_code,
.detail = (char*)"exit" };
TEST_ASSERT_EQUAL_INT(0, on_job_fail(&c, &fail));
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));
persist_stop(&c.persist);
persist_fini(&c.persist);
dereference(c.jobmgr);
xce_hook_fini(&c.hooksys);
}
/* Phase 2: fresh persist_t + cluster_t, recover from disk. */
{
persist_t* p2 = calloc(1, sizeof(persist_t));
cluster_t c2;
job_t* j2 = NULL;
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
persist_config_t cfg = persist_config_defaults();
TEST_ASSERT_NOT_NULL(p2);
cfg.wal_sync_ms = 5000;
persist_set_workdir(p2, tmp, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(p2));
memset(&c2, 0, sizeof(c2));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(c2.jobmgr);
p2->cluster = &c2;
TEST_ASSERT_EQUAL_INT(0, persist_recover(p2));
lock(c2.jobmgr, RTTI_LOCK_READ);
key_find(c2.jobmgr, (void*)jobid, (void**)&j2);
unlock(c2.jobmgr);
TEST_ASSERT_NOT_NULL(j2);
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_EXIT, j2->job.state);
wjfl_assert_exit_code(j2->job.exit_info, real_exit_code);
xce_hook_fini(&c2.hooksys);
dereference(c2.jobmgr);
persist_stop(p2);
persist_fini(p2);
free(p2);
}
char cmd[512];
snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
if (system(cmd) != 0)
fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
}
/* ── bug-1747: WAL_JOB_FAIL also carries the failure time ─────────────── */
/* persist_submit_job_fail_fact()'s full shape: header + int32_t code +
* uint64_t finish_time (seconds). */
typedef struct {
wal_jobstate_t js;
int32_t code;
uint64_t finish_time;
} __attribute__((packed)) wjfl_fail_payload_t2;
static wjfl_fail_payload_t2 wjfl_fail_payload2(uint64_t jobid, uint32_t stage,
int32_t code, uint64_t ft)
{
wjfl_fail_payload_t2 pl;
memset(&pl, 0, sizeof(pl));
pl.js.jobid = jobid;
pl.js._pad = WAL_JOB_FAIL_STAGE_FLAG | stage;
pl.code = code;
pl.finish_time = ft;
return pl;
}
static void test_wjfl_apply_committed_entry_adopts_wire_finish_time(void)
{
wjfl_setup();
job_t* j = wjfl_seed_job(74101, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
wjfl_fail_payload_t2 pl
= wjfl_fail_payload2(74101, JOB_EXEC_STAGE_MAIN, 42, 1700000500ULL);
TEST_ASSERT_EQUAL_INT(0, apply_committed_entry(&g_wjfl_p, WAL_JOB_FAIL,
410, &pl, sizeof(pl)));
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_EXIT, j->job.state);
wjfl_assert_exit_code(j->job.exit_info, 42);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(1700000500ULL,
(uint64_t)j->job.finish_time,
"bug-1747: a failed job's finish_time must be the wire-carried "
"failure time, not apply-time time(NULL)");
wjfl_teardown();
}
static void test_wjfl_apply_committed_entry_code_only_payload_compat(void)
{
wjfl_setup();
job_t* j = wjfl_seed_job(74102, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
time_t before = time(NULL);
wjfl_fail_payload_t pl = wjfl_fail_payload(74102, JOB_EXEC_STAGE_MAIN, 7);
TEST_ASSERT_EQUAL_INT(0, apply_committed_entry(&g_wjfl_p, WAL_JOB_FAIL,
411, &pl, sizeof(pl)));
wjfl_assert_exit_code(j->job.exit_info, 7);
TEST_ASSERT_TRUE_MESSAGE(j->job.finish_time >= before,
"a code-only (bug-1754-era) payload carries no time: finish_time "
"stays apply-time, as before");
wjfl_teardown();
}
static void test_wjfl_apply_wal_entry_adopts_wire_finish_time(void)
{
wjfl_setup();
job_t* j = wjfl_seed_job(74103, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
wjfl_fail_payload_t2 pl
= wjfl_fail_payload2(74103, JOB_EXEC_STAGE_MAIN, 5, 1700000600ULL);
TEST_ASSERT_EQUAL_INT(0, apply_wal_entry(&g_wjfl_p, WAL_JOB_FAIL, 412,
&pl, sizeof(pl)));
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_EXIT, j->job.state);
wjfl_assert_exit_code(j->job.exit_info, 5);
TEST_ASSERT_EQUAL_UINT64(1700000600ULL, (uint64_t)j->job.finish_time);
wjfl_teardown();
}
/* Real record_job_fail_fact_at() + WAL write + persist_recover(): a failure
* reported with an old time (as a rejoining worker reports one) keeps that
* time across a restart; a 0 time falls back to receipt time. */
static void test_wjfl_record_fail_fact_at_survives_restart(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_wjfl_failat_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
const uint64_t jobid_old = 74104;
const uint64_t jobid_zero = 74105;
const uint64_t old_s = (uint64_t)time(NULL) - 86400;
time_t before, after;
{
persist_config_t cfg;
cluster_t c;
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
const uint64_t ids[2] = { jobid_old, jobid_zero };
int k;
if (makeString((char*)"wjfl-failat-probe") == NULL)
xstring_table_init(64);
if (!g_wjfl_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
g_wjfl_jobobj_ready = 1;
}
memset(&c, 0, sizeof(c));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));
c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(c.jobmgr);
cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
persist_set_workdir(&c.persist, tmp, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));
for (k = 0; k < 2; k++) {
job_submit_data_t submit;
xce_job_t jdef;
uint8_t* buf = NULL;
size_t sz = 0;
job_t* j;
memset(&submit, 0, sizeof(submit));
submit.queue = (char*)"oq";
submit.user = (char*)"tester";
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = ids[k];
jdef.state = JOB_STAT_PEND;
jdef.submit = &submit;
TEST_ASSERT_EQUAL_INT(0, wjfl_encode_job(&jdef, &buf, &sz));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(&c.persist,
WAL_JOB_SUBMIT, buf, (uint32_t)sz));
free(buf);
jdef.state = JOB_STAT_RUN;
j = makeJob(c.jobmgr, &jdef);
TEST_ASSERT_NOT_NULL(j);
job_failinfo_t fail = { .jobid = ids[k],
.stage = JOB_EXEC_STAGE_MAIN,
.code = 3,
.detail = (char*)"exit" };
before = time(NULL);
lock(j, RTTI_LOCK_WRITE);
TEST_ASSERT_EQUAL_INT(0, record_job_fail_fact_at(&c, j, &fail,
(ids[k] == jobid_old) ? old_s * 1000ULL : 0));
unlock(j);
after = time(NULL);
}
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));
persist_stop(&c.persist);
persist_fini(&c.persist);
dereference(c.jobmgr);
xce_hook_fini(&c.hooksys);
}
{
persist_t* p2 = calloc(1, sizeof(persist_t));
cluster_t c2;
job_t* j_old = NULL;
job_t* j_zero = NULL;
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
persist_config_t cfg = persist_config_defaults();
TEST_ASSERT_NOT_NULL(p2);
cfg.wal_sync_ms = 5000;
persist_set_workdir(p2, tmp, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(p2));
memset(&c2, 0, sizeof(c2));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(c2.jobmgr);
p2->cluster = &c2;
TEST_ASSERT_EQUAL_INT(0, persist_recover(p2));
lock(c2.jobmgr, RTTI_LOCK_READ);
key_find(c2.jobmgr, (void*)jobid_old, (void**)&j_old);
key_find(c2.jobmgr, (void*)jobid_zero, (void**)&j_zero);
unlock(c2.jobmgr);
TEST_ASSERT_NOT_NULL(j_old);
TEST_ASSERT_NOT_NULL(j_zero);
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_EXIT, j_old->job.state);
wjfl_assert_exit_code(j_old->job.exit_info, 3);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(old_s, (uint64_t)j_old->job.finish_time,
"bug-1747: the reported failure time must survive restart "
"replay, not become replay-time time(NULL)");
TEST_ASSERT_TRUE_MESSAGE(j_zero->job.finish_time >= before
&& j_zero->job.finish_time <= after,
"a 0 (unknown) failure time falls back to receipt time");
xce_hook_fini(&c2.hooksys);
dereference(c2.jobmgr);
persist_stop(p2);
persist_fini(p2);
free(p2);
}
char cmd[512];
snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
if (system(cmd) != 0)
fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
}
/* ── Test group entry point ───────────────────────────────────────────── */
void test_wal_job_fail_fact(void)
{
UnitySetTestFile(__FILE__);
RUN_TEST(test_wjfl_apply_committed_entry_sets_exit_info);
RUN_TEST(test_wjfl_apply_committed_entry_post_exec_keeps_job_exit_info);
RUN_TEST(test_wjfl_apply_committed_entry_backward_compat_old_payload);
RUN_TEST(test_wjfl_apply_wal_entry_sets_state_and_exit_info);
RUN_TEST(test_wjfl_real_on_job_fail_survives_restart);
RUN_TEST(test_wjfl_apply_committed_entry_adopts_wire_finish_time);
RUN_TEST(test_wjfl_apply_committed_entry_code_only_payload_compat);
RUN_TEST(test_wjfl_apply_wal_entry_adopts_wire_finish_time);
RUN_TEST(test_wjfl_record_fail_fact_at_survives_restart);
}
Reference in a new issue View git blame Copy permalink