lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
myxce/src/controller/testsuite/test_worker_resync.c
Rongsong Shen 6876e4f6f2
fix(controller): persist a failed job's exit code through WAL_JOB_FAIL (bug-1754) 
A worker reports every nonzero job exit as a FAIL (xcecd_job_finished ->
xcecd_notify_job_fail(jobid, stage, exit_code)), not a FINISH. The deferred
WAL_JOB_FAIL fact carried only {jobid, STAGE_FLAG|stage} and no apply seam
touched job.exit_info, so exit_info stayed NULL for every failed job, live
and after replay, and HOOK_DETAIL_JOB_FAIL subscribers saw code 0.

- persist_submit_job_fail_fact(): header + int32_t code trailer, used by
  record_job_fail_fact(). Old entries (no trailer) decode as before.
- apply_job_fail_exit_code(): records the code in exit_info for pre-exit
  stages only; a POST_EXEC code is the script's and leaves exit_info alone.
  Kept out of apply_job_fail_transition() so mark_job_fail()'s internal
  -errno failures (not replayed with a code) stay live/replay consistent.
- Both apply seams adopt the code. apply_wal_entry()'s WAL_JOB_FAIL case
  also stops assigning the wire state 0 for STAGE_FLAG entries and runs the
  real transition, as apply_committed_entry() already does.

New test_wal_job_fail_fact.c (5 tests, 3 RED before the fix); the two
test_worker_resync.c assertions that pinned exit_info == NULL now assert the
worker-reported code. Controller suite: 1228 tests, 0 failures.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-26 03:32:54 +00:00

4832 lines
207 KiB
C
Raw Blame History

/* test_worker_resync.c — worker-rejoin-resync task 2.1
*
* Proves record_job_reconcile_pending_fact() (job.c) — the Requirement 4.6
* new fact-recording function that returns a worker-unrecognized job to
* PEND, releasing any resource allocation it was carrying (bug-1631's own
* deliberately-deferred trigger condition, closed here — see design.md's
* Error Handling section) and durably appending a plain WAL_JOB_UPDATE fact
* -- following dispatch_a_job_()'s eager-mutation-plus-explicit-notify
* pattern (design.md "Controller: job.c — record_job_reconcile_pending_
* fact, record_job_reconcile_assign_fact"), NOT record_job_finish_fact's
* deferred one.
*
* This task implements ONLY record_job_reconcile_pending_fact() itself --
* no live caller exists yet (dispatch_a_job_'s disconnected-worker split is
* task 2.3; resync.c's report-apply glue is a later task), so every test
* here directly exercises the exact sequence a real caller will perform:
* lock -> call record_job_reconcile_pending_fact() -> check the returned
* job_transition_result_t -> unlock -> (only when APPLIED) call
* persist_notify_job_updated_already_persisted() itself -- proving the full
* intended sequence end-to-end even though production wiring of that
* sequence is a later task's job (design.md's Service Interface
* Postconditions/Invariants: the caller, not this function, owns that
* post-unlock notify call).
*
* Fixture mirrors test_dispatch_commit.c's dc_setup/dc_teardown exactly: a
* bare cluster_t (jobmgr + hooksys) plus a real disk-backed persist_t
* initialized with cluster == NULL, so wal_hook_append_classified()
* (persist.c) takes the direct persist_wal_async() path for
* persist_submit_job_fact()'s plain RETAIN-mode WAL_JOB_UPDATE append -- no
* raft/commit-engine machinery needed. p->current_lsn before/after each
* call proves (or disproves) a durable append happened, mirroring
* test_dispatch_commit.c's test_dc_worker_not_found_leaves_job_pend_and_
* alloc_uncommitted's own LSN-delta idiom.
*/
#include <rtti.h>
#include <capnp_c.h>
#include <errno.h>
#include <pthread.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
#include "cluster.h"
#include "controller.capnp.h"
#include "hook.h"
#include "jobj.h"
#include "kvmgr.h"
#include "netio.h" /* worker-rejoin-resync task 7.1: endpoint_get/
* endpoint_deq/ITPTR_TO_PACKET, for real wire capture of
* the round-trip test's query send / report send */
#include "persist.h"
#include "request.h" /* worker-rejoin-resync task 7.1: xce_decode,
* xce_free_request, xce_job_state_report_req */
#include "resobj.h" /* worker-rejoin-resync task 7.1: makeResource/
* xce_resource_t, for the round-trip test's real
* resmgr (apply_join_state, reached via rejoin_worker,
* needs one) */
#include "resync.h" /* worker-rejoin-resync task 3.1 */
#include "wal_types.h"
#include "wobj.h" /* worker-rejoin-resync task 3.1: makeWorker, for a real
* RTTI-managed worker_t the timeout test can genuinely
* lock() */
#include "xce/cluster_def.h"
#include "xce/types.h"
#include "xcealloc.h"
#include "xstring.h"
/* worker-rejoin-resync task 7.1: rejoin_worker/worker_send_resync_query are
* non-static in worker.c when XCE_TESTING is defined -- forward-declared
* directly here rather than via a header, mirroring test_worker_join.c's
* own identical forward declarations exactly (these functions have no
* public header of their own; test_worker_join.c already establishes this
* as the file's own convention for driving the real join path from a
* test). collect_confvars (needed for worker.c to link, since on_worker_
* join -> worker_push_config -> collect_confvars) is already provided by
* test_worker_join.c elsewhere in this SAME controller_unit_test binary --
* not redefined here to avoid a duplicate-symbol link error; this test
* never calls worker_push_config anyway. */
int rejoin_worker(cluster_t* c, worker_t* w, endpoint_t* ep,
hostattrs_t* attrs, xce_hostconfig_t* hconfig);
int worker_send_resync_query(cluster_t* c, worker_t* w, endpoint_t* ep);
/* jobj.c's job type table and the process-global xstring table are shared
* across every test file in the controller_unit_test binary -- guard
* idempotently, mirroring every sibling fixture in this suite
* (test_wal_job_dispatch_fact.c's g_wjd_jobobj_ready). */
static int g_twr_jobobj_ready;
static cluster_t g_twr;
static char g_twr_tmpdir[256];
/* worker-rejoin-resync task 3.1: no-op kvmgr datakiller for g_twr.workermgr,
* mirroring production's own worker_obj_killer (cluster.c) -- a worker_t,
* once created, is never explicitly torn down for the life of the process
* (no key_del(c->workermgr, ...) call site exists anywhere in this
* codebase), so there is nothing for this to actually free. */
static void twr_worker_obj_killer(void* arg)
{
(void)arg;
}
static void twr_setup(void)
{
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
persist_config_t cfg;
if (makeString((char*)"twr-probe") == NULL)
xstring_table_init(64);
if (!g_twr_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
g_twr_jobobj_ready = 1;
}
memset(&g_twr, 0, sizeof(g_twr));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&g_twr.hooksys, XCE_CONTROLLER_HOOK_MAX));
g_twr.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(g_twr.jobmgr);
/* worker-rejoin-resync task 3.1: a real workermgr so the round-lifecycle
* tests below can create a genuine RTTI-managed worker_t (via
* makeWorker) where that matters (the timeout test, whose internal
* resync_on_timeout callback genuinely lock()s `w`) -- key type mirrors
* production's own workermgr_params (cluster.c, KVMGR_KEY_STRING,
* keyed by hostname). */
{
kvmgr_params_t workermgr_params
= { .key_type = KVMGR_KEY_STRING,
.nslots = 32,
.delta = 16,
.datakiller = twr_worker_obj_killer };
g_twr.workermgr = makeInstance(&xkvmgr_type, &workermgr_params);
TEST_ASSERT_NOT_NULL(g_twr.workermgr);
}
/* Real disk-backed persist, cluster == NULL: wal_hook_append_
* classified() (persist.c) takes the direct persist_wal_async() path --
* no raft/commit-engine machinery needed for a plain RETAIN-mode fact
* append (mirrors test_dispatch_commit.c's dc_setup exactly, which
* exercises dispatch_a_job_'s own real persist_submit_job_fact()-family
* call the same way). */
cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
strcpy(g_twr_tmpdir, "/tmp/xce_twr21_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(g_twr_tmpdir));
persist_set_workdir(&g_twr.persist, g_twr_tmpdir, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(&g_twr.persist, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(&g_twr.persist));
}
static void twr_teardown(void)
{
char cmd[512];
persist_stop(&g_twr.persist);
persist_fini(&g_twr.persist);
if (g_twr.jobmgr)
dereference(g_twr.jobmgr);
if (g_twr.workermgr)
dereference(g_twr.workermgr);
xce_hook_fini(&g_twr.hooksys);
snprintf(cmd, sizeof(cmd), "rm -rf %s", g_twr_tmpdir);
if (system(cmd) != 0)
fprintf(stderr, "cleanup warning: rm -rf %s failed\n", g_twr_tmpdir);
}
/* Creates a job directly in the fixture's jobmgr, bypassing dispatch/replay
* entirely -- for seeding fact-transition tests, mirroring
* test_wal_job_dispatch_fact.c's wjd_seed_job / test_dispatch_commit.c's
* dc_seed_ready. */
static job_submit_data_t g_twr_seed_submit;
static job_t* twr_seed_job(uint64_t jobid, uint32_t state)
{
xce_job_t jdef;
memset(&g_twr_seed_submit, 0, sizeof(g_twr_seed_submit));
g_twr_seed_submit.queue = (char*)"q";
g_twr_seed_submit.user = (char*)"tester";
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = jobid;
jdef.state = state;
jdef.submit = &g_twr_seed_submit;
return makeJob(g_twr.jobmgr, &jdef);
}
/* One host ("twr-worker"), one resource ("slots", value 1.0) -- the same
* minimal shape test_wal_job_dispatch_fact.c's/test_dispatch_commit.c's own
* dispatch-alloc tests use. Caller owns the result (release via
* free_JobAlloc_ptr, or hand ownership to job->job.allocs). */
static xce_job_alloc_t* twr_make_test_alloc(uint64_t jobid)
{
return make_job_alloc(jobid, 1,
make_host_allocs(1,
make_host_alloc((char*)"twr-worker", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
}
static uint64_t twr_current_lsn(void)
{
uint64_t lsn;
pthread_mutex_lock(&g_twr.persist.lock);
lsn = g_twr.persist.current_lsn;
pthread_mutex_unlock(&g_twr.persist.lock);
return lsn;
}
/* Spy hook: counts HOOK_AFTER_EVENT/HOOK_DETAIL_JOB_UPDATE fires -- the
* exact hook sched_job_update_fun/apiserver_job_update_fun subscribe to in
* production, mirroring test_dispatch_commit.c's dc_job_update_spy_hook. */
static int g_twr_update_hook_fires;
static int twr_job_update_spy_hook(int type, int situation, int detail,
void* data, va_list ap)
{
(void)type; (void)data; (void)ap;
if (situation == HOOK_AFTER_EVENT && detail == HOOK_DETAIL_JOB_UPDATE)
g_twr_update_hook_fires++;
return 0;
}
#define TWR_JOBID_APPLIED 81001ULL
#define TWR_JOBID_NOOP 81002ULL
#define TWR_JOBID_TERMINAL 81003ULL
/* THE core proof (Req 4.6, 7.1-7.3; bug-1631 regression): a job that was
* dispatched (real allocs committed, state RUN) and reconciles to
* unrecognized ends the call -- synchronously, BEFORE any commit round-trip,
* matching dispatch_a_job_'s own eager pattern, NOT record_job_finish_
* fact's deferred one -- with PEND state and no allocation, a durably
* appended fact, and (once the caller unlocks and notifies, exactly as a
* real caller must) the scheduler's notify path fired exactly once. */
static void test_reconcile_pending_applies_releases_allocs_and_notifies(void)
{
job_t* j;
xce_job_alloc_t* alloc;
int rc;
uint64_t lsn_before, lsn_after;
twr_setup();
g_twr_update_hook_fires = 0;
TEST_ASSERT_EQUAL_INT(0,
xce_add_hook(&g_twr.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook, NULL));
j = twr_seed_job(TWR_JOBID_APPLIED, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
alloc = twr_make_test_alloc(TWR_JOBID_APPLIED);
TEST_ASSERT_NOT_NULL(alloc);
j->job.allocs = alloc; /* simulate dispatch_a_job_'s own committed alloc,
* ownership transferred to job->job.allocs. */
lsn_before = twr_current_lsn();
lock(j, RTTI_LOCK_WRITE);
rc = record_job_reconcile_pending_fact(&g_twr, j);
/* Observable completion (design.md): PEND + no allocation, asserted
* IMMEDIATELY after the call returns, still under the write lock,
* before any commit round-trip -- the mutation is eager, not
* deferred. */
TEST_ASSERT_EQUAL_INT_MESSAGE(JOB_TRANSITION_APPLIED, rc,
"an unrecognized RUN job must apply the PEND transition");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state,
"job must be PEND synchronously, before any WAL commit/apply "
"round trip");
TEST_ASSERT_NULL_MESSAGE(j->job.allocs,
"bug-1631: a job reconciled from RUN back to PEND must release its "
"resource allocation immediately, so a later redispatch never "
"inherits a stale allocation from the prior dispatch");
unlock(j);
lsn_after = twr_current_lsn();
TEST_ASSERT_TRUE_MESSAGE(lsn_after > lsn_before,
"record_job_reconcile_pending_fact must durably append a "
"WAL_JOB_UPDATE fact for an applied transition");
/* Caller's responsibility, AFTER unlocking (design.md Postconditions/
* Invariants): notify the scheduler through the existing notify path,
* exactly once, for this actual mutation (Req 7.1-7.3). */
persist_notify_job_updated_already_persisted(&g_twr, j);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_twr_update_hook_fires,
"the scheduler notify path must fire exactly once for this "
"reconciled pending transition (Req 7.2) when the caller notifies "
"after unlocking -- omitting it would reproduce bug-1679 at this "
"new call site");
xce_disable_hook(&g_twr.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook);
twr_teardown();
printf("test_reconcile_pending_applies_releases_allocs_and_notifies: "
"PASS\n");
}
/* A job with no allocation at all (e.g. never dispatched, already PEND):
* reconciling it again must be a no-op per apply_job_update_transition's
* own duplicate-state guard (JOB_TRANSITION_NOOP_DUPLICATE) -- no mutation,
* no allocs touch (there is none to release), no durable fact, no crash
* (Req 4.3). */
static void test_reconcile_pending_noop_on_already_pending_job(void)
{
job_t* j;
int rc;
uint64_t lsn_before, lsn_after;
twr_setup();
j = twr_seed_job(TWR_JOBID_NOOP, JOB_STAT_PEND);
TEST_ASSERT_NOT_NULL(j);
TEST_ASSERT_NULL(j->job.allocs);
lsn_before = twr_current_lsn();
lock(j, RTTI_LOCK_WRITE);
rc = record_job_reconcile_pending_fact(&g_twr, j);
TEST_ASSERT_EQUAL_INT_MESSAGE(JOB_TRANSITION_NOOP_DUPLICATE, rc,
"reconciling an already-PEND job must be treated as a duplicate "
"no-op, matching apply_job_update_transition's own guard");
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, j->job.state);
TEST_ASSERT_NULL_MESSAGE(j->job.allocs,
"a job that never held an allocation must still have none after a "
"no-op reconcile");
unlock(j);
lsn_after = twr_current_lsn();
TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before, lsn_after,
"a no-op reconcile must append no duplicate WAL fact (Req 4.3)");
twr_teardown();
printf("test_reconcile_pending_noop_on_already_pending_job: PASS\n");
}
/* A job the controller already knows is terminal (DONE) must never be
* clobbered back to PEND by a reconciliation report (Req 4.4) -- the
* existing job_state_is_backward guard inside apply_job_update_transition,
* reused unmodified, rejects it. */
static void test_reconcile_pending_rejects_terminal_job(void)
{
job_t* j;
int rc;
uint64_t lsn_before, lsn_after;
twr_setup();
j = twr_seed_job(TWR_JOBID_TERMINAL, JOB_STAT_DONE);
TEST_ASSERT_NOT_NULL(j);
lsn_before = twr_current_lsn();
lock(j, RTTI_LOCK_WRITE);
rc = record_job_reconcile_pending_fact(&g_twr, j);
TEST_ASSERT_EQUAL_INT_MESSAGE(JOB_TRANSITION_REJECTED_BACKWARD, rc,
"a terminal job must reject the backward PEND transition, never "
"clobbering more-final state (Req 4.4)");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE, j->job.state,
"the job's terminal state must be preserved exactly");
unlock(j);
lsn_after = twr_current_lsn();
TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before, lsn_after,
"a rejected reconcile must append no WAL fact");
twr_teardown();
printf("test_reconcile_pending_rejects_terminal_job: PASS\n");
}
/* ── record_job_reconcile_assign_fact() tests (task 2.2) ─────────────────
*
* Proves record_job_reconcile_assign_fact() (job.c) -- the Requirement 4.8
* new fact-recording function that records a job the worker reports running
* that the controller has no existing non-terminal record of ("unqueried
* extra", Req 3.6) as assigned to the reporting worker, through the same
* dispatch_a_job_()-style eager-mutation-plus-explicit-notify pattern as
* record_job_reconcile_pending_fact above, reusing bug-1678's NULL-alloc
* persist_submit_job_dispatch_fact() path (design.md "Controller: job.c --
* record_job_reconcile_pending_fact, record_job_reconcile_assign_fact").
*
* As with task 2.1, no live caller exists yet (resync.c's report-apply glue
* is a later task), so every test here directly exercises the exact
* sequence a real caller will perform: lock -> call
* record_job_reconcile_assign_fact() -> check the result -> unlock -> call
* persist_notify_job_updated_already_persisted() itself. */
#define TWR_JOBID_ASSIGN 81004ULL
#define TWR_JOBID_ASSIGN_REPLAY 81005ULL
/* THE core proof (Req 4.1, 4.3, 4.4, 4.7, 4.8, 7.1-7.3): a job the
* controller never previously tracked, reported RUNNING by a worker, ends
* the call -- synchronously, before any commit round-trip -- recorded as
* running with no resource allocation (the controller genuinely does not
* know what the scheduler originally computed, design.md's "Documented
* scope decision"), a durably appended fact, and (once the caller unlocks
* and notifies, exactly as a real caller must) the scheduler's notify path
* fired exactly once. */
static void
test_reconcile_assign_applies_running_no_allocs_and_notifies(void)
{
job_t* j;
worker_t w;
xce_job_state_entry_t report;
int rc;
uint64_t lsn_before, lsn_after;
twr_setup();
g_twr_update_hook_fires = 0;
TEST_ASSERT_EQUAL_INT(0,
xce_add_hook(&g_twr.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook, NULL));
/* Seeded with no allocation, mirroring a job the caller (task 3.2, not
* yet implemented) constructed fresh for a jobid the controller had
* never seen before -- there is nothing to release here, only
* something new to record. */
j = twr_seed_job(TWR_JOBID_ASSIGN, JOB_STAT_PEND);
TEST_ASSERT_NOT_NULL(j);
TEST_ASSERT_NULL(j->job.allocs);
memset(&w, 0, sizeof(w));
w.host.name = (char*)"twr-worker";
report = (xce_job_state_entry_t){
.jobid = TWR_JOBID_ASSIGN,
.status = JobStateStatus_running,
.exit_code = 0,
.finish_time_ms = 0,
.queried = 0
};
lsn_before = twr_current_lsn();
lock(j, RTTI_LOCK_WRITE);
rc = record_job_reconcile_assign_fact(&g_twr, j, &w, &report);
/* Observable completion (design.md / this task): RUN + no allocation,
* asserted IMMEDIATELY after the call returns, still under the write
* lock, before any commit round-trip -- the mutation is eager, not
* deferred. */
TEST_ASSERT_EQUAL_INT_MESSAGE(JOB_TRANSITION_APPLIED, rc,
"an unqueried running job must always apply (no reject/no-op case)");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j->job.state,
"job must be RUN synchronously, before any WAL commit/apply round "
"trip");
TEST_ASSERT_NULL_MESSAGE(j->job.allocs,
"Req 4.8: a reconciliation-assigned job must never carry a "
"fabricated resource allocation -- the controller genuinely does "
"not know what the scheduler originally computed for it");
unlock(j);
lsn_after = twr_current_lsn();
TEST_ASSERT_TRUE_MESSAGE(lsn_after > lsn_before,
"record_job_reconcile_assign_fact must durably append a "
"WAL_JOB_DISPATCH fact");
/* Caller's responsibility, AFTER unlocking (design.md Postconditions/
* Invariants): notify the scheduler through the existing notify path,
* exactly once, for this actual mutation (Req 7.1-7.3). */
persist_notify_job_updated_already_persisted(&g_twr, j);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_twr_update_hook_fires,
"the scheduler notify path must fire exactly once for this "
"reconciled assignment (Req 7.2) when the caller notifies after "
"unlocking -- omitting it would reproduce bug-1679 at this new "
"call site");
xce_disable_hook(&g_twr.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook);
twr_teardown();
printf("test_reconcile_assign_applies_running_no_allocs_and_notifies: "
"PASS\n");
}
/* Encode helper mirroring test_dispatch_commit.c's dc_encode_job /
* test_wal_job_dispatch_fact.c's wjd_encode_job (capnp_encode_job is
* file-static in persist.c, so a direct WAL_JOB_SUBMIT seed needs its own
* encoder built from the exported encode_Job/new_Job/write_Job codec
* chain). */
static int twr_encode_job(const xce_job_t* j, uint8_t** out, size_t* out_len)
{
struct capn c;
struct capn_segment* cs;
struct Job cj;
Job_ptr jp;
capn_init_malloc(&c);
cs = capn_root(&c).seg;
encode_Job(cs, &cj, (xce_job_t*)j);
jp = new_Job(cs);
write_Job(&cj, jp);
capn_setp(capn_root(&c), 0, jp.p);
*out_len = capn_size(&c);
*out = malloc(*out_len);
if (!*out) { capn_free(&c); return -ENOMEM; }
capn_write_mem(&c, *out, *out_len, 0);
capn_free(&c);
return 0;
}
/* Fresh disk-backed persist_t pointed at an existing tmpdir -- mirrors
* test_dispatch_commit.c's dc_make_fresh_persist/dc_teardown_persist
* exactly, for this test's own Phase 2 (simulated restart). */
static persist_t* twr_make_fresh_persist(const char* tmpdir)
{
persist_t* p = calloc(1, sizeof(persist_t));
persist_config_t cfg;
if (!p) return NULL;
cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
persist_set_workdir(p, tmpdir, "ctrl-test");
if (persist_init(p, &cfg, NULL) != 0) {
free(p);
return NULL;
}
if (persist_start(p) != 0) {
persist_fini(p);
free(p);
return NULL;
}
return p;
}
static void twr_teardown_persist(persist_t* p)
{
persist_stop(p);
persist_fini(p);
free(p);
}
/* Restart-replay round trip -- this task's own explicit completion
* criterion ("durably survives a restart replay with the same result"),
* mirroring test_dispatch_commit.c's test_dc_success_appends_wal_job_
* dispatch_durably's Phase 1/Phase 2 shape exactly, but driving Phase 1's
* WAL_JOB_DISPATCH fact via record_job_reconcile_assign_fact() instead of
* dispatch_a_job_() -- proving bug-1678's NULL-alloc decode branch (already
* tested on both apply seams by test_wal_job_dispatch_fact.c) round-trips
* this new call site's real fact correctly too. Uses its own local
* cluster_t/persist_t (not the shared g_twr fixture) so Phase 1's disk
* state survives past that phase's own teardown, exactly like dc's
* original. */
static void test_reconcile_assign_survives_restart_replay(void)
{
char tmp[256];
uint64_t captured_start_time = 0;
strcpy(tmp, "/tmp/xce_twr22_replay_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Phase 1: real cluster + real disk persist. Seed a durable
* WAL_JOB_SUBMIT fact directly (so replay can reconstruct the job --
* this job was never dispatched through any controller, so there is no
* other WAL_JOB_SUBMIT/DISPATCH in this log), seed the in-memory job_t
* the same way twr_seed_job() does (bypassing dispatch/replay --
* record_job_reconcile_assign_fact() operates on a job_t* the caller
* has already located/constructed, not this function's concern), then
* drive the real function under test. */
{
cluster_t c;
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
persist_config_t cfg;
job_t* j;
worker_t w;
xce_job_state_entry_t report;
int rc;
if (makeString((char*)"twr22-probe") == NULL)
xstring_table_init(64);
if (!g_twr_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
g_twr_jobobj_ready = 1;
}
memset(&c, 0, sizeof(c));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));
c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(c.jobmgr);
cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
persist_set_workdir(&c.persist, tmp, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));
{
job_submit_data_t wal_submit;
xce_job_t wal_jdef;
uint8_t* buf = NULL;
size_t sz = 0;
memset(&wal_submit, 0, sizeof(wal_submit));
wal_submit.queue = (char*)"q";
wal_submit.user = (char*)"tester";
memset(&wal_jdef, 0, sizeof(wal_jdef));
wal_jdef.jobid = TWR_JOBID_ASSIGN_REPLAY;
wal_jdef.state = JOB_STAT_PEND;
wal_jdef.submit = &wal_submit;
TEST_ASSERT_EQUAL_INT(0, twr_encode_job(&wal_jdef, &buf, &sz));
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
(uint32_t)sz));
free(buf);
}
{
job_submit_data_t submit;
xce_job_t jdef;
memset(&submit, 0, sizeof(submit));
submit.queue = (char*)"q";
submit.user = (char*)"tester";
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = TWR_JOBID_ASSIGN_REPLAY;
jdef.state = JOB_STAT_PEND;
jdef.submit = &submit;
j = makeJob(c.jobmgr, &jdef);
TEST_ASSERT_NOT_NULL(j);
}
memset(&w, 0, sizeof(w));
w.host.name = (char*)"twr-worker-replay";
report = (xce_job_state_entry_t){
.jobid = TWR_JOBID_ASSIGN_REPLAY,
.status = JobStateStatus_running,
.exit_code = 0,
.finish_time_ms = 0,
.queried = 0
};
lock(j, RTTI_LOCK_WRITE);
rc = record_job_reconcile_assign_fact(&c, j, &w, &report);
TEST_ASSERT_EQUAL_INT(JOB_TRANSITION_APPLIED, rc);
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
TEST_ASSERT_NULL(j->job.allocs);
captured_start_time = (uint64_t)j->job.start_time;
TEST_ASSERT_NOT_EQUAL(0, captured_start_time);
unlock(j);
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));
persist_stop(&c.persist);
persist_fini(&c.persist);
dereference(c.jobmgr);
xce_hook_fini(&c.hooksys);
}
/* Phase 2: fresh persist_t + fresh cluster_t (jobmgr/hooksys only),
* recover from disk -- simulates a controller restart. */
{
persist_t* p2 = twr_make_fresh_persist(tmp);
cluster_t c2;
job_t* j2 = NULL;
int rc;
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
TEST_ASSERT_NOT_NULL(p2);
memset(&c2, 0, sizeof(c2));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(c2.jobmgr);
p2->cluster = &c2;
rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
lock(c2.jobmgr, RTTI_LOCK_READ);
key_find(c2.jobmgr, (void*)TWR_JOBID_ASSIGN_REPLAY, (void**)&j2);
unlock(c2.jobmgr);
TEST_ASSERT_NOT_NULL_MESSAGE(j2,
"restart replay must reconstruct the job via WAL_JOB_SUBMIT");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(TWR_JOBID_ASSIGN_REPLAY,
j2->job.jobid, "recovered job must be the same jobid");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j2->job.state,
"restart replay must recover the job as RUNNING from the "
"WAL_JOB_DISPATCH fact appended by record_job_reconcile_"
"assign_fact()'s real call (Req 4.8) -- no WAL_JOB_START "
"anywhere in this log");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(captured_start_time,
(uint64_t)j2->job.start_time,
"restart replay must recover the exact dispatch-time "
"start_time stamped by the real call in Phase 1");
TEST_ASSERT_NULL_MESSAGE(j2->job.allocs,
"Req 4.8 / design.md's documented scope decision: replay must "
"NOT fabricate a resource allocation for a reconciliation-"
"assigned job -- allocs stays NULL exactly as it was set "
"eagerly in Phase 1, exercising bug-1678's NULL-alloc decode "
"branch");
xce_hook_fini(&c2.hooksys);
if (c2.jobmgr) dereference(c2.jobmgr);
twr_teardown_persist(p2);
}
{
char cmd[512];
snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
if (system(cmd) != 0)
fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
}
printf("test_reconcile_assign_survives_restart_replay: PASS\n");
}
/* ── resync.c round lifecycle tests (task 3.1) ───────────────────────────
*
* Proves resync_arm_round()/resync_resolve_round()/resync_round_is_current()
* (resync.c) -- the per-worker job-state reconciliation round lifecycle:
* query-set construction (Req 1.1, 2.1, 2.2), a fresh monotonic round token
* per arm distinguishing this round from any prior one for the same worker
* (Req 6.1, 6.3), the XCE_WORKER_RECONCILING status bit (Req 5.1, 5.2), and
* the Req 5.3 bounded timeout resolving the round without any blocking
* call. Reuses the g_twr/twr_setup/twr_teardown fixture and twr_seed_job()
* from task 2.1/2.2's tests above -- resync_arm_round only needs a
* cluster_t with a real c->jobmgr (which g_twr already provides).
*
* No live caller exists yet (worker.c's rejoin_worker/apply_join_state
* wiring is task 6.1; the report-apply glue that would call
* resync_round_is_current for real is task 3.2) -- every test here calls
* resync_arm_round/resync_resolve_round/resync_round_is_current directly,
* exactly as a real caller (task 6.1) eventually will.
*
* resync_arm_round/resync_resolve_round/resync_round_is_current never
* lock/unlock `w` themselves (their documented precondition is that the
* CALLER already holds it) -- so a bare stack worker_t is sufficient and
* safe for those tests, exactly mirroring how task 2.2's own
* record_job_reconcile_assign_fact tests above already use a bare stack
* `worker_t w` with no locking. The ONE exception is the timeout test:
* resync_on_timeout (the internal delaymgr callback resync_test_fire_
* timeout drives directly) runs with no caller-held lock -- because a real
* delaymgr fire has none -- so it genuinely calls lock()/unlock() on `w`
* itself, which requires a real RTTI-managed worker_t (via makeWorker
* against g_twr.workermgr), not a bare stack struct. */
#define TWR_JOBID_RESYNC_A 81101ULL
#define TWR_JOBID_RESYNC_B 81102ULL
#define TWR_JOBID_RESYNC_OTHER 81103ULL
#define TWR_JOBID_RESYNC_DONE 81104ULL
#define TWR_JOBID_RESYNC_NOALLOC 81105ULL
#define TWR_JOBID_RESYNC_SOLO 81106ULL
#define TWR_JOBID_RESYNC_TIMEOUT 81107ULL
/* Same builder chain as twr_make_test_alloc() above, parameterized by host
* -- twr_make_test_alloc() itself hardcodes "twr-worker", which does not
* fit these tests' need for several DISTINCT hosts within one job table. */
static xce_job_alloc_t* twr_make_test_alloc_for_host(uint64_t jobid,
const char* host)
{
return make_job_alloc(jobid, 1,
make_host_allocs(1,
make_host_alloc((char*)host, 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
}
/* True iff `jobid` appears anywhere in ids[0..n) -- used below to prove the
* query set contains exactly the expected members, neither missing a real
* match nor carrying an extra one. */
static int twr_ids_contains(const uint64_t* ids, int n, uint64_t jobid)
{
int i;
for (i = 0; i < n; i++) {
if (ids[i] == jobid)
return 1;
}
return 0;
}
/* A real RTTI-managed worker_t (via makeWorker against g_twr.workermgr),
* for the one test below that needs genuine lock()/unlock() to succeed on
* `w` -- mirrors worker.c's own init_worker() construction
* (`tmp.name = (char*)host; tmp.attrs = attrs; w = makeWorker(c->workermgr,
* &tmp);`). */
static worker_t* twr_make_test_worker(const char* hostname)
{
hostattrs_t attrs;
xce_worker_t tmp;
memset(&attrs, 0, sizeof(attrs));
memset(&tmp, 0, sizeof(tmp));
tmp.name = (char*)hostname;
tmp.attrs = &attrs;
return makeWorker(g_twr.workermgr, &tmp);
}
/* Req 1.1, 2.1, 2.2: the query set matches EXACTLY the non-terminal jobs
* assigned to the given host -- not fewer (a genuine match omitted), not
* more (a different host's job, a terminal job on the SAME host, or a
* never-dispatched job with no host assignment at all, leaking in). */
static void test_resync_arm_round_query_set_matches_exactly(void)
{
job_t* ja;
job_t* jb;
job_t* jother;
job_t* jdone;
job_t* jnoalloc;
worker_t w;
int rc;
twr_setup();
ja = twr_seed_job(TWR_JOBID_RESYNC_A, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(ja);
ja->job.allocs
= twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_A, "resync-host-A");
jb = twr_seed_job(TWR_JOBID_RESYNC_B, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(jb);
jb->job.allocs
= twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_B, "resync-host-A");
/* Same non-terminal state, DIFFERENT host -- must NOT appear in
* resync-host-A's query set. */
jother = twr_seed_job(TWR_JOBID_RESYNC_OTHER, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(jother);
jother->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_OTHER,
"resync-host-B");
/* SAME host, but terminal (DONE) -- must NOT appear (Req 1.1's
* "non-terminal" qualifier). */
jdone = twr_seed_job(TWR_JOBID_RESYNC_DONE, JOB_STAT_DONE);
TEST_ASSERT_NOT_NULL(jdone);
jdone->job.allocs
= twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_DONE, "resync-host-A");
/* Never dispatched (no allocation at all) -- not assigned to ANY host,
* so must NOT appear regardless of state. */
jnoalloc = twr_seed_job(TWR_JOBID_RESYNC_NOALLOC, JOB_STAT_PEND);
TEST_ASSERT_NOT_NULL(jnoalloc);
TEST_ASSERT_NULL(jnoalloc->job.allocs);
memset(&w, 0, sizeof(w));
w.host.name = (char*)"resync-host-A";
rc = resync_arm_round(&g_twr, &w);
TEST_ASSERT_EQUAL_INT_MESSAGE(2, rc,
"exactly the 2 non-terminal resync-host-A jobs must be armed into "
"the round");
TEST_ASSERT_NOT_NULL(w.resync);
TEST_ASSERT_EQUAL_INT(2, w.resync->n_queried_jobids);
TEST_ASSERT_TRUE_MESSAGE(
twr_ids_contains(w.resync->queried_jobids, w.resync->n_queried_jobids,
TWR_JOBID_RESYNC_A),
"job A (non-terminal, resync-host-A) must be in the query set");
TEST_ASSERT_TRUE_MESSAGE(
twr_ids_contains(w.resync->queried_jobids, w.resync->n_queried_jobids,
TWR_JOBID_RESYNC_B),
"job B (non-terminal, resync-host-A) must be in the query set");
TEST_ASSERT_FALSE_MESSAGE(
twr_ids_contains(w.resync->queried_jobids, w.resync->n_queried_jobids,
TWR_JOBID_RESYNC_OTHER),
"a job assigned to a DIFFERENT host must not appear");
TEST_ASSERT_FALSE_MESSAGE(
twr_ids_contains(w.resync->queried_jobids, w.resync->n_queried_jobids,
TWR_JOBID_RESYNC_DONE),
"a TERMINAL job on the same host must not appear (Req 1.1)");
TEST_ASSERT_FALSE_MESSAGE(
twr_ids_contains(w.resync->queried_jobids, w.resync->n_queried_jobids,
TWR_JOBID_RESYNC_NOALLOC),
"a never-dispatched job (no host assignment at all) must not "
"appear");
TEST_ASSERT_TRUE_MESSAGE(IS_WORKER_RECONCILING(w.host.status),
"XCE_WORKER_RECONCILING must be set once a round is armed");
resync_resolve_round(&g_twr, &w);
TEST_ASSERT_NULL(w.resync);
TEST_ASSERT_FALSE(IS_WORKER_RECONCILING(w.host.status));
twr_teardown();
printf("test_resync_arm_round_query_set_matches_exactly: PASS\n");
}
/* Req 1.2: a worker with zero non-terminal jobs assigned is a documented
* no-op -- no round armed, no status bit set. */
static void test_resync_arm_round_no_jobs_is_noop(void)
{
worker_t w;
int rc;
twr_setup();
memset(&w, 0, sizeof(w));
w.host.name = (char*)"resync-host-empty";
rc = resync_arm_round(&g_twr, &w);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"a worker with no non-terminal jobs must arm zero jobs (Req 1.2)");
TEST_ASSERT_NULL_MESSAGE(w.resync,
"no resync_state_t must be attached for the Req 1.2 no-op case");
TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w.host.status),
"XCE_WORKER_RECONCILING must not be set for the Req 1.2 no-op case");
twr_teardown();
printf("test_resync_arm_round_no_jobs_is_noop: PASS\n");
}
/* Req 6.1, 6.3: arming a round for a worker that already carries one
* (simulating a disconnect interrupting an outstanding round, then a
* rejoin) discards the prior round and produces a FRESH, DIFFERENT token --
* and the original token is provably no longer current once superseded. */
static void test_resync_arm_round_fresh_token_supersedes_prior(void)
{
job_t* j;
worker_t w;
int rc;
uint64_t token1;
uint64_t token2;
twr_setup();
j = twr_seed_job(TWR_JOBID_RESYNC_SOLO, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_SOLO,
"resync-host-solo");
memset(&w, 0, sizeof(w));
w.host.name = (char*)"resync-host-solo";
rc = resync_arm_round(&g_twr, &w);
TEST_ASSERT_EQUAL_INT(1, rc);
TEST_ASSERT_NOT_NULL(w.resync);
token1 = w.resync->round_token;
TEST_ASSERT_TRUE_MESSAGE(resync_round_is_current(&w, token1),
"the round just armed must report its own token as current");
/* Simulated disconnect-then-rejoin: the round never resolved, but the
* worker rejoins and is armed again. */
rc = resync_arm_round(&g_twr, &w);
TEST_ASSERT_EQUAL_INT(1, rc);
TEST_ASSERT_NOT_NULL(w.resync);
token2 = w.resync->round_token;
TEST_ASSERT_NOT_EQUAL_MESSAGE(token1, token2,
"re-arming must produce a token distinct from the prior round's "
"(Req 6.1, 6.3)");
TEST_ASSERT_FALSE_MESSAGE(resync_round_is_current(&w, token1),
"the ORIGINAL token must now report stale/superseded (Req 6.3) -- "
"the mechanism a later report-apply caller uses to discard a "
"report carrying it");
TEST_ASSERT_TRUE_MESSAGE(resync_round_is_current(&w, token2),
"the NEW token must report current");
resync_resolve_round(&g_twr, &w);
twr_teardown();
printf("test_resync_arm_round_fresh_token_supersedes_prior: PASS\n");
}
/* Req 5.1, 5.2: XCE_WORKER_RECONCILING is set once a round is armed and
* cleared once resync_resolve_round runs -- resync_round_is_current also
* correctly reports the resolved round as no longer current. */
static void test_resync_reconciling_flag_set_after_arm_cleared_after_resolve(
void)
{
job_t* j;
worker_t w;
int rc;
uint64_t token;
twr_setup();
j = twr_seed_job(TWR_JOBID_RESYNC_SOLO, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_SOLO,
"resync-host-flag");
memset(&w, 0, sizeof(w));
w.host.name = (char*)"resync-host-flag";
TEST_ASSERT_FALSE(IS_WORKER_RECONCILING(w.host.status));
rc = resync_arm_round(&g_twr, &w);
TEST_ASSERT_EQUAL_INT(1, rc);
TEST_ASSERT_TRUE_MESSAGE(IS_WORKER_RECONCILING(w.host.status),
"XCE_WORKER_RECONCILING must be set once armed (Req 5.1)");
token = w.resync->round_token;
resync_resolve_round(&g_twr, &w);
TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w.host.status),
"XCE_WORKER_RECONCILING must clear once resolved (Req 5.2)");
TEST_ASSERT_NULL_MESSAGE(w.resync,
"resync_resolve_round must detach/free w->resync");
TEST_ASSERT_FALSE_MESSAGE(resync_round_is_current(&w, token),
"a resolved round's token must no longer report current");
/* Idempotent: resolving an already-resolved (NULL) round is a safe
* no-op, never a double-free. */
resync_resolve_round(&g_twr, &w);
TEST_ASSERT_NULL(w.resync);
twr_teardown();
printf("test_resync_reconciling_flag_set_after_arm_cleared_after_"
"resolve: PASS\n");
}
/* bug-1841 (new): resync_resolve_round() must notify HOOK_DETAIL_WORKER_
* UPDATE subscribers once RECONCILING actually clears -- otherwise the
* scheduler's own copy of this worker's status (fed purely from whatever
* WORKERS_PUSH it last received; core_events.c's core_upsert_worker does a
* bare `worker->status = w->status` with no other refresh path) stays
* RECONCILING forever, permanently freezing that worker's resInUse
* (sched_reset_worker_/sched_update_account's by-design bug-1799/1801
* freeze) even after every job on it has long since finished DONE --
* live-observed via a real scheduler-replay capture: a 22-slot worker
* stuck at used=22.00 for the rest of the run with zero RUN jobs left.
* Mirrors bug-1679's identical job-side gap and its
* test_reconcile_pending_applies_releases_allocs_and_notifies spy-hook
* proof above. */
static int g_twr_worker_update_hook_fires;
static int twr_worker_update_spy_hook(int type, int situation, int detail,
void* data, va_list ap)
{
(void)type; (void)data; (void)ap;
if (situation == HOOK_AFTER_EVENT && detail == HOOK_DETAIL_WORKER_UPDATE)
g_twr_worker_update_hook_fires++;
return 0;
}
static void test_resync_resolve_round_notifies_worker_update(void)
{
job_t* j;
worker_t w;
int rc;
twr_setup();
g_twr_worker_update_hook_fires = 0;
TEST_ASSERT_EQUAL_INT(0,
xce_add_hook(&g_twr.hooksys, XCE_CONTROLLER_HOOK_WORKER,
twr_worker_update_spy_hook, NULL));
j = twr_seed_job(TWR_JOBID_RESYNC_SOLO, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_SOLO,
"resync-host-notify");
memset(&w, 0, sizeof(w));
w.host.name = (char*)"resync-host-notify";
rc = resync_arm_round(&g_twr, &w);
TEST_ASSERT_EQUAL_INT(1, rc);
TEST_ASSERT_TRUE(IS_WORKER_RECONCILING(w.host.status));
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_twr_worker_update_hook_fires,
"arming a round must not itself fire HOOK_DETAIL_WORKER_UPDATE -- "
"only resolving one does");
resync_resolve_round(&g_twr, &w);
TEST_ASSERT_FALSE(IS_WORKER_RECONCILING(w.host.status));
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_twr_worker_update_hook_fires,
"resync_resolve_round must notify HOOK_DETAIL_WORKER_UPDATE "
"subscribers (scheduler/apiserver) once RECONCILING actually "
"clears -- otherwise the scheduler never learns and stays frozen");
/* Idempotent no-op: resolving an already-resolved round must NOT fire a
* second, spurious notification. */
resync_resolve_round(&g_twr, &w);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_twr_worker_update_hook_fires,
"resolving an already-resolved (NULL) round must not re-notify");
twr_teardown();
printf("test_resync_resolve_round_notifies_worker_update: PASS\n");
}
/* Req 5.3: the timeout resolves an outstanding round -- and does so via a
* plain, synchronous function call with no sleep/wait of any kind, proving
* the resolution itself never blocks (ha-async-io.md's delaymgr
* thread-role contract). g_twr never starts a real delaymgr (matching
* every other bare-cluster_t fixture in this file), so resync_arm_round's
* own `!c->delaymgr.running` guard already took the no-real-timer branch
* when the round above was armed; resync_test_fire_timeout below drives
* the EXACT SAME internal resolution logic
* (resync_on_timeout) a real timer firing would have. */
static void test_resync_timeout_resolves_round_without_blocking(void)
{
job_t* j;
worker_t* w;
int rc;
uint64_t token;
twr_setup();
j = twr_seed_job(TWR_JOBID_RESYNC_TIMEOUT, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_TIMEOUT,
"resync-host-timeout");
w = twr_make_test_worker("resync-host-timeout");
TEST_ASSERT_NOT_NULL(w);
rc = resync_arm_round(&g_twr, w);
TEST_ASSERT_EQUAL_INT(1, rc);
TEST_ASSERT_NOT_NULL(w->resync);
TEST_ASSERT_TRUE(IS_WORKER_RECONCILING(w->host.status));
token = w->resync->round_token;
/* resync_on_timeout (invoked via resync_test_fire_timeout) locks/
* unlocks `w` itself -- this call must NOT be wrapped in an external
* lock(w, ...), or it would deadlock against `w`'s own non-recursive
* write lock. */
resync_test_fire_timeout(&g_twr, w, token);
TEST_ASSERT_NULL_MESSAGE(w->resync,
"the timeout must resolve (free/detach) the round");
TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
"the timeout must clear XCE_WORKER_RECONCILING so the worker is "
"not withheld from dispatch indefinitely (Req 5.1, 5.2, 5.3)");
/* A second, late fire for the SAME (now-stale) token must be a safe
* no-op -- never a double-free/double-clear against an already-
* resolved round. This is exactly the round-token staleness check
* (Req 6.3) protecting against a real delaymgr task that could not be
* dequeued/cancelled (delay.c exposes no cancel primitive) firing late
* against a resync_state_t that is already gone or has already been
* replaced by a newer round. */
resync_test_fire_timeout(&g_twr, w, token);
TEST_ASSERT_NULL(w->resync);
TEST_ASSERT_FALSE(IS_WORKER_RECONCILING(w->host.status));
twr_teardown();
printf("test_resync_timeout_resolves_round_without_blocking: PASS\n");
}
/* ── resync_apply_report() tests (task 3.2) ──────────────────────────────
*
* Proves resync_apply_report() (resync.c) -- the report-apply glue wiring
* together task 2.1's record_job_reconcile_pending_fact, task 2.2's
* record_job_reconcile_assign_fact, and task 3.1's resync_arm_round/
* resync_resolve_round/resync_round_is_current. Every test here uses a real
* RTTI-managed worker_t (via twr_make_test_worker), since resync_apply_
* report locks/unlocks `w` itself (unlike resync_arm_round/resync_resolve_
* round/resync_round_is_current above, which require the caller to
* pre-hold `w`'s lock and so tolerate a bare stack struct). */
#define TWR_JOBID_REPORT_FINOK 81108ULL
#define TWR_JOBID_REPORT_FINFAIL 81109ULL
#define TWR_JOBID_REPORT_RUNNING 81110ULL
#define TWR_JOBID_REPORT_UNREC 81111ULL
#define TWR_JOBID_REPORT_EXTRA 81112ULL
#define TWR_JOBID_REPORT_DUP_A 81113ULL
#define TWR_JOBID_REPORT_DUP_B 81114ULL
#define TWR_JOBID_REPORT_STALE 81115ULL
#define TWR_JOBID_REPORT_PARTIAL_A 81116ULL
#define TWR_JOBID_REPORT_PARTIAL_B 81117ULL
#define TWR_JOBID_REPORT_EXTRANOOP_Q 81118ULL
#define TWR_JOBID_REPORT_EXTRANOOP_E 81119ULL
#define TWR_JOBID_REPORT_FINISHTIME_REPLAY 81120ULL
#define TWR_JOBID_REPORT_LOSTDISPATCH_Q 81121ULL
#define TWR_JOBID_REPORT_LOSTDISPATCH 81122ULL
/* THE core proof (Req 4.1-4.5, 6.1): a single report covering all five
* distinct outcome kinds in one pass -- finished-success, finished-failure,
* still-running (queried), unrecognized (queried), and an unqueried extra --
* leaves the job table in the correct state for each, routed through the
* correct existing/new fact function, and resolves the round (outstanding
* set empties -> XCE_WORKER_RECONCILING clears) once every QUERIED entry
* has been answered.
*
* The unqueried-extra entry here is seeded TERMINAL (DONE), exercising
* resync_apply_running_entry()'s branch (a) -- ALWAYS a no-op, regardless
* of host (Req 4.4: a terminal job is never assign-eligible). This is
* deliberate, not an oversight: round-2 review caught that this test
* ORIGINALLY seeded j_extra TERMINAL and then asserted it became RUN --
* i.e. it baked in, and certified as PASSING, the exact Req 4.4 regression
* a since-corrected two-way guard introduced (calling record_job_
* reconcile_assign_fact() on an already-terminal job, unconditionally
* clobbering DONE/EXIT back to RUN). Branch (c) -- non-terminal AND
* genuinely unassigned, the real "gets assigned" case -- is proven
* separately, with full restart-replay rigor, by test_apply_report_
* unqueried_extra_unassigned_pend_job_gets_assigned below; branch (b) --
* non-terminal but already assigned (to this host or elsewhere) -- by
* test_apply_report_unqueried_extra_already_tracked_is_noop. */
static void test_apply_report_all_five_outcomes_updates_job_table(void)
{
job_t* j_finok;
job_t* j_finfail;
job_t* j_running;
job_t* j_unrec;
job_t* j_extra;
worker_t* w;
int rc;
uint64_t token;
uint64_t lsn_before, lsn_after;
xce_job_state_entry_t e_finok, e_finfail, e_running, e_unrec, e_extra;
xce_job_state_entry_t* entries[5];
xce_job_state_report_t report;
twr_setup();
j_finok = twr_seed_job(TWR_JOBID_REPORT_FINOK, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j_finok);
j_finok->job.allocs = twr_make_test_alloc_for_host(
TWR_JOBID_REPORT_FINOK, "resync-report-host");
j_finfail = twr_seed_job(TWR_JOBID_REPORT_FINFAIL, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j_finfail);
j_finfail->job.allocs = twr_make_test_alloc_for_host(
TWR_JOBID_REPORT_FINFAIL, "resync-report-host");
j_running = twr_seed_job(TWR_JOBID_REPORT_RUNNING, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j_running);
j_running->job.allocs = twr_make_test_alloc_for_host(
TWR_JOBID_REPORT_RUNNING, "resync-report-host");
j_unrec = twr_seed_job(TWR_JOBID_REPORT_UNREC, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j_unrec);
j_unrec->job.allocs = twr_make_test_alloc_for_host(
TWR_JOBID_REPORT_UNREC, "resync-report-host");
/* Seeded TERMINAL (DONE), so it is NOT part of the round's query set
* (arm_round only collects non-terminal jobs), AND -- per Req 4.4 --
* must be left COMPLETELY untouched when later reported running as an
* unqueried extra: a terminal record is never assign-eligible, full
* stop, regardless of host (resync_apply_running_entry()'s branch (a),
* resync.c). */
j_extra = twr_seed_job(TWR_JOBID_REPORT_EXTRA, JOB_STAT_DONE);
TEST_ASSERT_NOT_NULL(j_extra);
TEST_ASSERT_NULL(j_extra->job.allocs);
w = twr_make_test_worker("resync-report-host");
TEST_ASSERT_NOT_NULL(w);
rc = resync_arm_round(&g_twr, w);
TEST_ASSERT_EQUAL_INT_MESSAGE(4, rc,
"exactly the 4 non-terminal jobs on this host must be armed");
TEST_ASSERT_NOT_NULL(w->resync);
token = w->resync->round_token;
e_finok = (xce_job_state_entry_t){
.jobid = TWR_JOBID_REPORT_FINOK, .status = JobStateStatus_finishedOk,
.exit_code = 0, .finish_time_ms = 1000, .queried = 1
};
e_finfail = (xce_job_state_entry_t){
.jobid = TWR_JOBID_REPORT_FINFAIL,
.status = JobStateStatus_finishedFail,
.exit_code = 7, .finish_time_ms = 2000, .queried = 1
};
e_running = (xce_job_state_entry_t){
.jobid = TWR_JOBID_REPORT_RUNNING, .status = JobStateStatus_running,
.exit_code = 0, .finish_time_ms = 0, .queried = 1
};
e_unrec = (xce_job_state_entry_t){
.jobid = TWR_JOBID_REPORT_UNREC, .status = JobStateStatus_unrecognized,
.exit_code = 0, .finish_time_ms = 0, .queried = 1
};
e_extra = (xce_job_state_entry_t){
.jobid = TWR_JOBID_REPORT_EXTRA, .status = JobStateStatus_running,
.exit_code = 0, .finish_time_ms = 0, .queried = 0
};
entries[0] = &e_finok;
entries[1] = &e_finfail;
entries[2] = &e_running;
entries[3] = &e_unrec;
entries[4] = &e_extra;
report = (xce_job_state_report_t){
.n_total = 5, .pos = 0, .count = 5, .entries = entries
};
lsn_before = twr_current_lsn();
rc = resync_apply_report(&g_twr, w, token, &report);
TEST_ASSERT_EQUAL_INT(0, rc);
lsn_after = twr_current_lsn();
/* record_job_finish_fact_at() (bug-1747 fix)/record_job_fail_fact
* (called via on_job_fail, unchanged -- Req 4.1) follow the SAME
* durable-append-only, DEFERRED pattern as any ordinary worker
* finish/fail report: the in-memory state mutation happens later, when
* apply_committed_entry() replays the committed fact -- NOT
* synchronously here (unlike record_job_reconcile_pending_fact/assign_
* fact's own EAGER pattern below, asserted synchronously exactly
* because task 2.1/2.2 built those two functions to mutate eagerly).
* This exact deferred behavior is already proven end-to-end (including
* via a real restart replay) by test_wal_job_finish_fact.c's own
* test_wjf_real_on_job_finish_survives_restart -- this task's own job
* is only to prove routing (the correct fact TYPE, with the correct
* worker-reported finish_time_ms for finishedOk, was durably appended
* through the correct call), not to re-prove that already-tested apply
* seam; test_apply_report_finished_success_uses_worker_reported_
* finish_time below proves the finish_time_ms plumbing specifically,
* via a real restart replay. */
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_finok->job.state,
"finished-success routes through the DEFERRED record_job_finish_"
"fact pattern -- state mutation happens later via apply_committed_"
"entry, not synchronously here");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_finfail->job.state,
"finished-failure routes through the DEFERRED record_job_fail_fact "
"pattern -- state mutation happens later via apply_committed_entry, "
"not synchronously here");
TEST_ASSERT_TRUE_MESSAGE(lsn_after >= lsn_before + 3,
"at least 3 durable facts must be appended for this report: "
"finished-success (WAL_JOB_FINISH), finished-failure (WAL_JOB_"
"FAIL), and unrecognized (WAL_JOB_UPDATE) -- the still-running "
"entry and the TERMINAL unqueried extra (Req 4.4 no-op) record "
"none");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_running->job.state,
"a queried still-running job must be left exactly as-is -- no fact "
"recorded");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j_unrec->job.state,
"unrecognized must apply through record_job_reconcile_pending_fact "
"(-> PEND)");
TEST_ASSERT_NULL_MESSAGE(j_unrec->job.allocs,
"bug-1631: reconciled-to-pending must release the allocation");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE, j_extra->job.state,
"Req 4.4: a TERMINAL job reported running as an unqueried extra "
"must be left COMPLETELY untouched -- never routed through "
"record_job_reconcile_assign_fact, regardless of host");
TEST_ASSERT_NULL_MESSAGE(j_extra->job.allocs,
"the terminal extra's allocation (already NULL) must stay "
"untouched too");
TEST_ASSERT_NULL_MESSAGE(w->resync,
"once every queried job (4 of them) has a reconciled outcome, the "
"round must resolve");
TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
"XCE_WORKER_RECONCILING must clear once the round resolves");
twr_teardown();
printf("test_apply_report_all_five_outcomes_updates_job_table: PASS\n");
}
/* Req 6.2: applying the identical report a second time (same round token)
* must produce the same result as applying it once -- no further job-table
* change, no additional WAL append. Since the first application already
* fully resolves the round (both entries here are queried), the round
* token is no longer current by the time the second, identical call
* arrives -- resync_apply_report's own Req 6.1/6.3 stale-round discard is
* exactly the mechanism that makes the duplicate a pure no-op. */
static void test_apply_report_duplicate_reapplication_is_noop(void)
{
job_t* ja;
job_t* jb;
worker_t* w;
int rc;
uint64_t token;
uint64_t lsn_before_first, lsn_after_first, lsn_after_second;
xce_job_state_entry_t ea, eb;
xce_job_state_entry_t* entries[2];
xce_job_state_report_t report;
twr_setup();
ja = twr_seed_job(TWR_JOBID_REPORT_DUP_A, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(ja);
ja->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_REPORT_DUP_A,
"resync-dup-host");
jb = twr_seed_job(TWR_JOBID_REPORT_DUP_B, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(jb);
jb->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_REPORT_DUP_B,
"resync-dup-host");
w = twr_make_test_worker("resync-dup-host");
TEST_ASSERT_NOT_NULL(w);
rc = resync_arm_round(&g_twr, w);
TEST_ASSERT_EQUAL_INT(2, rc);
token = w->resync->round_token;
ea = (xce_job_state_entry_t){
.jobid = TWR_JOBID_REPORT_DUP_A, .status = JobStateStatus_finishedOk,
.exit_code = 0, .finish_time_ms = 500, .queried = 1
};
eb = (xce_job_state_entry_t){
.jobid = TWR_JOBID_REPORT_DUP_B, .status = JobStateStatus_unrecognized,
.exit_code = 0, .finish_time_ms = 0, .queried = 1
};
entries[0] = &ea;
entries[1] = &eb;
report = (xce_job_state_report_t){
.n_total = 2, .pos = 0, .count = 2, .entries = entries
};
lsn_before_first = twr_current_lsn();
rc = resync_apply_report(&g_twr, w, token, &report);
TEST_ASSERT_EQUAL_INT(0, rc);
/* ea (finishedOk) routes through the DEFERRED record_job_finish_fact_at
* pattern (bug-1747 fix) -- state mutation happens later via
* apply_committed_entry, not synchronously here (see the "all five
* outcomes" test's own comment for the full rationale); ja's state is
* asserted unchanged (still RUN) below instead. */
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, ja->job.state);
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, jb->job.state);
TEST_ASSERT_NULL(jb->job.allocs);
TEST_ASSERT_NULL_MESSAGE(w->resync,
"the round must resolve after the first, complete application");
lsn_after_first = twr_current_lsn();
TEST_ASSERT_TRUE_MESSAGE(lsn_after_first >= lsn_before_first + 2,
"the first application must durably append a fact for each of the "
"2 queried entries");
/* Second, IDENTICAL application -- same token, same report contents. */
rc = resync_apply_report(&g_twr, w, token, &report);
TEST_ASSERT_EQUAL_INT(0, rc);
lsn_after_second = twr_current_lsn();
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, ja->job.state,
"a duplicate re-application must change nothing further");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, jb->job.state,
"a duplicate re-application must change nothing further");
TEST_ASSERT_NULL(jb->job.allocs);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_after_first, lsn_after_second,
"a duplicate re-application for an already-resolved round must "
"append no further WAL fact (Req 6.2)");
TEST_ASSERT_NULL(w->resync);
twr_teardown();
printf("test_apply_report_duplicate_reapplication_is_noop: PASS\n");
}
/* Req 6.1, 6.3: a report carrying a round token already superseded by a
* newer round for the SAME worker is dropped in its entirety -- the job
* table must be completely untouched, and the (unrelated, current) newer
* round must be left exactly as it was. */
static void test_apply_report_stale_round_token_is_discarded(void)
{
job_t* j;
worker_t* w;
int rc;
uint64_t token1, token2;
uint64_t lsn_before, lsn_after;
xce_job_state_entry_t e;
xce_job_state_entry_t* entries[1];
xce_job_state_report_t report;
twr_setup();
j = twr_seed_job(TWR_JOBID_REPORT_STALE, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_REPORT_STALE,
"resync-stale-host");
w = twr_make_test_worker("resync-stale-host");
TEST_ASSERT_NOT_NULL(w);
rc = resync_arm_round(&g_twr, w);
TEST_ASSERT_EQUAL_INT(1, rc);
token1 = w->resync->round_token;
/* Simulated disconnect-then-rejoin: a fresh round supersedes the first
* before its report ever arrives. */
rc = resync_arm_round(&g_twr, w);
TEST_ASSERT_EQUAL_INT(1, rc);
token2 = w->resync->round_token;
TEST_ASSERT_NOT_EQUAL(token1, token2);
e = (xce_job_state_entry_t){
.jobid = TWR_JOBID_REPORT_STALE, .status = JobStateStatus_finishedOk,
.exit_code = 0, .finish_time_ms = 999, .queried = 1
};
entries[0] = &e;
report = (xce_job_state_report_t){
.n_total = 1, .pos = 0, .count = 1, .entries = entries
};
lsn_before = twr_current_lsn();
/* A report carrying the now-stale token1 must be dropped entirely. */
rc = resync_apply_report(&g_twr, w, token1, &report);
TEST_ASSERT_EQUAL_INT(0, rc);
lsn_after = twr_current_lsn();
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j->job.state,
"a stale-round report must be dropped without touching the job "
"table");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before, lsn_after,
"a stale-round report must append no WAL fact");
TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
"the CURRENT (newer) round must be left outstanding, untouched by "
"the stale report");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(token2, w->resync->round_token,
"the current round's own token must be unaffected");
TEST_ASSERT_EQUAL_INT(1, w->resync->n_queried_jobids);
resync_resolve_round(&g_twr, w);
twr_teardown();
printf("test_apply_report_stale_round_token_is_discarded: PASS\n");
}
/* Req 5.1, 5.2: a partial report (not all queried jobs answered yet) must
* NOT resolve the round -- the outstanding set stays non-empty and
* XCE_WORKER_RECONCILING stays set -- until a LATER call with the
* remaining entries completes it, proving resolution is genuinely
* incremental against the outstanding set, not "resolve unconditionally
* after any report." */
static void test_apply_report_partial_report_does_not_resolve_round(void)
{
job_t* ja;
job_t* jb;
worker_t* w;
int rc;
uint64_t token;
uint64_t lsn_before, lsn_after;
xce_job_state_entry_t ea, eb;
xce_job_state_entry_t* entries_a[1];
xce_job_state_entry_t* entries_b[1];
xce_job_state_report_t report_a, report_b;
twr_setup();
ja = twr_seed_job(TWR_JOBID_REPORT_PARTIAL_A, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(ja);
ja->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_REPORT_PARTIAL_A,
"resync-partial-host");
jb = twr_seed_job(TWR_JOBID_REPORT_PARTIAL_B, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(jb);
jb->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_REPORT_PARTIAL_B,
"resync-partial-host");
w = twr_make_test_worker("resync-partial-host");
TEST_ASSERT_NOT_NULL(w);
rc = resync_arm_round(&g_twr, w);
TEST_ASSERT_EQUAL_INT(2, rc);
token = w->resync->round_token;
ea = (xce_job_state_entry_t){
.jobid = TWR_JOBID_REPORT_PARTIAL_A,
.status = JobStateStatus_finishedOk,
.exit_code = 0, .finish_time_ms = 111, .queried = 1
};
entries_a[0] = &ea;
report_a = (xce_job_state_report_t){
.n_total = 1, .pos = 0, .count = 1, .entries = entries_a
};
lsn_before = twr_current_lsn();
rc = resync_apply_report(&g_twr, w, token, &report_a);
TEST_ASSERT_EQUAL_INT(0, rc);
lsn_after = twr_current_lsn();
/* ea (finishedOk) routes through the DEFERRED record_job_finish_fact
* pattern (via on_job_finish, unchanged) -- state mutation happens
* later via apply_committed_entry, not synchronously here (see the
* "all five outcomes" test's own comment for the full rationale); a
* durable append is asserted instead. */
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, ja->job.state);
TEST_ASSERT_TRUE_MESSAGE(lsn_after > lsn_before,
"job A's finished-success entry must durably append a fact");
TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
"job B has not been answered yet -- the round must stay "
"outstanding");
TEST_ASSERT_EQUAL_INT(1, w->resync->n_queried_jobids);
TEST_ASSERT_TRUE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
"XCE_WORKER_RECONCILING must remain set while the round is still "
"partially outstanding");
eb = (xce_job_state_entry_t){
.jobid = TWR_JOBID_REPORT_PARTIAL_B, .status = JobStateStatus_running,
.exit_code = 0, .finish_time_ms = 0, .queried = 1
};
entries_b[0] = &eb;
report_b = (xce_job_state_report_t){
.n_total = 1, .pos = 0, .count = 1, .entries = entries_b
};
rc = resync_apply_report(&g_twr, w, token, &report_b);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, jb->job.state);
TEST_ASSERT_NULL_MESSAGE(w->resync,
"the round must resolve once the LAST outstanding queried job "
"(B) is answered");
TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
"XCE_WORKER_RECONCILING must clear once the round fully resolves");
twr_teardown();
printf("test_apply_report_partial_report_does_not_resolve_round: "
"PASS\n");
}
/* Req 3.6/4.7/4.4 (the crux branching this task's own brief flagged as
* needing careful resolution against job.c's record_job_reconcile_assign_
* fact doc comment; round-2 review strengthened this test after finding it
* could not actually distinguish "correctly no-op'd" from "incorrectly
* reassigned" -- see below): an unqueried extra reported RUNNING that the
* controller ALREADY has a non-terminal record of (here: assigned to a
* DIFFERENT host -- resync_apply_running_entry()'s branch (b), non-terminal
* AND job->job.allocs != NULL) must NOT be routed through record_job_
* reconcile_assign_fact -- it is already correctly represented by simply
* existing and being reported running.
*
* Proven three ways, precisely enough to actually catch a regression back
* to either of the two review-caught defects (round 1's Req 4.8-axis bug;
* round 2's Req 4.4-axis bug):
* 1. allocs survives UNTOUCHED -- record_job_reconcile_assign_fact always
* sets job->job.allocs to NULL (Req 4.8), so a surviving allocation
* proves the no-op branch fired instead of the assign branch.
* 2. start_time survives UNTOUCHED at its seeded sentinel value --
* record_job_reconcile_assign_fact ALSO unconditionally stamps
* job->job.start_time = time(NULL) (bug-1697), even though the net
* STATE effect (RUN -> RUN) looks harmless and is invisible to (1) or
* to a bare state-equality assertion; this is exactly what round 1's
* version of this test could not catch, since it never checked
* start_time and the job was already RUN before the call either way.
* 3. no additional WAL fact is appended (LSN unchanged) -- record_job_
* reconcile_assign_fact durably appends a fresh WAL_JOB_DISPATCH fact
* unconditionally; a no-op must append nothing (the paired queried
* "running" entry records nothing either, so the WHOLE report's LSN
* delta must be exactly zero). */
static void test_apply_report_unqueried_extra_already_tracked_is_noop(void)
{
job_t* j_queried;
job_t* j_extra;
worker_t* w;
int rc;
uint64_t token;
uint64_t lsn_before, lsn_after;
time_t seeded_start_time;
xce_job_state_entry_t e_queried, e_extra;
xce_job_state_entry_t* entries[2];
xce_job_state_report_t report;
twr_setup();
/* Keeps the round non-empty (arm_round needs >=1 non-terminal job on
* w's own host to arm at all) -- unrelated to the extra under test. */
j_queried = twr_seed_job(TWR_JOBID_REPORT_EXTRANOOP_Q, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j_queried);
j_queried->job.allocs = twr_make_test_alloc_for_host(
TWR_JOBID_REPORT_EXTRANOOP_Q, "resync-extranoop-host");
/* Non-terminal (RUN), already assigned -- to a DIFFERENT host, so
* arm_round's own query-set construction (non-terminal jobs assigned
* to w's specific host) never includes it; it arrives only as an
* unqueried extra in the report below. */
j_extra = twr_seed_job(TWR_JOBID_REPORT_EXTRANOOP_E, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j_extra);
j_extra->job.allocs = twr_make_test_alloc_for_host(
TWR_JOBID_REPORT_EXTRANOOP_E, "resync-extranoop-OTHER-host");
/* A deliberately-distinguishable sentinel, far from "now" -- so a
* wrongful record_job_reconcile_assign_fact() call (which stamps
* time(NULL)) is caught precisely, not masked by coincidence. */
seeded_start_time = (time_t)424242;
j_extra->job.start_time = seeded_start_time;
w = twr_make_test_worker("resync-extranoop-host");
TEST_ASSERT_NOT_NULL(w);
rc = resync_arm_round(&g_twr, w);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
"only j_queried is assigned to w's own host");
token = w->resync->round_token;
e_queried = (xce_job_state_entry_t){
.jobid = TWR_JOBID_REPORT_EXTRANOOP_Q,
.status = JobStateStatus_running,
.exit_code = 0, .finish_time_ms = 0, .queried = 1
};
e_extra = (xce_job_state_entry_t){
.jobid = TWR_JOBID_REPORT_EXTRANOOP_E,
.status = JobStateStatus_running,
.exit_code = 0, .finish_time_ms = 0, .queried = 0
};
entries[0] = &e_queried;
entries[1] = &e_extra;
report = (xce_job_state_report_t){
.n_total = 2, .pos = 0, .count = 2, .entries = entries
};
lsn_before = twr_current_lsn();
rc = resync_apply_report(&g_twr, w, token, &report);
TEST_ASSERT_EQUAL_INT(0, rc);
lsn_after = twr_current_lsn();
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_extra->job.state,
"an already-non-terminal extra must be left exactly as-is");
TEST_ASSERT_NOT_NULL_MESSAGE(j_extra->job.allocs,
"record_job_reconcile_assign_fact must NOT have been called -- its "
"own contract always sets allocs to NULL, so a surviving "
"allocation proves the no-op branch fired instead");
TEST_ASSERT_EQUAL_UINT64_MESSAGE((uint64_t)seeded_start_time,
(uint64_t)j_extra->job.start_time,
"record_job_reconcile_assign_fact must NOT have been called -- it "
"unconditionally stamps start_time = time(NULL) (bug-1697), so a "
"start_time still equal to the seeded sentinel (not \"now\") "
"proves the no-op branch fired, catching what a bare state/allocs "
"check alone cannot (round-2 review finding)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before, lsn_after,
"neither entry in this report may append a WAL fact: the queried "
"running entry never does, and the already-tracked unqueried "
"extra must not durably re-append a WAL_JOB_DISPATCH fact for a "
"job that was never actually lost/unassigned");
TEST_ASSERT_NULL_MESSAGE(w->resync,
"the round's only queried job (j_queried) was answered -- it must "
"resolve regardless of the unqueried extra");
twr_teardown();
printf("test_apply_report_unqueried_extra_already_tracked_is_noop: "
"PASS\n");
}
/* bug-1747 fix (Req 4.2): a finished-success reconciliation entry's
* WORKER-reported finish_time_ms must end up as the job's recorded
* finish_time after a real restart replay -- NOT reconciliation-processing
* wall-clock time. Mirrors test_reconcile_assign_survives_restart_replay's
* own Phase 1 (real cluster + disk persist, drive the real function under
* test, flush) / Phase 2 (fresh persist_t + persist_recover(), assert on
* the recovered job_t) shape, and test_wal_job_finish_fact.c's own
* test_wjf_real_on_job_finish_survives_restart precedent for proving a
* finish_time round-trip through a real WAL append + replay, not a
* hand-built payload. Uses its own local cluster_t/persist_t/tmpdir (Phase
* 1's disk state must survive past Phase 1's own teardown) -- only
* g_twr.workermgr (via twr_make_test_worker) is shared with the rest of
* this file's fixture, exactly as test_reconcile_assign_survives_restart_
* replay already establishes is safe (resync_apply_report's own locking
* only ever touches `w` and the `c` passed to it, never workermgr). */
static void
test_apply_report_finished_success_uses_worker_reported_finish_time(void)
{
char tmp[256];
uint64_t worker_reported_finish_time_s;
strcpy(tmp, "/tmp/xce_twr24_finishtime_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
twr_setup();
/* A finish time deliberately, clearly DIFFERENT from "now" -- one full
* day in the past -- so a reconciliation-processing-time bug
* (time(NULL)) cannot accidentally produce a matching value. */
worker_reported_finish_time_s = (uint64_t)time(NULL) - 86400;
/* Phase 1: real cluster + real disk persist. Seed a durable
* WAL_JOB_SUBMIT fact directly (so replay can reconstruct the job --
* this job was never dispatched through any controller in this log)
* and the in-memory job_t the same way (bypassing dispatch/replay),
* already RUN and assigned to the reconciling worker's host, then
* drive a real resync_apply_report() call carrying the worker-reported
* finish_time_ms. */
{
cluster_t c;
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
persist_config_t cfg;
job_t* j;
worker_t* w;
int rc;
uint64_t token;
xce_job_state_entry_t e;
xce_job_state_entry_t* entries[1];
xce_job_state_report_t report;
memset(&c, 0, sizeof(c));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));
c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(c.jobmgr);
cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
persist_set_workdir(&c.persist, tmp, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));
{
job_submit_data_t wal_submit;
xce_job_t wal_jdef;
uint8_t* buf = NULL;
size_t sz = 0;
memset(&wal_submit, 0, sizeof(wal_submit));
wal_submit.queue = (char*)"q";
wal_submit.user = (char*)"tester";
memset(&wal_jdef, 0, sizeof(wal_jdef));
wal_jdef.jobid = TWR_JOBID_REPORT_FINISHTIME_REPLAY;
wal_jdef.state = JOB_STAT_RUN;
wal_jdef.submit = &wal_submit;
TEST_ASSERT_EQUAL_INT(0, twr_encode_job(&wal_jdef, &buf, &sz));
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
(uint32_t)sz));
free(buf);
}
{
job_submit_data_t submit;
xce_job_t jdef;
memset(&submit, 0, sizeof(submit));
submit.queue = (char*)"q";
submit.user = (char*)"tester";
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = TWR_JOBID_REPORT_FINISHTIME_REPLAY;
jdef.state = JOB_STAT_RUN;
jdef.submit = &submit;
j = makeJob(c.jobmgr, &jdef);
TEST_ASSERT_NOT_NULL(j);
}
j->job.allocs = twr_make_test_alloc_for_host(
TWR_JOBID_REPORT_FINISHTIME_REPLAY, "resync-finishtime-host");
w = twr_make_test_worker("resync-finishtime-host");
TEST_ASSERT_NOT_NULL(w);
rc = resync_arm_round(&c, w);
TEST_ASSERT_EQUAL_INT(1, rc);
token = w->resync->round_token;
e = (xce_job_state_entry_t){
.jobid = TWR_JOBID_REPORT_FINISHTIME_REPLAY,
.status = JobStateStatus_finishedOk,
.exit_code = 0,
.finish_time_ms = worker_reported_finish_time_s * 1000ULL,
.queried = 1
};
entries[0] = &e;
report = (xce_job_state_report_t){
.n_total = 1, .pos = 0, .count = 1, .entries = entries
};
rc = resync_apply_report(&c, w, token, &report);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_NULL_MESSAGE(w->resync,
"the round's only queried job was answered -- it must resolve");
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));
persist_stop(&c.persist);
persist_fini(&c.persist);
dereference(c.jobmgr);
xce_hook_fini(&c.hooksys);
}
/* Phase 2: fresh persist_t + fresh cluster_t (jobmgr/hooksys only),
* recover from disk -- simulates a controller restart. */
{
persist_t* p2 = twr_make_fresh_persist(tmp);
cluster_t c2;
job_t* j2 = NULL;
int rc;
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
TEST_ASSERT_NOT_NULL(p2);
memset(&c2, 0, sizeof(c2));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(c2.jobmgr);
p2->cluster = &c2;
rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
lock(c2.jobmgr, RTTI_LOCK_READ);
key_find(c2.jobmgr, (void*)TWR_JOBID_REPORT_FINISHTIME_REPLAY,
(void**)&j2);
unlock(c2.jobmgr);
TEST_ASSERT_NOT_NULL_MESSAGE(j2,
"restart replay must reconstruct the job via WAL_JOB_SUBMIT");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE, j2->job.state,
"the finished-success fact must apply to DONE on replay");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(worker_reported_finish_time_s,
(uint64_t)j2->job.finish_time,
"bug-1747 (Req 4.2): the recovered finish_time must match the "
"WORKER-reported finish_time_ms, not reconciliation-processing "
"wall-clock time");
xce_hook_fini(&c2.hooksys);
if (c2.jobmgr) dereference(c2.jobmgr);
twr_teardown_persist(p2);
}
{
char cmd[512];
snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
if (system(cmd) != 0)
fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
}
twr_teardown();
printf("test_apply_report_finished_success_uses_worker_reported_"
"finish_time: PASS\n");
}
/* Review-caught regression fix (Req 4.8): a job the controller currently
* shows PEND (non-terminal, allocs == NULL, unassigned to any host) --
* exactly design.md's own "Documented scope decision" scenario, e.g. its
* WAL_JOB_DISPATCH fact never reached durable majority commit before a
* controller failover, even though dispatch_a_job_() already sent
* xce_startjob_req() to the worker and the worker really is running it --
* must NOT be left stuck at PEND when reported running as an unqueried
* extra. It must reconcile via record_job_reconcile_assign_fact() to RUN,
* matching the reporting worker's host, with allocs staying NULL (there is
* none to fabricate, Req 4.8's own documented scope decision). Mirrors
* test_reconcile_assign_survives_restart_replay's rigor: real WAL_JOB_
* SUBMIT seed + real resync_apply_report() call + real WAL flush, followed
* by a genuine persist_recover() restart replay -- not just an in-memory
* assertion -- so this proves the fix survives a real controller restart,
* not merely the live eager-mutation path. A second, unrelated queried job
* on the same host keeps the round non-empty (arm_round's own query set
* construction only ever finds jobs it already believes are non-terminal
* AND assigned to the host -- by definition NOT this PEND/unassigned job,
* which is exactly why it only ever arrives as an unqueried extra). */
static void
test_apply_report_unqueried_extra_unassigned_pend_job_gets_assigned(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_twr25_lostdispatch_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
twr_setup();
/* Phase 1: real cluster + real disk persist. */
{
cluster_t c;
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
persist_config_t cfg;
job_t* jq;
job_t* jlost;
worker_t* w;
int rc;
uint64_t token;
xce_job_state_entry_t eq, elost;
xce_job_state_entry_t* entries[2];
xce_job_state_report_t report;
memset(&c, 0, sizeof(c));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));
c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(c.jobmgr);
cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
persist_set_workdir(&c.persist, tmp, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));
/* Durable WAL_JOB_SUBMIT seeds -- jq at RUN (assigned, keeps the
* round non-empty), jlost at PEND (unassigned -- the job whose
* WAL_JOB_DISPATCH fact never durably committed). */
{
job_submit_data_t wal_submit;
xce_job_t wal_jdef;
uint8_t* buf = NULL;
size_t sz = 0;
memset(&wal_submit, 0, sizeof(wal_submit));
wal_submit.queue = (char*)"q";
wal_submit.user = (char*)"tester";
memset(&wal_jdef, 0, sizeof(wal_jdef));
wal_jdef.jobid = TWR_JOBID_REPORT_LOSTDISPATCH_Q;
wal_jdef.state = JOB_STAT_RUN;
wal_jdef.submit = &wal_submit;
TEST_ASSERT_EQUAL_INT(0, twr_encode_job(&wal_jdef, &buf, &sz));
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
(uint32_t)sz));
free(buf);
memset(&wal_jdef, 0, sizeof(wal_jdef));
wal_jdef.jobid = TWR_JOBID_REPORT_LOSTDISPATCH;
wal_jdef.state = JOB_STAT_PEND;
wal_jdef.submit = &wal_submit;
TEST_ASSERT_EQUAL_INT(0, twr_encode_job(&wal_jdef, &buf, &sz));
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
(uint32_t)sz));
free(buf);
}
/* Seed the in-memory job_t's the same way, bypassing dispatch/
* replay. */
{
job_submit_data_t submit;
xce_job_t jdef;
memset(&submit, 0, sizeof(submit));
submit.queue = (char*)"q";
submit.user = (char*)"tester";
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = TWR_JOBID_REPORT_LOSTDISPATCH_Q;
jdef.state = JOB_STAT_RUN;
jdef.submit = &submit;
jq = makeJob(c.jobmgr, &jdef);
TEST_ASSERT_NOT_NULL(jq);
jq->job.allocs = twr_make_test_alloc_for_host(
TWR_JOBID_REPORT_LOSTDISPATCH_Q, "resync-lostdispatch-host");
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = TWR_JOBID_REPORT_LOSTDISPATCH;
jdef.state = JOB_STAT_PEND;
jdef.submit = &submit;
jlost = makeJob(c.jobmgr, &jdef);
TEST_ASSERT_NOT_NULL(jlost);
TEST_ASSERT_NULL_MESSAGE(jlost->job.allocs,
"the lost-dispatch-fact job starts PEND and unassigned -- "
"no allocation was ever durably recorded for it");
}
w = twr_make_test_worker("resync-lostdispatch-host");
TEST_ASSERT_NOT_NULL(w);
rc = resync_arm_round(&c, w);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
"only jq (RUN, assigned) is in the controller's own query set "
"-- jlost (PEND, unassigned) is invisible to arm_round's own "
"scan, exactly why it can only ever arrive as an unqueried "
"extra");
token = w->resync->round_token;
eq = (xce_job_state_entry_t){
.jobid = TWR_JOBID_REPORT_LOSTDISPATCH_Q,
.status = JobStateStatus_running,
.exit_code = 0, .finish_time_ms = 0, .queried = 1
};
elost = (xce_job_state_entry_t){
.jobid = TWR_JOBID_REPORT_LOSTDISPATCH,
.status = JobStateStatus_running,
.exit_code = 0, .finish_time_ms = 0, .queried = 0
};
entries[0] = &eq;
entries[1] = &elost;
report = (xce_job_state_report_t){
.n_total = 2, .pos = 0, .count = 2, .entries = entries
};
rc = resync_apply_report(&c, w, token, &report);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Eager pattern (record_job_reconcile_assign_fact) -- observable
* synchronously, before any commit round-trip. */
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, jlost->job.state,
"a PEND, unassigned job reported running as an unqueried extra "
"must reconcile to RUN via record_job_reconcile_assign_fact -- "
"not be left stuck at PEND");
TEST_ASSERT_NULL_MESSAGE(jlost->job.allocs,
"Req 4.8: no allocation is fabricated for a reconciliation-"
"assigned job");
TEST_ASSERT_NULL_MESSAGE(w->resync,
"the round's only QUERIED job (jq) was answered -- it must "
"resolve regardless of the unqueried extra");
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));
persist_stop(&c.persist);
persist_fini(&c.persist);
dereference(c.jobmgr);
xce_hook_fini(&c.hooksys);
}
/* Phase 2: fresh persist_t + fresh cluster_t, recover from disk --
* proves the fix survives a real controller restart, not just the live
* eager-mutation path. */
{
persist_t* p2 = twr_make_fresh_persist(tmp);
cluster_t c2;
job_t* j2 = NULL;
int rc;
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
TEST_ASSERT_NOT_NULL(p2);
memset(&c2, 0, sizeof(c2));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(c2.jobmgr);
p2->cluster = &c2;
rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
lock(c2.jobmgr, RTTI_LOCK_READ);
key_find(c2.jobmgr, (void*)TWR_JOBID_REPORT_LOSTDISPATCH,
(void**)&j2);
unlock(c2.jobmgr);
TEST_ASSERT_NOT_NULL_MESSAGE(j2,
"restart replay must reconstruct the job via WAL_JOB_SUBMIT");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j2->job.state,
"restart replay must recover the job as RUNNING from the "
"WAL_JOB_DISPATCH fact appended by record_job_reconcile_"
"assign_fact()'s real call -- the fix must survive a real "
"controller restart, not just the live eager-mutation path");
TEST_ASSERT_NULL_MESSAGE(j2->job.allocs,
"Req 4.8: replay must NOT fabricate a resource allocation for "
"a reconciliation-assigned job");
xce_hook_fini(&c2.hooksys);
if (c2.jobmgr) dereference(c2.jobmgr);
twr_teardown_persist(p2);
}
{
char cmd[512];
snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
if (system(cmd) != 0)
fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
}
twr_teardown();
printf("test_apply_report_unqueried_extra_unassigned_pend_job_gets_"
"assigned: PASS\n");
}
/* ══════════════════════════════════════════════════════════════════════
* Task 6.2 (Req 8.1, 8.3): resync_handle_report_page() -- multi-page wire
* reassembly of an incoming OPCode_jobStateReport, ahead of resync_apply_
* report() (task 3.2, proven above). Every test here proves the exact
* gap this task's own brief called out: not just that the correct END
* STATE results, but that resync_apply_report() is invoked EXACTLY ONCE
* per report regardless of page count (resync_apply_report_call_count_
* for_test(), resync.h) -- job.c's own idempotent fact guards would make
* even a buggy once-per-page implementation LOOK correct on a happy-path
* report, exactly the class of bug task 4.3's own round-1 review missed
* with a purely behavioral test (tasks.md Implementation Notes).
* ══════════════════════════════════════════════════════════════════════ */
#define TWR_JOBID_MPAGE_FINOK 81201ULL
#define TWR_JOBID_MPAGE_UNREC 81202ULL
#define TWR_JOBID_MPAGE_RUNNING 81203ULL
#define TWR_JOBID_MPAGE_STALE 81210ULL
#define TWR_JOBID_MPAGE_BAD 81220ULL
#define TWR_JOBID_MPAGE_BAD2 81221ULL
/* Baseline: a single-page report (pos=0, count==n_total) delivered through
* resync_handle_report_page() -- not resync_apply_report() directly -- is
* reassembled trivially (one page IS the whole report) and applied exactly
* once. Establishes the expected end state the multi-page test below must
* independently reproduce via genuine reassembly. */
static void test_handle_report_page_single_page_baseline(void)
{
job_t* j_unrec;
worker_t* w;
int rc;
uint64_t token;
int calls_before, calls_after;
xce_job_state_entry_t e_unrec;
xce_job_state_entry_t* entries[1];
xce_job_state_report_t page;
twr_setup();
j_unrec = twr_seed_job(TWR_JOBID_MPAGE_UNREC, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j_unrec);
j_unrec->job.allocs
= twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_UNREC, "mpage-host-1");
w = twr_make_test_worker("mpage-host-1");
TEST_ASSERT_NOT_NULL(w);
rc = resync_arm_round(&g_twr, w);
TEST_ASSERT_EQUAL_INT(1, rc);
token = w->resync->round_token;
e_unrec = (xce_job_state_entry_t){
.jobid = TWR_JOBID_MPAGE_UNREC, .status = JobStateStatus_unrecognized,
.exit_code = 0, .finish_time_ms = 0, .queried = 1
};
entries[0] = &e_unrec;
page = (xce_job_state_report_t){
.n_total = 1, .pos = 0, .count = 1, .entries = entries
};
resync_apply_report_call_count_reset_for_test();
calls_before = resync_apply_report_call_count_for_test();
rc = resync_handle_report_page(&g_twr, w, &page);
calls_after = resync_apply_report_call_count_for_test();
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
"a single page whose count == n_total completes the report "
"immediately");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, calls_after - calls_before,
"resync_apply_report must be invoked exactly once");
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, j_unrec->job.state);
TEST_ASSERT_NULL(j_unrec->job.allocs);
TEST_ASSERT_NULL_MESSAGE(w->resync,
"the round must resolve once its only queried job is answered");
(void)token;
twr_teardown();
printf("test_handle_report_page_single_page_baseline: PASS\n");
}
/* THE core proof (task 6.2's own Observable Completion Criterion): the SAME
* report, split across two wire pages, must (a) leave the job table
* COMPLETELY untouched after the first (non-final) page -- no partial
* application -- (b) converge to EXACTLY the same end state the single-page
* baseline above reached, once the final page arrives, and (c) invoke
* resync_apply_report() EXACTLY ONCE for the whole report, not once per
* page (the structural proof a behavioral-only assertion cannot provide --
* see this section's own header comment). */
static void test_handle_report_page_multi_page_reassembles_and_applies_once(
void)
{
job_t* j_finok;
job_t* j_unrec;
job_t* j_running;
worker_t* w;
int rc;
int calls_before, calls_mid, calls_after;
xce_job_state_entry_t e_finok, e_unrec, e_running;
xce_job_state_entry_t* entries_p1[2];
xce_job_state_entry_t* entries_p2[1];
xce_job_state_report_t page1, page2;
twr_setup();
j_finok = twr_seed_job(TWR_JOBID_MPAGE_FINOK, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j_finok);
j_finok->job.allocs
= twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_FINOK, "mpage-host-2");
j_unrec = twr_seed_job(TWR_JOBID_MPAGE_UNREC, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j_unrec);
j_unrec->job.allocs
= twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_UNREC, "mpage-host-2");
j_running = twr_seed_job(TWR_JOBID_MPAGE_RUNNING, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j_running);
j_running->job.allocs = twr_make_test_alloc_for_host(
TWR_JOBID_MPAGE_RUNNING, "mpage-host-2");
w = twr_make_test_worker("mpage-host-2");
TEST_ASSERT_NOT_NULL(w);
rc = resync_arm_round(&g_twr, w);
TEST_ASSERT_EQUAL_INT_MESSAGE(3, rc,
"all 3 non-terminal jobs on this host must be armed");
e_finok = (xce_job_state_entry_t){
.jobid = TWR_JOBID_MPAGE_FINOK, .status = JobStateStatus_finishedOk,
.exit_code = 0, .finish_time_ms = 4242, .queried = 1
};
e_unrec = (xce_job_state_entry_t){
.jobid = TWR_JOBID_MPAGE_UNREC, .status = JobStateStatus_unrecognized,
.exit_code = 0, .finish_time_ms = 0, .queried = 1
};
e_running = (xce_job_state_entry_t){
.jobid = TWR_JOBID_MPAGE_RUNNING, .status = JobStateStatus_running,
.exit_code = 0, .finish_time_ms = 0, .queried = 1
};
/* Page 1 of 2: entries [0..1] (finok, unrec). Page 2 of 2: entry [2]
* (running). Mirrors xce_job_state_report_req()'s own {pos,count,
* n_total} pagination shape (request.c) -- this is exactly what a real
* multi-page OPCode_jobStateReport looks like on the wire. */
entries_p1[0] = &e_finok;
entries_p1[1] = &e_unrec;
page1 = (xce_job_state_report_t){
.n_total = 3, .pos = 0, .count = 2, .entries = entries_p1
};
entries_p2[0] = &e_running;
page2 = (xce_job_state_report_t){
.n_total = 3, .pos = 2, .count = 1, .entries = entries_p2
};
resync_apply_report_call_count_reset_for_test();
calls_before = resync_apply_report_call_count_for_test();
rc = resync_handle_report_page(&g_twr, w, &page1);
calls_mid = resync_apply_report_call_count_for_test();
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"page 1 of 2 alone must NOT complete the report");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, calls_mid - calls_before,
"resync_apply_report must NOT be called after a non-final page -- "
"the structural 'not once per page' proof");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_finok->job.state,
"no partial application: job A must be untouched after page 1");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_unrec->job.state,
"no partial application: job B must be untouched after page 1 -- "
"still RUN, NOT yet reconciled to PEND");
TEST_ASSERT_NOT_NULL_MESSAGE(j_unrec->job.allocs,
"no partial application: job B's allocation must not be released "
"until the report is fully assembled and applied");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_running->job.state,
"job C (only referenced in page 2, not yet arrived) is untouched");
TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
"the round must still be outstanding after only a partial report");
rc = resync_handle_report_page(&g_twr, w, &page2);
calls_after = resync_apply_report_call_count_for_test();
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
"page 2 of 2 completes the report");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, calls_after - calls_mid,
"resync_apply_report must be invoked EXACTLY ONCE, on the final "
"page, for the whole 3-entry report -- not once per page");
/* Equivalence (task 6.2's own Observable Completion Criterion): the SAME
* end state test_handle_report_page_single_page_baseline reached for an
* equivalent single-page report. */
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_finok->job.state,
"finished-success stays in the DEFERRED record_job_finish_fact_at "
"pattern -- state mutates later via apply_committed_entry, same as "
"an equivalent single-page report (test_apply_report_all_five_"
"outcomes_updates_job_table proves this deferred path directly)");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j_unrec->job.state,
"job B reconciles to PEND -- identical to the single-page baseline");
TEST_ASSERT_NULL_MESSAGE(j_unrec->job.allocs,
"job B's allocation is released -- identical to the single-page "
"baseline");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_running->job.state,
"job C (queried running) is left exactly as-is -- no fact "
"recorded, identical to the single-page baseline");
TEST_ASSERT_NULL_MESSAGE(w->resync,
"once every queried job across BOTH pages has a reconciled "
"outcome, the round must resolve -- identical to the single-page "
"baseline");
TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
"XCE_WORKER_RECONCILING must clear once the round resolves");
twr_teardown();
printf("test_handle_report_page_multi_page_reassembles_and_applies_once:"
" PASS\n");
}
/* Req 6.1/6.3 at the reassembly layer: a report that STARTS arriving under
* one round but finishes assembling only after that round has been
* superseded by a fresh resync_arm_round() call (simulating a disconnect-
* then-immediate-rejoin racing the tail of a still-in-flight report) must
* be discarded in its entirety -- not matched against whichever round
* happens to be current once the LAST page arrives. Proves the round_token
* CAPTURE point (page 1, resync_report_accum_t.round_token) is what makes
* this correct, per resync.h's own doc comment on why that capture is not
* an arbitrary choice. */
static void test_handle_report_page_stale_round_discarded_when_superseded(
void)
{
job_t* j;
worker_t* w;
int rc;
int calls_before, calls_after;
xce_job_state_entry_t e1, e2;
xce_job_state_entry_t* entries_p1[1];
xce_job_state_entry_t* entries_p2[1];
xce_job_state_report_t page1, page2;
twr_setup();
j = twr_seed_job(TWR_JOBID_MPAGE_STALE, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs
= twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_STALE, "mpage-host-3");
w = twr_make_test_worker("mpage-host-3");
TEST_ASSERT_NOT_NULL(w);
rc = resync_arm_round(&g_twr, w);
TEST_ASSERT_EQUAL_INT(1, rc);
e1 = (xce_job_state_entry_t){
.jobid = TWR_JOBID_MPAGE_STALE, .status = JobStateStatus_finishedOk,
.exit_code = 0, .finish_time_ms = 1, .queried = 1
};
entries_p1[0] = &e1;
page1 = (xce_job_state_report_t){
.n_total = 2, .pos = 0, .count = 1, .entries = entries_p1
};
rc = resync_handle_report_page(&g_twr, w, &page1);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Simulated disconnect-then-rejoin racing the tail of the in-flight
* report above: a fresh round supersedes the one page1 started under,
* BEFORE page 2 ever arrives. */
rc = resync_arm_round(&g_twr, w);
TEST_ASSERT_EQUAL_INT(1, rc);
TEST_ASSERT_NOT_NULL(w->resync);
/* A second entry, unrelated jobid, just to complete the (now-stale)
* report's page count -- its content is immaterial since the whole
* report must be discarded regardless. */
e2 = (xce_job_state_entry_t){
.jobid = TWR_JOBID_MPAGE_STALE, .status = JobStateStatus_finishedFail,
.exit_code = 0, .finish_time_ms = 2, .queried = 1
};
entries_p2[0] = &e2;
page2 = (xce_job_state_report_t){
.n_total = 2, .pos = 1, .count = 1, .entries = entries_p2
};
resync_apply_report_call_count_reset_for_test();
calls_before = resync_apply_report_call_count_for_test();
rc = resync_handle_report_page(&g_twr, w, &page2);
calls_after = resync_apply_report_call_count_for_test();
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
"the report completes (page 2 arrived) and IS handed to "
"resync_apply_report -- which then discards it internally as "
"stale, per its own Req 6.1/6.3 contract");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, calls_after - calls_before,
"resync_apply_report is still called exactly once -- discarding "
"happens INSIDE it (round-token mismatch), not by skipping the "
"call");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j->job.state,
"the stale report must never touch the job table");
TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
"the CURRENT (superseding) round must be left outstanding, "
"untouched by the stale report");
TEST_ASSERT_EQUAL_INT(1, w->resync->n_queried_jobids);
resync_resolve_round(&g_twr, w);
twr_teardown();
printf("test_handle_report_page_stale_round_discarded_when_superseded:"
" PASS\n");
}
/* Robustness: a page whose [pos, pos+count) range falls outside the
* report's own declared n_total is rejected (-EINVAL) and discards
* whatever partial accumulation was in progress -- proven by immediately
* starting a FRESH, valid report afterward and confirming it applies
* correctly, i.e. the corrupt attempt left no residue behind to corrupt a
* later, legitimate report on the same worker. */
static void test_handle_report_page_malformed_bounds_rejected_and_recovers(
void)
{
job_t* j;
worker_t* w;
int rc;
xce_job_state_entry_t e_bad, e_good;
xce_job_state_entry_t* entries_bad[1];
xce_job_state_entry_t* entries_good[1];
xce_job_state_report_t page_bad, page_good;
twr_setup();
j = twr_seed_job(TWR_JOBID_MPAGE_BAD, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs
= twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_BAD, "mpage-host-4");
w = twr_make_test_worker("mpage-host-4");
TEST_ASSERT_NOT_NULL(w);
rc = resync_arm_round(&g_twr, w);
TEST_ASSERT_EQUAL_INT(1, rc);
/* pos(1) + count(1) == 2 > n_total(1): out of bounds. */
e_bad = (xce_job_state_entry_t){
.jobid = TWR_JOBID_MPAGE_BAD2, .status = JobStateStatus_running,
.exit_code = 0, .finish_time_ms = 0, .queried = 0
};
entries_bad[0] = &e_bad;
page_bad = (xce_job_state_report_t){
.n_total = 1, .pos = 1, .count = 1, .entries = entries_bad
};
rc = resync_handle_report_page(&g_twr, w, &page_bad);
TEST_ASSERT_EQUAL_INT_MESSAGE(-EINVAL, rc,
"an out-of-bounds page must be rejected");
/* A fresh, valid, single-page report for the SAME worker right after --
* proves the malformed attempt above left no corrupted accumulator
* state behind. */
e_good = (xce_job_state_entry_t){
.jobid = TWR_JOBID_MPAGE_BAD, .status = JobStateStatus_unrecognized,
.exit_code = 0, .finish_time_ms = 0, .queried = 1
};
entries_good[0] = &e_good;
page_good = (xce_job_state_report_t){
.n_total = 1, .pos = 0, .count = 1, .entries = entries_good
};
rc = resync_handle_report_page(&g_twr, w, &page_good);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
"a subsequent valid report must apply normally");
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, j->job.state);
TEST_ASSERT_NULL(j->job.allocs);
TEST_ASSERT_NULL(w->resync);
twr_teardown();
printf("test_handle_report_page_malformed_bounds_rejected_and_recovers:"
" PASS\n");
}
/* ══════════════════════════════════════════════════════════════════════
* Task 7.1 (Req 1.1-1.3, 2.1-2.2, 3.1-3.3, 3.6, 4.1-4.8, 7.1-7.3): full
* rejoin-reconciliation round trip against a live controller+worker pair.
*
* design.md's own Testing Strategy files this under "Integration Tests" as
* "against a live controller+worker pair, mirroring test_worker_join.c's
* fixture conventions (setup_test_cluster/teardown_test_cluster, direct
* calls, one test per behavior)" -- NOT separate OS processes or real TCP
* daemons. test_jobstatereport_dispatch.c's own header comment (task 6.2)
* additionally confirms a genuine wire dispatch through myreq_handler /
* handle_jobstatereport() cannot resolve a REAL worker_t in THIS binary at
* all: get_ep_worker() (misc.c) is not linked into controller_unit_test --
* a fixed test double defined in test_commit_barrier_reply.c stands in for
* it, always returning NULL/a fixture-controlled stub, for every test file
* in this binary, this one included. So this test drives the exact same
* REAL production entry points test_worker_join.c's own task 6.1 tests use
* directly -- rejoin_worker() (the real join path) to arm the round and
* worker_send_resync_query() to send the real wire query -- then feeds a
* REAL wire-encoded-then-decoded OPCode_jobStateReport (via task 1.4's own
* xce_job_state_report_req()/xce_decode() codec) into resync_handle_report_
* page() (task 6.2's own dispatch/reassembly entry point -- never resync_
* apply_report() directly), exactly mirroring how a real worker's answer
* would arrive and be reassembled/dispatched, without needing get_ep_
* worker()'s (unavailable) wire-to-worker_t resolution.
*
* Proves COMPOSITION, not per-outcome correctness -- each individual outcome
* (still-running no-op, finished-success, finished-failure, unrecognized)
* is already exhaustively unit-tested in isolation by tasks 2.1/2.2/3.2 (see
* test_apply_report_all_five_outcomes_updates_job_table above, and task
* 6.1's own test_worker_join.c tests for arm+query-send in isolation). This
* test's own value-add is proving all of join -> arm -> real wire query ->
* real wire report -> dispatch/reassembly -> apply -> scheduler-notify
* compose correctly TOGETHER, for a MIX of four different job outcomes, in
* ONE round, in ONE test run -- and that the scheduler notify path is
* reached for every actual state change produced, across BOTH of this
* feature's two distinct notify patterns: the EAGER path (unrecognized ->
* pending, task 2.1's pattern -- notified synchronously within the same
* call, asserted in Phase 1 below) and the DEFERRED path (finished-success/
* failure, task 3.2's pattern -- notified only once the durable fact is
* later applied). This bare-persist RETAIN-mode fixture has no live commit-
* engine apply pump to do that second half automatically, so this test
* proves it the same way test_apply_report_finished_success_uses_worker_
* reported_finish_time and test_reconcile_assign_survives_restart_replay
* already establish as this file's own precedent for that exact situation:
* a genuine two-phase restart-replay (Phase 1: real cluster + disk persist,
* drive the real chain, flush, close; Phase 2: fresh persist_t + fresh
* cluster_t, persist_recover() -- the SAME apply_committed_entry() seam a
* live controller's commit-apply pump would invoke). */
#define TWR_JOBID_RT_RUNNING 81301ULL
#define TWR_JOBID_RT_FINOK 81302ULL
#define TWR_JOBID_RT_FINFAIL 81303ULL
#define TWR_JOBID_RT_UNREC 81304ULL
#define TWR_RT_N_RES 9
/* Mirrors test_worker_join.c's own new_test_endpoint/pop_and_decode/assert_
* queue_empty helpers exactly -- real (heap) endpoint_t via endpoint_get()
* (bug-378: never a hand-rolled/zeroed stack stand-in for anything actually
* sent/received over), for genuine wire capture of both the query this test
* sends TO the worker and the report it simulates receiving FROM one. */
static endpoint_t* twr_rt_new_test_endpoint(void)
{
endpoint_t* ep = endpoint_get();
TEST_ASSERT_NOT_NULL(ep);
return ep;
}
static void twr_rt_pop_and_decode(endpoint_t* ep, xce_request_t* out)
{
data_item_t* it = endpoint_deq(ep);
TEST_ASSERT_NOT_NULL(it);
io_packet_t* pkt = ITPTR_TO_PACKET(it);
memset(out, 0, sizeof(*out));
TEST_ASSERT_EQUAL_INT(0, xce_decode(pkt, out)); /* consumes/frees pkt */
}
static void test_full_rejoin_reconciliation_round_trip(void)
{
char tmp[256];
uint64_t worker_reported_finish_time_s;
int32_t worker_reported_exit_code = 42;
strcpy(tmp, "/tmp/xce_twr71_roundtrip_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Deliberately far from "now" so a reconciliation-processing-time bug
* (time(NULL)) could never accidentally produce a matching value --
* same defensive idiom test_apply_report_finished_success_uses_worker_
* reported_finish_time already establishes. */
worker_reported_finish_time_s = (uint64_t)time(NULL) - 3600;
/* Phase 1: real cluster (workermgr + resmgr + jobmgr + hooksys) + real
* disk persist -- drives the REAL join/query-send/report-apply chain.
* Own local cluster_t (not the shared g_twr fixture), mirroring every
* other restart-replay test in this file, since Phase 1's disk state
* must survive past Phase 1's own teardown into Phase 2 below. */
{
cluster_t c;
kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING,
.nslots = 64, .delta = 16 };
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
persist_config_t cfg;
worker_t* w;
job_t* jobs[4];
uint64_t jobids[4] = { TWR_JOBID_RT_RUNNING,
TWR_JOBID_RT_FINOK,
TWR_JOBID_RT_FINFAIL,
TWR_JOBID_RT_UNREC };
hostattrs_t attrs, new_attrs;
xce_hostconfig_t hconfig;
endpoint_t stack_ep;
endpoint_t* ep;
xce_request_t decoded_query, decoded_report;
int saw[4] = { 0, 0, 0, 0 };
int i, rc;
uint64_t lsn_before, lsn_after;
static const char* res_names[TWR_RT_N_RES] = {
"type", "model", "cpuf", "maxcpus", "maxmem",
"maxswap", "maxtmp", "ndisks", "slots"
};
static const int res_valtypes[TWR_RT_N_RES] = {
XCE_VAL_STRING, XCE_VAL_STRING,
XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC,
XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC,
XCE_VAL_NUMERIC
};
if (makeString((char*)"twr71-probe") == NULL)
xstring_table_init(64);
if (!g_twr_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
g_twr_jobobj_ready = 1;
}
memset(&c, 0, sizeof(c));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));
c.workermgr = makeInstance(&xkvmgr_type, &str_params);
TEST_ASSERT_NOT_NULL(c.workermgr);
c.resmgr = makeInstance(&xkvmgr_type, &str_params);
TEST_ASSERT_NOT_NULL(c.resmgr);
c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(c.jobmgr);
for (i = 0; i < TWR_RT_N_RES; i++) {
xce_resource_t rdef;
memset(&rdef, 0, sizeof(rdef));
rdef.name = (char*)res_names[i];
rdef.valtype = res_valtypes[i];
rdef.flags = 0;
rdef.idx = -1;
TEST_ASSERT_NOT_NULL(makeResource(c.resmgr, &rdef));
}
c.myhost_info.name = (char*)"twr71-controller";
c.myhost_info.key = (char*)"twr71-key";
c.myhost = &c.myhost_info;
cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
persist_set_workdir(&c.persist, tmp, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));
/* Seed a durable WAL_JOB_SUBMIT fact + a matching in-memory job_t
* for all 4 jobs -- all RUN, all assigned to the SAME worker host,
* exactly what the controller believes right before this rejoin
* (bypassing dispatch/replay, mirroring every sibling restart-
* replay test in this file). All 4 are non-terminal, so all 4 are
* expected in the round's query set (Req 2.1) -- "unrecognized" is
* not a DIFFERENT starting state from the controller's own point of
* view, only a different worker ANSWER. */
for (i = 0; i < 4; i++) {
job_submit_data_t wal_submit;
xce_job_t wal_jdef;
uint8_t* buf = NULL;
size_t sz = 0;
job_submit_data_t submit;
xce_job_t jdef;
memset(&wal_submit, 0, sizeof(wal_submit));
wal_submit.queue = (char*)"q";
wal_submit.user = (char*)"tester";
memset(&wal_jdef, 0, sizeof(wal_jdef));
wal_jdef.jobid = jobids[i];
wal_jdef.state = JOB_STAT_RUN;
wal_jdef.submit = &wal_submit;
TEST_ASSERT_EQUAL_INT(0, twr_encode_job(&wal_jdef, &buf, &sz));
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
(uint32_t)sz));
free(buf);
memset(&submit, 0, sizeof(submit));
submit.queue = (char*)"q";
submit.user = (char*)"tester";
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = jobids[i];
jdef.state = JOB_STAT_RUN;
jdef.submit = &submit;
jobs[i] = makeJob(c.jobmgr, &jdef);
TEST_ASSERT_NOT_NULL(jobs[i]);
jobs[i]->job.allocs = twr_make_test_alloc_for_host(
jobids[i], "resync-roundtrip-host");
TEST_ASSERT_NOT_NULL(jobs[i]->job.allocs);
}
/* ── Join: rejoin_worker (the REAL join entry point) arms the round,
* worker_send_resync_query sends the REAL wire query (Req 1.1-1.3,
* 2.1, 2.2). */
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
{
xce_worker_t host;
memset(&host, 0, sizeof(host));
host.name = (char*)"resync-roundtrip-host";
host.attrs = &attrs;
w = makeWorker(c.workermgr, &host);
}
TEST_ASSERT_NOT_NULL(w);
memset(&new_attrs, 0, sizeof(new_attrs));
new_attrs.hosttype = makeString("LINUX");
new_attrs.hostmodel = makeString("x86");
new_attrs.cpufactor = 1.0f;
new_attrs.maxcpus = 4;
new_attrs.maxmem = 16;
new_attrs.maxswap = 32;
new_attrs.maxtmp = 64;
new_attrs.ndisks = 1;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
memset(&stack_ep, 0, sizeof(stack_ep));
rc = rejoin_worker(&c, w, &stack_ep, &new_attrs, &hconfig);
TEST_ASSERT_GREATER_OR_EQUAL(0, rc);
TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
"a rejoin with 4 known non-terminal jobs on this host must arm "
"a reconciliation round (Req 1.1)");
TEST_ASSERT_EQUAL_INT_MESSAGE(4, w->resync->n_queried_jobids,
"the round's query set must match exactly the host's 4 "
"non-terminal jobs (Req 2.1)");
TEST_ASSERT_TRUE(IS_WORKER_RECONCILING(w->host.status));
ep = twr_rt_new_test_endpoint();
TEST_ASSERT_EQUAL_INT(0, worker_send_resync_query(&c, w, ep));
twr_rt_pop_and_decode(ep, &decoded_query);
TEST_ASSERT_EQUAL_INT(OPCode_jobStateQuery, decoded_query.opcode);
TEST_ASSERT_EQUAL_INT(Request_body_jobstatequerymsg,
decoded_query.body_which);
TEST_ASSERT_NOT_NULL(decoded_query.body.jobstatequerymsg);
TEST_ASSERT_EQUAL_INT_MESSAGE(4,
decoded_query.body.jobstatequerymsg->n_total,
"the REAL wire query (Req 2.1, 2.2) must list exactly the 4 "
"armed jobids");
for (i = 0; i < decoded_query.body.jobstatequerymsg->count; i++) {
uint64_t id = decoded_query.body.jobstatequerymsg->jobids[i];
if (id == TWR_JOBID_RT_RUNNING) saw[0] = 1;
if (id == TWR_JOBID_RT_FINOK) saw[1] = 1;
if (id == TWR_JOBID_RT_FINFAIL) saw[2] = 1;
if (id == TWR_JOBID_RT_UNREC) saw[3] = 1;
}
TEST_ASSERT_TRUE_MESSAGE(saw[0] && saw[1] && saw[2] && saw[3],
"the wire query must cover all 4 jobs by unique ID (Req 2.2)");
xce_free_request(&decoded_query);
/* ── Worker's answer: a REAL wire-encoded-then-decoded OPCode_
* jobStateReport (task 1.4's own xce_job_state_report_req()/xce_
* decode() codec), covering all 4 outcomes in one page, fed into
* resync_handle_report_page() -- task 6.2's own dispatch/
* reassembly entry point, never resync_apply_report() directly
* (Req 3.1-3.3, 3.6, 8.1-8.3). */
{
xce_job_state_entry_t e_running, e_finok, e_finfail, e_unrec;
xce_job_state_entry_t* entries[4];
xce_job_state_report_t report;
e_running = (xce_job_state_entry_t){
.jobid = TWR_JOBID_RT_RUNNING,
.status = JobStateStatus_running,
.exit_code = 0, .finish_time_ms = 0, .queried = 1
};
e_finok = (xce_job_state_entry_t){
.jobid = TWR_JOBID_RT_FINOK,
.status = JobStateStatus_finishedOk,
.exit_code = 0,
.finish_time_ms = worker_reported_finish_time_s * 1000ULL,
.queried = 1
};
e_finfail = (xce_job_state_entry_t){
.jobid = TWR_JOBID_RT_FINFAIL,
.status = JobStateStatus_finishedFail,
.exit_code = worker_reported_exit_code,
.finish_time_ms = 0, .queried = 1
};
e_unrec = (xce_job_state_entry_t){
.jobid = TWR_JOBID_RT_UNREC,
.status = JobStateStatus_unrecognized,
.exit_code = 0, .finish_time_ms = 0, .queried = 1
};
entries[0] = &e_running;
entries[1] = &e_finok;
entries[2] = &e_finfail;
entries[3] = &e_unrec;
report = (xce_job_state_report_t){
.n_total = 4, .pos = 0, .count = 4, .entries = entries
};
TEST_ASSERT_EQUAL_INT(0,
xce_job_state_report_req(ep, c.myhost, &report, NULL, NULL,
0));
}
twr_rt_pop_and_decode(ep, &decoded_report);
TEST_ASSERT_EQUAL_INT(OPCode_jobStateReport, decoded_report.opcode);
TEST_ASSERT_EQUAL_INT(Request_body_jobstatereportmsg,
decoded_report.body_which);
TEST_ASSERT_NOT_NULL(decoded_report.body.jobstatereportmsg);
g_twr_update_hook_fires = 0;
TEST_ASSERT_EQUAL_INT(0,
xce_add_hook(&c.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook, NULL));
pthread_mutex_lock(&c.persist.lock);
lsn_before = c.persist.current_lsn;
pthread_mutex_unlock(&c.persist.lock);
rc = resync_handle_report_page(&c, w,
decoded_report.body.jobstatereportmsg);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
"a single-page report (count == n_total) must complete the "
"report immediately");
pthread_mutex_lock(&c.persist.lock);
lsn_after = c.persist.current_lsn;
pthread_mutex_unlock(&c.persist.lock);
xce_free_request(&decoded_report);
/* ── End-to-end assertions (Phase 1 -- everything observable before
* any commit round trip, mirroring record_job_reconcile_pending_
* fact's own eager-mutation precedent). */
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, jobs[0]->job.state,
"still-running (queried) must be left exactly as-is -- no fact "
"recorded (Req 4.1)");
TEST_ASSERT_NOT_NULL(jobs[0]->job.allocs);
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, jobs[1]->job.state,
"finished-success stays in the DEFERRED record_job_finish_"
"fact_at pattern -- state mutates later via apply_committed_"
"entry (proven below, Phase 2), not synchronously here, exactly "
"as test_apply_report_all_five_outcomes_updates_job_table's own "
"established precedent documents");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, jobs[2]->job.state,
"finished-failure stays in the DEFERRED on_job_fail/record_job_"
"fail_fact pattern -- same deferred precedent as finished-"
"success");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, jobs[3]->job.state,
"unrecognized reconciles EAGERLY through record_job_reconcile_"
"pending_fact (Req 4.5, 4.6) -- PEND synchronously, before any "
"commit round trip");
TEST_ASSERT_NULL_MESSAGE(jobs[3]->job.allocs,
"bug-1631: reconciled-to-pending must release the allocation "
"synchronously");
TEST_ASSERT_NULL_MESSAGE(w->resync,
"once every one of the 4 queried jobs has a reconciled outcome, "
"the round must resolve (Req 5.1, 5.2)");
TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
"XCE_WORKER_RECONCILING must clear once the round resolves");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_twr_update_hook_fires,
"Req 7.1, 7.2, 7.3: the EAGER unrecognized-to-pending "
"transition must reach the scheduler's/apiserver's notify path "
"SYNCHRONOUSLY, within this same call -- the two DEFERRED "
"outcomes (finished-success/failure) have not applied yet at "
"this point, so they must NOT have notified yet either "
"(confirmed to notify later, in Phase 2 below)");
TEST_ASSERT_TRUE_MESSAGE(lsn_after >= lsn_before + 3,
"at least 3 durable facts must be appended for this report: "
"finished-success (WAL_JOB_FINISH), finished-failure (WAL_JOB_"
"FAIL), and unrecognized (WAL_JOB_UPDATE) -- the still-running "
"entry records none");
xce_disable_hook(&c.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook);
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));
persist_stop(&c.persist);
persist_fini(&c.persist);
if (c.workermgr) dereference(c.workermgr);
if (c.resmgr) dereference(c.resmgr);
if (c.jobmgr) dereference(c.jobmgr);
xce_hook_fini(&c.hooksys);
dereference(new_attrs.hosttype);
dereference(new_attrs.hostmodel);
endpoint_fini(ep);
free(ep);
}
/* Phase 2: fresh persist_t + fresh cluster_t (jobmgr/hooksys only),
* recover from disk -- simulates a controller restart, the SAME apply_
* committed_entry() seam a live controller's own commit-apply pump
* would invoke to settle the two DEFERRED outcomes above. Mirrors
* test_apply_report_finished_success_uses_worker_reported_finish_
* time's own established two-phase idiom exactly. */
{
persist_t* p2 = twr_make_fresh_persist(tmp);
cluster_t c2;
job_t* j2_running = NULL;
job_t* j2_finok = NULL;
job_t* j2_finfail = NULL;
job_t* j2_unrec = NULL;
int rc;
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
TEST_ASSERT_NOT_NULL(p2);
memset(&c2, 0, sizeof(c2));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(c2.jobmgr);
g_twr_update_hook_fires = 0;
TEST_ASSERT_EQUAL_INT(0,
xce_add_hook(&c2.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook, NULL));
p2->cluster = &c2;
rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
lock(c2.jobmgr, RTTI_LOCK_READ);
key_find(c2.jobmgr, (void*)TWR_JOBID_RT_RUNNING, (void**)&j2_running);
key_find(c2.jobmgr, (void*)TWR_JOBID_RT_FINOK, (void**)&j2_finok);
key_find(c2.jobmgr, (void*)TWR_JOBID_RT_FINFAIL, (void**)&j2_finfail);
key_find(c2.jobmgr, (void*)TWR_JOBID_RT_UNREC, (void**)&j2_unrec);
unlock(c2.jobmgr);
TEST_ASSERT_NOT_NULL(j2_running);
TEST_ASSERT_NOT_NULL(j2_finok);
TEST_ASSERT_NOT_NULL(j2_finfail);
TEST_ASSERT_NOT_NULL(j2_unrec);
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j2_running->job.state,
"still-running: no fact was ever recorded for it, so replay "
"leaves it exactly as WAL_JOB_SUBMIT alone reconstructs it "
"(Req 4.1)");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE, j2_finok->job.state,
"finished-success: the DEFERRED record_job_finish_fact_at fact "
"must apply to DONE on replay (Req 4.1)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(worker_reported_finish_time_s,
(uint64_t)j2_finok->job.finish_time,
"Req 4.2, 3.5: the recovered finish_time must be the WORKER-"
"reported finish_time_ms, not reconciliation-processing "
"wall-clock time");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_EXIT, j2_finfail->job.state,
"finished-failure: the DEFERRED on_job_fail/record_job_fail_"
"fact fact must apply to EXIT on replay (Req 4.1)");
/* Req 4.2 remains DOCUMENTED-unsatisfiable for the finished-
* FAILURE finish_time specifically (bug-1747) -- not asserted here.
* The worker-reported EXIT CODE, however, is durably recorded via
* WAL_JOB_FAIL's code trailer since bug-1754. */
TEST_ASSERT_NOT_NULL_MESSAGE(j2_finfail->job.exit_info,
"bug-1754: a finished-failure's worker-reported exit code must "
"survive restart replay in job.exit_info");
TEST_ASSERT_EQUAL_INT(worker_reported_exit_code,
j2_finfail->job.exit_info->exitcode);
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j2_unrec->job.state,
"unrecognized: the EAGER record_job_reconcile_pending_fact's "
"durable WAL_JOB_UPDATE fact must ALSO reconstruct correctly on "
"replay -- convergence via either the live eager path (Phase 1 "
"above) or a restart, not just the live path (Req 4.5, 4.6)");
TEST_ASSERT_NULL(j2_unrec->job.allocs);
TEST_ASSERT_EQUAL_INT_MESSAGE(3, g_twr_update_hook_fires,
"Req 7.1-7.3: replaying the durable facts this reconciliation "
"round produced must reach the scheduler's/apiserver's notify "
"path (HOOK_DETAIL_JOB_UPDATE) for EVERY actual state change -- "
"finished-success, finished-failure, and unrecognized-to-"
"pending -- exactly 3 times (still-running recorded no fact, "
"and WAL_JOB_SUBMIT fires HOOK_DETAIL_JOB_NEW, not JOB_UPDATE, "
"so it does not add to this count). Together with Phase 1's own "
"single synchronous fire, this confirms EVERY actual state "
"change this round produced -- both the eager and the deferred "
"ones -- reaches the same notify path, not just some of them");
xce_disable_hook(&c2.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook);
xce_hook_fini(&c2.hooksys);
if (c2.jobmgr) dereference(c2.jobmgr);
twr_teardown_persist(p2);
}
{
char cmd[512];
snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
if (system(cmd) != 0)
fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
}
printf("test_full_rejoin_reconciliation_round_trip: PASS\n");
}
/* ── Task 7.2: Failover-during-job recovery -- the feature's own core
* motivating scenario (bug-1741; requirements.md's Introduction; design.md
* "Testing Strategy" > "Integration Tests": "Failover-during-job: a job's
* finish notify is lost across a primary failover; the new primary's
* rejoin reconciliation recovers it (this is the feature's core bug-1741
* regression scenario)") ────────────────────────────────────────────────
*
* Proves the SIMPLEST version of bug-1741's own reported failure: one job,
* one lost finish report, one failover, one rejoin. bug-1741 itself was a
* burst of 7 near-simultaneous finish reports losing 6 of them; that
* burst/many-jobs shape is Requirement 8's own separate coverage (already
* proven by the multi-page tests above and by test_full_rejoin_
* reconciliation_round_trip's four-outcome round trip) -- this test's own
* job is only to name and prove the single-job case in isolation, not to
* re-prove the general round-trip machinery task 7.1 already covers.
*
* Per requirements.md's Introduction, today a worker's finish report is a
* single fire-and-forget XCECLI_FF_CALL send with no retry -- if it is
* lost (the worker briefly disconnects, or the controller it was talking
* to fails over to a new primary mid-flight), the job's true outcome is
* never recorded anywhere and its state (RUN) sticks forever. This test
* models that starting condition directly: a fresh persist state in which
* the job is durably recorded as RUN, assigned to a worker host, with NO
* finish fact of any kind ever appended for it -- exactly as if the real
* XCECLI_FF_CALL send had simply vanished before this feature ever
* existed. No second controller process or literal primary-to-primary
* handoff mechanism is simulated -- the OBSERVABLE starting condition (RUN
* job, no finish fact, worker has since actually finished it) is what
* this feature's own recovery mechanism must correct, regardless of how
* that condition came to exist; per Requirement 1.3, the controller's own
* trigger does not distinguish a first join, an ordinary reconnect, or a
* post-failover reconnect to a new primary -- the same rejoin path
* (`rejoin_worker`) is exercised below exactly as it would be for any of
* the three.
*
* Drives the SAME real rejoin -> query-send -> report-apply chain task
* 7.1's own round-trip test established (`rejoin_worker`, the REAL join
* entry point; `worker_send_resync_query`'s REAL wire OPCode_jobStateQuery;
* a REAL worker answer via `xce_job_state_report_req`'s REAL wire OPCode_
* jobStateReport; `resync_handle_report_page`, never `resync_apply_report`
* directly) -- narrowed to ONE job, ONE outcome (finished-success),
* because this test's whole point is to isolate the bug-1741 scenario, not
* to duplicate the four-outcome coverage task 7.1 already built.
*
* finished-success stays on record_job_finish_fact_at's DEFERRED pattern
* (task 3.2/bug-1747) -- so, like test_full_rejoin_reconciliation_round_
* trip's own Phase 1/Phase 2 split, this test needs a real restart-replay
* (Phase 2, the SAME apply_committed_entry() seam a live controller's own
* commit-apply pump -- or another controller instance recovering the same
* durable log -- would invoke) to observe the job actually reach DONE;
* this bare-persist test fixture has no live commit-engine apply pump of
* its own, exactly the situation task 7.1's own Implementation Notes
* document. */
#define TWR_JOBID_FAILOVER_LOST 81401ULL
static void test_failover_during_job_recovers_lost_finish(void)
{
char tmp[256];
uint64_t worker_reported_finish_time_s;
strcpy(tmp, "/tmp/xce_twr72_failover_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Deliberately far in the past -- the job genuinely finished well
* before this reconciliation round runs, so a reconciliation-
* processing-time bug (time(NULL)) could never accidentally produce a
* matching value. Same defensive idiom as this file's other finish-
* time tests. */
worker_reported_finish_time_s = (uint64_t)time(NULL) - 5400;
/* Phase 1: the (new) primary's own persist state -- job RUN, no
* finish fact anywhere in it (the lost-notify starting condition) --
* then the REAL rejoin/query/report chain. */
{
cluster_t c;
kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING,
.nslots = 64, .delta = 16 };
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
persist_config_t cfg;
worker_t* w;
job_t* job;
hostattrs_t attrs, new_attrs;
xce_hostconfig_t hconfig;
endpoint_t stack_ep;
endpoint_t* ep;
xce_request_t decoded_query, decoded_report;
int rc;
uint64_t lsn_before, lsn_after;
int i;
static const char* res_names[TWR_RT_N_RES] = {
"type", "model", "cpuf", "maxcpus", "maxmem",
"maxswap", "maxtmp", "ndisks", "slots"
};
static const int res_valtypes[TWR_RT_N_RES] = {
XCE_VAL_STRING, XCE_VAL_STRING,
XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC,
XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC,
XCE_VAL_NUMERIC
};
if (makeString((char*)"twr72-probe") == NULL)
xstring_table_init(64);
if (!g_twr_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
g_twr_jobobj_ready = 1;
}
memset(&c, 0, sizeof(c));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));
c.workermgr = makeInstance(&xkvmgr_type, &str_params);
TEST_ASSERT_NOT_NULL(c.workermgr);
c.resmgr = makeInstance(&xkvmgr_type, &str_params);
TEST_ASSERT_NOT_NULL(c.resmgr);
c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(c.jobmgr);
for (i = 0; i < TWR_RT_N_RES; i++) {
xce_resource_t rdef;
memset(&rdef, 0, sizeof(rdef));
rdef.name = (char*)res_names[i];
rdef.valtype = res_valtypes[i];
rdef.flags = 0;
rdef.idx = -1;
TEST_ASSERT_NOT_NULL(makeResource(c.resmgr, &rdef));
}
c.myhost_info.name = (char*)"twr72-controller";
c.myhost_info.key = (char*)"twr72-key";
c.myhost = &c.myhost_info;
cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
persist_set_workdir(&c.persist, tmp, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));
/* Seed a durable WAL_JOB_SUBMIT fact + a matching in-memory job_t,
* RUN, assigned to the worker host -- and NOTHING ELSE. No
* WAL_JOB_START/DISPATCH/FINISH is ever appended for this job in
* this log: the controller's own durable state simply has no
* record the job ever finished, exactly bug-1741's own reported
* gap. */
{
job_submit_data_t wal_submit;
xce_job_t wal_jdef;
uint8_t* buf = NULL;
size_t sz = 0;
job_submit_data_t submit;
xce_job_t jdef;
memset(&wal_submit, 0, sizeof(wal_submit));
wal_submit.queue = (char*)"q";
wal_submit.user = (char*)"tester";
memset(&wal_jdef, 0, sizeof(wal_jdef));
wal_jdef.jobid = TWR_JOBID_FAILOVER_LOST;
wal_jdef.state = JOB_STAT_RUN;
wal_jdef.submit = &wal_submit;
TEST_ASSERT_EQUAL_INT(0, twr_encode_job(&wal_jdef, &buf, &sz));
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
(uint32_t)sz));
free(buf);
memset(&submit, 0, sizeof(submit));
submit.queue = (char*)"q";
submit.user = (char*)"tester";
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = TWR_JOBID_FAILOVER_LOST;
jdef.state = JOB_STAT_RUN;
jdef.submit = &submit;
job = makeJob(c.jobmgr, &jdef);
TEST_ASSERT_NOT_NULL(job);
job->job.allocs = twr_make_test_alloc_for_host(
TWR_JOBID_FAILOVER_LOST, "resync-failover-host");
TEST_ASSERT_NOT_NULL(job->job.allocs);
}
/* ── Worker rejoin: the REAL join entry point. Requirement 1.3 --
* the controller does not need to know or care whether this is
* the host's first-ever join, an ordinary network reconnect, or
* (this scenario) a reconnect to a new primary after a controller
* failover; the same trigger fires because the controller still
* holds one non-terminal job for this host. */
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
{
xce_worker_t host;
memset(&host, 0, sizeof(host));
host.name = (char*)"resync-failover-host";
host.attrs = &attrs;
w = makeWorker(c.workermgr, &host);
}
TEST_ASSERT_NOT_NULL(w);
memset(&new_attrs, 0, sizeof(new_attrs));
new_attrs.hosttype = makeString("LINUX");
new_attrs.hostmodel = makeString("x86");
new_attrs.cpufactor = 1.0f;
new_attrs.maxcpus = 4;
new_attrs.maxmem = 16;
new_attrs.maxswap = 32;
new_attrs.maxtmp = 64;
new_attrs.ndisks = 1;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
memset(&stack_ep, 0, sizeof(stack_ep));
rc = rejoin_worker(&c, w, &stack_ep, &new_attrs, &hconfig);
TEST_ASSERT_GREATER_OR_EQUAL(0, rc);
TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
"a rejoin finding this host's one lost-finish job still "
"recorded as RUN must arm a reconciliation round (Req 1.1, "
"1.3)");
TEST_ASSERT_EQUAL_INT(1, w->resync->n_queried_jobids);
TEST_ASSERT_TRUE(IS_WORKER_RECONCILING(w->host.status));
ep = twr_rt_new_test_endpoint();
TEST_ASSERT_EQUAL_INT(0, worker_send_resync_query(&c, w, ep));
twr_rt_pop_and_decode(ep, &decoded_query);
TEST_ASSERT_EQUAL_INT(OPCode_jobStateQuery, decoded_query.opcode);
TEST_ASSERT_EQUAL_INT(Request_body_jobstatequerymsg,
decoded_query.body_which);
TEST_ASSERT_NOT_NULL(decoded_query.body.jobstatequerymsg);
TEST_ASSERT_EQUAL_INT_MESSAGE(1,
decoded_query.body.jobstatequerymsg->n_total,
"the REAL wire query must ask about exactly the one job the "
"controller still believes is running (Req 2.1)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(TWR_JOBID_FAILOVER_LOST,
decoded_query.body.jobstatequerymsg->jobids[0],
"the query must identify the job unambiguously by its ID "
"(Req 2.2)");
xce_free_request(&decoded_query);
/* ── The worker's REAL answer: it actually finished this job
* successfully, long before this rejoin -- the controller simply
* never heard about it (the lost XCECLI_FF_CALL). */
{
xce_job_state_entry_t e;
xce_job_state_entry_t* entries[1];
xce_job_state_report_t report;
e = (xce_job_state_entry_t){
.jobid = TWR_JOBID_FAILOVER_LOST,
.status = JobStateStatus_finishedOk,
.exit_code = 0,
.finish_time_ms = worker_reported_finish_time_s * 1000ULL,
.queried = 1
};
entries[0] = &e;
report = (xce_job_state_report_t){
.n_total = 1, .pos = 0, .count = 1, .entries = entries
};
TEST_ASSERT_EQUAL_INT(0,
xce_job_state_report_req(ep, c.myhost, &report, NULL, NULL,
0));
}
twr_rt_pop_and_decode(ep, &decoded_report);
TEST_ASSERT_EQUAL_INT(OPCode_jobStateReport, decoded_report.opcode);
TEST_ASSERT_EQUAL_INT(Request_body_jobstatereportmsg,
decoded_report.body_which);
TEST_ASSERT_NOT_NULL(decoded_report.body.jobstatereportmsg);
g_twr_update_hook_fires = 0;
TEST_ASSERT_EQUAL_INT(0,
xce_add_hook(&c.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook, NULL));
pthread_mutex_lock(&c.persist.lock);
lsn_before = c.persist.current_lsn;
pthread_mutex_unlock(&c.persist.lock);
rc = resync_handle_report_page(&c, w,
decoded_report.body.jobstatereportmsg);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
"a single-page report (count == n_total) must complete the "
"report immediately");
pthread_mutex_lock(&c.persist.lock);
lsn_after = c.persist.current_lsn;
pthread_mutex_unlock(&c.persist.lock);
xce_free_request(&decoded_report);
/* ── Phase 1 assertions: the round resolves and a durable fact
* capturing the worker's TRUE outcome is appended -- but the job
* is NOT yet observably DONE in this process's own memory. That
* is expected and correct: finished-success reconciles through
* record_job_finish_fact_at's DEFERRED pattern (task 3.2/bug-
* 1747), same as an ordinary finish report -- the state mutation
* happens later, when the fact is applied (Phase 2 below), never
* synchronously here. */
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, job->job.state,
"finished-success stays RUN synchronously -- the DEFERRED "
"record_job_finish_fact_at pattern applies the state change "
"later, via apply_committed_entry (Phase 2)");
TEST_ASSERT_NULL_MESSAGE(w->resync,
"the round's only queried job has a reconciled outcome -- it "
"must resolve (Req 5.1, 5.2)");
TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
"XCE_WORKER_RECONCILING must clear once the round resolves");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_twr_update_hook_fires,
"finished-success is DEFERRED -- no HOOK_DETAIL_JOB_UPDATE "
"fire is expected synchronously here; that fire is proven "
"below, in Phase 2, once the durable fact actually applies");
TEST_ASSERT_TRUE_MESSAGE(lsn_after >= lsn_before + 1,
"the worker's true finished-success outcome must be durably "
"appended (WAL_JOB_FINISH) as part of this report, even "
"though it has not applied to in-memory state yet");
xce_disable_hook(&c.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook);
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));
persist_stop(&c.persist);
persist_fini(&c.persist);
if (c.workermgr) dereference(c.workermgr);
if (c.resmgr) dereference(c.resmgr);
if (c.jobmgr) dereference(c.jobmgr);
xce_hook_fini(&c.hooksys);
dereference(new_attrs.hosttype);
dereference(new_attrs.hostmodel);
endpoint_fini(ep);
free(ep);
}
/* Phase 2: fresh persist_t + fresh cluster_t, persist_recover() -- the
* SAME apply_committed_entry() seam a live controller's own commit-
* apply pump (or another controller instance recovering this same
* durable log) would invoke. THIS is the test's core, observable
* completion criterion: the job the controller's own durable state
* showed stuck in RUN a moment ago, with its finish notify never
* having reached any controller instance, now reconstructs as
* genuinely finished -- DONE, with the worker's own true finish time,
* not reconciliation-processing wall-clock time -- because the
* worker rejoined and this feature's reconciliation round recovered
* it (Req 4.1, 4.2). */
{
persist_t* p2 = twr_make_fresh_persist(tmp);
cluster_t c2;
job_t* j2 = NULL;
int rc;
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
TEST_ASSERT_NOT_NULL(p2);
memset(&c2, 0, sizeof(c2));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(c2.jobmgr);
g_twr_update_hook_fires = 0;
TEST_ASSERT_EQUAL_INT(0,
xce_add_hook(&c2.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook, NULL));
p2->cluster = &c2;
rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
lock(c2.jobmgr, RTTI_LOCK_READ);
key_find(c2.jobmgr, (void*)TWR_JOBID_FAILOVER_LOST, (void**)&j2);
unlock(c2.jobmgr);
TEST_ASSERT_NOT_NULL_MESSAGE(j2,
"restart/rejoin-to-new-primary replay must reconstruct the "
"job from its WAL_JOB_SUBMIT fact");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE, j2->job.state,
"the feature's core completion criterion (bug-1741): a job "
"whose finish notify never reached any controller instance "
"ends up correctly finished, not stuck in RUN, once the "
"worker rejoins the new primary (Req 4.1)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(worker_reported_finish_time_s,
(uint64_t)j2->job.finish_time,
"the recovered finish_time must be the WORKER-reported "
"finish_time_ms from the reconciliation report, not "
"reconciliation-processing or replay wall-clock time "
"(Req 4.2, 3.5)");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_twr_update_hook_fires,
"replaying the durable fact this reconciliation round "
"produced must reach the scheduler's/apiserver's notify path "
"(HOOK_DETAIL_JOB_UPDATE) exactly once for this one real "
"state change (Req 7.1-7.3)");
xce_disable_hook(&c2.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook);
xce_hook_fini(&c2.hooksys);
if (c2.jobmgr) dereference(c2.jobmgr);
twr_teardown_persist(p2);
}
{
char cmd[512];
snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
if (system(cmd) != 0)
fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
}
printf("test_failover_during_job_recovers_lost_finish: PASS\n");
}
/* ── Task 7.3: Burst/array recovery unbounded by job count -- bug-1741's OWN
* exact originally-reported shape (requirements.md's Introduction: "a burst
* of near-simultaneous job completions on one worker host lost 6 of 7
* finish reports, leaving those jobs stuck in RUN forever while their
* siblings all completed normally") ────────────────────────────────────────
*
* Task 7.1 proved the general four-outcome round trip (one job per
* outcome); task 7.2 proved the single-job lost-finish recovery in
* isolation. Neither proves what THIS task is for: bug-1741's own reported
* shape was SEVERAL jobs at once, MOST lost, with recovery required to
* scale correctly across all of them together in ONE round -- not merely
* repeat the single-job case N times with no cross-job interaction check
* (tasks.md's own Observable Completion Criterion for this task).
*
* 7 jobs total on ONE worker host (bug-1741's own literal numbers):
* - 1 job (TWR_JOBID_BURST_ALREADY_DONE) already finished successfully
* BEFORE this rejoin -- a durable WAL_JOB_SUBMIT + WAL_JOB_FINISH pair
* is already on disk (the FINISH appended directly via persist_submit_
* job_finish_fact(), the same primitive record_job_finish_fact_at()
* itself calls -- modeling a finish report that DID reach and was
* durably committed by an earlier controller instance, before this
* rejoin), in-memory state already DONE. Being terminal, it must NOT be
* part of the round's query set (Req 1.1, 1.2, 2.1) -- but the worker's
* own retention window (Req 3.6, 3.7) would still recognize it as
* "recently finished" and report it AGAIN, as an unqueried extra, this
* time carrying a deliberately WRONG (bogus, recent) finish_time_ms.
* This is this task's own "none duplicated" Observable Completion
* Criterion, made concrete: this job's already-recorded outcome must
* survive completely unchanged -- both synchronously (Phase 1) and
* after a real restart replay (Phase 2) -- never overwritten by the
* bogus re-report, and with NO new WAL fact appended for it at all.
* - 6 jobs (TWR_JOBID_BURST_LOST_1..6) mirror bug-1741's own "6 of 7 lost"
* ratio exactly: each is durably recorded RUN with NO finish fact of
* any kind ever appended (the lost fire-and-forget XCECLI_FF_CALL, the
* same starting condition as task 7.2's single-job case) -- all 6 are
* non-terminal and assigned to the SAME host, so all 6 are armed into
* ONE round's query set together (Req 2.1, 8.1). Each carries its OWN
* distinct worker-reported finish_time_ms/exit_code (5 finished-success
* + 1 finished-failure, a genuine mixed-outcome burst) -- distinct per
* job specifically so a bug that mixes up entries across the loop (an
* off-by-one / shared-variable defect in resync_apply_report's
* per-entry routing) would surface as a WRONG value under a SPECIFIC
* jobid, never masked by every job sharing one identical expected
* outcome.
*
* Drives the SAME real rejoin -> query-send -> report-apply chain tasks
* 7.1/7.2 already established (rejoin_worker, worker_send_resync_query's
* real wire OPCode_jobStateQuery, a real worker answer via xce_job_state_
* report_req's real wire OPCode_jobStateReport, resync_handle_report_page,
* never resync_apply_report directly) -- one single-page report covering
* all 7 entries together (6 queried + 1 unqueried extra). Deliberately NOT
* forced across multiple wire pages: task 7.4's own separately-scoped job
* is proving PAGINATION boundary correctness at a much larger count; this
* task's job is proving RECOVERY LOGIC correctness at bug-1741's own
* "several jobs at once" scale, per tasks.md's explicit boundary note
* separating the two. */
#define TWR_JOBID_BURST_ALREADY_DONE 81501ULL
#define TWR_JOBID_BURST_LOST_1 81502ULL
#define TWR_JOBID_BURST_LOST_2 81503ULL
#define TWR_JOBID_BURST_LOST_3 81504ULL
#define TWR_JOBID_BURST_LOST_4 81505ULL
#define TWR_JOBID_BURST_LOST_5 81506ULL
#define TWR_JOBID_BURST_LOST_6 81507ULL /* finished-FAILURE, not OK */
#define TWR_BURST_N_LOST 6
#define TWR_BURST_N_TOTAL 7 /* 1 already-done + 6 lost -- bug-1741's own 7/6 numbers */
static void test_burst_recovery_unbounded_by_job_count(void)
{
char tmp[256];
uint64_t already_done_finish_time_s;
uint64_t bogus_reapply_finish_time_s;
uint64_t lost_finish_time_s[TWR_BURST_N_LOST - 1]; /* LOST_1..LOST_5 (finished-ok) */
int32_t lost_fail_exit_code = 66;
strcpy(tmp, "/tmp/xce_twr73_burst_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Deliberately distinct/far-from-"now" values throughout -- so a
* reconciliation-processing-time bug (time(NULL)) or a cross-job value
* mix-up could never accidentally produce a matching value, same
* defensive idiom this file's other finish-time tests already
* establish. */
already_done_finish_time_s = (uint64_t)time(NULL) - 7200;
bogus_reapply_finish_time_s = (uint64_t)time(NULL) - 30;
{
int k;
for (k = 0; k < TWR_BURST_N_LOST - 1; k++)
lost_finish_time_s[k]
= (uint64_t)time(NULL) - 3600 - (uint64_t)(k * 137);
}
/* Phase 1: real cluster (workermgr + resmgr + jobmgr + hooksys) + real
* disk persist -- drives the REAL join/query-send/report-apply chain
* over all 7 jobs together. Own local cluster_t (not the shared g_twr
* fixture), mirroring every other restart-replay test in this file. */
{
cluster_t c;
kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING,
.nslots = 64, .delta = 16 };
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
persist_config_t cfg;
worker_t* w;
job_t* j_done;
job_t* j_lost[TWR_BURST_N_LOST];
uint64_t lost_ids[TWR_BURST_N_LOST] = {
TWR_JOBID_BURST_LOST_1, TWR_JOBID_BURST_LOST_2,
TWR_JOBID_BURST_LOST_3, TWR_JOBID_BURST_LOST_4,
TWR_JOBID_BURST_LOST_5, TWR_JOBID_BURST_LOST_6
};
hostattrs_t attrs, new_attrs;
xce_hostconfig_t hconfig;
endpoint_t stack_ep;
endpoint_t* ep;
xce_request_t decoded_query, decoded_report;
int saw[TWR_BURST_N_LOST];
int i, rc;
uint64_t lsn_before, lsn_after;
static const char* res_names[TWR_RT_N_RES] = {
"type", "model", "cpuf", "maxcpus", "maxmem",
"maxswap", "maxtmp", "ndisks", "slots"
};
static const int res_valtypes[TWR_RT_N_RES] = {
XCE_VAL_STRING, XCE_VAL_STRING,
XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC,
XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC,
XCE_VAL_NUMERIC
};
if (makeString((char*)"twr73-probe") == NULL)
xstring_table_init(64);
if (!g_twr_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
g_twr_jobobj_ready = 1;
}
memset(&c, 0, sizeof(c));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));
c.workermgr = makeInstance(&xkvmgr_type, &str_params);
TEST_ASSERT_NOT_NULL(c.workermgr);
c.resmgr = makeInstance(&xkvmgr_type, &str_params);
TEST_ASSERT_NOT_NULL(c.resmgr);
c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(c.jobmgr);
for (i = 0; i < TWR_RT_N_RES; i++) {
xce_resource_t rdef;
memset(&rdef, 0, sizeof(rdef));
rdef.name = (char*)res_names[i];
rdef.valtype = res_valtypes[i];
rdef.flags = 0;
rdef.idx = -1;
TEST_ASSERT_NOT_NULL(makeResource(c.resmgr, &rdef));
}
c.myhost_info.name = (char*)"twr73-controller";
c.myhost_info.key = (char*)"twr73-key";
c.myhost = &c.myhost_info;
cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
persist_set_workdir(&c.persist, tmp, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));
/* Seed the ALREADY-DONE job: durable WAL_JOB_SUBMIT + a REAL,
* already-committed WAL_JOB_FINISH (appended directly via
* persist_submit_job_finish_fact -- modeling a finish report that
* reached and was durably recorded by an EARLIER controller
* instance, before this rejoin) -- plus a matching in-memory job_t
* already DONE, allocs already released, exactly what a genuinely
* finished job looks like. */
{
job_submit_data_t wal_submit;
xce_job_t wal_jdef;
uint8_t* buf = NULL;
size_t sz = 0;
job_submit_data_t submit;
xce_job_t jdef;
memset(&wal_submit, 0, sizeof(wal_submit));
wal_submit.queue = (char*)"q";
wal_submit.user = (char*)"tester";
memset(&wal_jdef, 0, sizeof(wal_jdef));
wal_jdef.jobid = TWR_JOBID_BURST_ALREADY_DONE;
wal_jdef.state = JOB_STAT_RUN;
wal_jdef.submit = &wal_submit;
TEST_ASSERT_EQUAL_INT(0, twr_encode_job(&wal_jdef, &buf, &sz));
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
(uint32_t)sz));
free(buf);
TEST_ASSERT_EQUAL_INT(0,
persist_submit_job_finish_fact(&c.persist,
TWR_JOBID_BURST_ALREADY_DONE, JOB_STAT_DONE,
already_done_finish_time_s, 0));
memset(&submit, 0, sizeof(submit));
submit.queue = (char*)"q";
submit.user = (char*)"tester";
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = TWR_JOBID_BURST_ALREADY_DONE;
jdef.state = JOB_STAT_DONE;
jdef.submit = &submit;
j_done = makeJob(c.jobmgr, &jdef);
TEST_ASSERT_NOT_NULL(j_done);
j_done->job.finish_time = (time_t)already_done_finish_time_s;
j_done->job.exit_info
= (job_exitinfo_t*)calloc(1, sizeof(job_exitinfo_t));
TEST_ASSERT_NOT_NULL(j_done->job.exit_info);
j_done->job.exit_info->exitcode = 0;
TEST_ASSERT_NULL(j_done->job.allocs);
}
/* Seed the 6 LOST jobs: durable WAL_JOB_SUBMIT only, RUN, all
* assigned to the SAME host -- NO finish fact of any kind, the
* lost-fire-and-forget-send starting condition, x6 (bug-1741's own
* "6 of 7 lost" ratio). */
for (i = 0; i < TWR_BURST_N_LOST; i++) {
job_submit_data_t wal_submit;
xce_job_t wal_jdef;
uint8_t* buf = NULL;
size_t sz = 0;
job_submit_data_t submit;
xce_job_t jdef;
memset(&wal_submit, 0, sizeof(wal_submit));
wal_submit.queue = (char*)"q";
wal_submit.user = (char*)"tester";
memset(&wal_jdef, 0, sizeof(wal_jdef));
wal_jdef.jobid = lost_ids[i];
wal_jdef.state = JOB_STAT_RUN;
wal_jdef.submit = &wal_submit;
TEST_ASSERT_EQUAL_INT(0, twr_encode_job(&wal_jdef, &buf, &sz));
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
(uint32_t)sz));
free(buf);
memset(&submit, 0, sizeof(submit));
submit.queue = (char*)"q";
submit.user = (char*)"tester";
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = lost_ids[i];
jdef.state = JOB_STAT_RUN;
jdef.submit = &submit;
j_lost[i] = makeJob(c.jobmgr, &jdef);
TEST_ASSERT_NOT_NULL(j_lost[i]);
j_lost[i]->job.allocs = twr_make_test_alloc_for_host(
lost_ids[i], "resync-burst-host");
TEST_ASSERT_NOT_NULL(j_lost[i]->job.allocs);
}
/* ── Join: rejoin_worker (the REAL join entry point) arms the round
* over exactly the 6 non-terminal (lost) jobs -- the 7th,
* already-DONE job must NOT be part of the query set (Req 1.1,
* 1.2, 2.1). */
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
{
xce_worker_t host;
memset(&host, 0, sizeof(host));
host.name = (char*)"resync-burst-host";
host.attrs = &attrs;
w = makeWorker(c.workermgr, &host);
}
TEST_ASSERT_NOT_NULL(w);
memset(&new_attrs, 0, sizeof(new_attrs));
new_attrs.hosttype = makeString("LINUX");
new_attrs.hostmodel = makeString("x86");
new_attrs.cpufactor = 1.0f;
new_attrs.maxcpus = 4;
new_attrs.maxmem = 16;
new_attrs.maxswap = 32;
new_attrs.maxtmp = 64;
new_attrs.ndisks = 1;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
memset(&stack_ep, 0, sizeof(stack_ep));
rc = rejoin_worker(&c, w, &stack_ep, &new_attrs, &hconfig);
TEST_ASSERT_GREATER_OR_EQUAL(0, rc);
TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
"a rejoin with 6 known non-terminal jobs on this host must arm "
"a reconciliation round (Req 1.1)");
TEST_ASSERT_EQUAL_INT_MESSAGE(TWR_BURST_N_LOST,
w->resync->n_queried_jobids,
"the round's query set must match exactly the host's 6 "
"non-terminal jobs -- the already-DONE 7th job must be "
"excluded (Req 1.1, 1.2, 2.1)");
TEST_ASSERT_TRUE(IS_WORKER_RECONCILING(w->host.status));
ep = twr_rt_new_test_endpoint();
TEST_ASSERT_EQUAL_INT(0, worker_send_resync_query(&c, w, ep));
twr_rt_pop_and_decode(ep, &decoded_query);
TEST_ASSERT_EQUAL_INT(OPCode_jobStateQuery, decoded_query.opcode);
TEST_ASSERT_EQUAL_INT(Request_body_jobstatequerymsg,
decoded_query.body_which);
TEST_ASSERT_NOT_NULL(decoded_query.body.jobstatequerymsg);
TEST_ASSERT_EQUAL_INT_MESSAGE(TWR_BURST_N_LOST,
decoded_query.body.jobstatequerymsg->n_total,
"the REAL wire query (Req 2.1, 2.2, 8.1) must list exactly the "
"6 lost jobs, regardless of the burst also containing a 7th, "
"already-terminal one");
for (i = 0; i < TWR_BURST_N_LOST; i++)
saw[i] = 0;
for (i = 0; i < decoded_query.body.jobstatequerymsg->count; i++) {
uint64_t id = decoded_query.body.jobstatequerymsg->jobids[i];
int k;
TEST_ASSERT_FALSE_MESSAGE(id == TWR_JOBID_BURST_ALREADY_DONE,
"the already-terminal job must never appear in the query "
"set (Req 1.2)");
for (k = 0; k < TWR_BURST_N_LOST; k++) {
if (id == lost_ids[k])
saw[k] = 1;
}
}
for (i = 0; i < TWR_BURST_N_LOST; i++) {
TEST_ASSERT_TRUE_MESSAGE(saw[i],
"every one of the 6 lost jobs must be covered by the wire "
"query, by unique ID (Req 2.2, 8.1)");
}
xce_free_request(&decoded_query);
/* ── Worker's real answer: all 6 lost jobs report their TRUE
* outcome (5 finished-success + 1 finished-failure, each with its
* OWN distinct finish_time_ms/exit_code -- Req 3.1, 3.4, 3.5), plus
* the 7th, already-DONE job reported AGAIN as an unqueried extra
* (Req 3.6) -- carrying a deliberately WRONG finish_time_ms, to
* prove reconciliation does not clobber/duplicate what was already
* correctly recorded (this task's own "none duplicated" Observable
* Completion Criterion). All 7 entries travel in ONE page (Req
* 8.1, 8.2) -- this task is not about page-boundary mechanics
* (task 7.4's own separate job). */
{
xce_job_state_entry_t e_lost[TWR_BURST_N_LOST];
xce_job_state_entry_t e_already_done;
xce_job_state_entry_t* entries[TWR_BURST_N_TOTAL];
xce_job_state_report_t report;
for (i = 0; i < TWR_BURST_N_LOST - 1; i++) {
e_lost[i] = (xce_job_state_entry_t){
.jobid = lost_ids[i],
.status = JobStateStatus_finishedOk,
.exit_code = 0,
.finish_time_ms = lost_finish_time_s[i] * 1000ULL,
.queried = 1
};
}
/* The 6th lost job: finished-FAILURE, not success -- a genuine
* mixed-outcome burst (this task's own scope), not a re-proof
* of 7.1's already-established per-outcome routing in
* isolation. */
e_lost[TWR_BURST_N_LOST - 1] = (xce_job_state_entry_t){
.jobid = lost_ids[TWR_BURST_N_LOST - 1],
.status = JobStateStatus_finishedFail,
.exit_code = lost_fail_exit_code,
.finish_time_ms = 0,
.queried = 1
};
e_already_done = (xce_job_state_entry_t){
.jobid = TWR_JOBID_BURST_ALREADY_DONE,
.status = JobStateStatus_finishedOk,
.exit_code = 0,
.finish_time_ms = bogus_reapply_finish_time_s * 1000ULL,
.queried = 0
};
for (i = 0; i < TWR_BURST_N_LOST; i++)
entries[i] = &e_lost[i];
entries[TWR_BURST_N_LOST] = &e_already_done;
report = (xce_job_state_report_t){
.n_total = TWR_BURST_N_TOTAL, .pos = 0,
.count = TWR_BURST_N_TOTAL, .entries = entries
};
TEST_ASSERT_EQUAL_INT(0,
xce_job_state_report_req(ep, c.myhost, &report, NULL, NULL,
0));
}
twr_rt_pop_and_decode(ep, &decoded_report);
TEST_ASSERT_EQUAL_INT(OPCode_jobStateReport, decoded_report.opcode);
TEST_ASSERT_EQUAL_INT(Request_body_jobstatereportmsg,
decoded_report.body_which);
TEST_ASSERT_NOT_NULL(decoded_report.body.jobstatereportmsg);
TEST_ASSERT_EQUAL_INT_MESSAGE(TWR_BURST_N_TOTAL,
decoded_report.body.jobstatereportmsg->n_total,
"the real wire report must carry all 7 entries in one page "
"(Req 8.1, 8.2)");
g_twr_update_hook_fires = 0;
TEST_ASSERT_EQUAL_INT(0,
xce_add_hook(&c.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook, NULL));
pthread_mutex_lock(&c.persist.lock);
lsn_before = c.persist.current_lsn;
pthread_mutex_unlock(&c.persist.lock);
rc = resync_handle_report_page(&c, w,
decoded_report.body.jobstatereportmsg);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
"a single-page report (count == n_total) must complete the "
"report immediately, regardless of the burst's job count "
"(Req 8.1)");
pthread_mutex_lock(&c.persist.lock);
lsn_after = c.persist.current_lsn;
pthread_mutex_unlock(&c.persist.lock);
xce_free_request(&decoded_report);
/* ── Phase 1 assertions: everything observable before any commit
* round trip. */
for (i = 0; i < TWR_BURST_N_LOST; i++) {
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN,
j_lost[i]->job.state,
"finished-success/failure both stay on the DEFERRED "
"record_job_finish_fact_at/record_job_fail_fact pattern -- "
"state mutates later via apply_committed_entry (Phase 2), "
"not synchronously here, for every one of the 6 lost "
"jobs");
}
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE, j_done->job.state,
"the already-DONE job must be left completely untouched by "
"its own bogus re-report (Req 4.3, 4.4 -- 'none duplicated')");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(already_done_finish_time_s,
(uint64_t)j_done->job.finish_time,
"the already-DONE job's ORIGINAL finish_time must survive "
"unchanged -- NOT overwritten by the bogus finish_time_ms "
"carried in its own unqueried re-report");
TEST_ASSERT_NOT_NULL(j_done->job.exit_info);
TEST_ASSERT_EQUAL_INT(0, j_done->job.exit_info->exitcode);
TEST_ASSERT_NULL(j_done->job.allocs);
TEST_ASSERT_NULL_MESSAGE(w->resync,
"once every one of the 6 queried jobs has a reconciled "
"outcome, the round must resolve regardless of the burst's "
"count (Req 5.1, 5.2, 8.1)");
TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
"XCE_WORKER_RECONCILING must clear once the round resolves");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_twr_update_hook_fires,
"no EAGER outcome is present in this burst (no unrecognized, "
"no genuinely-unassigned extra) -- the 6 deferred finished-"
"success/failure facts have not applied yet, and the "
"already-DONE job's re-report is correctly a pure no-op with "
"no notify at all (Req 4.3, 4.4)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
lsn_before + (uint64_t)TWR_BURST_N_LOST, lsn_after,
"EXACTLY one durable fact per lost job (5x WAL_JOB_FINISH + 1x "
"WAL_JOB_FAIL) must be appended for this report -- the "
"already-DONE job's duplicate re-report must append NONE, so "
"an exact count (not merely a lower bound) is the genuine "
"'none duplicated' proof: a regression that re-appended a "
"fact for the already-DONE job would silently pass a "
"'>=' check but fails this exact one");
xce_disable_hook(&c.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook);
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));
persist_stop(&c.persist);
persist_fini(&c.persist);
if (c.workermgr) dereference(c.workermgr);
if (c.resmgr) dereference(c.resmgr);
if (c.jobmgr) dereference(c.jobmgr);
xce_hook_fini(&c.hooksys);
dereference(new_attrs.hosttype);
dereference(new_attrs.hostmodel);
endpoint_fini(ep);
free(ep);
}
/* Phase 2: fresh persist_t + fresh cluster_t (jobmgr/hooksys only),
* recover from disk -- simulates a controller restart, the SAME apply_
* committed_entry() seam a live controller's own commit-apply pump
* would invoke to settle the 6 DEFERRED outcomes above. This is the
* test's core, observable completion criterion: every one of the 6
* lost jobs reaches its correct final state, individually and by its
* OWN distinct worker-reported value (not some other job's from the
* same burst), and the 7th job's already-correct outcome is confirmed
* to have survived completely untouched -- through a real restart
* replay, not just the live path. */
{
persist_t* p2 = twr_make_fresh_persist(tmp);
cluster_t c2;
job_t* j2_done = NULL;
job_t* j2_lost[TWR_BURST_N_LOST] = { NULL };
uint64_t lost_ids[TWR_BURST_N_LOST] = {
TWR_JOBID_BURST_LOST_1, TWR_JOBID_BURST_LOST_2,
TWR_JOBID_BURST_LOST_3, TWR_JOBID_BURST_LOST_4,
TWR_JOBID_BURST_LOST_5, TWR_JOBID_BURST_LOST_6
};
int rc, i;
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
TEST_ASSERT_NOT_NULL(p2);
memset(&c2, 0, sizeof(c2));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(c2.jobmgr);
g_twr_update_hook_fires = 0;
TEST_ASSERT_EQUAL_INT(0,
xce_add_hook(&c2.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook, NULL));
p2->cluster = &c2;
rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
lock(c2.jobmgr, RTTI_LOCK_READ);
key_find(c2.jobmgr, (void*)TWR_JOBID_BURST_ALREADY_DONE,
(void**)&j2_done);
for (i = 0; i < TWR_BURST_N_LOST; i++)
key_find(c2.jobmgr, (void*)lost_ids[i], (void**)&j2_lost[i]);
unlock(c2.jobmgr);
TEST_ASSERT_NOT_NULL(j2_done);
for (i = 0; i < TWR_BURST_N_LOST; i++) {
TEST_ASSERT_NOT_NULL_MESSAGE(j2_lost[i],
"every one of the 6 lost jobs must reconstruct from its "
"own WAL_JOB_SUBMIT fact");
}
/* The core completion criterion (bug-1741, 'none left stuck'):
* every one of the 6 lost jobs, individually, by its own jobid,
* reaches its correct final state AND its OWN distinct
* worker-reported value -- proving no cross-job mixing across the
* burst (Req 8.1). */
for (i = 0; i < TWR_BURST_N_LOST - 1; i++) {
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE,
j2_lost[i]->job.state,
"each lost finished-success job must recover to DONE, "
"regardless of the burst's count (Req 8.1)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(lost_finish_time_s[i],
(uint64_t)j2_lost[i]->job.finish_time,
"each lost job must recover ITS OWN worker-reported "
"finish_time -- not another job's from the same burst "
"(cross-job mixing regression check, Req 4.2, 3.5)");
}
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_EXIT,
j2_lost[TWR_BURST_N_LOST - 1]->job.state,
"the one finished-FAILURE job in the burst must recover to "
"EXIT, not DONE, distinctly from its 5 finished-success "
"siblings in the SAME round");
/* Req 4.2 remains documented-unsatisfiable for FAIL's own
* finish_time specifically (bug-1747). Its exit code is recorded
* since bug-1754. */
TEST_ASSERT_NOT_NULL(j2_lost[TWR_BURST_N_LOST - 1]->job.exit_info);
TEST_ASSERT_EQUAL_INT(lost_fail_exit_code,
j2_lost[TWR_BURST_N_LOST - 1]->job.exit_info->exitcode);
/* The 'none duplicated' completion criterion, confirmed to
* survive a restart replay too: the already-DONE job's ORIGINAL
* outcome, not the bogus re-reported one. */
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE, j2_done->job.state,
"the already-DONE job must still be DONE after replay");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(already_done_finish_time_s,
(uint64_t)j2_done->job.finish_time,
"the already-DONE job's finish_time after replay must still "
"be its ORIGINAL value -- the bogus re-report's "
"finish_time_ms must never have been durably recorded at "
"all");
TEST_ASSERT_EQUAL_INT_MESSAGE(TWR_BURST_N_TOTAL,
g_twr_update_hook_fires,
"replaying the durable facts in this log must reach the "
"scheduler's/apiserver's notify path exactly once per REAL "
"state change -- 6 for the 6 lost jobs' newly-recorded facts, "
"PLUS 1 for the already-DONE job's own ORIGINALLY-seeded "
"WAL_JOB_FINISH fact (recorded before this round even ran, "
"modeling an earlier controller instance's already-committed "
"finish): this bare-persist fixture runs no live apply pump, "
"so THIS replay is the first time apply_committed_entry ever "
"applies that pre-existing fact too, exactly like every other "
"seeded WAL_JOB_SUBMIT in this test -- NOT a second, "
"duplicate application of it. The genuine 'none duplicated' "
"proof is not this count (which is 7 either way) but that the "
"already-DONE job's finish_time below is still its ORIGINAL "
"value, and that Phase 1's own exact LSN-delta check already "
"proved no fact was durably appended for its bogus re-report "
"(Req 7.1-7.3)");
xce_disable_hook(&c2.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook);
xce_hook_fini(&c2.hooksys);
if (c2.jobmgr) dereference(c2.jobmgr);
twr_teardown_persist(p2);
}
{
char cmd[512];
snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
if (system(cmd) != 0)
fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
}
printf("test_burst_recovery_unbounded_by_job_count: PASS\n");
}
/* ── Task 7.4: Large job-count pagination correctness (Req 8.1-8.3) ──────
*
* design.md's own "Performance/Load" testing-strategy line: "Reconciliation
* exchange for a worker running an order of magnitude more jobs than a
* single message can carry completes correctly across multiple pages, not
* truncated (Requirement 8.1-8.3)." Tasks 7.1-7.3 all deliberately stayed
* within ONE wire page each (task 7.3's own Implementation Notes:
* "Deliberately kept to a single wire page (page-boundary correctness is
* task 7.4's separate concern)") -- THIS is that separate concern: proving
* BOTH directions of the exchange (controller->worker OPCode_jobStateQuery,
* worker->controller OPCode_jobStateReport) genuinely span MULTIPLE wire
* pages for a large job count, and that resync_handle_report_page()'s
* accumulator (task 6.2) reassembles all of them into exactly one applied
* report -- not merely re-proving task 7.1's four-outcome/task 7.3's burst
* recovery logic at a bigger N.
*
* Real production page-size constants (found by reading the actual call
* sites, not assumed):
* - REPORT direction (worker -> controller): xcecd's real production call
* site (controller_connector.c's handle_jobstatequery ->
* xcecd_resync_handle_query_page -> xcecd_resync_answer_query) passes
* XCECD_RESYNC_REPORT_PKTLIMIT == 100 (controller_connector.c) into
* xce_job_state_report_req() (task 1.4, request.c) -- a REAL, wired,
* production page cap.
* - QUERY direction (controller -> worker): the controller's own real
* production call site, worker_send_resync_query() (worker.c:730),
* hardcodes pktlimit=0 in its xce_job_state_query_req() call --
* xce_job_state_query_req()'s own pktlimit<=0 convention means
* "pktlimit = nrows", i.e. ALWAYS exactly ONE wire page, however many
* jobs are queried. There is no real production constant to cite for
* this direction -- see this task's own CONCERNS (bug-1755). To
* genuinely exercise the QUERY side's already-implemented pagination
* loop (task 1.4) at scale, this test calls xce_job_state_query_req()
* directly (bypassing worker_send_resync_query()'s hardcoded 0),
* supplying the SAME page size as the worker's own real report cap for
* symmetry -- the exact same production function, exercised with a
* real bound instead of an effectively-disabled one.
*
* TWR_LARGE_N_TOTAL (below) is an order of magnitude larger than
* TWR_LARGE_PKTLIMIT (100), yielding a double-digit page count in EACH
* direction, with a genuinely PARTIAL final page (the total, and each of
* the four outcome-group sizes, are deliberately NOT multiples of the page
* limit) -- so several wire pages straddle two different outcome groups, a
* genuine page-boundary-falls-mid-group case, and the very last page of
* each direction is short. Covers all four outcome kinds (running,
* finished-success, finished-failure, unrecognized) so this is not a
* degenerate single-outcome test.
*/
#define TWR_JOBID_LARGE_BASE 900000ULL
#define TWR_LARGE_N_RUNNING 260
#define TWR_LARGE_N_FINOK 245
#define TWR_LARGE_N_FINFAIL 235
#define TWR_LARGE_N_UNREC 302
#define TWR_LARGE_N_TOTAL 1042 /* 260+245+235+302 -- not a multiple of
* TWR_LARGE_PKTLIMIT, so the final wire
* page in each direction is genuinely
* partial (42 entries), not just N full
* pages */
/* Shared page-size cap for both directions in this test -- == the REAL
* production XCECD_RESYNC_REPORT_PKTLIMIT for the report direction; the
* query direction has no real production constant (see header comment
* above / CONCERNS, bug-1755), so this test uses the same value for
* symmetry while directly exercising xce_job_state_query_req()'s already-
* implemented pagination loop. */
#define TWR_LARGE_PKTLIMIT 100
#define TWR_LARGE_N_PAGES \
((TWR_LARGE_N_TOTAL + TWR_LARGE_PKTLIMIT - 1) / TWR_LARGE_PKTLIMIT)
static uint64_t twr_large_jobid(int i)
{
return TWR_JOBID_LARGE_BASE + (uint64_t)i;
}
/* 0 = running, 1 = finished-success, 2 = finished-failure, 3 =
* unrecognized -- see this test's own header comment on why the group
* boundaries are deliberately not page-aligned. */
static int twr_large_group(int i)
{
if (i < TWR_LARGE_N_RUNNING)
return 0;
if (i < TWR_LARGE_N_RUNNING + TWR_LARGE_N_FINOK)
return 1;
if (i < TWR_LARGE_N_RUNNING + TWR_LARGE_N_FINOK + TWR_LARGE_N_FINFAIL)
return 2;
return 3;
}
static void test_large_job_count_pagination_correctness(void)
{
char tmp[256];
uint64_t finok_anchor_s;
strcpy(tmp, "/tmp/xce_twr74_large_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Deliberately far from "now" -- same defensive idiom as every other
* finish-time test in this file. */
finok_anchor_s = (uint64_t)time(NULL) - 3600;
/* Phase 1: real cluster (workermgr + resmgr + jobmgr + hooksys) + real
* disk persist -- drives the REAL rejoin/query-construction chain over
* TWR_LARGE_N_TOTAL jobs, then a REAL, explicitly-bounded wire query
* send and a REAL, real-production-bounded wire report send, both
* genuinely multi-page. Own local cluster_t (not the shared g_twr
* fixture), mirroring every other restart-replay test in this file. */
{
cluster_t c;
kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING,
.nslots = 64, .delta = 16 };
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 1024, .delta = 256 };
persist_config_t cfg;
worker_t* w;
job_t** jobs;
hostattrs_t attrs, new_attrs;
xce_hostconfig_t hconfig;
endpoint_t stack_ep;
endpoint_t* ep;
xce_job_state_entry_t* storage;
xce_job_state_entry_t** entries;
xce_job_state_query_t query;
xce_job_state_report_t report;
int i, rc;
uint64_t lsn_before, lsn_after;
int calls_before, calls_after;
static const char* res_names[TWR_RT_N_RES] = {
"type", "model", "cpuf", "maxcpus", "maxmem",
"maxswap", "maxtmp", "ndisks", "slots"
};
static const int res_valtypes[TWR_RT_N_RES] = {
XCE_VAL_STRING, XCE_VAL_STRING,
XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC,
XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC,
XCE_VAL_NUMERIC
};
if (makeString((char*)"twr74-probe") == NULL)
xstring_table_init(64);
if (!g_twr_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
g_twr_jobobj_ready = 1;
}
memset(&c, 0, sizeof(c));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));
c.workermgr = makeInstance(&xkvmgr_type, &str_params);
TEST_ASSERT_NOT_NULL(c.workermgr);
c.resmgr = makeInstance(&xkvmgr_type, &str_params);
TEST_ASSERT_NOT_NULL(c.resmgr);
c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(c.jobmgr);
for (i = 0; i < TWR_RT_N_RES; i++) {
xce_resource_t rdef;
memset(&rdef, 0, sizeof(rdef));
rdef.name = (char*)res_names[i];
rdef.valtype = res_valtypes[i];
rdef.flags = 0;
rdef.idx = -1;
TEST_ASSERT_NOT_NULL(makeResource(c.resmgr, &rdef));
}
c.myhost_info.name = (char*)"twr74-controller";
c.myhost_info.key = (char*)"twr74-key";
c.myhost = &c.myhost_info;
cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
persist_set_workdir(&c.persist, tmp, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));
jobs = (job_t**)malloc(sizeof(job_t*) * TWR_LARGE_N_TOTAL);
TEST_ASSERT_NOT_NULL(jobs);
/* Seed a durable WAL_JOB_SUBMIT fact + a matching in-memory job_t
* for all TWR_LARGE_N_TOTAL jobs -- all RUN, all assigned to the
* SAME worker host, exactly what the controller believes right
* before this rejoin (bypassing dispatch/replay, mirroring every
* sibling restart-replay test in this file). */
for (i = 0; i < TWR_LARGE_N_TOTAL; i++) {
uint64_t jobid = twr_large_jobid(i);
job_submit_data_t wal_submit;
xce_job_t wal_jdef;
uint8_t* buf = NULL;
size_t sz = 0;
job_submit_data_t submit;
xce_job_t jdef;
memset(&wal_submit, 0, sizeof(wal_submit));
wal_submit.queue = (char*)"q";
wal_submit.user = (char*)"tester";
memset(&wal_jdef, 0, sizeof(wal_jdef));
wal_jdef.jobid = jobid;
wal_jdef.state = JOB_STAT_RUN;
wal_jdef.submit = &wal_submit;
TEST_ASSERT_EQUAL_INT(0, twr_encode_job(&wal_jdef, &buf, &sz));
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
(uint32_t)sz));
free(buf);
memset(&submit, 0, sizeof(submit));
submit.queue = (char*)"q";
submit.user = (char*)"tester";
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = jobid;
jdef.state = JOB_STAT_RUN;
jdef.submit = &submit;
jobs[i] = makeJob(c.jobmgr, &jdef);
TEST_ASSERT_NOT_NULL(jobs[i]);
jobs[i]->job.allocs
= twr_make_test_alloc_for_host(jobid, "resync-large-host");
TEST_ASSERT_NOT_NULL(jobs[i]->job.allocs);
}
/* ── Join: rejoin_worker (the REAL join entry point) arms the round
* over all TWR_LARGE_N_TOTAL non-terminal jobs -- Req 1.1, 2.1, and
* the "no artificial cap" proof (Req 8.1) that resync_collect_
* query_jobids' realloc-doubling growth (resync.c) genuinely has no
* fixed-size ceiling. */
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 4;
attrs.maxmem = 16;
attrs.maxswap = 32;
attrs.maxtmp = 64;
attrs.ndisks = 1;
{
xce_worker_t host;
memset(&host, 0, sizeof(host));
host.name = (char*)"resync-large-host";
host.attrs = &attrs;
w = makeWorker(c.workermgr, &host);
}
TEST_ASSERT_NOT_NULL(w);
memset(&new_attrs, 0, sizeof(new_attrs));
new_attrs.hosttype = makeString("LINUX");
new_attrs.hostmodel = makeString("x86");
new_attrs.cpufactor = 1.0f;
new_attrs.maxcpus = 4;
new_attrs.maxmem = 16;
new_attrs.maxswap = 32;
new_attrs.maxtmp = 64;
new_attrs.ndisks = 1;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = 4;
memset(&stack_ep, 0, sizeof(stack_ep));
rc = rejoin_worker(&c, w, &stack_ep, &new_attrs, &hconfig);
TEST_ASSERT_GREATER_OR_EQUAL(0, rc);
TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
"a rejoin with TWR_LARGE_N_TOTAL known non-terminal jobs on "
"this host must arm a reconciliation round (Req 1.1)");
TEST_ASSERT_EQUAL_INT_MESSAGE(TWR_LARGE_N_TOTAL,
w->resync->n_queried_jobids,
"the round's query set must match exactly the host's non-"
"terminal jobs, with no artificial cap on how many resync_"
"arm_round will collect (Req 1.1, 2.1, 8.1)");
TEST_ASSERT_TRUE(IS_WORKER_RECONCILING(w->host.status));
ep = twr_rt_new_test_endpoint();
/* ── QUERY direction: a REAL wire-encoded xce_job_state_query_req()
* send, called directly (not via worker_send_resync_query(), which
* hardcodes pktlimit=0 -- see this test's own header comment and
* CONCERNS) with an explicit page size, genuinely producing
* multiple wire pages for this job count. */
query = (xce_job_state_query_t){
.n_total = w->resync->n_queried_jobids,
.pos = 0,
.count = w->resync->n_queried_jobids,
.jobids = w->resync->queried_jobids
};
TEST_ASSERT_EQUAL_INT(0,
xce_job_state_query_req(ep, c.myhost, &query, NULL, NULL,
TWR_LARGE_PKTLIMIT));
{
bool seen_query[TWR_LARGE_N_TOTAL];
int running_total = 0;
int query_pages = 0;
memset(seen_query, 0, sizeof(seen_query));
while (running_total < TWR_LARGE_N_TOTAL) {
xce_request_t decoded_query;
int j;
twr_rt_pop_and_decode(ep, &decoded_query);
TEST_ASSERT_EQUAL_INT(OPCode_jobStateQuery,
decoded_query.opcode);
TEST_ASSERT_EQUAL_INT(Request_body_jobstatequerymsg,
decoded_query.body_which);
TEST_ASSERT_NOT_NULL(decoded_query.body.jobstatequerymsg);
TEST_ASSERT_EQUAL_INT_MESSAGE(TWR_LARGE_N_TOTAL,
decoded_query.body.jobstatequerymsg->n_total,
"Req 2.1: every query page must carry the SAME full "
"n_total, not a per-page total");
TEST_ASSERT_EQUAL_INT_MESSAGE(running_total,
decoded_query.body.jobstatequerymsg->pos,
"Req 8.3: each page's pos must pick up exactly where "
"the previous page's count left off -- no gap, no "
"overlap");
for (j = 0; j < decoded_query.body.jobstatequerymsg->count;
j++) {
uint64_t jobid
= decoded_query.body.jobstatequerymsg->jobids[j];
uint64_t idx = jobid - TWR_JOBID_LARGE_BASE;
TEST_ASSERT_TRUE_MESSAGE(
idx < (uint64_t)TWR_LARGE_N_TOTAL,
"a jobid appearing on the wire query must be one "
"of the armed round's own jobids");
TEST_ASSERT_FALSE_MESSAGE(seen_query[idx],
"Req 8.3: no jobid may be sent twice across query "
"pages");
seen_query[idx] = true;
}
running_total
+= decoded_query.body.jobstatequerymsg->count;
query_pages++;
xce_free_request(&decoded_query);
}
TEST_ASSERT_EQUAL_INT_MESSAGE(TWR_LARGE_N_PAGES, query_pages,
"Req 8.1-8.3: an order-of-magnitude-larger-than-one-page "
"query must genuinely span many wire pages, including a "
"partial final page, not silently collapse to one (the "
"real production worker_send_resync_query() call site "
"would collapse to one -- see CONCERNS, bug-1755 -- this "
"assertion is against xce_job_state_query_req()'s own "
"pagination mechanism, exercised directly)");
for (i = 0; i < TWR_LARGE_N_TOTAL; i++) {
TEST_ASSERT_TRUE_MESSAGE(seen_query[i],
"Req 8.3: every armed jobid must appear exactly once "
"across all query pages -- none silently dropped past "
"a page boundary");
}
}
/* ── Worker's answer: a REAL wire-encoded-then-decoded OPCode_
* jobStateReport, sent at the REAL production report page size
* (XCECD_RESYNC_REPORT_PKTLIMIT, controller_connector.c) --
* genuinely many wire pages -- covering all four outcome kinds,
* fed page-by-page into resync_handle_report_page() (task 6.2's
* real reassembly entry point), never resync_apply_report()
* directly. */
storage = (xce_job_state_entry_t*)malloc(
sizeof(xce_job_state_entry_t) * TWR_LARGE_N_TOTAL);
entries = (xce_job_state_entry_t**)malloc(
sizeof(xce_job_state_entry_t*) * TWR_LARGE_N_TOTAL);
TEST_ASSERT_NOT_NULL(storage);
TEST_ASSERT_NOT_NULL(entries);
for (i = 0; i < TWR_LARGE_N_TOTAL; i++) {
xce_job_state_entry_t* e = &storage[i];
memset(e, 0, sizeof(*e));
e->jobid = twr_large_jobid(i);
e->queried = 1;
switch (twr_large_group(i)) {
case 0:
e->status = JobStateStatus_running;
break;
case 1: {
int local = i - TWR_LARGE_N_RUNNING;
uint64_t fin_s = finok_anchor_s - (uint64_t)(local * 3);
e->status = JobStateStatus_finishedOk;
e->finish_time_ms = fin_s * 1000ULL;
break;
}
case 2: {
int local = i - TWR_LARGE_N_RUNNING - TWR_LARGE_N_FINOK;
e->status = JobStateStatus_finishedFail;
e->exit_code = 200 + local;
break;
}
default:
e->status = JobStateStatus_unrecognized;
break;
}
entries[i] = e;
}
report = (xce_job_state_report_t){
.n_total = TWR_LARGE_N_TOTAL,
.pos = 0,
.count = TWR_LARGE_N_TOTAL,
.entries = entries
};
TEST_ASSERT_EQUAL_INT(0,
xce_job_state_report_req(ep, c.myhost, &report, NULL, NULL,
TWR_LARGE_PKTLIMIT));
g_twr_update_hook_fires = 0;
TEST_ASSERT_EQUAL_INT(0,
xce_add_hook(&c.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook, NULL));
pthread_mutex_lock(&c.persist.lock);
lsn_before = c.persist.current_lsn;
pthread_mutex_unlock(&c.persist.lock);
resync_apply_report_call_count_reset_for_test();
calls_before = resync_apply_report_call_count_for_test();
{
int running_total = 0;
int report_pages = 0;
while (running_total < TWR_LARGE_N_TOTAL) {
xce_request_t decoded_report;
int rc_page;
twr_rt_pop_and_decode(ep, &decoded_report);
TEST_ASSERT_EQUAL_INT(OPCode_jobStateReport,
decoded_report.opcode);
TEST_ASSERT_EQUAL_INT(Request_body_jobstatereportmsg,
decoded_report.body_which);
TEST_ASSERT_NOT_NULL(decoded_report.body.jobstatereportmsg);
running_total
+= decoded_report.body.jobstatereportmsg->count;
report_pages++;
rc_page = resync_handle_report_page(
&c, w, decoded_report.body.jobstatereportmsg);
if (running_total < TWR_LARGE_N_TOTAL) {
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc_page,
"a non-final report page must not complete the "
"report");
} else {
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc_page,
"the final (partial) report page must complete "
"the report");
}
xce_free_request(&decoded_report);
}
TEST_ASSERT_EQUAL_INT_MESSAGE(TWR_LARGE_N_PAGES, report_pages,
"Req 8.1-8.3: at the real production per-page limit, this "
"job count must genuinely span many wire pages, including "
"a partial final page");
}
calls_after = resync_apply_report_call_count_for_test();
TEST_ASSERT_EQUAL_INT_MESSAGE(1, calls_after - calls_before,
"resync_apply_report must be invoked EXACTLY ONCE for the "
"whole large, many-page report -- not once per page (task "
"6.2's own structural proof, generalized from 2 pages to "
"double digits)");
pthread_mutex_lock(&c.persist.lock);
lsn_after = c.persist.current_lsn;
pthread_mutex_unlock(&c.persist.lock);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
lsn_before
+ (uint64_t)(TWR_LARGE_N_FINOK + TWR_LARGE_N_FINFAIL
+ TWR_LARGE_N_UNREC),
lsn_after,
"EXACTLY one durable fact per non-running job (finished-"
"success x WAL_JOB_FINISH + finished-failure x WAL_JOB_FAIL + "
"unrecognized x WAL_JOB_UPDATE), and NONE for the still-"
"running jobs -- an exact count (not merely a lower bound) is "
"the genuine 'none dropped, none duplicated across page "
"boundaries' proof");
TEST_ASSERT_EQUAL_INT_MESSAGE(TWR_LARGE_N_UNREC,
g_twr_update_hook_fires,
"Req 7.1-7.3: the EAGER unrecognized-to-pending transition "
"must reach the scheduler's/apiserver's notify path "
"synchronously, exactly once per unrecognized job, regardless "
"of which page it happened to land on");
TEST_ASSERT_NULL_MESSAGE(w->resync,
"once every queried job across every report page has a "
"reconciled outcome, the round must resolve (Req 5.1, 5.2)");
TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
"XCE_WORKER_RECONCILING must clear once the round resolves");
/* Phase 1 end-to-end per-job assertions (Req 8.1-8.3: EVERY job
* accounted for, not a sample) -- everything observable before any
* commit round-trip. */
for (i = 0; i < TWR_LARGE_N_TOTAL; i++) {
switch (twr_large_group(i)) {
case 0:
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN,
jobs[i]->job.state,
"still-running (queried) must be left exactly as-is");
TEST_ASSERT_NOT_NULL(jobs[i]->job.allocs);
break;
case 1:
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN,
jobs[i]->job.state,
"finished-success stays DEFERRED (record_job_finish_"
"fact_at) -- mutates later via apply_committed_entry, "
"not synchronously here");
break;
case 2:
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN,
jobs[i]->job.state,
"finished-failure stays DEFERRED (on_job_fail/record_"
"job_fail_fact), same as finished-success");
break;
default:
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND,
jobs[i]->job.state,
"unrecognized reconciles EAGERLY -- PEND "
"synchronously, before any commit round-trip");
TEST_ASSERT_NULL_MESSAGE(jobs[i]->job.allocs,
"bug-1631: reconciled-to-pending must release the "
"allocation synchronously, at scale, across page "
"boundaries");
break;
}
}
xce_disable_hook(&c.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook);
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));
persist_stop(&c.persist);
persist_fini(&c.persist);
if (c.workermgr) dereference(c.workermgr);
if (c.resmgr) dereference(c.resmgr);
if (c.jobmgr) dereference(c.jobmgr);
xce_hook_fini(&c.hooksys);
dereference(new_attrs.hosttype);
dereference(new_attrs.hostmodel);
endpoint_fini(ep);
free(ep);
free(jobs);
free(storage);
free(entries);
}
/* Phase 2: fresh persist_t + fresh cluster_t (jobmgr/hooksys only),
* recover from disk -- simulates a controller restart, settling the two
* DEFERRED outcomes (finished-success/failure) across every page they
* happened to land on, and re-confirming the EAGER unrecognized->
* pending outcome also round-trips through a real WAL replay, not just
* the live eager-mutation path -- for EVERY one of the TWR_LARGE_N_
* TOTAL jobs, individually, not a sample. */
{
persist_t* p2 = twr_make_fresh_persist(tmp);
cluster_t c2;
job_t** jobs2;
int rc, i;
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 1024, .delta = 256 };
TEST_ASSERT_NOT_NULL(p2);
memset(&c2, 0, sizeof(c2));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(c2.jobmgr);
g_twr_update_hook_fires = 0;
TEST_ASSERT_EQUAL_INT(0,
xce_add_hook(&c2.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook, NULL));
p2->cluster = &c2;
rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
jobs2 = (job_t**)malloc(sizeof(job_t*) * TWR_LARGE_N_TOTAL);
TEST_ASSERT_NOT_NULL(jobs2);
lock(c2.jobmgr, RTTI_LOCK_READ);
for (i = 0; i < TWR_LARGE_N_TOTAL; i++) {
jobs2[i] = NULL;
key_find(c2.jobmgr, (void*)twr_large_jobid(i),
(void**)&jobs2[i]);
}
unlock(c2.jobmgr);
for (i = 0; i < TWR_LARGE_N_TOTAL; i++) {
TEST_ASSERT_NOT_NULL_MESSAGE(jobs2[i],
"restart replay must reconstruct every job via its own "
"WAL_JOB_SUBMIT fact");
switch (twr_large_group(i)) {
case 0:
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN,
jobs2[i]->job.state,
"still-running: no fact was ever recorded, so replay "
"leaves it exactly as WAL_JOB_SUBMIT alone "
"reconstructs it");
break;
case 1: {
int local = i - TWR_LARGE_N_RUNNING;
uint64_t fin_s = finok_anchor_s - (uint64_t)(local * 3);
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE,
jobs2[i]->job.state,
"finished-success: the DEFERRED record_job_finish_"
"fact_at fact must apply to DONE on replay");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(fin_s,
(uint64_t)jobs2[i]->job.finish_time,
"each finished-success job must recover ITS OWN "
"worker-reported finish_time -- no cross-job mixing "
"across a page boundary");
break;
}
case 2:
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_EXIT,
jobs2[i]->job.state,
"finished-failure: the DEFERRED on_job_fail/record_"
"job_fail_fact fact must apply to EXIT on replay");
break;
default:
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND,
jobs2[i]->job.state,
"unrecognized: the EAGER record_job_reconcile_"
"pending_fact's durable WAL_JOB_UPDATE fact must ALSO "
"reconstruct correctly on replay");
TEST_ASSERT_NULL(jobs2[i]->job.allocs);
break;
}
}
TEST_ASSERT_EQUAL_INT_MESSAGE(
TWR_LARGE_N_FINOK + TWR_LARGE_N_FINFAIL + TWR_LARGE_N_UNREC,
g_twr_update_hook_fires,
"Req 7.1-7.3: replaying the durable facts this reconciliation "
"round produced must reach the scheduler's/apiserver's notify "
"path for EVERY actual state change -- finished-success, "
"finished-failure, and unrecognized-to-pending -- exactly "
"once each, across every job (still-running recorded no fact, "
"and WAL_JOB_SUBMIT fires HOOK_DETAIL_JOB_NEW, not JOB_UPDATE, "
"so neither adds to this count)");
xce_disable_hook(&c2.hooksys, XCE_CONTROLLER_HOOK_JOB,
twr_job_update_spy_hook);
xce_hook_fini(&c2.hooksys);
if (c2.jobmgr) dereference(c2.jobmgr);
twr_teardown_persist(p2);
free(jobs2);
}
{
char cmd[512];
snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
if (system(cmd) != 0)
fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
}
printf("test_large_job_count_pagination_correctness: PASS\n");
}
/* ── Test group entry point ───────────────────────────────────────────── */
void test_worker_resync(void)
{
UnitySetTestFile(__FILE__);
RUN_TEST(test_reconcile_pending_applies_releases_allocs_and_notifies);
RUN_TEST(test_reconcile_pending_noop_on_already_pending_job);
RUN_TEST(test_reconcile_pending_rejects_terminal_job);
RUN_TEST(test_reconcile_assign_applies_running_no_allocs_and_notifies);
RUN_TEST(test_reconcile_assign_survives_restart_replay);
RUN_TEST(test_resync_arm_round_query_set_matches_exactly);
RUN_TEST(test_resync_arm_round_no_jobs_is_noop);
RUN_TEST(test_resync_arm_round_fresh_token_supersedes_prior);
RUN_TEST(test_resync_reconciling_flag_set_after_arm_cleared_after_resolve);
RUN_TEST(test_resync_resolve_round_notifies_worker_update);
RUN_TEST(test_resync_timeout_resolves_round_without_blocking);
RUN_TEST(test_apply_report_all_five_outcomes_updates_job_table);
RUN_TEST(test_apply_report_duplicate_reapplication_is_noop);
RUN_TEST(test_apply_report_stale_round_token_is_discarded);
RUN_TEST(test_apply_report_partial_report_does_not_resolve_round);
RUN_TEST(test_apply_report_unqueried_extra_already_tracked_is_noop);
RUN_TEST(
test_apply_report_finished_success_uses_worker_reported_finish_time);
RUN_TEST(
test_apply_report_unqueried_extra_unassigned_pend_job_gets_assigned);
RUN_TEST(test_handle_report_page_single_page_baseline);
RUN_TEST(test_handle_report_page_multi_page_reassembles_and_applies_once);
RUN_TEST(test_handle_report_page_stale_round_discarded_when_superseded);
RUN_TEST(test_handle_report_page_malformed_bounds_rejected_and_recovers);
RUN_TEST(test_full_rejoin_reconciliation_round_trip);
RUN_TEST(test_failover_during_job_recovers_lost_finish);
RUN_TEST(test_burst_recovery_unbounded_by_job_count);
RUN_TEST(test_large_job_count_pagination_correctness);
}
Reference in a new issue View git blame Copy permalink