lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(controller): job.start_time never set -- dispatch/WAL_JOB_START RUN-state collision (bug-1697)

Browse source 
dispatch_a_job_() synchronously sets state=JOB_STAT_RUN at dispatch time
(scheduler-dispatch-commit task 3.1's deliberate no-worker-ack-wait
semantic). By the time the worker's later WAL_JOB_START fact commits,
apply_job_update_transition() sees state already == RUN and returns
NOOP_DUPLICATE instead of APPLIED -- so bug-1687's start_time/pidinfo
decode/adopt block, gated on result==APPLIED, could never run for any
job. start_time (and consequently run_time) stayed 0 forever. Same bug
class already fixed for WAL_JOB_RUSAGE/CKPT/PREASON in this identical
apply function; WAL_JOB_START was missed because bug-1687 predated the
dispatch-time RUN semantic change.

Moves start_time to dispatch-owned: dispatch_a_job_() now stamps
job.start_time alongside state=RUN, under the same write lock.
WAL_JOB_DISPATCH gains a flag-gated (WAL_JOB_DISPATCH_HAS_START_TIME_FLAG)
start_time field so it survives replay/restart; both apply seams
(apply_committed_entry, apply_wal_entry) and restore_job_dispatch_state
decode it when present, falling through to the old alloc-only shape
otherwise. WAL_JOB_START's own now-permanently-dead start_time write is
removed; pidinfo adoption made unconditional to match.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AgYRU3zDTqNGV5Jk8aiKLg
This commit is contained in:
Rongsong Shen 2026-08-27 08:25:40 +00:00
parent faedc6fc49
commit ac1e312e2f
No known key found for this signature in database
GPG key ID: 5BCDAAE0123E9FE2
9 changed files with 859 additions and 184 deletions
Download patch file Download diff file Expand all files Collapse all files

68
src/controller/testsuite/test_dispatch_commit.c
View file

@ -274,6 +274,60 @@ static void test_dc_success_commits_alloc_state_and_resets_counter(void)
printf("test_dc_success_commits_alloc_state_and_resets_counter: PASS\n");
}
#define DC_JOBID_START_TIME 91005ULL
/* bug-1697's core RED/GREEN proof for the "start_time is dispatch-owned
* now" half of the fix: a successful dispatch must stamp job.start_time
* synchronously, in the SAME lock-held operation that sets job.state =
* JOB_STAT_RUN and commits job.allocs -- no WAL commit/apply round trip
* needed, since this is a plain in-memory write under the job's own write
* lock. RED against the pre-fix code: j->job.start_time stays 0 forever
* (dispatch_a_job_() never touched it, and WAL_JOB_START's own adoption is
* permanently dead per bug-1697's root cause, so nothing else ever sets
* it either). GREEN after the fix: job.start_time is nonzero immediately
* after dispatch_a_job_test() returns. */
static void test_dc_success_stamps_start_time_synchronously(void)
{
job_t* j;
xce_job_alloc_t* alloc;
int rc;
time_t before, after;
dc_setup();
j = dc_seed_ready(DC_JOBID_START_TIME, 0, "dq", "wkr-dc-start-time");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, (uint64_t)j->job.start_time,
"fixture must seed a job with start_time still at its zero-"
"initialized default");
alloc = make_job_alloc(DC_JOBID_START_TIME, 1,
make_host_allocs(1,
make_host_alloc((char*)"wkr-dc-start-time", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(alloc);
before = time(NULL);
rc = dispatch_a_job_test(&g_dc, NULL, alloc);
after = time(NULL);
TEST_ASSERT_TRUE_MESSAGE(rc >= 0,
"a dispatch to a live, no-iomgr worker endpoint must succeed");
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
TEST_ASSERT_NOT_EQUAL_MESSAGE(0, (uint64_t)j->job.start_time,
"bug-1697: a successful dispatch must stamp job.start_time "
"synchronously -- it must never be left at 0");
TEST_ASSERT_TRUE_MESSAGE(
(j->job.start_time >= before) && (j->job.start_time <= after),
"job.start_time must be the dispatch-time clock, captured within "
"this call's own [before, after] window");
free_JobAlloc_ptr(&alloc);
dc_drain_ep(&g_dc_worker_ep);
dc_teardown();
printf("test_dc_success_stamps_start_time_synchronously: PASS\n");
}
/* bug-1679: spy hook counting HOOK_AFTER_EVENT/HOOK_DETAIL_JOB_UPDATE fires
* specifically -- the exact hook apiserver_job_update_fun (apiserver.c)
* subscribes to in production to push a job's current state to every
@ -506,7 +560,9 @@ static int dc_encode_job(const xce_job_t* j, uint8_t** out, size_t* out_len)
static void test_dc_success_appends_wal_job_dispatch_durably(void)
{
char tmp[256];
char tmp[256];
time_t captured_start_time = 0; /* bug-1697: set in Phase 1, checked
* against Phase 2's recovered value */
strcpy(tmp, "/tmp/xce_dc31_replay_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
@ -637,6 +693,10 @@ static void test_dc_success_appends_wal_job_dispatch_durably(void)
rc = dispatch_a_job_test(&c, NULL, alloc);
TEST_ASSERT_TRUE(rc >= 0);
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
TEST_ASSERT_NOT_EQUAL_MESSAGE(0, (uint64_t)j->job.start_time,
"bug-1697: a successful dispatch must stamp job.start_time "
"synchronously");
captured_start_time = j->job.start_time;
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));
@ -683,6 +743,11 @@ static void test_dc_success_appends_wal_job_dispatch_durably(void)
"restart replay must recover the job as RUNNING from the "
"WAL_JOB_DISPATCH fact appended by dispatch_a_job_()'s real "
"success path -- no WAL_JOB_START anywhere in this log");
TEST_ASSERT_EQUAL_UINT64_MESSAGE((uint64_t)captured_start_time,
(uint64_t)j2->job.start_time,
"bug-1697: restart replay must recover the exact dispatch-time "
"start_time stamped by the real dispatch_a_job_() call in "
"Phase 1, from the WAL_JOB_DISPATCH fact's extended payload");
/* bug-1678: restart replay must recover not just STATE but the
* resource allocation itself, or EXEC_HOST is permanently blank for
@ -731,6 +796,7 @@ void test_dispatch_commit(void)
"5.1)\n\n");
RUN_TEST(test_dc_success_commits_alloc_state_and_resets_counter);
RUN_TEST(test_dc_success_stamps_start_time_synchronously);
RUN_TEST(test_dc_success_notifies_job_update_hook_exactly_once);
RUN_TEST(test_dc_worker_not_found_leaves_job_pend_and_alloc_uncommitted);
RUN_TEST(test_dc_success_appends_wal_job_dispatch_durably);