lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(controller): persist a failed job's finish_time through WAL_JOB_FAIL (bug-1747)

Browse source 
The FAIL half of bug-1747: apply_job_fail_transition() stamps
finish_time = time(NULL) at APPLY time, and WAL_JOB_FAIL carried no time,
so a failed job's finish_time changed on every restart replay / standby
catch-up, and a worker-reported failure found on rejoin could never keep
the worker's own time.

- WAL_JOB_FAIL's trailer grows to int32_t code + uint64_t finish_time
  (seconds). Decoded by length: 16 B (pre-bug-1754), 20 B (code only,
  bug-1754) and 28 B (code + time) all apply as before for what they carry.
- record_job_fail_fact_at(c, job, failinfo, finish_time_ms), the fail-side
  sibling of record_job_finish_fact_at(): 0 = receipt time. on_job_fail()
  uses receipt time; resync's finishedFail branch passes the worker-
  reported finish_time_ms (Req 4.2 now holds for failures too).
- Both apply seams adopt the wire time after the transition.

4 new tests in test_wal_job_fail_fact.c (3 RED before the fix) and the
resync round-trip now asserts the worker-reported failure time (RED
before). Controller suite 1232/0. Factstream HA soak (2 rounds, 60 jobs)
PASS.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-09-26 09:33:55 +00:00
parent 21e72726f0
commit 1d6c04c28a
Signed by untrusted user: rshen
GPG key ID: 80533536B0EB651C
9 changed files with 349 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

48
src/controller/persist.c
View file

@ -4123,6 +4123,7 @@ int apply_wal_entry(persist_t* p, uint32_t type, uint64_t lsn,
& WAL_JOB_FAIL_STAGE_FLAG)) {
const wal_jobstate_t* js = (const wal_jobstate_t*)payload;
int32_t wire_code = 0;
uint64_t wire_time = 0;
bool have_code = payload_len
>= sizeof(wal_jobstate_t) + sizeof(wire_code);
job_t* j = NULL;
@ -4131,6 +4132,12 @@ int apply_wal_entry(persist_t* p, uint32_t type, uint64_t lsn,
memcpy(&wire_code,
(const uint8_t*)payload + sizeof(wal_jobstate_t),
sizeof(wire_code));
if (payload_len >= sizeof(wal_jobstate_t) + sizeof(wire_code)
+ sizeof(wire_time))
memcpy(&wire_time,
(const uint8_t*)payload + sizeof(wal_jobstate_t)
+ sizeof(wire_code),
sizeof(wire_time));
lock(p->cluster->jobmgr, RTTI_LOCK_READ);
key_find(p->cluster->jobmgr, (void*)js->jobid, (void**)&j);
@ -4145,9 +4152,12 @@ int apply_wal_entry(persist_t* p, uint32_t type, uint64_t lsn,
lock(j, RTTI_LOCK_WRITE);
if (apply_job_fail_transition(j, &failinfo)
== JOB_TRANSITION_APPLIED
&& have_code)
apply_job_fail_exit_code(j, &failinfo);
== JOB_TRANSITION_APPLIED) {
if (have_code)
apply_job_fail_exit_code(j, &failinfo);
if (wire_time != 0)
j->job.finish_time = (time_t)wire_time;
}
unlock(j);
}
break;
@ -5046,16 +5056,26 @@ int apply_committed_entry(persist_t* p, uint32_t wal_type, uint64_t lsn,
job_failinfo_t failinfo;
bool have_failinfo = false;
if (js->_pad & WAL_JOB_FAIL_STAGE_FLAG) {
int32_t wire_code = 0;
bool have_code = len >= sizeof(wal_jobstate_t)
+ sizeof(wire_code);
int32_t wire_code = 0;
uint64_t wire_time = 0;
bool have_code = len >= sizeof(wal_jobstate_t)
+ sizeof(wire_code);
bool have_time = len >= sizeof(wal_jobstate_t)
+ sizeof(wire_code)
+ sizeof(wire_time);
/* bug-1754: persist_submit_job_fail_fact()'s int32_t code
* trailer; absent on entries written before the fix. */
/* bug-1754/bug-1747: persist_submit_job_fail_fact()'s int32_t
* code + uint64_t finish_time trailer; older entries carry
* neither (pre-bug-1754) or only the code (bug-1754). */
if (have_code)
memcpy(&wire_code,
(const uint8_t*)payload + sizeof(wal_jobstate_t),
sizeof(wire_code));
if (have_time)
memcpy(&wire_time,
(const uint8_t*)payload + sizeof(wal_jobstate_t)
+ sizeof(wire_code),
sizeof(wire_time));
failinfo = (job_failinfo_t){
.jobid = js->jobid,
@ -5067,6 +5087,8 @@ int apply_committed_entry(persist_t* p, uint32_t wal_type, uint64_t lsn,
result = apply_job_fail_transition(j, &failinfo);
if (have_code && result == JOB_TRANSITION_APPLIED)
apply_job_fail_exit_code(j, &failinfo);
if (wire_time != 0 && result == JOB_TRANSITION_APPLIED)
j->job.finish_time = (time_t)wire_time;
} else {
result = apply_job_update_transition(j, (int)js->state);
}
@ -8115,18 +8137,20 @@ int persist_submit_job_finish_fact(persist_t* p, uint64_t jobid, int state,
payload, (uint32_t)sizeof(payload));
}
/* bug-1754: see persist.h's doc comment. Same native fixed-size trailer
* style as persist_submit_job_finish_fact() above. */
/* bug-1754/bug-1747: see persist.h's doc comment. Same native fixed-size
* trailer style as persist_submit_job_finish_fact() above. */
int persist_submit_job_fail_fact(persist_t* p, uint64_t jobid, uint32_t stage,
int code)
int code, uint64_t finish_time)
{
wal_jobstate_t js = { .jobid = jobid, .state = 0,
._pad = WAL_JOB_FAIL_STAGE_FLAG | stage };
int32_t c32 = (int32_t)code;
uint8_t payload[sizeof(js) + sizeof(c32)];
uint8_t payload[sizeof(js) + sizeof(c32) + sizeof(finish_time)];
memcpy(payload, &js, sizeof(js));
memcpy(payload + sizeof(js), &c32, sizeof(c32));
memcpy(payload + sizeof(js) + sizeof(c32), &finish_time,
sizeof(finish_time));
return wal_hook_append_classified(p, WAL_JOB_FAIL, WAL_MODE_RETAIN,
payload, (uint32_t)sizeof(payload));