lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(controller): extract job-submit commit barrier out of the hook fan-out — stop committed-job-loss (bug-1306)

Browse source 
raft-apply-decoupling item #1 (down payment). xce_run_hooks returns only the
LAST-registered hook's rc, so a later JOB AFTER_EVENT subscriber (sched/
apiserver) returning 0 clobbered persist_job_hook_fun's commit -ETIMEDOUT.
handle_jobsubmit then replied 'jobsubmit ok' for an entry that never reached
majority, and a new-term leader legally discarded it (committed-job-loss,
Commit Durability violation).

- persist.c/.h: new persist_submit_wal_and_commit(p, jlist) — the explicit raft
  step (WAL append + majority-commit barrier) returning the commit verdict
  directly. persist_job_hook_fun's WAL_MODE_WAIT branch is now a no-op.
- job.c handle_jobsubmit: call the explicit step, read commit_rc DIRECTLY,
  reject on <0 before firing the now notification-only AFTER hook
  (canonical commit -> notify -> reply).
- tests: +test_job_hook_new_is_notify_only regression pin; 4 hook tests
  relocated to drive the explicit step. persist_hooks 19/0, e2e_hooks 3/0,
  commit_barrier_reply 22/0, commit_durability 38/0.

Proven: 10-round chaotic soak committed_job_loss 5 -> 0 (same election code).
Residual soak FAILs (divergence catch-up lag + a round-4 election-escalation
tail) are NOT job loss — tracked as task 5.10.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JDYFbrgyDHbvNYcxApKhrX
This commit is contained in:
Rongsong Shen 2026-07-01 00:42:30 +00:00
parent 6bb9d81225
commit 15be5d94e8
8 changed files with 1592 additions and 2495 deletions
Download patch file Download diff file Expand all files Collapse all files

11
src/controller/testsuite/test_e2e_hooks.c
View file

@ -130,11 +130,11 @@ static void test_job_hook_writes_wal_before_lsn_advances(void)
uint64_t lsn_before = p->current_lsn;
/* JOB_NEW args: (cluster_t*, xlist_t* jlist) — list must have ≥1 job */
/* Submit durability moved out of the JOB_NEW hook into the explicit raft
* step persist_submit_wal_and_commit (raft-apply-decoupling item #1 /
* bug-1306); the hook is now notification-only. */
STUB_JOB_XLIST(jlist);
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_NEW, p,
(cluster_t*)NULL, &jlist);
int rc = persist_submit_wal_and_commit(p, &jlist);
TEST_ASSERT_EQUAL_INT(0, rc);
/* LSN must have advanced by exactly 1 */
@ -197,8 +197,7 @@ static void test_hook_sequence_preserves_lsn_order(void)
uint64_t lsn0 = p->current_lsn;
STUB_JOB_XLIST(jlist);
call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_NEW, p,
(cluster_t*)NULL, &jlist);
persist_submit_wal_and_commit(p, &jlist); /* JOB_NEW via explicit raft step */
uint64_t lsn1 = p->current_lsn;
call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_START, p,

43
src/controller/testsuite/test_persist_hooks.c
View file

@ -106,11 +106,12 @@ static void test_job_hook_new_writes_wal_entry(void)
uint64_t lsn_before = p->current_lsn;
/* JOB_NEW args: (cluster_t*, xlist_t* jlist) — list must have ≥1 job */
/* Submit durability moved OUT of the JOB_NEW hook into the explicit raft
* step persist_submit_wal_and_commit (raft-apply-decoupling item #1 /
* bug-1306) — the hook is now notification-only, so the WAL-append is
* asserted against the explicit step. */
STUB_JOB_XLIST(jlist);
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_NEW, p,
(cluster_t*)NULL, &jlist);
int rc = persist_submit_wal_and_commit(p, &jlist);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
@ -118,6 +119,33 @@ static void test_job_hook_new_writes_wal_entry(void)
printf("test_job_hook_new_writes_wal_entry: PASS\n");
}
/* Regression pin (bug-1306 / raft-apply-decoupling item #1): the JOB_NEW hook
* must be NOTIFICATION-ONLY — it must NOT perform the WAL append / commit
* barrier. If durability lived in the hook, xce_run_hooks (which returns only
* the last hook's rc) could let a later notify subscriber's 0 clobber a commit
* -ETIMEDOUT, acking the client for an uncommitted entry. Asserting the hook
* appends NOTHING structurally forecloses that clobber. */
static void test_job_hook_new_is_notify_only(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
STUB_JOB_XLIST(jlist);
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_NEW, p,
(cluster_t*)NULL, &jlist);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before, p->current_lsn,
"JOB_NEW hook must be notify-only — durability moved to "
"persist_submit_wal_and_commit (bug-1306); a hook that still appends "
"re-opens the xce_run_hooks last-rc clobber");
cleanup_hook_test_persist(p);
printf("test_job_hook_new_is_notify_only: PASS\n");
}
static void test_job_hook_start_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
@ -406,10 +434,10 @@ static void test_job_hook_new_writes_multiple_entries_sequentially(void)
uint64_t lsn_before = p->current_lsn;
/* Fire three hook calls — each must advance LSN */
/* JOB_NEW submit durability goes through the explicit raft step; START/EXIT
* facts still flow through the hook (RETAIN). All three advance LSN. */
STUB_JOB_XLIST(jlist);
call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_NEW, p,
(cluster_t*)NULL, &jlist);
persist_submit_wal_and_commit(p, &jlist);
call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_START, p,
(cluster_t*)NULL, (void*)NULL, &g_stub_job);
call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_EXIT, p,
@ -430,6 +458,7 @@ void test_persist_hooks(void)
printf("\n>> persist hook function tests\n\n");
RUN_TEST(test_job_hook_new_writes_wal_entry);
RUN_TEST(test_job_hook_new_is_notify_only);
RUN_TEST(test_job_hook_start_writes_wal_entry);
RUN_TEST(test_job_hook_exit_writes_wal_entry);
RUN_TEST(test_job_hook_fail_writes_wal_entry);