fix(controller): post-purge job facts must be benign skips, not fence-engaging failures (bug-1970)
A committed job-state fact (WAL_JOB_UPDATE/FAIL/START/...) whose jobid no
longer exists in the jobmgr used to return -ENOENT from
apply_committed_entry(). Both of its callers (replay_wal during boot,
persist_apply_pump_run live) halt-and-fence on rc < 0, engaging the Live
Apply Resource Fence. A finished job is purged from the live table once
finished_retain_s expires, but its committed WAL entries remain; a boot
replay walks the full log and re-hits a late post-finish UPDATE for the
purged jobid on EVERY restart. No in-memory state change can ever make a
purged job re-appear, so the fence can never self-heal: the controller
logs "promotion FENCED -- durable_floor < cluster HWM=0" every 30s
forever, rejects all component joins ("standby has no primary yet"), and
cannot serve. Observed live on the t1 rollout (2026-10-21): the old
primary had logged the same entry as "duplicate UPDATE to state 0x4 --
no-op" while the job was still in memory; the first restart after the
job's purge tripped the fence. Pre-existing since the July 2026
apply_committed_entry dispatch table (2fa955b9/387ea35c) -- exposed by
any controller restart, not a de-nix regression.
Fix: all three case groups (shared START/FINISH/STOP/RESUME/RUSAGE/CKPT/
PREASON/DISPATCH group, WAL_JOB_FAIL, WAL_JOB_UPDATE) now return 0 for a
missing job, mirroring apply_wal_entry()'s existing behavior on the same
seam (restore_job_state()'s rc discarded; the FAIL branch guards on
j != NULL) and this function's own WAL_QUEUE_CTRL benign-skip precedent.
Genuine construction/ENOMEM failures keep their fence semantics.
Also:
- test_apply_committed_entry: three new tests covering the skip for all
three groups (incl. the POSTDONE-mark variant), asserting rc == 0 and
no hook fires.
- lib/meson.build: ASAN_OPTIONS=detect_leaks=0 on the capnp generation
command. capnpc-c deliberately never frees its compile-time structures
(one-shot tool); under the de-nix asan profile the wrap-built generator
carries -fsanitize=address and LSAN's end-of-process check failed the
generation step. The legacy nix build never propagated -fsanitize to
the subproject. Same exemption convention as every other one-shot/
never-torn-down boundary in this repo.
- xce-package.sh: ALWAYS-bundle name list (libcyaml/libyaml/libtirpc/
libcgroup/libhwloc/libedit) alongside the ldd-location rule -- libcyaml
and libcgroup proved missing on the t2/t3 Ubuntu 24.04 runtime hosts
even though the build host resolves them from /usr/lib.
Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
lei.chen
parent
50cf424146
commit
6bf26cbab3
4 changed files with 178 additions and 10 deletions
|
|
@ -648,6 +648,116 @@ static void test_ace_job_fail_guard_rejected_fires_no_hook(void)
|
|||
printf("test_ace_job_fail_guard_rejected_fires_no_hook: PASS\n");
|
||||
}
|
||||
|
||||
/* ── bug-1970: post-purge stale facts must be benign skips ───────────────
|
||||
* A committed job-state fact whose jobid no longer exists in the jobmgr
|
||||
* (finished-retention purge removed the job, but its WAL entries remain)
|
||||
* must apply as a no-op SUCCESS (rc == 0, no hook), never a negative
|
||||
* return. The pre-fix -ENOENT propagated to apply_committed_entry()'s
|
||||
* callers, and both (replay_wal during boot, persist_apply_pump_run live)
|
||||
* halt-and-fence on rc < 0 -- engaging the Live Apply Resource Fence.
|
||||
* Unlike a genuine resource exhaustion, NO in-memory state change can ever
|
||||
* make a purged jobid re-appear, so the fence could never self-heal:
|
||||
* observed live as a controller refusing promotion forever after any
|
||||
* restart whose WAL held a post-purge fact ("durable_floor < cluster
|
||||
* HWM=0"). These tests cover all three case groups that carried the
|
||||
* -ENOENT: the shared START/.../DISPATCH group, WAL_JOB_FAIL, and
|
||||
* WAL_JOB_UPDATE. */
|
||||
|
||||
static void test_ace_job_fact_missing_job_is_benign_skip_dispatch_group(void)
|
||||
{
|
||||
tace_setup();
|
||||
|
||||
/* No job seeded -- jobid 701 was "purged". Apply a late RUSAGE-style
|
||||
* fact for it through the shared group (WAL_JOB_START is in the same
|
||||
* group; RUSAGE's trailing-decode block is also keyed on wal_type and
|
||||
* must simply not run). */
|
||||
wal_jobstate_t js = { .jobid = 701, .state = JOB_STAT_RUN };
|
||||
|
||||
xce_add_hook(&g_tace_c.hooksys, XCE_CONTROLLER_HOOK_JOB, tace_spy_hook,
|
||||
NULL);
|
||||
|
||||
int rc = apply_committed_entry(&g_tace_p, WAL_JOB_START, 27, &js,
|
||||
sizeof(js));
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
|
||||
"a committed fact for a purged jobid must be a benign skip (rc==0), "
|
||||
"never a fence-engaging negative (bug-1970)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_tace_hook_fires,
|
||||
"a skip must fire NO notification -- there is no job to notify "
|
||||
"about");
|
||||
|
||||
tace_teardown();
|
||||
printf("test_ace_job_fact_missing_job_is_benign_skip_dispatch_group: "
|
||||
"PASS\n");
|
||||
}
|
||||
|
||||
static void test_ace_job_fail_missing_job_is_benign_skip(void)
|
||||
{
|
||||
tace_setup();
|
||||
|
||||
/* STAGE_FLAG fail-fact for a purged jobid: apply_wal_entry()'s own
|
||||
* FAIL branch already guards on j != NULL and proceeds (returns 0) --
|
||||
* apply_committed_entry() must behave identically. */
|
||||
wal_jobstate_t js = { .jobid = 702,
|
||||
.state = 0,
|
||||
._pad = WAL_JOB_FAIL_STAGE_FLAG |
|
||||
JOB_EXEC_STAGE_POST_EXEC };
|
||||
|
||||
xce_add_hook(&g_tace_c.hooksys, XCE_CONTROLLER_HOOK_JOB,
|
||||
tace_faildetail_spy_hook, NULL);
|
||||
|
||||
int rc = apply_committed_entry(&g_tace_p, WAL_JOB_FAIL, 28, &js,
|
||||
sizeof(js));
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
|
||||
"a committed FAIL fact for a purged jobid must be a benign skip "
|
||||
"(rc==0), never a fence-engaging negative (bug-1970)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_tace_hook_fires,
|
||||
"a skip must fire NEITHER HOOK_DETAIL_JOB_FAIL nor JOB_UPDATE");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(-1, g_tace_last_fail_jobid,
|
||||
"no fail-detail fire means no job_failinfo_t was fabricated");
|
||||
|
||||
tace_teardown();
|
||||
printf("test_ace_job_fail_missing_job_is_benign_skip: PASS\n");
|
||||
}
|
||||
|
||||
static void test_ace_job_update_missing_job_is_benign_skip(void)
|
||||
{
|
||||
tace_setup();
|
||||
|
||||
/* The case observed live (bug-1970's actual trigger): a duplicate
|
||||
* post-finish UPDATE for a job purged between the old primary's
|
||||
* in-memory no-op logging and the next restart's replay. _pad == 0
|
||||
* (legacy absolute-state shape, as persist_submit_job_fact writes). */
|
||||
wal_jobstate_t js = { .jobid = 703, .state = JOB_STAT_EXIT, ._pad = 0 };
|
||||
|
||||
xce_add_hook(&g_tace_c.hooksys, XCE_CONTROLLER_HOOK_JOB, tace_spy_hook,
|
||||
NULL);
|
||||
|
||||
int rc = apply_committed_entry(&g_tace_p, WAL_JOB_UPDATE, 29, &js,
|
||||
sizeof(js));
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
|
||||
"a committed UPDATE for a purged jobid must be a benign skip "
|
||||
"(rc==0) -- the pre-fix -ENOENT here fenced a live controller out "
|
||||
"of promotion forever (bug-1970)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_tace_hook_fires,
|
||||
"a skip must fire NO notification");
|
||||
|
||||
/* The POSTDONE-mark variant must skip identically (same missing-job
|
||||
* lookup precedes the _pad dispatch). */
|
||||
wal_jobstate_t pd = { .jobid = 704,
|
||||
.state = 0,
|
||||
._pad = WAL_JOB_UPDATE_POSTDONE_MARK };
|
||||
rc = apply_committed_entry(&g_tace_p, WAL_JOB_UPDATE, 30, &pd,
|
||||
sizeof(pd));
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
|
||||
"a committed POSTDONE-mark UPDATE for a purged jobid must also be "
|
||||
"a benign skip (bug-1970)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_tace_hook_fires,
|
||||
"the POSTDONE skip must fire NO notification either");
|
||||
|
||||
tace_teardown();
|
||||
printf("test_ace_job_update_missing_job_is_benign_skip: PASS\n");
|
||||
}
|
||||
|
||||
/* ── WAL_QUEUE_CTRL ───────────────────────────────────────────────────── */
|
||||
|
||||
static void test_ace_queue_ctrl_updates_and_notifies(void)
|
||||
|
|
@ -1293,6 +1403,9 @@ void test_apply_committed_entry(void)
|
|||
RUN_TEST(test_ace_job_fail_stage_flag_fires_faildetail_and_update);
|
||||
RUN_TEST(test_ace_job_fail_legacy_pad_zero_fires_update_only);
|
||||
RUN_TEST(test_ace_job_fail_guard_rejected_fires_no_hook);
|
||||
RUN_TEST(test_ace_job_fact_missing_job_is_benign_skip_dispatch_group);
|
||||
RUN_TEST(test_ace_job_fail_missing_job_is_benign_skip);
|
||||
RUN_TEST(test_ace_job_update_missing_job_is_benign_skip);
|
||||
RUN_TEST(test_ace_queue_ctrl_updates_and_notifies);
|
||||
RUN_TEST(test_ace_worker_ctrl_join_restores_and_notifies);
|
||||
RUN_TEST(test_ace_worker_ctrl_update_applies_to_existing_worker);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue