diff --git a/src/controller/persist.c b/src/controller/persist.c index 4b3ebb12..fc79a42c 100644 --- a/src/controller/persist.c +++ b/src/controller/persist.c @@ -4798,8 +4798,27 @@ int apply_committed_entry(persist_t* p, uint32_t wal_type, uint64_t lsn, lock(c->jobmgr, RTTI_LOCK_READ); key_find(c->jobmgr, (void*)js->jobid, (void**)&j); unlock(c->jobmgr); - if (!j) - return -ENOENT; + if (!j) { + /* bug-1970: a committed fact for a jobid this node's jobmgr no + * longer holds is a benign stale fact, not an apply failure -- + * return 0 (skip) rather than -ENOENT. A finished job is purged + * from the live table once finished_retain_s expires, but its + * already-committed WAL entries (e.g. a late RUSAGE/UPDATE) + * remain in the log forever; boot replay and standby catch-up + * walk the FULL log, so they will re-hit such entries on every + * restart. Returning a negative here engages the Live Apply + * Resource Fence (this dispatch's callers halt the walk and + * fence on rc < 0), and because no in-memory state change can + * ever make the job re-appear, the fence can never self-heal -- + * observed live as a controller that refuses promotion forever + * ("durable_floor < cluster HWM=0") after any restart whose WAL + * contains a post-purge fact. This mirrors apply_wal_entry()'s + * existing behavior on this same seam (restore_job_state()'s + * rc is discarded; the FAIL branch guards on j != NULL) and + * restore_queue_state's -ENOENT being a benign skip in this + * function's own WAL_QUEUE_CTRL case below. */ + return 0; + } lock(j, RTTI_LOCK_WRITE); job_transition_result_t result = @@ -5186,8 +5205,15 @@ int apply_committed_entry(persist_t* p, uint32_t wal_type, uint64_t lsn, lock(c->jobmgr, RTTI_LOCK_READ); key_find(c->jobmgr, (void*)js->jobid, (void**)&j); unlock(c->jobmgr); - if (!j) - return -ENOENT; + if (!j) { + /* bug-1970: benign stale fact, not an apply failure -- see the + * full rationale in this function's shared START/.../DISPATCH + * group above (a post-purge committed fact must be a skip, + * never a fence-engaging negative return). Mirrors + * apply_wal_entry()'s own WAL_JOB_FAIL branch, which guards the + * whole merge on j != NULL and proceeds (returns 0) otherwise. */ + return 0; + } lock(j, RTTI_LOCK_WRITE); job_transition_result_t result; @@ -5287,8 +5313,19 @@ int apply_committed_entry(persist_t* p, uint32_t wal_type, uint64_t lsn, lock(c->jobmgr, RTTI_LOCK_READ); key_find(c->jobmgr, (void*)js->jobid, (void**)&j); unlock(c->jobmgr); - if (!j) - return -ENOENT; + if (!j) { + /* bug-1970: benign stale fact, not an apply failure -- see the + * full rationale in this function's shared START/.../DISPATCH + * group above. This is the case observed live: a duplicate + * post-finish UPDATE for a job whose finished-retention purge + * has already removed it (the pre-restart primary logged the + * same entry as "duplicate UPDATE -- no-op" while the job was + * still in memory; after a restart the job is gone, and the + * pre-fix -ENOENT here fenced the node out of promotion + * forever). The live-path sibling of this skip already exists: + * apply_job_update_transition's own NOOP_DUPLICATE logging. */ + return 0; + } lock(j, RTTI_LOCK_WRITE); job_transition_result_t result = diff --git a/src/controller/testsuite/test_apply_committed_entry.c b/src/controller/testsuite/test_apply_committed_entry.c index 57c40fd3..ab65044a 100644 --- a/src/controller/testsuite/test_apply_committed_entry.c +++ b/src/controller/testsuite/test_apply_committed_entry.c @@ -648,6 +648,116 @@ static void test_ace_job_fail_guard_rejected_fires_no_hook(void) printf("test_ace_job_fail_guard_rejected_fires_no_hook: PASS\n"); } +/* ── bug-1970: post-purge stale facts must be benign skips ─────────────── + * A committed job-state fact whose jobid no longer exists in the jobmgr + * (finished-retention purge removed the job, but its WAL entries remain) + * must apply as a no-op SUCCESS (rc == 0, no hook), never a negative + * return. The pre-fix -ENOENT propagated to apply_committed_entry()'s + * callers, and both (replay_wal during boot, persist_apply_pump_run live) + * halt-and-fence on rc < 0 -- engaging the Live Apply Resource Fence. + * Unlike a genuine resource exhaustion, NO in-memory state change can ever + * make a purged jobid re-appear, so the fence could never self-heal: + * observed live as a controller refusing promotion forever after any + * restart whose WAL held a post-purge fact ("durable_floor < cluster + * HWM=0"). These tests cover all three case groups that carried the + * -ENOENT: the shared START/.../DISPATCH group, WAL_JOB_FAIL, and + * WAL_JOB_UPDATE. */ + +static void test_ace_job_fact_missing_job_is_benign_skip_dispatch_group(void) +{ + tace_setup(); + + /* No job seeded -- jobid 701 was "purged". Apply a late RUSAGE-style + * fact for it through the shared group (WAL_JOB_START is in the same + * group; RUSAGE's trailing-decode block is also keyed on wal_type and + * must simply not run). */ + wal_jobstate_t js = { .jobid = 701, .state = JOB_STAT_RUN }; + + xce_add_hook(&g_tace_c.hooksys, XCE_CONTROLLER_HOOK_JOB, tace_spy_hook, + NULL); + + int rc = apply_committed_entry(&g_tace_p, WAL_JOB_START, 27, &js, + sizeof(js)); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, + "a committed fact for a purged jobid must be a benign skip (rc==0), " + "never a fence-engaging negative (bug-1970)"); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_tace_hook_fires, + "a skip must fire NO notification -- there is no job to notify " + "about"); + + tace_teardown(); + printf("test_ace_job_fact_missing_job_is_benign_skip_dispatch_group: " + "PASS\n"); +} + +static void test_ace_job_fail_missing_job_is_benign_skip(void) +{ + tace_setup(); + + /* STAGE_FLAG fail-fact for a purged jobid: apply_wal_entry()'s own + * FAIL branch already guards on j != NULL and proceeds (returns 0) -- + * apply_committed_entry() must behave identically. */ + wal_jobstate_t js = { .jobid = 702, + .state = 0, + ._pad = WAL_JOB_FAIL_STAGE_FLAG | + JOB_EXEC_STAGE_POST_EXEC }; + + xce_add_hook(&g_tace_c.hooksys, XCE_CONTROLLER_HOOK_JOB, + tace_faildetail_spy_hook, NULL); + + int rc = apply_committed_entry(&g_tace_p, WAL_JOB_FAIL, 28, &js, + sizeof(js)); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, + "a committed FAIL fact for a purged jobid must be a benign skip " + "(rc==0), never a fence-engaging negative (bug-1970)"); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_tace_hook_fires, + "a skip must fire NEITHER HOOK_DETAIL_JOB_FAIL nor JOB_UPDATE"); + TEST_ASSERT_EQUAL_INT_MESSAGE(-1, g_tace_last_fail_jobid, + "no fail-detail fire means no job_failinfo_t was fabricated"); + + tace_teardown(); + printf("test_ace_job_fail_missing_job_is_benign_skip: PASS\n"); +} + +static void test_ace_job_update_missing_job_is_benign_skip(void) +{ + tace_setup(); + + /* The case observed live (bug-1970's actual trigger): a duplicate + * post-finish UPDATE for a job purged between the old primary's + * in-memory no-op logging and the next restart's replay. _pad == 0 + * (legacy absolute-state shape, as persist_submit_job_fact writes). */ + wal_jobstate_t js = { .jobid = 703, .state = JOB_STAT_EXIT, ._pad = 0 }; + + xce_add_hook(&g_tace_c.hooksys, XCE_CONTROLLER_HOOK_JOB, tace_spy_hook, + NULL); + + int rc = apply_committed_entry(&g_tace_p, WAL_JOB_UPDATE, 29, &js, + sizeof(js)); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, + "a committed UPDATE for a purged jobid must be a benign skip " + "(rc==0) -- the pre-fix -ENOENT here fenced a live controller out " + "of promotion forever (bug-1970)"); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_tace_hook_fires, + "a skip must fire NO notification"); + + /* The POSTDONE-mark variant must skip identically (same missing-job + * lookup precedes the _pad dispatch). */ + wal_jobstate_t pd = { .jobid = 704, + .state = 0, + ._pad = WAL_JOB_UPDATE_POSTDONE_MARK }; + rc = apply_committed_entry(&g_tace_p, WAL_JOB_UPDATE, 30, &pd, + sizeof(pd)); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, + "a committed POSTDONE-mark UPDATE for a purged jobid must also be " + "a benign skip (bug-1970)"); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_tace_hook_fires, + "the POSTDONE skip must fire NO notification either"); + + tace_teardown(); + printf("test_ace_job_update_missing_job_is_benign_skip: PASS\n"); +} + /* ── WAL_QUEUE_CTRL ───────────────────────────────────────────────────── */ static void test_ace_queue_ctrl_updates_and_notifies(void) @@ -1293,6 +1403,9 @@ void test_apply_committed_entry(void) RUN_TEST(test_ace_job_fail_stage_flag_fires_faildetail_and_update); RUN_TEST(test_ace_job_fail_legacy_pad_zero_fires_update_only); RUN_TEST(test_ace_job_fail_guard_rejected_fires_no_hook); + RUN_TEST(test_ace_job_fact_missing_job_is_benign_skip_dispatch_group); + RUN_TEST(test_ace_job_fail_missing_job_is_benign_skip); + RUN_TEST(test_ace_job_update_missing_job_is_benign_skip); RUN_TEST(test_ace_queue_ctrl_updates_and_notifies); RUN_TEST(test_ace_worker_ctrl_join_restores_and_notifies); RUN_TEST(test_ace_worker_ctrl_update_applies_to_existing_worker); diff --git a/src/lib/meson.build b/src/lib/meson.build index e103c724..90a1c8ad 100644 --- a/src/lib/meson.build +++ b/src/lib/meson.build @@ -15,7 +15,17 @@ gen_srcs = custom_target( 'controllercapnp.[ch]', output:['controller.capnp.c', 'controller.capnp.h'], input: ['../schema/controller.capnp', capnpc_c], + # ASAN_OPTIONS=detect_leaks=0: capnpc-c (the code generator) deliberately + # never frees its compile-time structures -- a one-shot tool whose process + # exits immediately. Under the asan profile the wrap-subproject binary is + # built with -fsanitize=address, and LSAN's default end-of-process leak + # check then fails the generation step (exit 1, ~2MB reported across + # capnpc-c.c allocations) -- de-nix build path only; the legacy nix build + # never propagated -fsanitize to the subproject. Same exemption + # convention as every other "one-shot, never-torn-down" boundary in this + # repo (test('libxce test', ...) above uses the identical env var). command: [env_program, 'PATH=' + fs.parent(capnpc_c.full_path()) + ':/usr/bin:/bin', + 'ASAN_OPTIONS=detect_leaks=0', capnp, 'compile','-o' ,'c:lib', '-I', schema_dir, '--src-prefix', schema_dir, '@INPUT0@'] ) diff --git a/src/tools/xce-package.sh b/src/tools/xce-package.sh index 7c9c8004..9910645a 100755 --- a/src/tools/xce-package.sh +++ b/src/tools/xce-package.sh @@ -55,8 +55,13 @@ fi [ -f "$BUILD_DIR"/worker/xcecd_virtual ] && install -m 755 "$BUILD_DIR"/worker/xcecd_virtual "$STAGING/bin/" # --- bundled shared libraries --- -# Every DT_NEEDED whose resolved path falls outside the base system dirs is -# a wrap-subproject or non-baseline artifact -> copy + rewrite RPATH. +# Two collection rules (DEPS.md 运行期 bundle 规则): +# 1. ALWAYS-bundle list: libraries the target hosts may lack even though the +# BUILD host has them under /usr/lib (libcyaml, libcgroup proved missing +# on the t2/t3 Ubuntu 24.04 runtime hosts). The ldd-location heuristic +# cannot see this gap -- only an explicit name list can. +# 2. Location rule: every DT_NEEDED whose resolved path falls outside the +# base system dirs (wrap-subproject artifacts like libtlog.so). # # Collect from the BUILD-TREE binaries, not the staged copies: meson links # product binaries with a build-tree RPATH ($ORIGIN/../subprojects/), @@ -66,6 +71,9 @@ fi # Reading the ORIGINAL binary paths keeps ldd resolution intact; the # staged copies get a fresh $ORIGIN RPATH below. BASE_LIB_RE='^/(lib|usr/lib|lib64|usr/lib64)/' +# Non-glibc libs the tarball must carry regardless of the build host's +# ldconfig cache. libsystemd stays OUT (target-distro-provided, DEPS.md). +ALWAYS_BUNDLE_RE='^(libcyaml\.so|libyaml-0\.so|libtirpc\.so|libcgroup\.so|libhwloc\.so|libedit\.so)' declare -A BUNDLED stage_bins=( "$BUILD_DIR/controller/xce_controller" @@ -80,8 +88,8 @@ stage_bins=( for src in "${stage_bins[@]}"; do while IFS= read -r lib; do [ -z "$lib" ] && continue - if [[ ! "$lib" =~ $BASE_LIB_RE ]]; then - name="$(basename "$lib")" + name="$(basename "$lib")" + if [[ "$name" =~ $ALWAYS_BUNDLE_RE ]] || [[ ! "$lib" =~ $BASE_LIB_RE ]]; then if [ -z "${BUNDLED[$name]:-}" ]; then cp -L "$lib" "$STAGING/lib/" BUNDLED[$name]=1