fix(controller): close 6 data-safety races + apply-pump O(N^2) regression + boot TOCTOU
Bundles the fixes validated this session across ha.c/persist.c/worker.c: - raft-apply-decoupling Task 7.4: apply-pump O(N^2) throughput regression fixed via a segment-read cache. - bug-1416: boot-time TOCTOU race in persist_verify_apply_correctness() closed with a recovery-complete gate that closes at process-lifecycle start, not function entry. - bug-451: HOOK_DETAIL_JOB_RUSAGE_UPDATE now has its own persist hook arm instead of silently falling through the default job-state case. - bug-1426: trim_wal_segments() also caps its trim floor at the local apply pump's own progress, not just remote standby-acked positions. - bug-1427: persist_verify_apply_correctness() extended with a worker-count check mirroring the existing job-count check (count_jobmgr_entries and count_workermgr_entries now share one count_manager_entries(var) helper). - bug-1428: persist_register_pending_apply's critical section widened to close the actual race instead of patching a symptom. - bug-1472 (remainder): WORKER_DEAD replay in worker_from_capnp() now applies mark_worker_unavail()'s mutations via a shared apply_worker_dead_mutation() helper instead of duplicating them. - bug-1474 (angle-a): on_standby_connected's stale-duplicate detection fixed for promoted vote endpoints via a shared votes_idx_for_peer() index-scale converter. Each fix independently verified via TDD (RED before fix, GREEN after). Full ASAN rebuild + controller_unit_test: 1101 tests, 0 failures. Live single-node HA deployment validated Task 7.4 and bug-1416 across 5 restart/recovery cycles (45/45 job submissions, zero APPLY_DIVERGENT fences). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
3afe3d5549
commit
45400c3a28
SSH key fingerprint:
SHA256:8Vf2pZ3h6rJa7EhGK+WaAQ9Mta/apZvteW6g9U/WKKk
18 changed files with 1499 additions and 195 deletions
|
|
@ -480,7 +480,10 @@ static void test_ace_worker_ctrl_join_restores_and_notifies(void)
|
|||
|
||||
int rc = apply_committed_entry(&g_tace_p, WAL_WORKER_CTRL, 40, pkt,
|
||||
(uint32_t)total);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
|
||||
"bug-1427: a genuine new-worker JOIN must return 1 (not the old "
|
||||
"hardcoded 0), mirroring WAL_JOB_SUBMIT's apply_rc==1 creation "
|
||||
"signal — replay_wal()'s new-worker tally relies on this");
|
||||
|
||||
worker_t* w = NULL;
|
||||
lock(g_tace_c.workermgr, RTTI_LOCK_READ);
|
||||
|
|
@ -692,6 +695,102 @@ static void test_ace_worker_ctrl_join_duplicate_existing_is_noop(void)
|
|||
printf("test_ace_worker_ctrl_join_duplicate_existing_is_noop: PASS\n");
|
||||
}
|
||||
|
||||
/* bug-1472-remainder: a HOOK_DETAIL_WORKER_DEAD entry against an already-
|
||||
* existing worker previously returned -EEXIST unconditionally (a no-op,
|
||||
* same as JOIN's genuine idempotency case) -- so a dead-worker transition
|
||||
* detected by one member (mark_worker_unavail(), worker.c) never replicated
|
||||
* to standbys/this member's own WAL replay. Proves the fix replays
|
||||
* mark_worker_unavail()'s own local mutations: XCE_WORKER_UNAVAIL set,
|
||||
* dead_since taken from the decoded payload (the PRIMARY's own death
|
||||
* timestamp), and hostattrs reset to the "unknown"/0 sentinels -- mirroring
|
||||
* exactly what the primary itself did before firing this same WAL entry. */
|
||||
static void test_ace_worker_ctrl_dead_applies_to_existing_worker(void)
|
||||
{
|
||||
tace_setup();
|
||||
|
||||
/* Pre-existing worker with live (non-"unknown") attrs, as if genuinely
|
||||
* joined and reporting real hardware facts. */
|
||||
hostattrs_t attrs;
|
||||
memset(&attrs, 0, sizeof(attrs));
|
||||
attrs.hosttype = (char*)"LINUX";
|
||||
attrs.hostmodel = (char*)"x86_64";
|
||||
attrs.cpufactor = 1.0f;
|
||||
attrs.maxcpus = 8;
|
||||
attrs.maxmem = 16384;
|
||||
xce_worker_t wdef;
|
||||
memset(&wdef, 0, sizeof(wdef));
|
||||
wdef.name = (char*)"w-dead";
|
||||
wdef.attrs = &attrs;
|
||||
wdef.status = 0;
|
||||
TEST_ASSERT_NOT_NULL(makeWorker(g_tace_c.workermgr, &wdef));
|
||||
|
||||
/* Encode a DEAD entry the way mark_worker_unavail()'s own hook fire
|
||||
* would -- attrs already reset to "unknown"/0 (capnp_encode_worker
|
||||
* captures whatever the primary's worker_t held AT hook-fire time,
|
||||
* which is always post-reset for a real DEAD transition), a specific
|
||||
* non-zero deadSince (the primary's own death timestamp, not this
|
||||
* member's clock). */
|
||||
hostattrs_t dead_attrs;
|
||||
memset(&dead_attrs, 0, sizeof(dead_attrs));
|
||||
dead_attrs.hosttype = (char*)"unknown";
|
||||
dead_attrs.hostmodel = (char*)"unknown";
|
||||
xce_worker_t dead_wdef;
|
||||
memset(&dead_wdef, 0, sizeof(dead_wdef));
|
||||
dead_wdef.name = (char*)"w-dead";
|
||||
dead_wdef.attrs = &dead_attrs;
|
||||
dead_wdef.status = XCE_WORKER_UNAVAIL;
|
||||
dead_wdef.dead_since = 1755300000; /* arbitrary fixed epoch time */
|
||||
|
||||
uint8_t* wbuf = NULL;
|
||||
size_t wsize = 0;
|
||||
TEST_ASSERT_EQUAL_INT(0, tace_encode_worker(&dead_wdef, &wbuf, &wsize));
|
||||
|
||||
uint32_t disc = (uint32_t)HOOK_DETAIL_WORKER_DEAD;
|
||||
size_t total = sizeof(disc) + wsize;
|
||||
uint8_t* pkt = malloc(total);
|
||||
TEST_ASSERT_NOT_NULL(pkt);
|
||||
memcpy(pkt, &disc, sizeof(disc));
|
||||
memcpy(pkt + sizeof(disc), wbuf, wsize);
|
||||
free(wbuf);
|
||||
|
||||
xce_add_hook(&g_tace_c.hooksys, XCE_CONTROLLER_HOOK_WORKER, tace_spy_hook,
|
||||
NULL);
|
||||
|
||||
int rc = apply_committed_entry(&g_tace_p, WAL_WORKER_CTRL, 44, pkt,
|
||||
(uint32_t)total);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
|
||||
"bug-1472-remainder: a DEAD entry against an already-existing "
|
||||
"worker must succeed (0), not -EEXIST -- -EEXIST silently "
|
||||
"discarded the dead-worker transition entirely");
|
||||
|
||||
worker_t* w = NULL;
|
||||
lock(g_tace_c.workermgr, RTTI_LOCK_READ);
|
||||
key_find(g_tace_c.workermgr, (void*)"w-dead", (void**)&w);
|
||||
unlock(g_tace_c.workermgr);
|
||||
TEST_ASSERT_NOT_NULL(w);
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
(w->host.status & XCE_WORKER_UNAVAIL) != 0,
|
||||
"the existing worker must be marked XCE_WORKER_UNAVAIL");
|
||||
TEST_ASSERT_EQUAL_INT64_MESSAGE((int64_t)1755300000, (int64_t)w->host.dead_since,
|
||||
"dead_since must come from the decoded payload (the primary's own "
|
||||
"death timestamp), not be silently discarded or re-derived locally");
|
||||
TEST_ASSERT_EQUAL_STRING_MESSAGE("unknown", w->host.attrs->hosttype,
|
||||
"hostattrs must be reset to the \"unknown\" sentinel, mirroring "
|
||||
"mark_worker_unavail()'s own local effect on the primary");
|
||||
TEST_ASSERT_EQUAL_STRING_MESSAGE("unknown", w->host.attrs->hostmodel,
|
||||
"hostattrs must be reset to the \"unknown\" sentinel");
|
||||
TEST_ASSERT_EQUAL_FLOAT_MESSAGE(0, w->host.attrs->maxmem,
|
||||
"numeric hostattrs (maxmem and friends) must be zeroed, not left "
|
||||
"at their last-known-live values");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_tace_hook_fires,
|
||||
"a genuinely-applied DEAD transition must fire exactly one "
|
||||
"notification");
|
||||
|
||||
free(pkt);
|
||||
tace_teardown();
|
||||
printf("test_ace_worker_ctrl_dead_applies_to_existing_worker: PASS\n");
|
||||
}
|
||||
|
||||
/* ── WAL_JOB_ARRAY_SUBMIT (job-array-single-wal-record task 3.2) ────────── */
|
||||
|
||||
static job_submit_data_t tace_array_make_spec(void)
|
||||
|
|
@ -947,6 +1046,7 @@ void test_apply_committed_entry(void)
|
|||
RUN_TEST(test_ace_worker_ctrl_update_applies_to_existing_worker);
|
||||
RUN_TEST(test_ace_worker_ctrl_update_replaces_not_ors_admin_bits);
|
||||
RUN_TEST(test_ace_worker_ctrl_join_duplicate_existing_is_noop);
|
||||
RUN_TEST(test_ace_worker_ctrl_dead_applies_to_existing_worker);
|
||||
RUN_TEST(test_ace_job_array_submit_constructs_all_and_notifies_once);
|
||||
RUN_TEST(test_ace_job_array_submit_idempotent_replay_no_extra_notify);
|
||||
#ifdef XCE_TESTING
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue