fix(controller): close 6 data-safety races + apply-pump O(N^2) regression + boot TOCTOU
Bundles the fixes validated this session across ha.c/persist.c/worker.c: - raft-apply-decoupling Task 7.4: apply-pump O(N^2) throughput regression fixed via a segment-read cache. - bug-1416: boot-time TOCTOU race in persist_verify_apply_correctness() closed with a recovery-complete gate that closes at process-lifecycle start, not function entry. - bug-451: HOOK_DETAIL_JOB_RUSAGE_UPDATE now has its own persist hook arm instead of silently falling through the default job-state case. - bug-1426: trim_wal_segments() also caps its trim floor at the local apply pump's own progress, not just remote standby-acked positions. - bug-1427: persist_verify_apply_correctness() extended with a worker-count check mirroring the existing job-count check (count_jobmgr_entries and count_workermgr_entries now share one count_manager_entries(var) helper). - bug-1428: persist_register_pending_apply's critical section widened to close the actual race instead of patching a symptom. - bug-1472 (remainder): WORKER_DEAD replay in worker_from_capnp() now applies mark_worker_unavail()'s mutations via a shared apply_worker_dead_mutation() helper instead of duplicating them. - bug-1474 (angle-a): on_standby_connected's stale-duplicate detection fixed for promoted vote endpoints via a shared votes_idx_for_peer() index-scale converter. Each fix independently verified via TDD (RED before fix, GREEN after). Full ASAN rebuild + controller_unit_test: 1101 tests, 0 failures. Live single-node HA deployment validated Task 7.4 and bug-1416 across 5 restart/recovery cycles (45/45 job submissions, zero APPLY_DIVERGENT fences). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
3afe3d5549
commit
45400c3a28
18 changed files with 1499 additions and 195 deletions
|
|
@ -9209,6 +9209,19 @@ int standby_ep_err_handler_primary_for_test(endpoint_t *ep)
|
|||
}
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
/*
|
||||
* votes_idx_for_peer: convert an ABSOLUTE candidate index to the RELATIVE
|
||||
* votes[] index (self excluded) used by a promoted vote ep's peer_idx -- see
|
||||
* raft_cb_send_vote_request. Shared by on_standby_connected's stale-duplicate
|
||||
* check and primary_heartbeat_fn's reconnect-sweep check, which both need to
|
||||
* compare a standby_ep_ctx_t's peer_idx (absolute) against a promoted vote
|
||||
* ep's peer_idx (relative) on the same scale.
|
||||
*/
|
||||
static int votes_idx_for_peer(ha_state_t *ha, int abs_idx)
|
||||
{
|
||||
return (abs_idx < ha->my_priority) ? abs_idx : abs_idx - 1;
|
||||
}
|
||||
|
||||
/*
|
||||
* on_standby_connected: iomgr on_connected callback for outbound standby
|
||||
* endpoints created by the heartbeat task.
|
||||
|
|
@ -9248,6 +9261,25 @@ static void on_standby_connected(endpoint_t *ep)
|
|||
*/
|
||||
endpoint_t *stale_same_peer = NULL;
|
||||
if (ha->standby_endpoints) {
|
||||
/* bug-1474 (angle-a): sctx->peer_idx is ALWAYS the ABSOLUTE candidate
|
||||
* index (standby_ep_ctx_t's own contract, doc comment above). But a
|
||||
* DIFFERENT entry already in standby_endpoints may be a PROMOTED VOTE
|
||||
* EP (raft_cb_become_leader), whose peer_idx is the RELATIVE votes[]
|
||||
* index (self excluded) -- see raft_cb_send_vote_request. Comparing
|
||||
* the two peer_idx values directly, as this loop previously did, only
|
||||
* happens to work when cur is itself a standby ep (both absolute);
|
||||
* against a promoted vote ep for the SAME physical peer, the raw
|
||||
* comparison is comparing two different index scales and can never
|
||||
* match -- the stale/duplicate promoted-vote channel then survives
|
||||
* alongside this fresh one, exactly the "two live outbound state-push
|
||||
* channels to one peer" hazard this function's own doc comment above
|
||||
* describes and this loop exists to prevent. Convert sctx's absolute
|
||||
* index to the relative scale before comparing against a promoted
|
||||
* vote ep -- same votes_idx_for_peer() primary_heartbeat_fn's own
|
||||
* reconnect-sweep check already uses for the identical discrimination
|
||||
* (see its "Two endpoint types can be in standby_endpoints" comment,
|
||||
* the established precedent this mirrors). */
|
||||
int votes_idx_for_new = votes_idx_for_peer(ha, sctx->peer_idx);
|
||||
pthread_mutex_lock(&ha->lock);
|
||||
int already = 0;
|
||||
for (int i = 0; i < ha->standby_endpoints->count; i++) {
|
||||
|
|
@ -9256,7 +9288,12 @@ static void on_standby_connected(endpoint_t *ep)
|
|||
/* Different ep, same peer → stale duplicate to evict. */
|
||||
standby_ep_ctx_t *cur_sctx =
|
||||
cur ? (standby_ep_ctx_t *)cur->data : NULL;
|
||||
if (cur_sctx && cur_sctx->peer_idx == sctx->peer_idx) {
|
||||
if (!cur_sctx) continue;
|
||||
int cur_is_promoted_vote_ep = (cur_sctx->ctx != NULL);
|
||||
int match = cur_is_promoted_vote_ep
|
||||
? (cur_sctx->peer_idx == votes_idx_for_new)
|
||||
: (cur_sctx->peer_idx == sctx->peer_idx);
|
||||
if (match) {
|
||||
stale_same_peer = cur;
|
||||
zarray_del(ha->standby_endpoints, i);
|
||||
/* count shrank by one; the surviving scan continues but we
|
||||
|
|
@ -9793,7 +9830,7 @@ static int primary_heartbeat_fn(void *arg)
|
|||
* - Heartbeat standby eps: ep->data = standby_ep_ctx_t (ctx==NULL),
|
||||
* peer_idx is absolute candidate index. Compare directly with i.
|
||||
*/
|
||||
int votes_idx_for_i = (i < ha->my_priority) ? i : i - 1;
|
||||
int votes_idx_for_i = votes_idx_for_peer(ha, i);
|
||||
int found = 0;
|
||||
if (ha->standby_endpoints) {
|
||||
pthread_mutex_lock(&ha->lock);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue