lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(controller): close 6 data-safety races + apply-pump O(N^2) regression + boot TOCTOU

Browse source 
Bundles the fixes validated this session across ha.c/persist.c/worker.c:

- raft-apply-decoupling Task 7.4: apply-pump O(N^2) throughput regression
  fixed via a segment-read cache.
- bug-1416: boot-time TOCTOU race in persist_verify_apply_correctness()
  closed with a recovery-complete gate that closes at process-lifecycle
  start, not function entry.
- bug-451: HOOK_DETAIL_JOB_RUSAGE_UPDATE now has its own persist hook arm
  instead of silently falling through the default job-state case.
- bug-1426: trim_wal_segments() also caps its trim floor at the local
  apply pump's own progress, not just remote standby-acked positions.
- bug-1427: persist_verify_apply_correctness() extended with a worker-count
  check mirroring the existing job-count check (count_jobmgr_entries and
  count_workermgr_entries now share one count_manager_entries(var) helper).
- bug-1428: persist_register_pending_apply's critical section widened to
  close the actual race instead of patching a symptom.
- bug-1472 (remainder): WORKER_DEAD replay in worker_from_capnp() now
  applies mark_worker_unavail()'s mutations via a shared
  apply_worker_dead_mutation() helper instead of duplicating them.
- bug-1474 (angle-a): on_standby_connected's stale-duplicate detection
  fixed for promoted vote endpoints via a shared votes_idx_for_peer()
  index-scale converter.

Each fix independently verified via TDD (RED before fix, GREEN after).
Full ASAN rebuild + controller_unit_test: 1101 tests, 0 failures. Live
single-node HA deployment validated Task 7.4 and bug-1416 across 5
restart/recovery cycles (45/45 job submissions, zero APPLY_DIVERGENT
fences).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-08-16 10:20:02 +00:00
parent 3afe3d5549
commit 45400c3a28
Signed by: rshen
SSH key fingerprint: SHA256:8Vf2pZ3h6rJa7EhGK+WaAQ9Mta/apZvteW6g9U/WKKk
18 changed files with 1499 additions and 195 deletions
Download patch file Download diff file Expand all files Collapse all files

73
src/controller/testsuite/test_persist_hooks.c
View file

@ -8,10 +8,15 @@
#include <sys/stat.h>
#include <unistd.h>
#include <rtti.h>
#include <unity/unity.h>
#include "cluster.h"
#include "config.h"
#include "jobj.h"
#include "kvmgr.h"
#include "persist.h"
#include "typemap.h"
#include "xce/cluster_def.h"
#include "xlist.h"
@ -62,6 +67,41 @@ static worker_t g_stub_worker;
static job_t g_stub_job;
static queue_t g_stub_queue;
/* bug-451: HOOK_DETAIL_JOB_RUSAGE_UPDATE is the only job hook detail whose
* persist.c arm actually dereferences the cluster_t* it's called with
* (every other case discards it via `(void)va_arg(ap, cluster_t*)`), since
* it must look the job up by jobid via c->jobmgr rather than receiving a
* job_t* directly. Minimal real (non-NULL) cluster+jobmgr+job, built once
* on first use — every other test in this file keeps passing
* (cluster_t*)NULL safely, this is the one exception. */
static cluster_t g_hook_cluster;
static int g_hook_jobobj_ready;
static void ensure_hook_cluster_with_job(uint64_t jobid, unsigned int state)
{
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 64, .delta = 16 };
xce_job_t jdef;
job_submit_data_t submit;
if (!g_hook_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(64, 64));
g_hook_jobobj_ready = 1;
}
if (g_hook_cluster.jobmgr != NULL)
dereference(g_hook_cluster.jobmgr);
memset(&g_hook_cluster, 0, sizeof(g_hook_cluster));
g_hook_cluster.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(g_hook_cluster.jobmgr);
memset(&submit, 0, sizeof(submit));
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = jobid;
jdef.state = state;
jdef.submit = &submit;
TEST_ASSERT_NOT_NULL(makeJob(g_hook_cluster.jobmgr, &jdef));
}
/* Variadic wrappers so tests can pass typed args through a real va_list */
static int call_worker_hook(int type, int sit, int detail, persist_t* p, ...)
{
@ -276,9 +316,16 @@ static void test_job_hook_rusage_writes_wal_entry(void)
uint64_t lsn_before = p->current_lsn;
/* bug-451: RUSAGE_UPDATE args are (cluster_t*, xce_job_rusages_t*) —
* NOT (cluster_t*, job_t*) like STOP/RESUME/UPDATE/CKPT/PREASON below.
* A real cluster+jobmgr+job is required (see ensure_hook_cluster_with_
* job) since persist.c's arm looks the job up by jobid, it doesn't
* receive a job_t* directly. */
ensure_hook_cluster_with_job(42, JOB_STAT_RUN);
xce_job_rusages_t rusages = { .jobid = 42 };
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_RUSAGE_UPDATE, p,
(cluster_t*)NULL, &g_stub_job);
&g_hook_cluster, &rusages);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
@ -286,6 +333,29 @@ static void test_job_hook_rusage_writes_wal_entry(void)
printf("test_job_hook_rusage_writes_wal_entry: PASS\n");
}
static void test_job_hook_rusage_unknown_job_is_silently_skipped(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* jobid 999 was never inserted into g_hook_cluster's jobmgr — the job
* may have already been reaped by the time this hook fires. Must
* return 0 (no error) and append nothing, never crash on a NULL
* job_t*. */
ensure_hook_cluster_with_job(42, JOB_STAT_RUN);
xce_job_rusages_t rusages = { .jobid = 999 };
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_RUSAGE_UPDATE, p,
&g_hook_cluster, &rusages);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT64(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_rusage_unknown_job_is_silently_skipped: PASS\n");
}
static void test_job_hook_setpreason_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
@ -468,6 +538,7 @@ void test_persist_hooks(void)
RUN_TEST(test_job_hook_update_writes_wal_entry);
RUN_TEST(test_job_hook_ckpt_writes_wal_entry);
RUN_TEST(test_job_hook_rusage_writes_wal_entry);
RUN_TEST(test_job_hook_rusage_unknown_job_is_silently_skipped);
RUN_TEST(test_job_hook_setpreason_writes_wal_entry);
RUN_TEST(test_job_hook_clrpreason_writes_wal_entry);
RUN_TEST(test_queue_hook_update_writes_wal_entry);