lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
myxce/src/controller/testsuite/test_persist_hooks.c
Rongsong Shen 45400c3a28
fix(controller): close 6 data-safety races + apply-pump O(N^2) regression + boot TOCTOU 
Bundles the fixes validated this session across ha.c/persist.c/worker.c:

- raft-apply-decoupling Task 7.4: apply-pump O(N^2) throughput regression
  fixed via a segment-read cache.
- bug-1416: boot-time TOCTOU race in persist_verify_apply_correctness()
  closed with a recovery-complete gate that closes at process-lifecycle
  start, not function entry.
- bug-451: HOOK_DETAIL_JOB_RUSAGE_UPDATE now has its own persist hook arm
  instead of silently falling through the default job-state case.
- bug-1426: trim_wal_segments() also caps its trim floor at the local
  apply pump's own progress, not just remote standby-acked positions.
- bug-1427: persist_verify_apply_correctness() extended with a worker-count
  check mirroring the existing job-count check (count_jobmgr_entries and
  count_workermgr_entries now share one count_manager_entries(var) helper).
- bug-1428: persist_register_pending_apply's critical section widened to
  close the actual race instead of patching a symptom.
- bug-1472 (remainder): WORKER_DEAD replay in worker_from_capnp() now
  applies mark_worker_unavail()'s mutations via a shared
  apply_worker_dead_mutation() helper instead of duplicating them.
- bug-1474 (angle-a): on_standby_connected's stale-duplicate detection
  fixed for promoted vote endpoints via a shared votes_idx_for_peer()
  index-scale converter.

Each fix independently verified via TDD (RED before fix, GREEN after).
Full ASAN rebuild + controller_unit_test: 1101 tests, 0 failures. Live
single-node HA deployment validated Task 7.4 and bug-1416 across 5
restart/recovery cycles (45/45 job submissions, zero APPLY_DIVERGENT
fences).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-08-16 10:20:02 +00:00

551 lines
19 KiB
C
Raw Blame History

#define _GNU_SOURCE
#include <dirent.h>
#include <fcntl.h>
#include <inttypes.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#include <rtti.h>
#include <unity/unity.h>
#include "cluster.h"
#include "config.h"
#include "jobj.h"
#include "kvmgr.h"
#include "persist.h"
#include "typemap.h"
#include "xce/cluster_def.h"
#include "xlist.h"
/* ------------------------------------------------------------------ */
/* Test helpers */
/* ------------------------------------------------------------------ */
static char g_tmpdir[256];
static persist_t* make_hook_test_persist(void)
{
strcpy(g_tmpdir, "/tmp/xce_hook_test_XXXXXX");
if (!mkdtemp(g_tmpdir))
return NULL;
persist_t* p = calloc(1, sizeof(persist_t));
if (!p)
return NULL;
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 1; /* 1 ms — fsync thread wakes up immediately on stop */
persist_set_workdir(p, g_tmpdir, "hook-test");
if (persist_init(p, &cfg, NULL) != 0) {
free(p);
return NULL;
}
if (persist_start(p) != 0) {
persist_fini(p);
free(p);
return NULL;
}
return p;
}
static void cleanup_hook_test_persist(persist_t* p)
{
persist_stop(p);
persist_fini(p);
free(p);
}
/* Zero-initialized stubs — capnp encoders handle all-NULL fields safely */
static worker_t g_stub_worker;
static job_t g_stub_job;
static queue_t g_stub_queue;
/* bug-451: HOOK_DETAIL_JOB_RUSAGE_UPDATE is the only job hook detail whose
* persist.c arm actually dereferences the cluster_t* it's called with
* (every other case discards it via `(void)va_arg(ap, cluster_t*)`), since
* it must look the job up by jobid via c->jobmgr rather than receiving a
* job_t* directly. Minimal real (non-NULL) cluster+jobmgr+job, built once
* on first use — every other test in this file keeps passing
* (cluster_t*)NULL safely, this is the one exception. */
static cluster_t g_hook_cluster;
static int g_hook_jobobj_ready;
static void ensure_hook_cluster_with_job(uint64_t jobid, unsigned int state)
{
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 64, .delta = 16 };
xce_job_t jdef;
job_submit_data_t submit;
if (!g_hook_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(64, 64));
g_hook_jobobj_ready = 1;
}
if (g_hook_cluster.jobmgr != NULL)
dereference(g_hook_cluster.jobmgr);
memset(&g_hook_cluster, 0, sizeof(g_hook_cluster));
g_hook_cluster.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(g_hook_cluster.jobmgr);
memset(&submit, 0, sizeof(submit));
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = jobid;
jdef.state = state;
jdef.submit = &submit;
TEST_ASSERT_NOT_NULL(makeJob(g_hook_cluster.jobmgr, &jdef));
}
/* Variadic wrappers so tests can pass typed args through a real va_list */
static int call_worker_hook(int type, int sit, int detail, persist_t* p, ...)
{
va_list ap;
va_start(ap, p);
int rc = persist_worker_hook_fun(type, sit, detail, p, ap);
va_end(ap);
return rc;
}
static int call_job_hook(int type, int sit, int detail, persist_t* p, ...)
{
va_list ap;
va_start(ap, p);
int rc = persist_job_hook_fun(type, sit, detail, p, ap);
va_end(ap);
return rc;
}
static int call_queue_hook(int type, int sit, int detail, persist_t* p, ...)
{
va_list ap;
va_start(ap, p);
int rc = persist_queue_hook_fun(type, sit, detail, p, ap);
va_end(ap);
return rc;
}
/* One-element xlist backed by a stack pointer array — no heap allocation needed */
#define STUB_JOB_XLIST(name) \
void* name##_ptr_ = &g_stub_job; \
xlist_t name = { .count = 1, .caps = 1, .delta = 1, .data = &name##_ptr_ }
/* ------------------------------------------------------------------ */
/* Tests — RED phase: functions not yet implemented, must fail to link */
/* ------------------------------------------------------------------ */
static void test_job_hook_new_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* Submit durability moved OUT of the JOB_NEW hook into the explicit raft
* step persist_submit_wal_and_commit (raft-apply-decoupling item #1 /
* bug-1306) — the hook is now notification-only, so the WAL-append is
* asserted against the explicit step. */
STUB_JOB_XLIST(jlist);
int rc = persist_submit_wal_and_commit(p, &jlist);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_new_writes_wal_entry: PASS\n");
}
/* Regression pin (bug-1306 / raft-apply-decoupling item #1): the JOB_NEW hook
* must be NOTIFICATION-ONLY — it must NOT perform the WAL append / commit
* barrier. If durability lived in the hook, xce_run_hooks (which returns only
* the last hook's rc) could let a later notify subscriber's 0 clobber a commit
* -ETIMEDOUT, acking the client for an uncommitted entry. Asserting the hook
* appends NOTHING structurally forecloses that clobber. */
static void test_job_hook_new_is_notify_only(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
STUB_JOB_XLIST(jlist);
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_NEW, p,
(cluster_t*)NULL, &jlist);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before, p->current_lsn,
"JOB_NEW hook must be notify-only — durability moved to "
"persist_submit_wal_and_commit (bug-1306); a hook that still appends "
"re-opens the xce_run_hooks last-rc clobber");
cleanup_hook_test_persist(p);
printf("test_job_hook_new_is_notify_only: PASS\n");
}
static void test_job_hook_start_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* JOB_START args: (cluster_t*, job_pidinfo_t*, job_t*) */
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_START, p,
(cluster_t*)NULL, (void*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_start_writes_wal_entry: PASS\n");
}
static void test_job_hook_exit_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* JOB_EXIT args: (cluster_t*, xce_jobfinish_t*, job_t*) */
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_EXIT, p,
(cluster_t*)NULL, (void*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_exit_writes_wal_entry: PASS\n");
}
static void test_job_hook_fail_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* JOB_FAIL args: (cluster_t*, job_failinfo_t*, job_t*) */
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_FAIL, p,
(cluster_t*)NULL, (void*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_fail_writes_wal_entry: PASS\n");
}
static void test_job_hook_stop_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* STOP/RESUME/UPDATE/CKPT/PREASON/RUSAGE default args: (cluster_t*, job_t*) */
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_STOP, p,
(cluster_t*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_stop_writes_wal_entry: PASS\n");
}
static void test_job_hook_resume_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_RESUME, p,
(cluster_t*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_resume_writes_wal_entry: PASS\n");
}
static void test_job_hook_update_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_UPDATE, p,
(cluster_t*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_update_writes_wal_entry: PASS\n");
}
static void test_job_hook_ckpt_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_CKPT, p,
(cluster_t*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_ckpt_writes_wal_entry: PASS\n");
}
static void test_job_hook_rusage_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* bug-451: RUSAGE_UPDATE args are (cluster_t*, xce_job_rusages_t*) —
* NOT (cluster_t*, job_t*) like STOP/RESUME/UPDATE/CKPT/PREASON below.
* A real cluster+jobmgr+job is required (see ensure_hook_cluster_with_
* job) since persist.c's arm looks the job up by jobid, it doesn't
* receive a job_t* directly. */
ensure_hook_cluster_with_job(42, JOB_STAT_RUN);
xce_job_rusages_t rusages = { .jobid = 42 };
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_RUSAGE_UPDATE, p,
&g_hook_cluster, &rusages);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_rusage_writes_wal_entry: PASS\n");
}
static void test_job_hook_rusage_unknown_job_is_silently_skipped(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* jobid 999 was never inserted into g_hook_cluster's jobmgr — the job
* may have already been reaped by the time this hook fires. Must
* return 0 (no error) and append nothing, never crash on a NULL
* job_t*. */
ensure_hook_cluster_with_job(42, JOB_STAT_RUN);
xce_job_rusages_t rusages = { .jobid = 999 };
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_RUSAGE_UPDATE, p,
&g_hook_cluster, &rusages);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT64(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_rusage_unknown_job_is_silently_skipped: PASS\n");
}
static void test_job_hook_setpreason_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_SETPREASON, p,
(cluster_t*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_setpreason_writes_wal_entry: PASS\n");
}
static void test_job_hook_clrpreason_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_CLRPREASON, p,
(cluster_t*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_clrpreason_writes_wal_entry: PASS\n");
}
static void test_queue_hook_update_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* QUEUE_UPDATE args: (cluster_t*, queue_t*) */
int rc = call_queue_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_QUEUE_UPDATE, p,
(cluster_t*)NULL, &g_stub_queue);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_queue_hook_update_writes_wal_entry: PASS\n");
}
static void test_worker_hook_join_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* Unified JOIN signature: (cluster_t*, char* name, worker_t*) */
int rc = call_worker_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_WORKER_JOIN, p,
(cluster_t*)NULL, "stub-host", &g_stub_worker);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_worker_hook_join_writes_wal_entry: PASS\n");
}
static void test_worker_hook_update_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* UPDATE signature: (cluster_t*, worker_t*) */
int rc = call_worker_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_WORKER_UPDATE, p,
(cluster_t*)NULL, &g_stub_worker);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_worker_hook_update_writes_wal_entry: PASS\n");
}
static void test_worker_hook_dead_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* DEAD signature: (cluster_t*, worker_t*) */
int rc = call_worker_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_WORKER_DEAD, p,
(cluster_t*)NULL, &g_stub_worker);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_worker_hook_dead_writes_wal_entry: PASS\n");
}
static void test_job_hook_skips_unknown_detail(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* HOOK_DETAIL_SCHEDULER_JOIN has no WAL mapping — must be silently skipped */
int rc = persist_job_hook_fun(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_SCHEDULER_JOIN, p, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
/* LSN must NOT have advanced — no WAL entry written */
TEST_ASSERT_EQUAL_UINT64(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_skips_unknown_detail: PASS\n");
}
static void test_worker_hook_skips_unknown_detail(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
int rc = persist_worker_hook_fun(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_SCHEDULER_DEAD, p, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT64(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_worker_hook_skips_unknown_detail: PASS\n");
}
static void test_job_hook_new_writes_multiple_entries_sequentially(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* JOB_NEW submit durability goes through the explicit raft step; START/EXIT
* facts still flow through the hook (RETAIN). All three advance LSN. */
STUB_JOB_XLIST(jlist);
persist_submit_wal_and_commit(p, &jlist);
call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_START, p,
(cluster_t*)NULL, (void*)NULL, &g_stub_job);
call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_EXIT, p,
(cluster_t*)NULL, (void*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_UINT64(lsn_before + 3, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_new_writes_multiple_entries_sequentially: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Suite entry point */
/* ------------------------------------------------------------------ */
void test_persist_hooks(void)
{
UnitySetTestFile(__FILE__);
printf("\n>> persist hook function tests\n\n");
RUN_TEST(test_job_hook_new_writes_wal_entry);
RUN_TEST(test_job_hook_new_is_notify_only);
RUN_TEST(test_job_hook_start_writes_wal_entry);
RUN_TEST(test_job_hook_exit_writes_wal_entry);
RUN_TEST(test_job_hook_fail_writes_wal_entry);
RUN_TEST(test_job_hook_stop_writes_wal_entry);
RUN_TEST(test_job_hook_resume_writes_wal_entry);
RUN_TEST(test_job_hook_update_writes_wal_entry);
RUN_TEST(test_job_hook_ckpt_writes_wal_entry);
RUN_TEST(test_job_hook_rusage_writes_wal_entry);
RUN_TEST(test_job_hook_rusage_unknown_job_is_silently_skipped);
RUN_TEST(test_job_hook_setpreason_writes_wal_entry);
RUN_TEST(test_job_hook_clrpreason_writes_wal_entry);
RUN_TEST(test_queue_hook_update_writes_wal_entry);
RUN_TEST(test_worker_hook_join_writes_wal_entry);
RUN_TEST(test_worker_hook_update_writes_wal_entry);
RUN_TEST(test_worker_hook_dead_writes_wal_entry);
RUN_TEST(test_job_hook_skips_unknown_detail);
RUN_TEST(test_worker_hook_skips_unknown_detail);
RUN_TEST(test_job_hook_new_writes_multiple_entries_sequentially);
}
Reference in a new issue View git blame Copy permalink