myxce/src/controller/testsuite/test_persist_hooks.c
Rongsong Shen 15be5d94e8 fix(controller): extract job-submit commit barrier out of the hook fan-out — stop committed-job-loss (bug-1306)
raft-apply-decoupling item #1 (down payment). xce_run_hooks returns only the
LAST-registered hook's rc, so a later JOB AFTER_EVENT subscriber (sched/
apiserver) returning 0 clobbered persist_job_hook_fun's commit -ETIMEDOUT.
handle_jobsubmit then replied 'jobsubmit ok' for an entry that never reached
majority, and a new-term leader legally discarded it (committed-job-loss,
Commit Durability violation).

- persist.c/.h: new persist_submit_wal_and_commit(p, jlist) — the explicit raft
  step (WAL append + majority-commit barrier) returning the commit verdict
  directly. persist_job_hook_fun's WAL_MODE_WAIT branch is now a no-op.
- job.c handle_jobsubmit: call the explicit step, read commit_rc DIRECTLY,
  reject on <0 before firing the now notification-only AFTER hook
  (canonical commit -> notify -> reply).
- tests: +test_job_hook_new_is_notify_only regression pin; 4 hook tests
  relocated to drive the explicit step. persist_hooks 19/0, e2e_hooks 3/0,
  commit_barrier_reply 22/0, commit_durability 38/0.

Proven: 10-round chaotic soak committed_job_loss 5 -> 0 (same election code).
Residual soak FAILs (divergence catch-up lag + a round-4 election-escalation
tail) are NOT job loss — tracked as task 5.10.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JDYFbrgyDHbvNYcxApKhrX
2026-07-01 00:42:30 +00:00

479 lines
16 KiB
C

#define _GNU_SOURCE
#include <dirent.h>
#include <fcntl.h>
#include <inttypes.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#include <unity/unity.h>
#include "config.h"
#include "persist.h"
#include "xce/cluster_def.h"
#include "xlist.h"
/* ------------------------------------------------------------------ */
/* Test helpers */
/* ------------------------------------------------------------------ */
static char g_tmpdir[256];
static persist_t* make_hook_test_persist(void)
{
strcpy(g_tmpdir, "/tmp/xce_hook_test_XXXXXX");
if (!mkdtemp(g_tmpdir))
return NULL;
persist_t* p = calloc(1, sizeof(persist_t));
if (!p)
return NULL;
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 1; /* 1 ms — fsync thread wakes up immediately on stop */
persist_set_workdir(p, g_tmpdir, "hook-test");
if (persist_init(p, &cfg, NULL) != 0) {
free(p);
return NULL;
}
if (persist_start(p) != 0) {
persist_fini(p);
free(p);
return NULL;
}
return p;
}
static void cleanup_hook_test_persist(persist_t* p)
{
persist_stop(p);
persist_fini(p);
free(p);
}
/* Zero-initialized stubs — capnp encoders handle all-NULL fields safely */
static worker_t g_stub_worker;
static job_t g_stub_job;
static queue_t g_stub_queue;
/* Variadic wrappers so tests can pass typed args through a real va_list */
static int call_worker_hook(int type, int sit, int detail, persist_t* p, ...)
{
va_list ap;
va_start(ap, p);
int rc = persist_worker_hook_fun(type, sit, detail, p, ap);
va_end(ap);
return rc;
}
static int call_job_hook(int type, int sit, int detail, persist_t* p, ...)
{
va_list ap;
va_start(ap, p);
int rc = persist_job_hook_fun(type, sit, detail, p, ap);
va_end(ap);
return rc;
}
static int call_queue_hook(int type, int sit, int detail, persist_t* p, ...)
{
va_list ap;
va_start(ap, p);
int rc = persist_queue_hook_fun(type, sit, detail, p, ap);
va_end(ap);
return rc;
}
/* One-element xlist backed by a stack pointer array — no heap allocation needed */
#define STUB_JOB_XLIST(name) \
void* name##_ptr_ = &g_stub_job; \
xlist_t name = { .count = 1, .caps = 1, .delta = 1, .data = &name##_ptr_ }
/* ------------------------------------------------------------------ */
/* Tests — RED phase: functions not yet implemented, must fail to link */
/* ------------------------------------------------------------------ */
static void test_job_hook_new_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* Submit durability moved OUT of the JOB_NEW hook into the explicit raft
* step persist_submit_wal_and_commit (raft-apply-decoupling item #1 /
* bug-1306) — the hook is now notification-only, so the WAL-append is
* asserted against the explicit step. */
STUB_JOB_XLIST(jlist);
int rc = persist_submit_wal_and_commit(p, &jlist);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_new_writes_wal_entry: PASS\n");
}
/* Regression pin (bug-1306 / raft-apply-decoupling item #1): the JOB_NEW hook
* must be NOTIFICATION-ONLY — it must NOT perform the WAL append / commit
* barrier. If durability lived in the hook, xce_run_hooks (which returns only
* the last hook's rc) could let a later notify subscriber's 0 clobber a commit
* -ETIMEDOUT, acking the client for an uncommitted entry. Asserting the hook
* appends NOTHING structurally forecloses that clobber. */
static void test_job_hook_new_is_notify_only(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
STUB_JOB_XLIST(jlist);
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_NEW, p,
(cluster_t*)NULL, &jlist);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before, p->current_lsn,
"JOB_NEW hook must be notify-only — durability moved to "
"persist_submit_wal_and_commit (bug-1306); a hook that still appends "
"re-opens the xce_run_hooks last-rc clobber");
cleanup_hook_test_persist(p);
printf("test_job_hook_new_is_notify_only: PASS\n");
}
static void test_job_hook_start_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* JOB_START args: (cluster_t*, job_pidinfo_t*, job_t*) */
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_START, p,
(cluster_t*)NULL, (void*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_start_writes_wal_entry: PASS\n");
}
static void test_job_hook_exit_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* JOB_EXIT args: (cluster_t*, xce_jobfinish_t*, job_t*) */
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_EXIT, p,
(cluster_t*)NULL, (void*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_exit_writes_wal_entry: PASS\n");
}
static void test_job_hook_fail_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* JOB_FAIL args: (cluster_t*, job_failinfo_t*, job_t*) */
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_FAIL, p,
(cluster_t*)NULL, (void*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_fail_writes_wal_entry: PASS\n");
}
static void test_job_hook_stop_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* STOP/RESUME/UPDATE/CKPT/PREASON/RUSAGE default args: (cluster_t*, job_t*) */
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_STOP, p,
(cluster_t*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_stop_writes_wal_entry: PASS\n");
}
static void test_job_hook_resume_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_RESUME, p,
(cluster_t*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_resume_writes_wal_entry: PASS\n");
}
static void test_job_hook_update_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_UPDATE, p,
(cluster_t*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_update_writes_wal_entry: PASS\n");
}
static void test_job_hook_ckpt_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_CKPT, p,
(cluster_t*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_ckpt_writes_wal_entry: PASS\n");
}
static void test_job_hook_rusage_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_RUSAGE_UPDATE, p,
(cluster_t*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_rusage_writes_wal_entry: PASS\n");
}
static void test_job_hook_setpreason_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_SETPREASON, p,
(cluster_t*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_setpreason_writes_wal_entry: PASS\n");
}
static void test_job_hook_clrpreason_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_CLRPREASON, p,
(cluster_t*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_clrpreason_writes_wal_entry: PASS\n");
}
static void test_queue_hook_update_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* QUEUE_UPDATE args: (cluster_t*, queue_t*) */
int rc = call_queue_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_QUEUE_UPDATE, p,
(cluster_t*)NULL, &g_stub_queue);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_queue_hook_update_writes_wal_entry: PASS\n");
}
static void test_worker_hook_join_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* Unified JOIN signature: (cluster_t*, char* name, worker_t*) */
int rc = call_worker_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_WORKER_JOIN, p,
(cluster_t*)NULL, "stub-host", &g_stub_worker);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_worker_hook_join_writes_wal_entry: PASS\n");
}
static void test_worker_hook_update_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* UPDATE signature: (cluster_t*, worker_t*) */
int rc = call_worker_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_WORKER_UPDATE, p,
(cluster_t*)NULL, &g_stub_worker);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_worker_hook_update_writes_wal_entry: PASS\n");
}
static void test_worker_hook_dead_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* DEAD signature: (cluster_t*, worker_t*) */
int rc = call_worker_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_WORKER_DEAD, p,
(cluster_t*)NULL, &g_stub_worker);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_worker_hook_dead_writes_wal_entry: PASS\n");
}
static void test_job_hook_skips_unknown_detail(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* HOOK_DETAIL_SCHEDULER_JOIN has no WAL mapping — must be silently skipped */
int rc = persist_job_hook_fun(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_SCHEDULER_JOIN, p, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
/* LSN must NOT have advanced — no WAL entry written */
TEST_ASSERT_EQUAL_UINT64(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_skips_unknown_detail: PASS\n");
}
static void test_worker_hook_skips_unknown_detail(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
int rc = persist_worker_hook_fun(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_SCHEDULER_DEAD, p, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT64(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_worker_hook_skips_unknown_detail: PASS\n");
}
static void test_job_hook_new_writes_multiple_entries_sequentially(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* JOB_NEW submit durability goes through the explicit raft step; START/EXIT
* facts still flow through the hook (RETAIN). All three advance LSN. */
STUB_JOB_XLIST(jlist);
persist_submit_wal_and_commit(p, &jlist);
call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_START, p,
(cluster_t*)NULL, (void*)NULL, &g_stub_job);
call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_EXIT, p,
(cluster_t*)NULL, (void*)NULL, &g_stub_job);
TEST_ASSERT_EQUAL_UINT64(lsn_before + 3, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_new_writes_multiple_entries_sequentially: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Suite entry point */
/* ------------------------------------------------------------------ */
void test_persist_hooks(void)
{
printf("\n>> persist hook function tests\n\n");
RUN_TEST(test_job_hook_new_writes_wal_entry);
RUN_TEST(test_job_hook_new_is_notify_only);
RUN_TEST(test_job_hook_start_writes_wal_entry);
RUN_TEST(test_job_hook_exit_writes_wal_entry);
RUN_TEST(test_job_hook_fail_writes_wal_entry);
RUN_TEST(test_job_hook_stop_writes_wal_entry);
RUN_TEST(test_job_hook_resume_writes_wal_entry);
RUN_TEST(test_job_hook_update_writes_wal_entry);
RUN_TEST(test_job_hook_ckpt_writes_wal_entry);
RUN_TEST(test_job_hook_rusage_writes_wal_entry);
RUN_TEST(test_job_hook_setpreason_writes_wal_entry);
RUN_TEST(test_job_hook_clrpreason_writes_wal_entry);
RUN_TEST(test_queue_hook_update_writes_wal_entry);
RUN_TEST(test_worker_hook_join_writes_wal_entry);
RUN_TEST(test_worker_hook_update_writes_wal_entry);
RUN_TEST(test_worker_hook_dead_writes_wal_entry);
RUN_TEST(test_job_hook_skips_unknown_detail);
RUN_TEST(test_worker_hook_skips_unknown_detail);
RUN_TEST(test_job_hook_new_writes_multiple_entries_sequentially);
}