lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(worker-rejoin-resync): reconciliation round lifecycle (task 3.1)

Browse source 
New src/controller/resync.c/resync.h: resync_arm_round(cluster_t*,
worker_t*) enumerates every non-terminal job assigned to a host,
populates worker_t.resync (round_token, query job-ID set, deadline),
sets XCE_WORKER_RECONCILING, and schedules a non-blocking delaymgr
one-shot timeout. resync_resolve_round() clears the round; a fired
timeout resolves it too, via resync_round_is_current()'s by-value
round-token check -- delay.c/delay.h expose no true cancel/dequeue
primitive, so a stale timer from an already-superseded round becomes a
safe no-op instead of touching a newer round's live state (verified
safe against both misresolving the wrong round and use-after-free: the
timeout closure never holds a pointer into resync_state_t itself).

5 new tests: query-set correctness, fresh-token-per-arm, the
RECONCILING flag's set/clear lifecycle, and a direct (non-blocking)
timeout-fire proof via an XCE_TESTING-gated seam confirmed compiled out
of the production binary. Full suite: 1232 tests, 0 failures.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Y4M5MJM2zK6nU43da267Ku
This commit is contained in:
Rongsong Shen 2026-09-01 12:53:09 +00:00
parent ea0658ecf4
commit 442fcf4544
No known key found for this signature in database
GPG key ID: 5BCDAAE0123E9FE2
5 changed files with 927 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

387
src/controller/testsuite/test_worker_resync.c
View file

@ -52,7 +52,11 @@
#include "jobj.h"
#include "kvmgr.h"
#include "persist.h"
#include "resync.h" /* worker-rejoin-resync task 3.1 */
#include "wal_types.h"
#include "wobj.h" /* worker-rejoin-resync task 3.1: makeWorker, for a real
* RTTI-managed worker_t the timeout test can genuinely
* lock() */
#include "xce/cluster_def.h"
#include "xce/types.h"
#include "xcealloc.h"
@ -67,6 +71,16 @@ static int g_twr_jobobj_ready;
static cluster_t g_twr;
static char g_twr_tmpdir[256];
/* worker-rejoin-resync task 3.1: no-op kvmgr datakiller for g_twr.workermgr,
* mirroring production's own worker_obj_killer (cluster.c) -- a worker_t,
* once created, is never explicitly torn down for the life of the process
* (no key_del(c->workermgr, ...) call site exists anywhere in this
* codebase), so there is nothing for this to actually free. */
static void twr_worker_obj_killer(void* arg)
{
(void)arg;
}
static void twr_setup(void)
{
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
@ -88,6 +102,23 @@ static void twr_setup(void)
g_twr.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(g_twr.jobmgr);
/* worker-rejoin-resync task 3.1: a real workermgr so the round-lifecycle
* tests below can create a genuine RTTI-managed worker_t (via
* makeWorker) where that matters (the timeout test, whose internal
* resync_on_timeout callback genuinely lock()s `w`) -- key type mirrors
* production's own workermgr_params (cluster.c, KVMGR_KEY_STRING,
* keyed by hostname). */
{
kvmgr_params_t workermgr_params
= { .key_type = KVMGR_KEY_STRING,
.nslots = 32,
.delta = 16,
.datakiller = twr_worker_obj_killer };
g_twr.workermgr = makeInstance(&xkvmgr_type, &workermgr_params);
TEST_ASSERT_NOT_NULL(g_twr.workermgr);
}
/* Real disk-backed persist, cluster == NULL: wal_hook_append_
* classified() (persist.c) takes the direct persist_wal_async() path --
* no raft/commit-engine machinery needed for a plain RETAIN-mode fact
@ -112,6 +143,8 @@ static void twr_teardown(void)
if (g_twr.jobmgr)
dereference(g_twr.jobmgr);
if (g_twr.workermgr)
dereference(g_twr.workermgr);
xce_hook_fini(&g_twr.hooksys);
snprintf(cmd, sizeof(cmd), "rm -rf %s", g_twr_tmpdir);
@ -671,6 +704,355 @@ static void test_reconcile_assign_survives_restart_replay(void)
printf("test_reconcile_assign_survives_restart_replay: PASS\n");
}
/* ── resync.c round lifecycle tests (task 3.1) ───────────────────────────
*
* Proves resync_arm_round()/resync_resolve_round()/resync_round_is_current()
* (resync.c) -- the per-worker job-state reconciliation round lifecycle:
* query-set construction (Req 1.1, 2.1, 2.2), a fresh monotonic round token
* per arm distinguishing this round from any prior one for the same worker
* (Req 6.1, 6.3), the XCE_WORKER_RECONCILING status bit (Req 5.1, 5.2), and
* the Req 5.3 bounded timeout resolving the round without any blocking
* call. Reuses the g_twr/twr_setup/twr_teardown fixture and twr_seed_job()
* from task 2.1/2.2's tests above -- resync_arm_round only needs a
* cluster_t with a real c->jobmgr (which g_twr already provides).
*
* No live caller exists yet (worker.c's rejoin_worker/apply_join_state
* wiring is task 6.1; the report-apply glue that would call
* resync_round_is_current for real is task 3.2) -- every test here calls
* resync_arm_round/resync_resolve_round/resync_round_is_current directly,
* exactly as a real caller (task 6.1) eventually will.
*
* resync_arm_round/resync_resolve_round/resync_round_is_current never
* lock/unlock `w` themselves (their documented precondition is that the
* CALLER already holds it) -- so a bare stack worker_t is sufficient and
* safe for those tests, exactly mirroring how task 2.2's own
* record_job_reconcile_assign_fact tests above already use a bare stack
* `worker_t w` with no locking. The ONE exception is the timeout test:
* resync_on_timeout (the internal delaymgr callback resync_test_fire_
* timeout drives directly) runs with no caller-held lock -- because a real
* delaymgr fire has none -- so it genuinely calls lock()/unlock() on `w`
* itself, which requires a real RTTI-managed worker_t (via makeWorker
* against g_twr.workermgr), not a bare stack struct. */
#define TWR_JOBID_RESYNC_A 81101ULL
#define TWR_JOBID_RESYNC_B 81102ULL
#define TWR_JOBID_RESYNC_OTHER 81103ULL
#define TWR_JOBID_RESYNC_DONE 81104ULL
#define TWR_JOBID_RESYNC_NOALLOC 81105ULL
#define TWR_JOBID_RESYNC_SOLO 81106ULL
#define TWR_JOBID_RESYNC_TIMEOUT 81107ULL
/* Same builder chain as twr_make_test_alloc() above, parameterized by host
* -- twr_make_test_alloc() itself hardcodes "twr-worker", which does not
* fit these tests' need for several DISTINCT hosts within one job table. */
static xce_job_alloc_t* twr_make_test_alloc_for_host(uint64_t jobid,
const char* host)
{
return make_job_alloc(jobid, 1,
make_host_allocs(1,
make_host_alloc((char*)host, 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
}
/* True iff `jobid` appears anywhere in ids[0..n) -- used below to prove the
* query set contains exactly the expected members, neither missing a real
* match nor carrying an extra one. */
static int twr_ids_contains(const uint64_t* ids, int n, uint64_t jobid)
{
int i;
for (i = 0; i < n; i++) {
if (ids[i] == jobid)
return 1;
}
return 0;
}
/* A real RTTI-managed worker_t (via makeWorker against g_twr.workermgr),
* for the one test below that needs genuine lock()/unlock() to succeed on
* `w` -- mirrors worker.c's own init_worker() construction
* (`tmp.name = (char*)host; tmp.attrs = attrs; w = makeWorker(c->workermgr,
* &tmp);`). */
static worker_t* twr_make_test_worker(const char* hostname)
{
hostattrs_t attrs;
xce_worker_t tmp;
memset(&attrs, 0, sizeof(attrs));
memset(&tmp, 0, sizeof(tmp));
tmp.name = (char*)hostname;
tmp.attrs = &attrs;
return makeWorker(g_twr.workermgr, &tmp);
}
/* Req 1.1, 2.1, 2.2: the query set matches EXACTLY the non-terminal jobs
* assigned to the given host -- not fewer (a genuine match omitted), not
* more (a different host's job, a terminal job on the SAME host, or a
* never-dispatched job with no host assignment at all, leaking in). */
static void test_resync_arm_round_query_set_matches_exactly(void)
{
job_t* ja;
job_t* jb;
job_t* jother;
job_t* jdone;
job_t* jnoalloc;
worker_t w;
int rc;
twr_setup();
ja = twr_seed_job(TWR_JOBID_RESYNC_A, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(ja);
ja->job.allocs
= twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_A, "resync-host-A");
jb = twr_seed_job(TWR_JOBID_RESYNC_B, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(jb);
jb->job.allocs
= twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_B, "resync-host-A");
/* Same non-terminal state, DIFFERENT host -- must NOT appear in
* resync-host-A's query set. */
jother = twr_seed_job(TWR_JOBID_RESYNC_OTHER, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(jother);
jother->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_OTHER,
"resync-host-B");
/* SAME host, but terminal (DONE) -- must NOT appear (Req 1.1's
* "non-terminal" qualifier). */
jdone = twr_seed_job(TWR_JOBID_RESYNC_DONE, JOB_STAT_DONE);
TEST_ASSERT_NOT_NULL(jdone);
jdone->job.allocs
= twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_DONE, "resync-host-A");
/* Never dispatched (no allocation at all) -- not assigned to ANY host,
* so must NOT appear regardless of state. */
jnoalloc = twr_seed_job(TWR_JOBID_RESYNC_NOALLOC, JOB_STAT_PEND);
TEST_ASSERT_NOT_NULL(jnoalloc);
TEST_ASSERT_NULL(jnoalloc->job.allocs);
memset(&w, 0, sizeof(w));
w.host.name = (char*)"resync-host-A";
rc = resync_arm_round(&g_twr, &w);
TEST_ASSERT_EQUAL_INT_MESSAGE(2, rc,
"exactly the 2 non-terminal resync-host-A jobs must be armed into "
"the round");
TEST_ASSERT_NOT_NULL(w.resync);
TEST_ASSERT_EQUAL_INT(2, w.resync->n_queried_jobids);
TEST_ASSERT_TRUE_MESSAGE(
twr_ids_contains(w.resync->queried_jobids, w.resync->n_queried_jobids,
TWR_JOBID_RESYNC_A),
"job A (non-terminal, resync-host-A) must be in the query set");
TEST_ASSERT_TRUE_MESSAGE(
twr_ids_contains(w.resync->queried_jobids, w.resync->n_queried_jobids,
TWR_JOBID_RESYNC_B),
"job B (non-terminal, resync-host-A) must be in the query set");
TEST_ASSERT_FALSE_MESSAGE(
twr_ids_contains(w.resync->queried_jobids, w.resync->n_queried_jobids,
TWR_JOBID_RESYNC_OTHER),
"a job assigned to a DIFFERENT host must not appear");
TEST_ASSERT_FALSE_MESSAGE(
twr_ids_contains(w.resync->queried_jobids, w.resync->n_queried_jobids,
TWR_JOBID_RESYNC_DONE),
"a TERMINAL job on the same host must not appear (Req 1.1)");
TEST_ASSERT_FALSE_MESSAGE(
twr_ids_contains(w.resync->queried_jobids, w.resync->n_queried_jobids,
TWR_JOBID_RESYNC_NOALLOC),
"a never-dispatched job (no host assignment at all) must not "
"appear");
TEST_ASSERT_TRUE_MESSAGE(IS_WORKER_RECONCILING(w.host.status),
"XCE_WORKER_RECONCILING must be set once a round is armed");
resync_resolve_round(&g_twr, &w);
TEST_ASSERT_NULL(w.resync);
TEST_ASSERT_FALSE(IS_WORKER_RECONCILING(w.host.status));
twr_teardown();
printf("test_resync_arm_round_query_set_matches_exactly: PASS\n");
}
/* Req 1.2: a worker with zero non-terminal jobs assigned is a documented
* no-op -- no round armed, no status bit set. */
static void test_resync_arm_round_no_jobs_is_noop(void)
{
worker_t w;
int rc;
twr_setup();
memset(&w, 0, sizeof(w));
w.host.name = (char*)"resync-host-empty";
rc = resync_arm_round(&g_twr, &w);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"a worker with no non-terminal jobs must arm zero jobs (Req 1.2)");
TEST_ASSERT_NULL_MESSAGE(w.resync,
"no resync_state_t must be attached for the Req 1.2 no-op case");
TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w.host.status),
"XCE_WORKER_RECONCILING must not be set for the Req 1.2 no-op case");
twr_teardown();
printf("test_resync_arm_round_no_jobs_is_noop: PASS\n");
}
/* Req 6.1, 6.3: arming a round for a worker that already carries one
* (simulating a disconnect interrupting an outstanding round, then a
* rejoin) discards the prior round and produces a FRESH, DIFFERENT token --
* and the original token is provably no longer current once superseded. */
static void test_resync_arm_round_fresh_token_supersedes_prior(void)
{
job_t* j;
worker_t w;
int rc;
uint64_t token1;
uint64_t token2;
twr_setup();
j = twr_seed_job(TWR_JOBID_RESYNC_SOLO, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_SOLO,
"resync-host-solo");
memset(&w, 0, sizeof(w));
w.host.name = (char*)"resync-host-solo";
rc = resync_arm_round(&g_twr, &w);
TEST_ASSERT_EQUAL_INT(1, rc);
TEST_ASSERT_NOT_NULL(w.resync);
token1 = w.resync->round_token;
TEST_ASSERT_TRUE_MESSAGE(resync_round_is_current(&w, token1),
"the round just armed must report its own token as current");
/* Simulated disconnect-then-rejoin: the round never resolved, but the
* worker rejoins and is armed again. */
rc = resync_arm_round(&g_twr, &w);
TEST_ASSERT_EQUAL_INT(1, rc);
TEST_ASSERT_NOT_NULL(w.resync);
token2 = w.resync->round_token;
TEST_ASSERT_NOT_EQUAL_MESSAGE(token1, token2,
"re-arming must produce a token distinct from the prior round's "
"(Req 6.1, 6.3)");
TEST_ASSERT_FALSE_MESSAGE(resync_round_is_current(&w, token1),
"the ORIGINAL token must now report stale/superseded (Req 6.3) -- "
"the mechanism a later report-apply caller uses to discard a "
"report carrying it");
TEST_ASSERT_TRUE_MESSAGE(resync_round_is_current(&w, token2),
"the NEW token must report current");
resync_resolve_round(&g_twr, &w);
twr_teardown();
printf("test_resync_arm_round_fresh_token_supersedes_prior: PASS\n");
}
/* Req 5.1, 5.2: XCE_WORKER_RECONCILING is set once a round is armed and
* cleared once resync_resolve_round runs -- resync_round_is_current also
* correctly reports the resolved round as no longer current. */
static void test_resync_reconciling_flag_set_after_arm_cleared_after_resolve(
void)
{
job_t* j;
worker_t w;
int rc;
uint64_t token;
twr_setup();
j = twr_seed_job(TWR_JOBID_RESYNC_SOLO, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_SOLO,
"resync-host-flag");
memset(&w, 0, sizeof(w));
w.host.name = (char*)"resync-host-flag";
TEST_ASSERT_FALSE(IS_WORKER_RECONCILING(w.host.status));
rc = resync_arm_round(&g_twr, &w);
TEST_ASSERT_EQUAL_INT(1, rc);
TEST_ASSERT_TRUE_MESSAGE(IS_WORKER_RECONCILING(w.host.status),
"XCE_WORKER_RECONCILING must be set once armed (Req 5.1)");
token = w.resync->round_token;
resync_resolve_round(&g_twr, &w);
TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w.host.status),
"XCE_WORKER_RECONCILING must clear once resolved (Req 5.2)");
TEST_ASSERT_NULL_MESSAGE(w.resync,
"resync_resolve_round must detach/free w->resync");
TEST_ASSERT_FALSE_MESSAGE(resync_round_is_current(&w, token),
"a resolved round's token must no longer report current");
/* Idempotent: resolving an already-resolved (NULL) round is a safe
* no-op, never a double-free. */
resync_resolve_round(&g_twr, &w);
TEST_ASSERT_NULL(w.resync);
twr_teardown();
printf("test_resync_reconciling_flag_set_after_arm_cleared_after_"
"resolve: PASS\n");
}
/* Req 5.3: the timeout resolves an outstanding round -- and does so via a
* plain, synchronous function call with no sleep/wait of any kind, proving
* the resolution itself never blocks (ha-async-io.md's delaymgr
* thread-role contract). g_twr never starts a real delaymgr (matching
* every other bare-cluster_t fixture in this file), so resync_arm_round's
* own `!c->delaymgr.running` guard already took the no-real-timer branch
* when the round above was armed; resync_test_fire_timeout below drives
* the EXACT SAME internal resolution logic
* (resync_on_timeout) a real timer firing would have. */
static void test_resync_timeout_resolves_round_without_blocking(void)
{
job_t* j;
worker_t* w;
int rc;
uint64_t token;
twr_setup();
j = twr_seed_job(TWR_JOBID_RESYNC_TIMEOUT, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_TIMEOUT,
"resync-host-timeout");
w = twr_make_test_worker("resync-host-timeout");
TEST_ASSERT_NOT_NULL(w);
rc = resync_arm_round(&g_twr, w);
TEST_ASSERT_EQUAL_INT(1, rc);
TEST_ASSERT_NOT_NULL(w->resync);
TEST_ASSERT_TRUE(IS_WORKER_RECONCILING(w->host.status));
token = w->resync->round_token;
/* resync_on_timeout (invoked via resync_test_fire_timeout) locks/
* unlocks `w` itself -- this call must NOT be wrapped in an external
* lock(w, ...), or it would deadlock against `w`'s own non-recursive
* write lock. */
resync_test_fire_timeout(&g_twr, w, token);
TEST_ASSERT_NULL_MESSAGE(w->resync,
"the timeout must resolve (free/detach) the round");
TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
"the timeout must clear XCE_WORKER_RECONCILING so the worker is "
"not withheld from dispatch indefinitely (Req 5.1, 5.2, 5.3)");
/* A second, late fire for the SAME (now-stale) token must be a safe
* no-op -- never a double-free/double-clear against an already-
* resolved round. This is exactly the round-token staleness check
* (Req 6.3) protecting against a real delaymgr task that could not be
* dequeued/cancelled (delay.c exposes no cancel primitive) firing late
* against a resync_state_t that is already gone or has already been
* replaced by a newer round. */
resync_test_fire_timeout(&g_twr, w, token);
TEST_ASSERT_NULL(w->resync);
TEST_ASSERT_FALSE(IS_WORKER_RECONCILING(w->host.status));
twr_teardown();
printf("test_resync_timeout_resolves_round_without_blocking: PASS\n");
}
/* ── Test group entry point ───────────────────────────────────────────── */
void test_worker_resync(void)
@ -681,4 +1063,9 @@ void test_worker_resync(void)
RUN_TEST(test_reconcile_pending_rejects_terminal_job);
RUN_TEST(test_reconcile_assign_applies_running_no_allocs_and_notifies);
RUN_TEST(test_reconcile_assign_survives_restart_replay);
RUN_TEST(test_resync_arm_round_query_set_matches_exactly);
RUN_TEST(test_resync_arm_round_no_jobs_is_noop);
RUN_TEST(test_resync_arm_round_fresh_token_supersedes_prior);
RUN_TEST(test_resync_reconciling_flag_set_after_arm_cleared_after_resolve);
RUN_TEST(test_resync_timeout_resolves_round_without_blocking);
}