lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(worker-rejoin-resync): task 4.3 - wire query receipt and post-join gate

Browse source 
Add OPCode_jobStateQuery dispatch to myreq_handler with multi-page query
reassembly, routing to task 4.2's xcecd_resync_answer_query() (Req
8.1, 8.3). Gate the worker's resumption of normal post-join operation
on a resync_pending flag until the full report has sent (Req 3.8).

Fixes bug-1749 (xcecd_resync.c was never added to the worker's
production source lists - task 4.2's build only ever exercised it via
the test binary's white-box include).

Round 1 review caught bug-1750: the gate sat as checking_connection()'s
entry guard, so a connection drop mid multi-page query left
resync_pending stuck forever, permanently disabling the worker's only
reconnect mechanism (Req 6.1 violation). Round 2 narrows the gate to
guard only the rejoin action, and resets it from xcecd_on_reconnect()
so a stalled query can never wedge it past one reconnect cycle.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Y4M5MJM2zK6nU43da267Ku
This commit is contained in:
Rongsong Shen 2026-09-01 18:01:03 +00:00
parent cfe58c3dcf
commit 5b69973677
No known key found for this signature in database
GPG key ID: 5BCDAAE0123E9FE2
8 changed files with 1161 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

417
src/worker/testsuite/test_controller_connector.c
View file

@ -134,6 +134,53 @@ bool ipc_master_send_job_ckpt(uint64_t job_id, int kill, const char* method)
void xcecd_process_command(void* arg) { (void)arg; }
/* worker-rejoin-resync task 4.3 (Requirements 3.8, 8.1, 8.3): link-
* satisfying stubs for xcecd_resync.c's public API -- this file's white-box
* target is controller_connector.c only, it does not #include
* xcecd_resync.c (that module's own resync_pending lifecycle and
* multi-page query reassembly are covered against the real production
* code by test_xcecd_resync.c). g_test_resync_pending lets
* checking_connection()'s gate-check tests below control what the gate
* reads without a real xcecd_resync.c backing it; the call-count/last-page
* globals let the dispatch-wiring tests below observe that myreq_handler
* actually reaches this function, without needing a real report to be
* built and sent. */
static bool g_test_resync_pending = false;
static int g_resync_handle_query_page_call_count = 0;
static const xce_job_state_query_t* g_resync_handle_query_page_last_page = NULL;
/* worker-rejoin-resync task 4.3 remediation (round 2, Requirements 3.8,
* 6.1): lets the tests below observe that xcecd_on_reconnect() actually
* calls xcecd_resync_reset() (the round-2 fix that bounds a stalled
* multi-page query's worst case), and lets a deferred-rejoin test simulate
* the gate clearing on its own, independent of checking_connection() ever
* touching it directly. */
static int g_resync_reset_call_count = 0;
bool xcecd_resync_is_pending(void) { return g_test_resync_pending; }
void xcecd_resync_reset(void)
{
g_resync_reset_call_count++;
g_test_resync_pending = false;
}
int xcecd_resync_handle_query_page(xcecd_worker_t* worker_list, endpoint_t* ep,
xce_peerinfo_t* from,
const xce_job_state_query_t* page,
uint64_t now_ms, int pktlimit)
{
(void)worker_list;
(void)ep;
(void)from;
(void)now_ms;
(void)pktlimit;
g_resync_handle_query_page_call_count++;
g_resync_handle_query_page_last_page = page;
return (page != NULL) ? 1 : -EINVAL;
}
/* ════════════════════════════════════════════════════════════════════════
* Fixtures
* ════════════════════════════════════════════════════════════════════════ */
@ -147,6 +194,13 @@ void setUp(void)
/* Restore the real production call by default (task 2.2) -- individual
* rejoin-retry tests below override this to a controllable double. */
rejoin_call = perform_pending_rejoin;
/* worker-rejoin-resync task 4.3: default the resync gate to "not
* pending" and reset the dispatch-wiring spy globals. */
g_test_resync_pending = false;
g_resync_handle_query_page_call_count = 0;
g_resync_handle_query_page_last_page = NULL;
g_resync_reset_call_count = 0;
}
void tearDown(void) { return; }
@ -276,6 +330,35 @@ static void test_rejoin_callback_sets_flag_and_returns_immediately(void)
printf("test_rejoin_callback_sets_flag_and_returns_immediately: PASS\n");
}
/* worker-rejoin-resync task 4.3 remediation (round 2, Requirements 3.8,
* 6.1): the reconnect callback must also reset xcecd_resync.c's gate, not
* just set needs_rejoin -- this is what bounds a stalled multi-page
* query's worst case (see xcecd_resync_reset()'s own doc comment). Without
* this call, a query whose connection dropped mid-reassembly would leave
* resync_pending stuck true forever with nothing left to clear it. */
static void test_rejoin_callback_also_resets_resync_gate(void)
{
g_test_resync_pending = true;
g_resync_reset_call_count = 0;
int rc = xcecd_on_reconnect((xcecli_ctx_t)0x1, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_TRUE(needs_rejoin);
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, g_resync_reset_call_count,
"xcecd_on_reconnect() must call xcecd_resync_reset() exactly once "
"so a resync gate left stuck by a stalled multi-page query is "
"always cleared the instant the worker detects it has "
"reconnected");
TEST_ASSERT_FALSE_MESSAGE(
g_test_resync_pending,
"the resync gate must actually be cleared, not merely have its "
"reset function called");
printf("test_rejoin_callback_also_resets_resync_gate: PASS\n");
}
static void test_rejoin_callback_registered_via_public_api(void)
{
/* Register through the real public entry point the production code
@ -558,6 +641,267 @@ test_checking_connection_rejoin_retried_on_next_probe_after_failure(void)
"failure: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* Task 4.3 (worker-rejoin-resync, Requirements 3.8, 8.1, 8.3) — gating the
* needs_rejoin-triggered rejoin action on xcecd_resync.c's resync_pending
* flag (the "resume normal post-join operation" step Requirement 3.8
* actually names), and wiring OPCode_jobStateQuery's real receipt.
*
* Round-2 remediation note: round 1 gated checking_connection()'s ENTRY on
* resync_pending, which also silently skipped xcecli_hello() and the
* needs_rejoin check itself -- confirmed to permanently disable the
* worker's own connection-recovery mechanism whenever resync_pending got
* stuck true (a stalled multi-page query, e.g.). The gate now sits INSIDE
* the needs_rejoin branch, guarding only rejoin_call(); see
* test_checking_connection_survives_permanently_stuck_resync_pending() and
* test_checking_connection_hello_and_needs_rejoin_precede_resync_gate_in_
* source() further below for the regression coverage of that distinction.
*
* What THIS file proves is the two things that are genuinely
* controller_connector.c's own: (1) checking_connection() actually
* consults the gate before attempting the rejoin -- the same way it
* already consults needs_rejoin -- and (2) myreq_handler's dispatch switch
* actually routes OPCode_jobStateQuery to the resync module at all, the
* exact gap task 4.2 left unclosed (confirmed by inspection: no case
* existed for this opcode before this task). xcecd_resync.c's own
* resync_pending set/clear lifecycle and multi-page query reassembly are
* covered end-to-end, against the real production code and real wire
* encode/decode, by test_xcecd_resync.c -- not duplicated here.
* ════════════════════════════════════════════════════════════════════════ */
static void
test_checking_connection_defers_rejoin_action_when_resync_pending(void)
{
controller_connector_ctx_t fake_cc = { 0 };
fake_cc.running = true;
cc_ctx = &fake_cc;
xcecli_ctx = NULL;
needs_rejoin = true; /* would normally trigger a rejoin attempt */
g_rejoin_call_count = 0;
rejoin_call = fake_rejoin_always_succeeds;
g_test_resync_pending = true;
checking_connection();
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_rejoin_call_count,
"checking_connection() must not attempt a rejoin while a job-state "
"resync report is still being sent (Req 3.8) -- the worker's "
"resumption of normal post-join operation must not begin or "
"interleave with it");
TEST_ASSERT_TRUE_MESSAGE(
needs_rejoin,
"a deferred tick must leave needs_rejoin untouched, not clear it "
"as if a rejoin had been attempted");
printf(
"test_checking_connection_defers_rejoin_action_when_resync_pending: "
"PASS\n");
}
static void test_checking_connection_resumes_once_resync_no_longer_pending(void)
{
controller_connector_ctx_t fake_cc = { 0 };
fake_cc.running = true;
cc_ctx = &fake_cc;
xcecli_ctx = NULL;
needs_rejoin = true;
g_rejoin_call_count = 0;
rejoin_call = fake_rejoin_always_succeeds;
g_test_resync_pending = false; /* report already finished sending */
checking_connection();
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, g_rejoin_call_count,
"once the gate clears, checking_connection() must resume its "
"normal periodic activity on its very next tick");
TEST_ASSERT_FALSE(needs_rejoin);
printf("test_checking_connection_resumes_once_resync_no_longer_pending: "
"PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* Round-2 remediation regression guard (Requirements 3.8, 6.1): FINDING 1
* of the round-1 review -- gating checking_connection()'s ENTRY on
* resync_pending permanently disabled the worker's own connection-recovery
* mechanism whenever a multi-page job-state query's final page could never
* arrive (e.g. the worker's own connection to the controller dropped after
* a non-final page). resync_pending must now guard ONLY the
* needs_rejoin-triggered rejoin action -- xcecli_hello() (the worker's
* liveness probe / reconnect-detection trigger) and the needs_rejoin check
* itself must keep running every single tick regardless, extending
* test_checking_connection_never_permanently_disables_worker's exact
* pattern (many ticks, must never get permanently stuck) into this new
* adversarial condition: resync_pending genuinely stuck true (simulating a
* stalled multi-page query whose final page never arrives) for the whole
* loop, never cleared by anything inside checking_connection() itself.
* ════════════════════════════════════════════════════════════════════════ */
static void
test_checking_connection_survives_permanently_stuck_resync_pending(void)
{
controller_connector_ctx_t fake_cc = { 0 };
fake_cc.running = true;
cc_ctx = &fake_cc;
/* No connection at all -> xcecli_hello(NULL, ...) fails immediately
* with -EINVAL, same fast/deterministic setup as
* test_checking_connection_never_permanently_disables_worker. */
xcecli_ctx = NULL;
needs_rejoin = true; /* a rejoin is genuinely pending throughout */
g_rejoin_call_count = 0;
rejoin_call = fake_rejoin_always_succeeds;
/* Simulates a multi-page job-state query that stalled after a
* non-final page (the worker's own connection dropped mid-query, so no
* further page -- and therefore no call into
* xcecd_resync_handle_query_page() that could ever clear the gate --
* will ever arrive). Left true for the entire loop below: nothing in
* this test ever clears it, matching the round-1 defect's worst case
* exactly. */
g_test_resync_pending = true;
struct timespec t0, t1;
clock_gettime(CLOCK_MONOTONIC, &t0);
/* Far more iterations than the old MAX_RETRY_COUNT (5) -- see
* test_checking_connection_never_permanently_disables_worker. Every
* single tick must still leave the worker able to detect/recover its
* own connection (cc_ctx->running never forced false), and the pending
* rejoin must never be silently dropped (needs_rejoin stays set, exactly
* because it is legitimately still deferred by the still-pending
* resync, not because the worker gave up). */
for (int i = 0; i < 25; i++) {
checking_connection();
TEST_ASSERT_TRUE_MESSAGE(
cc_ctx->running,
"a permanently-stuck resync_pending must never permanently "
"disable the worker -- checking_connection()'s liveness probe "
"and needs_rejoin check must keep running every tick "
"regardless (Requirement 6.1)");
TEST_ASSERT_TRUE_MESSAGE(
needs_rejoin,
"the rejoin action stays correctly deferred tick after tick "
"while resync_pending is genuinely stuck true -- it must "
"never be silently cleared without actually being attempted");
}
clock_gettime(CLOCK_MONOTONIC, &t1);
long elapsed_ms = (t1.tv_sec - t0.tv_sec) * 1000L
+ (t1.tv_nsec - t0.tv_nsec) / 1000000L;
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_rejoin_call_count,
"the rejoin action itself must stay deferred for as long as "
"resync_pending is genuinely (if pathologically) stuck true -- "
"narrowing the gate must not mean it stops applying");
/* 25 ticks completing well under a second proves this loop never fell
* into any blocking wait -- same bound as the sibling invariant test. */
TEST_ASSERT_LESS_THAN_INT(1000, elapsed_ms);
printf(
"test_checking_connection_survives_permanently_stuck_resync_"
"pending: PASS (elapsed_ms=%ld)\n",
elapsed_ms);
}
/* A dummy backing buffer large enough for the real api_ctx_t
* (client_internal.h) that xcecli_get_my_endpoint()/xcecli_get_myinfo()
* (real functions, src/lib/client.c) cast xcecli_ctx to internally --
* this file must not include client_internal.h itself (its api_ctx_t
* conflicts with controller_connector.h's own unused same-named type, see
* this file's top-of-file comment), but neither accessor dereferences
* anything through the returned pointer in THIS translation unit, since
* xcecd_resync_handle_query_page() is a stub here that never dereferences
* its ep/from parameters either -- only their addresses need to be
* computable, which pure pointer arithmetic over a large enough zeroed
* buffer safely provides regardless of the real struct's exact layout. */
static uint8_t g_dummy_api_ctx[8192];
/* Req 8.1/8.3: myreq_handler's OPCode_jobStateQuery case must route an
* incoming query page to xcecd_resync_handle_query_page(). */
static void test_myreq_handler_routes_jobstatequery_to_resync_module(void)
{
controller_connector_ctx_t fake_cc = { 0 };
void* worker_list_head = NULL;
fake_cc.running = true;
fake_cc.worker_list = &worker_list_head;
cc_ctx = &fake_cc;
memset(g_dummy_api_ctx, 0, sizeof(g_dummy_api_ctx));
xcecli_ctx = (xcecli_ctx_t)g_dummy_api_ctx;
uint64_t jobids[2] = { 42, 43 };
xce_job_state_query_t page = { 0 };
page.n_total = 2;
page.pos = 0;
page.count = 2;
page.jobids = jobids;
xce_request_t request = { 0 };
request.opcode = OPCode_jobStateQuery;
request.body_which = Request_body_jobstatequerymsg;
request.body.jobstatequerymsg = &page;
int rc = myreq_handler(&request, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, g_resync_handle_query_page_call_count,
"OPCode_jobStateQuery must reach xcecd_resync_handle_query_page() "
"exactly once per incoming page -- this is the real dispatch "
"wiring task 4.2 left out (no case existed for this opcode before "
"this task)");
TEST_ASSERT_EQUAL_PTR_MESSAGE(
&page, g_resync_handle_query_page_last_page,
"the decoded query page must be passed through unchanged");
xcecli_ctx = NULL;
printf("test_myreq_handler_routes_jobstatequery_to_resync_module: PASS\n");
}
/* Without an active controller connection (xcecli_ctx == NULL), the
* dispatch case must fail fast rather than call into the resync module
* with a NULL endpoint/peerinfo. */
static void
test_myreq_handler_jobstatequery_without_connection_is_rejected(void)
{
controller_connector_ctx_t fake_cc = { 0 };
void* worker_list_head = NULL;
fake_cc.running = true;
fake_cc.worker_list = &worker_list_head;
cc_ctx = &fake_cc;
xcecli_ctx = NULL;
uint64_t jobids[1] = { 1 };
xce_job_state_query_t page = { 0 };
page.n_total = 1;
page.pos = 0;
page.count = 1;
page.jobids = jobids;
xce_request_t request = { 0 };
request.opcode = OPCode_jobStateQuery;
request.body_which = Request_body_jobstatequerymsg;
request.body.jobstatequerymsg = &page;
int rc = myreq_handler(&request, NULL);
TEST_ASSERT_TRUE(rc < 0);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_resync_handle_query_page_call_count,
"without an active controller connection, the resync module must "
"never be called");
printf("test_myreq_handler_jobstatequery_without_connection_is_rejected: "
"PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* Task 2.3 — structural regression guard: xcecli_set_reconnect_handler()
* must be called AFTER xcecli_start() succeeds, never before.
@ -637,6 +981,69 @@ test_reconnect_handler_registered_after_start_not_before_in_source(void)
"source: PASS\n");
}
/* Structural companion to
* test_checking_connection_survives_permanently_stuck_resync_pending()
* above: proves WHY checking_connection() can never again become
* permanently inert the way round 1's entry-guard placement did. Same
* source-text-scan genre as the xcecli_start()-before-
* xcecli_set_reconnect_handler() guard just above, because the behavioral
* test cannot, by itself, distinguish "the gate is checked first and
* returns early" from "the gate is checked last and defers only the
* rejoin action" when needs_rejoin/resync_pending start out already true
* in both cases -- both produce the same rejoin_call_count/needs_rejoin
* observations. The distinction is textual: xcecli_hello() and the
* needs_rejoin check must appear in checking_connection()'s body strictly
* BEFORE any xcecd_resync_is_pending() call, never gated behind it. */
static void
test_checking_connection_hello_and_needs_rejoin_precede_resync_gate_in_source(
void)
{
char* src = read_whole_file(XCE_CONTROLLER_CONNECTOR_C_PATH);
TEST_ASSERT_NOT_NULL_MESSAGE(
src, "could not read controller_connector.c source for scan");
const char* fn_start = strstr(src, "int checking_connection()");
TEST_ASSERT_NOT_NULL_MESSAGE(fn_start,
"checking_connection() definition not found");
/* Search only from the function definition onward, so an unrelated,
* earlier mention of any of these tokens elsewhere in the file (e.g. in
* a doc comment) cannot produce a false pass. */
const char* hello_call = strstr(fn_start, "xcecli_hello(xcecli_ctx");
const char* needs_rejoin_check = strstr(fn_start, "if (needs_rejoin)");
const char* pending_check
= strstr(fn_start, "xcecd_resync_is_pending()");
TEST_ASSERT_NOT_NULL_MESSAGE(hello_call,
"xcecli_hello(xcecli_ctx, ...) call site "
"not found in checking_connection()");
TEST_ASSERT_NOT_NULL_MESSAGE(
needs_rejoin_check,
"if (needs_rejoin) check not found in checking_connection()");
TEST_ASSERT_NOT_NULL_MESSAGE(
pending_check,
"xcecd_resync_is_pending() call not found in checking_connection()");
TEST_ASSERT_TRUE_MESSAGE(
hello_call < needs_rejoin_check,
"xcecli_hello() must run unconditionally, before the needs_rejoin "
"check -- never gated behind resync_pending (Requirement 6.1's "
"self-healing liveness probe)");
TEST_ASSERT_TRUE_MESSAGE(
needs_rejoin_check < pending_check,
"the resync_pending gate must be checked INSIDE the needs_rejoin "
"branch (guarding only the rejoin action itself), not before it -- "
"an entry-guard placement (round 1's confirmed defect) would put "
"xcecd_resync_is_pending() before both xcecli_hello() and the "
"needs_rejoin check, permanently disabling the worker's own "
"connection-recovery mechanism whenever resync_pending gets stuck");
FREEUP(src);
printf(
"test_checking_connection_hello_and_needs_rejoin_precede_resync_"
"gate_in_source: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* Task 2.3 — Req 1.6/5.2: all-candidates-unreachable failure identifies
* every attempted candidate, not a generic error.
@ -828,6 +1235,7 @@ int main(void)
test_build_cluster_config_single_entry_peers_matches_synthesized_shape);
RUN_TEST(test_build_cluster_config_multi_candidate_passthrough);
RUN_TEST(test_rejoin_callback_sets_flag_and_returns_immediately);
RUN_TEST(test_rejoin_callback_also_resets_resync_gate);
RUN_TEST(test_rejoin_callback_registered_via_public_api);
RUN_TEST(test_checking_connection_never_permanently_disables_worker);
RUN_TEST(test_checking_connection_skips_rejoin_when_not_pending);
@ -837,8 +1245,17 @@ int main(void)
test_checking_connection_rejoin_success_with_nonzero_index_clears_flag);
RUN_TEST(
test_checking_connection_rejoin_retried_on_next_probe_after_failure);
RUN_TEST(
test_checking_connection_defers_rejoin_action_when_resync_pending);
RUN_TEST(test_checking_connection_resumes_once_resync_no_longer_pending);
RUN_TEST(
test_checking_connection_survives_permanently_stuck_resync_pending);
RUN_TEST(test_myreq_handler_routes_jobstatequery_to_resync_module);
RUN_TEST(test_myreq_handler_jobstatequery_without_connection_is_rejected);
RUN_TEST(
test_reconnect_handler_registered_after_start_not_before_in_source);
RUN_TEST(
test_checking_connection_hello_and_needs_rejoin_precede_resync_gate_in_source);
RUN_TEST(test_format_candidate_list_names_every_candidate_in_order);
RUN_TEST(test_xcecd_join_cluster_all_unreachable_reports_every_candidate);
RUN_TEST(