lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(worker-rejoin-resync): task 6.2 - route job-state report to reconciliation

Browse source 
Add OPCode_jobStateReport dispatch in myreq_handler, reassembling a
report across multiple incoming pages before handing it to task 3.2's
resync_apply_report() exactly once - proven via a test-only call
counter, not just end-state equivalence (a double-apply could pass an
end-state check if idempotent).

Standby-redirect investigation (Req 8.1/8.3): confirmed
ha_dispatch_gate already redirects every rejected opcode uniformly, so
no opcode-specific code was needed for that. OPCode_jobStateQuery has
no controller-side receive path at all (only sent, never received)
so needed no gate change either.

That same investigation found a different, more consequential gap:
fixes bug-1753, ha_dispatch_gate's quorum-loss fact-admission allowlist
(OPCode_notify/update/setpreason/clrpreason) was missing
OPCode_jobStateReport, so a report arriving during a transient
quorum-loss window - this feature's own core failover scenario - would
be rejected outright with no retry (worker sends fire-and-forget).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Y4M5MJM2zK6nU43da267Ku
This commit is contained in:
Rongsong Shen 2026-09-01 21:06:42 +00:00
parent a3d12d37d9
commit 9f655c61c7
No known key found for this signature in database
GPG key ID: 5BCDAAE0123E9FE2
12 changed files with 1178 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/controller/testsuite/controller_test.c
View file

@ -103,6 +103,7 @@ extern void test_upgrade_snapshot_restore(void);
extern void test_upgrade_snapshot_apply(void);
extern void test_bug857_regression(void);
extern void test_sched_admin_dispatch(void);
extern void test_jobstatereport_dispatch(void);
extern void test_fetchfile_disconnect(void);
extern void test_job_dispatch(void);
extern void test_dispatch_commit(void);
@ -282,6 +283,7 @@ int main(int argc, char* argv[])
TEST(test_upgrade_snapshot_apply);
TEST(test_bug857_regression);
TEST(test_sched_admin_dispatch);
TEST(test_jobstatereport_dispatch);
TEST(test_fetchfile_disconnect);
TEST(test_job_dispatch);
TEST(test_dispatch_commit);

53
src/controller/testsuite/test_commit_barrier_rollout.c
View file

@ -982,6 +982,37 @@ static void test_gate_no_quorum_admits_clrpreason_fact(void)
printf("test_gate_no_quorum_admits_clrpreason_fact: PASS\n");
}
/* ── OPCode_jobStateReport admitted without write quorum ───────────────────
* (worker-rejoin-resync task 6.2, found live while auditing this exact gate
* for task 6.2's own standby-redirect investigation -- design.md only
* anticipated the STANDBY case, per bug-1700's precedent, and did not
* separately consider a PRIMARY that has temporarily lost write quorum.
* OPCode_jobStateReport is a worker's already-happened job-state fact, sent
* fire-and-forget (xce_job_state_report_req(), request.c) with no
* redelivery on rejection -- exactly the same shape as OPCode_notify, which
* this same allowlist already exists to protect from bug-396's silent-loss
* failure mode. Rejecting a reconciliation report outright during a
* transient quorum-loss window would reproduce that exact loss mode for
* the specific facts this whole feature exists to recover (bug-1741). */
static void test_gate_no_quorum_admits_jobstatereport_fact(void)
{
int rc;
/* Arrange: quorum absent */
fact_gate_setup(3, 0);
/* Act */
rc = ha_dispatch_gate(&g_fg_ha, OPCode_jobStateReport);
/* Assert */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"OPCode_jobStateReport (worker reconciliation fact) must be "
"admitted without write quorum, exactly like OPCode_notify");
fact_gate_teardown();
printf("test_gate_no_quorum_admits_jobstatereport_fact: PASS\n");
}
/* ── PIN: request-shaped opcodes stay rejected without quorum (Req 3.2) ───
* The allowlist is FACTS ONLY: every request-shaped opcode (a requester
* awaits confirmation, served by the task-3 handler barrier) must keep
@ -1018,16 +1049,17 @@ static void test_gate_no_quorum_still_rejects_request_opcodes(void)
static void test_gate_with_quorum_admits_all_opcodes(void)
{
static const struct { int opcode; const char *name; } ops[] = {
{ OPCode_notify, "OPCode_notify" },
{ OPCode_update, "OPCode_update" },
{ OPCode_setpreason, "OPCode_setpreason" },
{ OPCode_clrpreason, "OPCode_clrpreason" },
{ OPCode_jobsub, "OPCode_jobsub" },
{ OPCode_jobcontrol, "OPCode_jobcontrol" },
{ OPCode_queuecontrol, "OPCode_queuecontrol" },
{ OPCode_hostcontrol, "OPCode_hostcontrol" },
{ OPCode_jobdispatch, "OPCode_jobdispatch" },
{ OPCode_join, "OPCode_join" },
{ OPCode_notify, "OPCode_notify" },
{ OPCode_update, "OPCode_update" },
{ OPCode_setpreason, "OPCode_setpreason" },
{ OPCode_clrpreason, "OPCode_clrpreason" },
{ OPCode_jobStateReport, "OPCode_jobStateReport" },
{ OPCode_jobsub, "OPCode_jobsub" },
{ OPCode_jobcontrol, "OPCode_jobcontrol" },
{ OPCode_queuecontrol, "OPCode_queuecontrol" },
{ OPCode_hostcontrol, "OPCode_hostcontrol" },
{ OPCode_jobdispatch, "OPCode_jobdispatch" },
{ OPCode_join, "OPCode_join" },
};
size_t i;
@ -2459,6 +2491,7 @@ void test_commit_barrier_rollout(void)
RUN_TEST(test_gate_no_quorum_admits_update_fact);
RUN_TEST(test_gate_no_quorum_admits_setpreason_fact);
RUN_TEST(test_gate_no_quorum_admits_clrpreason_fact);
RUN_TEST(test_gate_no_quorum_admits_jobstatereport_fact);
RUN_TEST(test_gate_no_quorum_still_rejects_request_opcodes);
RUN_TEST(test_gate_with_quorum_admits_all_opcodes);
/* Task 5.1 — config barrier: configAck arithmetic + upgrade failure

322
src/controller/testsuite/test_jobstatereport_dispatch.c Normal file
View file

@ -0,0 +1,322 @@
/*
* test_jobstatereport_dispatch.c — Unity tests for the OPCode_jobStateReport
* controller dispatch case (worker-rejoin-resync spec, task 6.2, Req 8.1,
* 8.3).
*
* Exercises myreq_handler's OPCode_jobStateReport case through the real
* decode+dispatch entry point exposed for tests, myreq_handler_for_test
* (cluster.c, #ifdef XCE_TESTING) — same precedent test_sched_admin_
* dispatch.c already established for a comparable primary-only opcode.
*
* Covers task 6.2's own explicit standby-redirect investigation: a report
* (or, by the same generic mechanism, a query) sent to a non-primary
* controller must receive the SAME redirect behavior an equivalent
* existing primary-only message already receives, not a bare unrecoverable
* error — proven here exactly as test_sched_admin_dispatch.c's own
* test_schedadmin_standby_with_known_primary_redirects/test_schedadmin_
* standby_no_known_primary_no_redirect prove it for OPCode_schedAdmin, the
* "real existing primary-only opcode" ground truth this task's brief
* calls for. OPCode_jobStateReport is deliberately absent from ha_
* dispatch_gate()'s CTRL_STANDBY allow-list (ha.c) — exactly like OPCode_
* schedAdmin/OPCode_upgradeConfig — so this redirect behavior requires NO
* opcode-specific code in cluster.c's own dispatch case: it is entirely a
* property of the GENERIC gate-rejection block that already runs ahead of
* every opcode's switch case (cluster.c, the bug-1700 fix). These tests
* prove that generic mechanism actually covers this opcode too, rather
* than assuming it does.
*
* The OTHER half of task 6.2's own brief — multi-page reassembly, applied
* exactly once, equivalent to a single-page report — is proven against a
* REAL worker_t and REAL job table in test_worker_resync.c (resync_handle_
* report_page()'s own direct tests), not re-proven here through the wire.
* See this file's own comment further down (immediately before the RUN_
* TEST list) for exactly why a "real wire dispatch resolves a real
* worker_t" version of that proof cannot be built in THIS test binary, and
* why that is a pre-existing, unrelated architectural property, not a gap
* in this task's own coverage.
*/
#define _GNU_SOURCE
#include <errno.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
#include "cluster.h"
#include "controller.capnp.h"
#include "ha.h"
#include "netio.h"
#include "request.h"
#include "typemap.h"
#include "xce/errnox.h"
#include "xce/types.h"
/* myreq_handler exposed for tests (cluster.c) — same seam test_sched_admin_
* dispatch.c uses. `pkt` ownership: consumed (freed) by the call. */
extern int myreq_handler_for_test(endpoint_t* ep, io_packet_t* pkt);
/* ── Fixture ────────────────────────────────────────────────────────────
*
* A bare calloc'd cluster_t, exactly like test_sched_admin_dispatch.c's
* sa_make_cluster() — sufficient for every test below, none of which
* reaches past ha_dispatch_gate() or handle_jobstatereport()'s own first
* line (get_ep_worker()).
*/
static cluster_t* jsr_make_gate_cluster(void)
{
cluster_t* c = calloc(1, sizeof(cluster_t));
TEST_ASSERT_NOT_NULL(c);
mylock_init(&(c->sched.lock));
return c;
}
static void jsr_gate_teardown(cluster_t* c)
{
if (c == NULL)
return;
mylock_fini(&(c->sched.lock));
free(c);
}
/* Build a wire-encoded OPCode_jobStateReport request packet carrying a
* minimal single-entry, single-page report — the exact content is
* immaterial for the standby-redirect tests below (the request never
* reaches the dispatch case at all). */
static io_packet_t* jsr_build_request_pkt(uint64_t jobid)
{
xce_job_state_entry_t entry;
xce_job_state_entry_t* entries[1];
xce_job_state_report_t body;
memset(&entry, 0, sizeof(entry));
entry.jobid = jobid;
entry.status = JobStateStatus_running;
entry.queried = 1;
entries[0] = &entry;
memset(&body, 0, sizeof(body));
body.n_total = 1;
body.pos = 0;
body.count = 1;
body.entries = entries;
xce_request_t req;
memset(&req, 0, sizeof(req));
req.opcode = OPCode_jobStateReport;
req.body_which = Request_body_jobstatereportmsg;
req.body.jobstatereportmsg = &body;
io_packet_t* pkt = calloc(1, sizeof(*pkt));
TEST_ASSERT_NOT_NULL(pkt);
TEST_ASSERT_TRUE_MESSAGE(xce_encode(&req, pkt) >= 0,
"failed to encode JobStateReport packet");
return pkt;
}
/* Pop the single reply packet myreq_handler queued on ep->outq (if any) and
* decode it into *out. Mirrors test_sched_admin_dispatch.c's sa_decode_
* queued_reply exactly. */
static void jsr_decode_queued_reply(endpoint_t* ep, xce_request_t* out)
{
data_item_t* it = endpoint_deq(ep);
TEST_ASSERT_NOT_NULL_MESSAGE(it, "expected a reply packet on ep->outq");
io_packet_t* pkt = ITPTR_TO_PACKET(it);
memset(out, 0, sizeof(*out));
int rc = xce_decode(pkt, out);
TEST_ASSERT_EQUAL_INT(0, rc);
}
/* ── Standby redirect (task 6.2's own explicit investigation) ─────────────
*
* Ground truth: OPCode_schedAdmin's identical fixture/assertions
* (test_sched_admin_dispatch.c), the "real existing primary-only opcode"
* this task's brief calls for. */
static void test_jobstatereport_rejects_when_not_primary(void)
{
cluster_t* c = jsr_make_gate_cluster();
atomic_store(&c->ha.role, CTRL_STANDBY);
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
ep.owner = c;
io_packet_t* pkt = jsr_build_request_pkt(1001);
int rc = myreq_handler_for_test(&ep, pkt);
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rc);
xce_request_t decoded;
jsr_decode_queued_reply(&ep, &decoded);
TEST_ASSERT_EQUAL_INT(Request_body_replymsg, decoded.body_which);
TEST_ASSERT_NOT_NULL(decoded.body.replymsg);
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, decoded.body.replymsg->status);
TEST_ASSERT_EQUAL_INT(RequestReply_data_nodata,
decoded.body.replymsg->data_which);
xce_free_request(&decoded);
jsr_gate_teardown(c);
printf("test_jobstatereport_rejects_when_not_primary: PASS\n");
}
/* bug-1700 precedent: a standby dispatch-gate rejection with a KNOWN
* primary must carry a redirect hint — proven here with NO opcode-specific
* code added anywhere for OPCode_jobStateReport: it works purely because
* this opcode is absent from ha_dispatch_gate's CTRL_STANDBY allow-list,
* so the GENERIC post-gate rejection block (cluster.c) fills the redirect
* for it exactly as it already does for OPCode_schedAdmin/OPCode_join. */
static void test_jobstatereport_standby_with_known_primary_redirects(void)
{
cluster_t* c = jsr_make_gate_cluster();
atomic_store(&c->ha.role, CTRL_STANDBY);
strncpy(c->ha.primary_host, "10.0.1.9", sizeof(c->ha.primary_host) - 1);
c->ha.primary_port = 7003;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
ep.owner = c;
io_packet_t* pkt = jsr_build_request_pkt(1002);
int rc = myreq_handler_for_test(&ep, pkt);
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rc);
xce_request_t decoded;
jsr_decode_queued_reply(&ep, &decoded);
TEST_ASSERT_EQUAL_INT(Request_body_replymsg, decoded.body_which);
xce_reply_t* rr = decoded.body.replymsg;
TEST_ASSERT_NOT_NULL(rr);
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rr->status);
TEST_ASSERT_NOT_NULL_MESSAGE(
rr->redirect_host,
"a standby dispatch-gate rejection with a known primary must carry "
"that primary's address, exactly like every other primary-only "
"opcode already does (bug-1700)");
TEST_ASSERT_EQUAL_STRING("10.0.1.9", rr->redirect_host);
TEST_ASSERT_EQUAL_UINT16(7003, rr->redirect_port);
xce_free_request(&decoded);
jsr_gate_teardown(c);
printf("test_jobstatereport_standby_with_known_primary_redirects:"
" PASS\n");
}
/* Regression companion: no known primary yet must not crash and must
* decode as a non-redirecting reply (empty host / zero port). */
static void test_jobstatereport_standby_no_known_primary_no_redirect(void)
{
cluster_t* c = jsr_make_gate_cluster();
atomic_store(&c->ha.role, CTRL_STANDBY);
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
ep.owner = c;
io_packet_t* pkt = jsr_build_request_pkt(1003);
int rc = myreq_handler_for_test(&ep, pkt);
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rc);
xce_request_t decoded;
jsr_decode_queued_reply(&ep, &decoded);
TEST_ASSERT_EQUAL_INT(Request_body_replymsg, decoded.body_which);
xce_reply_t* rr = decoded.body.replymsg;
TEST_ASSERT_NOT_NULL(rr);
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rr->status);
TEST_ASSERT_TRUE_MESSAGE(
rr->redirect_host == NULL || rr->redirect_host[0] == '\0',
"no known primary: redirect_host must decode as empty/absent");
TEST_ASSERT_EQUAL_UINT16(0, rr->redirect_port);
xce_free_request(&decoded);
jsr_gate_teardown(c);
printf("test_jobstatereport_standby_no_known_primary_no_redirect:"
" PASS\n");
}
/* A PRIMARY with write quorum handles the request normally — dispatched
* past the gate into handle_jobstatereport(), which returns -ENOENT for an
* endpoint that does not resolve to a known worker (this bare fixture's
* `ep` was never registered as one) rather than any standby-style
* rejection. Proves the gate's happy path is unaffected by this task's
* change and that the new dispatch case is genuinely reached. */
static void test_jobstatereport_primary_reaches_dispatch_case(void)
{
cluster_t* c = jsr_make_gate_cluster();
atomic_store(&c->ha.role, CTRL_PRIMARY);
/* n_peers defaults to 0 (calloc) -- single-node cluster, trivially has
* write quorum (ha_has_write_quorum: n_peers <= 1 -> true). */
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
ep.owner = c;
/* ep.data left NULL -> get_ep_role()/get_ep_worker() treat this as "not
* a worker endpoint" and return NULL/-EINVAL, exercising handle_
* jobstatereport's own not-a-worker branch. */
io_packet_t* pkt = jsr_build_request_pkt(1004);
int rc = myreq_handler_for_test(&ep, pkt);
TEST_ASSERT_EQUAL_INT_MESSAGE(-ENOENT, rc,
"a primary with quorum must reach handle_jobstatereport() itself "
"(not a gate rejection) -- which reports -ENOENT for an endpoint "
"with no resolvable worker_t");
jsr_gate_teardown(c);
printf("test_jobstatereport_primary_reaches_dispatch_case: PASS\n");
}
/* ── Why no "real wire dispatch resolves a real worker_t" test lives here ──
*
* A genuine end-to-end test (two real wire-encoded OPCode_jobStateReport
* packets through myreq_handler_for_test, resolving a REAL worker_t via
* get_ep_worker() and converging a REAL job table) was attempted here and
* found structurally impossible in THIS binary: get_ep_worker() (misc.c)
* is NOT one of controller_unit_test_src's linked sources (meson.build) --
* test_commit_barrier_reply.c instead provides controller_unit_test's ONLY
* definition of it, a fixed test double (`return g_stub_ep_worker;`,
* file-static, defaulting to NULL and settable only by that file's own
* tests) that no other test file can point at a real worker_t. Confirmed
* live (not assumed) by direct instrumentation: a page built with a
* correct, verified-by-hand ep_private_t{role=HostRole_worker, idx=w->idx}
* -- independently proven resolvable via key_find()/vget() called directly
* against the same workermgr -- still made get_ep_worker() return NULL,
* because the symbol actually linked into this binary is the stub, not
* misc.c's real implementation.
*
* This is a pre-existing architectural property of controller_unit_test
* (several other cluster.c-adjacent functions -- notify(), update_host_
* runusage(), on_scheduler_join(), add_apiserver(), on_apiserver_join() --
* are stubbed the same way in the same file, for the same reason), not
* something this task introduced or can fix without either modifying
* shared test infrastructure many other tests depend on (out of this
* task's boundary) or standing up a new dedicated test binary that links
* the real misc.c (the pattern host_runusage_unit_test/resval_typeresolve_
* unit_test already establish for exactly this class of problem, bug-1735)
* -- a real option, but a disproportionate scope expansion for what the
* unit tests below plus test_worker_resync.c's own tests already prove
* between them: handle_jobstatereport() (cluster.c) is a two-line function
* (get_ep_worker() then resync_handle_report_page()) whose own correctness
* is not in question -- the two things worth proving are (a) cluster.c's
* switch genuinely routes OPCode_jobStateReport to it, proven above by
* test_jobstatereport_primary_reaches_dispatch_case's distinctive -ENOENT
* return (get_ep_worker's stub-NULL path, reached only via this exact
* call chain), and (b) multi-page reassembly + exactly-once apply +
* equivalence-to-single-page, proven exhaustively against a REAL worker_t
* and REAL job table by resync_handle_report_page()'s own direct tests in
* test_worker_resync.c (test_handle_report_page_multi_page_reassembles_
* and_applies_once and its siblings) -- the same function handle_
* jobstatereport() calls, unmodified.
*/
void test_jobstatereport_dispatch(void)
{
UnitySetTestFile(__FILE__);
RUN_TEST(test_jobstatereport_rejects_when_not_primary);
RUN_TEST(test_jobstatereport_standby_with_known_primary_redirects);
RUN_TEST(test_jobstatereport_standby_no_known_primary_no_redirect);
RUN_TEST(test_jobstatereport_primary_reaches_dispatch_case);
}

387
src/controller/testsuite/test_worker_resync.c
View file

@ -2052,6 +2052,389 @@ test_apply_report_unqueried_extra_unassigned_pend_job_gets_assigned(void)
"assigned: PASS\n");
}
/* ══════════════════════════════════════════════════════════════════════
* Task 6.2 (Req 8.1, 8.3): resync_handle_report_page() -- multi-page wire
* reassembly of an incoming OPCode_jobStateReport, ahead of resync_apply_
* report() (task 3.2, proven above). Every test here proves the exact
* gap this task's own brief called out: not just that the correct END
* STATE results, but that resync_apply_report() is invoked EXACTLY ONCE
* per report regardless of page count (resync_apply_report_call_count_
* for_test(), resync.h) -- job.c's own idempotent fact guards would make
* even a buggy once-per-page implementation LOOK correct on a happy-path
* report, exactly the class of bug task 4.3's own round-1 review missed
* with a purely behavioral test (tasks.md Implementation Notes).
* ══════════════════════════════════════════════════════════════════════ */
#define TWR_JOBID_MPAGE_FINOK 81201ULL
#define TWR_JOBID_MPAGE_UNREC 81202ULL
#define TWR_JOBID_MPAGE_RUNNING 81203ULL
#define TWR_JOBID_MPAGE_STALE 81210ULL
#define TWR_JOBID_MPAGE_BAD 81220ULL
#define TWR_JOBID_MPAGE_BAD2 81221ULL
/* Baseline: a single-page report (pos=0, count==n_total) delivered through
* resync_handle_report_page() -- not resync_apply_report() directly -- is
* reassembled trivially (one page IS the whole report) and applied exactly
* once. Establishes the expected end state the multi-page test below must
* independently reproduce via genuine reassembly. */
static void test_handle_report_page_single_page_baseline(void)
{
job_t* j_unrec;
worker_t* w;
int rc;
uint64_t token;
int calls_before, calls_after;
xce_job_state_entry_t e_unrec;
xce_job_state_entry_t* entries[1];
xce_job_state_report_t page;
twr_setup();
j_unrec = twr_seed_job(TWR_JOBID_MPAGE_UNREC, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j_unrec);
j_unrec->job.allocs
= twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_UNREC, "mpage-host-1");
w = twr_make_test_worker("mpage-host-1");
TEST_ASSERT_NOT_NULL(w);
rc = resync_arm_round(&g_twr, w);
TEST_ASSERT_EQUAL_INT(1, rc);
token = w->resync->round_token;
e_unrec = (xce_job_state_entry_t){
.jobid = TWR_JOBID_MPAGE_UNREC, .status = JobStateStatus_unrecognized,
.exit_code = 0, .finish_time_ms = 0, .queried = 1
};
entries[0] = &e_unrec;
page = (xce_job_state_report_t){
.n_total = 1, .pos = 0, .count = 1, .entries = entries
};
resync_apply_report_call_count_reset_for_test();
calls_before = resync_apply_report_call_count_for_test();
rc = resync_handle_report_page(&g_twr, w, &page);
calls_after = resync_apply_report_call_count_for_test();
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
"a single page whose count == n_total completes the report "
"immediately");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, calls_after - calls_before,
"resync_apply_report must be invoked exactly once");
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, j_unrec->job.state);
TEST_ASSERT_NULL(j_unrec->job.allocs);
TEST_ASSERT_NULL_MESSAGE(w->resync,
"the round must resolve once its only queried job is answered");
(void)token;
twr_teardown();
printf("test_handle_report_page_single_page_baseline: PASS\n");
}
/* THE core proof (task 6.2's own Observable Completion Criterion): the SAME
* report, split across two wire pages, must (a) leave the job table
* COMPLETELY untouched after the first (non-final) page -- no partial
* application -- (b) converge to EXACTLY the same end state the single-page
* baseline above reached, once the final page arrives, and (c) invoke
* resync_apply_report() EXACTLY ONCE for the whole report, not once per
* page (the structural proof a behavioral-only assertion cannot provide --
* see this section's own header comment). */
static void test_handle_report_page_multi_page_reassembles_and_applies_once(
void)
{
job_t* j_finok;
job_t* j_unrec;
job_t* j_running;
worker_t* w;
int rc;
int calls_before, calls_mid, calls_after;
xce_job_state_entry_t e_finok, e_unrec, e_running;
xce_job_state_entry_t* entries_p1[2];
xce_job_state_entry_t* entries_p2[1];
xce_job_state_report_t page1, page2;
twr_setup();
j_finok = twr_seed_job(TWR_JOBID_MPAGE_FINOK, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j_finok);
j_finok->job.allocs
= twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_FINOK, "mpage-host-2");
j_unrec = twr_seed_job(TWR_JOBID_MPAGE_UNREC, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j_unrec);
j_unrec->job.allocs
= twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_UNREC, "mpage-host-2");
j_running = twr_seed_job(TWR_JOBID_MPAGE_RUNNING, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j_running);
j_running->job.allocs = twr_make_test_alloc_for_host(
TWR_JOBID_MPAGE_RUNNING, "mpage-host-2");
w = twr_make_test_worker("mpage-host-2");
TEST_ASSERT_NOT_NULL(w);
rc = resync_arm_round(&g_twr, w);
TEST_ASSERT_EQUAL_INT_MESSAGE(3, rc,
"all 3 non-terminal jobs on this host must be armed");
e_finok = (xce_job_state_entry_t){
.jobid = TWR_JOBID_MPAGE_FINOK, .status = JobStateStatus_finishedOk,
.exit_code = 0, .finish_time_ms = 4242, .queried = 1
};
e_unrec = (xce_job_state_entry_t){
.jobid = TWR_JOBID_MPAGE_UNREC, .status = JobStateStatus_unrecognized,
.exit_code = 0, .finish_time_ms = 0, .queried = 1
};
e_running = (xce_job_state_entry_t){
.jobid = TWR_JOBID_MPAGE_RUNNING, .status = JobStateStatus_running,
.exit_code = 0, .finish_time_ms = 0, .queried = 1
};
/* Page 1 of 2: entries [0..1] (finok, unrec). Page 2 of 2: entry [2]
* (running). Mirrors xce_job_state_report_req()'s own {pos,count,
* n_total} pagination shape (request.c) -- this is exactly what a real
* multi-page OPCode_jobStateReport looks like on the wire. */
entries_p1[0] = &e_finok;
entries_p1[1] = &e_unrec;
page1 = (xce_job_state_report_t){
.n_total = 3, .pos = 0, .count = 2, .entries = entries_p1
};
entries_p2[0] = &e_running;
page2 = (xce_job_state_report_t){
.n_total = 3, .pos = 2, .count = 1, .entries = entries_p2
};
resync_apply_report_call_count_reset_for_test();
calls_before = resync_apply_report_call_count_for_test();
rc = resync_handle_report_page(&g_twr, w, &page1);
calls_mid = resync_apply_report_call_count_for_test();
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"page 1 of 2 alone must NOT complete the report");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, calls_mid - calls_before,
"resync_apply_report must NOT be called after a non-final page -- "
"the structural 'not once per page' proof");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_finok->job.state,
"no partial application: job A must be untouched after page 1");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_unrec->job.state,
"no partial application: job B must be untouched after page 1 -- "
"still RUN, NOT yet reconciled to PEND");
TEST_ASSERT_NOT_NULL_MESSAGE(j_unrec->job.allocs,
"no partial application: job B's allocation must not be released "
"until the report is fully assembled and applied");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_running->job.state,
"job C (only referenced in page 2, not yet arrived) is untouched");
TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
"the round must still be outstanding after only a partial report");
rc = resync_handle_report_page(&g_twr, w, &page2);
calls_after = resync_apply_report_call_count_for_test();
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
"page 2 of 2 completes the report");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, calls_after - calls_mid,
"resync_apply_report must be invoked EXACTLY ONCE, on the final "
"page, for the whole 3-entry report -- not once per page");
/* Equivalence (task 6.2's own Observable Completion Criterion): the SAME
* end state test_handle_report_page_single_page_baseline reached for an
* equivalent single-page report. */
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_finok->job.state,
"finished-success stays in the DEFERRED record_job_finish_fact_at "
"pattern -- state mutates later via apply_committed_entry, same as "
"an equivalent single-page report (test_apply_report_all_five_"
"outcomes_updates_job_table proves this deferred path directly)");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j_unrec->job.state,
"job B reconciles to PEND -- identical to the single-page baseline");
TEST_ASSERT_NULL_MESSAGE(j_unrec->job.allocs,
"job B's allocation is released -- identical to the single-page "
"baseline");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_running->job.state,
"job C (queried running) is left exactly as-is -- no fact "
"recorded, identical to the single-page baseline");
TEST_ASSERT_NULL_MESSAGE(w->resync,
"once every queried job across BOTH pages has a reconciled "
"outcome, the round must resolve -- identical to the single-page "
"baseline");
TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
"XCE_WORKER_RECONCILING must clear once the round resolves");
twr_teardown();
printf("test_handle_report_page_multi_page_reassembles_and_applies_once:"
" PASS\n");
}
/* Req 6.1/6.3 at the reassembly layer: a report that STARTS arriving under
* one round but finishes assembling only after that round has been
* superseded by a fresh resync_arm_round() call (simulating a disconnect-
* then-immediate-rejoin racing the tail of a still-in-flight report) must
* be discarded in its entirety -- not matched against whichever round
* happens to be current once the LAST page arrives. Proves the round_token
* CAPTURE point (page 1, resync_report_accum_t.round_token) is what makes
* this correct, per resync.h's own doc comment on why that capture is not
* an arbitrary choice. */
static void test_handle_report_page_stale_round_discarded_when_superseded(
void)
{
job_t* j;
worker_t* w;
int rc;
int calls_before, calls_after;
xce_job_state_entry_t e1, e2;
xce_job_state_entry_t* entries_p1[1];
xce_job_state_entry_t* entries_p2[1];
xce_job_state_report_t page1, page2;
twr_setup();
j = twr_seed_job(TWR_JOBID_MPAGE_STALE, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs
= twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_STALE, "mpage-host-3");
w = twr_make_test_worker("mpage-host-3");
TEST_ASSERT_NOT_NULL(w);
rc = resync_arm_round(&g_twr, w);
TEST_ASSERT_EQUAL_INT(1, rc);
e1 = (xce_job_state_entry_t){
.jobid = TWR_JOBID_MPAGE_STALE, .status = JobStateStatus_finishedOk,
.exit_code = 0, .finish_time_ms = 1, .queried = 1
};
entries_p1[0] = &e1;
page1 = (xce_job_state_report_t){
.n_total = 2, .pos = 0, .count = 1, .entries = entries_p1
};
rc = resync_handle_report_page(&g_twr, w, &page1);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Simulated disconnect-then-rejoin racing the tail of the in-flight
* report above: a fresh round supersedes the one page1 started under,
* BEFORE page 2 ever arrives. */
rc = resync_arm_round(&g_twr, w);
TEST_ASSERT_EQUAL_INT(1, rc);
TEST_ASSERT_NOT_NULL(w->resync);
/* A second entry, unrelated jobid, just to complete the (now-stale)
* report's page count -- its content is immaterial since the whole
* report must be discarded regardless. */
e2 = (xce_job_state_entry_t){
.jobid = TWR_JOBID_MPAGE_STALE, .status = JobStateStatus_finishedFail,
.exit_code = 0, .finish_time_ms = 2, .queried = 1
};
entries_p2[0] = &e2;
page2 = (xce_job_state_report_t){
.n_total = 2, .pos = 1, .count = 1, .entries = entries_p2
};
resync_apply_report_call_count_reset_for_test();
calls_before = resync_apply_report_call_count_for_test();
rc = resync_handle_report_page(&g_twr, w, &page2);
calls_after = resync_apply_report_call_count_for_test();
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
"the report completes (page 2 arrived) and IS handed to "
"resync_apply_report -- which then discards it internally as "
"stale, per its own Req 6.1/6.3 contract");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, calls_after - calls_before,
"resync_apply_report is still called exactly once -- discarding "
"happens INSIDE it (round-token mismatch), not by skipping the "
"call");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j->job.state,
"the stale report must never touch the job table");
TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
"the CURRENT (superseding) round must be left outstanding, "
"untouched by the stale report");
TEST_ASSERT_EQUAL_INT(1, w->resync->n_queried_jobids);
resync_resolve_round(&g_twr, w);
twr_teardown();
printf("test_handle_report_page_stale_round_discarded_when_superseded:"
" PASS\n");
}
/* Robustness: a page whose [pos, pos+count) range falls outside the
* report's own declared n_total is rejected (-EINVAL) and discards
* whatever partial accumulation was in progress -- proven by immediately
* starting a FRESH, valid report afterward and confirming it applies
* correctly, i.e. the corrupt attempt left no residue behind to corrupt a
* later, legitimate report on the same worker. */
static void test_handle_report_page_malformed_bounds_rejected_and_recovers(
void)
{
job_t* j;
worker_t* w;
int rc;
xce_job_state_entry_t e_bad, e_good;
xce_job_state_entry_t* entries_bad[1];
xce_job_state_entry_t* entries_good[1];
xce_job_state_report_t page_bad, page_good;
twr_setup();
j = twr_seed_job(TWR_JOBID_MPAGE_BAD, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs
= twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_BAD, "mpage-host-4");
w = twr_make_test_worker("mpage-host-4");
TEST_ASSERT_NOT_NULL(w);
rc = resync_arm_round(&g_twr, w);
TEST_ASSERT_EQUAL_INT(1, rc);
/* pos(1) + count(1) == 2 > n_total(1): out of bounds. */
e_bad = (xce_job_state_entry_t){
.jobid = TWR_JOBID_MPAGE_BAD2, .status = JobStateStatus_running,
.exit_code = 0, .finish_time_ms = 0, .queried = 0
};
entries_bad[0] = &e_bad;
page_bad = (xce_job_state_report_t){
.n_total = 1, .pos = 1, .count = 1, .entries = entries_bad
};
rc = resync_handle_report_page(&g_twr, w, &page_bad);
TEST_ASSERT_EQUAL_INT_MESSAGE(-EINVAL, rc,
"an out-of-bounds page must be rejected");
/* A fresh, valid, single-page report for the SAME worker right after --
* proves the malformed attempt above left no corrupted accumulator
* state behind. */
e_good = (xce_job_state_entry_t){
.jobid = TWR_JOBID_MPAGE_BAD, .status = JobStateStatus_unrecognized,
.exit_code = 0, .finish_time_ms = 0, .queried = 1
};
entries_good[0] = &e_good;
page_good = (xce_job_state_report_t){
.n_total = 1, .pos = 0, .count = 1, .entries = entries_good
};
rc = resync_handle_report_page(&g_twr, w, &page_good);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
"a subsequent valid report must apply normally");
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, j->job.state);
TEST_ASSERT_NULL(j->job.allocs);
TEST_ASSERT_NULL(w->resync);
twr_teardown();
printf("test_handle_report_page_malformed_bounds_rejected_and_recovers:"
" PASS\n");
}
/* ── Test group entry point ───────────────────────────────────────────── */
void test_worker_resync(void)
@ -2076,4 +2459,8 @@ void test_worker_resync(void)
test_apply_report_finished_success_uses_worker_reported_finish_time);
RUN_TEST(
test_apply_report_unqueried_extra_unassigned_pend_job_gets_assigned);
RUN_TEST(test_handle_report_page_single_page_baseline);
RUN_TEST(test_handle_report_page_multi_page_reassembles_and_applies_once);
RUN_TEST(test_handle_report_page_stale_round_discarded_when_superseded);
RUN_TEST(test_handle_report_page_malformed_bounds_rejected_and_recovers);
}