feat(worker-rejoin-resync): task 6.2 - route job-state report to reconciliation
Add OPCode_jobStateReport dispatch in myreq_handler, reassembling a report across multiple incoming pages before handing it to task 3.2's resync_apply_report() exactly once - proven via a test-only call counter, not just end-state equivalence (a double-apply could pass an end-state check if idempotent). Standby-redirect investigation (Req 8.1/8.3): confirmed ha_dispatch_gate already redirects every rejected opcode uniformly, so no opcode-specific code was needed for that. OPCode_jobStateQuery has no controller-side receive path at all (only sent, never received) so needed no gate change either. That same investigation found a different, more consequential gap: fixes bug-1753, ha_dispatch_gate's quorum-loss fact-admission allowlist (OPCode_notify/update/setpreason/clrpreason) was missing OPCode_jobStateReport, so a report arriving during a transient quorum-loss window - this feature's own core failover scenario - would be rejected outright with no retry (worker sends fire-and-forget). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Y4M5MJM2zK6nU43da267Ku
This commit is contained in:
parent
a3d12d37d9
commit
9f655c61c7
12 changed files with 1178 additions and 12 deletions
|
|
@ -2052,6 +2052,389 @@ test_apply_report_unqueried_extra_unassigned_pend_job_gets_assigned(void)
|
|||
"assigned: PASS\n");
|
||||
}
|
||||
|
||||
/* ══════════════════════════════════════════════════════════════════════
|
||||
* Task 6.2 (Req 8.1, 8.3): resync_handle_report_page() -- multi-page wire
|
||||
* reassembly of an incoming OPCode_jobStateReport, ahead of resync_apply_
|
||||
* report() (task 3.2, proven above). Every test here proves the exact
|
||||
* gap this task's own brief called out: not just that the correct END
|
||||
* STATE results, but that resync_apply_report() is invoked EXACTLY ONCE
|
||||
* per report regardless of page count (resync_apply_report_call_count_
|
||||
* for_test(), resync.h) -- job.c's own idempotent fact guards would make
|
||||
* even a buggy once-per-page implementation LOOK correct on a happy-path
|
||||
* report, exactly the class of bug task 4.3's own round-1 review missed
|
||||
* with a purely behavioral test (tasks.md Implementation Notes).
|
||||
* ══════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
#define TWR_JOBID_MPAGE_FINOK 81201ULL
|
||||
#define TWR_JOBID_MPAGE_UNREC 81202ULL
|
||||
#define TWR_JOBID_MPAGE_RUNNING 81203ULL
|
||||
#define TWR_JOBID_MPAGE_STALE 81210ULL
|
||||
#define TWR_JOBID_MPAGE_BAD 81220ULL
|
||||
#define TWR_JOBID_MPAGE_BAD2 81221ULL
|
||||
|
||||
/* Baseline: a single-page report (pos=0, count==n_total) delivered through
|
||||
* resync_handle_report_page() -- not resync_apply_report() directly -- is
|
||||
* reassembled trivially (one page IS the whole report) and applied exactly
|
||||
* once. Establishes the expected end state the multi-page test below must
|
||||
* independently reproduce via genuine reassembly. */
|
||||
static void test_handle_report_page_single_page_baseline(void)
|
||||
{
|
||||
job_t* j_unrec;
|
||||
worker_t* w;
|
||||
int rc;
|
||||
uint64_t token;
|
||||
int calls_before, calls_after;
|
||||
|
||||
xce_job_state_entry_t e_unrec;
|
||||
xce_job_state_entry_t* entries[1];
|
||||
xce_job_state_report_t page;
|
||||
|
||||
twr_setup();
|
||||
|
||||
j_unrec = twr_seed_job(TWR_JOBID_MPAGE_UNREC, JOB_STAT_RUN);
|
||||
TEST_ASSERT_NOT_NULL(j_unrec);
|
||||
j_unrec->job.allocs
|
||||
= twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_UNREC, "mpage-host-1");
|
||||
|
||||
w = twr_make_test_worker("mpage-host-1");
|
||||
TEST_ASSERT_NOT_NULL(w);
|
||||
|
||||
rc = resync_arm_round(&g_twr, w);
|
||||
TEST_ASSERT_EQUAL_INT(1, rc);
|
||||
token = w->resync->round_token;
|
||||
|
||||
e_unrec = (xce_job_state_entry_t){
|
||||
.jobid = TWR_JOBID_MPAGE_UNREC, .status = JobStateStatus_unrecognized,
|
||||
.exit_code = 0, .finish_time_ms = 0, .queried = 1
|
||||
};
|
||||
entries[0] = &e_unrec;
|
||||
page = (xce_job_state_report_t){
|
||||
.n_total = 1, .pos = 0, .count = 1, .entries = entries
|
||||
};
|
||||
|
||||
resync_apply_report_call_count_reset_for_test();
|
||||
calls_before = resync_apply_report_call_count_for_test();
|
||||
|
||||
rc = resync_handle_report_page(&g_twr, w, &page);
|
||||
|
||||
calls_after = resync_apply_report_call_count_for_test();
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
|
||||
"a single page whose count == n_total completes the report "
|
||||
"immediately");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, calls_after - calls_before,
|
||||
"resync_apply_report must be invoked exactly once");
|
||||
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, j_unrec->job.state);
|
||||
TEST_ASSERT_NULL(j_unrec->job.allocs);
|
||||
TEST_ASSERT_NULL_MESSAGE(w->resync,
|
||||
"the round must resolve once its only queried job is answered");
|
||||
|
||||
(void)token;
|
||||
twr_teardown();
|
||||
printf("test_handle_report_page_single_page_baseline: PASS\n");
|
||||
}
|
||||
|
||||
/* THE core proof (task 6.2's own Observable Completion Criterion): the SAME
|
||||
* report, split across two wire pages, must (a) leave the job table
|
||||
* COMPLETELY untouched after the first (non-final) page -- no partial
|
||||
* application -- (b) converge to EXACTLY the same end state the single-page
|
||||
* baseline above reached, once the final page arrives, and (c) invoke
|
||||
* resync_apply_report() EXACTLY ONCE for the whole report, not once per
|
||||
* page (the structural proof a behavioral-only assertion cannot provide --
|
||||
* see this section's own header comment). */
|
||||
static void test_handle_report_page_multi_page_reassembles_and_applies_once(
|
||||
void)
|
||||
{
|
||||
job_t* j_finok;
|
||||
job_t* j_unrec;
|
||||
job_t* j_running;
|
||||
worker_t* w;
|
||||
int rc;
|
||||
int calls_before, calls_mid, calls_after;
|
||||
|
||||
xce_job_state_entry_t e_finok, e_unrec, e_running;
|
||||
xce_job_state_entry_t* entries_p1[2];
|
||||
xce_job_state_entry_t* entries_p2[1];
|
||||
xce_job_state_report_t page1, page2;
|
||||
|
||||
twr_setup();
|
||||
|
||||
j_finok = twr_seed_job(TWR_JOBID_MPAGE_FINOK, JOB_STAT_RUN);
|
||||
TEST_ASSERT_NOT_NULL(j_finok);
|
||||
j_finok->job.allocs
|
||||
= twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_FINOK, "mpage-host-2");
|
||||
|
||||
j_unrec = twr_seed_job(TWR_JOBID_MPAGE_UNREC, JOB_STAT_RUN);
|
||||
TEST_ASSERT_NOT_NULL(j_unrec);
|
||||
j_unrec->job.allocs
|
||||
= twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_UNREC, "mpage-host-2");
|
||||
|
||||
j_running = twr_seed_job(TWR_JOBID_MPAGE_RUNNING, JOB_STAT_RUN);
|
||||
TEST_ASSERT_NOT_NULL(j_running);
|
||||
j_running->job.allocs = twr_make_test_alloc_for_host(
|
||||
TWR_JOBID_MPAGE_RUNNING, "mpage-host-2");
|
||||
|
||||
w = twr_make_test_worker("mpage-host-2");
|
||||
TEST_ASSERT_NOT_NULL(w);
|
||||
|
||||
rc = resync_arm_round(&g_twr, w);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(3, rc,
|
||||
"all 3 non-terminal jobs on this host must be armed");
|
||||
|
||||
e_finok = (xce_job_state_entry_t){
|
||||
.jobid = TWR_JOBID_MPAGE_FINOK, .status = JobStateStatus_finishedOk,
|
||||
.exit_code = 0, .finish_time_ms = 4242, .queried = 1
|
||||
};
|
||||
e_unrec = (xce_job_state_entry_t){
|
||||
.jobid = TWR_JOBID_MPAGE_UNREC, .status = JobStateStatus_unrecognized,
|
||||
.exit_code = 0, .finish_time_ms = 0, .queried = 1
|
||||
};
|
||||
e_running = (xce_job_state_entry_t){
|
||||
.jobid = TWR_JOBID_MPAGE_RUNNING, .status = JobStateStatus_running,
|
||||
.exit_code = 0, .finish_time_ms = 0, .queried = 1
|
||||
};
|
||||
|
||||
/* Page 1 of 2: entries [0..1] (finok, unrec). Page 2 of 2: entry [2]
|
||||
* (running). Mirrors xce_job_state_report_req()'s own {pos,count,
|
||||
* n_total} pagination shape (request.c) -- this is exactly what a real
|
||||
* multi-page OPCode_jobStateReport looks like on the wire. */
|
||||
entries_p1[0] = &e_finok;
|
||||
entries_p1[1] = &e_unrec;
|
||||
page1 = (xce_job_state_report_t){
|
||||
.n_total = 3, .pos = 0, .count = 2, .entries = entries_p1
|
||||
};
|
||||
|
||||
entries_p2[0] = &e_running;
|
||||
page2 = (xce_job_state_report_t){
|
||||
.n_total = 3, .pos = 2, .count = 1, .entries = entries_p2
|
||||
};
|
||||
|
||||
resync_apply_report_call_count_reset_for_test();
|
||||
calls_before = resync_apply_report_call_count_for_test();
|
||||
|
||||
rc = resync_handle_report_page(&g_twr, w, &page1);
|
||||
|
||||
calls_mid = resync_apply_report_call_count_for_test();
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
|
||||
"page 1 of 2 alone must NOT complete the report");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, calls_mid - calls_before,
|
||||
"resync_apply_report must NOT be called after a non-final page -- "
|
||||
"the structural 'not once per page' proof");
|
||||
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_finok->job.state,
|
||||
"no partial application: job A must be untouched after page 1");
|
||||
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_unrec->job.state,
|
||||
"no partial application: job B must be untouched after page 1 -- "
|
||||
"still RUN, NOT yet reconciled to PEND");
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(j_unrec->job.allocs,
|
||||
"no partial application: job B's allocation must not be released "
|
||||
"until the report is fully assembled and applied");
|
||||
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_running->job.state,
|
||||
"job C (only referenced in page 2, not yet arrived) is untouched");
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
|
||||
"the round must still be outstanding after only a partial report");
|
||||
|
||||
rc = resync_handle_report_page(&g_twr, w, &page2);
|
||||
|
||||
calls_after = resync_apply_report_call_count_for_test();
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
|
||||
"page 2 of 2 completes the report");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, calls_after - calls_mid,
|
||||
"resync_apply_report must be invoked EXACTLY ONCE, on the final "
|
||||
"page, for the whole 3-entry report -- not once per page");
|
||||
|
||||
/* Equivalence (task 6.2's own Observable Completion Criterion): the SAME
|
||||
* end state test_handle_report_page_single_page_baseline reached for an
|
||||
* equivalent single-page report. */
|
||||
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_finok->job.state,
|
||||
"finished-success stays in the DEFERRED record_job_finish_fact_at "
|
||||
"pattern -- state mutates later via apply_committed_entry, same as "
|
||||
"an equivalent single-page report (test_apply_report_all_five_"
|
||||
"outcomes_updates_job_table proves this deferred path directly)");
|
||||
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j_unrec->job.state,
|
||||
"job B reconciles to PEND -- identical to the single-page baseline");
|
||||
TEST_ASSERT_NULL_MESSAGE(j_unrec->job.allocs,
|
||||
"job B's allocation is released -- identical to the single-page "
|
||||
"baseline");
|
||||
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_running->job.state,
|
||||
"job C (queried running) is left exactly as-is -- no fact "
|
||||
"recorded, identical to the single-page baseline");
|
||||
TEST_ASSERT_NULL_MESSAGE(w->resync,
|
||||
"once every queried job across BOTH pages has a reconciled "
|
||||
"outcome, the round must resolve -- identical to the single-page "
|
||||
"baseline");
|
||||
TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
|
||||
"XCE_WORKER_RECONCILING must clear once the round resolves");
|
||||
|
||||
twr_teardown();
|
||||
printf("test_handle_report_page_multi_page_reassembles_and_applies_once:"
|
||||
" PASS\n");
|
||||
}
|
||||
|
||||
/* Req 6.1/6.3 at the reassembly layer: a report that STARTS arriving under
|
||||
* one round but finishes assembling only after that round has been
|
||||
* superseded by a fresh resync_arm_round() call (simulating a disconnect-
|
||||
* then-immediate-rejoin racing the tail of a still-in-flight report) must
|
||||
* be discarded in its entirety -- not matched against whichever round
|
||||
* happens to be current once the LAST page arrives. Proves the round_token
|
||||
* CAPTURE point (page 1, resync_report_accum_t.round_token) is what makes
|
||||
* this correct, per resync.h's own doc comment on why that capture is not
|
||||
* an arbitrary choice. */
|
||||
static void test_handle_report_page_stale_round_discarded_when_superseded(
|
||||
void)
|
||||
{
|
||||
job_t* j;
|
||||
worker_t* w;
|
||||
int rc;
|
||||
int calls_before, calls_after;
|
||||
|
||||
xce_job_state_entry_t e1, e2;
|
||||
xce_job_state_entry_t* entries_p1[1];
|
||||
xce_job_state_entry_t* entries_p2[1];
|
||||
xce_job_state_report_t page1, page2;
|
||||
|
||||
twr_setup();
|
||||
|
||||
j = twr_seed_job(TWR_JOBID_MPAGE_STALE, JOB_STAT_RUN);
|
||||
TEST_ASSERT_NOT_NULL(j);
|
||||
j->job.allocs
|
||||
= twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_STALE, "mpage-host-3");
|
||||
|
||||
w = twr_make_test_worker("mpage-host-3");
|
||||
TEST_ASSERT_NOT_NULL(w);
|
||||
|
||||
rc = resync_arm_round(&g_twr, w);
|
||||
TEST_ASSERT_EQUAL_INT(1, rc);
|
||||
|
||||
e1 = (xce_job_state_entry_t){
|
||||
.jobid = TWR_JOBID_MPAGE_STALE, .status = JobStateStatus_finishedOk,
|
||||
.exit_code = 0, .finish_time_ms = 1, .queried = 1
|
||||
};
|
||||
entries_p1[0] = &e1;
|
||||
page1 = (xce_job_state_report_t){
|
||||
.n_total = 2, .pos = 0, .count = 1, .entries = entries_p1
|
||||
};
|
||||
|
||||
rc = resync_handle_report_page(&g_twr, w, &page1);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
|
||||
/* Simulated disconnect-then-rejoin racing the tail of the in-flight
|
||||
* report above: a fresh round supersedes the one page1 started under,
|
||||
* BEFORE page 2 ever arrives. */
|
||||
rc = resync_arm_round(&g_twr, w);
|
||||
TEST_ASSERT_EQUAL_INT(1, rc);
|
||||
TEST_ASSERT_NOT_NULL(w->resync);
|
||||
|
||||
/* A second entry, unrelated jobid, just to complete the (now-stale)
|
||||
* report's page count -- its content is immaterial since the whole
|
||||
* report must be discarded regardless. */
|
||||
e2 = (xce_job_state_entry_t){
|
||||
.jobid = TWR_JOBID_MPAGE_STALE, .status = JobStateStatus_finishedFail,
|
||||
.exit_code = 0, .finish_time_ms = 2, .queried = 1
|
||||
};
|
||||
entries_p2[0] = &e2;
|
||||
page2 = (xce_job_state_report_t){
|
||||
.n_total = 2, .pos = 1, .count = 1, .entries = entries_p2
|
||||
};
|
||||
|
||||
resync_apply_report_call_count_reset_for_test();
|
||||
calls_before = resync_apply_report_call_count_for_test();
|
||||
|
||||
rc = resync_handle_report_page(&g_twr, w, &page2);
|
||||
|
||||
calls_after = resync_apply_report_call_count_for_test();
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
|
||||
"the report completes (page 2 arrived) and IS handed to "
|
||||
"resync_apply_report -- which then discards it internally as "
|
||||
"stale, per its own Req 6.1/6.3 contract");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, calls_after - calls_before,
|
||||
"resync_apply_report is still called exactly once -- discarding "
|
||||
"happens INSIDE it (round-token mismatch), not by skipping the "
|
||||
"call");
|
||||
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j->job.state,
|
||||
"the stale report must never touch the job table");
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
|
||||
"the CURRENT (superseding) round must be left outstanding, "
|
||||
"untouched by the stale report");
|
||||
TEST_ASSERT_EQUAL_INT(1, w->resync->n_queried_jobids);
|
||||
|
||||
resync_resolve_round(&g_twr, w);
|
||||
twr_teardown();
|
||||
printf("test_handle_report_page_stale_round_discarded_when_superseded:"
|
||||
" PASS\n");
|
||||
}
|
||||
|
||||
/* Robustness: a page whose [pos, pos+count) range falls outside the
|
||||
* report's own declared n_total is rejected (-EINVAL) and discards
|
||||
* whatever partial accumulation was in progress -- proven by immediately
|
||||
* starting a FRESH, valid report afterward and confirming it applies
|
||||
* correctly, i.e. the corrupt attempt left no residue behind to corrupt a
|
||||
* later, legitimate report on the same worker. */
|
||||
static void test_handle_report_page_malformed_bounds_rejected_and_recovers(
|
||||
void)
|
||||
{
|
||||
job_t* j;
|
||||
worker_t* w;
|
||||
int rc;
|
||||
|
||||
xce_job_state_entry_t e_bad, e_good;
|
||||
xce_job_state_entry_t* entries_bad[1];
|
||||
xce_job_state_entry_t* entries_good[1];
|
||||
xce_job_state_report_t page_bad, page_good;
|
||||
|
||||
twr_setup();
|
||||
|
||||
j = twr_seed_job(TWR_JOBID_MPAGE_BAD, JOB_STAT_RUN);
|
||||
TEST_ASSERT_NOT_NULL(j);
|
||||
j->job.allocs
|
||||
= twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_BAD, "mpage-host-4");
|
||||
|
||||
w = twr_make_test_worker("mpage-host-4");
|
||||
TEST_ASSERT_NOT_NULL(w);
|
||||
|
||||
rc = resync_arm_round(&g_twr, w);
|
||||
TEST_ASSERT_EQUAL_INT(1, rc);
|
||||
|
||||
/* pos(1) + count(1) == 2 > n_total(1): out of bounds. */
|
||||
e_bad = (xce_job_state_entry_t){
|
||||
.jobid = TWR_JOBID_MPAGE_BAD2, .status = JobStateStatus_running,
|
||||
.exit_code = 0, .finish_time_ms = 0, .queried = 0
|
||||
};
|
||||
entries_bad[0] = &e_bad;
|
||||
page_bad = (xce_job_state_report_t){
|
||||
.n_total = 1, .pos = 1, .count = 1, .entries = entries_bad
|
||||
};
|
||||
|
||||
rc = resync_handle_report_page(&g_twr, w, &page_bad);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(-EINVAL, rc,
|
||||
"an out-of-bounds page must be rejected");
|
||||
|
||||
/* A fresh, valid, single-page report for the SAME worker right after --
|
||||
* proves the malformed attempt above left no corrupted accumulator
|
||||
* state behind. */
|
||||
e_good = (xce_job_state_entry_t){
|
||||
.jobid = TWR_JOBID_MPAGE_BAD, .status = JobStateStatus_unrecognized,
|
||||
.exit_code = 0, .finish_time_ms = 0, .queried = 1
|
||||
};
|
||||
entries_good[0] = &e_good;
|
||||
page_good = (xce_job_state_report_t){
|
||||
.n_total = 1, .pos = 0, .count = 1, .entries = entries_good
|
||||
};
|
||||
|
||||
rc = resync_handle_report_page(&g_twr, w, &page_good);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
|
||||
"a subsequent valid report must apply normally");
|
||||
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, j->job.state);
|
||||
TEST_ASSERT_NULL(j->job.allocs);
|
||||
TEST_ASSERT_NULL(w->resync);
|
||||
|
||||
twr_teardown();
|
||||
printf("test_handle_report_page_malformed_bounds_rejected_and_recovers:"
|
||||
" PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test group entry point ───────────────────────────────────────────── */
|
||||
|
||||
void test_worker_resync(void)
|
||||
|
|
@ -2076,4 +2459,8 @@ void test_worker_resync(void)
|
|||
test_apply_report_finished_success_uses_worker_reported_finish_time);
|
||||
RUN_TEST(
|
||||
test_apply_report_unqueried_extra_unassigned_pend_job_gets_assigned);
|
||||
RUN_TEST(test_handle_report_page_single_page_baseline);
|
||||
RUN_TEST(test_handle_report_page_multi_page_reassembles_and_applies_once);
|
||||
RUN_TEST(test_handle_report_page_stale_round_discarded_when_superseded);
|
||||
RUN_TEST(test_handle_report_page_malformed_bounds_rejected_and_recovers);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue