lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(raft-apply-decoupling): fire command continuation post-apply (task 6.2)

Browse source 
Wire persist_apply_pump_run() to call persist_take_pending_apply()
immediately after each entry's apply_committed_entry() call,
invoking the registered continuation strictly after the mutation.
Adds jobsubmit_reply_success_once() (job.c) so a successful job
submission finally sends a client reply once every array element
applies -- closes the standing gap from task 3.2 where committed
submissions never got a response.

Fixes bug-1415: commit_resolve_pending() dispatches the commit
continuation and the apply-pump kick as two independent
VTASK_LIVE_BLOCK vtasks with no completion-order guarantee. On any
real multi-vcpu pool (the production default, n_vcpus=8), if the
apply pump won the race and advanced last_applied_lsn past an LSN
before its continuation registered, the registration was permanently
orphaned -- the job would be constructed and visible, but the client
would hang forever. Untested by every existing fixture in this spec
(all use pool=NULL, which trivially serializes and can't reproduce
the race).

Fixed with a self-healing check in persist_register_pending_apply():
if the LSN is already <= last_applied_lsn, invoke the callback
immediately instead of registering for later pickup. Verified by the
spec's first genuine multi-vcpu regression test (real vtaskmgr_t,
n_vcpus=4), independently RED->GREEN reproduced by two review rounds.
This commit is contained in:
Rongsong Shen 2026-07-05 00:38:02 +00:00
parent 2fa955b92e
commit 88577f4658
6 changed files with 719 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

245
src/controller/testsuite/test_commit_barrier_reply.c
View file

@ -200,6 +200,13 @@ typedef struct {
int n_replies; /* replymsg packets seen by this scan */
int32_t last_status; /* status of the last replymsg observed */
uint64_t committed_at_obs; /* committed_lsn sampled at observation time */
int32_t last_data_which; /* RequestReply_data_which of the last
* replymsg observed (raft-apply-decoupling
* task 6.2: distinguishes a jobid-bearing
* success reply from a nodata one) */
uint64_t last_jobid; /* rr.data.jobid of the last replymsg observed
* -- meaningful only when last_data_which ==
* RequestReply_data_jobid */
} cbrp_reply_obs_t;
static persist_t *g_obs_persist; /* sampled into committed_at_obs */
@ -224,6 +231,9 @@ static void cbrp_scan_replies(endpoint_t *ep, cbrp_reply_obs_t *obs)
read_RequestReply(&rr, rq.body.replymsg);
obs->n_replies += 1;
obs->last_status = rr.status;
obs->last_data_which = rr.data_which;
if (rr.data_which == RequestReply_data_jobid)
obs->last_jobid = rr.data.jobid;
obs->committed_at_obs = atomic_load(&g_obs_persist->committed_lsn);
}
capn_free(&c);
@ -1389,6 +1399,237 @@ static void test_jobsubmit_array_mixed_outcome_no_phantom_job_for_failed_element
" PASS\n");
}
/* ── handle_jobsubmit success path via the ordered apply pump
* (raft-apply-decoupling task 6.2, Req 1.1, 2.1-2.3) ────────────────────
*
* Everything above this point pins the COMMITTED-outcome path's gap left by
* task 3.2: a committed job submission is handed off to
* persist_register_pending_apply() and then has NO further observable
* effect (no reply, no job in c->jobmgr) — by design, until task 6 exists.
* Task 6.1 built the ordered apply pump that finally calls
* apply_committed_entry() live; task 6.2 (this section) wires the pump to
* consume that hand-off via persist_take_pending_apply() and fire the
* registered continuation (jobsubmit_apply_complete -> the client's success
* reply), strictly after the entry's mutation, never before or during it.
*
* Ordering proof mechanism: apply_committed_entry()'s WAL_JOB_SUBMIT case
* (persist.c) fires HOOK_DETAIL_JOB_NEW as an AFTER_EVENT, synchronously,
* immediately after inserting the job into c->jobmgr — and, by
* persist_apply_pump_run()'s own strict sequencing (task 6.2), strictly
* BEFORE the pump reaches its persist_take_pending_apply()/invoke step for
* that same lsn. A spy hook on this exact (situation, detail) pair can
* therefore observe, synchronously, whether a reply has already been
* enqueued on the requester's outq at the moment the mutation completes —
* mirroring this file's own pre-existing g_notify_reply_already_enqueued
* idiom (see notify() capture above) for the identical class of ordering
* assertion. */
static bool g_jobsubmit_order_spy_fired;
static bool g_jobsubmit_order_spy_reply_seen_early;
static uint64_t g_jobsubmit_order_spy_last_jobid;
static int g_jobsubmit_order_spy_fire_count;
static int cbrp_jobsubmit_order_spy(int type, int situation, int detail,
void *data, va_list ap)
{
(void)type; (void)data;
if (situation != HOOK_AFTER_EVENT || detail != HOOK_DETAIL_JOB_NEW)
return 0;
/* apply_committed_entry's WAL_JOB_SUBMIT branch fires this hook with
* (cluster_t*, xlist_t*) — mirroring test_apply_pump.c's tap_spy_hook,
* which decodes the identical detail the same way. */
(void)va_arg(ap, cluster_t *);
xlist_t *jlist = va_arg(ap, xlist_t *);
g_jobsubmit_order_spy_fired = true;
g_jobsubmit_order_spy_fire_count++;
if (jlist && jlist->count > 0) {
job_t *j = (job_t *)((void **)jlist->data)[0];
g_jobsubmit_order_spy_last_jobid = j->job.jobid;
}
/* THE ordering proof, latched sticky (OR'd) across every firing: once
* true, a reply was observably enqueued no later than one of these
* post-mutation notifications — a real ordering violation, regardless
* of which array element it happened on. */
if (!DATAQ_EMPTY(&g_req_ep.outq))
g_jobsubmit_order_spy_reply_seen_early = true;
return 0;
}
static void cbrp_jobsubmit_order_spy_reset(void)
{
g_jobsubmit_order_spy_fired = false;
g_jobsubmit_order_spy_reply_seen_early = false;
g_jobsubmit_order_spy_last_jobid = 0;
g_jobsubmit_order_spy_fire_count = 0;
}
/* Single-element submission: the success reply must carry the job id, and
* must be observable only after the job is visible in c->jobmgr. */
static void test_jobsubmit_success_reply_after_apply_carries_jobid(void)
{
const uint64_t jobid = 9200;
uint64_t lsn_before, watermark;
cbrp_reply_obs_t early = { 0 };
cbrp_reply_obs_t final = { 0 };
bool job_exists_before_ack, job_exists_at_reply;
cbrp_setup(2000);
TEST_ASSERT_EQUAL_INT(0, xce_add_hook(&g_hc.hooksys, XCE_CONTROLLER_HOOK_JOB,
cbrp_jobsubmit_order_spy, &g_hc));
cbrp_jobsubmit_order_spy_reset();
cbrp_arrange_jobsubmit(jobid, 1);
pthread_mutex_lock(&g_hc.persist.lock);
lsn_before = g_hc.persist.current_lsn;
pthread_mutex_unlock(&g_hc.persist.lock);
g_op_arg.fn = handle_jobsubmit;
g_op_arg.req = &g_jobsub_req;
g_op_arg.rc = -9999;
TEST_ASSERT_EQUAL_INT(0, pthread_create(&g_op_tid, NULL,
cbrp_op_thread_fn, &g_op_arg));
TEST_ASSERT_EQUAL_INT(0, pthread_join(g_op_tid, NULL));
watermark = cbrp_poll_append(&g_hc.persist, lsn_before, 2000);
/* Before any covering ACK: no reply, no job -- the pre-existing
* task-3.2 contract this task's own change must not disturb. */
cbrp_sleep_ms(150);
cbrp_scan_replies(&g_req_ep, &early);
job_exists_before_ack = cbrp_job_exists(jobid);
repl_on_ack(&g_hc.persist.repl, &g_hep_a, watermark);
cbrp_poll_reply(&g_req_ep, 2000, &final);
job_exists_at_reply = cbrp_job_exists(jobid);
cbrp_teardown();
TEST_ASSERT_TRUE_MESSAGE(watermark > 0,
"job submission must append its WAL entry");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, early.n_replies,
"no reply may be observable before the entry commits (Req 1.1)");
TEST_ASSERT_FALSE_MESSAGE(job_exists_before_ack,
"the job must not exist in the in-memory table before commit");
TEST_ASSERT_TRUE_MESSAGE(final.n_replies > 0,
"a successful job submission must reply once the ordered apply "
"pump consumes the commit hand-off (raft-apply-decoupling task "
"6.2 -- closes the standing gap left open since task 3.2)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, final.last_status,
"a committed, applied job submission must reply success");
TEST_ASSERT_EQUAL_INT_MESSAGE(RequestReply_data_jobid,
final.last_data_which,
"the success reply must carry the job id (RequestReply_data_jobid)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(jobid, final.last_jobid,
"the success reply's job id must match the submitted job");
TEST_ASSERT_TRUE_MESSAGE(job_exists_at_reply,
"the job must be visible in the in-memory job table by the time "
"the success reply is observed");
TEST_ASSERT_TRUE_MESSAGE(g_jobsubmit_order_spy_fired,
"apply_committed_entry's post-mutation HOOK_DETAIL_JOB_NEW "
"notification must fire for this job submission");
TEST_ASSERT_EQUAL_UINT64(jobid, g_jobsubmit_order_spy_last_jobid);
TEST_ASSERT_FALSE_MESSAGE(g_jobsubmit_order_spy_reply_seen_early,
"the client reply must be sent strictly AFTER the job's mutation "
"(apply_committed_entry's own post-mutation hook), never before or "
"concurrently with it -- task 6.2's own named observable "
"(Req 1.1, 2.1-2.3)");
printf("test_jobsubmit_success_reply_after_apply_carries_jobid: PASS\n");
}
/* Array submission (3 elements, base jobid 9210): a single covering ACK
* commits all three entries at once (mirroring test_apply_pump.c's
* burst-order coverage), and the pump applies + hands off each one
* independently, in ascending LSN order. Confirms task 3.2's per-element
* independent registration is correctly drained by task 6.2's per-element
* consumption: every element's own post-mutation notification fires before
* the single batch-level reply is ever observable, and every element is
* visible in c->jobmgr by the time that reply arrives. */
static void test_jobsubmit_array_success_all_elements_apply_before_single_reply(
void)
{
const uint64_t base_jobid = 9210;
const uint64_t array_size = 3;
uint64_t lsn_before, watermark;
cbrp_reply_obs_t final = { 0 };
bool job_exists[3];
cbrp_setup(2000);
TEST_ASSERT_EQUAL_INT(0, xce_add_hook(&g_hc.hooksys, XCE_CONTROLLER_HOOK_JOB,
cbrp_jobsubmit_order_spy, &g_hc));
cbrp_jobsubmit_order_spy_reset();
cbrp_arrange_jobsubmit(base_jobid, array_size);
pthread_mutex_lock(&g_hc.persist.lock);
lsn_before = g_hc.persist.current_lsn;
pthread_mutex_unlock(&g_hc.persist.lock);
g_op_arg.fn = handle_jobsubmit;
g_op_arg.req = &g_jobsub_req;
g_op_arg.rc = -9999;
TEST_ASSERT_EQUAL_INT(0, pthread_create(&g_op_tid, NULL,
cbrp_op_thread_fn, &g_op_arg));
TEST_ASSERT_EQUAL_INT(0, pthread_join(g_op_tid, NULL));
/* All three elements append back-to-back before handle_jobsubmit
* returns (task 3.2) -- one covering ACK past the last of them commits
* the whole batch in a single jump. */
watermark = cbrp_poll_append(&g_hc.persist, lsn_before + (array_size - 1),
2000);
repl_on_ack(&g_hc.persist.repl, &g_hep_a, watermark);
cbrp_poll_reply(&g_req_ep, 2000, &final);
job_exists[0] = cbrp_job_exists(base_jobid);
job_exists[1] = cbrp_job_exists(base_jobid + 1);
job_exists[2] = cbrp_job_exists(base_jobid + 2);
cbrp_teardown();
TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before + (array_size - 1), watermark,
"all three array elements must append before this poll returns");
TEST_ASSERT_TRUE_MESSAGE(final.n_replies > 0,
"the batch must eventually reply once every element applies");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, final.n_replies,
"an all-succeeding array batch must produce exactly ONE reply for "
"the whole request, not one per element (Req 9.3)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, final.last_status,
"the batch reply must report success");
TEST_ASSERT_EQUAL_INT_MESSAGE(RequestReply_data_jobid,
final.last_data_which,
"the batch success reply must carry a job id");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(base_jobid, final.last_jobid,
"the batch success reply must echo the BASE job id, matching "
"pre-task-3.1 semantics (git history commit 6f1fa97), never a "
"per-element id");
TEST_ASSERT_TRUE_MESSAGE(job_exists[0],
"array element 0 must be visible in c->jobmgr by reply time");
TEST_ASSERT_TRUE_MESSAGE(job_exists[1],
"array element 1 must be visible in c->jobmgr by reply time");
TEST_ASSERT_TRUE_MESSAGE(job_exists[2],
"array element 2 must be visible in c->jobmgr by reply time");
TEST_ASSERT_EQUAL_INT_MESSAGE(3, g_jobsubmit_order_spy_fire_count,
"each of the three array elements must independently fire its own "
"post-mutation notification (task 3.2's per-element registration, "
"drained one at a time by task 6.2)");
TEST_ASSERT_FALSE_MESSAGE(g_jobsubmit_order_spy_reply_seen_early,
"no array element's post-mutation notification may ever observe "
"the batch's single reply already enqueued -- the reply can only "
"follow the LAST element's own mutation, and every earlier "
"element's mutation must equally precede it");
printf("test_jobsubmit_array_success_all_elements_apply_before_single_"
"reply: PASS\n");
}
/* REMOVED (raft-apply-decoupling task 3.2): the array-partial-construction-
* failure remediation test (task 3.1 round-2 fix, bug-1408) that lived here
* drove handle_jobsubmit's OLD sequential, per-element BLOCKING append loop
@ -2942,6 +3183,10 @@ void test_commit_barrier_reply(void)
RUN_TEST(test_jobsubmit_repeated_timeout_failures_no_phantom_jobs);
RUN_TEST(test_jobsubmit_repeated_truncation_forced_failures_no_phantom_jobs);
RUN_TEST(test_jobsubmit_array_mixed_outcome_no_phantom_job_for_failed_elements);
/* handle_jobsubmit success path via the ordered apply pump
* (raft-apply-decoupling task 6.2, Req 1.1, 2.1-2.3) */
RUN_TEST(test_jobsubmit_success_reply_after_apply_carries_jobid);
RUN_TEST(test_jobsubmit_array_success_all_elements_apply_before_single_reply);
/* Receipt-ACK verdict pins (ha-commit-barrier-rollout task 3.3, Req 2.3)
* — green on landing */
RUN_TEST(test_jobdispatch_receipt_ack_no_wal_append);