lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

test(raft-apply-decoupling): command failure path leaves no trace under repetition (task 3.3)

Browse source 
Add 4 tests to test_commit_barrier_reply.c (25->29), no production
changes: explicit-FAILED via truncation (distinct from timeout),
repeated failure loops (5 cycles) for both timeout and truncation,
and an array_size>1 mixed-outcome test proving a committed sibling's
apply hand-off isn't blocked by failing siblings while producing
zero phantom jobs for the failures and exactly one batch-level
failure reply.

Closes out task-group 3 (command apply-after-commit ordering).
This commit is contained in:
Rongsong Shen 2026-07-04 06:05:06 +00:00
parent df4d2fa0ed
commit f357215a6d
2 changed files with 342 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

340
src/controller/testsuite/test_commit_barrier_reply.c
View file

@ -1058,6 +1058,337 @@ static void test_jobsubmit_returns_before_commit_on_failure_path_too(void)
printf("test_jobsubmit_returns_before_commit_on_failure_path_too: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* raft-apply-decoupling task 3.3 (Req 2.1, 2.3): the failure-path guarantee
* task 3.2 built, and the single-timeout tests above already proved, now
* needs proof that it (a) also holds for an explicit, non-timeout FAILED
* outcome, and (b) holds across REPEATED forced failures, not just once.
* Every test below follows the task-3.1-round-3 lesson: assertion-relevant
* observables are captured into locals, cbrp_teardown() runs exactly once
* and unconditionally, and every TEST_ASSERT fires only after teardown
* returns.
*
* Demotion is deliberately NOT used to force the non-timeout FAILED case
* here: raft_cb_become_follower's demotion branch resolves continuations
* on `c->commit` (the cluster_t-embedded commit_engine_t), but this file's
* cbrp_setup fixture wires job submission's continuations through a
* SEPARATE standalone `g_hce` (via repl_set_commit_engine) exactly like
* every other handler in this file -- calling the demotion seam against
* g_hc would touch the wrong, never-initialized engine and prove nothing.
* Truncation resolves through persist_truncate_wal_above ->
* p->on_truncate -> the SAME g_hce commit_engine_init wires
* (raft-apply-decoupling task 1.4), so it is the correct and only
* deterministic way to force an explicit FAILED outcome for job
* submission in this harness. Generic exactly-once demotion-resolve
* behavior already has dedicated coverage at the commit-engine level in
* test_demotion_convergence.c (task 1.4/1.5).
* ════════════════════════════════════════════════════════════════════════ */
/* Single-shot: an explicit FAILED outcome (rc=-ENOENT, forced via
* log-tail truncation) must leave no phantom job, exactly like the
* -ETIMEDOUT path -- but the reply status must carry the DISTINCT
* truncation vocabulary, not the timeout one, proving the continuation
* genuinely observed a different cause. */
static void test_jobsubmit_truncation_forces_failed_no_phantom_job(void)
{
uint64_t lsn_before, watermark;
int create_rc, join_rc, trunc_rc;
cbrp_reply_obs_t obs = { 0 };
bool job_exists;
cbrp_setup(5000); /* generous: truncation, not the timeout sweep, must
* be what resolves this entry */
cbrp_arrange_jobsubmit(9110, 1);
pthread_mutex_lock(&g_hc.persist.lock);
lsn_before = g_hc.persist.current_lsn;
pthread_mutex_unlock(&g_hc.persist.lock);
g_op_arg.fn = handle_jobsubmit;
g_op_arg.req = &g_jobsub_req;
g_op_arg.rc = -9999;
create_rc = pthread_create(&g_op_tid, NULL, cbrp_op_thread_fn, &g_op_arg);
join_rc = (create_rc == 0) ? pthread_join(g_op_tid, NULL) : -1;
watermark = cbrp_poll_append(&g_hc.persist, lsn_before, 2000);
/* Force an explicit FAILED outcome (rc=-ENOENT), NOT a timeout:
* truncate the entry that was just appended. Exercises
* commit_on_truncate -> commit_resolve_pending_above(ce, cutoff,
* -ENOENT) (task 1.4) through the real job-submit continuation
* (task 3.2), distinct from the -ETIMEDOUT path already covered by
* test_jobsubmit_returns_before_commit_on_failure_path_too. */
trunc_rc = (watermark > 0)
? persist_truncate_wal_above(&g_hc.persist, watermark - 1)
: -1;
cbrp_poll_reply(&g_req_ep, 2000, &obs);
job_exists = cbrp_job_exists(9110);
cbrp_teardown();
TEST_ASSERT_EQUAL_INT_MESSAGE(0, create_rc, "pthread_create must succeed");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, join_rc, "pthread_join must succeed");
TEST_ASSERT_TRUE_MESSAGE(watermark > 0,
"job submission must append its WAL entry before truncation can "
"force its continuation FAILED");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, trunc_rc,
"forcing truncation must succeed");
TEST_ASSERT_TRUE_MESSAGE(obs.n_replies > 0,
"a truncated commit must still reply to the client");
TEST_ASSERT_EQUAL_INT_MESSAGE(-ENOENT, obs.last_status,
"truncation must reply with -ENOENT, the existing "
"commit_resolve_pending_above error vocabulary (raft-apply-"
"decoupling task 1.4) -- distinct from the -ETIMEDOUT timeout "
"path already covered elsewhere in this file");
TEST_ASSERT_FALSE_MESSAGE(job_exists,
"no job may ever be constructed for an entry whose commit failed "
"via truncation, not just via timeout (raft-apply-decoupling task "
"3.3, Req 2.1/2.3)");
printf("test_jobsubmit_truncation_forces_failed_no_phantom_job: PASS\n");
}
#define CBRP_JOBSUBMIT_REPEAT_N 5
/* Proves the failure-path guarantee holds across MULTIPLE forced timeouts
* in sequence, not just once -- and that nothing about the fixture (the
* jobmgr, the per-request jobsubmit_async_ctx_t lifecycle) leaks or
* accumulates state across repeated failures. */
static void test_jobsubmit_repeated_timeout_failures_no_phantom_jobs(void)
{
const uint64_t base_jobid = 9120;
int create_rc[CBRP_JOBSUBMIT_REPEAT_N];
int join_rc[CBRP_JOBSUBMIT_REPEAT_N];
uint64_t watermark[CBRP_JOBSUBMIT_REPEAT_N];
int n_replies[CBRP_JOBSUBMIT_REPEAT_N];
int32_t last_status[CBRP_JOBSUBMIT_REPEAT_N];
bool job_exists[CBRP_JOBSUBMIT_REPEAT_N];
int i;
cbrp_setup(200); /* short timeout: bound the loop's wall time */
for (i = 0; i < CBRP_JOBSUBMIT_REPEAT_N; i++) {
uint64_t jobid = base_jobid + (uint64_t)i;
uint64_t lsn_before;
cbrp_reply_obs_t obs = { 0 };
cbrp_arrange_jobsubmit(jobid, 1);
pthread_mutex_lock(&g_hc.persist.lock);
lsn_before = g_hc.persist.current_lsn;
pthread_mutex_unlock(&g_hc.persist.lock);
g_op_arg.fn = handle_jobsubmit;
g_op_arg.req = &g_jobsub_req;
g_op_arg.rc = -9999;
create_rc[i] = pthread_create(&g_op_tid, NULL, cbrp_op_thread_fn,
&g_op_arg);
join_rc[i] = (create_rc[i] == 0) ? pthread_join(g_op_tid, NULL) : -1;
watermark[i] = cbrp_poll_append(&g_hc.persist, lsn_before, 1000);
/* No ACK is ever delivered: this iteration's entry must resolve
* FAILED via the timeout sweep (task 1.3), not majority ack. */
cbrp_poll_reply(&g_req_ep, 200 + 1500, &obs);
n_replies[i] = obs.n_replies;
last_status[i] = obs.last_status;
job_exists[i] = cbrp_job_exists(jobid);
}
cbrp_teardown();
for (i = 0; i < CBRP_JOBSUBMIT_REPEAT_N; i++) {
TEST_ASSERT_EQUAL_INT_MESSAGE(0, create_rc[i],
"pthread_create must succeed for every repeated iteration");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, join_rc[i],
"pthread_join must succeed for every repeated iteration");
TEST_ASSERT_TRUE_MESSAGE(watermark[i] > 0,
"each repeated submission must append its own WAL entry");
TEST_ASSERT_TRUE_MESSAGE(n_replies[i] > 0,
"every repeated forced-timeout submission must still reply");
TEST_ASSERT_EQUAL_INT_MESSAGE(-ETIMEDOUT, last_status[i],
"every repeated forced-timeout submission must reply "
"-ETIMEDOUT");
TEST_ASSERT_FALSE_MESSAGE(job_exists[i],
"no phantom job may exist for any repeated forced-timeout "
"failure (raft-apply-decoupling task 3.3, Req 2.1/2.3) -- "
"proves the guarantee is not a one-shot fluke");
}
printf("test_jobsubmit_repeated_timeout_failures_no_phantom_jobs: PASS\n");
}
/* The explicit-FAILED (non-timeout) sibling of the repeated-timeout test
* above: repeats the truncation-forced FAILED path
* CBRP_JOBSUBMIT_REPEAT_N times in the same fixture, proving the
* no-phantom-job guarantee holds for "repeated commit failures" in the
* literal sense the task text names (not just "repeated timeouts"), and
* that repeated FAILED resolutions release their jobsubmit_async_ctx_t
* cleanly each time (a fresh entry always starts from the same
* current_lsn==1 baseline the previous iteration's truncation left
* behind -- if a context or WAL state leaked, later iterations would
* observe a growing lsn_before/current_lsn or a stuck reply queue). */
static void test_jobsubmit_repeated_truncation_forced_failures_no_phantom_jobs(void)
{
const uint64_t base_jobid = 9130;
int create_rc[CBRP_JOBSUBMIT_REPEAT_N];
int join_rc[CBRP_JOBSUBMIT_REPEAT_N];
uint64_t watermark[CBRP_JOBSUBMIT_REPEAT_N];
int trunc_rc[CBRP_JOBSUBMIT_REPEAT_N];
int n_replies[CBRP_JOBSUBMIT_REPEAT_N];
int32_t last_status[CBRP_JOBSUBMIT_REPEAT_N];
bool job_exists[CBRP_JOBSUBMIT_REPEAT_N];
int i;
cbrp_setup(5000); /* generous: truncation, not the timeout sweep, must
* be what resolves each iteration */
for (i = 0; i < CBRP_JOBSUBMIT_REPEAT_N; i++) {
uint64_t jobid = base_jobid + (uint64_t)i;
uint64_t lsn_before;
cbrp_reply_obs_t obs = { 0 };
cbrp_arrange_jobsubmit(jobid, 1);
pthread_mutex_lock(&g_hc.persist.lock);
lsn_before = g_hc.persist.current_lsn;
pthread_mutex_unlock(&g_hc.persist.lock);
g_op_arg.fn = handle_jobsubmit;
g_op_arg.req = &g_jobsub_req;
g_op_arg.rc = -9999;
create_rc[i] = pthread_create(&g_op_tid, NULL, cbrp_op_thread_fn,
&g_op_arg);
join_rc[i] = (create_rc[i] == 0) ? pthread_join(g_op_tid, NULL) : -1;
watermark[i] = cbrp_poll_append(&g_hc.persist, lsn_before, 1000);
trunc_rc[i] = (watermark[i] > 0)
? persist_truncate_wal_above(&g_hc.persist, watermark[i] - 1)
: -1;
cbrp_poll_reply(&g_req_ep, 2000, &obs);
n_replies[i] = obs.n_replies;
last_status[i] = obs.last_status;
job_exists[i] = cbrp_job_exists(jobid);
}
cbrp_teardown();
for (i = 0; i < CBRP_JOBSUBMIT_REPEAT_N; i++) {
TEST_ASSERT_EQUAL_INT_MESSAGE(0, create_rc[i],
"pthread_create must succeed for every repeated iteration");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, join_rc[i],
"pthread_join must succeed for every repeated iteration");
TEST_ASSERT_TRUE_MESSAGE(watermark[i] > 0,
"each repeated submission must append its own WAL entry");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, trunc_rc[i],
"forcing truncation must succeed for every repeated iteration");
TEST_ASSERT_TRUE_MESSAGE(n_replies[i] > 0,
"every repeated forced-truncation submission must still "
"reply");
TEST_ASSERT_EQUAL_INT_MESSAGE(-ENOENT, last_status[i],
"every repeated forced-truncation submission must reply "
"-ENOENT, the explicit non-timeout FAILED vocabulary "
"(raft-apply-decoupling task 1.4)");
TEST_ASSERT_FALSE_MESSAGE(job_exists[i],
"no phantom job may exist for any repeated forced-truncation "
"failure (raft-apply-decoupling task 3.3, Req 2.1/2.3)");
}
printf("test_jobsubmit_repeated_truncation_forced_failures_no_phantom_jobs:"
" PASS\n");
}
/* Closes task 3.2's flagged gap ("no array_size > 1 mixed-outcome test
* exists yet"): element 1 commits (covering ACK) while elements 2 and 3
* are left to time out FAILED. Confirms the failed elements produce zero
* phantom jobs and that the committed element's independent resolution
* (hand-off to persist_register_pending_apply) is not blocked or skipped
* by its failing siblings -- task 3.2's per-element independence, proven
* by a real test rather than by code inspection alone. */
static void test_jobsubmit_array_mixed_outcome_no_phantom_job_for_failed_elements(void)
{
const uint64_t base_jobid = 9140;
uint64_t lsn_before, watermark, lsn1, t1;
int create_rc, join_rc;
bool has_pending1;
cbrp_reply_obs_t obs = { 0 };
bool job1_exists, job2_exists, job3_exists;
cbrp_setup(300); /* short: elements 2/3 must resolve via the timeout
* sweep within a bounded window */
cbrp_arrange_jobsubmit(base_jobid, 3);
pthread_mutex_lock(&g_hc.persist.lock);
lsn_before = g_hc.persist.current_lsn;
pthread_mutex_unlock(&g_hc.persist.lock);
lsn1 = lsn_before;
g_op_arg.fn = handle_jobsubmit;
g_op_arg.req = &g_jobsub_req;
g_op_arg.rc = -9999;
create_rc = pthread_create(&g_op_tid, NULL, cbrp_op_thread_fn, &g_op_arg);
join_rc = (create_rc == 0) ? pthread_join(g_op_tid, NULL) : -1;
/* All three array elements append back-to-back (task 3.2's
* non-blocking, no per-element ACK gate) before handle_jobsubmit
* returns -- by the time pthread_join above returns, every element's
* WAL entry already exists. */
watermark = cbrp_poll_append(&g_hc.persist, lsn_before + 2, 2000);
/* Cover ONLY the first element with a majority ACK; elements 2 and 3
* are left pending -- they must resolve FAILED via the timeout sweep,
* independently of element 1's success. */
repl_on_ack(&g_hc.persist.repl, &g_hep_a, lsn1);
has_pending1 = false;
t1 = cbrp_mono_ms();
while (cbrp_mono_ms() - t1 < 2000) {
if (persist_has_pending_apply(&g_hc.persist, lsn1)) {
has_pending1 = true;
break;
}
cbrp_sleep_ms(5);
}
cbrp_poll_reply(&g_req_ep, 300 + 1500, &obs);
job1_exists = cbrp_job_exists(base_jobid);
job2_exists = cbrp_job_exists(base_jobid + 1);
job3_exists = cbrp_job_exists(base_jobid + 2);
cbrp_teardown();
TEST_ASSERT_EQUAL_INT_MESSAGE(0, create_rc, "pthread_create must succeed");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, join_rc, "pthread_join must succeed");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before + 2, watermark,
"all three array elements must append before this poll returns");
TEST_ASSERT_TRUE_MESSAGE(has_pending1,
"a sibling array element that commits must still be handed off to "
"the pending-apply registry even though other siblings in the "
"same batch are failing (raft-apply-decoupling task 3.2's "
"per-element independent resolution)");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, obs.n_replies,
"a partially-failed array batch must produce exactly one failure "
"reply for the whole request (jobsubmit_reply_failure_once)");
TEST_ASSERT_EQUAL_INT_MESSAGE(-ETIMEDOUT, obs.last_status,
"the batch's single reply must carry the failing elements' "
"-ETIMEDOUT outcome");
TEST_ASSERT_FALSE_MESSAGE(job1_exists,
"no element may have a constructed job yet -- task 6's apply pump "
"does not exist, so even the committed element has no job; this "
"test proves independent per-element RESOLUTION, not apply");
TEST_ASSERT_FALSE_MESSAGE(job2_exists,
"a failed array element must never have a phantom job");
TEST_ASSERT_FALSE_MESSAGE(job3_exists,
"a failed array element must never have a phantom job");
printf("test_jobsubmit_array_mixed_outcome_no_phantom_job_for_failed_elements:"
" PASS\n");
}
/* REMOVED (raft-apply-decoupling task 3.2): the array-partial-construction-
* failure remediation test (task 3.1 round-2 fix, bug-1408) that lived here
* drove handle_jobsubmit's OLD sequential, per-element BLOCKING append loop
@ -2124,7 +2455,14 @@ void test_commit_barrier_reply(void)
RUN_TEST(test_jobsubmit_withheld_acks_no_phantom_job);
RUN_TEST(test_jobsubmit_returns_before_commit_and_registers_pending_apply);
RUN_TEST(test_jobsubmit_returns_before_commit_on_failure_path_too);
/* Receipt-ACK verdict pins (task 3.3, Req 2.3) — green on landing */
/* raft-apply-decoupling task 3.3 (Req 2.1, 2.3): repeated/explicit-
* FAILED failure-path coverage beyond the single-timeout tests above */
RUN_TEST(test_jobsubmit_truncation_forces_failed_no_phantom_job);
RUN_TEST(test_jobsubmit_repeated_timeout_failures_no_phantom_jobs);
RUN_TEST(test_jobsubmit_repeated_truncation_forced_failures_no_phantom_jobs);
RUN_TEST(test_jobsubmit_array_mixed_outcome_no_phantom_job_for_failed_elements);
/* Receipt-ACK verdict pins (ha-commit-barrier-rollout task 3.3, Req 2.3)
* — green on landing */
RUN_TEST(test_jobdispatch_receipt_ack_no_wal_append);
RUN_TEST(test_set_preason_receipt_reply_fact_retained);
RUN_TEST(test_clr_preason_receipt_reply_fact_retained);