lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(controller): apply worker finish arriving before job materializes (bug-1509)

Browse source 
A worker-reported finish may arrive before the job exists: job
construction is deferred to apply_committed_entry (the submit's WAL
entry), leaving a commit->apply window. on_job_finish dropped such
finishes ('can not find job'), so a fast job that finished in that
window stayed RUN forever (e.g. noexec-test/newcfg-test/verify-fix).

Add an in-memory unknown-finish registry: on_job_finish registers the
finish when the job isn't materialized, and apply_committed_entry's
WAL_JOB_SUBMIT case drains it right after restore_job — appending the
durable WAL_JOB_FINISH and mutating the job to DONE/EXIT immediately so
the scheduler never dispatches a job that already finished.

Design: docs/Job-Finish-Robustness-设计文档.md (Part A of A+B; Part B
worker-liveness + reconnect reconciliation deferred to later phases).

Unit tests (red->green): registry register/take, apply-once-job-
materializes (DONE), exit-code maps to EXIT, drain no-op without
registration.
This commit is contained in:
lei.chen 2026-08-06 05:40:56 +00:00
parent 8b81ba7336
commit a793015f74
5 changed files with 520 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

105
src/controller/testsuite/test_apply_pump.c
View file

@ -1635,11 +1635,116 @@ static void test_standby_truncate_before_commit_leaves_no_phantom_job(void)
"PASS\n");
}
/* ── 5. bug-1509: unknown-finish registry (finish arriving before the job is
* ── materialized must not be dropped — it must be applied once the job
* ── materializes, converging to DONE/EXIT instead of lingering as RUN). */
static void test_unknown_finish_registry_register_and_take(void)
{
tap_setup();
xce_jobfinish_t out;
/* Register a finish for a not-yet-materialized job. */
xce_jobfinish_t f = { .jobid = 903, .exit_code = 0 };
TEST_ASSERT_EQUAL_INT(0, test_unknown_finish_register(&f));
/* Take it back — must be found with the exact payload. */
TEST_ASSERT_EQUAL_INT(1, test_unknown_finish_take(903, &out));
TEST_ASSERT_EQUAL_UINT64(903, out.jobid);
TEST_ASSERT_EQUAL_INT(0, out.exit_code);
/* Take again — nothing left. */
TEST_ASSERT_EQUAL_INT(0, test_unknown_finish_take(903, &out));
/* A second jobid coexists independently. */
xce_jobfinish_t g = { .jobid = 904, .exit_code = 7 };
TEST_ASSERT_EQUAL_INT(0, test_unknown_finish_register(&g));
TEST_ASSERT_EQUAL_INT(1, test_unknown_finish_take(904, &out));
TEST_ASSERT_EQUAL_INT(7, out.exit_code);
TEST_ASSERT_EQUAL_INT(0, test_unknown_finish_take(903, &out));
tap_teardown();
printf("test_unknown_finish_registry_register_and_take: PASS\n");
}
/* The real bug: a worker finish for a job the controller has not yet
* materialized (submit committed but not applied). on_job_finish must
* register it (return 0), and once the job materializes,
* apply_registered_finish_if_any must converge it to DONE. */
static void test_unknown_finish_applied_once_job_materializes(void)
{
tap_setup();
/* No job 905 exists yet. */
TEST_ASSERT_NULL(tap_find_job(905));
xce_jobfinish_t finish = { .jobid = 905, .exit_code = 0 };
TEST_ASSERT_EQUAL_INT(0, on_job_finish(&g_tap_c, &finish));
/* Still no job (on_job_finish must not fabricate one). */
TEST_ASSERT_NULL(tap_find_job(905));
/* The submit's apply materializes the job (simulated restore_job). */
job_t* j = tap_seed_job(905, JOB_STAT_PEND);
TEST_ASSERT_NOT_NULL(j);
/* Draining applies the deferred finish: exit_code 0 -> DONE. */
TEST_ASSERT_EQUAL_INT(1, apply_registered_finish_if_any(&g_tap_c, j));
TEST_ASSERT_TRUE_MESSAGE(j->job.state & JOB_STAT_DONE,
"job must converge to DONE after deferred finish");
/* Draining again is a no-op. */
TEST_ASSERT_EQUAL_INT(0, apply_registered_finish_if_any(&g_tap_c, j));
tap_teardown();
printf("test_unknown_finish_applied_once_job_materializes: PASS\n");
}
/* Non-zero exit_code converges to EXIT, not DONE. */
static void test_unknown_finish_exit_code_maps_to_exit(void)
{
tap_setup();
xce_jobfinish_t finish = { .jobid = 906, .exit_code = 3 };
TEST_ASSERT_EQUAL_INT(0, on_job_finish(&g_tap_c, &finish));
job_t* j = tap_seed_job(906, JOB_STAT_PEND);
TEST_ASSERT_NOT_NULL(j);
TEST_ASSERT_EQUAL_INT(1, apply_registered_finish_if_any(&g_tap_c, j));
TEST_ASSERT_TRUE_MESSAGE(j->job.state & JOB_STAT_EXIT,
"exit_code!=0 must converge to EXIT");
TEST_ASSERT_FALSE_MESSAGE(j->job.state & JOB_STAT_DONE,
"EXIT and DONE must not both be set");
tap_teardown();
printf("test_unknown_finish_exit_code_maps_to_exit: PASS\n");
}
/* The drain must not touch a job with no registered finish. */
static void test_unknown_finish_drain_noop_without_registration(void)
{
tap_setup();
job_t* j = tap_seed_job(907, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
TEST_ASSERT_EQUAL_INT(0, apply_registered_finish_if_any(&g_tap_c, j));
TEST_ASSERT_TRUE_MESSAGE(j->job.state & JOB_STAT_RUN,
"job without a deferred finish must be untouched");
tap_teardown();
printf("test_unknown_finish_drain_noop_without_registration: PASS\n");
}
/* ── Test group entry point ───────────────────────────────────────────── */
void test_apply_pump(void)
{
UnitySetTestFile(__FILE__);
RUN_TEST(test_unknown_finish_registry_register_and_take);
RUN_TEST(test_unknown_finish_applied_once_job_materializes);
RUN_TEST(test_unknown_finish_exit_code_maps_to_exit);
RUN_TEST(test_unknown_finish_drain_noop_without_registration);
RUN_TEST(test_apply_pump_kick_applies_committed_burst_in_ascending_lsn_order);
RUN_TEST(test_apply_pump_run_applies_mixed_submit_and_fact_entries);
RUN_TEST(test_apply_pump_run_halts_at_failed_lsn_without_advancing);