lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(controller): resync_resolve_round never told scheduler RECONCILING cleared (bug-1841)

Browse source 
Worker-rejoin-resync sets XCE_WORKER_RECONCILING and pushes it to the
scheduler, which correctly freezes that worker's resInUse mid-reconciliation
(bug-1799/1801). But resync_resolve_round() only cleared the bit on the
controller's own worker_t, never notifying the scheduler -- so once a
rejoin round resolved, the scheduler's copy stayed RECONCILING forever,
permanently freezing that worker's resource accounting even after every
job on it finished. Live-observed via a replay capture: a 22-slot worker
stuck at used=22.00 for the rest of the run with zero RUN jobs left.

Added persist_notify_worker_status_updated() (mirrors bug-1679's job-side
persist_notify_job_updated_already_persisted()) and call it from
resync_resolve_round() so HOOK_DETAIL_WORKER_UPDATE fires without a
spurious WAL append. TDD: new spy-hook regression test, RED confirmed
against the unfixed code, GREEN after the fix. Full controller_unit_test
suite: 1220 Tests, 0 Failures, 0 Ignored.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-09-22 09:28:10 +00:00
parent e1da9f1e9b
commit 1f6f219da6
No known key found for this signature in database
GPG key ID: 5BCDAAE0123E9FE2
5 changed files with 147 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

69
src/controller/testsuite/test_worker_resync.c
View file

@ -1019,6 +1019,74 @@ static void test_resync_reconciling_flag_set_after_arm_cleared_after_resolve(
"resolve: PASS\n");
}
/* bug-1841 (new): resync_resolve_round() must notify HOOK_DETAIL_WORKER_
* UPDATE subscribers once RECONCILING actually clears -- otherwise the
* scheduler's own copy of this worker's status (fed purely from whatever
* WORKERS_PUSH it last received; core_events.c's core_upsert_worker does a
* bare `worker->status = w->status` with no other refresh path) stays
* RECONCILING forever, permanently freezing that worker's resInUse
* (sched_reset_worker_/sched_update_account's by-design bug-1799/1801
* freeze) even after every job on it has long since finished DONE --
* live-observed via a real scheduler-replay capture: a 22-slot worker
* stuck at used=22.00 for the rest of the run with zero RUN jobs left.
* Mirrors bug-1679's identical job-side gap and its
* test_reconcile_pending_applies_releases_allocs_and_notifies spy-hook
* proof above. */
static int g_twr_worker_update_hook_fires;
static int twr_worker_update_spy_hook(int type, int situation, int detail,
void* data, va_list ap)
{
(void)type; (void)data; (void)ap;
if (situation == HOOK_AFTER_EVENT && detail == HOOK_DETAIL_WORKER_UPDATE)
g_twr_worker_update_hook_fires++;
return 0;
}
static void test_resync_resolve_round_notifies_worker_update(void)
{
job_t* j;
worker_t w;
int rc;
twr_setup();
g_twr_worker_update_hook_fires = 0;
TEST_ASSERT_EQUAL_INT(0,
xce_add_hook(&g_twr.hooksys, XCE_CONTROLLER_HOOK_WORKER,
twr_worker_update_spy_hook, NULL));
j = twr_seed_job(TWR_JOBID_RESYNC_SOLO, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_SOLO,
"resync-host-notify");
memset(&w, 0, sizeof(w));
w.host.name = (char*)"resync-host-notify";
rc = resync_arm_round(&g_twr, &w);
TEST_ASSERT_EQUAL_INT(1, rc);
TEST_ASSERT_TRUE(IS_WORKER_RECONCILING(w.host.status));
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_twr_worker_update_hook_fires,
"arming a round must not itself fire HOOK_DETAIL_WORKER_UPDATE -- "
"only resolving one does");
resync_resolve_round(&g_twr, &w);
TEST_ASSERT_FALSE(IS_WORKER_RECONCILING(w.host.status));
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_twr_worker_update_hook_fires,
"resync_resolve_round must notify HOOK_DETAIL_WORKER_UPDATE "
"subscribers (scheduler/apiserver) once RECONCILING actually "
"clears -- otherwise the scheduler never learns and stays frozen");
/* Idempotent no-op: resolving an already-resolved round must NOT fire a
* second, spurious notification. */
resync_resolve_round(&g_twr, &w);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_twr_worker_update_hook_fires,
"resolving an already-resolved (NULL) round must not re-notify");
twr_teardown();
printf("test_resync_resolve_round_notifies_worker_update: PASS\n");
}
/* Req 5.3: the timeout resolves an outstanding round -- and does so via a
* plain, synchronous function call with no sleep/wait of any kind, proving
* the resolution itself never blocks (ha-async-io.md's delaymgr
@ -4759,6 +4827,7 @@ void test_worker_resync(void)
RUN_TEST(test_resync_arm_round_no_jobs_is_noop);
RUN_TEST(test_resync_arm_round_fresh_token_supersedes_prior);
RUN_TEST(test_resync_reconciling_flag_set_after_arm_cleared_after_resolve);
RUN_TEST(test_resync_resolve_round_notifies_worker_update);
RUN_TEST(test_resync_timeout_resolves_round_without_blocking);
RUN_TEST(test_apply_report_all_five_outcomes_updates_job_table);
RUN_TEST(test_apply_report_duplicate_reapplication_is_noop);