lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(controller): offload demotion convergence and catch-up streaming by default (bug-1450, bug-1451)

Browse source 
f88e4ef4 moved ha_demotion_convergence_fn's WAL truncate + self-restart
and ha_catchup_continuation_fn's WAL streaming off the delaymgr thread
(which drives heartbeats and election timeouts cluster-wide) onto the
slow lane, but left both flags OFF: flipping them ON hung pre-existing
unit fixtures that demote a primary or trigger catch-up without ever
starting c->slowlane -- the work was queued in a lane with no workers and
never ran.

- slowlane_is_started(): true while the lane has live workers.
- Both dispatchers run inline when the flag is OFF *or* the lane was never
  started. Production always starts the lane, so it always offloads; the
  lightweight fixtures keep their old inline behaviour, and neither
  one-shot action can be silently dropped.
- g_drive_async_demotion_offload / g_drive_async_catchup_offload now
  default to 1. The offload tests save and restore the previous value
  instead of forcing 0.

New test_catchup_offload_without_started_lane_runs_inline (RED before: the
catch-up body never ran, the stream gate was never reached). Controller
suite 1239/0 with both offloads ON by default.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-09-27 05:46:09 +00:00
parent 88c4310e11
commit 7aac018c00
Signed by untrusted user: rshen
GPG key ID: 80533536B0EB651C
6 changed files with 72 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

31
src/controller/testsuite/test_config_repl.c
View file

@ -7071,8 +7071,11 @@ static int cc_poll_flag(_Atomic int *flag, int timeout_ms)
return atomic_load(flag);
}
static void cc_run_offload_probe(int offload_on, const char *label)
static void cc_run_offload_probe_ex(int offload_on, int start_lane,
const char *label)
{
int saved_offload = ha_catchup_offload_enabled();
char tmpdir[256];
snprintf(tmpdir, sizeof(tmpdir), "/tmp/xce_cc_%s_XXXXXX",
offload_on ? "on" : "off");
@ -7103,7 +7106,8 @@ static void cc_run_offload_probe(int offload_on, const char *label)
.max_promotion_retries = 5,
};
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &hcfg, 0));
TEST_ASSERT_EQUAL_INT(0, slowlane_init(&c.slowlane, 2, 4));
if (start_lane)
TEST_ASSERT_EQUAL_INT(0, slowlane_init(&c.slowlane, 2, 4));
TEST_ASSERT_EQUAL_INT(0, delaymgr_init(&c.delaymgr, 5, cc_run_delay_task));
TEST_ASSERT_EQUAL_INT(0, delaymgr_start(&c.delaymgr));
@ -7126,7 +7130,7 @@ static void cc_run_offload_probe(int offload_on, const char *label)
TEST_ASSERT_EQUAL_INT(0,
delay_task_ms(&c.delaymgr, 1, 1, cc_probe_fn, &probe));
if (offload_on) {
if (offload_on && start_lane) {
TEST_ASSERT_TRUE_MESSAGE(cc_poll_flag(&probe.fired, 1000),
"GREEN: with catch-up offloaded to the slow lane, the delaymgr "
"worker thread must be free to run the NEXT queued task "
@ -7154,11 +7158,12 @@ static void cc_run_offload_probe(int offload_on, const char *label)
"the probe task must eventually fire once the gate is released");
ha_repl_stream_wal_hook = NULL;
ha_set_catchup_offload_for_test(0); /* restore production default */
ha_set_catchup_offload_for_test(saved_offload);
cc_gate_destroy(&g_cc_gate);
delaymgr_stop(&c.delaymgr);
delaymgr_fini(&c.delaymgr);
slowlane_fini(&c.slowlane);
if (start_lane)
slowlane_fini(&c.slowlane);
ha_fini(&c.ha);
persist_stop(&c.persist);
persist_fini(&c.persist);
@ -7166,6 +7171,21 @@ static void cc_run_offload_probe(int offload_on, const char *label)
printf("%s: PASS\n", label);
}
static void cc_run_offload_probe(int offload_on, const char *label)
{
cc_run_offload_probe_ex(offload_on, /*start_lane=*/1, label);
}
/* bug-1451: with the offload ON (now the production default) and a slow
* lane that was never started -- the lightweight fixtures that made the
* offload hang when it was first flipped ON -- the catch-up body must still
* run (inline), never sit forever in a lane with no workers. */
static void test_catchup_offload_without_started_lane_runs_inline(void)
{
cc_run_offload_probe_ex(1, /*start_lane=*/0,
"test_catchup_offload_without_started_lane_runs_inline");
}
static void test_catchup_offload_probe_runs_promptly(void)
{
cc_run_offload_probe(1, "test_catchup_offload_probe_runs_promptly");
@ -7556,6 +7576,7 @@ void test_config_repl(void)
/* xce-audit-invariants controller pass, thread-role violation 2 fix
* (2026-07-18): catch-up continuation slow-lane offload */
RUN_TEST(test_catchup_offload_probe_runs_promptly);
RUN_TEST(test_catchup_offload_without_started_lane_runs_inline);
RUN_TEST(test_catchup_inline_probe_blocks_RED);
/* bug-1476: concurrent load_primary_config() UAF/double-free race */
RUN_TEST(test_bug1476_concurrent_load_primary_config_no_uaf_or_double_free);

4
src/controller/testsuite/test_demotion_convergence.c
View file

@ -926,6 +926,8 @@ static void *dcb_run_thread(void *p)
static void dcb_run_offload(int offload_on, const char *label)
{
int saved_offload = ha_demotion_offload_enabled();
dc_setup("blocking", /*n_peers=*/3, /*n_standbys=*/2,
/*appended_lsn=*/5, /*n_vcpus=*/8);
atomic_store(&g_dc.persist.committed_lsn, 3);
@ -979,7 +981,7 @@ static void dcb_run_offload(int offload_on, const char *label)
/* Let the async completion (offload ON) finish before teardown. */
dc_poll_restart_calls(1, 2000);
ha_set_demotion_offload_for_test(0); /* restore production default */
ha_set_demotion_offload_for_test(saved_offload);
dcb_gate_destroy(&g_dcb_gate);
dc_teardown();
printf("%s: PASS\n", label);