wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity

fix(ha): retry cluster_init_phase2 on failure instead of latching phase2_done unconditionally (bug-1477)

Browse source 
A node that takes myapp_run's skip_election passive path (bug-849 --
"leader already known after the startup stagger") makes exactly ONE
cluster_init_phase2 attempt. All 5 call sites (myapp.c's skip_election
block, raft_cb_become_leader, raft_cb_become_follower,
ha_elect_evaluate_and_signal's two branches) latched ha->phase2_done = 1
unconditionally after calling cluster_init_phase2, discarding its return
value. If that one attempt raced config availability and failed, the node
was permanently stranded: commit_engine_init never ran, so
apply_committed_entry never applied any committed WAL entry for the rest
of the process's life -- and for a skip_election node specifically,
nothing else would ever retry it, since the other 4 call sites only fire
via that node's own raft-library state transitions, which never happen
for a passive standby whose primary keeps heartbeating normally.

Extracted a shared, idempotent retry (ha_phase2_retry_fn, mirrors
upgrade_materialization_retry_fn's shape) plus a delaymgr-scheduled,
self-rescheduling trigger (ha_schedule_phase2_retry) -- self-rescheduling
because, unlike config_activate_retry/upgrade_materialization_retry which
are re-triggered externally by recurring commit-advance events, nothing
else recurs for a node stuck on the passive path. Wired into all 5 call
sites; raft_cb_become_leader additionally refuses promotion on failure
(mirroring the sibling upgrade_materialization_fenced refusal already in
that function -- a promoted primary with an unwired commit engine is far
worse than a delayed promotion).

TDD: 4 new tests, RED confirmed by temporarily reverting the rc-gating.
GREEN: full suite 1044 tests, 0 failures. Live-verified via
run-ha-soak.sh --config-upgrade (the scenario that found this bug): the
retry fires for real (4 failed attempts, then success) and the soak's
checkpoint-identity/config-identity/host-admin-status assertions -- the
things bug-1477 blocked -- all pass.
This commit is contained in:
Rongsong Shen 2026-07-28 13:25:48 +00:00
parent 102e263cce
commit f1e54f3ba0
6 changed files with 384 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

177
src/controller/ha.c
View file

@ -406,6 +406,101 @@ int schedule_upgrade_materialization_retry(cluster_t *c)
return rc;
}
/* ha_phase2_retry_fn / ha_phase2_retry_vtask_entry /
* ha_phase2_retry_delaymgr_trampoline / ha_schedule_phase2_retry (bug-1477):
* see the doc comments in ha.h. Structurally identical to
* upgrade_materialization_retry_fn's own 3-layer shape (blocking retry_fn ->
* vtask entry -> delaymgr trampoline) just above, except the trampoline
* re-arms itself on failure since nothing else recurring would otherwise
* ever retry cluster_init_phase2 for a node stuck on myapp_run's passive
* skip_election path. */
int ha_phase2_retry_fn(void *arg)
{
cluster_t *c = (cluster_t *)arg;
if (!c)
return -EINVAL;
if (c->ha.phase2_done)
return 0; /* already done by another path -- nothing to retry */
int rc;
#ifdef XCE_TESTING
rc = (ha_phase2_hook != NULL) ? ha_phase2_hook(c) : 0;
#else
rc = cluster_init_phase2(c);
#endif
if (rc == 0) {
c->ha.phase2_done = 1;
INFO("HA: cluster_init_phase2 retry succeeded — commit engine now "
"wired (bug-1477)");
} else {
WARN("HA: cluster_init_phase2 retry rc=%d — member stays alive and "
"will retry again (bug-1477, never self-terminate)", rc);
ha_schedule_phase2_retry(c);
}
return rc;
}
static int ha_phase2_retry_vtask_entry(vtask_t *task)
{
cluster_t *c = (cluster_t *)task->data->u.generic;
ha_phase2_retry_fn(c);
return 0;
}
static int ha_phase2_retry_delaymgr_trampoline(void *arg)
{
cluster_t *c = (cluster_t *)arg;
if (!c)
return -EINVAL;
vtaskmgr_t *pool = &c->vtaskmgr;
vtask_data_t *data = calloc(1, sizeof(*data));
vtask_t *task = vtask_alloc();
if (data == NULL || task == NULL) {
WARN("HA: cluster_init_phase2 retry: vtask alloc failed — running "
"inline (stay alive)");
free(data);
free(task);
return ha_phase2_retry_fn(c);
}
data->vtype = VTASK_GENERIC;
data->u.generic = c;
vtask_init(task, (vtask_func_t)ha_phase2_retry_vtask_entry, data);
if (vtask_enq_classified(pool, task, VTASK_LIVE_BLOCK) < 0) {
WARN("HA: cluster_init_phase2 retry: vtask enqueue failed — running "
"inline (stay alive)");
free(data);
free(task);
return ha_phase2_retry_fn(c);
}
return 0;
}
int ha_schedule_phase2_retry(cluster_t *c)
{
if (!c)
return -EINVAL;
if (c->ha.phase2_done)
return 0; /* steady state: nothing to retry, schedule nothing */
if (!c->delaymgr.running)
return 0; /* delaymgr not running (test fixture) — caller drives directly */
uint32_t interval_ms = c->ha.config.heartbeat_interval_ms;
if (interval_ms == 0)
interval_ms = HA_DEFAULT_HEARTBEAT_INTERVAL_MS;
int rc = delay_task_ms(&c->delaymgr, (int)interval_ms, 1,
(delay_task_entry_fn_t)ha_phase2_retry_delaymgr_trampoline,
c);
if (rc < 0)
WARN("HA: cluster_init_phase2 retry: delay_task_ms failed: %d — "
"will retry on the next natural re-trigger", rc);
return rc;
}
static void on_vote_connected(endpoint_t *ep);
static void on_pre_vote_connected(endpoint_t *ep);
static int vote_ep_err_handler(void *ep_raw);
@ -1773,14 +1868,31 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
/* First boot: call cluster_init_phase2 */
if (is_first_boot) {
int p2rc = 0;
#ifdef XCE_TESTING
if (ha_phase2_hook != NULL)
ha_phase2_hook(c);
p2rc = ha_phase2_hook(c);
#else
if (c != NULL)
cluster_init_phase2(c);
p2rc = cluster_init_phase2(c);
#endif
ha->phase2_done = 1;
if (p2rc == 0) {
ha->phase2_done = 1;
} else {
/* bug-1477: do NOT latch phase2_done on failure, and do NOT
* proceed to CTRL_PRIMARY with an unwired commit engine — mirror
* the upgrade_materialization_fenced refusal just above (stay
* ALIVE, non-promotable, retry; a later promotion attempt
* succeeds once ha_schedule_phase2_retry completes). Before this
* fix, phase2_done was latched unconditionally, so this failure
* path never existed for the code below to worry about. */
WARN("HA: promotion REFUSED — cluster_init_phase2 failed "
"(rc=%d); member stays ALIVE, non-promotable, retrying "
"(bug-1477, never self-terminate)", p2rc);
if (c != NULL)
ha_schedule_phase2_retry(c);
return; /* do NOT become CTRL_PRIMARY with an unwired commit engine */
}
/* bug-1445 fix: cluster_init_phase2() above just ran this process's
* very FIRST persist_recover()/replay_wal() -- the ONLY thing that
@ -2001,14 +2113,26 @@ static void raft_cb_become_follower(void *ctx, uint64_t term, int leader_idx)
/* First boot: call cluster_init_phase2 so structures are ready */
if (is_first_boot) {
int p2rc = 0;
#ifdef XCE_TESTING
if (ha_phase2_hook != NULL)
ha_phase2_hook(c);
p2rc = ha_phase2_hook(c);
#else
if (c != NULL)
cluster_init_phase2(c);
p2rc = cluster_init_phase2(c);
#endif
ha->phase2_done = 1;
if (p2rc == 0) {
ha->phase2_done = 1;
} else {
/* bug-1477: do NOT latch phase2_done on failure — a standby
* (unlike a promoted primary) can safely stay passive while
* ha_schedule_phase2_retry retries; nothing below depends on
* phase2 having succeeded. */
WARN("HA: cluster_init_phase2 on become-follower failed (rc=%d) "
"— member stays alive, will retry (bug-1477)", p2rc);
if (c != NULL)
ha_schedule_phase2_retry(c);
}
}
/* Close all outgoing vote endpoints (loser path).
@ -5664,15 +5788,32 @@ apply_role:
/* First boot: call cluster_init_phase2 to load config+data (Req 9.2) */
if (is_first_boot) {
int p2rc = 0;
#ifdef XCE_TESTING
/* In test mode, use hook if set; skip real phase2 (cluster.c not linked) */
if (ha_phase2_hook != NULL)
ha_phase2_hook(c);
p2rc = ha_phase2_hook(c);
#else
if (c != NULL)
cluster_init_phase2(c);
p2rc = cluster_init_phase2(c);
#endif
ha->phase2_done = 1;
if (p2rc == 0) {
ha->phase2_done = 1;
} else {
/* bug-1477: do NOT latch phase2_done on failure. Unlike
* raft_cb_become_leader, this legacy evaluation path has
* ALREADY signaled the caller's ctx->done semaphore with
* result=1 above (before "apply_role:") — refusing
* promotion here would leave that signal inconsistent
* with never becoming CTRL_PRIMARY. Proceed to
* CTRL_PRIMARY as before; ha_schedule_phase2_retry wires
* the commit engine shortly after instead of never. */
WARN("HA FSM: cluster_init_phase2 failed on promotion "
"(rc=%d) — proceeding to CTRL_PRIMARY, retrying "
"phase2 (bug-1477)", p2rc);
if (c != NULL)
ha_schedule_phase2_retry(c);
}
}
/* ha-single-leader-role task 3.1 (Req 1.1, 1.5): enter CTRL_PRIMARY
@ -5745,15 +5886,27 @@ apply_role:
/* First boot: call cluster_init_phase2 so structures are ready;
* config arrives via primary's replication push (Req 9.2, 7.3) */
if (is_first_boot) {
int p2rc = 0;
#ifdef XCE_TESTING
/* In test mode, use hook if set; skip real phase2 (cluster.c not linked) */
if (ha_phase2_hook != NULL)
ha_phase2_hook(c);
p2rc = ha_phase2_hook(c);
#else
if (c != NULL)
cluster_init_phase2(c);
p2rc = cluster_init_phase2(c);
#endif
ha->phase2_done = 1;
if (p2rc == 0) {
ha->phase2_done = 1;
} else {
/* bug-1477: do NOT latch phase2_done on failure — a
* standby can safely stay passive while
* ha_schedule_phase2_retry retries. */
WARN("HA FSM: cluster_init_phase2 failed on "
"become-standby (rc=%d) — member stays alive, will "
"retry (bug-1477)", p2rc);
if (c != NULL)
ha_schedule_phase2_retry(c);
}
}
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY,