fix(ha-election,ha-rejoin): stop two infinite-election loops on failover and restart
Two distinct HA livelocks, both verified on a live 3-node ASAN deploy (kill+restart of primary and standby, plus SIGSTOP wedge injection). bug-339 — election freshness-floor asymmetry (RDHUP failover path): After the primary dies the two survivors could never elect a leader — both only ever became follower while terms climbed in lockstep. The RDHUP candidate advertised c->persist.committed_lsn, which a standby never advances (only the primary commit engine writes it), while every voter computed its freshness floor from max(applied_lsn, last_known_committed_lsn). So each survivor advertised a stale-low LSN against a high voter floor and the freshness gate denied on both sides. Fix: compute the candidate LSN via ha_vote_self_freshness so the advertised value equals the voter floor (ha_on_primary_ctrl_fd_rdhup); also route the boot-path election (myapp_run) through the same helper for symmetry. bug-340 — wedged standby endpoint suppressed reconnect (rejoin path): A killed/restarted peer's outbound endpoint on the primary could wedge (epoll re-arm ENOENT) yet stay in standby_endpoints, so primary_heartbeat_fn's reconnect scan treated the peer as still-connected and never re-established a ctrlhello channel — the restarted node never heard from the primary and re-elected every heartbeat_timeout (term climbing unbounded). xce_nop_req returns success on enqueue even when the send can't progress, so the heartbeat layer never noticed. Fix: primary_heartbeat_fn now evicts a standby that was responsive once (ts_last_in > 0) but has been silent past heartbeat_timeout — symmetric with the standby's own liveness poll — and the reconnect scan re-establishes the channel. Uses existing endpoint_t.ts_last_in (no libnetio change). Tests (RED->GREEN): - test_ha_rdhup_integration: candidate advertises freshness floor, not stale committed_lsn - test_ha_liveness: evicts_silent_standby + keeps_live/keeps_unconfirmed (no false eviction)
This commit is contained in:
parent
ca1d9ae42d
commit
a735a880ae
4 changed files with 271 additions and 10 deletions
|
|
@ -435,6 +435,117 @@ static void test_primary_hb_skips_nop_when_fresh(void)
|
|||
printf("test_primary_hb_skips_nop_when_fresh: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 9b: primary evicts a standby that has gone silent (dead/restarted) ──
|
||||
*
|
||||
* Regression for the "kill+restart a controller → it loops elections forever"
|
||||
* bug. When a peer is killed/restarted, its outbound standby endpoint on the
|
||||
* primary can wedge (epoll re-arm fails) yet stay in standby_endpoints, so the
|
||||
* reconnect scan treats the peer as still-connected and never re-establishes a
|
||||
* fresh ctrlhello channel — the restarted node never hears from the primary and
|
||||
* re-elects every heartbeat_timeout (term climbing unbounded).
|
||||
*
|
||||
* Fix: primary_heartbeat_fn evicts any standby endpoint that was responsive once
|
||||
* (ts_last_in > 0) but has been silent longer than heartbeat_timeout, so the
|
||||
* reconnect scan re-establishes the channel. Before the fix the ep persists.
|
||||
*/
|
||||
static void test_primary_hb_evicts_silent_standby(void)
|
||||
{
|
||||
cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
init_ha_primary_with_standby_eps(&c.ha);
|
||||
|
||||
endpoint_t ep;
|
||||
memset(&ep, 0, sizeof(ep));
|
||||
standby_ep_ctx_build_for_test(&ep, &c.ha, &c, 0);
|
||||
standby_ep_ctx_t_test *sctx = (standby_ep_ctx_t_test *)ep.data;
|
||||
sctx->last_push_time = mono_now(); /* recently pushed — not the trigger */
|
||||
|
||||
/* Responsive once, then silent well beyond heartbeat_timeout (wall clock,
|
||||
* matching endpoint_t.ts_last_in which handle_input stamps via time(NULL)). */
|
||||
uint32_t timeout_s = g_cfg.heartbeat_timeout_ms / 1000;
|
||||
ep.ts_last_in = time(NULL) - (time_t)(timeout_s + 5);
|
||||
|
||||
zarray_store(c.ha.standby_endpoints, &ep);
|
||||
TEST_ASSERT_EQUAL_INT(1, c.ha.standby_endpoints->count);
|
||||
|
||||
g_nop_calls = 0;
|
||||
ha_nop_req_hook = stub_nop_req;
|
||||
primary_heartbeat_fn_for_test(&c);
|
||||
ha_nop_req_hook = NULL;
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, c.ha.standby_endpoints->count,
|
||||
"primary must evict a standby silent beyond heartbeat_timeout so it reconnects");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_nop_calls,
|
||||
"no NOP should be sent to an evicted dead standby");
|
||||
|
||||
free(ep.data);
|
||||
fini_ha_primary_with_standby_eps(&c.ha);
|
||||
printf("test_primary_hb_evicts_silent_standby: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 9c: a responsive standby is NOT evicted (no false positives) ─────── */
|
||||
static void test_primary_hb_keeps_live_standby(void)
|
||||
{
|
||||
cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
init_ha_primary_with_standby_eps(&c.ha);
|
||||
|
||||
endpoint_t ep;
|
||||
memset(&ep, 0, sizeof(ep));
|
||||
standby_ep_ctx_build_for_test(&ep, &c.ha, &c, 0);
|
||||
standby_ep_ctx_t_test *sctx = (standby_ep_ctx_t_test *)ep.data;
|
||||
sctx->last_push_time = mono_now();
|
||||
|
||||
/* Recent inbound activity — within the timeout window. */
|
||||
ep.ts_last_in = time(NULL);
|
||||
|
||||
zarray_store(c.ha.standby_endpoints, &ep);
|
||||
|
||||
g_nop_calls = 0;
|
||||
ha_nop_req_hook = stub_nop_req;
|
||||
primary_heartbeat_fn_for_test(&c);
|
||||
ha_nop_req_hook = NULL;
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, c.ha.standby_endpoints->count,
|
||||
"a responsive standby (recent ts_last_in) must NOT be evicted");
|
||||
|
||||
free(ep.data);
|
||||
fini_ha_primary_with_standby_eps(&c.ha);
|
||||
printf("test_primary_hb_keeps_live_standby: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 9d: a never-responded standby is NOT evicted (grace for new joiners) ── */
|
||||
static void test_primary_hb_keeps_unconfirmed_standby(void)
|
||||
{
|
||||
cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
init_ha_primary_with_standby_eps(&c.ha);
|
||||
|
||||
endpoint_t ep;
|
||||
memset(&ep, 0, sizeof(ep));
|
||||
standby_ep_ctx_build_for_test(&ep, &c.ha, &c, 0);
|
||||
standby_ep_ctx_t_test *sctx = (standby_ep_ctx_t_test *)ep.data;
|
||||
sctx->last_push_time = mono_now();
|
||||
|
||||
/* Freshly connected, no reply yet: ts_last_in == 0. Must NOT be evicted
|
||||
* (eviction keys on ts_last_in > 0 so a just-connected joiner gets grace). */
|
||||
ep.ts_last_in = 0;
|
||||
|
||||
zarray_store(c.ha.standby_endpoints, &ep);
|
||||
|
||||
g_nop_calls = 0;
|
||||
ha_nop_req_hook = stub_nop_req;
|
||||
primary_heartbeat_fn_for_test(&c);
|
||||
ha_nop_req_hook = NULL;
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, c.ha.standby_endpoints->count,
|
||||
"a just-connected standby (ts_last_in==0) must NOT be evicted");
|
||||
|
||||
free(ep.data);
|
||||
fini_ha_primary_with_standby_eps(&c.ha);
|
||||
printf("test_primary_hb_keeps_unconfirmed_standby: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 10: on_standby_connected sets last_push_time ──────────────────── */
|
||||
|
||||
/*
|
||||
|
|
@ -648,6 +759,9 @@ void test_ha_liveness(void)
|
|||
RUN_TEST(test_ha_disarm_makes_subsequent_poll_noop);
|
||||
RUN_TEST(test_primary_hb_sends_nop_when_idle);
|
||||
RUN_TEST(test_primary_hb_skips_nop_when_fresh);
|
||||
RUN_TEST(test_primary_hb_evicts_silent_standby);
|
||||
RUN_TEST(test_primary_hb_keeps_live_standby);
|
||||
RUN_TEST(test_primary_hb_keeps_unconfirmed_standby);
|
||||
RUN_TEST(test_on_standby_connected_sets_last_push_time);
|
||||
RUN_TEST(test_cluster_dispatch_nop_resets_activity_when_standby);
|
||||
RUN_TEST(test_cluster_dispatch_nop_skips_when_primary);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue