fix(controller): cold start elects in ~1.7 s, not ~10 s; ctrlstatus appliedLsn on the primary (bug-1879, bug-1878)
bug-1879: on a fresh HA cluster the config-holding ctrl-1 started its term-1 election before its peers ran; connect failures count as "no" votes (bug-1423), so it lost at once and became a follower with no leader. ctrl-2/ctrl-3 hold no config and are voters only (by design), so nothing retried until ctrl-1's 10 s standby liveness timeout: every cold start had no primary for ~10 s. - ha_note_leaderless() marks a config-holding node that becomes a follower with no leader known; the liveness poll then retries the election (PreVote) after HA_LEADERLESS_RETRY_MS = 1 s, doubling up to heartbeat_timeout_ms. Primary contact or becoming leader clears it. - ha_heartbeat_is_fresh() answers "no" while leaderless: last_activity then only holds the watchdog's arm time, and the failover gates took it for a live primary. Live cold start: primary 1.7 s after the first node started (was ~10 s). bug-1878: ctrlstatus_fill() reported ha->applied_lsn for every role, a standby-only counter a primary never advances, so every primary showed "appliedLsn: 0". A primary now reports persist->last_applied_lsn. The controller-status design doc's "already correct in every role" note is corrected. Tests: 5 new in test_ha_failover_prevote.c, 1 new in test_ctrlstatus.c (RED: expected 4900, was 0); controller suite 1256/0. HA soaks: sequential 6 rounds PASS, chaotic 6 rounds PASS (zero divergence, zero job loss). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
16b422e303
commit
aa7fc65a36
GPG key ID:
80533536B0EB651C
6 changed files with 260 additions and 6 deletions
|
|
@ -90,6 +90,33 @@ static void cs_teardown_cluster(cluster_t* c)
|
|||
|
||||
/* ── tests ──────────────────────────────────────────────────── */
|
||||
|
||||
/* bug-1878: a primary never advances ha->applied_lsn (a standby-only
|
||||
* counter: how far replicated WAL has been received), so ctrlstatus showed
|
||||
* "committedLsn: 4900 / appliedLsn: 0" on every primary. A primary reports
|
||||
* its apply cursor, persist->last_applied_lsn (the last entry applied). */
|
||||
static void test_ctrlstatus_primary_reports_its_apply_cursor(void)
|
||||
{
|
||||
cluster_t c;
|
||||
myconf_t mc;
|
||||
cs_setup_cluster(&c, &mc);
|
||||
|
||||
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY,
|
||||
memory_order_release);
|
||||
atomic_store_explicit(&c.ha.applied_lsn, 0, memory_order_release);
|
||||
atomic_store_explicit(&c.persist.committed_lsn, 4900, memory_order_release);
|
||||
atomic_store_explicit(&c.persist.last_applied_lsn, 4900,
|
||||
memory_order_release);
|
||||
|
||||
xce_ctrlstatus_t st;
|
||||
ctrlstatus_fill(&c, &st);
|
||||
|
||||
TEST_ASSERT_EQUAL_UINT64(4900, st.committedLsn);
|
||||
TEST_ASSERT_EQUAL_UINT64(4900, st.appliedLsn);
|
||||
TEST_ASSERT_TRUE_MESSAGE(st.appliedLsn <= st.committedLsn,
|
||||
"a primary applies only committed entries");
|
||||
ctrlstatus_free_fields(&st);
|
||||
}
|
||||
|
||||
/* (a) PRIMARY: responderRole==PRIMARY, primaryName==self, peers/version
|
||||
* populated; committed/applied/term reflect the atomics. (Req 2.1, 3.1, 3.2,
|
||||
* 4.1, 4.2, 5.1, 6.1, 6.2, 6.4) */
|
||||
|
|
@ -102,7 +129,9 @@ static void test_ctrlstatus_primary(void)
|
|||
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY,
|
||||
memory_order_release);
|
||||
atomic_store_explicit(&c.ha.term, 7, memory_order_release);
|
||||
atomic_store_explicit(&c.ha.applied_lsn, 42, memory_order_release);
|
||||
/* bug-1878: a primary's applied position is persist.last_applied_lsn;
|
||||
* ha.applied_lsn (standby-only) is left at the 0 a live primary has */
|
||||
atomic_store_explicit(&c.persist.last_applied_lsn, 41, memory_order_release);
|
||||
atomic_store_explicit(&c.ha.committed_config_tag, 0xAABBCCDDEE001122ULL,
|
||||
memory_order_release);
|
||||
atomic_store_explicit(&c.persist.committed_lsn, 41, memory_order_release);
|
||||
|
|
@ -120,7 +149,7 @@ static void test_ctrlstatus_primary(void)
|
|||
TEST_ASSERT_NOT_NULL(st.primaryName);
|
||||
TEST_ASSERT_EQUAL_STRING("ctrlA", st.primaryName);
|
||||
TEST_ASSERT_EQUAL_UINT64(7, st.term);
|
||||
TEST_ASSERT_EQUAL_UINT64(42, st.appliedLsn);
|
||||
TEST_ASSERT_EQUAL_UINT64(41, st.appliedLsn);
|
||||
TEST_ASSERT_EQUAL_UINT64(41, st.committedLsn);
|
||||
TEST_ASSERT_EQUAL_UINT64(0xAABBCCDDEE001122ULL, st.configVersion);
|
||||
TEST_ASSERT_EQUAL_UINT64(0xAABBCCDDEE001122ULL, st.committedConfigVersion);
|
||||
|
|
@ -1098,6 +1127,7 @@ void test_ctrlstatus(void)
|
|||
UnitySetTestFile(__FILE__);
|
||||
RUN_TEST(test_ctrlstatus_format_time_ms);
|
||||
RUN_TEST(test_ctrlstatus_primary);
|
||||
RUN_TEST(test_ctrlstatus_primary_reports_its_apply_cursor);
|
||||
RUN_TEST(test_ctrlstatus_standby_known_primary);
|
||||
RUN_TEST(test_ctrlstatus_unknown_primary);
|
||||
RUN_TEST(test_ctrlstatus_standby_self_vote_is_unknown);
|
||||
|
|
|
|||
|
|
@ -352,6 +352,138 @@ static void test_config_less_node_does_not_start_prevote(void)
|
|||
printf("test_config_less_node_does_not_start_prevote: PASS\n");
|
||||
}
|
||||
|
||||
/* ── bug-1879: cold boot / lost election with no leader known ──────────── */
|
||||
|
||||
/*
|
||||
* A config-holding node that loses its first election because its peers are
|
||||
* not up yet (connect failures count as "no" votes, bug-1423) became a
|
||||
* follower with nobody to follow, and only the 10 s liveness timeout ever
|
||||
* retried: every cold start had no primary for ~10 s. Now such a node is
|
||||
* marked leaderless and the liveness poll retries the election (via PreVote)
|
||||
* after leaderless_retry_ms, doubling up to heartbeat_timeout_ms.
|
||||
*/
|
||||
static uint64_t now_ms(void)
|
||||
{
|
||||
struct timespec ts;
|
||||
clock_gettime(CLOCK_MONOTONIC, &ts);
|
||||
return (uint64_t)ts.tv_sec * 1000u + (uint64_t)ts.tv_nsec / 1000000u;
|
||||
}
|
||||
|
||||
static void test_leaderless_follower_retries_election_after_short_delay(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
||||
ha_do_demote(&ha);
|
||||
make_three_node(&ha);
|
||||
|
||||
struct timespec ts;
|
||||
clock_gettime(CLOCK_MONOTONIC, &ts);
|
||||
ha.last_activity = ts.tv_sec; /* the 6 s timeout is far away */
|
||||
atomic_store(&ha.leaderless, 1);
|
||||
ha.leaderless_retry_ms = HA_LEADERLESS_RETRY_MS;
|
||||
ha.leaderless_since_ms = now_ms() - HA_LEADERLESS_RETRY_MS - 50;
|
||||
|
||||
liveness_poll_ctx_t ctx;
|
||||
memset(&ctx, 0, sizeof(ctx));
|
||||
ctx.ha = &ha;
|
||||
atomic_init(&ctx.cancelled, 0);
|
||||
liveness_poll_fn_test_entry(&ctx);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
|
||||
"a leaderless config-holding node retries after leaderless_retry_ms");
|
||||
TEST_ASSERT_EQUAL_UINT32_MESSAGE(2 * HA_LEADERLESS_RETRY_MS,
|
||||
ha.leaderless_retry_ms, "the next retry waits twice as long");
|
||||
ha_fini(&ha);
|
||||
}
|
||||
|
||||
static void test_leaderless_retry_waits_for_its_delay(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
||||
ha_do_demote(&ha);
|
||||
make_three_node(&ha);
|
||||
|
||||
struct timespec ts;
|
||||
clock_gettime(CLOCK_MONOTONIC, &ts);
|
||||
ha.last_activity = ts.tv_sec;
|
||||
atomic_store(&ha.leaderless, 1);
|
||||
ha.leaderless_retry_ms = HA_LEADERLESS_RETRY_MS;
|
||||
ha.leaderless_since_ms = now_ms() - 100;
|
||||
|
||||
liveness_poll_ctx_t ctx;
|
||||
memset(&ctx, 0, sizeof(ctx));
|
||||
ctx.ha = &ha;
|
||||
atomic_init(&ctx.cancelled, 0);
|
||||
liveness_poll_fn_test_entry(&ctx);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, ha.raft.pre_vote_in_progress);
|
||||
ha_fini(&ha);
|
||||
}
|
||||
|
||||
static void test_leaderless_retry_backoff_is_capped(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
||||
ha_do_demote(&ha);
|
||||
make_three_node(&ha);
|
||||
|
||||
struct timespec ts;
|
||||
clock_gettime(CLOCK_MONOTONIC, &ts);
|
||||
ha.last_activity = ts.tv_sec;
|
||||
atomic_store(&ha.leaderless, 1);
|
||||
ha.leaderless_retry_ms = g_cfg.heartbeat_timeout_ms;
|
||||
ha.leaderless_since_ms = now_ms() - g_cfg.heartbeat_timeout_ms - 50;
|
||||
|
||||
liveness_poll_ctx_t ctx;
|
||||
memset(&ctx, 0, sizeof(ctx));
|
||||
ctx.ha = &ha;
|
||||
atomic_init(&ctx.cancelled, 0);
|
||||
liveness_poll_fn_test_entry(&ctx);
|
||||
|
||||
TEST_ASSERT_EQUAL_UINT32(g_cfg.heartbeat_timeout_ms, ha.leaderless_retry_ms);
|
||||
ha_fini(&ha);
|
||||
}
|
||||
|
||||
static void test_primary_contact_clears_leaderless(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
||||
ha_do_demote(&ha);
|
||||
atomic_store(&ha.leaderless, 1);
|
||||
ha.leaderless_retry_ms = 4 * HA_LEADERLESS_RETRY_MS;
|
||||
|
||||
ha_on_heartbeat_received(&ha);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, atomic_load(&ha.leaderless));
|
||||
TEST_ASSERT_EQUAL_UINT32(HA_LEADERLESS_RETRY_MS, ha.leaderless_retry_ms);
|
||||
ha_fini(&ha);
|
||||
}
|
||||
|
||||
static void test_only_a_config_holder_without_leader_is_leaderless(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
cluster_t c;
|
||||
myconf_t conf;
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
||||
memset(&c, 0, sizeof(c));
|
||||
memset(&conf, 0, sizeof(conf));
|
||||
c.myconf = &conf;
|
||||
|
||||
conf.config_version_tag = 0; /* voter only */
|
||||
ha_note_leaderless(&ha, &c, -1);
|
||||
TEST_ASSERT_EQUAL_INT(0, atomic_load(&ha.leaderless));
|
||||
|
||||
conf.config_version_tag = 0x1234; /* can lead, leader known */
|
||||
ha_note_leaderless(&ha, &c, 1);
|
||||
TEST_ASSERT_EQUAL_INT(0, atomic_load(&ha.leaderless));
|
||||
|
||||
ha_note_leaderless(&ha, &c, -1); /* can lead, no leader */
|
||||
TEST_ASSERT_EQUAL_INT(1, atomic_load(&ha.leaderless));
|
||||
TEST_ASSERT_EQUAL_UINT32(HA_LEADERLESS_RETRY_MS, ha.leaderless_retry_ms);
|
||||
ha_fini(&ha);
|
||||
}
|
||||
|
||||
/* ── Suite entry point ──────────────────────────────────────────────────── */
|
||||
|
||||
void test_ha_failover_prevote(void)
|
||||
|
|
@ -367,6 +499,11 @@ void test_ha_failover_prevote(void)
|
|||
RUN_TEST(test_heartbeat_without_round_is_plain_noop);
|
||||
RUN_TEST(test_failover_prevote_win_reaches_real_election);
|
||||
RUN_TEST(test_config_less_node_does_not_start_prevote);
|
||||
RUN_TEST(test_leaderless_follower_retries_election_after_short_delay);
|
||||
RUN_TEST(test_leaderless_retry_waits_for_its_delay);
|
||||
RUN_TEST(test_leaderless_retry_backoff_is_capped);
|
||||
RUN_TEST(test_primary_contact_clears_leaderless);
|
||||
RUN_TEST(test_only_a_config_holder_without_leader_is_leaderless);
|
||||
|
||||
ha_prevote_rearm_liveness_hook = NULL;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue