lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(controller-status): ctrlstatus_fill must prefer a live registry slot over a stale disconnected one (task 8.6, bug-1819)

Browse source 
Found live during task 8.5's optional catching-up-observation attempt:
a killed-then-reconnected standby can end up occupying a NEW
replicator_t.standbys[] slot while its OLD slot (host/port still
populated, ep == NULL) also matches the same configured peer.
ctrlstatus_fill's primary-side peer-status loop stopped at the first
host:port match, so it permanently reported that peer dead even
minutes after it had genuinely reconnected and fully caught up --
violating Requirement 13.3's "true, currently-known connection status"
guarantee.

Fix: scan all MAX_STANDBYS slots for a given peer and prefer any slot
with ep != NULL over a slot with ep == NULL; fall back to dead only
when no matching slot is live. Read-only, lock scope unchanged from
task 8.2.

TDD RED->GREEN (1216 Tests: 2 Failures -> 0), independently reproduced
by the reviewer (exact RED replay) plus two independent live
kill/backlog/restart/poll cycles on separate fresh clusters, both
confirming the previously-stuck standby now converges to ok.

design.md's Locked Decision 3 corrected to document the fix.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RLSNJhNoULvVcUhXEFafQC
This commit is contained in:
Rongsong Shen 2026-09-16 17:53:13 +00:00
parent 464489d972
commit 6c437e2bed
No known key found for this signature in database
GPG key ID: 5BCDAAE0123E9FE2
5 changed files with 790 additions and 440 deletions
Download patch file Download diff file Expand all files Collapse all files

49
src/controller/ctrlstatus.c
View file

@ -253,22 +253,43 @@ void ctrlstatus_fill(cluster_t* c, xce_ctrlstatus_t* st)
} else if (is_primary) {
/* Reuses repl_active_standby_count's exact ep != NULL &&
* !in_catch_up predicate (repl.c) rather than inventing a
* new one (Locked Decision 3). */
uint8_t status = 3u; /* default: no matching live slot */
* new one (Locked Decision 3).
*
* Scans ALL MAX_STANDBYS slots (does NOT stop at the first
* host/port match) and prefers any slot with ep != NULL
* over a slot with ep == NULL when multiple slots match the
* same configured peer's host:port (task 8.6 fix). A killed
* standby's slot is freed (ep cleared) but keeps its
* host/port populated; when it reconnects, repl_add_standby's
* ep-identity idempotency check cannot reuse that now-ep-NULL
* slot, so the reconnect lands in a NEW slot -- leaving two
* slots that share the same peer's host:port, one stale
* (ep == NULL) and one live (ep != NULL). A first-match-wins
* scan reported the stale, lower-index slot's dead status
* forever, even after the peer genuinely reconnected and
* caught up. If more than one live slot somehow matches, any
* one of them is an acceptable source -- they would all be
* observing the same live peer. */
uint8_t status = 3u; /* default: no matching slot at all */
int live_match = 0; /* found a slot with ep != NULL */
for (int j = 0; j < MAX_STANDBYS; j++) {
if (strcmp(repl->standbys[j].host,
mc->peers_cfg.candidates[i].hostname) == 0 &&
repl->standbys[j].port == p->port) {
if (repl->standbys[j].ep == NULL) {
status = 3u; /* matched slot, no live conn */
} else if (atomic_load_explicit(
&repl->standbys[j].in_catch_up,
memory_order_acquire)) {
status = 2u; /* catching-up */
} else {
status = 1u; /* ok */
}
break;
mc->peers_cfg.candidates[i].hostname) != 0 ||
repl->standbys[j].port != p->port) {
continue; /* not a match for this peer */
}
if (repl->standbys[j].ep == NULL) {
if (!live_match)
status = 3u; /* stale match; keep scanning for a live one */
continue;
}
if (!live_match) {
live_match = 1;
status = atomic_load_explicit(
&repl->standbys[j].in_catch_up,
memory_order_acquire)
? 2u /* live, catching-up */
: 1u; /* live, caught up: ok */
}
}
p->status = status;

107
src/controller/testsuite/test_ctrlstatus.c
View file

@ -986,6 +986,111 @@ static void test_ctrlstatus_peer_status_self_mismatch_falls_through(void)
printf("test_ctrlstatus_peer_status_self_mismatch_falls_through: PASS\n");
}
/* (r) PRIMARY: a killed-then-reconnected standby ends up owning TWO
* replicator_t.standbys[] slots for the SAME configured peer -- a
* lower-index slot left behind by the kill (ep == NULL, host/port still
* populated -- repl_add_standby's ep-identity idempotency check cannot
* reuse it, so the reconnect lands in a fresh, higher-index slot) and a
* higher-index slot for the genuine live reconnection (ep != NULL). A
* first-match-wins scan (pre-fix) reports the stale slot's dead status
* PERMANENTLY; the fix must scan every slot and prefer any live
* (ep != NULL) match over a stale (ep == NULL) one, regardless of which
* index the live slot lands in (Requirements 13.1, 13.3; task 8.6). */
static void test_ctrlstatus_peer_status_prefers_live_slot_over_stale_ok(void)
{
cluster_t c;
myconf_t mc;
cs_setup_cluster(&c, &mc);
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY,
memory_order_release);
TEST_ASSERT_EQUAL_INT(0, repl_init(&c.persist.repl));
c.ha.repl = &c.persist.repl;
/* Stale slot (lower index): ctrlB's OLD connection, killed -- ep
* cleared but host/port left behind, exactly as repl_remove_standby
* leaves it (this fixture models the pre-existing registry state,
* regardless of how it got there -- ctrlstatus_fill's boundary is
* reading it correctly, not preventing it). */
strncpy(c.persist.repl.standbys[0].host, "10.0.0.2",
sizeof(c.persist.repl.standbys[0].host) - 1);
c.persist.repl.standbys[0].port = 7002;
c.persist.repl.standbys[0].ep = NULL;
/* Live slot (higher index): ctrlB's NEW connection after reconnecting,
* fully caught up. */
int dummy_ep;
strncpy(c.persist.repl.standbys[1].host, "10.0.0.2",
sizeof(c.persist.repl.standbys[1].host) - 1);
c.persist.repl.standbys[1].port = 7002;
c.persist.repl.standbys[1].ep = (repl_ep_t)&dummy_ep;
atomic_store_explicit(&c.persist.repl.standbys[1].in_catch_up, 0,
memory_order_release);
/* ctrlC: no matching slot at all -- unaffected, still dead. */
xce_ctrlstatus_t st;
ctrlstatus_fill(&c, &st);
TEST_ASSERT_EQUAL_INT(3, st.peers.count);
TEST_ASSERT_EQUAL_STRING("ctrlA", st.peers.list[0]->name);
TEST_ASSERT_EQUAL_UINT8(1, st.peers.list[0]->status); /* self: ok */
TEST_ASSERT_EQUAL_STRING("ctrlB", st.peers.list[1]->name);
TEST_ASSERT_EQUAL_UINT8(1, st.peers.list[1]->status); /* live slot wins: ok */
TEST_ASSERT_EQUAL_STRING("ctrlC", st.peers.list[2]->name);
TEST_ASSERT_EQUAL_UINT8(3, st.peers.list[2]->status); /* unaffected: dead */
ctrlstatus_free_fields(&st);
repl_fini(&c.persist.repl);
cs_teardown_cluster(&c);
printf("test_ctrlstatus_peer_status_prefers_live_slot_over_stale_ok: PASS\n");
}
/* (s) Sibling of (r): the live slot is still catching up (not yet acked up
* to the cluster) -- the stale/live preference must still hold, and the
* reported status must be catching-up (2), not dead (3) and not the
* unconditional ok a naive "any live slot -> ok" fix would wrongly give. */
static void test_ctrlstatus_peer_status_prefers_live_slot_catching_up(void)
{
cluster_t c;
myconf_t mc;
cs_setup_cluster(&c, &mc);
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY,
memory_order_release);
TEST_ASSERT_EQUAL_INT(0, repl_init(&c.persist.repl));
c.ha.repl = &c.persist.repl;
/* Stale slot (lower index): ctrlB's OLD connection, killed. */
strncpy(c.persist.repl.standbys[0].host, "10.0.0.2",
sizeof(c.persist.repl.standbys[0].host) - 1);
c.persist.repl.standbys[0].port = 7002;
c.persist.repl.standbys[0].ep = NULL;
/* Live slot (higher index): ctrlB's NEW connection, still catching up. */
int dummy_ep;
strncpy(c.persist.repl.standbys[1].host, "10.0.0.2",
sizeof(c.persist.repl.standbys[1].host) - 1);
c.persist.repl.standbys[1].port = 7002;
c.persist.repl.standbys[1].ep = (repl_ep_t)&dummy_ep;
atomic_store_explicit(&c.persist.repl.standbys[1].in_catch_up, 1,
memory_order_release);
xce_ctrlstatus_t st;
ctrlstatus_fill(&c, &st);
TEST_ASSERT_EQUAL_INT(3, st.peers.count);
TEST_ASSERT_EQUAL_STRING("ctrlB", st.peers.list[1]->name);
TEST_ASSERT_EQUAL_UINT8(2, st.peers.list[1]->status); /* live slot wins: catching-up */
ctrlstatus_free_fields(&st);
repl_fini(&c.persist.repl);
cs_teardown_cluster(&c);
printf("test_ctrlstatus_peer_status_prefers_live_slot_catching_up: PASS\n");
}
/* ── suite entry point ─────────────────────────────────────── */
void test_ctrlstatus(void)
@ -1011,4 +1116,6 @@ void test_ctrlstatus(void)
RUN_TEST(test_ctrlstatus_peer_status_primary_sourced_from_registry);
RUN_TEST(test_ctrlstatus_peer_status_standby_unknown_except_self);
RUN_TEST(test_ctrlstatus_peer_status_self_mismatch_falls_through);
RUN_TEST(test_ctrlstatus_peer_status_prefers_live_slot_over_stale_ok);
RUN_TEST(test_ctrlstatus_peer_status_prefers_live_slot_catching_up);
}