lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(controller-status): ctrlstatus_fill must prefer a live registry slot over a stale disconnected one (task 8.6, bug-1819)

Browse source 
Found live during task 8.5's optional catching-up-observation attempt:
a killed-then-reconnected standby can end up occupying a NEW
replicator_t.standbys[] slot while its OLD slot (host/port still
populated, ep == NULL) also matches the same configured peer.
ctrlstatus_fill's primary-side peer-status loop stopped at the first
host:port match, so it permanently reported that peer dead even
minutes after it had genuinely reconnected and fully caught up --
violating Requirement 13.3's "true, currently-known connection status"
guarantee.

Fix: scan all MAX_STANDBYS slots for a given peer and prefer any slot
with ep != NULL over a slot with ep == NULL; fall back to dead only
when no matching slot is live. Read-only, lock scope unchanged from
task 8.2.

TDD RED->GREEN (1216 Tests: 2 Failures -> 0), independently reproduced
by the reviewer (exact RED replay) plus two independent live
kill/backlog/restart/poll cycles on separate fresh clusters, both
confirming the previously-stuck standby now converges to ok.

design.md's Locked Decision 3 corrected to document the fix.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RLSNJhNoULvVcUhXEFafQC
This commit is contained in:
Rongsong Shen 2026-09-16 17:53:13 +00:00
parent 464489d972
commit 6c437e2bed
No known key found for this signature in database
GPG key ID: 5BCDAAE0123E9FE2
5 changed files with 790 additions and 440 deletions
Download patch file Download diff file Expand all files Collapse all files

107
src/controller/testsuite/test_ctrlstatus.c
View file

@ -986,6 +986,111 @@ static void test_ctrlstatus_peer_status_self_mismatch_falls_through(void)
printf("test_ctrlstatus_peer_status_self_mismatch_falls_through: PASS\n");
}
/* (r) PRIMARY: a killed-then-reconnected standby ends up owning TWO
* replicator_t.standbys[] slots for the SAME configured peer -- a
* lower-index slot left behind by the kill (ep == NULL, host/port still
* populated -- repl_add_standby's ep-identity idempotency check cannot
* reuse it, so the reconnect lands in a fresh, higher-index slot) and a
* higher-index slot for the genuine live reconnection (ep != NULL). A
* first-match-wins scan (pre-fix) reports the stale slot's dead status
* PERMANENTLY; the fix must scan every slot and prefer any live
* (ep != NULL) match over a stale (ep == NULL) one, regardless of which
* index the live slot lands in (Requirements 13.1, 13.3; task 8.6). */
static void test_ctrlstatus_peer_status_prefers_live_slot_over_stale_ok(void)
{
cluster_t c;
myconf_t mc;
cs_setup_cluster(&c, &mc);
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY,
memory_order_release);
TEST_ASSERT_EQUAL_INT(0, repl_init(&c.persist.repl));
c.ha.repl = &c.persist.repl;
/* Stale slot (lower index): ctrlB's OLD connection, killed -- ep
* cleared but host/port left behind, exactly as repl_remove_standby
* leaves it (this fixture models the pre-existing registry state,
* regardless of how it got there -- ctrlstatus_fill's boundary is
* reading it correctly, not preventing it). */
strncpy(c.persist.repl.standbys[0].host, "10.0.0.2",
sizeof(c.persist.repl.standbys[0].host) - 1);
c.persist.repl.standbys[0].port = 7002;
c.persist.repl.standbys[0].ep = NULL;
/* Live slot (higher index): ctrlB's NEW connection after reconnecting,
* fully caught up. */
int dummy_ep;
strncpy(c.persist.repl.standbys[1].host, "10.0.0.2",
sizeof(c.persist.repl.standbys[1].host) - 1);
c.persist.repl.standbys[1].port = 7002;
c.persist.repl.standbys[1].ep = (repl_ep_t)&dummy_ep;
atomic_store_explicit(&c.persist.repl.standbys[1].in_catch_up, 0,
memory_order_release);
/* ctrlC: no matching slot at all -- unaffected, still dead. */
xce_ctrlstatus_t st;
ctrlstatus_fill(&c, &st);
TEST_ASSERT_EQUAL_INT(3, st.peers.count);
TEST_ASSERT_EQUAL_STRING("ctrlA", st.peers.list[0]->name);
TEST_ASSERT_EQUAL_UINT8(1, st.peers.list[0]->status); /* self: ok */
TEST_ASSERT_EQUAL_STRING("ctrlB", st.peers.list[1]->name);
TEST_ASSERT_EQUAL_UINT8(1, st.peers.list[1]->status); /* live slot wins: ok */
TEST_ASSERT_EQUAL_STRING("ctrlC", st.peers.list[2]->name);
TEST_ASSERT_EQUAL_UINT8(3, st.peers.list[2]->status); /* unaffected: dead */
ctrlstatus_free_fields(&st);
repl_fini(&c.persist.repl);
cs_teardown_cluster(&c);
printf("test_ctrlstatus_peer_status_prefers_live_slot_over_stale_ok: PASS\n");
}
/* (s) Sibling of (r): the live slot is still catching up (not yet acked up
* to the cluster) -- the stale/live preference must still hold, and the
* reported status must be catching-up (2), not dead (3) and not the
* unconditional ok a naive "any live slot -> ok" fix would wrongly give. */
static void test_ctrlstatus_peer_status_prefers_live_slot_catching_up(void)
{
cluster_t c;
myconf_t mc;
cs_setup_cluster(&c, &mc);
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY,
memory_order_release);
TEST_ASSERT_EQUAL_INT(0, repl_init(&c.persist.repl));
c.ha.repl = &c.persist.repl;
/* Stale slot (lower index): ctrlB's OLD connection, killed. */
strncpy(c.persist.repl.standbys[0].host, "10.0.0.2",
sizeof(c.persist.repl.standbys[0].host) - 1);
c.persist.repl.standbys[0].port = 7002;
c.persist.repl.standbys[0].ep = NULL;
/* Live slot (higher index): ctrlB's NEW connection, still catching up. */
int dummy_ep;
strncpy(c.persist.repl.standbys[1].host, "10.0.0.2",
sizeof(c.persist.repl.standbys[1].host) - 1);
c.persist.repl.standbys[1].port = 7002;
c.persist.repl.standbys[1].ep = (repl_ep_t)&dummy_ep;
atomic_store_explicit(&c.persist.repl.standbys[1].in_catch_up, 1,
memory_order_release);
xce_ctrlstatus_t st;
ctrlstatus_fill(&c, &st);
TEST_ASSERT_EQUAL_INT(3, st.peers.count);
TEST_ASSERT_EQUAL_STRING("ctrlB", st.peers.list[1]->name);
TEST_ASSERT_EQUAL_UINT8(2, st.peers.list[1]->status); /* live slot wins: catching-up */
ctrlstatus_free_fields(&st);
repl_fini(&c.persist.repl);
cs_teardown_cluster(&c);
printf("test_ctrlstatus_peer_status_prefers_live_slot_catching_up: PASS\n");
}
/* ── suite entry point ─────────────────────────────────────── */
void test_ctrlstatus(void)
@ -1011,4 +1116,6 @@ void test_ctrlstatus(void)
RUN_TEST(test_ctrlstatus_peer_status_primary_sourced_from_registry);
RUN_TEST(test_ctrlstatus_peer_status_standby_unknown_except_self);
RUN_TEST(test_ctrlstatus_peer_status_self_mismatch_falls_through);
RUN_TEST(test_ctrlstatus_peer_status_prefers_live_slot_over_stale_ok);
RUN_TEST(test_ctrlstatus_peer_status_prefers_live_slot_catching_up);
}