fix(controller): standby dispatch-gate rejections now redirect to a known primary (bug-1700, bug-1701)
xceadmin scheduler record status/start/stop against a standby failed with the bare "controller returned status -10014" (E_XCE_STANDBY) with no redirect, forcing the caller to guess the primary manually. bug-1700: myreq_handler's generic ha_dispatch_gate rejection path (cluster.c) now calls fill_redirect_reply() when rc == -XCE_ERR_STANDBY, mirroring the same helper OPCode_join's handler already uses -- -XCE_ERR_NOT_READY (CTRL_INIT/CTRL_ELECTING, no settled primary to redirect to) is deliberately excluded. Landing this alone had zero observable effect live: ha_state_t.primary_host/primary_port were never written by any production path on a standby (only test fixtures set them) -- fill_redirect_reply() always saw an empty/zero redirect target in a real deployment. bug-1701: closes that data gap with a pure local lookup, no new wire field. handle_ctrlhello() (config_repl.c) now resolves the ctrlhello sender's name against this node's own statically-known peers_cfg (parsed once from --peers at boot) and populates primary_host/port at the exact moment a standby first learns who primary is. Live-verified end to end on a fresh 3-node cluster: xceadmin scheduler record status against any standby now prints "redirected: <standby> -> <primary>" and returns the real status, confirmed again after a re-election changed which node was primary. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AgYRU3zDTqNGV5Jk8aiKLg
This commit is contained in:
parent
ac1e312e2f
commit
f886c80930
4 changed files with 305 additions and 0 deletions
|
|
@ -235,6 +235,106 @@ static void test_schedadmin_rejects_when_not_primary(void)
|
|||
printf("test_schedadmin_rejects_when_not_primary: PASS\n");
|
||||
}
|
||||
|
||||
/* ── bug-1700: a standby dispatch-gate rejection must redirect to the known
|
||||
* primary ──────────────────────────────────────────────────────────────────
|
||||
*
|
||||
* ha_dispatch_gate() rejects OPCode_schedAdmin on a CTRL_STANDBY node with
|
||||
* -XCE_ERR_STANDBY (ha.c's CTRL_STANDBY case default branch, ~line 4774) --
|
||||
* this node MAY know who the primary is (c->ha.primary_host/primary_port).
|
||||
* Before the fix, myreq_handler's generic ha_dispatch_gate rejection block
|
||||
* (cluster.c ~406-414) never called fill_redirect_reply() (the exact same
|
||||
* one-line helper OPCode_join's handler already uses, join.c:277), so
|
||||
* redirect_host/redirect_port were left at their zero-value defaults on
|
||||
* EVERY dispatch-gate rejection -- xceadmin's client-side, already fully
|
||||
* implemented and generic follow_redirects() had nothing to follow and
|
||||
* surfaced the bare -XCE_ERR_STANDBY status as-is instead of transparently
|
||||
* retrying against the primary. This is the exact scenario the user hit
|
||||
* running `xceadmin scheduler record status` against a standby. */
|
||||
static void test_schedadmin_standby_with_known_primary_redirects(void)
|
||||
{
|
||||
cluster_t* c = sa_make_cluster();
|
||||
atomic_store(&c->ha.role, CTRL_STANDBY);
|
||||
strncpy(c->ha.primary_host, "10.0.1.7", sizeof(c->ha.primary_host) - 1);
|
||||
c->ha.primary_port = 7001;
|
||||
|
||||
/* Poison sentinel mirrors test_schedadmin_rejects_when_not_primary above
|
||||
* -- a gate rejection must never reach the forwarding code. */
|
||||
endpoint_t* poison = (endpoint_t*)(uintptr_t)0x1;
|
||||
c->sched.ep = poison;
|
||||
|
||||
endpoint_t ep;
|
||||
memset(&ep, 0, sizeof(ep));
|
||||
ep.owner = c;
|
||||
|
||||
io_packet_t* pkt = sa_build_request_pkt();
|
||||
int rc = myreq_handler_for_test(&ep, pkt);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rc);
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
c->sched.ep == poison,
|
||||
"c->sched.ep must be untouched -- a gate rejection never forwards");
|
||||
|
||||
xce_request_t decoded;
|
||||
sa_decode_queued_reply(&ep, &decoded);
|
||||
TEST_ASSERT_EQUAL_INT(Request_body_replymsg, decoded.body_which);
|
||||
xce_reply_t* rr = decoded.body.replymsg;
|
||||
TEST_ASSERT_NOT_NULL(rr);
|
||||
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rr->status);
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(
|
||||
rr->redirect_host,
|
||||
"bug-1700: a standby dispatch-gate rejection with a known primary "
|
||||
"must carry that primary's address so xceadmin's follow_redirects() "
|
||||
"can transparently retry there");
|
||||
TEST_ASSERT_EQUAL_STRING("10.0.1.7", rr->redirect_host);
|
||||
TEST_ASSERT_EQUAL_UINT16(7001, rr->redirect_port);
|
||||
xce_free_request(&decoded);
|
||||
|
||||
sa_teardown(c);
|
||||
printf("test_schedadmin_standby_with_known_primary_redirects: PASS\n");
|
||||
}
|
||||
|
||||
/* Regression companion: a standby that does NOT yet know any live primary
|
||||
* (fresh ha_state_t straight out of calloc -- primary_host empty,
|
||||
* primary_port 0, e.g. very early boot / mid-election / primary just died
|
||||
* with no successor elected yet) must NOT crash and must produce a reply
|
||||
* the client correctly treats as non-redirecting: discovery.py's
|
||||
* `_redirect_target()` requires BOTH a non-empty host AND a non-zero port
|
||||
* to treat a reply as a redirect, so an empty host / zero port here is
|
||||
* harmless and falls back to today's bare-rejection behavior. Proves the
|
||||
* fix does not regress the genuinely-no-primary-yet case. */
|
||||
static void test_schedadmin_standby_no_known_primary_no_redirect(void)
|
||||
{
|
||||
cluster_t* c = sa_make_cluster();
|
||||
atomic_store(&c->ha.role, CTRL_STANDBY);
|
||||
/* c->ha.primary_host/primary_port left at their calloc'd zero value. */
|
||||
|
||||
endpoint_t ep;
|
||||
memset(&ep, 0, sizeof(ep));
|
||||
ep.owner = c;
|
||||
|
||||
io_packet_t* pkt = sa_build_request_pkt();
|
||||
int rc = myreq_handler_for_test(&ep, pkt);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rc);
|
||||
|
||||
xce_request_t decoded;
|
||||
sa_decode_queued_reply(&ep, &decoded);
|
||||
TEST_ASSERT_EQUAL_INT(Request_body_replymsg, decoded.body_which);
|
||||
xce_reply_t* rr = decoded.body.replymsg;
|
||||
TEST_ASSERT_NOT_NULL(rr);
|
||||
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rr->status);
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
rr->redirect_host == NULL || rr->redirect_host[0] == '\0',
|
||||
"no known primary: redirect_host must decode as empty/absent, "
|
||||
"never garbage -- the client's _redirect_target() treats an empty "
|
||||
"host as 'no redirect, use this reply as-is'");
|
||||
TEST_ASSERT_EQUAL_UINT16(0, rr->redirect_port);
|
||||
xce_free_request(&decoded);
|
||||
|
||||
sa_teardown(c);
|
||||
printf("test_schedadmin_standby_no_known_primary_no_redirect: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Observable 2: primary with no scheduler connected rejects immediately ──
|
||||
*/
|
||||
static void test_schedadmin_rejects_when_not_connected(void)
|
||||
|
|
@ -775,6 +875,8 @@ void test_sched_admin_dispatch(void)
|
|||
{
|
||||
UnitySetTestFile(__FILE__);
|
||||
RUN_TEST(test_schedadmin_rejects_when_not_primary);
|
||||
RUN_TEST(test_schedadmin_standby_with_known_primary_redirects);
|
||||
RUN_TEST(test_schedadmin_standby_no_known_primary_no_redirect);
|
||||
RUN_TEST(test_schedadmin_rejects_when_not_connected);
|
||||
RUN_TEST(test_schedadmin_fast_rejects_when_admission_exhausted);
|
||||
RUN_TEST(test_schedadmin_round_trip_success);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue