lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(scheduler): close post-failover rejoin detection gap (bug-1760)

Browse source 
An idle scheduler (nothing queued to publish) never noticed a dead
controller connection: sched_needs_rejoin was only ever set as a side
effect of an active client-library call, and an idle round makes none.
After any controller failover with nothing already in flight, jobs
submitted to the new primary stayed PEND indefinitely.

Adds xcecli_is_connection_broken() (libxce) -- a pure, socket-free read
of the transport's already-async-maintained dead/alive flag, set for
free by the existing RDHUP disconnect handler with zero application
calls. sched_check_pending_rejoin() now retries xcecli_join() on either
the existing reactive flag or this new passive signal, so a healthy
idle scheduler still makes zero extra network calls while a genuinely
dead connection self-heals within one ~1s round -- live-verified
against a 3-node HA cluster (rejoin fired ~300ms after new-primary
election with nothing submitted in between).

Also closes out bug-1735 (job memory/CPU usage reporting): confirmed
already fixed on this branch since 2026-08-30, just never marked in
the bug log.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XYdtrEWWZfKRRGED7raEYg
This commit is contained in:
Rongsong Shen 2026-09-04 04:37:11 +00:00
parent 48587368ca
commit 88b0ac9937
No known key found for this signature in database
GPG key ID: 5BCDAAE0123E9FE2
9 changed files with 441 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

9
src/lib/client.c
View file

@ -264,6 +264,15 @@ int xcecli_heartbeat(xcecli_ctx_t c, int hb_interval)
return XCECLI_FF_CALL(ctx, xce_heartbeat(&(ctx->ctx), hb_interval));
}
bool xcecli_is_connection_broken(xcecli_ctx_t c)
{
/* Delegates to the exact predicate XCECLI_FF_CALL already uses
* internally (client_internal.h) -- same NULL-safe, socket-free,
* side-effect-free read of ctx->ctx.state, just exposed publicly (see
* client.h's doc comment for the full rationale, bug-1760). */
return xcecli_ff_needs_reconnect((api_ctx_t*)c);
}
int xcecli_shutdown(xcecli_ctx_t c)
{
api_ctx_t* ctx = (api_ctx_t*)c;

43
src/lib/testsuite/test_libxce_retry.c
View file

@ -1700,6 +1700,46 @@ static void test_ff_needs_reconnect_null_safe(void)
printf("test_ff_needs_reconnect_null_safe: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* bug-1760 — xcecli_is_connection_broken(): the public, xcecli_ctx_t-typed
* wrapper around xcecli_ff_needs_reconnect() above (client.c), letting an
* idle consumer loop (e.g. the Scheduler's round loop) passively check for
* a transport already known-dead via the async RDHUP disconnect path
* (handle_disconnect(), xcectx.c) -- zero network I/O, zero blocking risk --
* instead of periodically probing a possibly-healthy connection. Same
* ERR/RUN/DOWN/INIT state matrix as the tests immediately above; this group
* only proves the public wrapper delegates correctly, not the predicate
* logic itself (already fully covered above).
* ════════════════════════════════════════════════════════════════════════ */
static void test_is_connection_broken_true_on_err(void)
{
api_ctx_t ac;
memset(&ac, 0, sizeof(ac));
ac.ctx.state = XCE_CTX_ERR;
TEST_ASSERT_TRUE(xcecli_is_connection_broken((xcecli_ctx_t)&ac));
printf("test_is_connection_broken_true_on_err: PASS\n");
}
static void test_is_connection_broken_false_on_run(void)
{
api_ctx_t ac;
memset(&ac, 0, sizeof(ac));
ac.ctx.state = XCE_CTX_RUN;
TEST_ASSERT_FALSE(xcecli_is_connection_broken((xcecli_ctx_t)&ac));
printf("test_is_connection_broken_false_on_run: PASS\n");
}
static void test_is_connection_broken_null_safe(void)
{
TEST_ASSERT_FALSE(xcecli_is_connection_broken(NULL));
printf("test_is_connection_broken_null_safe: PASS\n");
}
/* Req 5.3 (structural): the fire-and-forget path is single-shot — it makes AT
* MOST ONE reconnect attempt and NEVER consumes the no-primary retry budget /
* sleeps. We prove this structurally: the FF macro drives the body exactly once
@ -1800,6 +1840,9 @@ void test_libxce_retry(void)
RUN_TEST(test_ff_no_reconnect_on_run);
RUN_TEST(test_ff_no_reconnect_on_down_or_init);
RUN_TEST(test_ff_needs_reconnect_null_safe);
RUN_TEST(test_is_connection_broken_true_on_err);
RUN_TEST(test_is_connection_broken_false_on_run);
RUN_TEST(test_is_connection_broken_null_safe);
RUN_TEST(test_ff_call_is_single_shot_no_budget);
UNITY_END();
}