fix(scheduler): close post-failover rejoin detection gap (bug-1760)
An idle scheduler (nothing queued to publish) never noticed a dead controller connection: sched_needs_rejoin was only ever set as a side effect of an active client-library call, and an idle round makes none. After any controller failover with nothing already in flight, jobs submitted to the new primary stayed PEND indefinitely. Adds xcecli_is_connection_broken() (libxce) -- a pure, socket-free read of the transport's already-async-maintained dead/alive flag, set for free by the existing RDHUP disconnect handler with zero application calls. sched_check_pending_rejoin() now retries xcecli_join() on either the existing reactive flag or this new passive signal, so a healthy idle scheduler still makes zero extra network calls while a genuinely dead connection self-heals within one ~1s round -- live-verified against a 3-node HA cluster (rejoin fired ~300ms after new-primary election with nothing submitted in between). Also closes out bug-1735 (job memory/CPU usage reporting): confirmed already fixed on this branch since 2026-08-30, just never marked in the bug log. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XYdtrEWWZfKRRGED7raEYg
This commit is contained in:
parent
48587368ca
commit
88b0ac9937
9 changed files with 441 additions and 28 deletions
File diff suppressed because one or more lines are too long
|
|
@ -51,6 +51,20 @@ extern int xcecli_wakeup(xcecli_ctx_t, int);
|
||||||
extern int xcecli_hello(xcecli_ctx_t, char*);
|
extern int xcecli_hello(xcecli_ctx_t, char*);
|
||||||
extern int xcecli_nop(xcecli_ctx_t);
|
extern int xcecli_nop(xcecli_ctx_t);
|
||||||
extern int xcecli_heartbeat(xcecli_ctx_t, int);
|
extern int xcecli_heartbeat(xcecli_ctx_t, int);
|
||||||
|
/*
|
||||||
|
* xcecli_is_connection_broken (bug-1760) - PURE, socket-free, side-effect-free
|
||||||
|
* query: has the transport been detected dead since the last successful
|
||||||
|
* (re)connect? Mirrors the private xcecli_ff_needs_reconnect() predicate
|
||||||
|
* (client_internal.h) exactly, exposed publicly so a consumer's own idle
|
||||||
|
* loop can passively check for a dead connection -- with ZERO network I/O
|
||||||
|
* and zero blocking risk -- instead of periodically probing a possibly-
|
||||||
|
* healthy one. The underlying signal (ctx->state == XCE_CTX_ERR) is set
|
||||||
|
* asynchronously by the transport's own RDHUP/EPOLLERR handler
|
||||||
|
* (handle_disconnect(), xcectx.c) the instant the connection drops, with no
|
||||||
|
* application call involved -- this function only ever reads that
|
||||||
|
* already-computed flag. Returns false for a NULL ctx.
|
||||||
|
*/
|
||||||
|
extern bool xcecli_is_connection_broken(xcecli_ctx_t);
|
||||||
extern int xcecli_shutdown(xcecli_ctx_t);
|
extern int xcecli_shutdown(xcecli_ctx_t);
|
||||||
extern int xcecli_reslist(xcecli_ctx_t, xce_resource_t***);
|
extern int xcecli_reslist(xcecli_ctx_t, xce_resource_t***);
|
||||||
extern int xcecli_join(xcecli_ctx_t, int, hostattrs_t*);
|
extern int xcecli_join(xcecli_ctx_t, int, hostattrs_t*);
|
||||||
|
|
|
||||||
|
|
@ -264,6 +264,15 @@ int xcecli_heartbeat(xcecli_ctx_t c, int hb_interval)
|
||||||
return XCECLI_FF_CALL(ctx, xce_heartbeat(&(ctx->ctx), hb_interval));
|
return XCECLI_FF_CALL(ctx, xce_heartbeat(&(ctx->ctx), hb_interval));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool xcecli_is_connection_broken(xcecli_ctx_t c)
|
||||||
|
{
|
||||||
|
/* Delegates to the exact predicate XCECLI_FF_CALL already uses
|
||||||
|
* internally (client_internal.h) -- same NULL-safe, socket-free,
|
||||||
|
* side-effect-free read of ctx->ctx.state, just exposed publicly (see
|
||||||
|
* client.h's doc comment for the full rationale, bug-1760). */
|
||||||
|
return xcecli_ff_needs_reconnect((api_ctx_t*)c);
|
||||||
|
}
|
||||||
|
|
||||||
int xcecli_shutdown(xcecli_ctx_t c)
|
int xcecli_shutdown(xcecli_ctx_t c)
|
||||||
{
|
{
|
||||||
api_ctx_t* ctx = (api_ctx_t*)c;
|
api_ctx_t* ctx = (api_ctx_t*)c;
|
||||||
|
|
|
||||||
|
|
@ -1700,6 +1700,46 @@ static void test_ff_needs_reconnect_null_safe(void)
|
||||||
printf("test_ff_needs_reconnect_null_safe: PASS\n");
|
printf("test_ff_needs_reconnect_null_safe: PASS\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ════════════════════════════════════════════════════════════════════════
|
||||||
|
* bug-1760 — xcecli_is_connection_broken(): the public, xcecli_ctx_t-typed
|
||||||
|
* wrapper around xcecli_ff_needs_reconnect() above (client.c), letting an
|
||||||
|
* idle consumer loop (e.g. the Scheduler's round loop) passively check for
|
||||||
|
* a transport already known-dead via the async RDHUP disconnect path
|
||||||
|
* (handle_disconnect(), xcectx.c) -- zero network I/O, zero blocking risk --
|
||||||
|
* instead of periodically probing a possibly-healthy connection. Same
|
||||||
|
* ERR/RUN/DOWN/INIT state matrix as the tests immediately above; this group
|
||||||
|
* only proves the public wrapper delegates correctly, not the predicate
|
||||||
|
* logic itself (already fully covered above).
|
||||||
|
* ════════════════════════════════════════════════════════════════════════ */
|
||||||
|
|
||||||
|
static void test_is_connection_broken_true_on_err(void)
|
||||||
|
{
|
||||||
|
api_ctx_t ac;
|
||||||
|
memset(&ac, 0, sizeof(ac));
|
||||||
|
ac.ctx.state = XCE_CTX_ERR;
|
||||||
|
|
||||||
|
TEST_ASSERT_TRUE(xcecli_is_connection_broken((xcecli_ctx_t)&ac));
|
||||||
|
|
||||||
|
printf("test_is_connection_broken_true_on_err: PASS\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_is_connection_broken_false_on_run(void)
|
||||||
|
{
|
||||||
|
api_ctx_t ac;
|
||||||
|
memset(&ac, 0, sizeof(ac));
|
||||||
|
ac.ctx.state = XCE_CTX_RUN;
|
||||||
|
|
||||||
|
TEST_ASSERT_FALSE(xcecli_is_connection_broken((xcecli_ctx_t)&ac));
|
||||||
|
|
||||||
|
printf("test_is_connection_broken_false_on_run: PASS\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_is_connection_broken_null_safe(void)
|
||||||
|
{
|
||||||
|
TEST_ASSERT_FALSE(xcecli_is_connection_broken(NULL));
|
||||||
|
printf("test_is_connection_broken_null_safe: PASS\n");
|
||||||
|
}
|
||||||
|
|
||||||
/* Req 5.3 (structural): the fire-and-forget path is single-shot — it makes AT
|
/* Req 5.3 (structural): the fire-and-forget path is single-shot — it makes AT
|
||||||
* MOST ONE reconnect attempt and NEVER consumes the no-primary retry budget /
|
* MOST ONE reconnect attempt and NEVER consumes the no-primary retry budget /
|
||||||
* sleeps. We prove this structurally: the FF macro drives the body exactly once
|
* sleeps. We prove this structurally: the FF macro drives the body exactly once
|
||||||
|
|
@ -1800,6 +1840,9 @@ void test_libxce_retry(void)
|
||||||
RUN_TEST(test_ff_no_reconnect_on_run);
|
RUN_TEST(test_ff_no_reconnect_on_run);
|
||||||
RUN_TEST(test_ff_no_reconnect_on_down_or_init);
|
RUN_TEST(test_ff_no_reconnect_on_down_or_init);
|
||||||
RUN_TEST(test_ff_needs_reconnect_null_safe);
|
RUN_TEST(test_ff_needs_reconnect_null_safe);
|
||||||
|
RUN_TEST(test_is_connection_broken_true_on_err);
|
||||||
|
RUN_TEST(test_is_connection_broken_false_on_run);
|
||||||
|
RUN_TEST(test_is_connection_broken_null_safe);
|
||||||
RUN_TEST(test_ff_call_is_single_shot_no_budget);
|
RUN_TEST(test_ff_call_is_single_shot_no_budget);
|
||||||
UNITY_END();
|
UNITY_END();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -271,6 +271,12 @@ extern int sched_apply_job_rusage(xce_scheduler_t*, xce_job_rusages_t*);
|
||||||
extern int sched_commit_decision(xce_scheduler_t*, xce_decision_t*);
|
extern int sched_commit_decision(xce_scheduler_t*, xce_decision_t*);
|
||||||
extern int sched_publish_pend_reason(xce_scheduler_t*, xce_preason_set_t*);
|
extern int sched_publish_pend_reason(xce_scheduler_t*, xce_preason_set_t*);
|
||||||
extern int sched_publish_dispatch(xce_scheduler_t*, xce_job_dispatch_t*);
|
extern int sched_publish_dispatch(xce_scheduler_t*, xce_job_dispatch_t*);
|
||||||
|
/* bug-1760: exposed (was `static sched_publish_check_rejoin`) so
|
||||||
|
* xce_sched_driver_run_normal_mode_round() (driver.c) can call it
|
||||||
|
* unconditionally once per round, not only as a side effect of a
|
||||||
|
* successful decision publish -- see scheduler.c's doc comment on this
|
||||||
|
* function for the chicken-and-egg deadlock this closes. */
|
||||||
|
extern void sched_check_pending_rejoin(xce_scheduler_t*);
|
||||||
extern void sched_worker_release_host_ctx(xce_sched_worker_t*);
|
extern void sched_worker_release_host_ctx(xce_sched_worker_t*);
|
||||||
extern int sched_meet_host_selection(xce_scheduler_t*, xce_sched_worker_t*,
|
extern int sched_meet_host_selection(xce_scheduler_t*, xce_sched_worker_t*,
|
||||||
xce_select_t*);
|
xce_select_t*);
|
||||||
|
|
|
||||||
|
|
@ -468,6 +468,30 @@ int main(int argc, char* argv[])
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* bug-1760: check for a pending rejoin UNCONDITIONALLY, once per
|
||||||
|
* ~1s round, regardless of whether this round published anything.
|
||||||
|
* sched_check_pending_rejoin() was previously only invoked
|
||||||
|
* reactively, as a side effect of a successful decision publish
|
||||||
|
* (sched_publish_retry_loop(), scheduler.c) -- but after a
|
||||||
|
* controller failover, the scheduler can have nothing queued to
|
||||||
|
* publish at all (the new primary has no record of this scheduler
|
||||||
|
* ever having joined it, so it never routes any job to this
|
||||||
|
* scheduler's picker), so the reactive-only call could never fire
|
||||||
|
* and a pending rejoin could be stuck forever: a chicken-and-egg
|
||||||
|
* deadlock -- closed by sched_check_pending_rejoin() itself now
|
||||||
|
* also checking a passive, zero-I/O "connection already known
|
||||||
|
* dead" signal in addition to this flag; see that function's own
|
||||||
|
* doc comment (scheduler.c) for the full mechanism. Placed here
|
||||||
|
* (main.c), not inside driver.c's mode-agnostic
|
||||||
|
* xce_sched_driver_run_normal_mode_round(), because
|
||||||
|
* driver.c is linked by several sim-mode test binaries that never
|
||||||
|
* link scheduler.c (where this function and sched_needs_rejoin
|
||||||
|
* live) -- calling a scheduler.c symbol from driver.c broke those
|
||||||
|
* links under -Wl,--no-undefined. main.c is normal-mode-only and
|
||||||
|
* not linked into any test binary, so this call site has no such
|
||||||
|
* cross-boundary cost. */
|
||||||
|
sched_check_pending_rejoin(&sched);
|
||||||
|
|
||||||
sleep(1);
|
sleep(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1455,34 +1455,70 @@ int sched_commit_decision(xce_scheduler_t* sched, xce_decision_t* decision)
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* sched_publish_check_rejoin - after a decision publish call finally
|
* sched_check_pending_rejoin - re-issue xcecli_join() as a follow-up if
|
||||||
* succeeds, re-issue xcecli_join() as a separate follow-up if the client
|
* either (a) the client library's reconnect callback (sched_on_reconnect(),
|
||||||
* library's reconnect callback (sched_on_reconnect(), above) flagged a
|
* above) flagged a pending rejoin (component-failover-adoption, task 3.2,
|
||||||
* pending rejoin (component-failover-adoption, task 3.2, Req 2 AC5).
|
* Req 2 AC5), or (b) the connection is passively known to be broken right
|
||||||
|
* now (bug-1760, see below) -- either condition alone is sufficient.
|
||||||
|
*
|
||||||
|
* Called from TWO places: sched_publish_retry_loop() below calls it
|
||||||
|
* reactively right after a successful decision publish (the original task
|
||||||
|
* 3.2 behavior, lowest possible rejoin latency when there IS something to
|
||||||
|
* publish); main()'s round loop (main.c) ALSO calls it unconditionally,
|
||||||
|
* once per ~1s scheduling round, regardless of whether that round
|
||||||
|
* publishes anything.
|
||||||
|
*
|
||||||
|
* bug-1760 fix, condition (b): sched_needs_rejoin ALONE cannot close the
|
||||||
|
* chicken-and-egg deadlock where the scheduler has nothing queued to
|
||||||
|
* publish -- it is only ever set by sched_on_reconnect(), which itself
|
||||||
|
* only fires as a side effect of an ACTUAL xcecli_* call detecting a dead
|
||||||
|
* connection (XCECLI_FAILOVER_CALL/XCECLI_FF_CALL, client_internal.h), and
|
||||||
|
* an idle round makes no such call. Confirmed live (2026-09-04, 3-node HA
|
||||||
|
* cluster): with condition (a) alone, a post-failover job submitted to the
|
||||||
|
* new primary stayed PEND indefinitely with zero scheduler.log rejoin
|
||||||
|
* activity, even though this function was already being called every
|
||||||
|
* round.
|
||||||
|
*
|
||||||
|
* Condition (b) -- xcecli_is_connection_broken(sched->apictx) -- closes
|
||||||
|
* this without adding any periodic active-liveness-probe call (which would
|
||||||
|
* trip test_scheduler_connect.c's Req-3-AC3 structural guard against
|
||||||
|
* introducing the worker's own periodic-probe pattern here, and carries a
|
||||||
|
* real, separate hazard: a stalled reconnect retry inside such a probe
|
||||||
|
* could itself delay/stall the scheduler's own round loop). Instead it
|
||||||
|
* reads a flag the transport ALREADY maintains for free: handle_disconnect()
|
||||||
|
* (xcectx.c) sets ctx->state = XCE_CTX_ERR asynchronously, from the iomgr's
|
||||||
|
* own epoll/RDHUP handling, the instant the connection drops -- zero
|
||||||
|
* application calls, zero blocking. xcecli_is_connection_broken() (client.h)
|
||||||
|
* is a pure, socket-free, side-effect-free read of that flag. A healthy,
|
||||||
|
* idle scheduler therefore makes zero extra calls of any kind; this branch
|
||||||
|
* only ever reaches xcecli_join() below when the transport is ALREADY
|
||||||
|
* known-dead, at which point blocking through xcecli_join()'s own
|
||||||
|
* XCECLI_FAILOVER_CALL retry/rotate logic is the same cost the round loop
|
||||||
|
* already accepts whenever it has a live decision to publish during a
|
||||||
|
* failover -- not a new category of stall.
|
||||||
*
|
*
|
||||||
* Deliberately NOT its own retry loop: on failure, sched_needs_rejoin
|
* Deliberately NOT its own retry loop: on failure, sched_needs_rejoin
|
||||||
* stays set, so the NEXT decision's publish call retries this same
|
* stays set (if it was already set) and/or xcecli_is_connection_broken()
|
||||||
* rejoin -- the per-decision retry cadence sched_publish_retry_loop()
|
* keeps reporting true until a real reconnect succeeds, so the NEXT call
|
||||||
* below already establishes IS the retry mechanism (design.md's
|
* (whichever call site reaches it first) retries this same rejoin.
|
||||||
* "Rejoin-check interaction" note).
|
|
||||||
*
|
*
|
||||||
* Uses the same rc<0-is-the-only-failure-test convention sched_init()'s
|
* Uses the same rc<0-is-the-only-failure-test convention sched_init()'s
|
||||||
* own xcecli_join() check uses above (component-failover-adoption task
|
* own xcecli_join() check uses above (component-failover-adoption task
|
||||||
* 2.3's worker lesson: a successful join returns the assigned slot index
|
* 2.3's worker lesson: a successful join returns the assigned slot index
|
||||||
* as reply.status, which is not always exactly 0).
|
* as reply.status, which is not always exactly 0).
|
||||||
*/
|
*/
|
||||||
static void sched_publish_check_rejoin(xce_scheduler_t* sched)
|
void sched_check_pending_rejoin(xce_scheduler_t* sched)
|
||||||
{
|
{
|
||||||
int rc;
|
int rc;
|
||||||
|
|
||||||
if (!sched_needs_rejoin) {
|
if (!sched_needs_rejoin && !xcecli_is_connection_broken(sched->apictx)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
rc = xcecli_join(sched->apictx, HostRole_scheduler, NULL);
|
rc = xcecli_join(sched->apictx, HostRole_scheduler, NULL);
|
||||||
if (rc < 0) {
|
if (rc < 0) {
|
||||||
WARN("%s: pending rejoin retry failed (rc=%d) -- will retry on the "
|
WARN("%s: pending rejoin retry failed (rc=%d) -- will retry on the "
|
||||||
"next decision publish",
|
"next check",
|
||||||
__func__, rc);
|
__func__, rc);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
@ -1562,7 +1598,7 @@ static int sched_publish_retry_loop(xce_scheduler_t* sched, void* payload,
|
||||||
}
|
}
|
||||||
|
|
||||||
if (rc >= 0) {
|
if (rc >= 0) {
|
||||||
sched_publish_check_rejoin(sched);
|
sched_check_pending_rejoin(sched);
|
||||||
}
|
}
|
||||||
|
|
||||||
return rc;
|
return rc;
|
||||||
|
|
|
||||||
|
|
@ -1313,11 +1313,15 @@ sched_scheduler_decision_publish_retry_test = executable(
|
||||||
scheduler_decision_publish_retry_test_srcs,
|
scheduler_decision_publish_retry_test_srcs,
|
||||||
include_directories:[incdir],
|
include_directories:[incdir],
|
||||||
dependencies: [scheduler_deps, unity_dep],
|
dependencies: [scheduler_deps, unity_dep],
|
||||||
|
c_args: ['-DXCE_MAIN_C_PATH="'
|
||||||
|
+ (meson.current_source_dir() / '..' / 'src' / 'main.c')
|
||||||
|
+ '"'],
|
||||||
link_args: ['-Wl,--wrap=xcecli_jobdispatch',
|
link_args: ['-Wl,--wrap=xcecli_jobdispatch',
|
||||||
'-Wl,--wrap=xcecli_set_pend_reason',
|
'-Wl,--wrap=xcecli_set_pend_reason',
|
||||||
'-Wl,--wrap=xcecli_join',
|
'-Wl,--wrap=xcecli_join',
|
||||||
'-Wl,--wrap=xcecli_start',
|
'-Wl,--wrap=xcecli_start',
|
||||||
'-Wl,--wrap=xcecli_set_reconnect_handler'],
|
'-Wl,--wrap=xcecli_set_reconnect_handler',
|
||||||
|
'-Wl,--wrap=xcecli_is_connection_broken'],
|
||||||
install: false)
|
install: false)
|
||||||
|
|
||||||
# Worst-case decision/allocation backpressure test (component-failover-
|
# Worst-case decision/allocation backpressure test (component-failover-
|
||||||
|
|
|
||||||
|
|
@ -182,6 +182,28 @@ int __wrap_xcecli_join(xcecli_ctx_t c, int role, hostattrs_t* attrs)
|
||||||
return rc;
|
return rc;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ════════════════════════════════════════════════════════════════════════
|
||||||
|
* Test double: __wrap_xcecli_is_connection_broken (bug-1760)
|
||||||
|
*
|
||||||
|
* Defaults to false so every EXISTING test in this file (which never
|
||||||
|
* touches this double) sees the connection as healthy, exactly matching
|
||||||
|
* this file's pre-bug-1760 behavior -- sched_check_pending_rejoin()'s new
|
||||||
|
* OR'd condition is a pure no-op unless a test explicitly sets this true.
|
||||||
|
* ════════════════════════════════════════════════════════════════════════ */
|
||||||
|
|
||||||
|
static bool g_connection_broken;
|
||||||
|
|
||||||
|
static void reset_connection_broken_double(void)
|
||||||
|
{
|
||||||
|
g_connection_broken = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool __wrap_xcecli_is_connection_broken(xcecli_ctx_t c)
|
||||||
|
{
|
||||||
|
(void)c;
|
||||||
|
return g_connection_broken;
|
||||||
|
}
|
||||||
|
|
||||||
/* ════════════════════════════════════════════════════════════════════════
|
/* ════════════════════════════════════════════════════════════════════════
|
||||||
* Test doubles: __wrap_xcecli_start / __wrap_xcecli_set_reconnect_handler
|
* Test doubles: __wrap_xcecli_start / __wrap_xcecli_set_reconnect_handler
|
||||||
* -- setup-only doubles for the real sched_init() call the rejoin-
|
* -- setup-only doubles for the real sched_init() call the rejoin-
|
||||||
|
|
@ -215,6 +237,7 @@ void setUp(void)
|
||||||
reset_dispatch_double();
|
reset_dispatch_double();
|
||||||
reset_pend_double();
|
reset_pend_double();
|
||||||
reset_join_double();
|
reset_join_double();
|
||||||
|
reset_connection_broken_double();
|
||||||
}
|
}
|
||||||
|
|
||||||
void tearDown(void) { return; }
|
void tearDown(void) { return; }
|
||||||
|
|
@ -562,6 +585,243 @@ static void test_rejoin_composition_retries_across_publish_calls(void)
|
||||||
printf("test_rejoin_composition_retries_across_publish_calls: PASS\n");
|
printf("test_rejoin_composition_retries_across_publish_calls: PASS\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ════════════════════════════════════════════════════════════════════════
|
||||||
|
* bug-1760 — sched_check_pending_rejoin() must work as a standalone public
|
||||||
|
* entry point, invocable with NO decision ever having been published.
|
||||||
|
* This is the direct proof that the chicken-and-egg deadlock (a pending
|
||||||
|
* rejoin that could only ever be checked as a side effect of a successful
|
||||||
|
* publish) is closed: the function is now callable, and does the right
|
||||||
|
* thing, entirely independent of publish activity. The actual production
|
||||||
|
* call site added by this fix (xce_sched_driver_run_normal_mode_round(),
|
||||||
|
* driver.c, called unconditionally once per ~1s round) is verified by
|
||||||
|
* source-text scan below, since driving that function through a full round
|
||||||
|
* would require live picker/allocator threads this fixture deliberately
|
||||||
|
* never starts (see this file's header comment).
|
||||||
|
* ════════════════════════════════════════════════════════════════════════ */
|
||||||
|
|
||||||
|
static void test_check_pending_rejoin_invoked_directly_without_a_publish(void)
|
||||||
|
{
|
||||||
|
xce_scheduler_t sched;
|
||||||
|
xce_client_config_t config;
|
||||||
|
xce_cluster_config_t cluster_cfg;
|
||||||
|
int rc;
|
||||||
|
|
||||||
|
/* Same fixture setup as test_rejoin_composition_retries_across_publish_
|
||||||
|
* calls above: reach a real sched_init() far enough to register the
|
||||||
|
* reconnect handler, force its own join to fail so no picker/allocator
|
||||||
|
* thread starts, then simulate a transparent reconnect. */
|
||||||
|
memset(&config, 0, sizeof(config));
|
||||||
|
config.nmasters = 1;
|
||||||
|
config.masters[0] = "127.0.0.1";
|
||||||
|
config.master_ports[0] = 19999;
|
||||||
|
config.from = "sched_pubretry_direct_test";
|
||||||
|
config.key = "keepsecret$";
|
||||||
|
|
||||||
|
memset(&cluster_cfg, 0, sizeof(cluster_cfg));
|
||||||
|
cluster_cfg.candidates_count = 1;
|
||||||
|
strncpy(cluster_cfg.candidates[0].hostname, "127.0.0.1",
|
||||||
|
XCE_MAXHOSTNAME_LEN - 1);
|
||||||
|
cluster_cfg.candidates[0].port = 19999;
|
||||||
|
strncpy(cluster_cfg.identity, "sched_pubretry_direct_test",
|
||||||
|
sizeof(cluster_cfg.identity) - 1);
|
||||||
|
|
||||||
|
g_join_returns[0] = -EINVAL; /* sched_init()'s own join: fail on purpose */
|
||||||
|
g_join_nreturns = 1;
|
||||||
|
|
||||||
|
memset(&sched, 0, sizeof(sched));
|
||||||
|
rc = sched_init(&sched, &config, &cluster_cfg, scheduler_req_handler);
|
||||||
|
TEST_ASSERT_TRUE_MESSAGE(
|
||||||
|
rc < 0,
|
||||||
|
"fixture: sched_init()'s own join was deliberately made to fail so "
|
||||||
|
"no picker/allocator thread is spun up by this setup step");
|
||||||
|
TEST_ASSERT_NOT_NULL_MESSAGE(
|
||||||
|
g_captured_reconnect_cb,
|
||||||
|
"fixture: sched_init() must register the reconnect handler before "
|
||||||
|
"its xcecli_join() call");
|
||||||
|
|
||||||
|
/* No decision has EVER been published at this point -- unlike every
|
||||||
|
* other test in this file, sched_publish_dispatch()/
|
||||||
|
* sched_publish_pend_reason() are never called here at all. */
|
||||||
|
g_captured_reconnect_cb(sched.apictx, g_captured_reconnect_arg);
|
||||||
|
|
||||||
|
reset_join_double();
|
||||||
|
g_join_returns[0] = -E_XCE_NOT_READY; /* first direct check: still fails */
|
||||||
|
g_join_nreturns = 1;
|
||||||
|
|
||||||
|
sched_check_pending_rejoin(&sched);
|
||||||
|
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||||
|
1, g_join_call_count,
|
||||||
|
"sched_check_pending_rejoin() must be callable on its own and "
|
||||||
|
"attempt xcecli_join() when a rejoin is pending, with zero "
|
||||||
|
"decisions ever published");
|
||||||
|
|
||||||
|
g_join_returns[1] = 0; /* second direct check: succeeds */
|
||||||
|
g_join_nreturns = 2;
|
||||||
|
|
||||||
|
sched_check_pending_rejoin(&sched);
|
||||||
|
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||||
|
2, g_join_call_count,
|
||||||
|
"a still-pending rejoin must be retried on the next direct call");
|
||||||
|
|
||||||
|
/* Flag must now be clear -- a further direct call must not re-attempt
|
||||||
|
* the join. */
|
||||||
|
sched_check_pending_rejoin(&sched);
|
||||||
|
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||||
|
2, g_join_call_count,
|
||||||
|
"once the rejoin has succeeded, a further direct call must be a "
|
||||||
|
"no-op -- no third xcecli_join() call");
|
||||||
|
|
||||||
|
printf("test_check_pending_rejoin_invoked_directly_without_a_publish: "
|
||||||
|
"PASS\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ════════════════════════════════════════════════════════════════════════
|
||||||
|
* bug-1760 — passive-detection condition: sched_check_pending_rejoin() must
|
||||||
|
* attempt xcecli_join() when the transport is passively known-broken, even
|
||||||
|
* when sched_needs_rejoin was NEVER set (the exact idle-scheduler scenario
|
||||||
|
* a reactive-only check cannot detect on its own) -- and must make ZERO
|
||||||
|
* calls when neither condition holds, so a healthy idle scheduler never
|
||||||
|
* probes its connection. Uses a minimal zeroed fixture (no sched_init()
|
||||||
|
* needed): sched_check_pending_rejoin() only reads sched->apictx to pass
|
||||||
|
* verbatim into the wrapped xcecli_join()/xcecli_is_connection_broken()
|
||||||
|
* doubles, neither of which dereferences it.
|
||||||
|
* ════════════════════════════════════════════════════════════════════════ */
|
||||||
|
|
||||||
|
static void test_check_pending_rejoin_triggers_on_passive_broken_signal(void)
|
||||||
|
{
|
||||||
|
xce_scheduler_t sched;
|
||||||
|
|
||||||
|
memset(&sched, 0, sizeof(sched));
|
||||||
|
|
||||||
|
g_connection_broken = true;
|
||||||
|
g_join_returns[0] = 0;
|
||||||
|
g_join_nreturns = 1;
|
||||||
|
|
||||||
|
sched_check_pending_rejoin(&sched);
|
||||||
|
|
||||||
|
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||||
|
1, g_join_call_count,
|
||||||
|
"a passively-detected broken connection must trigger xcecli_join() "
|
||||||
|
"even when sched_needs_rejoin was never set -- this closes the "
|
||||||
|
"idle-scheduler chicken-and-egg gap a reactive-only check cannot");
|
||||||
|
|
||||||
|
printf("test_check_pending_rejoin_triggers_on_passive_broken_signal: "
|
||||||
|
"PASS\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_check_pending_rejoin_no_op_when_healthy_and_no_flag(void)
|
||||||
|
{
|
||||||
|
xce_scheduler_t sched;
|
||||||
|
|
||||||
|
memset(&sched, 0, sizeof(sched));
|
||||||
|
/* g_connection_broken defaults to false (setUp); sched_needs_rejoin
|
||||||
|
* (scheduler.c-private) is not touched by this test and is already
|
||||||
|
* false by this point in the file's test order. */
|
||||||
|
|
||||||
|
sched_check_pending_rejoin(&sched);
|
||||||
|
|
||||||
|
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||||
|
0, g_join_call_count,
|
||||||
|
"a healthy, idle scheduler (no flag set, connection reports "
|
||||||
|
"healthy) must make ZERO xcecli_join() calls -- this is the "
|
||||||
|
"no-periodic-probe-to-a-healthy-connection guarantee");
|
||||||
|
|
||||||
|
printf("test_check_pending_rejoin_no_op_when_healthy_and_no_flag: "
|
||||||
|
"PASS\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ════════════════════════════════════════════════════════════════════════
|
||||||
|
* bug-1760 — production wiring: main()'s round loop (scheduler/src/main.c)
|
||||||
|
* must call sched_check_pending_rejoin() UNCONDITIONALLY on every
|
||||||
|
* iteration, not only as a side effect of a successful decision publish.
|
||||||
|
* Verified by source-text scan (same technique test_scheduler_decision_
|
||||||
|
* redelivery.c's test_no_worker_resync_symbols_in_scheduler_publish_path
|
||||||
|
* uses) rather than driving main() itself, which isn't linked into any
|
||||||
|
* test binary (main.c is excluded from every scheduler_srcs test target
|
||||||
|
* in meson.build -- see this file's own foreach-exclusion, shared by
|
||||||
|
* every test target in this directory).
|
||||||
|
* ════════════════════════════════════════════════════════════════════════ */
|
||||||
|
|
||||||
|
static char* read_whole_file(const char* path)
|
||||||
|
{
|
||||||
|
FILE* f = fopen(path, "r");
|
||||||
|
long size;
|
||||||
|
char* buf;
|
||||||
|
size_t nread;
|
||||||
|
|
||||||
|
if (!f) {
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
fseek(f, 0, SEEK_END);
|
||||||
|
size = ftell(f);
|
||||||
|
fseek(f, 0, SEEK_SET);
|
||||||
|
if (size < 0) {
|
||||||
|
fclose(f);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
buf = (char*)malloc((size_t)size + 1);
|
||||||
|
if (!buf) {
|
||||||
|
fclose(f);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
nread = fread(buf, 1, (size_t)size, f);
|
||||||
|
buf[nread] = '\0';
|
||||||
|
fclose(f);
|
||||||
|
return buf;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_main_loop_checks_pending_rejoin_unconditionally(void)
|
||||||
|
{
|
||||||
|
char* main_src = read_whole_file(XCE_MAIN_C_PATH);
|
||||||
|
const char* func_sig = "int main(int argc, char* argv[])";
|
||||||
|
const char* call = "sched_check_pending_rejoin(&sched);";
|
||||||
|
const char* round_call
|
||||||
|
= "xce_sched_driver_run_normal_mode_round(&sched);";
|
||||||
|
const char* stop_check = "if (sched.core.stop)";
|
||||||
|
char* func_start;
|
||||||
|
char* call_pos;
|
||||||
|
char* round_pos;
|
||||||
|
char* stop_pos;
|
||||||
|
|
||||||
|
TEST_ASSERT_NOT_NULL_MESSAGE(main_src,
|
||||||
|
"could not read main.c source for scan");
|
||||||
|
|
||||||
|
func_start = strstr(main_src, func_sig);
|
||||||
|
TEST_ASSERT_NOT_NULL_MESSAGE(func_start,
|
||||||
|
"main.c must still define int main(...)");
|
||||||
|
|
||||||
|
round_pos = strstr(func_start, round_call);
|
||||||
|
stop_pos = strstr(func_start, stop_check);
|
||||||
|
call_pos = strstr(func_start, call);
|
||||||
|
|
||||||
|
TEST_ASSERT_NOT_NULL_MESSAGE(
|
||||||
|
round_pos, "sanity: the round loop's xce_sched_driver_run_normal_"
|
||||||
|
"mode_round(&sched) call must still exist");
|
||||||
|
TEST_ASSERT_NOT_NULL_MESSAGE(
|
||||||
|
stop_pos, "sanity: the round loop's core.stop check must still "
|
||||||
|
"exist");
|
||||||
|
TEST_ASSERT_NOT_NULL_MESSAGE(
|
||||||
|
call_pos,
|
||||||
|
"bug-1760: main()'s round loop must call "
|
||||||
|
"sched_check_pending_rejoin(&sched) on every iteration -- without "
|
||||||
|
"this, a pending rejoin after a controller failover can only ever "
|
||||||
|
"be retried as a side effect of a successful decision publish, "
|
||||||
|
"which may never happen (chicken-and-egg deadlock)");
|
||||||
|
TEST_ASSERT_TRUE_MESSAGE(
|
||||||
|
(round_pos < call_pos) && (stop_pos < call_pos),
|
||||||
|
"sched_check_pending_rejoin() must run AFTER the round's own "
|
||||||
|
"stop-check (never called on the iteration that is about to "
|
||||||
|
"break/shut down) but is placed unconditionally on every "
|
||||||
|
"surviving iteration regardless of what that round scheduled");
|
||||||
|
|
||||||
|
free(main_src);
|
||||||
|
printf("test_main_loop_checks_pending_rejoin_unconditionally: "
|
||||||
|
"PASS\n");
|
||||||
|
}
|
||||||
|
|
||||||
int main(void)
|
int main(void)
|
||||||
{
|
{
|
||||||
UNITY_BEGIN();
|
UNITY_BEGIN();
|
||||||
|
|
@ -571,5 +831,9 @@ int main(void)
|
||||||
RUN_TEST(test_extended_retry_success_logs_distinctly);
|
RUN_TEST(test_extended_retry_success_logs_distinctly);
|
||||||
RUN_TEST(test_first_attempt_success_does_not_log_extended_retry);
|
RUN_TEST(test_first_attempt_success_does_not_log_extended_retry);
|
||||||
RUN_TEST(test_rejoin_composition_retries_across_publish_calls);
|
RUN_TEST(test_rejoin_composition_retries_across_publish_calls);
|
||||||
|
RUN_TEST(test_check_pending_rejoin_invoked_directly_without_a_publish);
|
||||||
|
RUN_TEST(test_check_pending_rejoin_triggers_on_passive_broken_signal);
|
||||||
|
RUN_TEST(test_check_pending_rejoin_no_op_when_healthy_and_no_flag);
|
||||||
|
RUN_TEST(test_main_loop_checks_pending_rejoin_unconditionally);
|
||||||
return UNITY_END();
|
return UNITY_END();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue