fix(scheduler): close post-failover rejoin detection gap (bug-1760)

An idle scheduler (nothing queued to publish) never noticed a dead
controller connection: sched_needs_rejoin was only ever set as a side
effect of an active client-library call, and an idle round makes none.
After any controller failover with nothing already in flight, jobs
submitted to the new primary stayed PEND indefinitely.

Adds xcecli_is_connection_broken() (libxce) -- a pure, socket-free read
of the transport's already-async-maintained dead/alive flag, set for
free by the existing RDHUP disconnect handler with zero application
calls. sched_check_pending_rejoin() now retries xcecli_join() on either
the existing reactive flag or this new passive signal, so a healthy
idle scheduler still makes zero extra network calls while a genuinely
dead connection self-heals within one ~1s round -- live-verified
against a 3-node HA cluster (rejoin fired ~300ms after new-primary
election with nothing submitted in between).

Also closes out bug-1735 (job memory/CPU usage reporting): confirmed
already fixed on this branch since 2026-08-30, just never marked in
the bug log.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XYdtrEWWZfKRRGED7raEYg
This commit is contained in:
Rongsong Shen 2026-09-04 04:37:11 +00:00
parent 48587368ca
commit 88b0ac9937
No known key found for this signature in database
GPG key ID: 5BCDAAE0123E9FE2
9 changed files with 441 additions and 28 deletions

File diff suppressed because one or more lines are too long

View file

@ -51,6 +51,20 @@ extern int xcecli_wakeup(xcecli_ctx_t, int);
extern int xcecli_hello(xcecli_ctx_t, char*); extern int xcecli_hello(xcecli_ctx_t, char*);
extern int xcecli_nop(xcecli_ctx_t); extern int xcecli_nop(xcecli_ctx_t);
extern int xcecli_heartbeat(xcecli_ctx_t, int); extern int xcecli_heartbeat(xcecli_ctx_t, int);
/*
* xcecli_is_connection_broken (bug-1760) - PURE, socket-free, side-effect-free
* query: has the transport been detected dead since the last successful
* (re)connect? Mirrors the private xcecli_ff_needs_reconnect() predicate
* (client_internal.h) exactly, exposed publicly so a consumer's own idle
* loop can passively check for a dead connection -- with ZERO network I/O
* and zero blocking risk -- instead of periodically probing a possibly-
* healthy one. The underlying signal (ctx->state == XCE_CTX_ERR) is set
* asynchronously by the transport's own RDHUP/EPOLLERR handler
* (handle_disconnect(), xcectx.c) the instant the connection drops, with no
* application call involved -- this function only ever reads that
* already-computed flag. Returns false for a NULL ctx.
*/
extern bool xcecli_is_connection_broken(xcecli_ctx_t);
extern int xcecli_shutdown(xcecli_ctx_t); extern int xcecli_shutdown(xcecli_ctx_t);
extern int xcecli_reslist(xcecli_ctx_t, xce_resource_t***); extern int xcecli_reslist(xcecli_ctx_t, xce_resource_t***);
extern int xcecli_join(xcecli_ctx_t, int, hostattrs_t*); extern int xcecli_join(xcecli_ctx_t, int, hostattrs_t*);

View file

@ -264,6 +264,15 @@ int xcecli_heartbeat(xcecli_ctx_t c, int hb_interval)
return XCECLI_FF_CALL(ctx, xce_heartbeat(&(ctx->ctx), hb_interval)); return XCECLI_FF_CALL(ctx, xce_heartbeat(&(ctx->ctx), hb_interval));
} }
bool xcecli_is_connection_broken(xcecli_ctx_t c)
{
/* Delegates to the exact predicate XCECLI_FF_CALL already uses
* internally (client_internal.h) -- same NULL-safe, socket-free,
* side-effect-free read of ctx->ctx.state, just exposed publicly (see
* client.h's doc comment for the full rationale, bug-1760). */
return xcecli_ff_needs_reconnect((api_ctx_t*)c);
}
int xcecli_shutdown(xcecli_ctx_t c) int xcecli_shutdown(xcecli_ctx_t c)
{ {
api_ctx_t* ctx = (api_ctx_t*)c; api_ctx_t* ctx = (api_ctx_t*)c;

View file

@ -1700,6 +1700,46 @@ static void test_ff_needs_reconnect_null_safe(void)
printf("test_ff_needs_reconnect_null_safe: PASS\n"); printf("test_ff_needs_reconnect_null_safe: PASS\n");
} }
/* ════════════════════════════════════════════════════════════════════════
* bug-1760 — xcecli_is_connection_broken(): the public, xcecli_ctx_t-typed
* wrapper around xcecli_ff_needs_reconnect() above (client.c), letting an
* idle consumer loop (e.g. the Scheduler's round loop) passively check for
* a transport already known-dead via the async RDHUP disconnect path
* (handle_disconnect(), xcectx.c) -- zero network I/O, zero blocking risk --
* instead of periodically probing a possibly-healthy connection. Same
* ERR/RUN/DOWN/INIT state matrix as the tests immediately above; this group
* only proves the public wrapper delegates correctly, not the predicate
* logic itself (already fully covered above).
* ════════════════════════════════════════════════════════════════════════ */
static void test_is_connection_broken_true_on_err(void)
{
api_ctx_t ac;
memset(&ac, 0, sizeof(ac));
ac.ctx.state = XCE_CTX_ERR;
TEST_ASSERT_TRUE(xcecli_is_connection_broken((xcecli_ctx_t)&ac));
printf("test_is_connection_broken_true_on_err: PASS\n");
}
static void test_is_connection_broken_false_on_run(void)
{
api_ctx_t ac;
memset(&ac, 0, sizeof(ac));
ac.ctx.state = XCE_CTX_RUN;
TEST_ASSERT_FALSE(xcecli_is_connection_broken((xcecli_ctx_t)&ac));
printf("test_is_connection_broken_false_on_run: PASS\n");
}
static void test_is_connection_broken_null_safe(void)
{
TEST_ASSERT_FALSE(xcecli_is_connection_broken(NULL));
printf("test_is_connection_broken_null_safe: PASS\n");
}
/* Req 5.3 (structural): the fire-and-forget path is single-shot — it makes AT /* Req 5.3 (structural): the fire-and-forget path is single-shot — it makes AT
* MOST ONE reconnect attempt and NEVER consumes the no-primary retry budget / * MOST ONE reconnect attempt and NEVER consumes the no-primary retry budget /
* sleeps. We prove this structurally: the FF macro drives the body exactly once * sleeps. We prove this structurally: the FF macro drives the body exactly once
@ -1800,6 +1840,9 @@ void test_libxce_retry(void)
RUN_TEST(test_ff_no_reconnect_on_run); RUN_TEST(test_ff_no_reconnect_on_run);
RUN_TEST(test_ff_no_reconnect_on_down_or_init); RUN_TEST(test_ff_no_reconnect_on_down_or_init);
RUN_TEST(test_ff_needs_reconnect_null_safe); RUN_TEST(test_ff_needs_reconnect_null_safe);
RUN_TEST(test_is_connection_broken_true_on_err);
RUN_TEST(test_is_connection_broken_false_on_run);
RUN_TEST(test_is_connection_broken_null_safe);
RUN_TEST(test_ff_call_is_single_shot_no_budget); RUN_TEST(test_ff_call_is_single_shot_no_budget);
UNITY_END(); UNITY_END();
} }

View file

@ -271,6 +271,12 @@ extern int sched_apply_job_rusage(xce_scheduler_t*, xce_job_rusages_t*);
extern int sched_commit_decision(xce_scheduler_t*, xce_decision_t*); extern int sched_commit_decision(xce_scheduler_t*, xce_decision_t*);
extern int sched_publish_pend_reason(xce_scheduler_t*, xce_preason_set_t*); extern int sched_publish_pend_reason(xce_scheduler_t*, xce_preason_set_t*);
extern int sched_publish_dispatch(xce_scheduler_t*, xce_job_dispatch_t*); extern int sched_publish_dispatch(xce_scheduler_t*, xce_job_dispatch_t*);
/* bug-1760: exposed (was `static sched_publish_check_rejoin`) so
* xce_sched_driver_run_normal_mode_round() (driver.c) can call it
* unconditionally once per round, not only as a side effect of a
* successful decision publish -- see scheduler.c's doc comment on this
* function for the chicken-and-egg deadlock this closes. */
extern void sched_check_pending_rejoin(xce_scheduler_t*);
extern void sched_worker_release_host_ctx(xce_sched_worker_t*); extern void sched_worker_release_host_ctx(xce_sched_worker_t*);
extern int sched_meet_host_selection(xce_scheduler_t*, xce_sched_worker_t*, extern int sched_meet_host_selection(xce_scheduler_t*, xce_sched_worker_t*,
xce_select_t*); xce_select_t*);

View file

@ -468,6 +468,30 @@ int main(int argc, char* argv[])
break; break;
} }
/* bug-1760: check for a pending rejoin UNCONDITIONALLY, once per
* ~1s round, regardless of whether this round published anything.
* sched_check_pending_rejoin() was previously only invoked
* reactively, as a side effect of a successful decision publish
* (sched_publish_retry_loop(), scheduler.c) -- but after a
* controller failover, the scheduler can have nothing queued to
* publish at all (the new primary has no record of this scheduler
* ever having joined it, so it never routes any job to this
* scheduler's picker), so the reactive-only call could never fire
* and a pending rejoin could be stuck forever: a chicken-and-egg
* deadlock -- closed by sched_check_pending_rejoin() itself now
* also checking a passive, zero-I/O "connection already known
* dead" signal in addition to this flag; see that function's own
* doc comment (scheduler.c) for the full mechanism. Placed here
* (main.c), not inside driver.c's mode-agnostic
* xce_sched_driver_run_normal_mode_round(), because
* driver.c is linked by several sim-mode test binaries that never
* link scheduler.c (where this function and sched_needs_rejoin
* live) -- calling a scheduler.c symbol from driver.c broke those
* links under -Wl,--no-undefined. main.c is normal-mode-only and
* not linked into any test binary, so this call site has no such
* cross-boundary cost. */
sched_check_pending_rejoin(&sched);
sleep(1); sleep(1);
} }

View file

@ -1455,34 +1455,70 @@ int sched_commit_decision(xce_scheduler_t* sched, xce_decision_t* decision)
} }
/* /*
* sched_publish_check_rejoin - after a decision publish call finally * sched_check_pending_rejoin - re-issue xcecli_join() as a follow-up if
* succeeds, re-issue xcecli_join() as a separate follow-up if the client * either (a) the client library's reconnect callback (sched_on_reconnect(),
* library's reconnect callback (sched_on_reconnect(), above) flagged a * above) flagged a pending rejoin (component-failover-adoption, task 3.2,
* pending rejoin (component-failover-adoption, task 3.2, Req 2 AC5). * Req 2 AC5), or (b) the connection is passively known to be broken right
* now (bug-1760, see below) -- either condition alone is sufficient.
*
* Called from TWO places: sched_publish_retry_loop() below calls it
* reactively right after a successful decision publish (the original task
* 3.2 behavior, lowest possible rejoin latency when there IS something to
* publish); main()'s round loop (main.c) ALSO calls it unconditionally,
* once per ~1s scheduling round, regardless of whether that round
* publishes anything.
*
* bug-1760 fix, condition (b): sched_needs_rejoin ALONE cannot close the
* chicken-and-egg deadlock where the scheduler has nothing queued to
* publish -- it is only ever set by sched_on_reconnect(), which itself
* only fires as a side effect of an ACTUAL xcecli_* call detecting a dead
* connection (XCECLI_FAILOVER_CALL/XCECLI_FF_CALL, client_internal.h), and
* an idle round makes no such call. Confirmed live (2026-09-04, 3-node HA
* cluster): with condition (a) alone, a post-failover job submitted to the
* new primary stayed PEND indefinitely with zero scheduler.log rejoin
* activity, even though this function was already being called every
* round.
*
* Condition (b) -- xcecli_is_connection_broken(sched->apictx) -- closes
* this without adding any periodic active-liveness-probe call (which would
* trip test_scheduler_connect.c's Req-3-AC3 structural guard against
* introducing the worker's own periodic-probe pattern here, and carries a
* real, separate hazard: a stalled reconnect retry inside such a probe
* could itself delay/stall the scheduler's own round loop). Instead it
* reads a flag the transport ALREADY maintains for free: handle_disconnect()
* (xcectx.c) sets ctx->state = XCE_CTX_ERR asynchronously, from the iomgr's
* own epoll/RDHUP handling, the instant the connection drops -- zero
* application calls, zero blocking. xcecli_is_connection_broken() (client.h)
* is a pure, socket-free, side-effect-free read of that flag. A healthy,
* idle scheduler therefore makes zero extra calls of any kind; this branch
* only ever reaches xcecli_join() below when the transport is ALREADY
* known-dead, at which point blocking through xcecli_join()'s own
* XCECLI_FAILOVER_CALL retry/rotate logic is the same cost the round loop
* already accepts whenever it has a live decision to publish during a
* failover -- not a new category of stall.
* *
* Deliberately NOT its own retry loop: on failure, sched_needs_rejoin * Deliberately NOT its own retry loop: on failure, sched_needs_rejoin
* stays set, so the NEXT decision's publish call retries this same * stays set (if it was already set) and/or xcecli_is_connection_broken()
* rejoin -- the per-decision retry cadence sched_publish_retry_loop() * keeps reporting true until a real reconnect succeeds, so the NEXT call
* below already establishes IS the retry mechanism (design.md's * (whichever call site reaches it first) retries this same rejoin.
* "Rejoin-check interaction" note).
* *
* Uses the same rc<0-is-the-only-failure-test convention sched_init()'s * Uses the same rc<0-is-the-only-failure-test convention sched_init()'s
* own xcecli_join() check uses above (component-failover-adoption task * own xcecli_join() check uses above (component-failover-adoption task
* 2.3's worker lesson: a successful join returns the assigned slot index * 2.3's worker lesson: a successful join returns the assigned slot index
* as reply.status, which is not always exactly 0). * as reply.status, which is not always exactly 0).
*/ */
static void sched_publish_check_rejoin(xce_scheduler_t* sched) void sched_check_pending_rejoin(xce_scheduler_t* sched)
{ {
int rc; int rc;
if (!sched_needs_rejoin) { if (!sched_needs_rejoin && !xcecli_is_connection_broken(sched->apictx)) {
return; return;
} }
rc = xcecli_join(sched->apictx, HostRole_scheduler, NULL); rc = xcecli_join(sched->apictx, HostRole_scheduler, NULL);
if (rc < 0) { if (rc < 0) {
WARN("%s: pending rejoin retry failed (rc=%d) -- will retry on the " WARN("%s: pending rejoin retry failed (rc=%d) -- will retry on the "
"next decision publish", "next check",
__func__, rc); __func__, rc);
return; return;
} }
@ -1562,7 +1598,7 @@ static int sched_publish_retry_loop(xce_scheduler_t* sched, void* payload,
} }
if (rc >= 0) { if (rc >= 0) {
sched_publish_check_rejoin(sched); sched_check_pending_rejoin(sched);
} }
return rc; return rc;

View file

@ -1313,11 +1313,15 @@ sched_scheduler_decision_publish_retry_test = executable(
scheduler_decision_publish_retry_test_srcs, scheduler_decision_publish_retry_test_srcs,
include_directories:[incdir], include_directories:[incdir],
dependencies: [scheduler_deps, unity_dep], dependencies: [scheduler_deps, unity_dep],
c_args: ['-DXCE_MAIN_C_PATH="'
+ (meson.current_source_dir() / '..' / 'src' / 'main.c')
+ '"'],
link_args: ['-Wl,--wrap=xcecli_jobdispatch', link_args: ['-Wl,--wrap=xcecli_jobdispatch',
'-Wl,--wrap=xcecli_set_pend_reason', '-Wl,--wrap=xcecli_set_pend_reason',
'-Wl,--wrap=xcecli_join', '-Wl,--wrap=xcecli_join',
'-Wl,--wrap=xcecli_start', '-Wl,--wrap=xcecli_start',
'-Wl,--wrap=xcecli_set_reconnect_handler'], '-Wl,--wrap=xcecli_set_reconnect_handler',
'-Wl,--wrap=xcecli_is_connection_broken'],
install: false) install: false)
# Worst-case decision/allocation backpressure test (component-failover- # Worst-case decision/allocation backpressure test (component-failover-

View file

@ -182,6 +182,28 @@ int __wrap_xcecli_join(xcecli_ctx_t c, int role, hostattrs_t* attrs)
return rc; return rc;
} }
/* ════════════════════════════════════════════════════════════════════════
* Test double: __wrap_xcecli_is_connection_broken (bug-1760)
*
* Defaults to false so every EXISTING test in this file (which never
* touches this double) sees the connection as healthy, exactly matching
* this file's pre-bug-1760 behavior -- sched_check_pending_rejoin()'s new
* OR'd condition is a pure no-op unless a test explicitly sets this true.
* ════════════════════════════════════════════════════════════════════════ */
static bool g_connection_broken;
static void reset_connection_broken_double(void)
{
g_connection_broken = false;
}
bool __wrap_xcecli_is_connection_broken(xcecli_ctx_t c)
{
(void)c;
return g_connection_broken;
}
/* ════════════════════════════════════════════════════════════════════════ /* ════════════════════════════════════════════════════════════════════════
* Test doubles: __wrap_xcecli_start / __wrap_xcecli_set_reconnect_handler * Test doubles: __wrap_xcecli_start / __wrap_xcecli_set_reconnect_handler
* -- setup-only doubles for the real sched_init() call the rejoin- * -- setup-only doubles for the real sched_init() call the rejoin-
@ -215,6 +237,7 @@ void setUp(void)
reset_dispatch_double(); reset_dispatch_double();
reset_pend_double(); reset_pend_double();
reset_join_double(); reset_join_double();
reset_connection_broken_double();
} }
void tearDown(void) { return; } void tearDown(void) { return; }
@ -562,6 +585,243 @@ static void test_rejoin_composition_retries_across_publish_calls(void)
printf("test_rejoin_composition_retries_across_publish_calls: PASS\n"); printf("test_rejoin_composition_retries_across_publish_calls: PASS\n");
} }
/* ════════════════════════════════════════════════════════════════════════
* bug-1760 — sched_check_pending_rejoin() must work as a standalone public
* entry point, invocable with NO decision ever having been published.
* This is the direct proof that the chicken-and-egg deadlock (a pending
* rejoin that could only ever be checked as a side effect of a successful
* publish) is closed: the function is now callable, and does the right
* thing, entirely independent of publish activity. The actual production
* call site added by this fix (xce_sched_driver_run_normal_mode_round(),
* driver.c, called unconditionally once per ~1s round) is verified by
* source-text scan below, since driving that function through a full round
* would require live picker/allocator threads this fixture deliberately
* never starts (see this file's header comment).
* ════════════════════════════════════════════════════════════════════════ */
static void test_check_pending_rejoin_invoked_directly_without_a_publish(void)
{
xce_scheduler_t sched;
xce_client_config_t config;
xce_cluster_config_t cluster_cfg;
int rc;
/* Same fixture setup as test_rejoin_composition_retries_across_publish_
* calls above: reach a real sched_init() far enough to register the
* reconnect handler, force its own join to fail so no picker/allocator
* thread starts, then simulate a transparent reconnect. */
memset(&config, 0, sizeof(config));
config.nmasters = 1;
config.masters[0] = "127.0.0.1";
config.master_ports[0] = 19999;
config.from = "sched_pubretry_direct_test";
config.key = "keepsecret$";
memset(&cluster_cfg, 0, sizeof(cluster_cfg));
cluster_cfg.candidates_count = 1;
strncpy(cluster_cfg.candidates[0].hostname, "127.0.0.1",
XCE_MAXHOSTNAME_LEN - 1);
cluster_cfg.candidates[0].port = 19999;
strncpy(cluster_cfg.identity, "sched_pubretry_direct_test",
sizeof(cluster_cfg.identity) - 1);
g_join_returns[0] = -EINVAL; /* sched_init()'s own join: fail on purpose */
g_join_nreturns = 1;
memset(&sched, 0, sizeof(sched));
rc = sched_init(&sched, &config, &cluster_cfg, scheduler_req_handler);
TEST_ASSERT_TRUE_MESSAGE(
rc < 0,
"fixture: sched_init()'s own join was deliberately made to fail so "
"no picker/allocator thread is spun up by this setup step");
TEST_ASSERT_NOT_NULL_MESSAGE(
g_captured_reconnect_cb,
"fixture: sched_init() must register the reconnect handler before "
"its xcecli_join() call");
/* No decision has EVER been published at this point -- unlike every
* other test in this file, sched_publish_dispatch()/
* sched_publish_pend_reason() are never called here at all. */
g_captured_reconnect_cb(sched.apictx, g_captured_reconnect_arg);
reset_join_double();
g_join_returns[0] = -E_XCE_NOT_READY; /* first direct check: still fails */
g_join_nreturns = 1;
sched_check_pending_rejoin(&sched);
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, g_join_call_count,
"sched_check_pending_rejoin() must be callable on its own and "
"attempt xcecli_join() when a rejoin is pending, with zero "
"decisions ever published");
g_join_returns[1] = 0; /* second direct check: succeeds */
g_join_nreturns = 2;
sched_check_pending_rejoin(&sched);
TEST_ASSERT_EQUAL_INT_MESSAGE(
2, g_join_call_count,
"a still-pending rejoin must be retried on the next direct call");
/* Flag must now be clear -- a further direct call must not re-attempt
* the join. */
sched_check_pending_rejoin(&sched);
TEST_ASSERT_EQUAL_INT_MESSAGE(
2, g_join_call_count,
"once the rejoin has succeeded, a further direct call must be a "
"no-op -- no third xcecli_join() call");
printf("test_check_pending_rejoin_invoked_directly_without_a_publish: "
"PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* bug-1760 — passive-detection condition: sched_check_pending_rejoin() must
* attempt xcecli_join() when the transport is passively known-broken, even
* when sched_needs_rejoin was NEVER set (the exact idle-scheduler scenario
* a reactive-only check cannot detect on its own) -- and must make ZERO
* calls when neither condition holds, so a healthy idle scheduler never
* probes its connection. Uses a minimal zeroed fixture (no sched_init()
* needed): sched_check_pending_rejoin() only reads sched->apictx to pass
* verbatim into the wrapped xcecli_join()/xcecli_is_connection_broken()
* doubles, neither of which dereferences it.
* ════════════════════════════════════════════════════════════════════════ */
static void test_check_pending_rejoin_triggers_on_passive_broken_signal(void)
{
xce_scheduler_t sched;
memset(&sched, 0, sizeof(sched));
g_connection_broken = true;
g_join_returns[0] = 0;
g_join_nreturns = 1;
sched_check_pending_rejoin(&sched);
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, g_join_call_count,
"a passively-detected broken connection must trigger xcecli_join() "
"even when sched_needs_rejoin was never set -- this closes the "
"idle-scheduler chicken-and-egg gap a reactive-only check cannot");
printf("test_check_pending_rejoin_triggers_on_passive_broken_signal: "
"PASS\n");
}
static void test_check_pending_rejoin_no_op_when_healthy_and_no_flag(void)
{
xce_scheduler_t sched;
memset(&sched, 0, sizeof(sched));
/* g_connection_broken defaults to false (setUp); sched_needs_rejoin
* (scheduler.c-private) is not touched by this test and is already
* false by this point in the file's test order. */
sched_check_pending_rejoin(&sched);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_join_call_count,
"a healthy, idle scheduler (no flag set, connection reports "
"healthy) must make ZERO xcecli_join() calls -- this is the "
"no-periodic-probe-to-a-healthy-connection guarantee");
printf("test_check_pending_rejoin_no_op_when_healthy_and_no_flag: "
"PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* bug-1760 — production wiring: main()'s round loop (scheduler/src/main.c)
* must call sched_check_pending_rejoin() UNCONDITIONALLY on every
* iteration, not only as a side effect of a successful decision publish.
* Verified by source-text scan (same technique test_scheduler_decision_
* redelivery.c's test_no_worker_resync_symbols_in_scheduler_publish_path
* uses) rather than driving main() itself, which isn't linked into any
* test binary (main.c is excluded from every scheduler_srcs test target
* in meson.build -- see this file's own foreach-exclusion, shared by
* every test target in this directory).
* ════════════════════════════════════════════════════════════════════════ */
static char* read_whole_file(const char* path)
{
FILE* f = fopen(path, "r");
long size;
char* buf;
size_t nread;
if (!f) {
return NULL;
}
fseek(f, 0, SEEK_END);
size = ftell(f);
fseek(f, 0, SEEK_SET);
if (size < 0) {
fclose(f);
return NULL;
}
buf = (char*)malloc((size_t)size + 1);
if (!buf) {
fclose(f);
return NULL;
}
nread = fread(buf, 1, (size_t)size, f);
buf[nread] = '\0';
fclose(f);
return buf;
}
static void test_main_loop_checks_pending_rejoin_unconditionally(void)
{
char* main_src = read_whole_file(XCE_MAIN_C_PATH);
const char* func_sig = "int main(int argc, char* argv[])";
const char* call = "sched_check_pending_rejoin(&sched);";
const char* round_call
= "xce_sched_driver_run_normal_mode_round(&sched);";
const char* stop_check = "if (sched.core.stop)";
char* func_start;
char* call_pos;
char* round_pos;
char* stop_pos;
TEST_ASSERT_NOT_NULL_MESSAGE(main_src,
"could not read main.c source for scan");
func_start = strstr(main_src, func_sig);
TEST_ASSERT_NOT_NULL_MESSAGE(func_start,
"main.c must still define int main(...)");
round_pos = strstr(func_start, round_call);
stop_pos = strstr(func_start, stop_check);
call_pos = strstr(func_start, call);
TEST_ASSERT_NOT_NULL_MESSAGE(
round_pos, "sanity: the round loop's xce_sched_driver_run_normal_"
"mode_round(&sched) call must still exist");
TEST_ASSERT_NOT_NULL_MESSAGE(
stop_pos, "sanity: the round loop's core.stop check must still "
"exist");
TEST_ASSERT_NOT_NULL_MESSAGE(
call_pos,
"bug-1760: main()'s round loop must call "
"sched_check_pending_rejoin(&sched) on every iteration -- without "
"this, a pending rejoin after a controller failover can only ever "
"be retried as a side effect of a successful decision publish, "
"which may never happen (chicken-and-egg deadlock)");
TEST_ASSERT_TRUE_MESSAGE(
(round_pos < call_pos) && (stop_pos < call_pos),
"sched_check_pending_rejoin() must run AFTER the round's own "
"stop-check (never called on the iteration that is about to "
"break/shut down) but is placed unconditionally on every "
"surviving iteration regardless of what that round scheduled");
free(main_src);
printf("test_main_loop_checks_pending_rejoin_unconditionally: "
"PASS\n");
}
int main(void) int main(void)
{ {
UNITY_BEGIN(); UNITY_BEGIN();
@ -571,5 +831,9 @@ int main(void)
RUN_TEST(test_extended_retry_success_logs_distinctly); RUN_TEST(test_extended_retry_success_logs_distinctly);
RUN_TEST(test_first_attempt_success_does_not_log_extended_retry); RUN_TEST(test_first_attempt_success_does_not_log_extended_retry);
RUN_TEST(test_rejoin_composition_retries_across_publish_calls); RUN_TEST(test_rejoin_composition_retries_across_publish_calls);
RUN_TEST(test_check_pending_rejoin_invoked_directly_without_a_publish);
RUN_TEST(test_check_pending_rejoin_triggers_on_passive_broken_signal);
RUN_TEST(test_check_pending_rejoin_no_op_when_healthy_and_no_flag);
RUN_TEST(test_main_loop_checks_pending_rejoin_unconditionally);
return UNITY_END(); return UNITY_END();
} }