Rongsong Shen
3eba7c16bf
Violates this project's own recorded Decision Log entry: "No client reply-wait timeout as a fix for a hung request -- fail loud at the real transport/source instead of masking with a deadline backstop." OPCode_schedAdmin (cluster.c) and on_fetchfile (job.c, the worker proxy -- cluster.c's own comment says schedAdmin "mirrors on_fetchfile's proxy pattern", confirming one shared design mistake, not two independent bugs) both blocked on a multi-second pthread_cond_timedwait, replying "timeout" on expiry. handle_scheduler_leave()/handle_worker_leave() (the actual disconnect handlers) only nulled the endpoint pointer -- neither ever signaled the pending waiter, so a mid-wait disconnect was invisible until the deadline lapsed. The timeout was the ONLY failure-detection mechanism, exactly what the decision forbids. Both sites already had a working signal function on their success path (schedadmin_signal(), fetchfile_signal()) -- just never wired to the disconnect handlers. Fixed by calling them from handle_scheduler_leave()/handle_worker_leave() with a distinguishable -ECONNRESET outcome, so the waiter wakes immediately on disconnect instead of only on the deadline. Timeouts kept as defensive backstops for a genuinely wedged peer that never disconnects cleanly either -- this fix is about not relying SOLELY on the timeout, not removing it. schedadmin_signal_disconnect() must be called BEFORE handle_scheduler_leave() acquires c->sched.lock -- the dispatch wait holds that same lock for its entire duration, so signaling after acquiring it would deadlock. Found by an /xce-audit-invariants controller pass (Decision Log fidelity dimension). New test_fetchfile_disconnect.c plus extensions to test_sched_admin_dispatch.c/test_sched_worker_change.c; RED/GREEN independently verified against real elapsed-time assertions (pre-fix: 5001ms full-timeout wait; post-fix: <1s immediate wake). Full controller suite: 1181 tests, 2 failures -- both the pre-existing, unrelated bug-1561 overlayfs sandbox flake, confirmed via a clean meson-driven run. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AgYRU3zDTqNGV5Jk8aiKLg
288 lines
9.1 KiB
C
288 lines
9.1 KiB
C
#include <argp.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include <unity/unity.h>
|
|
|
|
extern void test_persist_types(void);
|
|
extern void test_persist_wal(void);
|
|
extern void test_persist_schema(void);
|
|
extern void test_persist_checkpoint(void);
|
|
extern void test_persist_recovery(void);
|
|
extern void test_hooks(void);
|
|
extern void test_persist_hooks(void);
|
|
extern void test_gc_retention(void);
|
|
extern void test_cluster_persist(void);
|
|
extern void test_e2e_wal(void);
|
|
extern void test_e2e_checkpoint(void);
|
|
extern void test_e2e_hooks(void);
|
|
extern void test_e2e_gc(void);
|
|
extern void test_e2e_capnp(void);
|
|
extern void test_ha_role(void);
|
|
extern void test_ha_role_lifecycle(void);
|
|
extern void test_ha_cluster(void);
|
|
extern void test_ha_redirect(void);
|
|
extern void test_join_redirect(void);
|
|
extern void test_p1_integration(void);
|
|
extern void test_repl_unit(void);
|
|
extern void test_repl_apply(void);
|
|
extern void test_persist_repl(void);
|
|
extern void test_repl_catchup(void);
|
|
extern void test_p2_integration(void);
|
|
extern void test_p3_hardening(void);
|
|
extern void test_p3_manual_failover(void);
|
|
extern void test_ctrl_metrics(void);
|
|
extern void test_config_repl(void);
|
|
extern void test_worker_join(void);
|
|
extern void test_ha_startup(void);
|
|
extern void test_ha_election(void);
|
|
extern void test_ha_elect_ctx_lifecycle(void);
|
|
extern void test_ha_bootstrap(void);
|
|
extern void test_ha_durability(void);
|
|
extern void test_repl_majority(void);
|
|
extern void test_persist_truncate(void);
|
|
extern void test_ha_ctrlhello(void);
|
|
extern void test_ha_failover(void);
|
|
extern void test_ha_vote_send(void);
|
|
extern void test_ha_start_election_async(void);
|
|
extern void test_ha_vote_gate(void);
|
|
extern void test_ha_repl_push(void);
|
|
extern void test_ha_connection_reuse(void);
|
|
extern void test_ha_rdhup_integration(void);
|
|
extern void test_ha_liveness(void);
|
|
extern void test_ha_prevote(void);
|
|
extern void test_ha_failover_prevote(void);
|
|
extern void test_commit_engine(void);
|
|
extern void test_commit_barrier_rollout(void);
|
|
extern void test_commit_barrier_reply(void);
|
|
extern void test_egress_filter(void);
|
|
extern void test_demotion_convergence(void);
|
|
extern void test_convergence_verification(void);
|
|
extern void test_coldstart_lsn0(void);
|
|
extern void test_commit_durability(void);
|
|
extern void test_ctrlstatus(void);
|
|
extern void test_ha_heartbeat_committed(void);
|
|
extern void test_ha_vote_floor_heartbeat(void);
|
|
extern void test_term_safety_adapter(void);
|
|
extern void test_waldump(void);
|
|
extern void test_walcompare(void);
|
|
extern void test_snapverify(void);
|
|
extern void test_snapdiff(void);
|
|
extern void test_configcompare(void);
|
|
extern void test_drive_async_config(void);
|
|
extern void test_slowlane(void);
|
|
extern void test_commit_admission_pool(void);
|
|
extern void test_commit_resolve(void);
|
|
extern void test_ha_vep_teardown(void);
|
|
extern void test_ha_dispatch_integration(void);
|
|
extern void test_term_adoption_vectors(void);
|
|
extern void test_ha_core_safety(void);
|
|
extern void test_ha_partition(void);
|
|
extern void test_persist_snapshot_entities(void);
|
|
extern void test_apply_seam(void);
|
|
extern void test_apply_committed_entry(void);
|
|
extern void test_replay_apply_seam(void);
|
|
extern void test_wal_job_dispatch_fact(void);
|
|
extern void test_wal_job_start_fact(void);
|
|
extern void test_wal_job_finish_fact(void);
|
|
extern void test_wal_job_rusage_fact(void);
|
|
extern void test_wal_job_ckpt_fact(void);
|
|
extern void test_wal_job_preason_fact(void);
|
|
extern void test_job_dispatch_fail_count(void);
|
|
extern void test_apply_reapply_idempotency(void);
|
|
extern void test_apply_correctness_verification(void);
|
|
extern void test_apply_pump(void);
|
|
extern void test_apply_pump_integration(void);
|
|
extern void test_job_array_apply(void);
|
|
extern void test_job_array_fence(void);
|
|
extern void test_job_array_replay_fence(void);
|
|
extern void test_upgrade_snapshot_restore(void);
|
|
extern void test_upgrade_snapshot_apply(void);
|
|
extern void test_bug857_regression(void);
|
|
extern void test_sched_admin_dispatch(void);
|
|
extern void test_fetchfile_disconnect(void);
|
|
extern void test_job_dispatch(void);
|
|
extern void test_dispatch_commit(void);
|
|
extern void test_dispatch_fail_terminal(void);
|
|
extern void test_dispatch_fail_retry(void);
|
|
extern void test_jobdispatch_no_double_notify(void);
|
|
|
|
void setUp(void) { return; }
|
|
|
|
void tearDown(void)
|
|
{
|
|
printf("\n-----------------------------------------------------\n");
|
|
return;
|
|
}
|
|
|
|
static char doc[] = "controller_test";
|
|
static char args_doc[] = "";
|
|
|
|
#define MAX_TESTS (256)
|
|
|
|
struct arguments {
|
|
int ntests;
|
|
char* tests[MAX_TESTS];
|
|
};
|
|
|
|
static struct argp_option options[] = {
|
|
{ "run", 't', "STR", 0, "run test" },
|
|
{ 0 }
|
|
};
|
|
|
|
static error_t parse_opt(int key, char* arg, struct argp_state* state)
|
|
{
|
|
struct arguments* arguments = state->input;
|
|
|
|
switch (key) {
|
|
case 't':
|
|
if (arguments->ntests >= MAX_TESTS) {
|
|
fprintf(stderr, "Too many tests\n");
|
|
exit(-1);
|
|
}
|
|
arguments->tests[arguments->ntests] = arg;
|
|
arguments->ntests++;
|
|
break;
|
|
default:
|
|
return ARGP_ERR_UNKNOWN;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static struct argp argp = { options, parse_opt, args_doc, doc };
|
|
struct arguments arguments;
|
|
|
|
static int check_test(struct arguments* args, char* x)
|
|
{
|
|
int i;
|
|
|
|
if (args->ntests == 0) {
|
|
return 1;
|
|
}
|
|
for (i = 0; i < args->ntests; i++) {
|
|
if (strcmp(x, args->tests[i]) == 0) {
|
|
return 1;
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
#define TEST(x) \
|
|
do { \
|
|
if (check_test(&arguments, #x)) { \
|
|
x(); \
|
|
} \
|
|
} while (0)
|
|
|
|
int main(int argc, char* argv[])
|
|
{
|
|
argp_parse(&argp, argc, argv, 0, 0, &arguments);
|
|
|
|
UNITY_BEGIN();
|
|
TEST(test_persist_types);
|
|
TEST(test_persist_wal);
|
|
TEST(test_persist_schema);
|
|
TEST(test_persist_checkpoint);
|
|
TEST(test_persist_recovery);
|
|
TEST(test_hooks);
|
|
TEST(test_persist_hooks);
|
|
TEST(test_gc_retention);
|
|
TEST(test_cluster_persist);
|
|
TEST(test_e2e_wal);
|
|
TEST(test_e2e_checkpoint);
|
|
TEST(test_e2e_hooks);
|
|
TEST(test_e2e_gc);
|
|
TEST(test_e2e_capnp);
|
|
TEST(test_ha_role);
|
|
TEST(test_ha_role_lifecycle);
|
|
TEST(test_ha_cluster);
|
|
TEST(test_ha_redirect);
|
|
TEST(test_join_redirect);
|
|
TEST(test_p1_integration);
|
|
TEST(test_repl_unit);
|
|
TEST(test_repl_apply);
|
|
TEST(test_persist_repl);
|
|
TEST(test_repl_catchup);
|
|
TEST(test_p2_integration);
|
|
TEST(test_p3_hardening);
|
|
TEST(test_p3_manual_failover);
|
|
TEST(test_ctrl_metrics);
|
|
TEST(test_config_repl);
|
|
TEST(test_worker_join);
|
|
TEST(test_ha_startup);
|
|
TEST(test_ha_election);
|
|
TEST(test_ha_elect_ctx_lifecycle);
|
|
TEST(test_ha_bootstrap);
|
|
TEST(test_ha_durability);
|
|
TEST(test_repl_majority);
|
|
TEST(test_persist_truncate);
|
|
TEST(test_ha_ctrlhello);
|
|
TEST(test_ha_failover);
|
|
TEST(test_ha_vote_send);
|
|
TEST(test_ha_start_election_async);
|
|
TEST(test_ha_vote_gate);
|
|
TEST(test_ha_repl_push);
|
|
TEST(test_ha_connection_reuse);
|
|
TEST(test_ha_rdhup_integration);
|
|
TEST(test_ha_liveness);
|
|
TEST(test_ha_prevote);
|
|
TEST(test_ha_failover_prevote);
|
|
TEST(test_commit_engine);
|
|
TEST(test_commit_barrier_rollout);
|
|
TEST(test_commit_barrier_reply);
|
|
TEST(test_egress_filter);
|
|
TEST(test_demotion_convergence);
|
|
TEST(test_convergence_verification);
|
|
TEST(test_coldstart_lsn0);
|
|
TEST(test_commit_durability);
|
|
TEST(test_ctrlstatus);
|
|
TEST(test_ha_heartbeat_committed);
|
|
TEST(test_ha_vote_floor_heartbeat);
|
|
TEST(test_term_safety_adapter);
|
|
TEST(test_waldump);
|
|
TEST(test_walcompare);
|
|
TEST(test_snapverify);
|
|
TEST(test_snapdiff);
|
|
TEST(test_configcompare);
|
|
TEST(test_drive_async_config);
|
|
TEST(test_slowlane);
|
|
TEST(test_commit_admission_pool);
|
|
TEST(test_commit_resolve);
|
|
TEST(test_ha_vep_teardown);
|
|
TEST(test_ha_dispatch_integration);
|
|
TEST(test_term_adoption_vectors);
|
|
TEST(test_ha_core_safety);
|
|
TEST(test_ha_partition);
|
|
TEST(test_persist_snapshot_entities);
|
|
TEST(test_apply_seam);
|
|
TEST(test_apply_committed_entry);
|
|
TEST(test_replay_apply_seam);
|
|
TEST(test_wal_job_dispatch_fact);
|
|
TEST(test_wal_job_start_fact);
|
|
TEST(test_wal_job_finish_fact);
|
|
TEST(test_wal_job_rusage_fact);
|
|
TEST(test_wal_job_ckpt_fact);
|
|
TEST(test_wal_job_preason_fact);
|
|
TEST(test_job_dispatch_fail_count);
|
|
TEST(test_apply_reapply_idempotency);
|
|
TEST(test_apply_correctness_verification);
|
|
TEST(test_apply_pump);
|
|
TEST(test_apply_pump_integration);
|
|
TEST(test_job_array_apply);
|
|
TEST(test_job_array_fence);
|
|
TEST(test_job_array_replay_fence);
|
|
TEST(test_upgrade_snapshot_restore);
|
|
TEST(test_upgrade_snapshot_apply);
|
|
TEST(test_bug857_regression);
|
|
TEST(test_sched_admin_dispatch);
|
|
TEST(test_fetchfile_disconnect);
|
|
TEST(test_job_dispatch);
|
|
TEST(test_dispatch_commit);
|
|
TEST(test_dispatch_fail_terminal);
|
|
TEST(test_dispatch_fail_retry);
|
|
TEST(test_jobdispatch_no_double_notify);
|
|
|
|
return UNITY_END();
|
|
}
|