lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

merge: sync Rongsong's scheduler-framework into chatbot

Browse source 
Full merge of origin/scheduler-framework (scheduler-core restructure,
failover-adoption, WAL_JOB_DISPATCH, apiserver backend health/heartbeat,
build-time versioning, bug-1673/1676/1678/1679/1680).

Conflict resolution:
- scheduler: adopt Rongsong's scheduler-core architecture (driver loop,
  queue project/picker fields, resource bug-1569 fix)
- worker/controller_connector.c: keep Rongsong's reconnect/rejoin
  structure, PORT our bug-1509 resend_job_status onto the rejoin success
  path (his reconnect didn't re-report job state)
- apiserver backend: take Rongsong's health/heartbeat (supersedes our
  e505066 circuit breaker)
- group.c: keep our bug-1513 comment
- misc: take theirs for additive fixes (go_build versioning,
  test_group.c, persist.c RUSAGE_UPDATE, FilterRows bug-1680)

NOTE: our scheduler fixes (bug-1494 family dispatch-fail requeue,
PEND-reason classification) are NOT in Rongsong's branch — needs
verification/porting onto scheduler-core (separate follow-up).

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
lei.chen 2026-08-25 12:33:16 +00:00
commit 4e894d2184
395 changed files with 88517 additions and 5742 deletions
Download patch file Download diff file Expand all files Collapse all files

18
src/controller/testsuite/controller_test.c
View file

@ -76,12 +76,15 @@ extern void test_commit_admission_pool(void);
extern void test_commit_resolve(void);
extern void test_ha_vep_teardown(void);
extern void test_ha_dispatch_integration(void);
extern void test_term_adoption_vectors(void);
extern void test_ha_core_safety(void);
extern void test_ha_partition(void);
extern void test_persist_snapshot_entities(void);
extern void test_apply_seam(void);
extern void test_apply_committed_entry(void);
extern void test_replay_apply_seam(void);
extern void test_wal_job_dispatch_fact(void);
extern void test_job_dispatch_fail_count(void);
extern void test_apply_reapply_idempotency(void);
extern void test_apply_correctness_verification(void);
extern void test_apply_pump(void);
@ -92,6 +95,12 @@ extern void test_job_array_replay_fence(void);
extern void test_upgrade_snapshot_restore(void);
extern void test_upgrade_snapshot_apply(void);
extern void test_bug857_regression(void);
extern void test_sched_admin_dispatch(void);
extern void test_job_dispatch(void);
extern void test_dispatch_commit(void);
extern void test_dispatch_fail_terminal(void);
extern void test_dispatch_fail_retry(void);
extern void test_jobdispatch_no_double_notify(void);
void setUp(void) { return; }
@ -237,12 +246,15 @@ int main(int argc, char* argv[])
TEST(test_commit_resolve);
TEST(test_ha_vep_teardown);
TEST(test_ha_dispatch_integration);
TEST(test_term_adoption_vectors);
TEST(test_ha_core_safety);
TEST(test_ha_partition);
TEST(test_persist_snapshot_entities);
TEST(test_apply_seam);
TEST(test_apply_committed_entry);
TEST(test_replay_apply_seam);
TEST(test_wal_job_dispatch_fact);
TEST(test_job_dispatch_fail_count);
TEST(test_apply_reapply_idempotency);
TEST(test_apply_correctness_verification);
TEST(test_apply_pump);
@ -253,6 +265,12 @@ int main(int argc, char* argv[])
TEST(test_upgrade_snapshot_restore);
TEST(test_upgrade_snapshot_apply);
TEST(test_bug857_regression);
TEST(test_sched_admin_dispatch);
TEST(test_job_dispatch);
TEST(test_dispatch_commit);
TEST(test_dispatch_fail_terminal);
TEST(test_dispatch_fail_retry);
TEST(test_jobdispatch_no_double_notify);
return UNITY_END();
}

57
src/controller/testsuite/resval_typeresolve.c Normal file
View file

@ -0,0 +1,57 @@
/* resval_typeresolve.c -- resource-value-verification task 1.1
*
* See resval_typeresolve.h for the full rationale and TASK 1.1 STATUS note.
*/
#include <string.h>
#include "resval_typeresolve.h"
int resval_resolve_type(const char* name, int type_override)
{
/* Verbatim extraction of set_resval()'s resinfo[] table (test.c),
* unchanged: type/model are Value_u_svalue; everything else here is
* Value_u_fvalue. */
static const struct {
const char* name;
int valtype;
} resinfo[] = {
{ "r15s", Value_u_fvalue },
{ "r1m", Value_u_fvalue },
{ "slots", Value_u_fvalue },
{ "ut", Value_u_fvalue },
{ "mem", Value_u_fvalue },
{ "maxcpus", Value_u_fvalue },
{ "ndisks", Value_u_fvalue },
{ "cpuf", Value_u_fvalue },
{ "maxswap", Value_u_fvalue },
{ "maxtmp", Value_u_fvalue },
{ "type", Value_u_svalue },
{ "model", Value_u_svalue },
{ "verilog", Value_u_fvalue },
{ "dyna", Value_u_fvalue },
{ NULL, 0 }
};
int i;
if (name == NULL) {
return RESVAL_TYPE_UNKNOWN;
}
for (i = 0; resinfo[i].name != NULL; i++) {
if (strcmp(resinfo[i].name, name) == 0) {
/* Known name: exact existing behavior, type_override ignored
* (Requirement 1.4). */
return resinfo[i].valtype;
}
}
/* task 2.1: unrecognized name -- Requirement 1.1's fallback. Default to
* numeric (Value_u_fvalue) unless the caller passed an explicit
* type_override (any valid Value_u_which tag other than the
* RESVAL_TYPE_UNKNOWN sentinel), in which case honor the override. */
if (type_override != RESVAL_TYPE_UNKNOWN) {
return type_override;
}
return Value_u_fvalue;
}

53
src/controller/testsuite/resval_typeresolve.h Normal file
View file

@ -0,0 +1,53 @@
#ifndef RESVAL_TYPERESOLVE_H
#define RESVAL_TYPERESOLVE_H
/* resval_typeresolve.h -- resource-value-verification task 1.1
*
* Extracted from test.c's set_resval(): pure resource-name -> wire-type
* resolution logic, with NO networking, REPL, or global-state dependency
* (unlike set_resval() itself, which also allocates xce_val_t and tags
* .host via get_setting("myname") -- that stays in test.c since it needs
* REPL state). This isolation is what makes the logic unit-testable outside
* of test.c, which cannot be white-box #include'd into a Unity test (4298
* lines, its own main(), heavy libedit/network dependencies).
*
* The resolved type is a Value_u_which tag (controller.capnp.h) -- e.g.
* Value_u_fvalue for a numeric resource, Value_u_svalue for a string one.
*/
#include "controller.capnp.h" /* enum Value_u_which */
/* Sentinel: not a valid Value_u_which value. Used both as:
* - the RETURN value when `name` is not present in the known-resource
* table and no override applies;
* - the `type_override` INPUT value meaning "no override requested".
*/
#define RESVAL_TYPE_UNKNOWN (-1)
/* Resolve `name` to its Value_u_which wire type.
*
* `type_override`: pass a valid Value_u_which value (Value_u_fvalue,
* Value_u_svalue, ...) to request that type for a name not in the known-
* resource table, or RESVAL_TYPE_UNKNOWN for no override.
*
* Returns:
* - the table's type, for any of the 14 currently-known resource names
* (r15s, r1m, slots, ut, mem, maxcpus, ndisks, cpuf, maxswap, maxtmp,
* type, model, verilog, dyna) -- `type_override` is ignored for a known
* name, exactly as set_resval() behaves today.
* - `type_override`, for any name NOT in the table, when `type_override`
* is not RESVAL_TYPE_UNKNOWN (Requirement 1.1's `-t` escape hatch).
* - Value_u_fvalue (numeric), for any name NOT in the table when no
* override was requested (Requirement 1.1's default fallback).
* - RESVAL_TYPE_UNKNOWN, only for `name` == NULL.
*
* *** TASK 2.1 STATUS (fallback implemented) ***
* Task 1.1 was a byte-for-byte extraction of set_resval()'s existing
* resinfo[] table lookup, with the unrecognized-name fallback left
* unimplemented (RED tests in test_resval_typeresolve.c). Task 2.1 adds the
* numeric-by-default fallback and `type_override` handling described above
* -- behavior for all 14 known names remains byte-for-byte unchanged.
*/
int resval_resolve_type(const char* name, int type_override);
#endif /* RESVAL_TYPERESOLVE_H */

1003
src/controller/testsuite/test.c
View file

File diff suppressed because it is too large Load diff

160
src/controller/testsuite/test_apply_committed_entry.c
View file

@ -31,6 +31,7 @@
#include <stdarg.h>
#include <stdbool.h>
#include <string.h>
#include <time.h>
#include <unity/unity.h>
@ -402,6 +403,61 @@ test_ace_job_state_transition_rejects_backward_suppresses_notify(void)
"notify: PASS\n");
}
static void test_ace_job_start_sets_start_time(void)
{
tace_setup();
job_t* j = tace_seed_job(603, JOB_STAT_PEND);
TEST_ASSERT_NOT_NULL(j);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, (uint64_t)j->job.start_time,
"fixture must seed a job with start_time still unset");
wal_jobstate_t js = { .jobid = 603, .state = JOB_STAT_RUN };
time_t before = time(NULL);
int rc = apply_committed_entry(&g_tace_p, WAL_JOB_START, 22, &js,
sizeof(js));
time_t after = time(NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
TEST_ASSERT_TRUE_MESSAGE(
(j->job.start_time >= before) && (j->job.start_time <= after),
"WAL_JOB_START must stamp start_time to the apply-time clock");
tace_teardown();
printf("test_ace_job_start_sets_start_time: PASS\n");
}
static void test_ace_job_finish_sets_finish_time_and_run_time(void)
{
tace_setup();
job_t* j = tace_seed_job(604, JOB_STAT_RUN);
TEST_ASSERT_NOT_NULL(j);
j->job.start_time = 1000;
wal_jobstate_t js = { .jobid = 604, .state = JOB_STAT_DONE };
time_t before = time(NULL);
int rc = apply_committed_entry(&g_tace_p, WAL_JOB_FINISH, 23, &js,
sizeof(js));
time_t after = time(NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_DONE, j->job.state);
TEST_ASSERT_TRUE_MESSAGE(
(j->job.finish_time >= before) && (j->job.finish_time <= after),
"WAL_JOB_FINISH must stamp finish_time to the apply-time clock");
TEST_ASSERT_EQUAL_INT_MESSAGE((int)(j->job.finish_time - 1000),
j->job.run_time,
"run_time must be derived from the stamped finish_time minus the "
"job's start_time");
tace_teardown();
printf("test_ace_job_finish_sets_finish_time_and_run_time: PASS\n");
}
/* ── WAL_QUEUE_CTRL ───────────────────────────────────────────────────── */
static void test_ace_queue_ctrl_updates_and_notifies(void)
@ -480,7 +536,10 @@ static void test_ace_worker_ctrl_join_restores_and_notifies(void)
int rc = apply_committed_entry(&g_tace_p, WAL_WORKER_CTRL, 40, pkt,
(uint32_t)total);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
"bug-1427: a genuine new-worker JOIN must return 1 (not the old "
"hardcoded 0), mirroring WAL_JOB_SUBMIT's apply_rc==1 creation "
"signal — replay_wal()'s new-worker tally relies on this");
worker_t* w = NULL;
lock(g_tace_c.workermgr, RTTI_LOCK_READ);
@ -692,6 +751,102 @@ static void test_ace_worker_ctrl_join_duplicate_existing_is_noop(void)
printf("test_ace_worker_ctrl_join_duplicate_existing_is_noop: PASS\n");
}
/* bug-1472-remainder: a HOOK_DETAIL_WORKER_DEAD entry against an already-
* existing worker previously returned -EEXIST unconditionally (a no-op,
* same as JOIN's genuine idempotency case) -- so a dead-worker transition
* detected by one member (mark_worker_unavail(), worker.c) never replicated
* to standbys/this member's own WAL replay. Proves the fix replays
* mark_worker_unavail()'s own local mutations: XCE_WORKER_UNAVAIL set,
* dead_since taken from the decoded payload (the PRIMARY's own death
* timestamp), and hostattrs reset to the "unknown"/0 sentinels -- mirroring
* exactly what the primary itself did before firing this same WAL entry. */
static void test_ace_worker_ctrl_dead_applies_to_existing_worker(void)
{
tace_setup();
/* Pre-existing worker with live (non-"unknown") attrs, as if genuinely
* joined and reporting real hardware facts. */
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = (char*)"LINUX";
attrs.hostmodel = (char*)"x86_64";
attrs.cpufactor = 1.0f;
attrs.maxcpus = 8;
attrs.maxmem = 16384;
xce_worker_t wdef;
memset(&wdef, 0, sizeof(wdef));
wdef.name = (char*)"w-dead";
wdef.attrs = &attrs;
wdef.status = 0;
TEST_ASSERT_NOT_NULL(makeWorker(g_tace_c.workermgr, &wdef));
/* Encode a DEAD entry the way mark_worker_unavail()'s own hook fire
* would -- attrs already reset to "unknown"/0 (capnp_encode_worker
* captures whatever the primary's worker_t held AT hook-fire time,
* which is always post-reset for a real DEAD transition), a specific
* non-zero deadSince (the primary's own death timestamp, not this
* member's clock). */
hostattrs_t dead_attrs;
memset(&dead_attrs, 0, sizeof(dead_attrs));
dead_attrs.hosttype = (char*)"unknown";
dead_attrs.hostmodel = (char*)"unknown";
xce_worker_t dead_wdef;
memset(&dead_wdef, 0, sizeof(dead_wdef));
dead_wdef.name = (char*)"w-dead";
dead_wdef.attrs = &dead_attrs;
dead_wdef.status = XCE_WORKER_UNAVAIL;
dead_wdef.dead_since = 1755300000; /* arbitrary fixed epoch time */
uint8_t* wbuf = NULL;
size_t wsize = 0;
TEST_ASSERT_EQUAL_INT(0, tace_encode_worker(&dead_wdef, &wbuf, &wsize));
uint32_t disc = (uint32_t)HOOK_DETAIL_WORKER_DEAD;
size_t total = sizeof(disc) + wsize;
uint8_t* pkt = malloc(total);
TEST_ASSERT_NOT_NULL(pkt);
memcpy(pkt, &disc, sizeof(disc));
memcpy(pkt + sizeof(disc), wbuf, wsize);
free(wbuf);
xce_add_hook(&g_tace_c.hooksys, XCE_CONTROLLER_HOOK_WORKER, tace_spy_hook,
NULL);
int rc = apply_committed_entry(&g_tace_p, WAL_WORKER_CTRL, 44, pkt,
(uint32_t)total);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"bug-1472-remainder: a DEAD entry against an already-existing "
"worker must succeed (0), not -EEXIST -- -EEXIST silently "
"discarded the dead-worker transition entirely");
worker_t* w = NULL;
lock(g_tace_c.workermgr, RTTI_LOCK_READ);
key_find(g_tace_c.workermgr, (void*)"w-dead", (void**)&w);
unlock(g_tace_c.workermgr);
TEST_ASSERT_NOT_NULL(w);
TEST_ASSERT_TRUE_MESSAGE(
(w->host.status & XCE_WORKER_UNAVAIL) != 0,
"the existing worker must be marked XCE_WORKER_UNAVAIL");
TEST_ASSERT_EQUAL_INT64_MESSAGE((int64_t)1755300000, (int64_t)w->host.dead_since,
"dead_since must come from the decoded payload (the primary's own "
"death timestamp), not be silently discarded or re-derived locally");
TEST_ASSERT_EQUAL_STRING_MESSAGE("unknown", w->host.attrs->hosttype,
"hostattrs must be reset to the \"unknown\" sentinel, mirroring "
"mark_worker_unavail()'s own local effect on the primary");
TEST_ASSERT_EQUAL_STRING_MESSAGE("unknown", w->host.attrs->hostmodel,
"hostattrs must be reset to the \"unknown\" sentinel");
TEST_ASSERT_EQUAL_FLOAT_MESSAGE(0, w->host.attrs->maxmem,
"numeric hostattrs (maxmem and friends) must be zeroed, not left "
"at their last-known-live values");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_tace_hook_fires,
"a genuinely-applied DEAD transition must fire exactly one "
"notification");
free(pkt);
tace_teardown();
printf("test_ace_worker_ctrl_dead_applies_to_existing_worker: PASS\n");
}
/* ── WAL_JOB_ARRAY_SUBMIT (job-array-single-wal-record task 3.2) ────────── */
static job_submit_data_t tace_array_make_spec(void)
@ -942,11 +1097,14 @@ void test_apply_committed_entry(void)
RUN_TEST(test_ace_job_submit_duplicate_is_noop_no_extra_notify);
RUN_TEST(test_ace_job_state_transition_applies_forward_and_notifies);
RUN_TEST(test_ace_job_state_transition_rejects_backward_suppresses_notify);
RUN_TEST(test_ace_job_start_sets_start_time);
RUN_TEST(test_ace_job_finish_sets_finish_time_and_run_time);
RUN_TEST(test_ace_queue_ctrl_updates_and_notifies);
RUN_TEST(test_ace_worker_ctrl_join_restores_and_notifies);
RUN_TEST(test_ace_worker_ctrl_update_applies_to_existing_worker);
RUN_TEST(test_ace_worker_ctrl_update_replaces_not_ors_admin_bits);
RUN_TEST(test_ace_worker_ctrl_join_duplicate_existing_is_noop);
RUN_TEST(test_ace_worker_ctrl_dead_applies_to_existing_worker);
RUN_TEST(test_ace_job_array_submit_constructs_all_and_notifies_once);
RUN_TEST(test_ace_job_array_submit_idempotent_replay_no_extra_notify);
#ifdef XCE_TESTING

101
src/controller/testsuite/test_apply_correctness_verification.c
View file

@ -49,6 +49,7 @@
#include "jobj.h"
#include "persist.h"
#include "wal_types.h"
#include "wobj.h"
#include "xce/cluster_def.h"
#include "xce/types.h"
#include "xstring.h"
@ -436,12 +437,16 @@ static void test_checkpoint_tail_with_no_snapshot_is_apply_clean(void)
* which needs none of Fixture A's full cluster machinery either. ──────── */
static var g_tacv_bare_jobmgr;
static var g_tacv_bare_workermgr; /* bug-1427 */
static void tacv_bare_setup(void)
{
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 16,
.delta = 16 };
kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING, /* bug-1427 */
.nslots = 16,
.delta = 16 };
if (makeString((char*)"tacv-bare-probe") == NULL)
xstring_table_init(64);
@ -453,6 +458,8 @@ static void tacv_bare_setup(void)
g_tacv_bare_jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(g_tacv_bare_jobmgr);
g_tacv_bare_workermgr = makeInstance(&xkvmgr_type, &str_params);
TEST_ASSERT_NOT_NULL(g_tacv_bare_workermgr);
}
static void tacv_bare_teardown(void)
@ -461,6 +468,28 @@ static void tacv_bare_teardown(void)
dereference(g_tacv_bare_jobmgr);
g_tacv_bare_jobmgr = NULL;
}
if (g_tacv_bare_workermgr != NULL) {
dereference(g_tacv_bare_workermgr);
g_tacv_bare_workermgr = NULL;
}
}
/* bug-1427: worker analog of tacv_bare_make_job below. */
static worker_t* tacv_bare_make_worker(const char* name)
{
xce_worker_t wdef;
hostattrs_t* attrs = calloc(1, sizeof(*attrs));
TEST_ASSERT_NOT_NULL(attrs);
attrs->hosttype = makeString((char*)"LINUX");
attrs->hostmodel = makeString((char*)"x86");
attrs->cpufactor = 1.0f;
attrs->maxcpus = 4;
memset(&wdef, 0, sizeof(wdef));
wdef.name = (char*)name;
wdef.attrs = attrs;
return makeWorker(g_tacv_bare_workermgr, &wdef);
}
static job_t* tacv_bare_make_job(uint64_t jobid)
@ -513,7 +542,7 @@ static void test_verify_apply_correctness_detects_job_count_mismatch(void)
.expected_new_job_count = 2,
};
TEST_ASSERT_EQUAL_INT_MESSAGE(APPLY_CLEAN,
persist_verify_apply_correctness(&p, &exp_match, jobmgr_count_before),
persist_verify_apply_correctness(&p, &exp_match, jobmgr_count_before, 0),
"matching expectation must report APPLY_CLEAN");
/* Mismatched expectation: log says 3 jobs should exist, only 2 do. */
@ -523,7 +552,7 @@ static void test_verify_apply_correctness_detects_job_count_mismatch(void)
};
TEST_ASSERT_EQUAL_INT_MESSAGE(APPLY_DIVERGENT,
persist_verify_apply_correctness(&p, &exp_count_mismatch,
jobmgr_count_before),
jobmgr_count_before, 0),
"a job-count mismatch must be detected as APPLY_DIVERGENT");
/* Mismatched expectation: position disagrees, count matches. */
@ -533,7 +562,7 @@ static void test_verify_apply_correctness_detects_job_count_mismatch(void)
};
TEST_ASSERT_EQUAL_INT_MESSAGE(APPLY_DIVERGENT,
persist_verify_apply_correctness(&p, &exp_position_mismatch,
jobmgr_count_before),
jobmgr_count_before, 0),
"an apply-cursor position mismatch must be detected as "
"APPLY_DIVERGENT even when the job count matches");
@ -541,6 +570,69 @@ static void test_verify_apply_correctness_detects_job_count_mismatch(void)
printf("test_verify_apply_correctness_detects_job_count_mismatch: PASS\n");
}
/*
* bug-1427: worker analog of the job-count test above. Before this fix,
* persist_verify_apply_correctness() had no worker-count check at all — a
* worker-creation divergence between the log-derived expectation and the
* live workermgr table went completely undetected. Also proves the two
* checks are independent: a job-count-only expectation still passes even
* with workers present (worker check only engages when workermgr is set
* AND expected_new_worker_count disagrees).
*/
static void test_verify_apply_correctness_detects_worker_count_mismatch(void)
{
tacv_bare_setup();
cluster_t c;
memset(&c, 0, sizeof(c));
c.jobmgr = g_tacv_bare_jobmgr;
c.workermgr = g_tacv_bare_workermgr;
persist_t p;
memset(&p, 0, sizeof(p));
p.cluster = &c;
uint64_t jobmgr_count_before = 0;
uint64_t workermgr_count_before = 0; /* empty table before "replay" */
/* Two workers materialize — mimics what a real replay would have done
* via apply_committed_entry()'s WAL_WORKER_CTRL/HOOK_DETAIL_WORKER_JOIN
* route (worker_from_capnp() returning 1 for each genuine creation). */
TEST_ASSERT_NOT_NULL(tacv_bare_make_worker("wkr-a"));
TEST_ASSERT_NOT_NULL(tacv_bare_make_worker("wkr-b"));
atomic_store_explicit(&p.last_applied_lsn, 42, memory_order_relaxed);
/* Matching expectation: 0 new jobs, 2 new workers, position 42 — CLEAN. */
apply_expectation_t exp_match = {
.expected_last_applied_lsn = 42,
.expected_new_job_count = 0,
.expected_new_worker_count = 2,
};
TEST_ASSERT_EQUAL_INT_MESSAGE(APPLY_CLEAN,
persist_verify_apply_correctness(&p, &exp_match, jobmgr_count_before,
workermgr_count_before),
"matching expectation must report APPLY_CLEAN");
/* Mismatched expectation: log says 3 workers should exist, only 2 do —
* the exact bug-1427 gap: a worker-creation divergence must now be
* caught, not silently invisible to this check. */
apply_expectation_t exp_count_mismatch = {
.expected_last_applied_lsn = 42,
.expected_new_job_count = 0,
.expected_new_worker_count = 3,
};
TEST_ASSERT_EQUAL_INT_MESSAGE(APPLY_DIVERGENT,
persist_verify_apply_correctness(&p, &exp_count_mismatch,
jobmgr_count_before,
workermgr_count_before),
"a worker-count mismatch must be detected as APPLY_DIVERGENT "
"(bug-1427)");
tacv_bare_teardown();
printf("test_verify_apply_correctness_detects_worker_count_mismatch: PASS\n");
}
/*
* Defensive: no live cluster to query (a bare unit-test persist_t, or any
* caller with nothing to dispatch into) must degrade to APPLY_CLEAN rather
@ -560,7 +652,7 @@ static void test_verify_apply_correctness_clean_when_no_cluster(void)
.expected_new_job_count = 5, /* irrelevant -- no cluster to check */
};
TEST_ASSERT_EQUAL_INT_MESSAGE(APPLY_CLEAN,
persist_verify_apply_correctness(&p, &exp, 0),
persist_verify_apply_correctness(&p, &exp, 0, 0),
"no live cluster to query must degrade to APPLY_CLEAN, not a false "
"DIVERGENT");
@ -1500,6 +1592,7 @@ void test_apply_correctness_verification(void)
RUN_TEST(test_clean_replay_produces_apply_clean_alongside_crc_verdict);
RUN_TEST(test_checkpoint_tail_with_no_snapshot_is_apply_clean);
RUN_TEST(test_verify_apply_correctness_detects_job_count_mismatch);
RUN_TEST(test_verify_apply_correctness_detects_worker_count_mismatch);
RUN_TEST(test_verify_apply_correctness_clean_when_no_cluster);
RUN_TEST(test_apply_divergent_decision_forces_unconditional_fence);
RUN_TEST(test_apply_divergent_loud_fence_sets_sticky_flag_and_stays_alive);

112
src/controller/testsuite/test_apply_pump.c
View file

@ -68,6 +68,7 @@
#include <errno.h>
#include <pthread.h>
#include <signal.h>
#include <stdatomic.h>
#include <stdbool.h>
#include <stdio.h>
#include <string.h>
@ -1406,6 +1407,116 @@ static void test_apply_pump_late_pending_apply_registration_self_heals(void)
"PASS\n");
}
/* bug-1428: a narrower TOCTOU residual of the SAME self-heal mechanism the
* test above proves. There, the register call arrives after the walk has
* ALREADY finished applying `lsn` -- the well-known bug-1415 case, closed
* by the fast-path self-heal check. This test targets the tighter,
* previously-unclosed window: the register call's check runs BEFORE the
* walk applies `lsn` (sees "not yet applied") but the walk's own
* advance-then-take sequence completes entirely BEFORE the register call
* reaches its insert -- so the walk's persist_take_pending_apply() finds
* nothing (register hasn't inserted yet), and the register call then
* inserts a node the walk will never revisit. Permanently orphaned,
* identical failure mode to bug-1415, different trigger window.
*
* Reproduced deterministically (no real-time race dependent on timing)
* by driving the exact bad ordering through controlled state, mirroring
* how the register thread's old code observed state before its lock and
* how the walk observed/mutated it independently:
* 1. Main thread holds pending_applies_lock (standing in for "the walk's
* advance-then-take sequence is in flight, about to complete").
* 2. A second thread calls the real persist_register_pending_apply() for
* an lsn NOT YET applied (last_applied_lsn is still below it) -- under
* the pre-fix code this thread's unlocked check would already have
* run and concluded "not yet applied" by this point; under the fixed
* code it blocks entirely on the lock, check included.
* 3. Main thread (still holding the lock) advances last_applied_lsn past
* lsn -- simulating the walk finishing its apply for this lsn while
* the register call is blocked/in-flight.
* 4. Main thread releases the lock.
* 5. The second thread's persist_register_pending_apply() call proceeds.
*
* Post-fix: the register call's self-heal check runs AFTER acquiring the
* lock in step 5, observes the now-current last_applied_lsn from step 3,
* and fires on_applied directly -- no insert, no orphan. Pre-fix, the
* check would have already run (unlocked) back in step 2 against the
* STALE last_applied_lsn, concluded "not yet applied", and inserted
* unconditionally once the lock freed in step 5 -- orphaning it forever
* (on_applied never fires; the node stays linked on pending_applies).
*/
typedef struct {
persist_t* p;
uint64_t lsn;
int fired;
uint64_t fired_lsn;
} trpa_ctx_t;
static void trpa_on_applied(void* ctx, uint64_t lsn)
{
trpa_ctx_t* c = (trpa_ctx_t*)ctx;
c->fired = 1;
c->fired_lsn = lsn;
}
static void* trpa_register_thread(void* arg)
{
trpa_ctx_t* c = (trpa_ctx_t*)arg;
persist_register_pending_apply(c->p, c->lsn, trpa_on_applied, c);
return NULL;
}
static void test_register_pending_apply_race_against_concurrent_apply_self_heals(void)
{
persist_t p;
memset(&p, 0, sizeof(p));
TEST_ASSERT_EQUAL_INT(0, pthread_mutex_init(&p.pending_applies_lock, NULL));
atomic_store_explicit(&p.last_applied_lsn, 40, memory_order_relaxed);
const uint64_t target_lsn = 42; /* not yet applied at the moment the
* register thread's call begins */
trpa_ctx_t rctx = { .p = &p, .lsn = target_lsn, .fired = 0, .fired_lsn = 0 };
pthread_mutex_lock(&p.pending_applies_lock);
pthread_t th;
TEST_ASSERT_EQUAL_INT(0,
pthread_create(&th, NULL, trpa_register_thread, &rctx));
/* Give the register thread every opportunity to run past an unlocked
* check (pre-fix) before the walk's simulated advance below — a bounded
* sleep is fine here since the correctness assertion is deterministic
* regardless of exactly how far it got; this only maximizes pre-fix
* failure detection sensitivity. */
usleep(20000);
/* Simulate the walk finishing its apply of target_lsn while the
* register call is blocked on (or, pre-fix, already past) the check. */
atomic_store_explicit(&p.last_applied_lsn, target_lsn, memory_order_relaxed);
pthread_mutex_unlock(&p.pending_applies_lock);
struct timespec ts = { .tv_sec = 0, .tv_nsec = 200000000L };
nanosleep(&ts, NULL);
TEST_ASSERT_EQUAL_INT(0, pthread_join(th, NULL));
int has_pending = persist_has_pending_apply(&p, target_lsn);
pthread_mutex_destroy(&p.pending_applies_lock);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rctx.fired,
"bug-1428: a registration racing a concurrent apply of the same lsn "
"must still fire its hand-off (self-heal) exactly once -- 0 means "
"the TOCTOU orphaned it permanently");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(target_lsn, rctx.fired_lsn,
"the hand-off must fire for the correct lsn");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, has_pending,
"no orphaned registration may remain linked on pending_applies");
printf("test_register_pending_apply_race_against_concurrent_apply_self_heals: "
"PASS\n");
}
/* ── 6. Standby-ingest commit-then-apply (2026-07-10 fix) ────────────────
*
* Regression coverage for ha.c's standby WAL-replication ingest path.
@ -1878,6 +1989,7 @@ void test_apply_pump(void)
RUN_TEST(test_apply_pump_guard_does_not_break_job_ctrl_live_append);
RUN_TEST(test_apply_pump_fail_racing_finish_merges_against_current_state);
RUN_TEST(test_apply_pump_late_pending_apply_registration_self_heals);
RUN_TEST(test_register_pending_apply_race_against_concurrent_apply_self_heals);
RUN_TEST(test_standby_apply_deferred_until_committed_then_materializes);
RUN_TEST(test_standby_truncate_before_commit_leaves_no_phantom_job);
}

232
src/controller/testsuite/test_commit_barrier_reply.c
View file

@ -133,6 +133,43 @@ static void cbrp_notify_capture_reset(void)
sizeof(g_notify_reply_already_enqueued));
}
/* scheduler-dispatch-commit task 3.3: minimal read-only accessors exposing
* this file's notify() capture (above) to other test files sharing this
* same controller_unit_test binary. This file owns the sole notify()
* definition linked into that binary (dataex.c's real one is excluded --
* see the "Link stubs" comment at the top of this file), so it is the only
* place any test in the binary can observe a notify_dispatch() (job.c)
* call. Read-only: does not alter this file's own capture/reset behavior
* or any of its own tests. Consumed by test_dispatch_fail_retry.c to
* confirm notify_dispatch() fires on the new retryable-resolution path. */
int cbrp_notify_capture_count_ext(void)
{
return g_notify_count;
}
int cbrp_notify_capture_event_ext(int idx)
{
return (idx >= 0 && idx < g_notify_count && idx < CBRP_MAX_NOTIFY)
? g_notify_events[idx] : -1;
}
uint64_t cbrp_notify_capture_jobid_ext(int idx)
{
return (idx >= 0 && idx < g_notify_count && idx < CBRP_MAX_NOTIFY)
? g_notify_jobids[idx] : 0;
}
int cbrp_notify_capture_code_ext(int idx)
{
return (idx >= 0 && idx < g_notify_count && idx < CBRP_MAX_NOTIFY)
? g_notify_codes[idx] : 0;
}
void cbrp_notify_capture_reset_ext(void)
{
cbrp_notify_capture_reset();
}
static worker_t *g_stub_ep_worker; /* NULL by default (3.1 behavior) */
/* clang-format off */
@ -1831,14 +1868,30 @@ static xce_job_dispatch_t g_dispatchmsg;
static xce_job_alloc_t *g_dispatch_allocs[2];
static xce_request_t g_dispatch_req;
#define CBRP_DISP_JOBID 60u
#define CBRP_DISP_JOBID 60u
#define CBRP_DISP_JOBID_RETRY 61u
/* Full dispatch fixture: queue (get_queue_exec), pending job with a real
* submit (dispatch_a_job_ reads submit->queue and submit->start_info),
* worker with a live no-iomgr endpoint (the startjob request must be
* enqueueable), and a 2-job dispatch message exercising BOTH outcomes:
* allocs[0] — dispatchable to wkr-disp → notify_dispatch(ok, code 0)
* allocs[1] — nhosts == 0 → notify_dispatch(fail, -EINVAL)
/* Full dispatch fixture: queue (get_queue_exec), two pending jobs each with
* a real submit (dispatch_a_job_ reads submit->queue and submit->
* start_info), worker with a live no-iomgr endpoint (the startjob request
* must be enqueueable), and a 2-job dispatch message exercising BOTH
* outcomes reachable through dispatch_a_job_()'s post-task-3.1/3.3
* resolution (scheduler-dispatch-commit task 5.4 — see this file's
* "Implementation Notes" history for CBRP_DISP_JOBID's own scenario, which
* this replaces):
* allocs[0] — dispatchable to wkr-disp → succeeds: job.state
* becomes RUN, one WAL_JOB_DISPATCH fact is appended, no
* notify_dispatch() call (success is never notified).
* allocs[1] — host "wkr-disp-missing" is not registered in
* g_hc.workermgr (XCE_DISPATCH_FAIL_SITE_WORKER_NULL,
* mirroring test_dispatch_fail_retry.c's own
* test_worker_notfound_resolves_retryable fixture) → a
* genuinely RETRYABLE dispatch failure: no WAL append
* (nothing was ever committed), exactly one
* notify_dispatch(jobDispatchFail) call. Kept on a SEPARATE
* job id (CBRP_DISP_JOBID_RETRY) so this scenario doesn't
* depend on ordering effects from allocs[0]'s own state
* transition on the same job.
* The allocs are heap-built via xcealloc helpers because the handler
* takes ownership (MOVE_PTR + free_JobAlloc_ptr per job). */
static void cbrp_arrange_jobdispatch(void)
@ -1859,6 +1912,11 @@ static void cbrp_arrange_jobdispatch(void)
g_disp_submit.user = (char *)"tester";
g_disp_submit.start_info = &g_disp_startinfo;
cbrp_make_job(CBRP_DISP_JOBID, &g_disp_submit);
/* scheduler-dispatch-commit task 5.4: a second job backing allocs[1]'s
* retryable-failure scenario below — cbrp_make_job's own comment
* confirms submit is deep-copied by makeJob/makeJobsub, so reusing
* g_disp_submit/g_disp_startinfo for both jobs is safe. */
cbrp_make_job(CBRP_DISP_JOBID_RETRY, &g_disp_submit);
attrs = calloc(1, sizeof(*attrs));
TEST_ASSERT_NOT_NULL(attrs);
@ -1887,7 +1945,19 @@ static void cbrp_arrange_jobdispatch(void)
make_allocs(1,
make_alloc((char *)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(g_dispatch_allocs[0]);
g_dispatch_allocs[1] = make_job_alloc(CBRP_DISP_JOBID, 0, NULL);
/* scheduler-dispatch-commit task 5.4 (was: make_job_alloc(CBRP_DISP_
* JOBID, 0, NULL), a terminal nhosts==0 failure — see the updated
* fixture-header comment above for why this changed): a genuinely
* RETRYABLE dispatch failure (unregistered worker host), so this test
* still drives exactly one real notify_dispatch() call through
* handle_jobdispatch() end-to-end, preserving the reply-before-
* processing ordering check below. */
g_dispatch_allocs[1] = make_job_alloc(
CBRP_DISP_JOBID_RETRY, 1,
make_host_allocs(1,
make_host_alloc((char *)"wkr-disp-missing", 1,
make_allocs(1,
make_alloc((char *)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(g_dispatch_allocs[1]);
memset(&g_dispatchmsg, 0, sizeof(g_dispatchmsg));
@ -1903,13 +1973,36 @@ static void cbrp_arrange_jobdispatch(void)
/* VERDICT PIN (Req 2.3): handle_jobdispatch (job.c:468-500, re-verified
* 2026-06-12) replies status=0 IMMEDIATELY — an ACK-of-receipt per the
* umbrella Reply-Ordering Contract (see section header) — and a full
* dispatch cycle (one successful + one failed allocation) appends NO WAL
* entries: allocation results return via notify_dispatch and job-state
* changes arrive later as worker facts. ACKs are withheld for the whole
* test, so a regression that adds a barrier wait trips the latency bound
* and any WAL append from dispatch processing trips the unchanged-LSN
* assert (an unclassified type would abort in wal_classify first). */
* umbrella Reply-Ordering Contract (see section header). ACKs are withheld
* for the whole test, so a regression that adds a barrier wait trips the
* latency bound.
*
* UPDATED (scheduler-dispatch-commit task 5.4, Req 5.3): the WAL/notify
* expectations below were re-pinned to reflect the dispatch-commit
* feature's new timing (Req 1.1/2.1) — they are no longer part of THIS
* test's own pin (which is only about the receipt ACK being immediate and
* ordered before per-job processing), just its supporting scenario:
* - dispatch processing is no longer guaranteed to append NO WAL entries
* — a job whose dispatch to the worker actually succeeds now commits
* its running-state transition synchronously (task 3.1), appending
* exactly one WAL_JOB_DISPATCH fact per successful dispatch. This
* test's batch has exactly one such successful dispatch (allocs[0]),
* so lsn_after must be lsn_before + 1, not lsn_before + 0. A
* genuinely retryable failure (allocs[1]) still appends nothing
* (nothing was ever committed for it — vacuously true by
* construction, per job.c's own task 3.3 comments), and the
* wal_classify XCE_TESTING guard still backstops any unclassified
* WAL type.
* - success is never notified (unchanged, predates this feature) and a
* terminal failure now resolves via mark_job_fail()'s own hook chain,
* not notify_dispatch() (task 3.2/4.1) — so only a genuinely
* RETRYABLE failure still produces a notify_dispatch(jobDispatchFail)
* call. This test's fixture was updated (see cbrp_arrange_jobdispatch)
* to give allocs[1] a retryable-classified failure specifically so
* this pin keeps exercising one real notify_dispatch() call end-to-end
* through handle_jobdispatch(), preserving the reply-before-processing
* ordering check below (which needs at least one notify call to prove
* anything). */
static void test_jobdispatch_receipt_ack_no_wal_append(void)
{
cbrp_reply_obs_t obs = { 0 };
@ -1943,31 +2036,46 @@ static void test_jobdispatch_receipt_ack_no_wal_append(void)
TEST_ASSERT_EQUAL_INT_MESSAGE(0, obs.last_status,
"the receipt ACK reports acceptance of the message (status 0)");
/* THE classification pin: dispatch processing appends no WAL entries,
* so there is nothing the receipt ACK could have falsely confirmed */
TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before, lsn_after,
"a dispatch cycle must append NO WAL entries (Req 2.3 — any "
"future append from dispatch processing must move behind the "
"barrier or be re-classified; the wal_classify XCE_TESTING "
"guard backstops unclassified types)");
/* scheduler-dispatch-commit task 5.4 (Req 5.3): re-pinned WAL-count
* expectation — allocs[0]'s dispatch to the live worker now succeeds
* and commits its running-state transition synchronously (task 3.1),
* appending exactly one WAL_JOB_DISPATCH fact; allocs[1]'s retryable
* failure appends nothing (see the VERDICT PIN comment above). Any
* OTHER/unexpected WAL append still trips this assert, and an
* unclassified type would still abort in wal_classify first. */
TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before + 1, lsn_after,
"a dispatch cycle with one successful dispatch must append "
"exactly one WAL_JOB_DISPATCH fact (Req 2.4) and nothing else "
"(Req 2.3 — any further append from dispatch processing must "
"move behind the barrier or be re-classified; the wal_classify "
"XCE_TESTING guard backstops unclassified types)");
/* notify_dispatch outcomes unaffected: one ok + one fail, in order */
TEST_ASSERT_EQUAL_INT_MESSAGE(2, g_notify_count,
"both allocations must produce their per-job outcome notification");
/* scheduler-dispatch-commit task 5.4 (Req 5.3): re-pinned notify-count
* expectation — success (allocs[0]) is never notified, and a terminal
* failure resolves via mark_job_fail()'s own hook chain rather than
* notify_dispatch() (tasks 3.2/4.1); only allocs[1]'s genuinely
* retryable failure produces a notify_dispatch(jobDispatchFail) call,
* so the batch total is exactly one, not two. */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_notify_count,
"only allocs[1]'s retryable dispatch failure notifies — success "
"(allocs[0]) and a terminal failure are never reported via "
"notify_dispatch()");
TEST_ASSERT_EQUAL_INT(NotifyData_data_jobDispatchFail,
g_notify_events[0]);
TEST_ASSERT_EQUAL_INT(NotifyData_data_jobDispatchFail,
g_notify_events[1]);
TEST_ASSERT_EQUAL_UINT64(CBRP_DISP_JOBID, g_notify_jobids[0]);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_notify_codes[0],
"allocs[0] dispatch to the live worker must succeed (code 0)");
TEST_ASSERT_EQUAL_UINT64(CBRP_DISP_JOBID, g_notify_jobids[1]);
TEST_ASSERT_EQUAL_INT_MESSAGE(-EINVAL, g_notify_codes[1],
"allocs[1] (nhosts == 0) must fail with -EINVAL");
TEST_ASSERT_EQUAL_UINT64(CBRP_DISP_JOBID_RETRY, g_notify_jobids[0]);
TEST_ASSERT_TRUE_MESSAGE(g_notify_codes[0] < 0,
"allocs[1]'s dispatch to an unregistered worker must fail with a "
"negative code (exact errno value is librtti key_find()'s own "
"internal, opaque choice — not asserted, mirroring "
"test_dispatch_fail_retry.c's established precedent)");
/* Ordering: the receipt ACK was already enqueued when the FIRST
* dispatch outcome fired — reply precedes processing (ACK-of-receipt,
* not ACK-of-outcome) */
/* Ordering (this test's actual pin, unchanged by task 5.4): the
* receipt ACK was already enqueued when allocs[1]'s dispatch outcome
* fired — reply precedes processing (ACK-of-receipt, not
* ACK-of-outcome). This is why the fixture keeps exactly one
* notify_dispatch() call reachable in this batch (allocs[1]) — with
* zero calls there would be nothing left to prove this ordering
* against. */
TEST_ASSERT_TRUE_MESSAGE(g_notify_reply_already_enqueued[0],
"the receipt ACK must be enqueued before any dispatch outcome "
"notification (reply-before-processing is the verdict)");
@ -2136,17 +2244,22 @@ static void test_clr_preason_receipt_reply_fact_retained(void)
/* ── do_update_op rusage path fixture ────────────────────────────────── */
/* KNOWN PRE-EXISTING MISMATCH (discovered while pinning, reported in the
* task evidence — NOT fixed here, production is out of boundary):
* update_job_rusages fires the AFTER_EVENT HOOK_DETAIL_JOB_RUSAGE_UPDATE
* hook with (cluster_t*, xce_job_rusages_t*) [job.c:1015-1016], but
* persist_job_hook_fun's default arm reads the second vararg as a job_t*
* [persist.c:1354-1359] and dereferences j->job.state at offset 56 —
* exactly sizeof(xce_job_rusages_t) — an out-of-bounds read on an
* exactly-sized allocation (the WAL_JOB_RUSAGE payload's state field is
* garbage). The padded box below keeps THIS pin's read in-bounds under
* ASAN without masking the report; the jobid field (offset 0 in both
* types) lines up by accident. */
/* bug-451, FIXED: update_job_rusages fires the AFTER_EVENT
* HOOK_DETAIL_JOB_RUSAGE_UPDATE hook with (cluster_t*, xce_job_rusages_t*)
* — a different va_list shape than every other job hook detail — because
* sched_job_rusage_update_fun/apiserver_job_rusage_update_fun (this hook's
* other two listeners) genuinely need the full rusages payload
* (pids/pgids/usages), not just a job_t*. persist_job_hook_fun previously
* had no dedicated arm for this detail and fell through to the default
* arm, which misread the xce_job_rusages_t* as a job_t* and dereferenced
* j->job.state at offset 56 — exactly sizeof(xce_job_rusages_t) — an
* out-of-bounds read on an exactly-sized allocation. Now fixed with a
* dedicated arm that reads the real xce_job_rusages_t* and looks the job
* up by jobid via the cluster_t* the hook already carries. The padded box
* below predates the fix (kept so this box's own read stays in-bounds
* under ASAN regardless); the jobid field (offset 0 in both types) lined
* up by accident even under the old bug — see the new assertion below for
* proof the state field (which did NOT line up) is now correct too. */
static union {
xce_job_rusages_t r;
unsigned char pad[sizeof(xce_job_rusages_t) + 64];
@ -2198,12 +2311,35 @@ static void cbrp_arrange_update_rusage(uint64_t jobid)
* WAL_JOB_UPDATE, HOOK_DETAIL_JOB_RUSAGE_UPDATE → WAL_JOB_RUSAGE, both
* RETAIN per wal_classify) stay pending until ACK-covered. Umbrella
* Reply-Ordering Contract: receipt-only ACK for fact-shaped delivery. */
/* Forward-declared: defined further below, alongside its _ex sibling and
* every other caller (this test is the only one that needs it this early
* in the file). */
static int cbrp_count_jobstate_entries(persist_t *p, uint32_t wal_type,
uint64_t jobid, uint32_t *state_out);
static void test_update_op_rusage_receipt_reply_facts_retained(void)
{
uint32_t rusage_state = 0xdeadbeef;
int rusage_count;
cbrp_setup(3000);
cbrp_arrange_jobctrl(72, 1); /* job 72 exists; jobctrl req unused */
cbrp_arrange_jobctrl(72, 1); /* job 72 exists, state = JOB_STAT_PEND */
cbrp_arrange_update_rusage(72);
cbrp_drive_receipt_fact(do_update_op, &g_update_req, 2);
/* bug-451 regression pin: the WAL_JOB_RUSAGE fact's jobid AND state
* must both be correct — under the pre-fix bug, jobid happened to
* line up by accident (offset 0 in both misread types) but state
* (offset 56, past xce_job_rusages_t's end) read back as 0 from the
* zeroed g_rusages_box padding, not the job's real JOB_STAT_PEND. */
rusage_count = cbrp_count_jobstate_entries(&g_hc.persist, WAL_JOB_RUSAGE,
72, &rusage_state);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, rusage_count,
"WAL_JOB_RUSAGE fact for job 72 must be found by jobid");
TEST_ASSERT_EQUAL_UINT32_MESSAGE((uint32_t)JOB_STAT_PEND, rusage_state,
"bug-451: WAL_JOB_RUSAGE's state must be the job's real state, not "
"a type-confused OOB read past xce_job_rusages_t's end");
g_stub_ep_worker = NULL;
cbrp_teardown();
printf("test_update_op_rusage_receipt_reply_facts_retained: PASS\n");

66
src/controller/testsuite/test_commit_durability.c
View file

@ -975,6 +975,71 @@ static void test_durable_majority_commit_end_to_end(void)
printf("test_durable_majority_commit_end_to_end: PASS\n");
}
/* ------------------------------------------------------------------ *
* bug-1564: persist_flush() must fire the durable-advance callback, *
* same as the async fsync_worker does on its own successful flush. *
* *
* persist_flush is the synchronous durability point tests/clean *
* shutdown use in place of waiting for the background flusher. Before *
* this fix it advanced durable_lsn but never notified *
* on_durable_advance -- any WAIT-blocked commit caller relying on that *
* callback would only ever be woken by fsync_worker's OWN next *
* successful flush, never by this call. bug-1563's un-throttled-retry *
* fix (fsync_worker now backs off between failed attempts instead of *
* spinning) is what turned this from a latent, never-triggered gap *
* into a real, deterministic timeout *
* (test_durable_majority_commit_end_to_end, above) -- pre-1563, *
* fsync_worker's busy-spin happened to notice and fire the callback *
* itself within microseconds of any state change, masking the gap. *
* ------------------------------------------------------------------ */
static _Atomic int g_durable_advance_fire_count;
static void durable_advance_count_cb(void* ctx)
{
(void)ctx;
atomic_fetch_add(&g_durable_advance_fire_count, 1);
}
static void test_persist_flush_fires_durable_advance_callback(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durflushcb_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
atomic_store(&g_durable_advance_fire_count, 0);
persist_set_durable_cb(p, durable_advance_count_cb, NULL);
/* Slow background cadence (wal_sync_ms=5000, make_durability_persist)
* means the async fsync_worker will not race in and fire this on its
* own within the test's lifetime -- persist_flush is the only thing
* that can advance durable_lsn here. */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, atomic_load(&g_durable_advance_fire_count));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
TEST_ASSERT_TRUE_MESSAGE(
atomic_load(&g_durable_advance_fire_count) >= 1,
"persist_flush must fire on_durable_advance when it advances "
"durable_lsn (bug-1564), same as fsync_worker's own successful "
"flush -- otherwise a WAIT-blocked commit caller relying on this "
"callback is never woken by this path");
/* A second flush with nothing new to sync must NOT re-fire (mirrors
* fsync_worker's own "only on actual advance" gate). */
int before_second = atomic_load(&g_durable_advance_fire_count);
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
TEST_ASSERT_EQUAL_INT_MESSAGE(
before_second, atomic_load(&g_durable_advance_fire_count),
"a flush with nothing new to sync must not re-fire the callback");
teardown_durability_persist(p);
printf("test_persist_flush_fires_durable_advance_callback: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 5.1b (Req 1.5 / 2.5): losing the sole durable contributor *
* leaves the acknowledged op present on a surviving member. *
@ -2517,6 +2582,7 @@ void test_commit_durability(void)
RUN_TEST(test_commit_never_exceeds_durable_invariant);
/* task 5.1 — durable-majority commit wired end to end */
RUN_TEST(test_durable_majority_commit_end_to_end);
RUN_TEST(test_persist_flush_fires_durable_advance_callback);
RUN_TEST(test_sole_durable_contributor_loss_survives);
RUN_TEST(test_durability_work_off_iomgr_preserves_liveness);
RUN_TEST(test_recover_verdict_clean_intact_log);

737
src/controller/testsuite/test_dispatch_commit.c Normal file
View file

@ -0,0 +1,737 @@
/* test_dispatch_commit.c — scheduler-dispatch-commit task 3.1
*
* Proves dispatch_a_job_()'s (job.c) restructured commit: the job's
* resource allocation and its transition to a running state are committed
* together, ONLY once the dispatch request to the worker actually succeeds
* (Req 2.1-2.4, 5.1) -- gating a commit that used to run unconditionally
* regardless of xce_startjob_req()'s outcome. Also proves the consecutive
* retryable-failure counter (job_t.dispatch_fail_count, task 1.3) resets to
* zero on that same successful outcome (Req 4.3), and that the new durable
* WAL_JOB_DISPATCH fact (task 1.2) is appended as part of it (Req 2.4).
*
* dispatch_a_job_() is static/file-local (its only caller, handle_
* jobdispatch(), is untouched by this task); reached here through
* dispatch_a_job_test(), an XCE_TESTING-only wrapper this task adds --
* mirroring xce_classify_dispatch_fail_test()'s existing precedent in the
* same file (test_job_dispatch.c, task 1.1).
*
* No live peer / synthetic reply thread is needed to make a dispatch
* "succeed": xce_startjob_req() (request.c) is a genuinely non-blocking,
* fire-and-forget send -- xce_request() only capnp-encodes the request and
* endpoint_enq()s it onto the endpoint's outq, it never waits for a reply.
* endpoint_enq() succeeds unconditionally against a zeroed endpoint_t with
* no iomgr registered: dataq_enq() (dataq.c) has no failure path, and
* endpoint_update_epoll() (netio.c) is a pure no-op whenever ep->iomgr ==
* NULL. This exact "zeroed, no-iomgr endpoint" pattern is already
* established and proven live by test_commit_barrier_reply.c's
* cbrp_arrange_jobdispatch()/g_wkr_disp_ep fixture (see that file's own
* header comment on its reply-observation harness) -- this file's fixtures
* mirror that queue/job/worker/alloc construction, trimmed to what
* dispatch_a_job_() itself reads (no HA commit-engine/standby-registry
* machinery, which is only needed there for that file's own, unrelated
* reply-ordering assertions).
*
* Two fixture tiers, mirroring test_wal_job_dispatch_fact.c's own split:
* - dc_setup/dc_teardown: cluster_t (jobmgr/workermgr/qmgr/hooksys) + a
* real disk-backed persist_t initialized with cluster == NULL, so
* wal_hook_append_classified() (persist.c) takes the direct
* persist_wal_async() path -- no raft/commit-engine machinery needed
* for a plain RETAIN-mode fact append.
* - the durability test builds its own two short-lived cluster_t/
* persist_t pairs directly, mirroring test_wal_job_dispatch_fact.c's
* wjd_make_fresh_persist/restart-replay pattern exactly.
*/
#include <rtti.h>
#include <capnp_c.h>
#include <errno.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
#include "cluster.h"
#include "controller.capnp.h"
#include "dataq.h"
#include "hook.h"
#include "jobj.h"
#include "kvmgr.h"
#include "netio.h"
#include "persist.h"
#include "qobj.h"
#include "wal_types.h"
#include "wobj.h"
#include "xce/cluster_def.h"
#include "xce/types.h"
#include "xcealloc.h"
#include "xstring.h"
/* dispatch_a_job_test: task 3.1's XCE_TESTING-only wrapper (job.c), thinly
* exposing the static dispatch_a_job_() for direct testing -- see this
* file's header and job.c's own comment beside the wrapper. */
extern int dispatch_a_job_test(cluster_t* c, endpoint_t* ep,
xce_job_alloc_t* alloc);
/* jobj.c's job type table and the process-global xstring table are shared
* across every test file in the controller_unit_test binary -- guard
* idempotently, mirroring every sibling fixture in this suite
* (test_wal_job_dispatch_fact.c's g_wjd_jobobj_ready). */
static int g_dc_jobobj_ready;
static cluster_t g_dc;
static char g_dc_tmpdir[256];
/* Worker endpoint: zeroed, no iomgr -- xce_startjob_req()'s enqueue always
* succeeds against it (see file header). File-static, not stack-local, so
* it stays valid for the whole fixture's lifetime regardless of which
* function last touched worker_t.ep (mirrors test_commit_barrier_reply.c's
* g_wkr_disp_ep). */
static endpoint_t g_dc_worker_ep;
static void dc_setup(void)
{
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING,
.nslots = 32, .delta = 16 };
persist_config_t cfg;
if (makeString((char*)"dc-probe") == NULL)
xstring_table_init(64);
if (!g_dc_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
g_dc_jobobj_ready = 1;
}
memset(&g_dc, 0, sizeof(g_dc));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&g_dc.hooksys, XCE_CONTROLLER_HOOK_MAX));
g_dc.jobmgr = makeInstance(&xkvmgr_type, &int_params);
g_dc.workermgr = makeInstance(&xkvmgr_type, &str_params);
g_dc.qmgr = makeInstance(&xkvmgr_type, &str_params);
TEST_ASSERT_NOT_NULL(g_dc.jobmgr);
TEST_ASSERT_NOT_NULL(g_dc.workermgr);
TEST_ASSERT_NOT_NULL(g_dc.qmgr);
/* Real disk-backed persist, cluster == NULL: wal_hook_append_
* classified() takes the direct persist_wal_async() path -- no raft/
* commit-engine machinery needed for a plain fact append (this exact
* routing is what lets a bare cluster_t exercise dispatch_a_job_()'s
* real persist_submit_job_fact() call without a full HA harness). */
cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
strcpy(g_dc_tmpdir, "/tmp/xce_dc31_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(g_dc_tmpdir));
persist_set_workdir(&g_dc.persist, g_dc_tmpdir, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(&g_dc.persist, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(&g_dc.persist));
memset(&g_dc_worker_ep, 0, sizeof(g_dc_worker_ep));
g_dc_worker_ep.fd = -1;
}
static void dc_teardown(void)
{
char cmd[512];
persist_stop(&g_dc.persist);
persist_fini(&g_dc.persist);
if (g_dc.workermgr) dereference(g_dc.workermgr);
if (g_dc.qmgr) dereference(g_dc.qmgr);
if (g_dc.jobmgr) dereference(g_dc.jobmgr);
xce_hook_fini(&g_dc.hooksys);
snprintf(cmd, sizeof(cmd), "rm -rf %s", g_dc_tmpdir);
if (system(cmd) != 0)
fprintf(stderr, "cleanup warning: rm -rf %s failed\n", g_dc_tmpdir);
}
/* Frees every packet left queued on an endpoint's outq (the startjob
* request xce_startjob_req() enqueued) -- mirrors test_commit_barrier_
* reply.c's cbrp_scan_replies, trimmed: this file never needs to decode the
* packets, only reclaim them. */
static void dc_drain_ep(endpoint_t* ep)
{
data_item_t* it;
while ((it = endpoint_deq(ep)) != NULL) {
io_packet_t* pkt = ITPTR_TO_PACKET(it);
free_packet(pkt);
free(pkt);
}
}
/* Seeds a queue, a PEND job with a real submit/start_info (deep-copied by
* makeJob via makeJobsub -- dispatch_a_job_() reads submit->queue and
* submit->start_info), and a ready worker whose ep is g_dc_worker_ep.
* Mirrors test_commit_barrier_reply.c's cbrp_arrange_jobdispatch fixture,
* built directly against g_dc rather than through handle_jobdispatch().
* Returns the seeded job_t* (borrowed -- g_dc.jobmgr owns it). */
static job_t* dc_seed_ready(uint64_t jobid, uint32_t start_fail_count,
const char* qname, const char* wname)
{
xce_queue_t qdef;
xce_worker_t wdef;
hostattrs_t* attrs;
job_startinfo_t startinfo;
job_submit_data_t submit;
xce_job_t jdef;
job_t* j = NULL;
worker_t* w = NULL;
memset(&qdef, 0, sizeof(qdef));
qdef.queue = (char*)qname;
TEST_ASSERT_NOT_NULL(makeQueue(g_dc.qmgr, &qdef));
memset(&startinfo, 0, sizeof(startinfo));
startinfo.cmdline = (char*)"/bin/true";
memset(&submit, 0, sizeof(submit));
submit.queue = (char*)qname;
submit.user = (char*)"tester";
submit.start_info = &startinfo;
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = jobid;
jdef.state = JOB_STAT_PEND;
jdef.submit = &submit;
j = makeJob(g_dc.jobmgr, &jdef);
TEST_ASSERT_NOT_NULL(j);
j->dispatch_fail_count = start_fail_count;
attrs = calloc(1, sizeof(*attrs));
TEST_ASSERT_NOT_NULL(attrs);
attrs->hosttype = makeString("LINUX");
attrs->hostmodel = makeString("x86");
attrs->cpufactor = 1.0f;
attrs->maxcpus = 4;
memset(&wdef, 0, sizeof(wdef));
wdef.name = (char*)wname;
wdef.attrs = attrs;
TEST_ASSERT_NOT_NULL(makeWorker(g_dc.workermgr, &wdef));
lock(g_dc.workermgr, RTTI_LOCK_READ);
key_find(g_dc.workermgr, (void*)wname, (void**)&w);
unlock(g_dc.workermgr);
TEST_ASSERT_NOT_NULL(w);
w->ep = &g_dc_worker_ep; /* host.status defaults to 0 (ready) via calloc */
return j;
}
#define DC_JOBID_SUCCESS 91001ULL
/* THE core proof (Req 2.1, 2.3, 4.3): a successful dispatch commits
* job.allocs and transitions job.state to RUN together, and resets
* dispatch_fail_count to 0 from a nonzero starting value -- all in one
* operation, with no dependence on any later worker confirmation. */
static void test_dc_success_commits_alloc_state_and_resets_counter(void)
{
job_t* j;
xce_job_alloc_t* alloc;
int rc;
dc_setup();
j = dc_seed_ready(DC_JOBID_SUCCESS, 3 /* nonzero starting fail count */,
"dq", "wkr-dc");
alloc = make_job_alloc(DC_JOBID_SUCCESS, 1,
make_host_allocs(1,
make_host_alloc((char*)"wkr-dc", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(alloc);
rc = dispatch_a_job_test(&g_dc, NULL, alloc);
TEST_ASSERT_TRUE_MESSAGE(rc >= 0,
"a dispatch to a live, no-iomgr worker endpoint must succeed -- "
"xce_startjob_req() is a fire-and-forget enqueue, not a blocking "
"round trip");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j->job.state,
"the job must transition to RUN in the same operation that commits "
"the allocation, unaffected by whether the worker has separately "
"confirmed job start yet (Req 2.1, 2.3)");
TEST_ASSERT_NOT_NULL(j->job.allocs);
TEST_ASSERT_EQUAL_UINT64(DC_JOBID_SUCCESS, j->job.allocs->jobid);
TEST_ASSERT_EQUAL_INT(1, j->job.allocs->nhosts);
TEST_ASSERT_NOT_NULL(j->job.allocs->allocs);
TEST_ASSERT_EQUAL_STRING("wkr-dc", j->job.allocs->allocs[0]->host);
TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, j->dispatch_fail_count,
"a successful dispatch must reset the consecutive retryable-"
"failure counter to zero (Req 4.3)");
free_JobAlloc_ptr(&alloc); /* allocs already MOVE_PTR'd out -- top-level
* struct only */
dc_drain_ep(&g_dc_worker_ep);
dc_teardown();
printf("test_dc_success_commits_alloc_state_and_resets_counter: PASS\n");
}
/* bug-1679: spy hook counting HOOK_AFTER_EVENT/HOOK_DETAIL_JOB_UPDATE fires
* specifically -- the exact hook apiserver_job_update_fun (apiserver.c)
* subscribes to in production to push a job's current state to every
* connected apiserver. Mirrors test_apply_committed_entry.c's tace_spy_hook
* pattern, filtered to this one detail rather than every AFTER_EVENT. */
static int g_dc_update_hook_fires;
static int dc_job_update_spy_hook(int type, int situation, int detail,
void* data, va_list ap)
{
(void)type; (void)data; (void)ap;
if (situation == HOOK_AFTER_EVENT && detail == HOOK_DETAIL_JOB_UPDATE)
g_dc_update_hook_fires++;
return 0;
}
#define DC_JOBID_NOTIFY 91004ULL
/* THE core proof for bug-1679 (Req: a live successful dispatch must notify
* the apiserver, restart or not): dispatch_a_job_()'s success path fires
* HOOK_DETAIL_JOB_UPDATE's AFTER_EVENT exactly once, synchronously, in the
* SAME operation that commits state+allocs -- unlike before this fix, where
* dispatch_a_job_() fired no hooks at all and relied entirely on
* apply_committed_entry()'s later (here, always-suppressed-by-duplicate-
* state) notify. RED before the fix: g_dc_update_hook_fires stays 0. */
static void test_dc_success_notifies_job_update_hook_exactly_once(void)
{
job_t* j;
xce_job_alloc_t* alloc;
int rc;
dc_setup();
g_dc_update_hook_fires = 0;
TEST_ASSERT_EQUAL_INT(0,
xce_add_hook(&g_dc.hooksys, XCE_CONTROLLER_HOOK_JOB,
dc_job_update_spy_hook, NULL));
j = dc_seed_ready(DC_JOBID_NOTIFY, 0, "dq", "wkr-dc-notify");
alloc = make_job_alloc(DC_JOBID_NOTIFY, 1,
make_host_allocs(1,
make_host_alloc((char*)"wkr-dc-notify", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(alloc);
rc = dispatch_a_job_test(&g_dc, NULL, alloc);
TEST_ASSERT_TRUE(rc >= 0);
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_dc_update_hook_fires,
"a live successful dispatch must fire HOOK_DETAIL_JOB_UPDATE's "
"AFTER_EVENT exactly once (bug-1679) -- apiserver_job_update_fun's "
"subscription is the ONLY mechanism that pushes the job's new "
"RUN state and allocation to the apiserver; without this fire the "
"job is stuck at PEND from the apiserver's perspective forever");
free_JobAlloc_ptr(&alloc);
dc_drain_ep(&g_dc_worker_ep);
xce_disable_hook(&g_dc.hooksys, XCE_CONTROLLER_HOOK_JOB,
dc_job_update_spy_hook);
dc_teardown();
printf("test_dc_success_notifies_job_update_hook_exactly_once: PASS\n");
}
#define DC_JOBID_NOTFOUND 91002ULL
/* Regression safety net (untouched failure branch): a dispatch whose
* target worker cannot be found must leave the job's ALLOCATION exactly
* as it was before the attempt -- still PEND, allocs still uncommitted --
* and must append no WAL entry. Confirms this task's restructuring did
* not disturb the five failure branches ahead of xce_startjob_req().
*
* scheduler-dispatch-commit task 3.3 update: dispatch_fail_count is no
* longer asserted unchanged here -- task 3.3 wired this exact failure
* branch (whichever of XCE_DISPATCH_FAIL_SITE_WORKER_LOOKUP/WORKER_NULL
* the runtime takes for an unregistered worker key) to increment the
* counter by exactly 1 as part of its retryable resolution (Req 4.3),
* which this test's own original comment explicitly anticipated
* ("incrementing it is task 3.3's job, not 3.1's"). The counter's own
* increment-by-1/bound behavior is covered in depth by
* test_dispatch_fail_retry.c; this test only re-confirms the seeded
* starting value (2) plus this task's own increment (+1) lands at 3,
* still within the bound (3), so the job stays PEND -- not a regression
* in this test's own subject (allocation-gating), just an update to an
* assertion that depended on timing task 3.3 had not landed yet. */
static void test_dc_worker_not_found_leaves_job_pend_and_alloc_uncommitted(void)
{
job_t* j;
xce_job_alloc_t* alloc;
int rc;
uint64_t lsn_before, lsn_after;
dc_setup();
/* Seed only the job/queue -- deliberately no matching worker in
* g_dc.workermgr, so key_find() finds nothing (w stays NULL) and
* dispatch_a_job_() takes its WORKER_NULL failure branch. */
{
job_startinfo_t startinfo;
job_submit_data_t submit;
xce_job_t jdef;
xce_queue_t qdef;
memset(&qdef, 0, sizeof(qdef));
qdef.queue = (char*)"dq2";
TEST_ASSERT_NOT_NULL(makeQueue(g_dc.qmgr, &qdef));
memset(&startinfo, 0, sizeof(startinfo));
startinfo.cmdline = (char*)"/bin/true";
memset(&submit, 0, sizeof(submit));
submit.queue = (char*)"dq2";
submit.user = (char*)"tester";
submit.start_info = &startinfo;
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = DC_JOBID_NOTFOUND;
jdef.state = JOB_STAT_PEND;
jdef.submit = &submit;
j = makeJob(g_dc.jobmgr, &jdef);
TEST_ASSERT_NOT_NULL(j);
j->dispatch_fail_count = 2;
}
alloc = make_job_alloc(DC_JOBID_NOTFOUND, 1,
make_host_allocs(1,
make_host_alloc((char*)"no-such-worker", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(alloc);
pthread_mutex_lock(&g_dc.persist.lock);
lsn_before = g_dc.persist.current_lsn;
pthread_mutex_unlock(&g_dc.persist.lock);
rc = dispatch_a_job_test(&g_dc, NULL, alloc);
TEST_ASSERT_TRUE_MESSAGE(rc < 0, "dispatch to a nonexistent worker must "
"fail");
pthread_mutex_lock(&g_dc.persist.lock);
lsn_after = g_dc.persist.current_lsn;
pthread_mutex_unlock(&g_dc.persist.lock);
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state,
"a dispatch failure ahead of xce_startjob_req() must leave the job "
"state untouched (still PEND) -- this task only gates the SUCCESS "
"commit, it does not change any failure branch's behavior");
TEST_ASSERT_NULL_MESSAGE(j->job.allocs,
"job.allocs must remain uncommitted on this failure path");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(3, j->dispatch_fail_count,
"task 3.3: this retryable failure path increments the counter by "
"exactly 1 from its seeded starting value (2 -> 3), still within "
"the bound (3)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before, lsn_after,
"a failed dispatch must append no WAL entry");
free_JobAlloc_ptr(&alloc);
dc_teardown();
printf("test_dc_worker_not_found_leaves_job_pend_and_alloc_uncommitted: "
"PASS\n");
}
/* ── Durability: WAL_JOB_DISPATCH survives a restart replay ─────────────
*
* Mirrors test_wal_job_dispatch_fact.c's own restart-replay test exactly,
* except the WAL_JOB_DISPATCH fact is produced by a REAL successful
* dispatch_a_job_() call (this task's actual new code), not a hand-built
* wal_jobstate_t -- proving the fact this task's restructuring appends is
* durable and independently sufficient to recover the job as RUNNING, with
* no WAL_JOB_START anywhere in the log. */
static persist_t* dc_make_fresh_persist(const char* tmpdir)
{
persist_t* p = calloc(1, sizeof(persist_t));
persist_config_t cfg;
if (!p) return NULL;
cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
persist_set_workdir(p, tmpdir, "ctrl-test");
if (persist_init(p, &cfg, NULL) != 0) {
free(p);
return NULL;
}
if (persist_start(p) != 0) {
persist_fini(p);
free(p);
return NULL;
}
return p;
}
static void dc_teardown_persist(persist_t* p)
{
persist_stop(p);
persist_fini(p);
free(p);
}
/* Encode helper mirroring test_wal_job_dispatch_fact.c's wjd_encode_job
* (capnp_encode_job is file-static in persist.c, so a direct WAL_JOB_SUBMIT
* seed needs its own encoder built from the exported encode_Job/new_Job/
* write_Job codec chain). */
static int dc_encode_job(const xce_job_t* j, uint8_t** out, size_t* out_len)
{
struct capn c;
struct capn_segment* cs;
struct Job cj;
Job_ptr jp;
capn_init_malloc(&c);
cs = capn_root(&c).seg;
encode_Job(cs, &cj, (xce_job_t*)j);
jp = new_Job(cs);
write_Job(&cj, jp);
capn_setp(capn_root(&c), 0, jp.p);
*out_len = capn_size(&c);
*out = malloc(*out_len);
if (!*out) { capn_free(&c); return -ENOMEM; }
capn_write_mem(&c, *out, *out_len, 0);
capn_free(&c);
return 0;
}
#define DC_JOBID_REPLAY 91003ULL
static void test_dc_success_appends_wal_job_dispatch_durably(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_dc31_replay_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Phase 1: real cluster + real disk persist. Seed WAL_JOB_SUBMIT
* directly (so replay can reconstruct the job), then drive an actual
* successful dispatch_a_job_() call and let IT append WAL_JOB_DISPATCH
* -- this is the production code path under test, not a hand-built
* WAL entry. */
{
cluster_t c;
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING,
.nslots = 32, .delta = 16 };
persist_config_t cfg;
endpoint_t wep;
job_t* j;
worker_t* w = NULL;
xce_job_alloc_t* alloc;
int rc;
if (makeString((char*)"dc-probe") == NULL)
xstring_table_init(64);
if (!g_dc_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
g_dc_jobobj_ready = 1;
}
memset(&c, 0, sizeof(c));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));
c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
c.workermgr = makeInstance(&xkvmgr_type, &str_params);
c.qmgr = makeInstance(&xkvmgr_type, &str_params);
TEST_ASSERT_NOT_NULL(c.jobmgr);
TEST_ASSERT_NOT_NULL(c.workermgr);
TEST_ASSERT_NOT_NULL(c.qmgr);
cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
persist_set_workdir(&c.persist, tmp, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));
/* Durable WAL_JOB_SUBMIT seed -- no WAL_JOB_START anywhere in this
* log; the recovered RUN state must come from WAL_JOB_DISPATCH
* alone. */
{
job_submit_data_t wal_submit;
xce_job_t wal_jdef;
uint8_t* buf = NULL;
size_t sz = 0;
memset(&wal_submit, 0, sizeof(wal_submit));
wal_submit.queue = (char*)"dq";
wal_submit.user = (char*)"tester";
memset(&wal_jdef, 0, sizeof(wal_jdef));
wal_jdef.jobid = DC_JOBID_REPLAY;
wal_jdef.state = JOB_STAT_PEND;
wal_jdef.submit = &wal_submit;
TEST_ASSERT_EQUAL_INT(0, dc_encode_job(&wal_jdef, &buf, &sz));
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
(uint32_t)sz));
free(buf);
}
/* dc_seed_ready() operates against the module-global g_dc fixture,
* not usable here (this test's cluster_t is a local `c`, and
* dc_setup()/dc_teardown() are deliberately not called in this
* test) -- so the queue/job/worker are seeded directly below. */
{
xce_queue_t qdef;
job_startinfo_t startinfo;
job_submit_data_t submit;
xce_job_t jdef;
hostattrs_t* attrs;
xce_worker_t wdef;
memset(&qdef, 0, sizeof(qdef));
qdef.queue = (char*)"dq";
TEST_ASSERT_NOT_NULL(makeQueue(c.qmgr, &qdef));
memset(&startinfo, 0, sizeof(startinfo));
startinfo.cmdline = (char*)"/bin/true";
memset(&submit, 0, sizeof(submit));
submit.queue = (char*)"dq";
submit.user = (char*)"tester";
submit.start_info = &startinfo;
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = DC_JOBID_REPLAY;
jdef.state = JOB_STAT_PEND;
jdef.submit = &submit;
j = makeJob(c.jobmgr, &jdef);
TEST_ASSERT_NOT_NULL(j);
attrs = calloc(1, sizeof(*attrs));
TEST_ASSERT_NOT_NULL(attrs);
attrs->hosttype = makeString("LINUX");
attrs->hostmodel = makeString("x86");
attrs->cpufactor = 1.0f;
attrs->maxcpus = 4;
memset(&wdef, 0, sizeof(wdef));
wdef.name = (char*)"wkr-replay";
wdef.attrs = attrs;
TEST_ASSERT_NOT_NULL(makeWorker(c.workermgr, &wdef));
}
memset(&wep, 0, sizeof(wep));
wep.fd = -1;
lock(c.workermgr, RTTI_LOCK_READ);
key_find(c.workermgr, (void*)"wkr-replay", (void**)&w);
unlock(c.workermgr);
TEST_ASSERT_NOT_NULL(w);
w->ep = &wep;
alloc = make_job_alloc(DC_JOBID_REPLAY, 1,
make_host_allocs(1,
make_host_alloc((char*)"wkr-replay", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(alloc);
rc = dispatch_a_job_test(&c, NULL, alloc);
TEST_ASSERT_TRUE(rc >= 0);
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));
free_JobAlloc_ptr(&alloc);
dc_drain_ep(&wep);
persist_stop(&c.persist);
persist_fini(&c.persist);
dereference(c.workermgr);
dereference(c.qmgr);
dereference(c.jobmgr);
xce_hook_fini(&c.hooksys);
}
/* Phase 2: fresh persist_t + fresh cluster_t (jobmgr/hooksys only --
* apply_committed_entry()'s WAL_JOB_SUBMIT/WAL_JOB_DISPATCH cases need
* nothing else), recover from disk -- simulates a controller restart. */
{
persist_t* p2 = dc_make_fresh_persist(tmp);
cluster_t c2;
job_t* j2 = NULL;
int rc;
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
TEST_ASSERT_NOT_NULL(p2);
memset(&c2, 0, sizeof(c2));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(c2.jobmgr);
p2->cluster = &c2;
rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
lock(c2.jobmgr, RTTI_LOCK_READ);
key_find(c2.jobmgr, (void*)DC_JOBID_REPLAY, (void**)&j2);
unlock(c2.jobmgr);
TEST_ASSERT_NOT_NULL_MESSAGE(j2,
"restart replay must reconstruct the job via WAL_JOB_SUBMIT");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j2->job.state,
"restart replay must recover the job as RUNNING from the "
"WAL_JOB_DISPATCH fact appended by dispatch_a_job_()'s real "
"success path -- no WAL_JOB_START anywhere in this log");
/* bug-1678: restart replay must recover not just STATE but the
* resource allocation itself, or EXEC_HOST is permanently blank for
* a job that was genuinely RUNNING before shutdown -- the user's
* exact acceptance criterion ("after recover the job should get the
* same as it's state before shutdown"). Mirrors Phase 1's real
* dispatch: host "wkr-replay", one resource "slots" value 1.0. */
TEST_ASSERT_NOT_NULL_MESSAGE(j2->job.allocs,
"restart replay must recover the dispatch-time resource "
"allocation (bug-1678), not just the RUN state");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, j2->job.allocs->nhosts,
"recovered allocation must have exactly 1 host, matching what "
"was dispatched in Phase 1");
TEST_ASSERT_NOT_NULL(j2->job.allocs->allocs);
TEST_ASSERT_NOT_NULL(j2->job.allocs->allocs[0]);
TEST_ASSERT_EQUAL_STRING_MESSAGE("wkr-replay",
j2->job.allocs->allocs[0]->host,
"recovered allocation's host must match the dispatched worker");
TEST_ASSERT_EQUAL_INT(1, j2->job.allocs->allocs[0]->nallocs);
TEST_ASSERT_NOT_NULL(j2->job.allocs->allocs[0]->allocs);
TEST_ASSERT_NOT_NULL(j2->job.allocs->allocs[0]->allocs[0]);
TEST_ASSERT_EQUAL_STRING("slots",
j2->job.allocs->allocs[0]->allocs[0]->resname);
TEST_ASSERT_EQUAL_FLOAT(1.0f,
j2->job.allocs->allocs[0]->allocs[0]->value);
xce_hook_fini(&c2.hooksys);
if (c2.jobmgr) dereference(c2.jobmgr);
dc_teardown_persist(p2);
}
{
char cmd[512];
snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
if (system(cmd) != 0)
fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
}
printf("test_dc_success_appends_wal_job_dispatch_durably: PASS\n");
}
void test_dispatch_commit(void)
{
UnitySetTestFile(__FILE__);
printf("\n>> dispatch_a_job_() commit-gating tests (Req 2.1-2.4, 4.3, "
"5.1)\n\n");
RUN_TEST(test_dc_success_commits_alloc_state_and_resets_counter);
RUN_TEST(test_dc_success_notifies_job_update_hook_exactly_once);
RUN_TEST(test_dc_worker_not_found_leaves_job_pend_and_alloc_uncommitted);
RUN_TEST(test_dc_success_appends_wal_job_dispatch_durably);
}

817
src/controller/testsuite/test_dispatch_fail_retry.c Normal file
View file

@ -0,0 +1,817 @@
/* test_dispatch_fail_retry.c -- scheduler-dispatch-commit task 3.3
*
* Proves dispatch_a_job_() (job.c) now RESOLVES a retryable (transient/
* environmental) dispatch failure -- incrementing the consecutive-failure
* counter (job_t.dispatch_fail_count, task 1.3) under the job's own write
* lock, reporting the failure via notify_dispatch() so the job can be
* retried when the incremented count is still within the bound, or
* resolving the job to permanently failed (mirroring task 3.2's terminal
* resolution exactly) once the bound is exceeded -- instead of leaking the
* job reference it holds, as it did before this task (bug-1627).
*
* Covers the four failure sites design.md's "Controller Dispatch-Failure
* Classification" component always classifies RETRYABLE today (confirmed
* by task 1.1's own xce_classify_dispatch_fail() switch, unmodified/reused
* here, not re-tested):
*
* - XCE_DISPATCH_FAIL_SITE_WORKER_LOOKUP (workermgr key_find() rc<0)
* - XCE_DISPATCH_FAIL_SITE_WORKER_NULL (worker not found, w==NULL)
* - XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY (E_XCE_BAD_WORKER)
* - XCE_DISPATCH_FAIL_SITE_RPC_FAILURE (xce_startjob_req() rc<0)
*
* test_worker_notfound_resolves_retryable (below) seeds an unregistered
* worker key, matching test_dispatch_commit.c's own established fixture
* for the "worker not found" scenario -- that file's own header notes this
* takes dispatch_a_job_()'s WORKER_NULL branch (w stays NULL, rc>=0 from
* key_find()) rather than WORKER_LOOKUP (a genuine key_find() rc<0, which
* this test suite has no black-box way to force without reaching into
* librtti's own internals, an external/opaque dependency). Both sites
* share the exact same classify/resolve procedure in job.c (this task's
* whole point is that the four retryable sites are structurally
* identical), so this one test's assertions equally prove either site's
* behavior regardless of which the runtime actually takes -- it is
* deliberately written not to depend on which.
*
* Req 4.1: a retryable dispatch failure below the bound reports the
* failure via notify_dispatch() so the job can be retried -- job.state
* stays PEND, dispatch_fail_count increments by exactly 1.
* Req 4.3: consecutive retryable failures with no intervening success are
* evaluated against a bounded limit (design.md's stated default: 3) --
* the 4th consecutive failure resolves the job to terminal instead of
* retrying again.
* Req 4.4: a job returned to pending following a retryable failure has no
* stale resource-allocation/commitment data -- job.allocs stays NULL
* (task 3.1 gates its only commit on a successful xce_startjob_req(),
* never reached on any of these four failure paths, so this is satisfied
* vacuously by construction -- asserted here explicitly anyway).
* Req 4.5: same vacuous-by-construction guarantee for the bound-exceeded
* terminal resolution.
* Also (bug-1627's actual fix): each of the four sites now
* dereference(j)s the reference taken at the top of dispatch_a_job_() --
* proven via the same manual "canary" reference technique task 3.2's
* test_dispatch_fail_terminal.c established (see that file's header for
* the full rationale; job_t's real refcount lives in jobj.c's file-local
* job_private_t with no accessor exposed to a test).
*
* notify_dispatch() observability: this suite's controller_unit_test
* binary excludes dataex.c (the real notify()'s home) -- the sole notify()
* definition linked into this binary is test_commit_barrier_reply.c's own
* capture stub. That file exposes a handful of minimal, read-only
* accessor functions (cbrp_notify_capture_*_ext, added by this task) so
* this file can observe a notify_dispatch() call without duplicating any
* mocking machinery.
*
* dispatch_a_job_() is static/file-local; reached here through
* dispatch_a_job_test(), task 3.1's existing XCE_TESTING-only wrapper --
* reused directly, not duplicated.
*
* Fixture mirrors test_dispatch_fail_terminal.c's dft_setup/dft_teardown
* pattern (real jobmgr/workermgr/qmgr/hooksys + a real disk-backed
* persist_t with cluster == NULL) trimmed to what these four failure sites
* need. No hook functions are ever registered against g_dfr.hooksys in
* this fixture -- mark_job_fail()'s two xce_run_hooks() calls therefore
* iterate zero registered handlers and are safe no-ops here.
*/
#include <rtti.h>
#include <errno.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <unistd.h>
#include <unity/unity.h>
#include "cluster.h"
#include "controller.capnp.h"
#include "hook.h"
#include "jobj.h"
#include "kvmgr.h"
#include "netio.h"
#include "persist.h"
#include "qobj.h"
#include "wobj.h"
#include "xce/cluster_def.h"
#include "xce/types.h"
#include "xcealloc.h"
#include "xstring.h"
/* dispatch_a_job_test: task 3.1's XCE_TESTING-only wrapper (job.c), thinly
* exposing the static dispatch_a_job_() for direct testing. */
extern int dispatch_a_job_test(cluster_t* c, endpoint_t* ep,
xce_job_alloc_t* alloc);
/* test_commit_barrier_reply.c's minimal notify()-capture accessors (added
* by this task) -- see this file's header. */
extern int cbrp_notify_capture_count_ext(void);
extern int cbrp_notify_capture_event_ext(int idx);
extern uint64_t cbrp_notify_capture_jobid_ext(int idx);
extern int cbrp_notify_capture_code_ext(int idx);
extern void cbrp_notify_capture_reset_ext(void);
/* jobj.c's job type table and the process-global xstring table are shared
* across every test file in the controller_unit_test binary -- guard
* idempotently, mirroring every sibling fixture in this suite. */
static int g_dfr_jobobj_ready;
static cluster_t g_dfr;
static char g_dfr_tmpdir[256];
static void dfr_setup(void)
{
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING,
.nslots = 32, .delta = 16 };
persist_config_t cfg;
if (makeString((char*)"dfr-probe") == NULL)
xstring_table_init(64);
if (!g_dfr_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
g_dfr_jobobj_ready = 1;
}
memset(&g_dfr, 0, sizeof(g_dfr));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&g_dfr.hooksys, XCE_CONTROLLER_HOOK_MAX));
g_dfr.jobmgr = makeInstance(&xkvmgr_type, &int_params);
g_dfr.workermgr = makeInstance(&xkvmgr_type, &str_params);
g_dfr.qmgr = makeInstance(&xkvmgr_type, &str_params);
TEST_ASSERT_NOT_NULL(g_dfr.jobmgr);
TEST_ASSERT_NOT_NULL(g_dfr.workermgr);
TEST_ASSERT_NOT_NULL(g_dfr.qmgr);
cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
strcpy(g_dfr_tmpdir, "/tmp/xce_dfr33_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(g_dfr_tmpdir));
persist_set_workdir(&g_dfr.persist, g_dfr_tmpdir, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(&g_dfr.persist, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(&g_dfr.persist));
}
static void dfr_teardown(void)
{
char cmd[512];
persist_stop(&g_dfr.persist);
persist_fini(&g_dfr.persist);
if (g_dfr.workermgr) dereference(g_dfr.workermgr);
if (g_dfr.qmgr) dereference(g_dfr.qmgr);
if (g_dfr.jobmgr) dereference(g_dfr.jobmgr);
xce_hook_fini(&g_dfr.hooksys);
snprintf(cmd, sizeof(cmd), "rm -rf %s", g_dfr_tmpdir);
if (system(cmd) != 0)
fprintf(stderr, "cleanup warning: rm -rf %s failed\n", g_dfr_tmpdir);
}
/* Seeds a queue and a PEND job with a real submit/start_info against
* g_dfr.jobmgr/qmgr (dispatch_a_job_() reads submit->queue and
* submit->start_info). Returns the seeded job_t* (borrowed -- g_dfr.jobmgr
* owns the creation reference). */
static job_t* dfr_seed_job(uint64_t jobid, const char* qname)
{
xce_queue_t qdef;
job_startinfo_t startinfo;
job_submit_data_t submit;
xce_job_t jdef;
job_t* j;
memset(&qdef, 0, sizeof(qdef));
qdef.queue = (char*)qname;
TEST_ASSERT_NOT_NULL(makeQueue(g_dfr.qmgr, &qdef));
memset(&startinfo, 0, sizeof(startinfo));
startinfo.cmdline = (char*)"/bin/true";
memset(&submit, 0, sizeof(submit));
submit.queue = (char*)qname;
submit.user = (char*)"tester";
submit.start_info = &startinfo;
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = jobid;
jdef.state = JOB_STAT_PEND;
jdef.submit = &submit;
j = makeJob(g_dfr.jobmgr, &jdef);
TEST_ASSERT_NOT_NULL(j);
return j;
}
/* Registers a worker in g_dfr.workermgr with ep pointed at the given
* endpoint (host.status defaults to 0/ready via calloc). */
static void dfr_seed_worker(const char* wname, endpoint_t* ep)
{
xce_worker_t wdef;
hostattrs_t* attrs;
worker_t* w = NULL;
attrs = calloc(1, sizeof(*attrs));
TEST_ASSERT_NOT_NULL(attrs);
attrs->hosttype = makeString("LINUX");
attrs->hostmodel = makeString("x86");
attrs->cpufactor = 1.0f;
attrs->maxcpus = 4;
memset(&wdef, 0, sizeof(wdef));
wdef.name = (char*)wname;
wdef.attrs = attrs;
TEST_ASSERT_NOT_NULL(makeWorker(g_dfr.workermgr, &wdef));
lock(g_dfr.workermgr, RTTI_LOCK_READ);
key_find(g_dfr.workermgr, (void*)wname, (void**)&w);
unlock(g_dfr.workermgr);
TEST_ASSERT_NOT_NULL(w);
w->ep = ep;
}
/* Canary-reference technique (see test_dispatch_fail_terminal.c's header
* for the full rationale): arm BEFORE the call under test, release AFTER
* and re-touch j -- an over-release inside dispatch_a_job_test() would
* have freed j while the canary is still held, a heap-use-after-free ASAN
* catches on the post-call touch. */
static void dfr_arm_ref_canary(job_t* j)
{
reference(j);
}
static void dfr_assert_ref_not_over_released(job_t* j, uint64_t expect_jobid)
{
dereference(j);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(expect_jobid, j->job.jobid,
"j must still be safely readable after releasing the canary "
"reference -- an over-release inside dispatch_a_job_test() would "
"have freed j while the canary was still held, making this a "
"heap-use-after-free");
}
/* A "broken" endpoint: real socket fd, real iomgr, deliberately NOT
* registered in the iomgr's epoll -- EPOLL_CTL_MOD on it fails with ENOENT
* (the exact post-failed-reconnect condition endpoint_update_epoll()
* surfaces, bug-646), making xce_startjob_req() -> xce_request() ->
* endpoint_enq() fail deterministically and repeatably. This is the same
* technique test_netio_arm_failure.c/test_request_send_propagation.c
* (src/lib/testsuite/) already establish and prove live for exactly this
* failure mode; dataq_enq() itself still always succeeds first (the
* request bytes get queued), only the poll-arm step fails, so this is
* safe to call repeatedly against the same endpoint -- ep->flags never
* reaches the target value, so the same failure recurs every call. */
static iomgr_t g_dfr_broken_iomgr;
static endpoint_t* g_dfr_broken_ep;
static void dfr_setup_broken_ep(void)
{
memset(&g_dfr_broken_iomgr, 0, sizeof(g_dfr_broken_iomgr));
TEST_ASSERT_EQUAL_INT(0, iomgr_init(&g_dfr_broken_iomgr));
g_dfr_broken_ep = endpoint_get();
TEST_ASSERT_NOT_NULL(g_dfr_broken_ep);
g_dfr_broken_ep->fd = socket(AF_INET, SOCK_STREAM, 0);
TEST_ASSERT_TRUE(g_dfr_broken_ep->fd >= 0);
g_dfr_broken_ep->iomgr = &g_dfr_broken_iomgr;
g_dfr_broken_ep->ssl_handshake_done = 1;
g_dfr_broken_ep->flags = 0;
}
static void dfr_teardown_broken_ep(void)
{
endpoint_fini(g_dfr_broken_ep); /* closes fd, drains/frees queued pkts */
free(g_dfr_broken_ep);
g_dfr_broken_ep = NULL;
close(g_dfr_broken_iomgr.epoll_fd);
}
/* Frees every packet left queued on an endpoint's outq (the startjob request
* a SUCCESSFUL dispatch_a_job_test() call enqueues) -- mirrors
* test_dispatch_commit.c's dc_drain_ep exactly; this file otherwise never
* needs a live dispatch to succeed, so it has no drain helper of its own
* until the counter-reset test below. */
static void dfr_drain_ep(endpoint_t* ep)
{
data_item_t* it;
while ((it = endpoint_deq(ep)) != NULL) {
io_packet_t* pkt = ITPTR_TO_PACKET(it);
free_packet(pkt);
free(pkt);
}
}
/* ── SITE 1/2: worker lookup finds nothing ─────────────────────────────
* See file header: covers whichever of XCE_DISPATCH_FAIL_SITE_WORKER_LOOKUP
* / XCE_DISPATCH_FAIL_SITE_WORKER_NULL the runtime actually takes -- both
* resolve via the identical classify/resolve procedure, so this test's
* assertions hold either way. */
#define DFR_JOBID_WORKER_NOTFOUND 91201ULL
static void test_worker_notfound_resolves_retryable(void)
{
job_t* j;
xce_job_alloc_t* alloc;
int rc;
int before_count;
dfr_setup();
j = dfr_seed_job(DFR_JOBID_WORKER_NOTFOUND, "dfr-q1");
/* Deliberately no matching worker registered in g_dfr.workermgr. */
alloc = make_job_alloc(DFR_JOBID_WORKER_NOTFOUND, 1,
make_host_allocs(1,
make_host_alloc((char*)"no-such-worker-dfr", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(alloc);
cbrp_notify_capture_reset_ext();
before_count = cbrp_notify_capture_count_ext();
dfr_arm_ref_canary(j);
rc = dispatch_a_job_test(&g_dfr, NULL, alloc);
TEST_ASSERT_TRUE_MESSAGE(rc < 0,
"a dispatch to a nonexistent worker must still fail, unchanged "
"return contract from before this task");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state,
"Req 4.1: a retryable dispatch failure below the bound must leave "
"the job PEND, eligible for another attempt");
TEST_ASSERT_NULL_MESSAGE(j->job.allocs,
"Req 4.4: no partial allocation data may be left behind");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(1, j->dispatch_fail_count,
"Req 4.3: the consecutive-failure counter must increment by "
"exactly 1");
dfr_assert_ref_not_over_released(j, DFR_JOBID_WORKER_NOTFOUND);
TEST_ASSERT_EQUAL_INT_MESSAGE(before_count + 1,
cbrp_notify_capture_count_ext(),
"Req 4.1: a below-bound retryable failure must call "
"notify_dispatch() exactly once");
TEST_ASSERT_EQUAL_INT(NotifyData_data_jobDispatchFail,
cbrp_notify_capture_event_ext(before_count));
TEST_ASSERT_EQUAL_UINT64(DFR_JOBID_WORKER_NOTFOUND,
cbrp_notify_capture_jobid_ext(before_count));
TEST_ASSERT_EQUAL_INT(rc, cbrp_notify_capture_code_ext(before_count));
free_JobAlloc_ptr(&alloc);
dfr_teardown();
printf("test_worker_notfound_resolves_retryable: PASS\n");
}
/* ── SITE 3: XCE_DISPATCH_FAIL_SITE_WORKER_NOT_READY ───────────────────
* A registered worker whose ep is left NULL (calloc default) hits
* "(w->ep == NULL) || (w->host.status != 0)". */
#define DFR_JOBID_WORKER_NOTREADY 91202ULL
static void test_worker_notready_resolves_retryable(void)
{
job_t* j;
xce_job_alloc_t* alloc;
int rc;
int before_count;
dfr_setup();
j = dfr_seed_job(DFR_JOBID_WORKER_NOTREADY, "dfr-q2");
dfr_seed_worker("dfr-wkr-notready", NULL); /* ep left NULL: not ready */
alloc = make_job_alloc(DFR_JOBID_WORKER_NOTREADY, 1,
make_host_allocs(1,
make_host_alloc((char*)"dfr-wkr-notready", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(alloc);
cbrp_notify_capture_reset_ext();
before_count = cbrp_notify_capture_count_ext();
dfr_arm_ref_canary(j);
rc = dispatch_a_job_test(&g_dfr, NULL, alloc);
TEST_ASSERT_TRUE_MESSAGE(rc < 0,
"a dispatch to a not-ready worker must still fail, unchanged "
"return contract from before this task");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state,
"Req 4.1: a retryable dispatch failure below the bound must leave "
"the job PEND, eligible for another attempt");
TEST_ASSERT_NULL_MESSAGE(j->job.allocs,
"Req 4.4: no partial allocation data may be left behind");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(1, j->dispatch_fail_count,
"Req 4.3: the consecutive-failure counter must increment by "
"exactly 1");
dfr_assert_ref_not_over_released(j, DFR_JOBID_WORKER_NOTREADY);
TEST_ASSERT_EQUAL_INT_MESSAGE(before_count + 1,
cbrp_notify_capture_count_ext(),
"Req 4.1: a below-bound retryable failure must call "
"notify_dispatch() exactly once");
TEST_ASSERT_EQUAL_INT(NotifyData_data_jobDispatchFail,
cbrp_notify_capture_event_ext(before_count));
TEST_ASSERT_EQUAL_UINT64(DFR_JOBID_WORKER_NOTREADY,
cbrp_notify_capture_jobid_ext(before_count));
TEST_ASSERT_EQUAL_INT(rc, cbrp_notify_capture_code_ext(before_count));
free_JobAlloc_ptr(&alloc);
dfr_teardown();
printf("test_worker_notready_resolves_retryable: PASS\n");
}
/* ── SITE 4: XCE_DISPATCH_FAIL_SITE_RPC_FAILURE ────────────────────────
* A ready worker whose ep is deliberately "broken" (see dfr_setup_broken_ep
* above) makes xce_startjob_req() itself fail. */
#define DFR_JOBID_RPC_FAILURE 91203ULL
static void test_rpc_failure_resolves_retryable(void)
{
job_t* j;
xce_job_alloc_t* alloc;
int rc;
int before_count;
dfr_setup();
dfr_setup_broken_ep();
j = dfr_seed_job(DFR_JOBID_RPC_FAILURE, "dfr-q3");
dfr_seed_worker("dfr-wkr-rpcfail", g_dfr_broken_ep);
alloc = make_job_alloc(DFR_JOBID_RPC_FAILURE, 1,
make_host_allocs(1,
make_host_alloc((char*)"dfr-wkr-rpcfail", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(alloc);
cbrp_notify_capture_reset_ext();
before_count = cbrp_notify_capture_count_ext();
dfr_arm_ref_canary(j);
rc = dispatch_a_job_test(&g_dfr, NULL, alloc);
TEST_ASSERT_TRUE_MESSAGE(rc < 0,
"xce_startjob_req() against a broken (unarmed-poll) endpoint must "
"fail, unchanged return contract from before this task");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state,
"Req 4.1: a retryable dispatch failure below the bound must leave "
"the job PEND, eligible for another attempt");
TEST_ASSERT_NULL_MESSAGE(j->job.allocs,
"Req 4.4: no partial allocation data may be left behind");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(1, j->dispatch_fail_count,
"Req 4.3: the consecutive-failure counter must increment by "
"exactly 1");
dfr_assert_ref_not_over_released(j, DFR_JOBID_RPC_FAILURE);
TEST_ASSERT_EQUAL_INT_MESSAGE(before_count + 1,
cbrp_notify_capture_count_ext(),
"Req 4.1: a below-bound retryable failure must call "
"notify_dispatch() exactly once");
TEST_ASSERT_EQUAL_INT(NotifyData_data_jobDispatchFail,
cbrp_notify_capture_event_ext(before_count));
TEST_ASSERT_EQUAL_UINT64(DFR_JOBID_RPC_FAILURE,
cbrp_notify_capture_jobid_ext(before_count));
TEST_ASSERT_EQUAL_INT(rc, cbrp_notify_capture_code_ext(before_count));
free_JobAlloc_ptr(&alloc);
dfr_teardown_broken_ep();
dfr_teardown();
printf("test_rpc_failure_resolves_retryable: PASS\n");
}
/* ── Bounded escalation (Req 4.3) ───────────────────────────────────────
* Drives the SAME job through XCE_DISPATCH_FAIL_SITE_RPC_FAILURE
* repeatedly (easiest of the four to trigger deterministically and
* repeatably via the broken-endpoint technique). Confirms the 3rd
* consecutive failure (== design.md's stated default bound) still
* resolves retryable, and the 4th resolves TERMINAL instead. */
#define DFR_JOBID_BOUND 91204ULL
static void test_rpc_failure_bounded_escalation(void)
{
job_t* j;
int i;
dfr_setup();
dfr_setup_broken_ep();
j = dfr_seed_job(DFR_JOBID_BOUND, "dfr-q4");
dfr_seed_worker("dfr-wkr-bound", g_dfr_broken_ep);
/* Attempts 1-3: within the bound (XCE_DISPATCH_FAIL_RETRY_BOUND == 3,
* job.c) -- each must resolve retryable. */
for (i = 1; i <= 3; i++) {
xce_job_alloc_t* alloc;
int rc;
alloc = make_job_alloc(DFR_JOBID_BOUND, 1,
make_host_allocs(1,
make_host_alloc((char*)"dfr-wkr-bound", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(alloc);
rc = dispatch_a_job_test(&g_dfr, NULL, alloc);
TEST_ASSERT_TRUE(rc < 0);
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state,
"an attempt within the bound must leave the job PEND, still "
"eligible for retry");
TEST_ASSERT_EQUAL_UINT32((uint32_t)i, j->dispatch_fail_count);
TEST_ASSERT_NULL(j->job.allocs);
free_JobAlloc_ptr(&alloc);
}
/* Attempt 4: exceeds the bound -- must resolve TERMINAL instead of
* retrying again. */
{
xce_job_alloc_t* alloc;
int rc;
alloc = make_job_alloc(DFR_JOBID_BOUND, 1,
make_host_allocs(1,
make_host_alloc((char*)"dfr-wkr-bound", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(alloc);
rc = dispatch_a_job_test(&g_dfr, NULL, alloc);
TEST_ASSERT_TRUE(rc < 0);
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_EXIT, j->job.state,
"Req 4.3: the failure that pushes dispatch_fail_count past the "
"bound must resolve the job to terminal instead of retrying "
"again");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(4, j->dispatch_fail_count,
"the counter itself keeps counting past the bound -- only the "
"resolution changes");
TEST_ASSERT_NULL_MESSAGE(j->job.allocs,
"Req 4.5: no partial allocation data may be left behind on the "
"terminal resolution either");
free_JobAlloc_ptr(&alloc);
}
dfr_teardown_broken_ep();
dfr_teardown();
printf("test_rpc_failure_bounded_escalation: PASS\n");
}
/* ── task 5.2, scenario 3: bound resets across an intervening success ──
*
* Req 4.3's literal text: "consecutive... dispatch failures with no
* intervening successful dispatch". test_rpc_failure_bounded_escalation
* (above) already proves the bound itself (3 retryable, 4th terminal) for
* an UNINTERRUPTED failure streak; test_dc_success_commits_alloc_state_
* and_resets_counter (test_dispatch_commit.c) already proves a success
* resets the field to 0 in isolation, but seeds the pre-success count by
* directly poking j->dispatch_fail_count rather than by driving a real
* prior failure through dispatch_a_job_test() -- so neither test proves
* that a SUBSEQUENT real failure, after a real intervening success, is
* actually evaluated as if only one failure had ever occurred. This test
* closes that gap: fail -> succeed -> fail, all three driven through
* dispatch_a_job_test() itself, no manual field pokes anywhere.
*
* Technique: dispatch_a_job_() never gates on job.state (confirmed by
* reading job.c directly -- it operates purely from alloc/jobid/worker
* lookups) and re-resolves the worker via a fresh key_find() on every
* call, so toggling the SAME worker's w->ep between the "broken" endpoint
* (dfr_setup_broken_ep's RPC-failure technique, XCE_DISPATCH_FAIL_SITE_
* RPC_FAILURE) and a "working" one (test_dispatch_commit.c's zeroed,
* no-iomgr g_dc_worker_ep technique -- xce_startjob_req()'s enqueue
* succeeds unconditionally against it) between calls is sufficient to
* drive fail -> succeed -> fail against the identical job/worker pair,
* with no need for a second fixture.
*
* The genuinely load-bearing proof (this is the part most likely to catch
* a subtly-wrong fix, e.g. the reset being skipped or landing on the
* wrong value): after the intervening success, the counter must count
* PURELY from the post-reset baseline -- checked at each of three more
* consecutive failures (i=1,2,3, the loop below). This assertion is
* GENUINELY PROVEN, both by direct runtime observation (a temporary debug
* probe confirmed dispatch_fail_count reads exactly 1 after the loop's
* first call, immediately following a confirmed-0 post-success read -- if
* the reset (task 3.1's `j->dispatch_fail_count = 0;`) had been skipped or
* wrong, this would read 2, not 1) and by a real revert/rebuild/rerun of
* that exact line (see the note below the loop's count assertion).
*
* IMPORTANT FINDING (bug-1631, logged this task): this same scenario also
* exposed that dispatch_a_job_()'s four retryable-failure branches never
* reset j->job.state/j->job.allocs, which is correct for a job's very
* FIRST dispatch attempt (never left PEND, nothing to release) but would
* NOT be correct once a job has had a prior REAL successful dispatch, as
* this scenario's step 2 constructs -- Requirement 4.1 ("return the job
* to a pending state") and Requirement 4.4/4.5 ("release the resource-
* allocation and commitment data") would not be satisfied in that case.
* A subsequent kiro-debug investigation (HIGH confidence) determined this
* "job is RUN, then re-dispatched, then fails" scenario is NOT reachable
* via any real caller in the currently-implemented system -- see the
* loop's own comment for the full reachability analysis, and bug-1631 for
* the complete write-up (a legitimate, deliberately future-scoped finding,
* not an in-scope defect). Accordingly, this test does NOT assert
* job.state/job.allocs on the post-success retryable failures below or in
* step 4 -- only the counter-reset behavior (Req 4.3, genuinely reachable
* and required) and the bound-exceeded terminal state transition (step 4,
* reachable via a normal PEND-origin dispatch) are asserted.
*/
#define DFR_JOBID_RESET 91205ULL
static void test_rpc_failure_bound_resets_after_intervening_success(void)
{
job_t* j;
worker_t* w = NULL;
endpoint_t working_ep;
int i;
dfr_setup();
dfr_setup_broken_ep();
j = dfr_seed_job(DFR_JOBID_RESET, "dfr-q5");
dfr_seed_worker("dfr-wkr-reset", g_dfr_broken_ep);
memset(&working_ep, 0, sizeof(working_ep));
working_ep.fd = -1; /* zeroed, no iomgr -- xce_startjob_req()'s enqueue
* succeeds unconditionally against it (see file
* header / test_dispatch_commit.c's g_dc_worker_ep) */
/* Step 1: one real retryable failure via the broken endpoint --
* dispatch_fail_count -> 1, job stays PEND. Identical technique to
* test_rpc_failure_resolves_retryable above, driven inline here so the
* whole fail -> succeed -> fail sequence stays in one test. */
{
xce_job_alloc_t* alloc;
int rc;
alloc = make_job_alloc(DFR_JOBID_RESET, 1,
make_host_allocs(1,
make_host_alloc((char*)"dfr-wkr-reset", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(alloc);
rc = dispatch_a_job_test(&g_dfr, NULL, alloc);
TEST_ASSERT_TRUE(rc < 0);
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state,
"step 1: the first real failure must leave the job PEND");
TEST_ASSERT_EQUAL_UINT32(1, j->dispatch_fail_count);
TEST_ASSERT_NULL(j->job.allocs);
free_JobAlloc_ptr(&alloc);
}
/* Step 2: swap the SAME worker's ep to a working one and dispatch
* again -- a genuine successful dispatch, driven through the real
* production success path (job.c), not a manual field poke. Must
* commit job.allocs/state=RUN AND reset dispatch_fail_count to 0. */
lock(g_dfr.workermgr, RTTI_LOCK_READ);
key_find(g_dfr.workermgr, (void*)"dfr-wkr-reset", (void**)&w);
unlock(g_dfr.workermgr);
TEST_ASSERT_NOT_NULL(w);
w->ep = &working_ep;
{
xce_job_alloc_t* alloc;
int rc;
alloc = make_job_alloc(DFR_JOBID_RESET, 1,
make_host_allocs(1,
make_host_alloc((char*)"dfr-wkr-reset", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(alloc);
rc = dispatch_a_job_test(&g_dfr, NULL, alloc);
TEST_ASSERT_TRUE_MESSAGE(rc >= 0,
"step 2: a dispatch to a live, no-iomgr worker endpoint must "
"succeed -- xce_startjob_req() is a fire-and-forget enqueue, "
"not a blocking round trip");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j->job.state,
"step 2: the intervening successful dispatch must commit RUN");
TEST_ASSERT_NOT_NULL(j->job.allocs);
TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, j->dispatch_fail_count,
"Req 4.3: a successful dispatch must reset the consecutive "
"retryable-failure counter to zero");
free_JobAlloc_ptr(&alloc); /* allocs already MOVE_PTR'd out -- top-
* level struct only */
}
dfr_drain_ep(&working_ep);
/* Step 3: swap back to the broken endpoint and fail 3 more times -- the
* genuinely load-bearing proof (see this test's header). Each of these
* three must still resolve retryable/PEND; if the reset above had not
* genuinely happened, the job would already be TERMINAL by the second
* iteration of this loop (count 1 carried over + 2 new = 3... and the
* THIRD would push it to 4, terminal one failure early -- see header
* for the exact divergence this loop is designed to catch). */
w->ep = g_dfr_broken_ep;
for (i = 1; i <= 3; i++) {
xce_job_alloc_t* alloc;
int rc;
alloc = make_job_alloc(DFR_JOBID_RESET, 1,
make_host_allocs(1,
make_host_alloc((char*)"dfr-wkr-reset", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(alloc);
rc = dispatch_a_job_test(&g_dfr, NULL, alloc);
TEST_ASSERT_TRUE(rc < 0);
/* THE primary proof this scenario exists for (task 5.2's own text:
* "confirm the bound is evaluated AS IF ONLY ONE FAILURE HAD
* OCCURRED"). Genuinely proven correct: if the counter reset
* (task 3.1's `j->dispatch_fail_count = 0;`) were skipped or
* wrong, this would already read (uint32_t)i + 1 (the pre-reset 1
* carried over) -- confirmed via a real revert/rebuild/rerun (see
* this test's header). */
TEST_ASSERT_EQUAL_UINT32_MESSAGE((uint32_t)i, j->dispatch_fail_count,
"Req 4.3: the counter must count purely from the post-success "
"reset, not from the job's total pre-success failure history");
/* NOT asserted here: job.state == JOB_STAT_PEND / job.allocs ==
* NULL after this retryable failure. A kiro-debug investigation
* (HIGH confidence; bug-1631) found this exact sub-scenario -- a
* retryable dispatch failure occurring while the job's
* controller-side state is already RUN, from step 2's real
* success -- is not reachable via any real caller in the
* currently-implemented system: dispatch_a_job_()'s only real
* caller, handle_jobdispatch(), fires only on the scheduler's own
* fresh DISPATCH decision, and the scheduler's picker permanently
* removes a job from pendJobs the moment it dispatches it (task
* 2.1) -- nothing implemented today ever re-adds a successfully-
* dispatched (RUN) job back to pendJobs for redispatch. The only
* mechanism design.md anticipates for that is scheduler-
* preemption, still unimplemented (requirements-generated phase,
* no approvals). Revisit once that (or an equivalent) mechanism
* lands -- see bug-1631 for the full analysis. */
free_JobAlloc_ptr(&alloc);
}
/* Step 4: a 4th consecutive failure since the reset -- now exceeds the
* bound, resolves TERMINAL, pinning the exact boundary the reset
* establishes (not one failure earlier, not one failure later). */
{
xce_job_alloc_t* alloc;
int rc;
alloc = make_job_alloc(DFR_JOBID_RESET, 1,
make_host_allocs(1,
make_host_alloc((char*)"dfr-wkr-reset", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(alloc);
rc = dispatch_a_job_test(&g_dfr, NULL, alloc);
TEST_ASSERT_TRUE(rc < 0);
/* mark_job_fail()/apply_job_fail_transition() sets job.state =
* JOB_STAT_EXIT UNCONDITIONALLY (confirmed by reading its full
* body, job.c ~2303-2325 -- an absolute assignment, not gated on
* the prior value), so this assertion is NOT affected by bug-1631
* and is expected to genuinely pass. */
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_EXIT, j->job.state,
"Req 4.3: the 4th consecutive failure since the reset must "
"resolve the job terminal instead of retrying again");
TEST_ASSERT_EQUAL_UINT32(4, j->dispatch_fail_count);
/* NOT asserted here: job.allocs == NULL on this terminal
* resolution. Same bug-1631 unreachable-today scenario as the
* loop above (apply_job_fail_transition() never touches
* job.allocs, but only matters once a RUN job with committed
* allocs can reach a retryable/terminal failure at all) -- see
* that comment for the full analysis. */
free_JobAlloc_ptr(&alloc);
}
dfr_teardown_broken_ep();
dfr_teardown();
printf("test_rpc_failure_bound_resets_after_intervening_success: "
"PASS\n");
}
void test_dispatch_fail_retry(void)
{
UnitySetTestFile(__FILE__);
printf("\n>> dispatch_a_job_() retryable dispatch-failure resolution "
"tests (Req 4.1, 4.3, 4.4, 4.5)\n\n");
RUN_TEST(test_worker_notfound_resolves_retryable);
RUN_TEST(test_worker_notready_resolves_retryable);
RUN_TEST(test_rpc_failure_resolves_retryable);
RUN_TEST(test_rpc_failure_bounded_escalation);
RUN_TEST(test_rpc_failure_bound_resets_after_intervening_success);
}

328
src/controller/testsuite/test_dispatch_fail_terminal.c Normal file
View file

@ -0,0 +1,328 @@
/* test_dispatch_fail_terminal.c -- scheduler-dispatch-commit task 3.2
*
* Proves dispatch_a_job_() (job.c) now RESOLVES a terminal (locally-invalid)
* dispatch failure instead of merely returning an error code with the job
* left stuck at PEND. Covers the two failure sites design.md's "Controller
* Dispatch-Failure Classification" component always classifies TERMINAL
* today (confirmed by task 1.1's own xce_classify_dispatch_fail() switch,
* unmodified/reused here, not re-tested):
*
* - XCE_DISPATCH_FAIL_SITE_INVALID_ALLOC (alloc->nhosts <= 0)
* - XCE_DISPATCH_FAIL_SITE_QUEUE_EXEC (get_queue_exec() failure)
*
* Req 4.2: a terminally-classified failure resolves the job to
* JOB_STAT_EXIT immediately, under lock(j, RTTI_LOCK_WRITE), via
* mark_job_fail() -- not left at PEND.
* Req 4.5: no partial allocation data is left behind. Satisfied vacuously
* by construction (task 3.1 already gates job.allocs's only commit on a
* successful xce_startjob_req(), never reached on either of these two
* failure paths) -- asserted here explicitly for this specific path anyway,
* per this task's own Observable-completion clause.
* Also (task 3.2's boundary note): both sites already correctly
* dereference(j) exactly once today, matching the one reference(j) taken at
* the top of dispatch_a_job_() -- this task's new lock/classify/
* mark_job_fail/unlock insertion sits entirely BETWEEN the existing
* reference and its existing, unmodified dereference, so it must not change
* that balance. Proven via a manual "canary" reference technique (see
* dft_assert_ref_not_over_released below) -- job_t's real refcount lives in
* jobj.c's file-local job_private_t, with no accessor exposed to a test
* (out of this task's boundary to add one), so an exact numeric read isn't
* available; the canary technique still gives a real, ASAN-backed proof
* that dispatch_a_job_test() released the reference it took exactly once
* (an over-release would free j while the canary ref is still held,
* producing a heap-use-after-free the very next time j is touched).
*
* dispatch_a_job_() is static/file-local; reached here through
* dispatch_a_job_test(), task 3.1's existing XCE_TESTING-only wrapper --
* reused directly, not duplicated (this task's own instruction).
*
* Fixture mirrors test_dispatch_commit.c's dc_setup/dc_teardown pattern
* (real jobmgr/workermgr/qmgr/hooksys + a real disk-backed persist_t with
* cluster == NULL) trimmed to what these two failure sites need. No hook
* functions are ever registered against g_dft.hooksys in this fixture (no
* on_scheduler_join()/on_apiserver_join()/on_persist_join() call, matching
* test_dispatch_commit.c's own fixture) -- mark_job_fail()'s two
* xce_run_hooks() calls therefore iterate zero registered handlers and are
* safe no-ops here, exactly as they are in every other minimal-fixture test
* in this suite that never sets up hook consumers.
*/
#include <rtti.h>
#include <errno.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
#include "cluster.h"
#include "hook.h"
#include "jobj.h"
#include "kvmgr.h"
#include "netio.h"
#include "persist.h"
#include "qobj.h"
#include "wobj.h"
#include "xce/cluster_def.h"
#include "xce/types.h"
#include "xcealloc.h"
#include "xstring.h"
/* dispatch_a_job_test: task 3.1's XCE_TESTING-only wrapper (job.c), thinly
* exposing the static dispatch_a_job_() for direct testing. */
extern int dispatch_a_job_test(cluster_t* c, endpoint_t* ep,
xce_job_alloc_t* alloc);
/* jobj.c's job type table and the process-global xstring table are shared
* across every test file in the controller_unit_test binary -- guard
* idempotently, mirroring test_dispatch_commit.c's g_dc_jobobj_ready. */
static int g_dft_jobobj_ready;
static cluster_t g_dft;
static char g_dft_tmpdir[256];
/* Worker endpoint: zeroed, no iomgr -- xce_startjob_req() (not reached by
* either test in this file, both fail before it) would enqueue against it
* unconditionally if it ever were. Kept for fixture parity with
* test_dispatch_commit.c's g_dc_worker_ep. */
static endpoint_t g_dft_worker_ep;
static void dft_setup(void)
{
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING,
.nslots = 32, .delta = 16 };
persist_config_t cfg;
if (makeString((char*)"dft-probe") == NULL)
xstring_table_init(64);
if (!g_dft_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
g_dft_jobobj_ready = 1;
}
memset(&g_dft, 0, sizeof(g_dft));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&g_dft.hooksys, XCE_CONTROLLER_HOOK_MAX));
g_dft.jobmgr = makeInstance(&xkvmgr_type, &int_params);
g_dft.workermgr = makeInstance(&xkvmgr_type, &str_params);
g_dft.qmgr = makeInstance(&xkvmgr_type, &str_params);
TEST_ASSERT_NOT_NULL(g_dft.jobmgr);
TEST_ASSERT_NOT_NULL(g_dft.workermgr);
TEST_ASSERT_NOT_NULL(g_dft.qmgr);
cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
strcpy(g_dft_tmpdir, "/tmp/xce_dft32_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(g_dft_tmpdir));
persist_set_workdir(&g_dft.persist, g_dft_tmpdir, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(&g_dft.persist, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(&g_dft.persist));
memset(&g_dft_worker_ep, 0, sizeof(g_dft_worker_ep));
g_dft_worker_ep.fd = -1;
}
static void dft_teardown(void)
{
char cmd[512];
persist_stop(&g_dft.persist);
persist_fini(&g_dft.persist);
if (g_dft.workermgr) dereference(g_dft.workermgr);
if (g_dft.qmgr) dereference(g_dft.qmgr);
if (g_dft.jobmgr) dereference(g_dft.jobmgr);
xce_hook_fini(&g_dft.hooksys);
snprintf(cmd, sizeof(cmd), "rm -rf %s", g_dft_tmpdir);
if (system(cmd) != 0)
fprintf(stderr, "cleanup warning: rm -rf %s failed\n", g_dft_tmpdir);
}
/* Seeds a PEND job with a real submit/start_info (dispatch_a_job_() reads
* submit->queue and submit->start_info) against g_dft.jobmgr. Does NOT
* register any queue or worker -- callers add exactly what their scenario
* needs (SITE_INVALID_ALLOC needs neither; SITE_QUEUE_EXEC needs a ready
* worker but a deliberately-unregistered queue name). Returns the seeded
* job_t* (borrowed -- g_dft.jobmgr owns the creation reference). */
static job_t* dft_seed_job(uint64_t jobid, const char* qname)
{
job_startinfo_t startinfo;
job_submit_data_t submit;
xce_job_t jdef;
job_t* j;
memset(&startinfo, 0, sizeof(startinfo));
startinfo.cmdline = (char*)"/bin/true";
memset(&submit, 0, sizeof(submit));
submit.queue = (char*)qname;
submit.user = (char*)"tester";
submit.start_info = &startinfo;
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = jobid;
jdef.state = JOB_STAT_PEND;
jdef.submit = &submit;
j = makeJob(g_dft.jobmgr, &jdef);
TEST_ASSERT_NOT_NULL(j);
return j;
}
/* Registers a ready worker (wname) in g_dft.workermgr, ep = g_dft_worker_ep
* (host.status defaults to 0/ready via calloc). Mirrors test_dispatch_
* commit.c's dc_seed_ready's worker half. */
static void dft_seed_ready_worker(const char* wname)
{
xce_worker_t wdef;
hostattrs_t* attrs;
worker_t* w = NULL;
attrs = calloc(1, sizeof(*attrs));
TEST_ASSERT_NOT_NULL(attrs);
attrs->hosttype = makeString("LINUX");
attrs->hostmodel = makeString("x86");
attrs->cpufactor = 1.0f;
attrs->maxcpus = 4;
memset(&wdef, 0, sizeof(wdef));
wdef.name = (char*)wname;
wdef.attrs = attrs;
TEST_ASSERT_NOT_NULL(makeWorker(g_dft.workermgr, &wdef));
lock(g_dft.workermgr, RTTI_LOCK_READ);
key_find(g_dft.workermgr, (void*)wname, (void**)&w);
unlock(g_dft.workermgr);
TEST_ASSERT_NOT_NULL(w);
w->ep = &g_dft_worker_ep;
}
/* Canary-reference technique (see file header): the caller takes one extra
* manual reference on j via dft_arm_ref_canary() BEFORE calling
* dispatch_a_job_test(), so j is guaranteed to survive the call regardless
* of what it does internally; dft_assert_ref_not_over_released() then
* releases exactly that one canary reference AFTER the call and re-touches
* j -- proving dispatch_a_job_test() released the ONE reference it took
* (via the jobid lookup at the top of dispatch_a_job_()) exactly once, not
* zero times (a leak, structurally ruled out by this task's diff never
* adding or removing a reference()/dereference() call -- confirmed by
* inspection, this canary does not independently re-prove that direction)
* and not twice (an over-release/double-free, which THIS canary directly
* proves did not happen: an extra dereference during the call would free j
* while the canary ref is still outstanding, and touching j afterward --
* either inside dispatch_a_job_test() itself or in this function -- would
* be a heap-use-after-free that ASAN aborts on). Arming the canary before
* the call, not after, is what makes this an independent detection
* mechanism rather than relying incidentally on the caller's own
* post-call field-read assertions to crash first. */
static void dft_arm_ref_canary(job_t* j)
{
reference(j);
}
static void dft_assert_ref_not_over_released(job_t* j, uint64_t expect_jobid)
{
dereference(j);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(expect_jobid, j->job.jobid,
"j must still be safely readable after releasing the canary "
"reference -- an over-release inside dispatch_a_job_test() would "
"have freed j while the canary was still held, making this a "
"heap-use-after-free");
}
#define DFT_JOBID_INVALID_ALLOC 91101ULL
/* SITE 1: XCE_DISPATCH_FAIL_SITE_INVALID_ALLOC (alloc->nhosts <= 0).
* Fails before any worker lookup -- no worker/queue registration needed
* beyond the job itself. */
static void test_invalid_alloc_resolves_job_to_exit(void)
{
job_t* j;
xce_job_alloc_t* alloc;
int rc;
dft_setup();
j = dft_seed_job(DFT_JOBID_INVALID_ALLOC, "dft-q1");
alloc = make_job_alloc(DFT_JOBID_INVALID_ALLOC, 0, NULL);
TEST_ASSERT_NOT_NULL(alloc);
dft_arm_ref_canary(j);
rc = dispatch_a_job_test(&g_dft, NULL, alloc);
TEST_ASSERT_EQUAL_INT_MESSAGE(-EINVAL, rc,
"an invalid allocation (nhosts <= 0) must still return -EINVAL, "
"unchanged from before this task");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_EXIT, j->job.state,
"Req 4.2: a terminally-classified dispatch failure must resolve "
"the job to EXIT immediately, not leave it at PEND");
TEST_ASSERT_NULL_MESSAGE(j->job.allocs,
"Req 4.5: no partial allocation data may be left behind");
dft_assert_ref_not_over_released(j, DFT_JOBID_INVALID_ALLOC);
free_JobAlloc_ptr(&alloc);
dft_teardown();
printf("test_invalid_alloc_resolves_job_to_exit: PASS\n");
}
#define DFT_JOBID_QUEUE_EXEC 91102ULL
/* SITE 2: XCE_DISPATCH_FAIL_SITE_QUEUE_EXEC (get_queue_exec() failure).
* Needs a real, ready worker (so the function reaches past the worker
* checks) but a job whose submit->queue names a queue deliberately never
* registered in g_dft.qmgr, so get_queue_exec()'s key_find() fails with
* -ENOENT (queue.c:get_queue_exec, unmodified by this task). */
static void test_queue_exec_failure_resolves_job_to_exit(void)
{
job_t* j;
xce_job_alloc_t* alloc;
int rc;
dft_setup();
j = dft_seed_job(DFT_JOBID_QUEUE_EXEC, "dft-nonexistent-queue");
dft_seed_ready_worker("dft-wkr");
alloc = make_job_alloc(DFT_JOBID_QUEUE_EXEC, 1,
make_host_allocs(1,
make_host_alloc((char*)"dft-wkr", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(alloc);
dft_arm_ref_canary(j);
rc = dispatch_a_job_test(&g_dft, NULL, alloc);
TEST_ASSERT_EQUAL_INT_MESSAGE(-ENOENT, rc,
"get_queue_exec() failing on an unregistered queue name must "
"still return -ENOENT, unchanged from before this task");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_EXIT, j->job.state,
"Req 4.2: a terminally-classified dispatch failure must resolve "
"the job to EXIT immediately, not leave it at PEND");
TEST_ASSERT_NULL_MESSAGE(j->job.allocs,
"Req 4.5: no partial allocation data may be left behind");
dft_assert_ref_not_over_released(j, DFT_JOBID_QUEUE_EXEC);
free_JobAlloc_ptr(&alloc);
dft_teardown();
printf("test_queue_exec_failure_resolves_job_to_exit: PASS\n");
}
void test_dispatch_fail_terminal(void)
{
UnitySetTestFile(__FILE__);
printf("\n>> dispatch_a_job_() terminal dispatch-failure resolution "
"tests (Req 4.2, 4.5)\n\n");
RUN_TEST(test_invalid_alloc_resolves_job_to_exit);
RUN_TEST(test_queue_exec_failure_resolves_job_to_exit);
}

1407
src/controller/testsuite/test_e2e_capnp.c
View file

File diff suppressed because it is too large Load diff

89
src/controller/testsuite/test_e2e_checkpoint.c
View file

@ -158,14 +158,23 @@ static void test_checkpoint_wal_trim_after_checkpoint(void)
persist_t* p = make_ckpt_e2e(tmpdir, sizeof(tmpdir), 2);
TEST_ASSERT_NOT_NULL(p);
/* Write enough entries and checkpoint once */
/* Write enough entries and checkpoint once. No live apply pump runs in
* this cluster-less fixture, so last_applied_lsn stays 0 unless
* simulated -- bug-1426's trim floor now also caps at last_applied_lsn
* (the local apply cursor), so it must be advanced here to reflect the
* pump having caught up, or trim_wal_segments would correctly refuse
* to delete anything this test expects trimmed. */
for (int i = 0; i < 5; i++)
persist_wal_async(p, WAL_JOB_UPDATE, "trim", 4);
atomic_store_explicit(&p->last_applied_lsn, p->current_lsn - 1,
memory_order_relaxed);
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
/* Write a few more entries then checkpoint again */
for (int i = 0; i < 3; i++)
persist_wal_async(p, WAL_JOB_UPDATE, "trim2", 5);
atomic_store_explicit(&p->last_applied_lsn, p->current_lsn - 1,
memory_order_relaxed);
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
/* After two checkpoints the wal directory must still be functional
@ -259,6 +268,14 @@ static void test_trim_guard_preserves_standby_segments(void)
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "y", 1));
TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));
/* bug-1426: simulate the local apply pump having caught up to the
* active segment tip -- trim_wal_segments now also caps its floor at
* last_applied_lsn (this test targets the standby-floor guard
* specifically, not the apply-cursor one, so the pump must not be the
* limiting factor here). */
atomic_store_explicit(&p->last_applied_lsn, p->current_lsn - 1,
memory_order_relaxed);
/* Trigger checkpoint → trim_wal_segments runs */
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
@ -307,6 +324,10 @@ static void test_trim_without_standbys_proceeds_normally(void)
TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));
/* bug-1426: simulate the local apply pump having caught up (see
* test_trim_guard_preserves_standby_segments's identical note above). */
atomic_store_explicit(&p->last_applied_lsn, p->current_lsn - 1,
memory_order_relaxed);
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
char wal_path[512];
@ -331,6 +352,71 @@ static void test_trim_without_standbys_proceeds_normally(void)
printf("test_trim_without_standbys_proceeds_normally: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Test 6: local apply-cursor trim guard — bug-1426. With no standbys */
/* registered (so the standby-floor guard alone would allow trimming */
/* everything below the active segment), a LAGGING local apply pump */
/* (last_applied_lsn below the active segment) must still block */
/* trimming of segments it hasn't caught up to yet. */
/* */
/* RED proof: revert the last_applied_lsn floor cap in */
/* trim_wal_segments -> seg-at-2 (which the lagging pump hasn't read */
/* yet) is deleted anyway, because the standby-only floor (= active_seg */
/* _lsn=3, no standbys) never considers local apply progress -> a */
/* concurrent live apply-pump reader relying on an already-open cached */
/* fd into that segment (POSIX unlink-keeps-fd-alive) would silently */
/* keep succeeding, while any FRESH read of the same range (a restart's */
/* persist_recover() replay) would find it already gone -- the apply- */
/* correctness gap this bug describes. */
/* ------------------------------------------------------------------ */
static void test_trim_guard_preserves_unapplied_local_segments(void)
{
char tmpdir[256];
persist_t* p = make_ckpt_e2e(tmpdir, sizeof(tmpdir), 2);
TEST_ASSERT_NOT_NULL(p);
/* No standbys registered — the pre-bug-1426 floor would be
* active_seg_lsn=3 alone, same as test_trim_without_standbys_proceeds_
* normally above. This time, deliberately advance last_applied_lsn only
* through the entry in seg-at-1 (LSN 1), NOT the one in seg-at-2 (LSN
* 2) — simulating a live apply pump that is still lagging (e.g. mid
* catch-up burst) when the checkpoint's trim fires. seg-at-1 is fully
* consumed and safe to delete; seg-at-2 is not — the pump hasn't
* reached its entry yet. */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));
atomic_store_explicit(&p->last_applied_lsn, 1, memory_order_relaxed);
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
char wal_path[512];
snprintf(wal_path, sizeof(wal_path), "%s/e2e-ckpt/data/wal", tmpdir);
/* bug-1426 floor = min(active_seg_lsn=3, last_applied_lsn+1=2) = 2.
* seg_at_1 (start=1) < 2 → deleted: its one entry (LSN 1) is fully
* applied, safe to reclaim. seg_at_2 (start=2) < 2 is FALSE →
* preserved: its entry (LSN 2) has NOT been applied yet — deleting it
* would strand data the local apply pump still needs. */
TEST_ASSERT_FALSE_MESSAGE(
seg_file_exists(wal_path, 1),
"seg-at-1: fully applied (last_applied_lsn=1), safe to trim even "
"though no standby needs it either");
TEST_ASSERT_TRUE_MESSAGE(
seg_file_exists(wal_path, 2),
"seg-at-2: NOT yet applied by the lagging local pump "
"(last_applied_lsn=1 < 2), MUST be preserved (bug-1426 guard) even "
"though no standby needs it");
TEST_ASSERT_TRUE_MESSAGE(
seg_file_exists(wal_path, 3),
"seg-at-3: active segment, MUST always be preserved");
teardown_ckpt_e2e(p);
printf("test_trim_guard_preserves_unapplied_local_segments: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Suite entry point */
/* ------------------------------------------------------------------ */
@ -345,4 +431,5 @@ void test_e2e_checkpoint(void)
RUN_TEST(test_snapshot_rotation_keeps_snap_keep_n);
RUN_TEST(test_trim_guard_preserves_standby_segments);
RUN_TEST(test_trim_without_standbys_proceeds_normally);
RUN_TEST(test_trim_guard_preserves_unapplied_local_segments);
}

71
src/controller/testsuite/test_ha_liveness.c
View file

@ -1359,6 +1359,17 @@ extern int standby_ep_err_handler_primary_for_test(endpoint_t *ep);
* evicted while resolving ownership (raft-drive-async Task 2.2). */
extern _Atomic int ha_standby_evicted_count_for_test;
/* vote_ep_ctx_build_for_test (bug-1474): build a promoted-vote-ep test
* double, mirroring test_ha_connection_reuse.c's declaration. */
extern endpoint_t *vote_ep_ctx_build_for_test(endpoint_t *ep,
ha_state_t *ha,
ha_elect_ctx_t *ctx,
int peer_idx,
uint64_t my_term,
uint64_t my_tag,
uint64_t my_lsn,
const char *my_name);
/* Allocate + init a heap endpoint wired as a standby channel for peer_idx, the
* way ha_connect_prep_peer / a promoted vote-ep would leave it (production
* teardown via standby_ep_err_handler_primary frees ep->data + endpoint, so the
@ -1442,6 +1453,65 @@ static void test_reconnect_resolves_dual_channel_same_peer(void)
printf("test_reconnect_resolves_dual_channel_same_peer: PASS\n");
}
/*
* bug-1474 (angle-a): the SAME dual-channel resolution as the test above, but
* for the mixed case that test never exercised — the stale entry is a
* PROMOTED VOTE EP (raft_cb_become_leader), not another plain standby ep.
* A promoted vote ep's peer_idx is the RELATIVE votes[] index (self
* excluded); a fresh standby_ep_ctx_t's peer_idx is always the ABSOLUTE
* candidate index. Comparing them directly (the pre-fix code) can never
* match for the SAME physical peer, so the stale promoted-vote channel
* survives alongside the fresh one — two live outbound channels to one
* peer, exactly the hazard on_standby_connected's own doc comment
* describes. own_setup_primary sets my_priority=0 (self is candidate 0);
* peer 2's relative votes[] index is (2 < 0) ? 2 : 2-1 = 1.
*
* RED (before the fix): count == 2 (both channels survive, no eviction).
* GREEN (after the fix): count == 1 (fresh survives, stale evicted).
*/
static void test_reconnect_resolves_dual_channel_promoted_vote_ep(void)
{
cluster_t c;
myconf_t conf;
own_setup_primary(&c, &conf);
atomic_store(&ha_standby_evicted_count_for_test, 0);
/* Stale entry: a promoted vote ep for peer 2, relative index 1. */
ha_elect_ctx_t *dummy_ectx = calloc(1, sizeof(*dummy_ectx));
TEST_ASSERT_NOT_NULL(dummy_ectx);
endpoint_t *stale = endpoint_get();
TEST_ASSERT_NOT_NULL(stale);
endpoint_init(stale, AF_INET, SOCK_STREAM, 0, 0);
stale->iomgr = NULL;
vote_ep_ctx_build_for_test(stale, &c.ha, dummy_ectx, /*peer_idx=*/1,
/*my_term=*/1, /*my_tag=*/0, /*my_lsn=*/0,
"test");
zarray_store(c.ha.standby_endpoints, stale);
TEST_ASSERT_EQUAL_INT(1, c.ha.standby_endpoints->count);
/* Fresh reconnect to the SAME physical peer (absolute index 2). */
endpoint_t *fresh = own_make_standby_ep(&c.ha, &c, 2);
on_standby_connected_for_test(fresh);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, c.ha.standby_endpoints->count,
"bug-1474: a reconnect to a peer whose ONLY existing channel is a "
"promoted vote ep must still resolve to exactly ONE live channel");
TEST_ASSERT_EQUAL_PTR_MESSAGE(fresh, zarray_get(c.ha.standby_endpoints, 0),
"the survivor must be the freshly-connected channel");
TEST_ASSERT_EQUAL_INT_MESSAGE(1,
atomic_load(&ha_standby_evicted_count_for_test),
"the stale promoted-vote-ep channel must be evicted exactly once");
standby_ep_err_handler_primary_for_test(fresh);
endpoint_fini(fresh);
free(fresh);
free(dummy_ectx);
fini_ha_primary_with_standby_eps(&c.ha);
printf("test_reconnect_resolves_dual_channel_promoted_vote_ep: PASS\n");
}
/*
* Test 19 (Req 2.6): registering a NEW channel for a DIFFERENT peer does NOT
* evict an existing channel for another peer — resolution is strictly
@ -2334,6 +2404,7 @@ void test_ha_liveness(void)
RUN_TEST(test_reconnect_offload_dedups_per_peer);
/* raft-drive-async Task 2.2 — inbound/outbound ownership (Req 2.6, 4.4) */
RUN_TEST(test_reconnect_resolves_dual_channel_same_peer);
RUN_TEST(test_reconnect_resolves_dual_channel_promoted_vote_ep);
RUN_TEST(test_reconnect_preserves_other_peer_channel);
/* raft-drive-async Task 2.3 — reconnect-offload INTEGRATION (Req 2.1, 2.2, 2.5) */
RUN_TEST(test_reconnect_offload_sustains_survivor_heartbeats);

45
src/controller/testsuite/test_ha_prevote.c
View file

@ -63,6 +63,12 @@ extern void ha_prevote_round_timeout_fire_for_test(ha_state_t *ha, cluster_t *c)
* LOST outcome without a live delaymgr. */
extern int (*ha_prevote_rearm_liveness_hook)(ha_state_t *ha, cluster_t *c);
/*
* liveness_ctx_get_for_test: opaque accessor for ha.c's static s_liveness_ctx
* (bug-1483 regression lock — see test below).
*/
extern void *liveness_ctx_get_for_test(void);
/* ── Task 5.1 transport-route seams ──────────────────────────────────────── */
/*
@ -280,6 +286,44 @@ static void test_prevote_round_timeout_is_loss(void)
printf("test_prevote_round_timeout_is_loss: PASS\n");
}
/* ── Test 4b: LOST round with no test hook must not leak the liveness ctx
* (bug-1483) ──────────────────────────────────────────────────────────── */
/*
* Every other LOST-path test in this file stubs ha_prevote_rearm_liveness_hook
* to avoid driving the real ha_arm_heartbeat_timeout_vtask(ha, c) with c==NULL
* — that unstubbed shape is exactly what bug-1483 found: the real function
* callocs a liveness_poll_ctx_t and publishes it to the static s_liveness_ctx,
* but with c==NULL there is no delaymgr to schedule liveness_poll_fn on, so
* nothing ever frees it — a permanent leak/dangling-publish on every such
* call. This test deliberately does NOT stub the hook, driving the real path,
* and asserts the ctx is never left published once arming without a
* scheduler is impossible.
*/
static void test_bug1483_lost_round_without_hook_does_not_leak_liveness_ctx(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
/* Deliberately leave ha_prevote_rearm_liveness_hook == NULL so the LOST
* branch calls the real ha_arm_heartbeat_timeout_vtask(&ha, NULL). */
TEST_ASSERT_NULL_MESSAGE(ha_prevote_rearm_liveness_hook,
"precondition: hook must be unstubbed to drive the real path");
ha_start_prevote_async(&ha, NULL);
ha_prevote_round_timeout_fire_for_test(&ha, NULL);
TEST_ASSERT_NULL_MESSAGE(liveness_ctx_get_for_test(),
"bug-1483: arming the liveness watchdog with c==NULL (no delaymgr "
"to poll on) must not leave an allocated ctx published as the "
"\"active\" s_liveness_ctx -- nothing would ever free it");
ha_fini(&ha);
printf("test_bug1483_lost_round_without_hook_does_not_leak_liveness_ctx: PASS\n");
}
/* ── Test 5: a subsequent genuine failure still triggers a new round ─────── */
/*
@ -876,6 +920,7 @@ void test_ha_prevote(void)
RUN_TEST(test_prevote_majority_grant_starts_real_election);
RUN_TEST(test_prevote_no_majority_stays_standby_and_rearms);
RUN_TEST(test_prevote_round_timeout_is_loss);
RUN_TEST(test_bug1483_lost_round_without_hook_does_not_leak_liveness_ctx);
RUN_TEST(test_prevote_reusable_after_loss);
RUN_TEST(test_prevote_pending_is_noop);
RUN_TEST(test_prevote_reply_over_real_transport_reaches_tally);

72
src/controller/testsuite/test_ha_rdhup_integration.c
View file

@ -642,11 +642,83 @@ static void test_rdhup_within_cooldown_is_debounced(void)
printf("test_rdhup_within_cooldown_is_debounced: PASS\n");
}
/* ═══════════════════════════════════════════════════════════════════════════
* Test: a debounced RDHUP must still re-arm the liveness watchdog (bug-1562)
*
* The 5.10b debounce branch (sibling test above) correctly suppresses a
* fresh failover attempt when an RDHUP lands within the cooldown window. But
* unlike the 5.6 guard right above it in the function (which only skips
* re-arming when it has VERIFIED an election is actually in flight,
* role==CTRL_ELECTING || pre_vote_in_progress), the 5.10b branch only checks
* elapsed time since last_failover_ms -- it never checks whether an election
* is still running. On a lone cold-booting node, the first PreVote round
* loses within milliseconds (no peers reachable yet -> connect failures
* count as "no" votes), which re-arms the watchdog once. When that watchdog
* fires again ~heartbeat_timeout_ms later, last_failover_ms is still inside
* the 2x-election_timeout cooldown even though no election is in flight
* anymore -- so pre-fix, the debounce branch returns without re-arming
* anything. That is the LAST timer ever scheduled in the process: a live
* repro (matching a real user's failed 3-node deploy exactly) confirmed
* every vtask worker thread idling on an empty queue and the io_worker
* parked in epoll_wait with zero pending connections, 9+ minutes of total
* silence -- the node is permanently stuck CTRL_STANDBY, and since it is the
* only node holding a local config, the whole cluster never elects a
* primary.
*
* ha_arm_heartbeat_timeout_vtask stamps ha->last_activity to "now" as its
* first, delaymgr-independent side effect (ha.c) -- the same observable
* test_ha_arm_no_crash_no_thread (test_ha_liveness.c) already uses to prove
* arming happened without a live delaymgr thread (c == NULL here).
* ═════════════════════════════════════════════════════════════════════════ */
static void test_rdhup_debounce_still_rearms_liveness_watchdog(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha); /* role = CTRL_STANDBY */
/* Sentinel the debounce branch itself never writes -- a change proves
* ha_arm_heartbeat_timeout_vtask actually ran. */
ha.last_activity = 1;
/* A failover was triggered "just now" -> this RDHUP lands inside the
* cooldown window and must be debounced (mirrors the sibling test). */
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
uint64_t now_ms = (uint64_t)ts.tv_sec * 1000u + (uint64_t)(ts.tv_nsec / 1000000L);
atomic_store_explicit(&ha.last_failover_ms, now_ms, memory_order_release);
int sv[2];
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
ha.primary_ctrl_fd = sv[0];
ha_on_primary_ctrl_fd_rdhup(&ha, NULL);
close(sv[1]);
/* Still debounced -- no fresh failover, stays CTRL_STANDBY (same
* guarantee as the sibling test). */
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"a debounced RDHUP must not itself trigger a fresh failover");
/* The watchdog must have been re-armed even though this trigger was
* debounced -- last_activity must have moved off the sentinel. */
TEST_ASSERT_TRUE_MESSAGE((int)ha.last_activity != 1,
"a debounced RDHUP must still re-arm the liveness watchdog (bug-1562) "
"-- otherwise a lone node whose first PreVote loses fast then hits a "
"second liveness timeout inside the cooldown window freezes forever, "
"with no timer ever scheduled again");
ha_disarm_heartbeat_timeout_vtask(&ha);
ha_fini(&ha);
printf("test_rdhup_debounce_still_rearms_liveness_watchdog: PASS\n");
}
void test_ha_rdhup_integration(void)
{
UnitySetTestFile(__FILE__);
RUN_TEST(test_rdhup_candidate_advertises_freshness_floor_not_stale_committed);
RUN_TEST(test_rdhup_within_cooldown_is_debounced);
RUN_TEST(test_rdhup_debounce_still_rearms_liveness_watchdog);
RUN_TEST(test_rdhup_full_path_reaches_primary);
/* ack-driven promotion test deleted (ha-single-leader-role 4.1:
* ack-driven promotion machinery removed, no shim) */

114
src/controller/testsuite/test_job_dispatch.c Normal file
View file

@ -0,0 +1,114 @@
/* test_job_dispatch.c -- scheduler-dispatch-commit task 1.1
*
* Unit tests for the Controller Dispatch-Failure Classification component
* (design.md, "Controller Dispatch-Failure Classification"): a pure, static,
* file-local function in job.c that maps each of dispatch_a_job_()'s six
* post-lookup failure points (xce_dispatch_fail_site_t) to a retryable-vs-
* terminal classification (xce_dispatch_fail_class_t), per Requirements
* 4.1/4.2.
*
* xce_classify_dispatch_fail() and its enum types are deliberately NOT
* declared in any header (design.md's own placement-correction note: nothing
* outside job.c consumes them). This test file reaches the static function
* through a thin, int-typed, XCE_TESTING-only wrapper exported from job.c
* (xce_classify_dispatch_fail_test), the same pattern job.c already uses for
* job_array_apply_test_set_force_fail_at()/friends -- and, like test_job_
* array_fence.c's ha_arm_promotion_fence_timer_for_test, the extern
* declaration lives directly in this test file rather than in a shared
* header.
*
* No cluster_t/persist/worker fixture is needed: the classification is a
* pure decision with no side effects, independently testable against every
* known failure case without a live dispatch attempt (task 1.1's own
* Observable-completion clause).
*/
#include <stdio.h>
#include <unity/unity.h>
/* Mirrors design.md's Service Interface enum values exactly (job.c's own
* xce_dispatch_fail_site_t/xce_dispatch_fail_class_t are file-local and not
* shared via a header -- this test drives the wrapper by the same plain-int
* values documented there, not by including a private type). */
#define SITE_WORKER_LOOKUP 0
#define SITE_WORKER_NULL 1
#define SITE_WORKER_NOT_READY 2
#define SITE_INVALID_ALLOC 3
#define SITE_QUEUE_EXEC 4
#define SITE_RPC_FAILURE 5
#define CLASS_RETRYABLE 0
#define CLASS_TERMINAL 1
extern int xce_classify_dispatch_fail_test(int site);
static void test_worker_lookup_is_retryable(void)
{
TEST_ASSERT_EQUAL_INT(CLASS_RETRYABLE,
xce_classify_dispatch_fail_test(SITE_WORKER_LOOKUP));
printf("test_worker_lookup_is_retryable: PASS\n");
}
static void test_worker_null_is_retryable(void)
{
TEST_ASSERT_EQUAL_INT(CLASS_RETRYABLE,
xce_classify_dispatch_fail_test(SITE_WORKER_NULL));
printf("test_worker_null_is_retryable: PASS\n");
}
static void test_worker_not_ready_is_retryable(void)
{
TEST_ASSERT_EQUAL_INT(CLASS_RETRYABLE, xce_classify_dispatch_fail_test(
SITE_WORKER_NOT_READY));
printf("test_worker_not_ready_is_retryable: PASS\n");
}
static void test_rpc_failure_is_retryable(void)
{
TEST_ASSERT_EQUAL_INT(CLASS_RETRYABLE,
xce_classify_dispatch_fail_test(SITE_RPC_FAILURE));
printf("test_rpc_failure_is_retryable: PASS\n");
}
static void test_invalid_alloc_is_terminal(void)
{
TEST_ASSERT_EQUAL_INT(CLASS_TERMINAL,
xce_classify_dispatch_fail_test(SITE_INVALID_ALLOC));
printf("test_invalid_alloc_is_terminal: PASS\n");
}
static void test_queue_exec_is_terminal(void)
{
TEST_ASSERT_EQUAL_INT(CLASS_TERMINAL,
xce_classify_dispatch_fail_test(SITE_QUEUE_EXEC));
printf("test_queue_exec_is_terminal: PASS\n");
}
/* Invariant (design.md Service Interface Postconditions): the same site
* always classifies the same way -- no hidden state, no time-dependence. */
static void test_classification_is_stable_across_repeated_calls(void)
{
int first = xce_classify_dispatch_fail_test(SITE_WORKER_NOT_READY);
int second = xce_classify_dispatch_fail_test(SITE_WORKER_NOT_READY);
int third = xce_classify_dispatch_fail_test(SITE_QUEUE_EXEC);
int fourth = xce_classify_dispatch_fail_test(SITE_QUEUE_EXEC);
TEST_ASSERT_EQUAL_INT(first, second);
TEST_ASSERT_EQUAL_INT(third, fourth);
printf("test_classification_is_stable_across_repeated_calls: PASS\n");
}
void test_job_dispatch(void)
{
UnitySetTestFile(__FILE__);
printf("\n>> dispatch-failure classification tests (Req 4.1, 4.2)\n\n");
RUN_TEST(test_worker_lookup_is_retryable);
RUN_TEST(test_worker_null_is_retryable);
RUN_TEST(test_worker_not_ready_is_retryable);
RUN_TEST(test_rpc_failure_is_retryable);
RUN_TEST(test_invalid_alloc_is_terminal);
RUN_TEST(test_queue_exec_is_terminal);
RUN_TEST(test_classification_is_stable_across_repeated_calls);
}

98
src/controller/testsuite/test_job_dispatch_fail_count.c Normal file
View file

@ -0,0 +1,98 @@
/* test_job_dispatch_fail_count.c — scheduler-dispatch-commit task 1.3
*
* Proves job_t.dispatch_fail_count (xce/types.h) -- a new in-memory-only
* uint32_t counter on the job_t wrapper ({xce_job_t job; uint64_t
* submit_lsn;}), NOT on xce_job_t itself -- exists and behaves as a plain,
* directly settable/readable/resettable scalar (Requirement 4.3, design.md
* "Logical Data Model": "in-memory only... following submit_lsn's exact
* precedent... recomputed... never serialized").
*
* This task is the data-model addition ONLY. It does NOT wire any
* increment/reset call site into dispatch_a_job_() (task 3.1/3.3's job) and
* does NOT implement the bounded-retry threshold check (also task 3.3).
* Every test here manipulates the field directly on a bare stack-local
* job_t -- no jobmgr/RTTI/makeJob machinery -- mirroring
* test_persist_types.c's test_slot_key_t_fields precedent for a plain
* struct-field addition, since this is not a behavioral/dispatch-path test.
*/
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <unity/unity.h>
#include "xce/types.h"
static void test_dispatch_fail_count_default_zero(void)
{
job_t j;
memset(&j, 0, sizeof(j));
TEST_ASSERT_EQUAL_UINT32(0, j.dispatch_fail_count);
printf("test_dispatch_fail_count_default_zero: PASS\n");
}
static void test_dispatch_fail_count_increments_independently(void)
{
job_t j;
memset(&j, 0, sizeof(j));
/* Simulates 3 consecutive retryable dispatch failures with no
* intervening successful dispatch (Requirement 4.3) -- incremented
* directly, with no actual dispatch attempt involved. */
j.dispatch_fail_count++;
TEST_ASSERT_EQUAL_UINT32(1, j.dispatch_fail_count);
j.dispatch_fail_count++;
j.dispatch_fail_count++;
TEST_ASSERT_EQUAL_UINT32_MESSAGE(3, j.dispatch_fail_count,
"counter must be incrementable directly, independent of any actual "
"dispatch attempt (Observable completion)");
printf("test_dispatch_fail_count_increments_independently: PASS\n");
}
static void test_dispatch_fail_count_resets_to_zero(void)
{
job_t j;
memset(&j, 0, sizeof(j));
j.dispatch_fail_count = 7;
TEST_ASSERT_EQUAL_UINT32(7, j.dispatch_fail_count);
/* A successful dispatch resets the counter to zero -- task 3.1/3.3 wire
* the real call site; this proves the field itself supports the
* reset-to-zero operation the design requires, independent of any
* actual dispatch attempt. */
j.dispatch_fail_count = 0;
TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, j.dispatch_fail_count,
"counter must be independently resettable to zero, matching "
"submit_lsn's in-memory-only precedent (no durability obligation, "
"safe to lose across a controller failover)");
printf("test_dispatch_fail_count_resets_to_zero: PASS\n");
}
static void test_dispatch_fail_count_does_not_alias_submit_lsn(void)
{
job_t j;
memset(&j, 0, sizeof(j));
j.submit_lsn = 12345;
j.dispatch_fail_count = 6;
TEST_ASSERT_EQUAL_UINT64(12345, j.submit_lsn);
TEST_ASSERT_EQUAL_UINT32_MESSAGE(6, j.dispatch_fail_count,
"dispatch_fail_count must be its own field on job_t, distinct from "
"submit_lsn (both in-memory-only siblings on the wrapper struct)");
printf("test_dispatch_fail_count_does_not_alias_submit_lsn: PASS\n");
}
void test_job_dispatch_fail_count(void)
{
UnitySetTestFile(__FILE__);
printf("\n>> job_t.dispatch_fail_count test\n\n");
RUN_TEST(test_dispatch_fail_count_default_zero);
RUN_TEST(test_dispatch_fail_count_increments_independently);
RUN_TEST(test_dispatch_fail_count_resets_to_zero);
RUN_TEST(test_dispatch_fail_count_does_not_alias_submit_lsn);
}

482
src/controller/testsuite/test_jobdispatch_no_double_notify.c Normal file
View file

@ -0,0 +1,482 @@
/* test_jobdispatch_no_double_notify.c -- scheduler-dispatch-commit task 4.1
*
* Proves handle_jobdispatch() (job.c) no longer duplicates the dispatch-
* outcome reporting that dispatch_a_job_() (tasks 3.1-3.3) now owns
* entirely. Before this task, handle_jobdispatch()'s loop called
* notify_dispatch() itself for EVERY job in the batch regardless of
* dispatch_a_job_()'s own outcome/classification -- on top of whatever
* dispatch_a_job_() had already decided to do (nothing on success or on a
* terminal failure, exactly one notify_dispatch() call on a retryable
* failure). That produced a conflicting double report for a terminal
* failure specifically (a caller-added jobDispatchFail alongside
* mark_job_fail()'s own schedFailJob -- design.md's round-2 finding) and a
* redundant extra notify_dispatch() call for a retryable failure.
*
* This test drives handle_jobdispatch() directly (not dispatch_a_job_test())
* with a 3-job batch covering all three of dispatch_a_job_()'s own
* notify_dispatch() behaviors:
* - success (a live, ready worker) -> dispatch_a_job_() itself
* calls notify_dispatch() 0 times
* - retryable failure (unregistered worker) -> dispatch_a_job_() itself
* calls notify_dispatch() exactly 1 time (below the retry bound)
* - terminal failure (nhosts == 0) -> dispatch_a_job_() itself
* calls notify_dispatch() 0 times (mark_job_fail()'s hook chain is the
* sole notification for this case)
*
* dispatch_a_job_() alone (already proven correct by tasks 3.1-3.3's own
* test_dispatch_commit.c / test_dispatch_fail_terminal.c /
* test_dispatch_fail_retry.c) therefore produces exactly ONE
* notify_dispatch() call for this batch. The fix under test is that
* handle_jobdispatch() must not add anything on top of that -- the total
* observed via notify() (test_commit_barrier_reply.c's capture stub, the
* sole notify() definition linked into this binary) must be exactly 1, not
* 4 (1 extra per job x 3, on top of the 1 dispatch_a_job_() already
* produces) as it was before this task.
*
* Fixture mirrors test_dispatch_commit.c's dc_setup/dc_teardown (cluster_t
* with a real disk-backed persist_t, cluster == NULL routing so
* persist_submit_job_fact() takes the direct persist_wal_async() path, no
* HA/commit-engine machinery needed) plus test_commit_barrier_reply.c's
* g_req_ep / c->myhost requester-endpoint pattern (zeroed, no iomgr --
* xce_reply()'s enqueue always succeeds against it, proven by that file's
* own header) so handle_jobdispatch()'s receipt-ACK xce_reply() call has
* somewhere to enqueue to.
*
* notify_dispatch() observability: see test_dispatch_fail_retry.c's header
* for the full rationale -- this binary's sole notify() definition lives in
* test_commit_barrier_reply.c, exposed read-only via its
* cbrp_notify_capture_*_ext() accessors (added by task 3.3).
*
* scheduler-dispatch-commit task 5.3 (Req 4.2, 4.5) adds
* test_handle_jobdispatch_terminal_no_notify_no_leak, below. It is a
* sharper-focused sibling of this file's original (task 4.1) 3-job-batch
* test: that test proves the BATCH's total notify_dispatch() count is 1,
* attributable to the retryable job (an aggregate proof the terminal job
* contributed zero calls only by subtraction/arithmetic against the other
* two jobs' known outcomes) and never exercises reference-count cleanliness
* for any job. The new test instead drives a SINGLE terminal-classified job
* through this same real handle_jobdispatch() entry point (not
* dispatch_a_job_test()'s bypass, unlike test_dispatch_fail_terminal.c's
* task-3.2 tests) and combines, in ONE test run: (a) JOB_STAT_EXIT
* resolution, (b) a direct (not subtracted) zero-notify assertion for this
* job's own batch, and (c) test_dispatch_fail_terminal.c's canary-reference
* technique proving the reference count returns to its pre-attempt value.
*/
#include <rtti.h>
#include <errno.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
#include "cluster.h"
#include "controller.capnp.h"
#include "dataq.h"
#include "hook.h"
#include "jobj.h"
#include "kvmgr.h"
#include "netio.h"
#include "persist.h"
#include "qobj.h"
#include "typemap.h"
#include "wobj.h"
#include "xce/cluster_def.h"
#include "xce/types.h"
#include "xcealloc.h"
#include "xstring.h"
/* test_commit_barrier_reply.c's minimal notify()-capture accessors (task
* 3.3) -- see this file's header. This binary's sole notify() definition
* lives there (dataex.c's real one is excluded from the unit-test link). */
extern int cbrp_notify_capture_count_ext(void);
extern int cbrp_notify_capture_event_ext(int idx);
extern uint64_t cbrp_notify_capture_jobid_ext(int idx);
extern int cbrp_notify_capture_code_ext(int idx);
extern void cbrp_notify_capture_reset_ext(void);
/* jobj.c's job type table and the process-global xstring table are shared
* across every test file in the controller_unit_test binary -- guard
* idempotently, mirroring every sibling fixture in this suite. */
static int g_jnd_jobobj_ready;
static cluster_t g_jnd;
static char g_jnd_tmpdir[256];
static endpoint_t g_jnd_req_ep; /* requester endpoint (receipt ACK) */
static endpoint_t g_jnd_worker_ep; /* the one live worker's endpoint */
static xce_peerinfo_t g_jnd_req_from;
static void jnd_setup(void)
{
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32, .delta = 16 };
kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING,
.nslots = 32, .delta = 16 };
persist_config_t cfg;
if (makeString((char*)"jnd-probe") == NULL)
xstring_table_init(64);
if (!g_jnd_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
g_jnd_jobobj_ready = 1;
}
memset(&g_jnd, 0, sizeof(g_jnd));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&g_jnd.hooksys, XCE_CONTROLLER_HOOK_MAX));
g_jnd.jobmgr = makeInstance(&xkvmgr_type, &int_params);
g_jnd.workermgr = makeInstance(&xkvmgr_type, &str_params);
g_jnd.qmgr = makeInstance(&xkvmgr_type, &str_params);
TEST_ASSERT_NOT_NULL(g_jnd.jobmgr);
TEST_ASSERT_NOT_NULL(g_jnd.workermgr);
TEST_ASSERT_NOT_NULL(g_jnd.qmgr);
/* Real disk-backed persist, cluster == NULL: wal_hook_append_
* classified() takes the direct persist_wal_async() path -- no raft/
* commit-engine machinery needed for the successful job's plain
* WAL_JOB_DISPATCH fact append (mirrors test_dispatch_commit.c). */
cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
strcpy(g_jnd_tmpdir, "/tmp/xce_jnd41_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(g_jnd_tmpdir));
persist_set_workdir(&g_jnd.persist, g_jnd_tmpdir, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(&g_jnd.persist, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(&g_jnd.persist));
g_jnd.myhost = &g_jnd.myhost_info;
g_jnd.myhost_info.name = (char*)"ctrl-test";
g_jnd.myhost_info.key = (char*)"";
/* Zeroed, no iomgr -- xce_reply()'s/xce_startjob_req()'s enqueue always
* succeeds against an endpoint like this (test_dispatch_commit.c's and
* test_commit_barrier_reply.c's own established, proven pattern). */
memset(&g_jnd_req_ep, 0, sizeof(g_jnd_req_ep));
g_jnd_req_ep.fd = -1;
memset(&g_jnd_worker_ep, 0, sizeof(g_jnd_worker_ep));
g_jnd_worker_ep.fd = -1;
g_jnd_req_from.name = (char*)"requester";
g_jnd_req_from.key = (char*)"";
}
/* Frees every packet left queued on an endpoint's outq -- mirrors
* test_dispatch_commit.c's dc_drain_ep. */
static void jnd_drain_ep(endpoint_t* ep)
{
data_item_t* it;
while ((it = endpoint_deq(ep)) != NULL) {
io_packet_t* pkt = ITPTR_TO_PACKET(it);
free_packet(pkt);
free(pkt);
}
}
static void jnd_teardown(void)
{
char cmd[512];
jnd_drain_ep(&g_jnd_req_ep);
jnd_drain_ep(&g_jnd_worker_ep);
persist_stop(&g_jnd.persist);
persist_fini(&g_jnd.persist);
if (g_jnd.workermgr) dereference(g_jnd.workermgr);
if (g_jnd.qmgr) dereference(g_jnd.qmgr);
if (g_jnd.jobmgr) dereference(g_jnd.jobmgr);
xce_hook_fini(&g_jnd.hooksys);
snprintf(cmd, sizeof(cmd), "rm -rf %s", g_jnd_tmpdir);
if (system(cmd) != 0)
fprintf(stderr, "cleanup warning: rm -rf %s failed\n", g_jnd_tmpdir);
}
/* Seeds a queue and a PEND job with a real submit/start_info against
* g_jnd.jobmgr/qmgr (dispatch_a_job_() reads submit->queue and
* submit->start_info). Returns the seeded job_t* (borrowed -- g_jnd.jobmgr
* owns the creation reference). Mirrors test_dispatch_fail_retry.c's
* dfr_seed_job. */
static job_t* jnd_seed_job(uint64_t jobid, const char* qname)
{
xce_queue_t qdef;
job_startinfo_t startinfo;
job_submit_data_t submit;
xce_job_t jdef;
job_t* j;
memset(&qdef, 0, sizeof(qdef));
qdef.queue = (char*)qname;
TEST_ASSERT_NOT_NULL(makeQueue(g_jnd.qmgr, &qdef));
memset(&startinfo, 0, sizeof(startinfo));
startinfo.cmdline = (char*)"/bin/true";
memset(&submit, 0, sizeof(submit));
submit.queue = (char*)qname;
submit.user = (char*)"tester";
submit.start_info = &startinfo;
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = jobid;
jdef.state = JOB_STAT_PEND;
jdef.submit = &submit;
j = makeJob(g_jnd.jobmgr, &jdef);
TEST_ASSERT_NOT_NULL(j);
return j;
}
/* Registers a ready worker (host.status defaults to 0 via calloc) in
* g_jnd.workermgr with ep pointed at the given endpoint. Mirrors
* test_dispatch_commit.c's dc_seed_ready's worker half. */
static void jnd_seed_ready_worker(const char* wname, endpoint_t* ep)
{
xce_worker_t wdef;
hostattrs_t* attrs;
worker_t* w = NULL;
attrs = calloc(1, sizeof(*attrs));
TEST_ASSERT_NOT_NULL(attrs);
attrs->hosttype = makeString("LINUX");
attrs->hostmodel = makeString("x86");
attrs->cpufactor = 1.0f;
attrs->maxcpus = 4;
memset(&wdef, 0, sizeof(wdef));
wdef.name = (char*)wname;
wdef.attrs = attrs;
TEST_ASSERT_NOT_NULL(makeWorker(g_jnd.workermgr, &wdef));
lock(g_jnd.workermgr, RTTI_LOCK_READ);
key_find(g_jnd.workermgr, (void*)wname, (void**)&w);
unlock(g_jnd.workermgr);
TEST_ASSERT_NOT_NULL(w);
w->ep = ep;
}
#define JND_JOBID_SUCCESS 92101ULL
#define JND_JOBID_RETRYABLE 92102ULL
#define JND_JOBID_TERMINAL 92103ULL
/* THE core proof (Req 3.1, 4.1, 4.2): a batch mixing all three of
* dispatch_a_job_()'s own notify_dispatch() behaviors must produce EXACTLY
* what dispatch_a_job_() itself produces (1 -- the retryable job's own
* call), not 1 extra per job from handle_jobdispatch()'s own loop. */
static void test_handle_jobdispatch_matches_dispatch_a_job_notify_count(void)
{
xce_job_alloc_t* allocs[3];
xce_job_dispatch_t dispatchmsg;
xce_request_t req;
job_t* jsucc;
job_t* jretry;
job_t* jterm;
int rc;
jnd_setup();
jsucc = jnd_seed_job(JND_JOBID_SUCCESS, "jnd-q1");
jretry = jnd_seed_job(JND_JOBID_RETRYABLE, "jnd-q2");
jterm = jnd_seed_job(JND_JOBID_TERMINAL, "jnd-q3");
jnd_seed_ready_worker("jnd-wkr-ok", &g_jnd_worker_ep);
/* jretry deliberately targets an unregistered worker name -- no
* corresponding makeWorker() call, matching test_dispatch_fail_retry.c's
* own established WORKER_NULL/WORKER_LOOKUP fixture. */
allocs[0] = make_job_alloc(JND_JOBID_SUCCESS, 1,
make_host_allocs(1,
make_host_alloc((char*)"jnd-wkr-ok", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(allocs[0]);
allocs[1] = make_job_alloc(JND_JOBID_RETRYABLE, 1,
make_host_allocs(1,
make_host_alloc((char*)"jnd-no-such-worker", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
TEST_ASSERT_NOT_NULL(allocs[1]);
/* nhosts == 0: XCE_DISPATCH_FAIL_SITE_INVALID_ALLOC, terminal --
* mirrors test_commit_barrier_reply.c's cbrp_arrange_jobdispatch()
* second allocation exactly. */
allocs[2] = make_job_alloc(JND_JOBID_TERMINAL, 0, NULL);
TEST_ASSERT_NOT_NULL(allocs[2]);
memset(&dispatchmsg, 0, sizeof(dispatchmsg));
dispatchmsg.njobs = 3;
dispatchmsg.allocs = allocs;
memset(&req, 0, sizeof(req));
req.opcode = OPCode_jobdispatch;
req.seq = 42;
req.from = &g_jnd_req_from;
req.body_which = Request_body_dispatchmsg;
req.body.dispatchmsg = &dispatchmsg;
cbrp_notify_capture_reset_ext();
/* Act: handle_jobdispatch() consumes/frees every alloc itself via
* MOVE_PTR + free_JobAlloc_ptr() in its own loop -- nothing left for
* this test to free afterward. */
rc = handle_jobdispatch(&g_jnd, &g_jnd_req_ep, &req);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "handle_jobdispatch's own return "
"is unconditionally 0, unaffected by any per-job outcome");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, jsucc->job.state,
"the successful dispatch must still commit RUN (unaffected by this "
"task -- only the caller's own notification logic changes)");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, jretry->job.state,
"a below-bound retryable failure must still leave the job PEND");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(1, jretry->dispatch_fail_count,
"the retryable failure's own counter increment is unaffected by "
"this task");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_EXIT, jterm->job.state,
"the terminal (nhosts == 0) failure must still resolve to EXIT via "
"mark_job_fail(), unaffected by this task");
/* THE assertion this task exists for: dispatch_a_job_() alone (tasks
* 3.1-3.3, already proven correct) produces exactly ONE
* notify_dispatch() call for this exact batch -- the retryable job's
* own call. Before this task, handle_jobdispatch()'s own unconditional
* per-job notify_dispatch() calls would add one more per job (3), for a
* total of 4 -- a conflicting double (and, for the terminal job,
* mismatched) report. */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, cbrp_notify_capture_count_ext(),
"handle_jobdispatch() must not add any notify_dispatch() call on "
"top of what dispatch_a_job_() itself already produced for this "
"batch (Req 3.1, 4.1, 4.2) -- exactly the retryable job's own "
"call, none for the success or terminal jobs");
TEST_ASSERT_EQUAL_INT(NotifyData_data_jobDispatchFail,
cbrp_notify_capture_event_ext(0));
TEST_ASSERT_EQUAL_UINT64(JND_JOBID_RETRYABLE,
cbrp_notify_capture_jobid_ext(0));
jnd_teardown();
printf("test_handle_jobdispatch_matches_dispatch_a_job_notify_count: "
"PASS\n");
}
#define JND_JOBID_TERMINAL_SOLO 92201ULL
/* Canary-reference technique, reused verbatim from
* test_dispatch_fail_terminal.c's dft_arm_ref_canary()/dft_assert_ref_
* not_over_released() (task 3.2): armed immediately BEFORE the call under
* test (not after -- arming after would rely incidentally on other
* assertions crashing first, which doesn't independently prove anything),
* released and the job re-touched immediately after. An over-release during
* the call would free j while the canary reference is still held, making
* the post-call touch a real, ASAN-caught heap-use-after-free. job_t's real
* refcount lives in jobj.c's file-local job_private_t with no accessor
* exposed to a test, so this indirect technique is the only available
* proof. */
static void jnd_arm_ref_canary(job_t* j)
{
reference(j);
}
static void jnd_assert_ref_not_over_released(job_t* j, uint64_t expect_jobid)
{
dereference(j);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(expect_jobid, j->job.jobid,
"Req 4.5: j must still be safely readable after releasing the "
"canary reference -- an over-release inside "
"handle_jobdispatch()/dispatch_a_job_() would have freed j while "
"the canary was still held, making this a heap-use-after-free "
"(this terminal site already dereferences correctly even before "
"bug-1627's fix, which was scoped to the four retryable sites -- "
"this check guards the same class of risk, not that specific bug)");
}
/* Task 5.3's focused proof (Req 4.2, 4.5) -- see this file's header for how
* this differs from test_handle_jobdispatch_matches_dispatch_a_job_notify_
* count above. A single-job batch, terminal-classified (nhosts == 0),
* driven through the real handle_jobdispatch() entry point. */
static void test_handle_jobdispatch_terminal_no_notify_no_leak(void)
{
xce_job_alloc_t* allocs[1];
xce_job_dispatch_t dispatchmsg;
xce_request_t req;
job_t* jterm;
int rc;
jnd_setup();
jterm = jnd_seed_job(JND_JOBID_TERMINAL_SOLO, "jnd-q-solo");
/* nhosts == 0: XCE_DISPATCH_FAIL_SITE_INVALID_ALLOC, terminal --
* mirrors this file's own JND_JOBID_TERMINAL allocation and
* test_dispatch_fail_terminal.c's test_invalid_alloc_resolves_job_
* to_exit, but reached this time through the real handle_jobdispatch()
* entry point instead of dispatch_a_job_test()'s bypass. */
allocs[0] = make_job_alloc(JND_JOBID_TERMINAL_SOLO, 0, NULL);
TEST_ASSERT_NOT_NULL(allocs[0]);
memset(&dispatchmsg, 0, sizeof(dispatchmsg));
dispatchmsg.njobs = 1;
dispatchmsg.allocs = allocs;
memset(&req, 0, sizeof(req));
req.opcode = OPCode_jobdispatch;
req.seq = 43;
req.from = &g_jnd_req_from;
req.body_which = Request_body_dispatchmsg;
req.body.dispatchmsg = &dispatchmsg;
cbrp_notify_capture_reset_ext();
jnd_arm_ref_canary(jterm);
/* Act: handle_jobdispatch() consumes/frees allocs[0] itself via
* MOVE_PTR + free_JobAlloc_ptr() in its own loop -- nothing left for
* this test to free afterward. */
rc = handle_jobdispatch(&g_jnd, &g_jnd_req_ep, &req);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "handle_jobdispatch's own return "
"is unconditionally 0, unaffected by any per-job outcome");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_EXIT, jterm->job.state,
"Req 4.2: a terminally-classified dispatch failure must resolve "
"the job to EXIT, reached this time through the real "
"handle_jobdispatch() entry point rather than the "
"dispatch_a_job_test() bypass test_dispatch_fail_terminal.c uses");
TEST_ASSERT_NULL_MESSAGE(jterm->job.allocs,
"Req 4.5: no partial allocation data may be left behind");
/* THE assertion this task exists for: a terminal dispatch failure must
* not ALSO produce a retry-eligible jobDispatchFail report for the same
* failure -- mark_job_fail()'s own schedFailJob hook chain (already
* fired above) is the sole notification. Unlike test_handle_jobdispatch_
* matches_dispatch_a_job_notify_count's 3-job batch (where the terminal
* job's zero contribution is inferred by the total equaling exactly the
* retryable job's own 1 call), this single-job batch makes the zero
* count direct: any notify_dispatch() call surfacing here can only have
* come from this one terminal job, whether from handle_jobdispatch()'s
* old per-job loop call (task 4.1's regression) or from dispatch_a_
* job_() itself misclassifying/misrouting (task 3.2's regression). */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, cbrp_notify_capture_count_ext(),
"a terminal dispatch failure must not also produce a "
"notify_dispatch() report for this same failure (regression test "
"for the round-2 double-notify finding, design.md's Integration "
"Tests third bullet)");
jnd_assert_ref_not_over_released(jterm, JND_JOBID_TERMINAL_SOLO);
jnd_teardown();
printf("test_handle_jobdispatch_terminal_no_notify_no_leak: PASS\n");
}
void test_jobdispatch_no_double_notify(void)
{
UnitySetTestFile(__FILE__);
printf("\n>> handle_jobdispatch() no-double-notify tests (Req 3.1, 4.1, "
"4.2)\n\n");
RUN_TEST(test_handle_jobdispatch_matches_dispatch_a_job_notify_count);
RUN_TEST(test_handle_jobdispatch_terminal_no_notify_no_leak);
}

84
src/controller/testsuite/test_persist_hooks.c
View file

@ -8,13 +8,17 @@
#include <sys/stat.h>
#include <unistd.h>
#include <rtti.h>
#include <unity/unity.h>
#include "cluster.h"
#include "config.h"
#include "jobj.h"
#include "kvmgr.h"
#include "persist.h"
#include "typemap.h"
#include "xce/cluster_def.h"
#include "xlist.h"
#include "controller.capnp.h" /* xce_job_rusages_t (bug-1502 regression test) */
/* ------------------------------------------------------------------ */
/* Test helpers */
@ -63,6 +67,41 @@ static worker_t g_stub_worker;
static job_t g_stub_job;
static queue_t g_stub_queue;
/* bug-451: HOOK_DETAIL_JOB_RUSAGE_UPDATE is the only job hook detail whose
* persist.c arm actually dereferences the cluster_t* it's called with
* (every other case discards it via `(void)va_arg(ap, cluster_t*)`), since
* it must look the job up by jobid via c->jobmgr rather than receiving a
* job_t* directly. Minimal real (non-NULL) cluster+jobmgr+job, built once
* on first use — every other test in this file keeps passing
* (cluster_t*)NULL safely, this is the one exception. */
static cluster_t g_hook_cluster;
static int g_hook_jobobj_ready;
static void ensure_hook_cluster_with_job(uint64_t jobid, unsigned int state)
{
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 64, .delta = 16 };
xce_job_t jdef;
job_submit_data_t submit;
if (!g_hook_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(64, 64));
g_hook_jobobj_ready = 1;
}
if (g_hook_cluster.jobmgr != NULL)
dereference(g_hook_cluster.jobmgr);
memset(&g_hook_cluster, 0, sizeof(g_hook_cluster));
g_hook_cluster.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(g_hook_cluster.jobmgr);
memset(&submit, 0, sizeof(submit));
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = jobid;
jdef.state = state;
jdef.submit = &submit;
TEST_ASSERT_NOT_NULL(makeJob(g_hook_cluster.jobmgr, &jdef));
}
/* Variadic wrappers so tests can pass typed args through a real va_list */
static int call_worker_hook(int type, int sit, int detail, persist_t* p, ...)
{
@ -277,19 +316,16 @@ static void test_job_hook_rusage_writes_wal_entry(void)
uint64_t lsn_before = p->current_lsn;
/* bug-1502 regression: production fires RUSAGE_UPDATE with a real
* xce_job_rusages_t* (56 bytes, NO job-state field) as the second hook
* arg — NOT a job_t*. The old persist_job_hook_fun default case read
* j->job.state at offset 56, one byte past this struct → ASAN
* heap-buffer-overflow. Passing the real minimal struct here (instead of
* the old &g_stub_job, a full job_t that masked the overflow) reproduces
* the production shape exactly. */
xce_job_rusages_t rusages = {0};
rusages.jobid = 0x1234567890ABCDEFULL;
/* bug-451: RUSAGE_UPDATE args are (cluster_t*, xce_job_rusages_t*) —
* NOT (cluster_t*, job_t*) like STOP/RESUME/UPDATE/CKPT/PREASON below.
* A real cluster+jobmgr+job is required (see ensure_hook_cluster_with_
* job) since persist.c's arm looks the job up by jobid, it doesn't
* receive a job_t* directly. */
ensure_hook_cluster_with_job(42, JOB_STAT_RUN);
xce_job_rusages_t rusages = { .jobid = 42 };
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_RUSAGE_UPDATE, p,
(cluster_t*)NULL, &rusages);
&g_hook_cluster, &rusages);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
@ -297,6 +333,29 @@ static void test_job_hook_rusage_writes_wal_entry(void)
printf("test_job_hook_rusage_writes_wal_entry: PASS\n");
}
static void test_job_hook_rusage_unknown_job_is_silently_skipped(void)
{
persist_t* p = make_hook_test_persist();
TEST_ASSERT_NOT_NULL(p);
uint64_t lsn_before = p->current_lsn;
/* jobid 999 was never inserted into g_hook_cluster's jobmgr — the job
* may have already been reaped by the time this hook fires. Must
* return 0 (no error) and append nothing, never crash on a NULL
* job_t*. */
ensure_hook_cluster_with_job(42, JOB_STAT_RUN);
xce_job_rusages_t rusages = { .jobid = 999 };
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_RUSAGE_UPDATE, p,
&g_hook_cluster, &rusages);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT64(lsn_before, p->current_lsn);
cleanup_hook_test_persist(p);
printf("test_job_hook_rusage_unknown_job_is_silently_skipped: PASS\n");
}
static void test_job_hook_setpreason_writes_wal_entry(void)
{
persist_t* p = make_hook_test_persist();
@ -479,6 +538,7 @@ void test_persist_hooks(void)
RUN_TEST(test_job_hook_update_writes_wal_entry);
RUN_TEST(test_job_hook_ckpt_writes_wal_entry);
RUN_TEST(test_job_hook_rusage_writes_wal_entry);
RUN_TEST(test_job_hook_rusage_unknown_job_is_silently_skipped);
RUN_TEST(test_job_hook_setpreason_writes_wal_entry);
RUN_TEST(test_job_hook_clrpreason_writes_wal_entry);
RUN_TEST(test_queue_hook_update_writes_wal_entry);

8
src/controller/testsuite/test_repl_apply.c
View file

@ -50,6 +50,14 @@ int restore_job_state(cluster_t* c, uint64_t jobid, uint32_t state)
return 0;
}
__attribute__((weak))
int restore_job_dispatch_state(cluster_t* c, uint64_t jobid, uint32_t state,
xce_job_alloc_t* alloc)
{
(void)c; (void)jobid; (void)state; (void)alloc;
return 0;
}
__attribute__((weak))
int restore_queue_state(cluster_t* c, const char* name, uint32_t status)
{

105
src/controller/testsuite/test_resval_typeresolve.c Normal file
View file

@ -0,0 +1,105 @@
/* test_resval_typeresolve.c -- resource-value-verification task 1.1
*
* Unity unit tests for resval_typeresolve.c's resource-name -> Value_u_which
* resolution, extracted from test.c's set_resval() (design.md "Testing
* Strategy"). Four cases, matching tasks.md task 1.1 exactly:
*
* 1. A currently-known numeric name (mem) resolves Value_u_fvalue, exactly
* as today (Requirement 1.4) -- GREEN.
* 2. A currently-known string name (type) resolves Value_u_svalue, exactly
* as today (Requirement 1.4) -- GREEN.
* 3. An unrecognized name (r15m) defaults to numeric (Requirement 1.1) --
* RED: task 2.1 has not yet implemented the fallback.
* 4. An unrecognized name with an explicit type-override resolves to the
* requested type (Requirement 1.1) -- RED: task 2.1 has not yet wired
* `type_override` in.
*
* Cases 3 and 4 are INTENTIONALLY FAILING (RED) evidence for this task --
* see resval_typeresolve.h's "TASK 1.1 STATUS" note and tasks.md task 2.1,
* which implements the behavior these two cases assert.
*/
#include <stdio.h>
#include <unity/unity.h>
#include "resval_typeresolve.h"
/* ------------------------------------------------------------------ */
/* Tests */
/* ------------------------------------------------------------------ */
/* 1. Currently-known numeric name -- unchanged behavior (GREEN). */
static void test_known_numeric_name_resolves_fvalue(void)
{
int rc = resval_resolve_type("mem", RESVAL_TYPE_UNKNOWN);
TEST_ASSERT_EQUAL_INT_MESSAGE(
Value_u_fvalue, rc,
"a currently-known numeric resource name (mem) must still resolve "
"Value_u_fvalue exactly as set_resval() does today");
printf("test_known_numeric_name_resolves_fvalue: PASS\n");
}
/* 2. Currently-known string name -- unchanged behavior (GREEN). */
static void test_known_string_name_resolves_svalue(void)
{
int rc = resval_resolve_type("type", RESVAL_TYPE_UNKNOWN);
TEST_ASSERT_EQUAL_INT_MESSAGE(
Value_u_svalue, rc,
"a currently-known string resource name (type) must still resolve "
"Value_u_svalue exactly as set_resval() does today");
printf("test_known_string_name_resolves_svalue: PASS\n");
}
/* 3. Unrecognized name defaults to numeric -- RED (task 2.1 implements). */
static void test_unrecognized_name_defaults_to_numeric(void)
{
int rc = resval_resolve_type("r15m", RESVAL_TYPE_UNKNOWN);
TEST_ASSERT_EQUAL_INT_MESSAGE(
Value_u_fvalue, rc,
"an unrecognized resource name (r15m) with no type override must "
"default to Value_u_fvalue (Requirement 1.1) -- NOT YET "
"IMPLEMENTED, expected RED until task 2.1");
printf("test_unrecognized_name_defaults_to_numeric: PASS\n");
}
/* 4. Unrecognized name + explicit override -- RED (task 2.1 implements). */
static void test_unrecognized_name_with_override_resolves_requested_type(
void)
{
int rc = resval_resolve_type("r15m", Value_u_svalue);
TEST_ASSERT_EQUAL_INT_MESSAGE(
Value_u_svalue, rc,
"an unrecognized resource name (r15m) with an explicit type "
"override must resolve to the requested type (Requirement 1.1) -- "
"NOT YET IMPLEMENTED, expected RED until task 2.1");
printf(
"test_unrecognized_name_with_override_resolves_requested_type: "
"PASS\n");
}
/* ------------------------------------------------------------------ */
/* Standalone Unity main (this binary is its own test executable -- see */
/* meson.build's resval_typeresolve_unit_test) */
/* ------------------------------------------------------------------ */
void setUp(void) { return; }
void tearDown(void) { return; }
int main(void)
{
printf("\n>> resource-value-verification task 1.1: resval_typeresolve "
"type resolution\n\n");
UNITY_BEGIN();
RUN_TEST(test_known_numeric_name_resolves_fvalue);
RUN_TEST(test_known_string_name_resolves_svalue);
RUN_TEST(test_unrecognized_name_defaults_to_numeric);
RUN_TEST(test_unrecognized_name_with_override_resolves_requested_type);
return UNITY_END();
}

681
src/controller/testsuite/test_sched_admin_dispatch.c Normal file
View file

@ -0,0 +1,681 @@
/*
* test_sched_admin_dispatch.c — Unity tests for the OPCode_schedAdmin
* controller dispatch case (scheduler-core spec, task 10.1, Req 5.10).
*
* Exercises myreq_handler's OPCode_schedAdmin case through the real
* decode+dispatch entry point exposed for tests, myreq_handler_for_test
* (cluster.c, #ifdef XCE_TESTING) — same precedent as test_config_repl.c's
* config-push-reply term-adoption tests (myreq_handler_for_test's own doc
* comment cites peer_ctrl_msg_handler_for_test, ha.c, as the original
* precedent for this style of test seam).
*
* Covers this task's two literal Observables:
* - a request against a non-primary controller returns the not-primary
* error immediately, with NO forwarding attempt (c->sched.ep is never
* touched — proven by pointing it at an intentionally-invalid sentinel
* that would crash the process on any dereference, not just asserted on
* the returned status);
* - a request against a primary with no scheduler connected returns the
* not-connected error immediately, carrying an explicit
* SchedAdminReply{error="scheduler not connected"} payload.
*
* Also includes a basic successful forward/reply round trip (register ->
* forward -> a second, independently-dispatched OPCode_reply carrying a
* SchedAdminReply wakes the blocked waiter), since the test scaffolding
* built for the two Observables above made it straightforward. The full
* dispatch test matrix (malformed requests, concurrent/serialized
* round-trips, etc.) belongs to task 10.2.
*
* ── Task 10.2 additions (Req 5.10) ──────────────────────────────────────
*
* Task 10.2's literal scenario list is "standby rejection, disconnected-
* scheduler rejection, forward timeout, and a successful forward/reply
* round trip against the real scheduler-side handler from 9.4". The first
* two are the Observables above (already task 10.1's). This file adds:
*
* - test_schedadmin_forward_timeout: the genuinely-missing scenario --
* no existing test drove the 5s deadline to actually expire. Uses
* schedadmin_set_timeout_ms_for_test (cluster.c, XCE_TESTING-only) to
* shrink the real deadline so pthread_cond_timedwait genuinely times
* out on a fast schedule, rather than mocking the wait away or eating
* a real 5s per test run (no precedent for that in this suite -- see
* the seam's own doc comment in cluster.c for the survey).
*
* - test_schedadmin_round_trip_rpc_status_only /
* test_schedadmin_round_trip_business_error: task 10.1's round-trip
* test only exercises ONE of the three outcome branches in the
* schedAdmin case's post-wait dispatch (cluster.c: got_done+reply,
* got_done+no-reply, timeout) -- the got_done+reply "success" shape.
* These two cover the other real, previously-uncovered branch
* (got_done, no SchedAdminReply payload -- an RPC-level outcome) and a
* business-level error shape (ok=false, e.g. "already recording" --
* requirements.md's "Reply field semantics by action" error rows),
* proving reply.status stays 0 for a completed request that merely
* carries a domain error, never conflated with an RPC/timeout failure.
*
* ON "against the real scheduler-side handler from 9.4": task 9.4's own
* scheduler_req_handler()/sched_handle_admin_request() are proven for
* real, end-to-end, including sched_record_status()'s actual field
* construction, by scheduler/testsuite/test_admin_request.c (already
* landed with task 9.4) -- that file white-box #includes handler.c,
* recording.c, and their scheduler-internal dependencies (core.c,
* driver.c, snapshot*.c) directly into the SCHEDULER's own test binary
* and spies on xcecli_reply_schedadmin() to capture the real reply. This
* controller test binary (controller_unit_test) is a wholly separate
* meson target/executable with its own main() and its own global state
* (cluster_t/ha_state_t fixtures, not xce_scheduler_t) -- pulling
* scheduler/src/handler.c + recording.c (which itself needs core.h and
* the rest of xce_scheduler_t's dependency graph) into THIS binary to
* invoke scheduler_req_handler() literally would require the same kind
* of careful selective-source-linking test_admin_request.c's own header
* comment documents doing for the scheduler side, replicated cross-
* directory inside an already-large, HA-critical controller test target
* -- real risk (symbol/global collisions between two "whole daemon"
* white-box bundles sharing one process image) for no correctness gain,
* since 9.4's own suite already covers that ground rigorously. A genuine
* two-process E2E (real controller + real scheduler binaries, a real
* connection between them) is an E2E/soak-test concern, not this task's
* scope. So: this file proves the CONTROLLER's forward/relay mechanism
* against the real wire contract (xce_encode/xce_decode, the same
* functions production uses) and the exact reply shapes 9.4's real
* sched_record_status()/sched_handle_admin_request() are proven (by
* their own suite) to produce -- not a second, redundant proof that the
* scheduler-side handler itself is correct.
*/
#define _GNU_SOURCE
#include <errno.h>
#include <pthread.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <unity/unity.h>
#include "cluster.h"
#include "controller.capnp.h"
#include "ha.h"
#include "netio.h"
#include "request.h"
#include "typemap.h"
#include "xce/errnox.h"
/* myreq_handler exposed for tests (cluster.c). Packet ownership follows
* myreq_handler's own contract — see its doc comment in cluster.c: `pkt` is
* consumed (freed) by the call, never touch/free it afterward. */
extern int myreq_handler_for_test(endpoint_t* ep, io_packet_t* pkt);
/* Test-only override for the OPCode_schedAdmin case's forward-and-block
* deadline (cluster.c, XCE_TESTING-only, task 10.2) -- see its own doc
* comment in cluster.c for why this seam exists. */
extern void schedadmin_set_timeout_ms_for_test(long ms);
/* ── Fixture ──────────────────────────────────────────────────────────────
*
* The OPCode_schedAdmin dispatch case (and the ha_dispatch_gate primary/
* standby gate ahead of it) only touch c->ha.role, c->ha.config_serve_fence /
* c->ha.upgrade_materialization_fence (both NULL-safe no-ops when
* c->ha.raft_c is left NULL — config_serve_fenced/upgrade_materialization_
* fenced both guard on `if (!c) return 0;`), c->ha.n_peers (0 → single-node
* quorum trivially satisfied, ha_has_write_quorum), c->sched.{lock,ep}, and
* c->myhost (NULL-safe — xce_reply's encode path null-checks its `from`
* argument, matching every other encode_*_ptr in controller.capnp.c). None
* of this needs persist_init/persist_start (unlike test_config_repl.c's
* t42_make_cluster, which exists for config-marker/WAL-touching tests) — a
* bare calloc'd cluster_t is sufficient and keeps this file free of
* temp-directory bookkeeping. */
static cluster_t* sa_make_cluster(void)
{
cluster_t* c = calloc(1, sizeof(cluster_t));
TEST_ASSERT_NOT_NULL(c);
mylock_init(&(c->sched.lock));
return c;
}
static void sa_teardown(cluster_t* c)
{
if (c == NULL)
return;
mylock_fini(&(c->sched.lock));
free(c);
}
/* Build a wire-encoded OPCode_schedAdmin request packet carrying a minimal
* SchedAdminRequest — task 10.1 passes every field through untouched, so the
* exact action/recordFile content is immaterial to these tests. */
static io_packet_t* sa_build_request_pkt(void)
{
sched_admin_req_t areq;
memset(&areq, 0, sizeof(areq));
areq.action = SchedAdminAction_recordStatus;
xce_request_t req;
memset(&req, 0, sizeof(req));
req.opcode = OPCode_schedAdmin;
req.body_which = Request_body_schedadminmsg;
req.body.schedadminmsg = &areq;
io_packet_t* pkt = calloc(1, sizeof(*pkt));
TEST_ASSERT_NOT_NULL(pkt);
TEST_ASSERT_TRUE_MESSAGE(xce_encode(&req, pkt) >= 0,
"failed to encode SchedAdminRequest packet");
return pkt;
}
/* Pop the single reply packet myreq_handler queued on ep->outq and decode it
* into *out (caller must xce_free_request(out) when done with it). */
static void sa_decode_queued_reply(endpoint_t* ep, xce_request_t* out)
{
data_item_t* it = endpoint_deq(ep);
TEST_ASSERT_NOT_NULL_MESSAGE(it, "expected a reply packet on ep->outq");
io_packet_t* pkt = ITPTR_TO_PACKET(it);
memset(out, 0, sizeof(*out));
int rc = xce_decode(pkt, out);
TEST_ASSERT_EQUAL_INT(0, rc);
}
/* Monotonic milliseconds -- for measuring that the forward-timeout test
* genuinely waited out its (shrunk) deadline rather than returning early.
* Same construction as test_commit_resolve.c's mono_ms_r(). */
static uint64_t sa_mono_ms(void)
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (uint64_t)ts.tv_sec * 1000u + (uint64_t)ts.tv_nsec / 1000000u;
}
/* ── Observable 1: non-primary rejects immediately, no forwarding attempt ──
*
* ha_dispatch_gate (ha.c) denies OPCode_schedAdmin for every non-primary
* role — it is absent from the CTRL_STANDBY/CTRL_INIT/CTRL_ELECTING
* allow-lists, exactly like OPCode_upgradeConfig — so the request is
* rejected BEFORE myreq_handler's switch is ever reached; the
* OPCode_schedAdmin case's own ha_is_primary() check is pure defensive
* belt-and-suspenders for a role change racing between the gate and the
* switch. c->sched.ep is set to a deliberately-invalid, non-NULL sentinel:
* if a regression ever let this request reach the forwarding code (both
* mylock_lock and the `sched.ep == NULL` check would pass on a garbage
* non-NULL pointer), xce_schedadmin_req would dereference it and crash —
* making this a self-verifying proof that no forwarding attempt was made,
* not just an assertion on the returned status. */
static void test_schedadmin_rejects_when_not_primary(void)
{
cluster_t* c = sa_make_cluster();
atomic_store(&c->ha.role, CTRL_STANDBY);
endpoint_t* poison = (endpoint_t*)(uintptr_t)0x1;
c->sched.ep = poison;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
ep.owner = c;
io_packet_t* pkt = sa_build_request_pkt();
int rc = myreq_handler_for_test(&ep, pkt);
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rc);
TEST_ASSERT_TRUE_MESSAGE(
c->sched.ep == poison,
"c->sched.ep must be untouched when the request never reaches the "
"dispatch case");
xce_request_t decoded;
sa_decode_queued_reply(&ep, &decoded);
TEST_ASSERT_EQUAL_INT(Request_body_replymsg, decoded.body_which);
TEST_ASSERT_NOT_NULL(decoded.body.replymsg);
TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, decoded.body.replymsg->status);
TEST_ASSERT_EQUAL_INT(RequestReply_data_nodata,
decoded.body.replymsg->data_which);
xce_free_request(&decoded);
sa_teardown(c);
printf("test_schedadmin_rejects_when_not_primary: PASS\n");
}
/* ── Observable 2: primary with no scheduler connected rejects immediately ──
*/
static void test_schedadmin_rejects_when_not_connected(void)
{
cluster_t* c = sa_make_cluster();
atomic_store(&c->ha.role, CTRL_PRIMARY);
c->sched.ep = NULL;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
ep.owner = c;
io_packet_t* pkt = sa_build_request_pkt();
int rc = myreq_handler_for_test(&ep, pkt);
TEST_ASSERT_EQUAL_INT(0, rc);
xce_request_t decoded;
sa_decode_queued_reply(&ep, &decoded);
TEST_ASSERT_EQUAL_INT(Request_body_replymsg, decoded.body_which);
xce_reply_t* rr = decoded.body.replymsg;
TEST_ASSERT_NOT_NULL(rr);
TEST_ASSERT_EQUAL_INT(0, rr->status);
TEST_ASSERT_EQUAL_INT(RequestReply_data_schedadminreplymsg, rr->data_which);
TEST_ASSERT_NOT_NULL(rr->data.schedadminreplymsg);
TEST_ASSERT_EQUAL_STRING("scheduler not connected",
rr->data.schedadminreplymsg->error);
xce_free_request(&decoded);
sa_teardown(c);
printf("test_schedadmin_rejects_when_not_connected: PASS\n");
}
/* ── Observable 2b (design review follow-up, 2026-08-19): admission-
* exhausted requests fast-reject instead of blocking a vtask thread ────────
*
* ha-async-io.md Rule 3: bounded blocking on a vtask thread requires an
* admission reservation. c->vtaskmgr starts gate-DISABLED by a bare
* sa_make_cluster() (liveness_reserved defaults to 0 on a calloc'd
* vtaskmgr_t -- vtaskmgr_blocking_acquire's own "Gate disabled: admit
* uncounted (legacy: nothing reserved)" branch, vtaskmgr.c), matching
* production behavior before vtaskmgr_set_liveness_reserved is called
* (cluster.c's real init path) -- this is why every OTHER test in this
* file, which never calls that setter, is completely unaffected by this
* feature. This test explicitly enables the gate with a 1-slot cap
* (vtaskmgr_set_liveness_reserved(..., 1) floors cap at 1 regardless of
* n_vcpus, vtaskmgr_blocking_cap's own doc comment) and pre-saturates that
* one slot -- simulating a concurrent commit-wait or another in-flight
* schedAdmin request already holding it -- before dispatching. The poison
* c->sched.ep sentinel (same technique test_schedadmin_rejects_when_not_
* primary uses above) proves the fast-reject happens BEFORE
* mylock_lock(&c->sched.lock)/any forwarding attempt, not merely that the
* eventual reply looks right. */
static void test_schedadmin_fast_rejects_when_admission_exhausted(void)
{
cluster_t* c = sa_make_cluster();
atomic_store(&c->ha.role, CTRL_PRIMARY);
endpoint_t* poison = (endpoint_t*)(uintptr_t)0x1;
c->sched.ep = poison;
vtaskmgr_set_liveness_reserved(&c->vtaskmgr, 1);
TEST_ASSERT_EQUAL_INT(0, vtaskmgr_blocking_acquire(&c->vtaskmgr));
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
ep.owner = c;
io_packet_t* pkt = sa_build_request_pkt();
int rc = myreq_handler_for_test(&ep, pkt);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_TRUE_MESSAGE(
c->sched.ep == poison,
"c->sched.ep must be untouched -- admission-exhausted requests must "
"fast-reject before ever locking c->sched.lock or forwarding");
xce_request_t decoded;
sa_decode_queued_reply(&ep, &decoded);
TEST_ASSERT_EQUAL_INT(Request_body_replymsg, decoded.body_which);
xce_reply_t* rr = decoded.body.replymsg;
TEST_ASSERT_NOT_NULL(rr);
TEST_ASSERT_EQUAL_INT(0, rr->status);
TEST_ASSERT_EQUAL_INT(RequestReply_data_schedadminreplymsg, rr->data_which);
TEST_ASSERT_NOT_NULL(rr->data.schedadminreplymsg);
TEST_ASSERT_EQUAL_STRING("server busy, try again",
rr->data.schedadminreplymsg->error);
xce_free_request(&decoded);
vtaskmgr_blocking_release(&c->vtaskmgr); /* release our pre-saturating slot */
sa_teardown(c);
printf("test_schedadmin_fast_rejects_when_admission_exhausted: PASS\n");
}
/* ── Basic round trip: forward -> scheduler reply wakes the waiter ──────────
*
* c->sched.ep points at a real (if unconnected) endpoint_t so the dispatch
* case's forward+wait path is exercised for real: xce_schedadmin_req just
* encodes+enqueues (like xce_reply — see request.c), so a zeroed,
* non-socket-backed endpoint_t is sufficient — the same technique the two
* tests above already rely on for the caller-side ep. A second thread plays
* the scheduler: after a short delay it independently dispatches an
* OPCode_reply/SchedAdminReply through myreq_handler_for_test on the SAME
* cluster_t, exactly how the real reply arrives in production (a second
* vtask handling c->sched.ep's inbound traffic) — this drives the real
* schedadmin_signal()/demux path in the OPCode_reply case, not a shortcut. */
typedef struct {
cluster_t* c;
} sa_reply_thread_arg_t;
static void* sa_deliver_scheduler_reply(void* argp)
{
sa_reply_thread_arg_t* arg = (sa_reply_thread_arg_t*)argp;
/* Give the main thread time to register the pending waiter and start
* blocking on the condvar — comfortably inside the dispatch case's 5s
* deadline. */
usleep(200000);
sched_admin_reply_t sreply;
memset(&sreply, 0, sizeof(sreply));
sreply.ok = 1;
sreply.recording = 1;
sreply.file = "/tmp/xce_sa_roundtrip.replay";
sreply.eventCount = 42;
sreply.startedAt = 1234;
xce_reply_t rr;
memset(&rr, 0, sizeof(rr));
rr.opcode = OPCode_schedAdmin;
rr.status = 0;
rr.data_which = RequestReply_data_schedadminreplymsg;
rr.data.schedadminreplymsg = &sreply;
xce_request_t req;
memset(&req, 0, sizeof(req));
req.opcode = OPCode_reply;
req.body_which = Request_body_replymsg;
req.body.replymsg = &rr;
io_packet_t* pkt = calloc(1, sizeof(*pkt));
TEST_ASSERT_NOT_NULL(pkt);
TEST_ASSERT_TRUE(xce_encode(&req, pkt) >= 0);
endpoint_t sched_reply_ep;
memset(&sched_reply_ep, 0, sizeof(sched_reply_ep));
sched_reply_ep.owner = arg->c;
myreq_handler_for_test(&sched_reply_ep, pkt);
return NULL;
}
static void test_schedadmin_round_trip_success(void)
{
cluster_t* c = sa_make_cluster();
atomic_store(&c->ha.role, CTRL_PRIMARY);
endpoint_t sched_conn;
memset(&sched_conn, 0, sizeof(sched_conn));
c->sched.ep = &sched_conn;
sa_reply_thread_arg_t arg = { .c = c };
pthread_t tid;
TEST_ASSERT_EQUAL_INT(
0, pthread_create(&tid, NULL, sa_deliver_scheduler_reply, &arg));
endpoint_t caller_ep;
memset(&caller_ep, 0, sizeof(caller_ep));
caller_ep.owner = c;
io_packet_t* pkt = sa_build_request_pkt();
int rc = myreq_handler_for_test(&caller_ep, pkt);
pthread_join(tid, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, rc,
"the round trip must complete (woken by the reply), not time out");
xce_request_t decoded;
sa_decode_queued_reply(&caller_ep, &decoded);
xce_reply_t* rr = decoded.body.replymsg;
TEST_ASSERT_NOT_NULL(rr);
TEST_ASSERT_EQUAL_INT(0, rr->status);
TEST_ASSERT_EQUAL_INT(RequestReply_data_schedadminreplymsg, rr->data_which);
sched_admin_reply_t* got = rr->data.schedadminreplymsg;
TEST_ASSERT_NOT_NULL(got);
TEST_ASSERT_EQUAL_INT(1, got->ok);
TEST_ASSERT_EQUAL_INT(1, got->recording);
TEST_ASSERT_EQUAL_UINT64(42, got->eventCount);
TEST_ASSERT_EQUAL_UINT64(1234, got->startedAt);
TEST_ASSERT_EQUAL_STRING("/tmp/xce_sa_roundtrip.replay", got->file);
xce_free_request(&decoded);
sa_teardown(c);
printf("test_schedadmin_round_trip_success: PASS\n");
}
/* ── Task 10.2: forward timeout ─────────────────────────────────────────────
*
* c->sched.ep points at a real (if unconnected) endpoint_t, exactly like the
* round-trip test above, so xce_schedadmin_req really forwards and the
* dispatch case really blocks on pthread_cond_timedwait -- but NO second
* thread ever delivers a reply. schedadmin_set_timeout_ms_for_test shrinks
* the deadline so the real wait-then-give-up path completes on a fast test
* schedule; the override is restored to the production default IMMEDIATELY
* after myreq_handler_for_test returns, before any TEST_ASSERT below can
* longjmp out on failure (Unity's assertion macros unwind via longjmp) --
* so a failing assertion in this test can never leave a later test in this
* binary running against a shrunk deadline. */
static void test_schedadmin_forward_timeout(void)
{
cluster_t* c = sa_make_cluster();
atomic_store(&c->ha.role, CTRL_PRIMARY);
endpoint_t sched_conn;
memset(&sched_conn, 0, sizeof(sched_conn));
c->sched.ep = &sched_conn;
endpoint_t caller_ep;
memset(&caller_ep, 0, sizeof(caller_ep));
caller_ep.owner = c;
io_packet_t* pkt = sa_build_request_pkt();
const long timeout_ms = 200;
schedadmin_set_timeout_ms_for_test(timeout_ms);
uint64_t t0 = sa_mono_ms();
int rc = myreq_handler_for_test(&caller_ep, pkt);
uint64_t took = sa_mono_ms() - t0;
schedadmin_set_timeout_ms_for_test(5000); /* restore before any assert */
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, rc,
"the dispatch case always replies -- it does not propagate "
"-ETIMEDOUT as its own return, even on a genuine timeout");
TEST_ASSERT_TRUE_MESSAGE(
took + 20 >= (uint64_t)timeout_ms,
"must not give up meaningfully before the (shrunk) deadline -- this "
"is what proves the real pthread_cond_timedwait deadline fired, not "
"a shortcut that skipped the wait");
TEST_ASSERT_TRUE_MESSAGE(
took < (uint64_t)timeout_ms + 2000,
"must give up within a bounded window after the deadline, not hang");
xce_request_t decoded;
sa_decode_queued_reply(&caller_ep, &decoded);
TEST_ASSERT_EQUAL_INT(Request_body_replymsg, decoded.body_which);
xce_reply_t* rr = decoded.body.replymsg;
TEST_ASSERT_NOT_NULL(rr);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, rr->status,
"a timeout is relayed as an ok-RPC SchedAdminReply carrying "
"error=\"request timeout\", per requirements.md AC 10 -- not a "
"negative RPC status");
TEST_ASSERT_EQUAL_INT(RequestReply_data_schedadminreplymsg, rr->data_which);
sched_admin_reply_t* got = rr->data.schedadminreplymsg;
TEST_ASSERT_NOT_NULL(got);
TEST_ASSERT_EQUAL_INT(0, got->ok);
TEST_ASSERT_EQUAL_STRING("request timeout", got->error);
xce_free_request(&decoded);
sa_teardown(c);
printf("test_schedadmin_forward_timeout: PASS\n");
}
/* ── Task 10.2: additional round-trip outcome branches ──────────────────────
*
* Shared "second thread plays the scheduler" helper, generalized from
* test_schedadmin_round_trip_success's sa_deliver_scheduler_reply above so
* both new tests below can drive either outcome (a SchedAdminReply payload,
* or a bare RPC status with none) through the SAME real
* schedadmin_signal()/demux path, without duplicating the thread/encode
* boilerplate. Task 10.1's own sa_deliver_scheduler_reply is left untouched. */
typedef struct {
cluster_t* c;
int status;
sched_admin_reply_t* payload; /* NULL => RequestReply_data_nodata */
} sa_deliver_arg_t;
static void* sa_deliver_reply_generic(void* argp)
{
sa_deliver_arg_t* arg = (sa_deliver_arg_t*)argp;
usleep(200000); /* same margin as sa_deliver_scheduler_reply above */
xce_reply_t rr;
memset(&rr, 0, sizeof(rr));
rr.opcode = OPCode_schedAdmin;
rr.status = arg->status;
if (arg->payload != NULL) {
rr.data_which = RequestReply_data_schedadminreplymsg;
rr.data.schedadminreplymsg = arg->payload;
} else {
rr.data_which = RequestReply_data_nodata;
}
xce_request_t req;
memset(&req, 0, sizeof(req));
req.opcode = OPCode_reply;
req.body_which = Request_body_replymsg;
req.body.replymsg = &rr;
io_packet_t* pkt = calloc(1, sizeof(*pkt));
TEST_ASSERT_NOT_NULL(pkt);
TEST_ASSERT_TRUE(xce_encode(&req, pkt) >= 0);
endpoint_t sched_reply_ep;
memset(&sched_reply_ep, 0, sizeof(sched_reply_ep));
sched_reply_ep.owner = arg->c;
myreq_handler_for_test(&sched_reply_ep, pkt);
return NULL;
}
/* Exercises the got_done && got_reply==NULL branch (cluster.c, OPCode_reply
* case): the scheduler's OPCode_reply can legitimately carry no
* SchedAdminReply payload (an RPC-level outcome) -- the controller must
* still reply promptly, relaying the bare status, not treat it as a
* timeout. Never exercised by task 10.1's round-trip test, which always
* supplies a payload. */
static void test_schedadmin_round_trip_rpc_status_only(void)
{
cluster_t* c = sa_make_cluster();
atomic_store(&c->ha.role, CTRL_PRIMARY);
endpoint_t sched_conn;
memset(&sched_conn, 0, sizeof(sched_conn));
c->sched.ep = &sched_conn;
sa_deliver_arg_t arg = { .c = c, .status = -EIO, .payload = NULL };
pthread_t tid;
TEST_ASSERT_EQUAL_INT(
0, pthread_create(&tid, NULL, sa_deliver_reply_generic, &arg));
endpoint_t caller_ep;
memset(&caller_ep, 0, sizeof(caller_ep));
caller_ep.owner = c;
io_packet_t* pkt = sa_build_request_pkt();
int rc = myreq_handler_for_test(&caller_ep, pkt);
pthread_join(tid, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, rc,
"the dispatch case always replies, even when the scheduler's "
"OPCode_reply carries no SchedAdminReply payload");
xce_request_t decoded;
sa_decode_queued_reply(&caller_ep, &decoded);
xce_reply_t* rr = decoded.body.replymsg;
TEST_ASSERT_NOT_NULL(rr);
TEST_ASSERT_EQUAL_INT_MESSAGE(
-EIO, rr->status,
"a payload-less scheduler reply relays its bare RPC status verbatim");
TEST_ASSERT_EQUAL_INT(RequestReply_data_nodata, rr->data_which);
xce_free_request(&decoded);
sa_teardown(c);
printf("test_schedadmin_round_trip_rpc_status_only: PASS\n");
}
/* A business-level error from the scheduler (ok=false, e.g. "already
* recording" -- requirements.md's "Reply field semantics by action" error
* rows) is a NORMAL completed round trip, not an RPC failure. Proves the
* controller relays that shape verbatim with reply.status == 0 -- a
* domain error is never conflated with an RPC/timeout error. */
static void test_schedadmin_round_trip_business_error(void)
{
cluster_t* c = sa_make_cluster();
atomic_store(&c->ha.role, CTRL_PRIMARY);
endpoint_t sched_conn;
memset(&sched_conn, 0, sizeof(sched_conn));
c->sched.ep = &sched_conn;
sched_admin_reply_t sreply;
memset(&sreply, 0, sizeof(sreply));
sreply.ok = 0;
sreply.error = "already recording";
sreply.recording = 1;
sreply.file = "/var/log/xce/scheduler-replay-x.replay";
sreply.eventCount = 7;
sreply.startedAt = 999;
sa_deliver_arg_t arg = { .c = c, .status = 0, .payload = &sreply };
pthread_t tid;
TEST_ASSERT_EQUAL_INT(
0, pthread_create(&tid, NULL, sa_deliver_reply_generic, &arg));
endpoint_t caller_ep;
memset(&caller_ep, 0, sizeof(caller_ep));
caller_ep.owner = c;
io_packet_t* pkt = sa_build_request_pkt();
int rc = myreq_handler_for_test(&caller_ep, pkt);
pthread_join(tid, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, rc,
"a business-level error from the scheduler is still a completed "
"round trip");
xce_request_t decoded;
sa_decode_queued_reply(&caller_ep, &decoded);
xce_reply_t* rr = decoded.body.replymsg;
TEST_ASSERT_NOT_NULL(rr);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, rr->status,
"ok=false is a business error, not an RPC/timeout error -- "
"reply.status must stay 0");
TEST_ASSERT_EQUAL_INT(RequestReply_data_schedadminreplymsg, rr->data_which);
sched_admin_reply_t* got = rr->data.schedadminreplymsg;
TEST_ASSERT_NOT_NULL(got);
TEST_ASSERT_EQUAL_INT(0, got->ok);
TEST_ASSERT_EQUAL_STRING("already recording", got->error);
TEST_ASSERT_EQUAL_INT(1, got->recording);
TEST_ASSERT_EQUAL_STRING("/var/log/xce/scheduler-replay-x.replay",
got->file);
TEST_ASSERT_EQUAL_UINT64(7, got->eventCount);
TEST_ASSERT_EQUAL_UINT64(999, got->startedAt);
xce_free_request(&decoded);
sa_teardown(c);
printf("test_schedadmin_round_trip_business_error: PASS\n");
}
/* ── Suite entry point ──────────────────────────────────────────────────── */
void test_sched_admin_dispatch(void)
{
UnitySetTestFile(__FILE__);
RUN_TEST(test_schedadmin_rejects_when_not_primary);
RUN_TEST(test_schedadmin_rejects_when_not_connected);
RUN_TEST(test_schedadmin_fast_rejects_when_admission_exhausted);
RUN_TEST(test_schedadmin_round_trip_success);
RUN_TEST(test_schedadmin_forward_timeout);
RUN_TEST(test_schedadmin_round_trip_rpc_status_only);
RUN_TEST(test_schedadmin_round_trip_business_error);
}

360
src/controller/testsuite/test_sched_worker_change.c Normal file
View file

@ -0,0 +1,360 @@
/* test_sched_worker_change.c — regression test for bug-1578
*
* bug-1578: sched_worker_change() (sched.c) aliases w->host.resvals_to_sched
* / nresvals_to_sched directly onto a local xlist_t's backing array
* (resvalist.data), pushes it to the scheduler connection, then calls
* xlist_fini(&resvalist) -- which frees that backing array -- WITHOUT first
* resetting w->host.resvals_to_sched to NULL / nresvals_to_sched to 0. That
* left a DANGLING pointer + a stale nonzero count on the worker_t struct for
* the rest of that worker's lifetime; the next code to read
* resvals_to_sched (production: persist_worker_hook_fun ->
* capnp_encode_worker -> encode_Worker -> encode_NameValue_list ->
* encode_NameValue, on worker disconnect) dereferenced freed memory ->
* SIGSEGV.
*
* This bug had ONE PRIOR fix attempt (commit e5fde7b5) that silently never
* took effect because it edited code inside the same dead "#if 0" (FIXME-
* commented) block sched_worker_change() has always had (introduced
* already-disabled by commit acfbec875) -- i.e. this exact bug
* already regressed once before, silently, for lack of test coverage. The
* real fix (2026-08-18) replaces that dead block with a direct
* `w->host.nresvals_to_sched = 0; w->host.resvals_to_sched = NULL;` before
* `xlist_fini(&resvalist);`, mirroring the sibling apiserver_worker_change()
* (apiserver.c) that never had this bug.
*
* Fixture: sched_worker_change()/sched_worker_dead() are static to sched.c,
* so this file drives them through the same public entry points production
* code uses, exactly mirroring test_job_array_sched_push.c's established
* pattern for this same standalone-binary strategy (see meson.build's
* comment on sched_push_unit_test for why sched.c/dataex.c/egress.c must be
* linked for REAL here rather than folded into controller_unit_test, and
* why check_worker_() below is stubbed rather than linking all of opget.c):
* on_scheduler_join() installs sched_worker_hook_fun into a real
* cluster_t's hooksys (once per process -- hook_installed is a static
* inside on_scheduler_join), and xce_run_hooks() fires
* XCE_CONTROLLER_HOOK_WORKER / HOOK_DETAIL_WORKER_DEAD exactly as
* handle_worker_leave() -> mark_worker_unavail() does in production.
*
* Test 1 asserts the exact condition the fix guarantees and the bug
* violated: after the hook fires, w->host.resvals_to_sched == NULL and
* w->host.nresvals_to_sched == 0 -- never a dangling pointer + stale count.
*
* Test 2 goes further and proves it under ASAN: immediately after the hook
* fires, it calls the REAL production encoder encode_Worker() (controller.
* capnp.c, the exact function bug-1578's crash backtrace bottoms out in --
* capnp_encode_worker/persist_worker_hook_fun's own callee) directly on the
* SAME worker, exactly as persist_worker_hook_fun does on worker disconnect.
* Post-fix this is a correct, harmless 0-entry encode. If the fix ever
* regresses (nresvals_to_sched stale-nonzero, resvals_to_sched dangling),
* encode_NameValue_list's loop reads resvals_to_sched[i] out of the freed
* backing array -- an immediate ASAN heap-use-after-free, not just a failed
* assertion.
*/
#include <capnp_c.h>
#include <errno.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
#include "cluster.h"
#include "controller.capnp.h"
#include "hook.h"
#include "kvmgr.h"
#include "netio.h"
#include "request.h"
#include "resobj.h"
#include "xce/cluster_def.h"
#include "xce/types.h"
#include "xlist.h"
#include "xstring.h"
#include "zarray.h"
extern int on_scheduler_join(cluster_t*);
/* This file builds a STANDALONE test binary (meson.build's
* sched_worker_change_unit_test) linking the real sched.c + its real
* dataex.c/egress.c dependencies, mirroring sched_push_unit_test exactly
* (see meson.build's comment on that target for why: several existing
* controller_unit_test_src files already provide their OWN local fake
* definitions of on_scheduler_join()/notify()/collect_confvars(), so linking
* the real sched.c/dataex.c into that shared binary collides at link time).
* opget.c's check_worker_() is sched.c's only remaining unresolved
* reference (from xcesched_push_workers(), reached only through
* on_scheduler_join()'s push-on-join branch, which this file's setup
* deliberately never triggers at runtime -- c->sched.ep is NULL during that
* call -- but the symbol must still resolve at link time). Stubbed locally
* rather than linking all of opget.c, identical to test_job_array_sched_
* push.c's own stub. */
int check_worker_(xce_worker_t* w) { return (w != NULL) ? 1 : 0; }
/* ------------------------------------------------------------------ */
/* Fixture */
/* ------------------------------------------------------------------ */
static cluster_t g_swc_c;
static endpoint_t g_swc_ep;
static int g_swc_hook_wired;
static var g_swc_resmgr;
static void swc_val_killer(void* p) { free(p); }
static void swc_drain_ep(endpoint_t* ep)
{
data_item_t* it;
while ((it = endpoint_deq(ep)) != NULL) {
io_packet_t* pkt = ITPTR_TO_PACKET(it);
free_packet(pkt);
free(pkt);
}
}
/* Registers one resource ("mem", numeric) in a fresh resmgr and returns its
* xce_resource_t* (idx assigned by makeResource/key_add, matching production
* cluster_build_resources()'s own makeResource() call). */
static xce_resource_t* swc_setup(void)
{
kvmgr_params_t resmgr_params
= { .key_type = KVMGR_KEY_STRING,
.nslots = 64,
.delta = 16,
.datakiller = NULL };
xce_resource_t rdef;
xce_resource_t* r;
if (!g_swc_hook_wired) {
memset(&g_swc_c, 0, sizeof(g_swc_c));
TEST_ASSERT_EQUAL_INT(
0, xce_hook_init(&g_swc_c.hooksys, XCE_CONTROLLER_HOOK_MAX));
/* sched.ep == NULL here so on_scheduler_join's push-on-join branch
* is a no-op -- this call's only effect is installing
* sched_worker_hook_fun (and siblings) into g_swc_c.hooksys, once,
* process-wide (hook_installed is a static inside
* on_scheduler_join). */
g_swc_c.sched.ep = NULL;
TEST_ASSERT_EQUAL_INT(0, on_scheduler_join(&g_swc_c));
g_swc_hook_wired = 1;
}
g_swc_resmgr = makeInstance(&xkvmgr_type, &resmgr_params);
TEST_ASSERT_NOT_NULL(g_swc_resmgr);
memset(&rdef, 0, sizeof(rdef));
rdef.name = (char*)"mem";
rdef.valtype = XCE_VAL_NUMERIC;
rdef.idx = -1;
r = makeResource(g_swc_resmgr, &rdef);
TEST_ASSERT_NOT_NULL(r);
memset(&g_swc_ep, 0, sizeof(g_swc_ep));
g_swc_ep.fd = -1;
/* c->sched.ep != NULL is sched_worker_change()'s guard for actually
* pushing to the scheduler (situation == HOOK_AFTER_EVENT branch) --
* required to reach the collect_resval()/resvalist code path under
* test at all. A zeroed, fd=-1 stack endpoint_t queues packets on
* ep->outq without touching a real socket, exactly as test_job_array_
* sched_push.c's own fixture does for the JOB hook. */
g_swc_c.sched.ep = &g_swc_ep;
g_swc_c.resmgr = g_swc_resmgr;
g_swc_c.running = 1;
g_swc_c.myhost = &g_swc_c.myhost_info;
g_swc_c.myhost_info.name = (char*)"ctrl-swc";
g_swc_c.myhost_info.key = (char*)"";
return r;
}
static void swc_teardown(void)
{
swc_drain_ep(&g_swc_ep);
g_swc_c.sched.ep = NULL;
g_swc_c.resmgr = NULL;
if (g_swc_resmgr != NULL) {
dereference(g_swc_resmgr);
g_swc_resmgr = NULL;
}
}
/* Builds a plain, stack-owned worker_t with one populated resval -- NOT
* routed through makeWorker()/wobj.c's RTTI worker-object machinery
* (unneeded: sched_worker_change() only ever reads w->host.name/resvals and
* w->idx, none of which requires a real registered worker instance),
* mirroring test_job_array_sched_push.c's identical "plain struct, not
* through the real object constructor" rationale for its job_t fixtures.
* The resval is written the same way update_val_res() (worker.c) writes a
* real resupdate: a heap xce_val_t* stored via zarray_put() at the
* resource's own idx. */
static void swc_make_worker(worker_t* w, const char* name, xce_resource_t* r,
double fval)
{
xce_val_t* v;
memset(w, 0, sizeof(*w));
w->host.name = (char*)name;
w->idx = 0;
zarray_init(&(w->host.resvals), 16, swc_val_killer);
v = (xce_val_t*)calloc(1, sizeof(*v));
TEST_ASSERT_NOT_NULL(v);
v->which = Value_u_fvalue;
v->u.fvalue = fval;
TEST_ASSERT_TRUE(zarray_put(&(w->host.resvals), r->idx, v) >= 0);
}
static void swc_free_worker(worker_t* w) { zarray_fini(&(w->host.resvals)); }
static int swc_fire_worker_dead(worker_t* w)
{
return xce_run_hooks(&g_swc_c.hooksys, XCE_CONTROLLER_HOOK_WORKER,
HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_DEAD, &g_swc_c,
w);
}
/* Mirrors capnp_encode_worker() (persist.c) / tace_encode_worker()
* (test_apply_committed_entry.c) exactly -- the real production helper that
* wraps encode_Worker() for a malloc'd Cap'n Proto buffer. Not linking
* persist.c itself (heavy, unrelated dependency chain for this fix); calling
* encode_Worker() directly reaches the identical bug-1578 crash-site
* function (controller.capnp.c) with zero loss of fidelity -- persist.c's
* capnp_encode_worker() is a 20-line pass-through to it. */
static int swc_encode_worker(const xce_worker_t* w, uint8_t** out,
size_t* out_len)
{
struct capn c;
struct capn_segment* cs;
struct Worker cw;
Worker_ptr wp;
capn_init_malloc(&c);
cs = capn_root(&c).seg;
encode_Worker(cs, &cw, (xce_worker_t*)w);
wp = new_Worker(cs);
write_Worker(&cw, wp);
capn_setp(capn_root(&c), 0, wp.p);
*out_len = capn_size(&c);
*out = malloc(*out_len);
if (*out == NULL) {
capn_free(&c);
return -ENOMEM;
}
capn_write_mem(&c, *out, *out_len, 0);
capn_free(&c);
return 0;
}
/* ------------------------------------------------------------------ */
/* Test 1: the exact condition the fix guarantees */
/* ------------------------------------------------------------------ */
static void test_worker_dead_leaves_resvals_to_sched_null_and_zero(void)
{
xce_resource_t* r;
worker_t w;
int rc;
r = swc_setup();
swc_make_worker(&w, "swc-worker-1", r, 42.5);
rc = swc_fire_worker_dead(&w);
TEST_ASSERT_EQUAL_INT(0, rc);
/* bug-1578: before the fix, sched_worker_change() aliased these two
* fields directly onto a local xlist_t's backing array, then freed that
* array (xlist_fini) WITHOUT resetting them -- leaving
* resvals_to_sched a dangling pointer and nresvals_to_sched a stale
* nonzero count. This is the exact condition the fix guarantees. */
TEST_ASSERT_NULL_MESSAGE(
w.host.resvals_to_sched,
"bug-1578 regression: w->host.resvals_to_sched must be NULL after "
"sched_worker_change() completes -- a non-NULL value here is a "
"dangling pointer into memory sched_worker_change() already freed "
"via xlist_fini(&resvalist)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, w.host.nresvals_to_sched,
"bug-1578 regression: w->host.nresvals_to_sched must be 0 after "
"sched_worker_change() completes -- a nonzero stale count paired "
"with the dangling pointer above is exactly what let production "
"code (persist_worker_hook_fun -> capnp_encode_worker) iterate "
"into freed memory and SIGSEGV");
swc_free_worker(&w);
swc_teardown();
printf(
"test_worker_dead_leaves_resvals_to_sched_null_and_zero: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Test 2: ASAN-strength proof -- re-run the real crash-site encoder */
/* ------------------------------------------------------------------ */
static void test_worker_dead_then_encode_worker_is_asan_clean(void)
{
xce_resource_t* r;
worker_t w;
int rc;
uint8_t* buf = NULL;
size_t len = 0;
r = swc_setup();
swc_make_worker(&w, "swc-worker-2", r, 7.0);
rc = swc_fire_worker_dead(&w);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Exactly what persist_worker_hook_fun() does on the SAME
* HOOK_DETAIL_WORKER_DEAD event in production (capnp_encode_worker ->
* encode_Worker -> encode_NameValue_list -> encode_NameValue). Post-fix,
* w.host.nresvals_to_sched == 0, so encode_NameValue_list's loop never
* executes -- a correct, harmless empty encode. If sched_worker_change()
* ever regresses to the pre-fix behavior (dangling resvals_to_sched +
* stale nonzero nresvals_to_sched), this call reads resvals_to_sched[i]
* out of memory sched_worker_change() already freed -- an ASAN
* heap-use-after-free abort, not merely a failed assertion. */
rc = swc_encode_worker(&w.host, &buf, &len);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, rc, "encode_Worker() of the just-processed worker must succeed");
TEST_ASSERT_NOT_NULL(buf);
TEST_ASSERT_TRUE(len > 0);
free(buf);
swc_free_worker(&w);
swc_teardown();
printf(
"test_worker_dead_then_encode_worker_is_asan_clean: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Standalone Unity main (this binary is its own test executable -- see */
/* meson.build's sched_worker_change_unit_test) */
/* ------------------------------------------------------------------ */
void setUp(void) { return; }
void tearDown(void) { return; }
int main(void)
{
printf("\n>> bug-1578: sched_worker_change() dangling-pointer "
"regression test\n\n");
/* makeResource()/makeString() (resobj.c/xstring.c) need the global
* xstring intern table initialized before first use -- same requirement
* test_worker_join.c documents at its own RUN_TEST call site. This is a
* standalone binary (no other test file shares process init with it),
* so it must be done exactly once here. */
xstring_table_init(139);
UNITY_BEGIN();
RUN_TEST(test_worker_dead_leaves_resvals_to_sched_null_and_zero);
RUN_TEST(test_worker_dead_then_encode_worker_is_asan_clean);
return UNITY_END();
}

250
src/controller/testsuite/test_term_adoption_vectors.c Normal file
View file

@ -0,0 +1,250 @@
/*
* test_term_adoption_vectors.c — bug-1484 regression lock.
*
* bug-1421 added two term-discovery fallback vectors so a stale-term primary
* steps down instead of retrying AppendEntries against a follower that will
* reject forever:
* (a) heartbeat-reply state-report term adoption (ha.c ~6690-6711)
* (b) WAL-NACK reply_term adoption inside repl_ack_decode_and_dispatch
* (ha.c ~6805-6814)
*
* Neither vector had a test driving the real repl_ack_decode_and_dispatch
* path and asserting raft_on_higher_term_observed actually fires --
* test_intake_higher_term_heartbeat_adopts_term (test_term_safety_adapter.c)
* looks like a match but drives a different code path (the AE-request
* UpdateTerm vector, not this AE-reply/state-report fallback). This file
* closes that gap, mirroring the pattern task 2.2's
* test_config_push_reply_higher_term_adopts_and_steps_down (test_config_repl.c)
* already established for the third (config-push) vector.
*
* Requirements: raft-adapter-conformance Req 7.5 (bug-1484 residual).
*/
#define _GNU_SOURCE
#include <capnp_c.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
#include "controller.capnp.h"
#include "ha.h"
#include "netio.h"
#include "xce/cluster_def.h"
/* ── ha.c test seams ─────────────────────────────────────────────────────── */
extern int peer_ctrl_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt);
extern endpoint_t *standby_ep_ctx_build_for_test(endpoint_t *ep, ha_state_t *ha,
cluster_t *c, int peer_idx);
static ha_config_t g_cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 6000,
.election_timeout_ms = 500,
.probe_connect_timeout_ms = 50,
.max_promotion_retries = 3,
.election_jitter_max_ms = 0,
.election_priority_step_ms = 500,
};
/* Build Request{opcode=reply, body.replymsg=RequestReply{data_which=
* stateReportMsg, data.stateReportMsg=StateReport{appliedLsn,configTag,term}}}
* -- the exact wire shape ha_nop_reply_state_report populates in production
* (ha.c:3802-3811), wrapped by xce_reply() the same way cpr22's config-push
* reply builder mirrors handle_repl_config_push's send_config_push_reply. */
static io_packet_t *build_state_report_pkt(uint64_t applied_lsn,
uint64_t config_tag, uint64_t term)
{
struct capn cs;
struct capn_segment *seg;
struct StateReport sr_capn;
StateReport_ptr sr_ptr;
struct RequestReply rr_capn;
RequestReply_ptr rr_ptr;
struct Request rq_capn;
Request_ptr rq_ptr;
io_packet_t *pkt;
capn_init_malloc(&cs);
seg = capn_root(&cs).seg;
memset(&sr_capn, 0, sizeof(sr_capn));
sr_capn.appliedLsn = applied_lsn;
sr_capn.configTag = config_tag;
sr_capn.term = term;
sr_ptr = new_StateReport(seg);
write_StateReport(&sr_capn, sr_ptr);
memset(&rr_capn, 0, sizeof(rr_capn));
rr_capn.seq = 1;
rr_capn.opcode = OPCode_nop;
rr_capn.status = 0;
rr_capn.data_which = RequestReply_data_stateReportMsg;
rr_capn.data.stateReportMsg = sr_ptr;
rr_ptr = new_RequestReply(seg);
write_RequestReply(&rr_capn, rr_ptr);
memset(&rq_capn, 0, sizeof(rq_capn));
rq_capn.opcode = OPCode_reply;
rq_capn.seq = 1;
rq_capn.body_which = Request_body_replymsg;
rq_capn.body.replymsg = rr_ptr;
rq_ptr = new_Request(seg);
write_Request(&rq_capn, rq_ptr);
capn_setp(capn_root(&cs), 0, rq_ptr.p);
pkt = calloc(1, sizeof(*pkt));
TEST_ASSERT_NOT_NULL_MESSAGE(pkt, "state-report pkt alloc");
pkt->sz = capn_size(&cs);
TEST_ASSERT_TRUE_MESSAGE(alloc_packet(pkt) >= 0, "state-report pkt data alloc");
capn_write_mem(&cs, (uint8_t *)pkt->data, (size_t)pkt->sz, 0);
capn_free(&cs);
return pkt;
}
/* Build Request{opcode=reply, body.replymsg=RequestReply{data_which=
* walackreplymsg, data.walackreplymsg=WalAckReply{term,lsn,success=0}}} --
* a WAL catch-up entry NACK carrying a higher term. */
static io_packet_t *build_wal_nack_pkt(uint64_t lsn, uint64_t term)
{
struct capn cs;
struct capn_segment *seg;
struct WalAckReply wa_capn;
WalAckReply_ptr wa_ptr;
struct RequestReply rr_capn;
RequestReply_ptr rr_ptr;
struct Request rq_capn;
Request_ptr rq_ptr;
io_packet_t *pkt;
capn_init_malloc(&cs);
seg = capn_root(&cs).seg;
memset(&wa_capn, 0, sizeof(wa_capn));
wa_capn.term = term;
wa_capn.lsn = lsn;
wa_capn.success = 0;
wa_ptr = new_WalAckReply(seg);
write_WalAckReply(&wa_capn, wa_ptr);
memset(&rr_capn, 0, sizeof(rr_capn));
rr_capn.seq = 1;
rr_capn.opcode = OPCode_nop;
rr_capn.status = 0;
rr_capn.data_which = RequestReply_data_walackreplymsg;
rr_capn.data.walackreplymsg = wa_ptr;
rr_ptr = new_RequestReply(seg);
write_RequestReply(&rr_capn, rr_ptr);
memset(&rq_capn, 0, sizeof(rq_capn));
rq_capn.opcode = OPCode_reply;
rq_capn.seq = 1;
rq_capn.body_which = Request_body_replymsg;
rq_capn.body.replymsg = rr_ptr;
rq_ptr = new_Request(seg);
write_Request(&rq_capn, rq_ptr);
capn_setp(capn_root(&cs), 0, rq_ptr.p);
pkt = calloc(1, sizeof(*pkt));
TEST_ASSERT_NOT_NULL_MESSAGE(pkt, "wal-nack pkt alloc");
pkt->sz = capn_size(&cs);
TEST_ASSERT_TRUE_MESSAGE(alloc_packet(pkt) >= 0, "wal-nack pkt data alloc");
capn_write_mem(&cs, (uint8_t *)pkt->data, (size_t)pkt->sz, 0);
capn_free(&cs);
return pkt;
}
/* Shared fixture: a CTRL_PRIMARY node holding leadership at term 5, with a
* promoted "heartbeat standby" peer endpoint (vctx->ctx == NULL, peer_idx
* absolute -- ha.c:9126's documented shape) routed via
* peer_ctrl_msg_handler_inline's CTRL_PRIMARY branch (repl_ack_decode_and_
* dispatch). c is deliberately NULL: every c-dependent step downstream of
* the term check is guarded (`if (c) {...}`), so this drives the exact same
* term-adoption code the production wire path runs, without needing a live
* cluster_t/persist/replicator fixture. */
static void tav_setup(ha_state_t *ha, endpoint_t *ep)
{
TEST_ASSERT_EQUAL_INT(0, ha_init(ha, &g_cfg, 0));
atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY, memory_order_release);
ha_init_raft(ha, NULL, 3);
ha->raft.term = 5;
ha->raft.role = RAFT_LEADER;
memset(ep, 0, sizeof(*ep));
standby_ep_ctx_build_for_test(ep, ha, NULL, 0);
}
static void tav_teardown(ha_state_t *ha, endpoint_t *ep)
{
free(ep->data);
ep->data = NULL;
ha_fini(ha);
}
/* ── Vector (a): heartbeat-reply state-report term adoption ─────────────── */
static void test_heartbeat_state_report_higher_term_adopts_and_steps_down(void)
{
ha_state_t ha;
endpoint_t ep;
tav_setup(&ha, &ep);
io_packet_t *pkt = build_state_report_pkt(/*applied_lsn*/0, /*config_tag*/0,
/*term*/9);
TEST_ASSERT_NOT_NULL(pkt);
int rc = peer_ctrl_msg_handler_for_test(&ep, pkt);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(9, ha.raft.term,
"bug-1484/bug-1421 vector (a): a heartbeat-reply state report's higher "
"term must be adopted via raft_on_higher_term_observed through the "
"real repl_ack_decode_and_dispatch path");
TEST_ASSERT_NOT_EQUAL_INT_MESSAGE(RAFT_LEADER, ha.raft.role,
"bug-1484/bug-1421 vector (a): leadership held at the lower term must "
"be stepped down once a higher term is observed via the heartbeat-"
"reply fallback");
tav_teardown(&ha, &ep);
printf("test_heartbeat_state_report_higher_term_adopts_and_steps_down: PASS\n");
}
/* ── Vector (b): WAL-NACK reply_term adoption ────────────────────────────── */
static void test_wal_nack_higher_term_adopts_and_steps_down(void)
{
ha_state_t ha;
endpoint_t ep;
tav_setup(&ha, &ep);
io_packet_t *pkt = build_wal_nack_pkt(/*lsn*/3, /*term*/9);
TEST_ASSERT_NOT_NULL(pkt);
int rc = peer_ctrl_msg_handler_for_test(&ep, pkt);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(9, ha.raft.term,
"bug-1484/bug-1421 vector (b): a WAL catch-up NACK's higher reply_term "
"must be adopted via raft_on_higher_term_observed through the real "
"repl_ack_decode_and_dispatch path");
TEST_ASSERT_NOT_EQUAL_INT_MESSAGE(RAFT_LEADER, ha.raft.role,
"bug-1484/bug-1421 vector (b): leadership held at the lower term must "
"be stepped down once a higher term is observed via the WAL-NACK "
"fallback");
tav_teardown(&ha, &ep);
printf("test_wal_nack_higher_term_adopts_and_steps_down: PASS\n");
}
void test_term_adoption_vectors(void)
{
UnitySetTestFile(__FILE__);
RUN_TEST(test_heartbeat_state_report_higher_term_adopts_and_steps_down);
RUN_TEST(test_wal_nack_higher_term_adopts_and_steps_down);
}

77
src/controller/testsuite/test_term_safety_adapter.c
View file

@ -2340,6 +2340,82 @@ static void test_every_callback_and_marker_path_end_to_end(void)
printf("test_every_callback_and_marker_path_end_to_end: PASS\n");
}
/* ================================================================== */
/* bug-1563 — fsync_worker must back off on a sustained fdatasync fail */
/* ================================================================== */
/*
* Live 40-round chaotic /xce-test-ha soak evidence: a freshly-promoted
* primary's local fdatasync started failing (errno=22) and never recovered.
* flusher_work_pending_locked() (durable_lsn < tip) stays true forever once
* fdatasync starts failing -- the fail-loud gate in flusher_flush_once()
* deliberately never advances durable_lsn on failure (that part is correct
* and unchanged). But fsync_worker's loop only waits (pthread_cond_timedwait)
* when NOT pending -- "pending" looks identical whether it means "genuine
* fresh work just landed" or "still stuck from last attempt", so a sustained
* failure makes the loop skip the wait every single iteration and spin
* flusher_flush_once() as fast as the CPU/lock allow: the live soak hit
* 300k+ failed attempts in well under a minute on one node (one core pinned,
* log flooded). This test reproduces the mechanism directly (force every
* fdatasync to fail via the existing test_force_fsync_fail seam, on a
* PRIMARY so the entry's only sync path is the async flusher under test --
* a standby's on_persist_entry does its own separate inline persist-before-
* ACK sync, see test_on_persist_entry_failure_propagates_no_ack) and asserts
* the attempt count over a fixed wall-clock window stays close to the
* wal_sync_ms cadence, not an unbounded spin.
*/
static void test_flusher_backs_off_on_sustained_fsync_failure(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_flusher_backoff_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = calloc(1, sizeof(cluster_t));
TEST_ASSERT_NOT_NULL(c);
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 20; /* short cadence so backoff is observable quickly */
cfg.snap_keep_n = 2;
persist_set_workdir(&c->persist, tmp, "ctrl-test");
TEST_ASSERT_EQUAL_INT(0, persist_init(&c->persist, &cfg, c));
TEST_ASSERT_EQUAL_INT(0, persist_start(&c->persist));
c->ha.raft_c = c;
atomic_store(&c->ha.role, CTRL_PRIMARY);
/* Force every fdatasync to fail BEFORE appending, so the entry created
* below is genuinely pending/undurable for the entire observation
* window -- the async fsync_worker thread is the only thing that could
* ever sync it. */
atomic_store(&c->persist.test_force_fsync_fail, 1);
uint8_t blob[64];
size_t n = pack_entry(blob, WAL_JOB_UPDATE, "y", 1);
TEST_ASSERT_EQUAL_INT(0,
ha_cb_persist_entry_for_test(&c->ha, /*index*/1, /*term*/1, blob, n));
struct timespec ts = { .tv_sec = 0, .tv_nsec = 300 * 1000 * 1000L };
nanosleep(&ts, NULL);
unsigned attempts =
atomic_load(&c->persist.test_flusher_flush_attempt_count);
atomic_store(&c->persist.test_force_fsync_fail, 0);
teardown_cluster(c);
/* 300ms / 20ms cadence -> ~15 attempts expected once backed off; a
* generous upper bound absorbs scheduling jitter without being loose
* enough to miss a genuine busy-spin (pre-fix: thousands of attempts in
* the same window). */
TEST_ASSERT_TRUE_MESSAGE(attempts < 60,
"fsync_worker must back off between failed fdatasync attempts "
"(bug-1563) -- a sustained sync failure must not spin the flusher "
"thread at full CPU with zero delay between retries");
printf("test_flusher_backs_off_on_sustained_fsync_failure: PASS "
"(attempts=%u)\n", attempts);
}
/* ------------------------------------------------------------------ */
/* Suite entry point */
/* ------------------------------------------------------------------ */
@ -2355,6 +2431,7 @@ void test_term_safety_adapter(void)
RUN_TEST(test_on_persist_entry_stamps_term_standby);
RUN_TEST(test_on_persist_entry_stamps_term_primary);
RUN_TEST(test_on_persist_entry_failure_propagates_no_ack);
RUN_TEST(test_flusher_backs_off_on_sustained_fsync_failure);
RUN_TEST(test_on_get_entry_term_present_legacy_absent);
RUN_TEST(test_on_truncate_from_drops_suffix);
RUN_TEST(test_on_truncate_from_removes_all);

566
src/controller/testsuite/test_wal_job_dispatch_fact.c Normal file
View file

@ -0,0 +1,566 @@
/* test_wal_job_dispatch_fact.c — scheduler-dispatch-commit task 1.2
*
* Proves the new WAL_JOB_DISPATCH fact type (wal_types.h) -- the durable
* record of a job's dispatch-time transition to a running state (Req 2.4,
* design.md "WAL Dispatch Fact") -- is wired into BOTH of persist.c's apply
* seams:
*
* - apply_committed_entry() (live-commit apply / boot replay dispatch)
* - apply_wal_entry() (standby catch-up / replicated-apply dispatch)
*
* and that appending ONLY a WAL_JOB_DISPATCH fact (no accompanying
* WAL_JOB_START) and replaying it -- simulating a controller restart via a
* real persist_recover() over a real on-disk WAL -- recovers the job as
* running (this task's "Observable completion").
*
* This task does NOT wire any live caller that appends this fact type
* (dispatch_a_job_() is untouched -- that's task 3.1's job); every test here
* constructs and applies/replays a WAL_JOB_DISPATCH entry directly.
*
* Fixtures mirror this suite's established conventions exactly:
* - wjd_setup/wjd_teardown: bare cluster_t (jobmgr + hooksys) + in-memory
* persist_t, mirroring test_apply_committed_entry.c's tace_setup and
* test_job_array_apply.c's tajd_setup.
* - wjd_make_fresh_persist/wjd_teardown_persist: real disk-backed
* persist_t + persist_recover(), mirroring test_replay_apply_seam.c's
* trr_make_fresh_persist/trr_teardown_persist.
*/
#include <rtti.h>
#include <capnp_c.h>
#include <errno.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
#include "cluster.h"
#include "controller.capnp.h"
#include "hook.h"
#include "jobj.h"
#include "persist.h"
#include "wal_types.h"
#include "xce/cluster_def.h"
#include "xce/types.h"
#include "xcealloc.h"
#include "xstring.h"
/* jobj.c's jobsub table and the process-global xstring table are shared
* across every test file in the controller_unit_test binary — guard
* idempotently, mirroring every sibling fixture in this suite. */
static int g_wjd_jobobj_ready;
static cluster_t g_wjd_c;
static persist_t g_wjd_p;
/* ── Spy hook: counts AFTER_EVENT firings, ignores all payload args ─────── */
static int g_wjd_hook_fires;
static int wjd_spy_hook(int type, int situation, int detail, void* data,
va_list ap)
{
(void)type; (void)detail; (void)data; (void)ap;
if (situation == HOOK_AFTER_EVENT)
g_wjd_hook_fires++;
return 0;
}
static void wjd_setup(void)
{
kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
.nslots = 32,
.delta = 16 };
if (makeString((char*)"wjd-probe") == NULL)
xstring_table_init(64);
if (!g_wjd_jobobj_ready) {
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
g_wjd_jobobj_ready = 1;
}
memset(&g_wjd_c, 0, sizeof(g_wjd_c));
TEST_ASSERT_EQUAL_INT(0,
xce_hook_init(&g_wjd_c.hooksys, XCE_CONTROLLER_HOOK_MAX));
g_wjd_c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
TEST_ASSERT_NOT_NULL(g_wjd_c.jobmgr);
memset(&g_wjd_p, 0, sizeof(g_wjd_p));
g_wjd_p.cluster = &g_wjd_c;
g_wjd_hook_fires = 0;
}
static void wjd_teardown(void)
{
xce_hook_fini(&g_wjd_c.hooksys);
if (g_wjd_c.jobmgr)
dereference(g_wjd_c.jobmgr);
}
/* Look up a job_t wrapper by jobid (NULL if absent). */
static job_t* wjd_find_job(uint64_t jobid)
{
job_t* j = NULL;
lock(g_wjd_c.jobmgr, RTTI_LOCK_READ);
int found = key_find(g_wjd_c.jobmgr, (void*)jobid, (void**)&j);
unlock(g_wjd_c.jobmgr);
return (found > 0) ? j : NULL;
}
/* Creates a job directly in the fixture's jobmgr, bypassing dispatch/replay
* entirely — for seeding fact-transition tests, mirroring
* test_apply_committed_entry.c's tace_seed_job. */
static job_submit_data_t g_wjd_seed_submit;
static job_t* wjd_seed_job(uint64_t jobid, uint32_t state)
{
xce_job_t jdef;
memset(&g_wjd_seed_submit, 0, sizeof(g_wjd_seed_submit));
g_wjd_seed_submit.queue = (char*)"q";
g_wjd_seed_submit.user = (char*)"tester";
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = jobid;
jdef.state = state;
jdef.submit = &g_wjd_seed_submit;
return makeJob(g_wjd_c.jobmgr, &jdef);
}
/* ── apply_committed_entry(): live-commit apply / boot replay dispatch ──── */
static void test_wjd_apply_committed_entry_transitions_pend_to_run(void)
{
wjd_setup();
job_t* j = wjd_seed_job(71001, JOB_STAT_PEND);
TEST_ASSERT_NOT_NULL(j);
wal_jobstate_t js = { .jobid = 71001, .state = JOB_STAT_RUN, ._pad = 0 };
xce_add_hook(&g_wjd_c.hooksys, XCE_CONTROLLER_HOOK_JOB, wjd_spy_hook,
NULL);
int rc = apply_committed_entry(&g_wjd_p, WAL_JOB_DISPATCH, 100, &js,
sizeof(js));
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j->job.state,
"WAL_JOB_DISPATCH must apply via the same absolute-value guarded "
"core as WAL_JOB_START/etc, joining their fall-through group");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_wjd_hook_fires,
"a forward WAL_JOB_DISPATCH transition must fire exactly one "
"notification, matching every other type in its shared group");
wjd_teardown();
printf("test_wjd_apply_committed_entry_transitions_pend_to_run: PASS\n");
}
/* bug-1678: build a wal_jobstate_t header + trailing Cap'n Proto JobAlloc
* bytes -- persist_submit_job_dispatch_fact()'s payload shape -- mirroring
* capnp_encode_joballoc (persist.c, file-static) since job.c/persist.c's own
* static helpers aren't linkable from a test TU. Caller owns *out. */
static int wjd_build_dispatch_payload(uint64_t jobid, uint32_t state,
xce_job_alloc_t* alloc,
uint8_t** out, size_t* out_len)
{
struct capn c;
struct capn_segment* cs;
struct JobAlloc cja;
JobAlloc_ptr jap;
uint8_t* allocbuf;
size_t alloclen;
wal_jobstate_t js = { .jobid = jobid, .state = state, ._pad = 0 };
capn_init_malloc(&c);
cs = capn_root(&c).seg;
encode_JobAlloc(cs, &cja, alloc);
jap = new_JobAlloc(cs);
write_JobAlloc(&cja, jap);
capn_setp(capn_root(&c), 0, jap.p);
alloclen = capn_size(&c);
allocbuf = malloc(alloclen);
if (!allocbuf) { capn_free(&c); return -ENOMEM; }
capn_write_mem(&c, allocbuf, alloclen, 0);
capn_free(&c);
*out_len = sizeof(js) + alloclen;
*out = malloc(*out_len);
if (!*out) { free(allocbuf); return -ENOMEM; }
memcpy(*out, &js, sizeof(js));
memcpy(*out + sizeof(js), allocbuf, alloclen);
free(allocbuf);
return 0;
}
/* One host ("wjd-worker"), one resource ("slots", value 1.0) -- the same
* minimal shape test_dispatch_commit.c's own dispatch tests use. Caller owns
* the result (release via free_JobAlloc_ptr). */
static xce_job_alloc_t* wjd_make_test_alloc(uint64_t jobid)
{
return make_job_alloc(jobid, 1,
make_host_allocs(1,
make_host_alloc((char*)"wjd-worker", 1,
make_allocs(1,
make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
}
static void test_wjd_apply_committed_entry_transitions_and_sets_allocs(void)
{
wjd_setup();
job_t* j = wjd_seed_job(71004, JOB_STAT_PEND);
TEST_ASSERT_NOT_NULL(j);
xce_job_alloc_t* alloc = wjd_make_test_alloc(71004);
TEST_ASSERT_NOT_NULL(alloc);
uint8_t* payload = NULL;
size_t plen = 0;
TEST_ASSERT_EQUAL_INT(0,
wjd_build_dispatch_payload(71004, JOB_STAT_RUN, alloc, &payload,
&plen));
free_JobAlloc_ptr(&alloc);
int rc = apply_committed_entry(&g_wjd_p, WAL_JOB_DISPATCH, 102, payload,
(uint32_t)plen);
free(payload);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
TEST_ASSERT_NOT_NULL_MESSAGE(j->job.allocs,
"a WAL_JOB_DISPATCH payload with trailing JobAlloc bytes must set "
"job.allocs (bug-1678), not just state");
TEST_ASSERT_EQUAL_INT(1, j->job.allocs->nhosts);
TEST_ASSERT_EQUAL_STRING("wjd-worker", j->job.allocs->allocs[0]->host);
TEST_ASSERT_EQUAL_STRING("slots",
j->job.allocs->allocs[0]->allocs[0]->resname);
wjd_teardown();
printf("test_wjd_apply_committed_entry_transitions_and_sets_allocs: "
"PASS\n");
}
/* ── apply_wal_entry(): standby catch-up / replicated-apply dispatch ────── */
static void
test_wjd_apply_wal_entry_transitions_pend_to_run_no_hook(void)
{
wjd_setup();
job_t* j = wjd_seed_job(71002, JOB_STAT_PEND);
TEST_ASSERT_NOT_NULL(j);
wal_jobstate_t js = { .jobid = 71002, .state = JOB_STAT_RUN, ._pad = 0 };
xce_add_hook(&g_wjd_c.hooksys, XCE_CONTROLLER_HOOK_JOB, wjd_spy_hook,
NULL);
int rc = apply_wal_entry(&g_wjd_p, WAL_JOB_DISPATCH, 101, &js,
sizeof(js));
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j->job.state,
"WAL_JOB_DISPATCH must apply via restore_job_state() through "
"apply_wal_entry's catch-up dispatch, joining the same "
"absolute-value fall-through group as WAL_JOB_START/etc");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_wjd_hook_fires,
"apply_wal_entry's catch-up dispatch must never fire a notification "
"hook, matching every other type in its group");
wjd_teardown();
printf("test_wjd_apply_wal_entry_transitions_pend_to_run_no_hook: "
"PASS\n");
}
static void test_wjd_apply_wal_entry_transitions_and_sets_allocs(void)
{
wjd_setup();
job_t* j = wjd_seed_job(71005, JOB_STAT_PEND);
TEST_ASSERT_NOT_NULL(j);
xce_job_alloc_t* alloc = wjd_make_test_alloc(71005);
TEST_ASSERT_NOT_NULL(alloc);
uint8_t* payload = NULL;
size_t plen = 0;
TEST_ASSERT_EQUAL_INT(0,
wjd_build_dispatch_payload(71005, JOB_STAT_RUN, alloc, &payload,
&plen));
free_JobAlloc_ptr(&alloc);
int rc = apply_wal_entry(&g_wjd_p, WAL_JOB_DISPATCH, 103, payload,
(uint32_t)plen);
free(payload);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
TEST_ASSERT_NOT_NULL_MESSAGE(j->job.allocs,
"apply_wal_entry's catch-up dispatch must also recover job.allocs "
"from a WAL_JOB_DISPATCH payload's trailing JobAlloc bytes "
"(bug-1678) -- required for a promoted standby, not just the "
"primary's own boot replay");
TEST_ASSERT_EQUAL_INT(1, j->job.allocs->nhosts);
TEST_ASSERT_EQUAL_STRING("wjd-worker", j->job.allocs->allocs[0]->host);
wjd_teardown();
printf("test_wjd_apply_wal_entry_transitions_and_sets_allocs: PASS\n");
}
/* ── Restart-replay Observable (this task's completion criterion) ───────── */
static persist_t* wjd_make_fresh_persist(const char* tmpdir)
{
persist_t* p = calloc(1, sizeof(persist_t));
if (!p) return NULL;
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000; /* slow fsync — no races in tests */
persist_set_workdir(p, tmpdir, "ctrl-test");
if (persist_init(p, &cfg, NULL) != 0) {
free(p);
return NULL;
}
if (persist_start(p) != 0) {
persist_fini(p);
free(p);
return NULL;
}
return p;
}
static void wjd_teardown_persist(persist_t* p)
{
persist_stop(p);
persist_fini(p);
free(p);
}
/* Encode helper mirroring test_apply_committed_entry.c's tace_encode_job /
* test_replay_apply_seam.c's trr_encode_job (capnp_encode_job is file-static
* in persist.c). */
static int wjd_encode_job(const xce_job_t* j, uint8_t** out, size_t* out_len)
{
struct capn c;
struct capn_segment* cs;
struct Job cj;
Job_ptr jp;
capn_init_malloc(&c);
cs = capn_root(&c).seg;
encode_Job(cs, &cj, (xce_job_t*)j);
jp = new_Job(cs);
write_Job(&cj, jp);
capn_setp(capn_root(&c), 0, jp.p);
*out_len = capn_size(&c);
*out = malloc(*out_len);
if (!*out) { capn_free(&c); return -ENOMEM; }
capn_write_mem(&c, *out, *out_len, 0);
capn_free(&c);
return 0;
}
/* Appends ONLY WAL_JOB_SUBMIT (jobid PEND) + WAL_JOB_DISPATCH -- no
* WAL_JOB_START anywhere in this log -- then replays it via a fresh
* persist_t's persist_recover() (a real restart). This is the task's own
* "Observable completion": the dispatch-time transition survives a restart
* standing entirely on its own, with no dependence on the separate
* worker-confirmed-start fact. */
static void
test_wjd_restart_replay_recovers_running_without_start_fact(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_wjd_replay_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Phase 1: write the log to real disk. No cluster needed —
* persist_wal_async is a physical-write-only path. */
{
persist_t* p = wjd_make_fresh_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
job_submit_data_t submit;
memset(&submit, 0, sizeof(submit));
submit.queue = (char*)"q";
submit.user = (char*)"tester";
xce_job_t jdef;
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = 71003;
jdef.state = JOB_STAT_PEND;
jdef.submit = &submit;
uint8_t* job_payload = NULL;
size_t job_plen = 0;
TEST_ASSERT_EQUAL_INT(0,
wjd_encode_job(&jdef, &job_payload, &job_plen));
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(p, WAL_JOB_SUBMIT, job_payload,
(uint32_t)job_plen));
free(job_payload);
/* The dispatch-time fact — no WAL_JOB_START anywhere in this log. */
wal_jobstate_t js = { .jobid = 71003, .state = JOB_STAT_RUN,
._pad = 0 };
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(p, WAL_JOB_DISPATCH, &js, sizeof(js)));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
wjd_teardown_persist(p);
}
/* Phase 2: fresh persist_t + fresh cluster_t, recover from disk —
* simulates a controller restart. */
{
persist_t* p2 = wjd_make_fresh_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
wjd_setup();
p2->cluster = &g_wjd_c;
int rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
job_t* j = wjd_find_job(71003);
TEST_ASSERT_NOT_NULL_MESSAGE(j,
"restart replay must construct the job via WAL_JOB_SUBMIT");
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j->job.state,
"restart replay must recover the job as RUNNING from the "
"WAL_JOB_DISPATCH fact alone, with no WAL_JOB_START present "
"in the log at all");
wjd_teardown();
wjd_teardown_persist(p2);
}
char cmd[512];
snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
if (system(cmd) != 0)
fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
printf("test_wjd_restart_replay_recovers_running_without_start_fact: "
"PASS\n");
}
/* bug-1678: sibling of the test above, proving the user's exact acceptance
* criterion ("after recover the job should get the same as it's state
* before shutdown") at this file's own direct-payload granularity: a
* WAL_JOB_DISPATCH fact carrying trailing JobAlloc bytes survives a real
* restart (persist_recover over real on-disk WAL) with job.allocs intact,
* not just job.state. test_dispatch_commit.c's
* test_dc_success_appends_wal_job_dispatch_durably covers the same
* observable through the REAL dispatch_a_job_() production call site; this
* test isolates it at the WAL fact level, mirroring how this file's other
* tests already isolate apply_committed_entry/apply_wal_entry from the live
* caller. */
static void
test_wjd_restart_replay_recovers_running_and_allocs_with_dispatch_fact(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_wjd_replay2_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Phase 1: write the log to real disk. */
{
persist_t* p = wjd_make_fresh_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
job_submit_data_t submit;
memset(&submit, 0, sizeof(submit));
submit.queue = (char*)"q";
submit.user = (char*)"tester";
xce_job_t jdef;
memset(&jdef, 0, sizeof(jdef));
jdef.jobid = 71006;
jdef.state = JOB_STAT_PEND;
jdef.submit = &submit;
uint8_t* job_payload = NULL;
size_t job_plen = 0;
TEST_ASSERT_EQUAL_INT(0,
wjd_encode_job(&jdef, &job_payload, &job_plen));
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(p, WAL_JOB_SUBMIT, job_payload,
(uint32_t)job_plen));
free(job_payload);
xce_job_alloc_t* alloc = wjd_make_test_alloc(71006);
TEST_ASSERT_NOT_NULL(alloc);
uint8_t* dispatch_payload = NULL;
size_t dispatch_plen = 0;
TEST_ASSERT_EQUAL_INT(0,
wjd_build_dispatch_payload(71006, JOB_STAT_RUN, alloc,
&dispatch_payload, &dispatch_plen));
free_JobAlloc_ptr(&alloc);
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(p, WAL_JOB_DISPATCH, dispatch_payload,
(uint32_t)dispatch_plen));
free(dispatch_payload);
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
wjd_teardown_persist(p);
}
/* Phase 2: fresh persist_t + fresh cluster_t, recover from disk —
* simulates a controller restart. */
{
persist_t* p2 = wjd_make_fresh_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
wjd_setup();
p2->cluster = &g_wjd_c;
int rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
job_t* j = wjd_find_job(71006);
TEST_ASSERT_NOT_NULL_MESSAGE(j,
"restart replay must construct the job via WAL_JOB_SUBMIT");
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
TEST_ASSERT_NOT_NULL_MESSAGE(j->job.allocs,
"restart replay must recover job.allocs from the "
"WAL_JOB_DISPATCH fact's trailing JobAlloc bytes (bug-1678), "
"not just job.state");
TEST_ASSERT_EQUAL_INT(1, j->job.allocs->nhosts);
TEST_ASSERT_EQUAL_STRING("wjd-worker",
j->job.allocs->allocs[0]->host);
wjd_teardown();
wjd_teardown_persist(p2);
}
char cmd[512];
snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
if (system(cmd) != 0)
fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
printf("test_wjd_restart_replay_recovers_running_and_allocs_with_"
"dispatch_fact: PASS\n");
}
/* ── Test group entry point ───────────────────────────────────────────── */
void test_wal_job_dispatch_fact(void)
{
UnitySetTestFile(__FILE__);
RUN_TEST(test_wjd_apply_committed_entry_transitions_pend_to_run);
RUN_TEST(test_wjd_apply_committed_entry_transitions_and_sets_allocs);
RUN_TEST(test_wjd_apply_wal_entry_transitions_pend_to_run_no_hook);
RUN_TEST(test_wjd_apply_wal_entry_transitions_and_sets_allocs);
RUN_TEST(test_wjd_restart_replay_recovers_running_without_start_fact);
RUN_TEST(
test_wjd_restart_replay_recovers_running_and_allocs_with_dispatch_fact);
}

184
src/controller/testsuite/test_worker_join.c
View file

@ -17,6 +17,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <unity/unity.h>
@ -152,6 +153,132 @@ static float get_worker_res_fval(worker_t* w, const char* name)
return (v != NULL) ? v->u.fvalue : -1.0f;
}
/*
* Capture whatever update_resvals()/update_resval_list() print during a
* failing update_resval() call, as raw runtime output rather than a
* source-text scan. This test binary never calls mylog_init()/tlog_init()
* (only the real daemon main()s do -- see test_apply_correctness_
* verification.c's own header comment, and the identical established
* precedent in scheduler/testsuite/test_scheduler_connect.c's
* capture_stdout_during_sched_init()), so tlog's internal root log handle
* stays NULL and every ERROR()/WARN() call falls through tlog's early-
* print path, which writes the fully formatted line directly to
* STDOUT_FILENO via a raw write() -- genuine runtime log capture, not a
* guess about which stream PERROR/ERROR happens to use.
*/
static char* capture_stdout_during(void (*fn)(void* ctx), void* ctx)
{
char tmp[] = "/tmp/xce_worker_join_logcap_XXXXXX";
int fd = mkstemp(tmp);
TEST_ASSERT_TRUE_MESSAGE(fd >= 0,
"fixture: mkstemp for stdout capture must succeed");
fflush(stdout);
int saved_stdout = dup(STDOUT_FILENO);
TEST_ASSERT_TRUE_MESSAGE(saved_stdout >= 0,
"fixture: dup(STDOUT_FILENO) must succeed");
TEST_ASSERT_TRUE_MESSAGE(dup2(fd, STDOUT_FILENO) >= 0,
"fixture: dup2 redirect of STDOUT_FILENO must "
"succeed");
fn(ctx);
fflush(stdout);
dup2(saved_stdout, STDOUT_FILENO);
close(saved_stdout);
off_t len = lseek(fd, 0, SEEK_CUR);
TEST_ASSERT_TRUE_MESSAGE(len >= 0,
"fixture: lseek on capture file must succeed");
char* buf = (char*)calloc(1, (size_t)len + 1);
TEST_ASSERT_NOT_NULL(buf);
if (len > 0) {
TEST_ASSERT_TRUE_MESSAGE(lseek(fd, 0, SEEK_SET) == 0,
"fixture: rewind of capture file must "
"succeed");
ssize_t nread = read(fd, buf, (size_t)len);
TEST_ASSERT_TRUE_MESSAGE(nread == len,
"fixture: reading back captured buffer "
"must succeed");
}
close(fd);
unlink(tmp);
return buf;
}
typedef struct {
cluster_t* c;
worker_t* w;
xce_resval_t* val;
} update_resvals_call_ctx_t;
static void call_update_resvals_1(void* ctx_)
{
update_resvals_call_ctx_t* ctx = (update_resvals_call_ctx_t*)ctx_;
update_resvals(ctx->c, ctx->w, 1, ctx->val);
}
/*
* update_resval() returns its failure reason as a raw negative code
* directly (key_find/zarray_put convention -- never via errno), so
* update_resvals()/update_resval_list() logging it via PERROR (which reads
* the process-global errno) reported whatever errno happened to be left
* over from an unrelated earlier call: "Success" if nothing had touched
* it, or a stray EAGAIN/etc from some other syscall in the same event-loop
* iteration -- never the real reason a resource update failed. This test
* reproduces that exact symptom (ambient errno=EAGAIN, unrelated to the
* genuine failure) and proves the fix decodes the actual rc instead.
*/
static void test_update_resvals_logs_actual_rc_not_stale_errno(void)
{
setup_test_cluster();
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
worker_t* w = make_test_worker("host-resval-log", &attrs);
TEST_ASSERT_NOT_NULL(w);
xce_val_t fval;
memset(&fval, 0, sizeof(fval));
fval.which = Value_u_fvalue;
fval.u.fvalue = 1.0f;
xce_resval_t val;
val.name = "does-not-exist-in-resmgr";
val.val = &fval;
/* Simulate the exact reported symptom: ambient errno unrelated to the
* real failure, left over from some other unrelated call. */
errno = EAGAIN;
update_resvals_call_ctx_t ctx = { .c = &g_tc.c, .w = w, .val = &val };
char* captured = capture_stdout_during(call_update_resvals_1, &ctx);
TEST_ASSERT_NOT_NULL(captured);
TEST_ASSERT_NULL_MESSAGE(
strstr(captured, "Resource temporarily unavailable"),
"must not log the ambient errno's string (EAGAIN) -- that is not "
"why update_resval failed");
TEST_ASSERT_NULL_MESSAGE(
strstr(captured, "Success"),
"must not log \"Success\" for a call that returned rc<0");
TEST_ASSERT_NOT_NULL_MESSAGE(
strstr(captured, "does-not-exist-in-resmgr"),
"must name which resource failed");
TEST_ASSERT_NOT_NULL_MESSAGE(
strstr(captured, "No such file or directory"),
"must log the real reason (-ENOENT, from key_find on an "
"unregistered resource name)");
free(captured);
teardown_test_cluster();
printf("test_update_resvals_logs_actual_rc_not_stale_errno: PASS\n");
}
/* ── Task 1.1: Tests for apply_join_state ──────────────────────────── */
/*
@ -295,6 +422,60 @@ static void test_apply_join_state_updates_resource_values(void)
printf("test_apply_join_state_updates_resource_values: PASS\n");
}
/*
* hconfig.mxj == -1 is the schema's documented "auto-detect" sentinel
* (hosts.schema.json: "max_slot": {"minimum": -1, "description": "-1 means
* auto-detect"}) -- apply_join_state must fall back to the worker's own
* reported maxcpus, NOT persist -1 (or, before the xceadmin mapping fix,
* silently ship 0 and permanently disable scheduling on that host: mxj is
* UInt32 on the wire, so an unset/absent max_slot used to decode as 0,
* which is not < 0 and so never reached this fallback -- see
* xcecommon.mapping.HOSTCONFIG_MXJ_AUTO_DETECT). This is the ONLY test in
* this file that exercises the nslots<0 branch; every sibling test above
* uses a positive hconfig.mxj and so cannot catch a regression here.
*/
static void test_apply_join_state_mxj_negative_one_falls_back_to_maxcpus(void)
{
setup_test_cluster();
hostattrs_t attrs;
memset(&attrs, 0, sizeof(attrs));
attrs.hosttype = makeString("LINUX");
attrs.hostmodel = makeString("x86");
attrs.cpufactor = 1.0f;
attrs.maxcpus = 12;
attrs.maxmem = 64;
attrs.maxswap = 128;
attrs.maxtmp = 256;
attrs.ndisks = 2;
worker_t* w = make_test_worker("host-ajs-autodetect", &attrs);
TEST_ASSERT_NOT_NULL(w);
xce_hostconfig_t hconfig;
memset(&hconfig, 0, sizeof(hconfig));
hconfig.hosttype = (char*)"!";
hconfig.hostmodel = (char*)"!";
hconfig.mxj = -1;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
lock(w, RTTI_LOCK_WRITE);
int rc = apply_join_state(&g_tc.c, w, &ep, &hconfig);
unlock(w);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_FLOAT_MESSAGE(
12.0f, get_worker_res_fval(w, "slots"),
"mxj=-1 must auto-detect slots from the worker's reported maxcpus, "
"not persist -1 or silently leave slots at 0");
teardown_test_cluster();
printf("test_apply_join_state_mxj_negative_one_falls_back_to_maxcpus: "
"PASS\n");
}
/* ── Task 1.2: Tests for init_worker ────────────────────────────────── */
/*
@ -520,10 +701,13 @@ void test_worker_join(void)
/* xstring_table_init must be called once before any makeString usage */
xstring_table_init(139);
RUN_TEST(test_update_resvals_logs_actual_rc_not_stale_errno);
/* Task 1.1: apply_join_state (shared status/resource update) */
RUN_TEST(test_apply_join_state_clears_unavail);
RUN_TEST(test_apply_join_state_applies_hosttype_override);
RUN_TEST(test_apply_join_state_updates_resource_values);
RUN_TEST(test_apply_join_state_mxj_negative_one_falls_back_to_maxcpus);
/* Task 1.2: init_worker (first-time join) */
RUN_TEST(test_init_worker_creates_new_entry);