feat(worker-rejoin-resync): task 4.1 - bounded finished-job retention on worker
Retain a job's exit status/code/true finish time for a configurable window after update_worker_status() detects its exit, before cleanup_worker_list() would otherwise discard all trace of it (Requirements 3.4, 3.5, 3.7). Hooked into the single choke point both production exit-detection paths (libuv IPC callback, dbus polling) converge through, so retention happens-before any reap regardless of which mechanism is active. Also prunes on retain (not just on lookup), closing a review finding: lookup currently has zero production callers until task 4.2 lands, so without this the store would grow unboundedly for the daemon's uptime. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Y4M5MJM2zK6nU43da267Ku
This commit is contained in:
parent
087b8c1804
commit
e31012e8a7
5 changed files with 613 additions and 1 deletions
|
|
@ -93,3 +93,17 @@ test('worker uv_ipc job_exit_severity test', uv_ipc_job_exit_severity_test,
|
|||
suite: 'worker',
|
||||
timeout: 60,
|
||||
env: ['ASAN_OPTIONS=detect_leaks=0'])
|
||||
|
||||
# worker-rejoin-resync task 4.1 (Requirements 3.4, 3.5, 3.7): bounded
|
||||
# finished-job retention -- white-box #include of xcecd_worker_utils.c,
|
||||
# same convention as resource_sync_test above.
|
||||
finished_job_retention_test = executable('finished_job_retention_test',
|
||||
'test_finished_job_retention.c',
|
||||
include_directories: [worker_test_incdir],
|
||||
dependencies: worker_test_deps,
|
||||
install: false)
|
||||
|
||||
test('worker finished_job_retention test', finished_job_retention_test,
|
||||
suite: 'worker',
|
||||
timeout: 60,
|
||||
env: ['ASAN_OPTIONS=detect_leaks=0'])
|
||||
|
|
|
|||
341
src/worker/testsuite/test_finished_job_retention.c
Normal file
341
src/worker/testsuite/test_finished_job_retention.c
Normal file
|
|
@ -0,0 +1,341 @@
|
|||
/*
|
||||
* test_finished_job_retention.c -- worker-rejoin-resync task 4.1
|
||||
* (Requirements 3.4, 3.5, 3.7): a job's true outcome (exit status, exit
|
||||
* code, real finish time) must remain queryable for a configurable bounded
|
||||
* window after the job exits, instead of being discarded the moment
|
||||
* cleanup_worker_list() reaps the xcecd_worker_t entry on its next pass.
|
||||
*
|
||||
* White-box #include of ../src/xcecd_worker_utils.c (same convention as
|
||||
* this suite's test_resource_sync.c/test_dbus_workers_update.c) so the
|
||||
* REAL update_worker_status() -- the exact job-exit-detection hook point
|
||||
* this task adds its retention call to -- and the real retention
|
||||
* functions under test both run for real, not against reimplemented
|
||||
* stand-ins. Link-satisfying stubs below cover every OTHER external
|
||||
* symbol xcecd_worker_utils.c references (dbus_worker_clean,
|
||||
* xcecd_job_finished, xcecd_db_update_worker/delete_worker,
|
||||
* ipc_master_check_connection, block_signal/unblock_signal) -- none of
|
||||
* which this task's own retention logic depends on.
|
||||
*/
|
||||
#include <stdbool.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
|
||||
#include <unity/unity.h>
|
||||
|
||||
#include "../src/xcecd_worker_utils.c"
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════
|
||||
* Link-satisfying stubs -- none exercised by the assertions below; they
|
||||
* only need to let update_worker_status()'s other, unrelated branches run
|
||||
* to completion without touching real sqlite3/dbus/IPC.
|
||||
* ════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
bool xcecd_reply_startjob(xcecd_job_t* job)
|
||||
{
|
||||
(void)job;
|
||||
return true;
|
||||
}
|
||||
|
||||
int xcecd_job_stopped(uint64_t job_id)
|
||||
{
|
||||
(void)job_id;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int xcecd_job_resumed(uint64_t job_id)
|
||||
{
|
||||
(void)job_id;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int xcecd_job_finished(xcecd_job_t* job)
|
||||
{
|
||||
(void)job;
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool xcecd_db_update_worker(sqlite3* db, int worker_pid, int worker_status,
|
||||
int job_status)
|
||||
{
|
||||
(void)db;
|
||||
(void)worker_pid;
|
||||
(void)worker_status;
|
||||
(void)job_status;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool xcecd_db_delete_worker(sqlite3* db, int worker_pid)
|
||||
{
|
||||
(void)db;
|
||||
(void)worker_pid;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool dbus_worker_clean(xcecd_worker_t* worker, xcecd_job_t* job)
|
||||
{
|
||||
(void)worker;
|
||||
(void)job;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* ipc_master_check_connection() gates update_worker_status()'s own
|
||||
* exit(1) fatal path -- must return true, or every test below would kill
|
||||
* the test process. */
|
||||
bool ipc_master_check_connection() { return true; }
|
||||
|
||||
/* block_signal()/unblock_signal() (xcecd_sig.c) -- not #include'd here
|
||||
* (only xcecd_worker_utils.c is), no-op stand-ins matching the header
|
||||
* signatures exactly. */
|
||||
void block_signal(int sig) { (void)sig; }
|
||||
void unblock_signal(int sig) { (void)sig; }
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════
|
||||
* Fixtures
|
||||
* ════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
static uint64_t test_wallclock_now_ms(void)
|
||||
{
|
||||
struct timespec ts;
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, clock_gettime(CLOCK_REALTIME, &ts));
|
||||
return (uint64_t)ts.tv_sec * 1000ULL + (uint64_t)ts.tv_nsec / 1000000ULL;
|
||||
}
|
||||
|
||||
static xcecd_worker_t* make_running_worker(int pid, uint64_t jobid)
|
||||
{
|
||||
xcecd_worker_t* w = (xcecd_worker_t*)calloc(1, sizeof(*w));
|
||||
w->worker_pid = pid;
|
||||
w->worker_status = WORKER_RUNNING;
|
||||
w->job_status = JOB_RUNNING;
|
||||
w->job_info.jobid = jobid;
|
||||
return w;
|
||||
}
|
||||
|
||||
void setUp(void)
|
||||
{
|
||||
xcecd_finished_job_retention_reset();
|
||||
}
|
||||
|
||||
void tearDown(void)
|
||||
{
|
||||
xcecd_finished_job_retention_reset();
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════
|
||||
* Population: a job's exit (via the real update_worker_status()
|
||||
* WORKER_DONE/WORKER_EXITED hook this task adds) must populate the
|
||||
* retention store with the correct outcome.
|
||||
* ════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
static void test_worker_done_populates_retention_with_correct_outcome(void)
|
||||
{
|
||||
xcecd_worker_t* worker = make_running_worker(4001, 90001);
|
||||
xcecd_job_t job = { .job_id = 0,
|
||||
.job_pid = 0,
|
||||
.job_status = JOB_DONE,
|
||||
.exit_code = 0 };
|
||||
|
||||
uint64_t before_ms = test_wallclock_now_ms();
|
||||
int rc = update_worker_status(worker, 4001, WORKER_DONE, &job, NULL);
|
||||
uint64_t after_ms = test_wallclock_now_ms();
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(XCECD_OK, rc);
|
||||
|
||||
xcecd_finished_job_t out;
|
||||
memset(&out, 0, sizeof(out));
|
||||
bool found = xcecd_finished_job_lookup(90001, after_ms, &out);
|
||||
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
found, "job 90001's exit (WORKER_DONE) must be retained");
|
||||
TEST_ASSERT_EQUAL_UINT64(90001, out.jobid);
|
||||
TEST_ASSERT_EQUAL_INT(WORKER_DONE, out.exit_status);
|
||||
TEST_ASSERT_EQUAL_INT(0, out.exit_code);
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
out.finish_time_ms >= before_ms && out.finish_time_ms <= after_ms,
|
||||
"finish_time_ms must be the real exit-detection time, not an "
|
||||
"arbitrary value -- it must fall within the [before, after] "
|
||||
"wall-clock bound taken around the update_worker_status() call, "
|
||||
"not the time this assertion happens to run");
|
||||
|
||||
free(worker);
|
||||
}
|
||||
|
||||
static void test_worker_exited_populates_retention_with_exit_code(void)
|
||||
{
|
||||
xcecd_worker_t* worker = make_running_worker(4002, 90002);
|
||||
xcecd_job_t job = { .job_id = 0,
|
||||
.job_pid = 0,
|
||||
.job_status = JOB_EXITED,
|
||||
.exit_code = 137 };
|
||||
|
||||
uint64_t before_ms = test_wallclock_now_ms();
|
||||
int rc
|
||||
= update_worker_status(worker, 4002, WORKER_EXITED, &job, NULL);
|
||||
uint64_t after_ms = test_wallclock_now_ms();
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(XCECD_OK, rc);
|
||||
|
||||
xcecd_finished_job_t out;
|
||||
memset(&out, 0, sizeof(out));
|
||||
bool found = xcecd_finished_job_lookup(90002, after_ms, &out);
|
||||
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
found, "job 90002's exit (WORKER_EXITED) must be retained");
|
||||
TEST_ASSERT_EQUAL_UINT64(90002, out.jobid);
|
||||
TEST_ASSERT_EQUAL_INT(WORKER_EXITED, out.exit_status);
|
||||
TEST_ASSERT_EQUAL_INT(137, out.exit_code);
|
||||
TEST_ASSERT_TRUE(out.finish_time_ms >= before_ms
|
||||
&& out.finish_time_ms <= after_ms);
|
||||
|
||||
free(worker);
|
||||
}
|
||||
|
||||
/* Confirms this task's own critical constraint: cleanup_worker_list()'s
|
||||
* existing in-memory reaping behavior for the xcecd_worker_t entry is
|
||||
* completely unmodified by the retention hook -- the worker is still
|
||||
* exactly as eligible for reap (worker_status == WORKER_DONE) as before,
|
||||
* and the retention entry exists as a SEPARATE, additional record, not a
|
||||
* replacement for it. */
|
||||
static void
|
||||
test_retention_hook_does_not_change_cleanup_worker_list_eligibility(void)
|
||||
{
|
||||
xcecd_worker_t* worker = make_running_worker(4003, 90003);
|
||||
xcecd_job_t job = { .job_id = 0, .job_pid = 0,
|
||||
.job_status = JOB_DONE, .exit_code = 0 };
|
||||
|
||||
int rc = update_worker_status(worker, 4003, WORKER_DONE, &job, NULL);
|
||||
TEST_ASSERT_EQUAL_INT(XCECD_OK, rc);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(WORKER_DONE, worker->worker_status);
|
||||
|
||||
xcecd_worker_t* list = worker;
|
||||
cleanup_worker_list(&list, NULL);
|
||||
|
||||
TEST_ASSERT_NULL_MESSAGE(
|
||||
list, "cleanup_worker_list() must still reap a WORKER_DONE entry "
|
||||
"exactly as before -- the retention hook must not interfere "
|
||||
"with its existing behavior");
|
||||
|
||||
xcecd_finished_job_t out;
|
||||
memset(&out, 0, sizeof(out));
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
xcecd_finished_job_lookup(90003, test_wallclock_now_ms(), &out),
|
||||
"the retention entry must survive cleanup_worker_list()'s reap of "
|
||||
"the live xcecd_worker_t entry -- that is the entire point of "
|
||||
"this task");
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════
|
||||
* Query: unknown jobid.
|
||||
* ════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
static void test_lookup_unknown_jobid_returns_not_found(void)
|
||||
{
|
||||
xcecd_finished_job_t out;
|
||||
memset(&out, 0, sizeof(out));
|
||||
|
||||
bool found = xcecd_finished_job_lookup(123456789, test_wallclock_now_ms(),
|
||||
&out);
|
||||
|
||||
TEST_ASSERT_FALSE_MESSAGE(
|
||||
found, "a jobid that was never retained must report not-found");
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════
|
||||
* Expiry: the configured window boundary, driven deterministically via
|
||||
* explicit finish_time_ms/now_ms values (no sleeping in the test).
|
||||
* ════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
static void test_lookup_within_window_returns_outcome(void)
|
||||
{
|
||||
const uint64_t finish_time_ms = 1000000ULL;
|
||||
const uint64_t window_ms = 5000ULL;
|
||||
|
||||
xcecd_finished_job_retention_set_window_ms(window_ms);
|
||||
xcecd_retain_finished_job(70001, WORKER_EXITED, 42, finish_time_ms);
|
||||
|
||||
xcecd_finished_job_t out;
|
||||
memset(&out, 0, sizeof(out));
|
||||
|
||||
/* Exactly at the boundary (elapsed == window_ms) is still within the
|
||||
* window -- inclusive, per xcecd_worker_utils.c's
|
||||
* finished_job_entry_expired() ("> window_ms" is expired, not
|
||||
* ">="). */
|
||||
bool found
|
||||
= xcecd_finished_job_lookup(70001, finish_time_ms + window_ms, &out);
|
||||
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
found, "a job retained within its configured window must still be "
|
||||
"queryable, including exactly at the boundary");
|
||||
TEST_ASSERT_EQUAL_UINT64(70001, out.jobid);
|
||||
TEST_ASSERT_EQUAL_INT(WORKER_EXITED, out.exit_status);
|
||||
TEST_ASSERT_EQUAL_INT(42, out.exit_code);
|
||||
TEST_ASSERT_EQUAL_UINT64(finish_time_ms, out.finish_time_ms);
|
||||
}
|
||||
|
||||
static void test_lookup_just_outside_window_returns_not_found(void)
|
||||
{
|
||||
const uint64_t finish_time_ms = 2000000ULL;
|
||||
const uint64_t window_ms = 5000ULL;
|
||||
|
||||
xcecd_finished_job_retention_set_window_ms(window_ms);
|
||||
xcecd_retain_finished_job(70002, WORKER_DONE, 0, finish_time_ms);
|
||||
|
||||
xcecd_finished_job_t out;
|
||||
memset(&out, 0, sizeof(out));
|
||||
|
||||
/* One millisecond past the boundary -- must no longer be returned. */
|
||||
bool found = xcecd_finished_job_lookup(
|
||||
70002, finish_time_ms + window_ms + 1, &out);
|
||||
|
||||
TEST_ASSERT_FALSE_MESSAGE(
|
||||
found, "an entry one millisecond past its configured retention "
|
||||
"window must no longer be queryable (Observable completion: "
|
||||
"'no longer available once the window elapses')");
|
||||
}
|
||||
|
||||
static void test_lookup_after_expiry_prunes_entry_permanently(void)
|
||||
{
|
||||
const uint64_t finish_time_ms = 3000000ULL;
|
||||
const uint64_t window_ms = 1000ULL;
|
||||
|
||||
xcecd_finished_job_retention_set_window_ms(window_ms);
|
||||
xcecd_retain_finished_job(70003, WORKER_DONE, 0, finish_time_ms);
|
||||
|
||||
xcecd_finished_job_t out;
|
||||
memset(&out, 0, sizeof(out));
|
||||
|
||||
/* First query, well past expiry, prunes the entry as a side effect
|
||||
* (lazy expiry-on-query, see xcecd_worker_utils.c). A second query,
|
||||
* even with a now_ms that would have been within the window relative
|
||||
* to the original finish_time_ms, must still report not-found -- the
|
||||
* entry is gone, not merely "currently judged expired". */
|
||||
TEST_ASSERT_FALSE(xcecd_finished_job_lookup(
|
||||
70003, finish_time_ms + window_ms + 100, &out));
|
||||
TEST_ASSERT_FALSE(
|
||||
xcecd_finished_job_lookup(70003, finish_time_ms, &out));
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════
|
||||
* main
|
||||
* ════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
int main(void)
|
||||
{
|
||||
xce_init_xcecd_log("/tmp/test_finished_job_retention.log", 1024 * 1024,
|
||||
2);
|
||||
|
||||
UNITY_BEGIN();
|
||||
|
||||
RUN_TEST(test_worker_done_populates_retention_with_correct_outcome);
|
||||
RUN_TEST(test_worker_exited_populates_retention_with_exit_code);
|
||||
RUN_TEST(test_retention_hook_does_not_change_cleanup_worker_list_eligibility);
|
||||
RUN_TEST(test_lookup_unknown_jobid_returns_not_found);
|
||||
RUN_TEST(test_lookup_within_window_returns_outcome);
|
||||
RUN_TEST(test_lookup_just_outside_window_returns_not_found);
|
||||
RUN_TEST(test_lookup_after_expiry_prunes_entry_permanently);
|
||||
|
||||
return UNITY_END();
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue