From 2e62e73d843002dd21e3be1f5082d5da491e6f30 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Tue, 14 Jul 2026 06:47:14 +0000 Subject: [PATCH] feat(job-array-single-wal-record): add WAL_JOB_ARRAY_SUBMIT record type Adds the array-shaped WAL entry type (distinct from WAL_JOB_SUBMIT, value 11 in the unused WAL_JOB_* gap) and classifies it WAIT-mode via wal_classify, matching the single-job type's commit-barrier semantics. No dispatch/apply wiring yet -- foundation for task 2.x/3.x. Task 1.1 of job-array-single-wal-record. --- .../job-array-single-wal-record/tasks.md | 169 ++++++++++++++++++ src/controller/persist.c | 6 + .../testsuite/test_commit_barrier_rollout.c | 18 ++ src/include/wal_types.h | 4 + 4 files changed, 197 insertions(+) create mode 100644 .kiro/specs/job-array-single-wal-record/tasks.md diff --git a/.kiro/specs/job-array-single-wal-record/tasks.md b/.kiro/specs/job-array-single-wal-record/tasks.md new file mode 100644 index 0000000..1013da8 --- /dev/null +++ b/.kiro/specs/job-array-single-wal-record/tasks.md @@ -0,0 +1,169 @@ +# Implementation Plan: job-array-single-wal-record + +- [ ] 1. Foundation: WAL/schema additions and jobj.c hardening +- [x] 1.1 (P) Add the array WAL record type + - Add a new `WAL_JOB_ARRAY_SUBMIT` value to the WAL entry type set, distinct from `WAL_JOB_SUBMIT` + - Classify it as a WAIT-mode (requester-awaiting) commit, same as the existing single-job type + - Observable: the new type is recognized by the WAL commit-mode classifier and compiles cleanly; the existing single-job type's classification is unchanged + - _Requirements: 1.1, 1.2, 1.3_ + - _Boundary: WAL Array Submission_ + +- [ ] 1.2 (P) Add the compact array push wire variant + - Add one new additive variant to the existing controller-to-scheduler/apiserver push message, reusing the client submit request's existing (base jobid, array size, one spec) shape verbatim — no new struct + - Observable: the schema compiles and generates encode/decode functions for the new variant without touching any existing variant's wire layout + - _Requirements: 6.1, 6.2_ + - _Boundary: Compact Scheduler/API-Server Push_ + +- [ ] 1.3 (P) jobj.c hardening + - Make job construction check the interning/string-construction return value and propagate failure instead of silently storing a NULL success + - Root-cause and fix the previously observed construction-table initialization-ordering gap; if root-causing shows the actual gap is outside this file's own init/call ordering (e.g. controller boot sequencing), stop and flag it as a separate follow-up rather than forcing a fix into this task's boundary + - Verify (with a test) that the destructive-move fields on a freshly-built job payload are always NULL, so repeated use against a shared template is provably benign + - Fix the known leak on job teardown for the one field that was never freed + - Verify (with a test) that the field claimed to be an undocumented leak risk actually has no wire representation to leak in the first place + - Observable: a new construction-failure path returns an error instead of a silently-broken success; the leak-fix is visible under a leak-detection test run; all five items have a passing test each + - _Requirements: 10.1, 10.2, 10.3, 10.4, 10.5_ + - _Boundary: jobj.c Hardening_ + +- [ ] 2. WAL Array Submission: encode, append, commit +- [ ] 2.1 Implement the single-record array append and commit path + - Encode one compact payload carrying (base jobid, array size, one shared job spec) — the same shape as the client's own submit request + - Append it as exactly one `WAL_JOB_ARRAY_SUBMIT` WAL entry and register exactly one commit continuation for it, regardless of array size + - Observable: submitting an array produces one WAL append call and one registered commit continuation in a controlled test, not one per element + - _Requirements: 1.1, 1.2, 1.3, 4.2, 9.2_ + - _Boundary: WAL Array Submission_ + +- [ ] 2.2 Wire array submissions into the request handler + - Branch the submission handler on array size: exactly 1 continues through today's existing single-job code path completely unmodified; greater than 1 routes to 2.1's single-append path + - Preserve the existing pre-commit range validation (collision check plus in-flight claim) unchanged and in the same order, still rejecting strictly before any WAL append + - Reuse the existing batched-reply mechanism, setting it to expect exactly one outcome instead of one per element + - Observable: a single-job submission (array size 1) is byte-identical in behavior to before this feature; an array submission returns exactly one reply after exactly one outcome resolves + - _Requirements: 1.1, 1.2, 1.3, 8.1, 9.1, 9.2_ + - _Boundary: WAL Array Submission_ + - _Depends: 1.1_ + +- [ ] 2.3 Unit tests for the array append/commit path + - Verify exactly one WAL entry and one commit continuation are produced for an array submission, for several array sizes + - Verify the encoded payload's size does not grow with array size (stays constant regardless of element count) + - Verify a single-job (array size 1) submission produces a byte-identical WAL record, commit-barrier count, and reply compared to the pre-feature baseline + - Observable: a passing test suite demonstrating all three properties above + - _Requirements: 1.1, 1.2, 1.3, 7.1, 7.2, 8.1_ + - _Boundary: WAL Array Submission_ + +- [ ] 3. Array Apply Expansion: shared N-way expansion and dispatch wiring +- [ ] 3.1 Implement the shared array expansion primitive + - Decode the committed array payload and, for each index in the array's range, build a job entry without yet making it visible to any reader + - Skip building an entry for any index that already exists (idempotent re-apply); if every index already exists, treat the whole record as an already-applied no-op + - Once every index has successfully built, make the whole set visible in one atomic step; if any index fails to build, discard everything built in this attempt without ever making any of it visible — a retry starts over from the first index + - Include a test-only, forceable failure point (compiled out in production) that a test can use to force a failure at a chosen index + - Observable: a test can force a mid-array failure and confirm zero elements are visible afterward, then force success and confirm the full set becomes visible in one step + - _Requirements: 2.1, 2.2, 2.3, 3.3, 4.1, 5.1, 5.2, 5.3, 9.3, 9.6_ + - _Boundary: Array Apply Expansion_ + - _Depends: 1.3_ + +- [ ] 3.2 Wire array expansion into live apply + - Call 3.1's primitive from the live-apply/boot-replay dispatch path for the new WAL record type + - After a successful expansion, fire the existing job-created notification exactly once, carrying every newly created element in one list — never once per element + - Observable: applying one committed array record produces exactly one notification event carrying all N elements + - _Requirements: 3.1, 3.4_ + - _Boundary: Array Apply Expansion_ + +- [ ] 3.3 Wire array expansion into standby catch-up + - Call 3.1's same primitive from the separate standby catch-up/replicated-apply dispatch path for the new WAL record type, without firing any notification (matching that path's existing hook-free behavior for the single-job case) + - Observable: a test applies the same committed array record through both the live-apply dispatch and the standby catch-up dispatch and confirms byte-identical resulting job state either way + - _Requirements: 3.1, 3.2, 4.1_ + - _Boundary: Array Apply Expansion_ + +- [ ] 3.4 Unit tests for array expansion + - Verify N-way expansion produces the correct contiguous set of element identities for several array sizes + - Verify idempotent re-apply of an already-fully-applied record is a no-op that creates nothing new + - Verify, using 3.1's forceable failure point, that a failure partway through leaves nothing visible, and that a subsequent successful retry produces the complete, non-duplicated set + - Observable: a passing test suite covering all three properties above + - _Requirements: 2.1, 2.2, 2.3, 3.3, 5.1, 5.2, 5.3, 9.4, 9.6_ + - _Boundary: Array Apply Expansion_ + +- [ ] 4. Live Apply Resource Fence +- [ ] 4.1 Make the apply pump respect apply failures + - Stop discarding the array-expansion (and every other WAL type's) apply outcome; on a failure, halt the apply walk at that position without advancing past it, and log a loud, operator-visible error naming the stuck position + - Observable: a simulated apply failure at a given position causes the walk to stop exactly there and not proceed to later entries + - _Requirements: 9.4_ + - _Boundary: Live Apply Resource Fence_ + +- [ ] 4.2 Add the live, self-healing fence + - Add a new piece of state (distinct from the existing boot-only unrecoverable-fence flag) recording "a position is currently stuck failing to apply" + - Engage it when 4.1's halt condition triggers; clear it automatically, from inside the apply walk itself, the moment a retry of that same stuck position succeeds + - Observable: after a forced failure the new state reads "fenced"; after the same position later succeeds on retry, the state reads "clear" without any external action + - _Requirements: 9.4, 9.5_ + - _Boundary: Live Apply Resource Fence_ + +- [ ] 4.3 Gate promotion on the live fence + - Add the new fence state as a required condition at both existing points where a node's eligibility for promotion is decided (the periodic release check and the election-time check) + - Observable: a node with the fence engaged is refused promotion at both points; once the fence clears, the node is eligible again without any other change + - _Requirements: 9.4, 9.5_ + - _Boundary: Live Apply Resource Fence_ + +- [ ] 4.4 Fault-injection test for the live fence + - Using 3.1's forceable failure point, force a resource-exhaustion-style failure during array expansion and verify: the node fences itself and is refused promotion, constructs no partial or divergent set of elements, does not affect any other (simulated) node's independent progress, and self-heals — clearing the fence and completing the apply — once the forced condition is removed and the retry succeeds + - Observable: a single passing test exercising the full engage-refuse-clear-recover sequence + - _Requirements: 9.4, 9.5, 9.6_ + - _Boundary: Live Apply Resource Fence_ + - _Depends: 3.1_ + +- [ ] 5. Compact Scheduler/API-Server Push +- [ ] 5.1 (P) Build the compact scheduler push + - Add a function that encodes the compact (base jobid, array size, one spec) message for the scheduler and branch the existing job-created push handler to call it, instead of the existing per-job push, whenever the notification carries more than one element + - Leave the single-element (ordinary single-job) branch completely unmodified + - Observable: a synthetic multi-element notification produces exactly one compact push call, not N individually-encoded job structs; a synthetic single-element notification is unaffected + - _Requirements: 6.1, 6.3, 6.4_ + - _Boundary: Compact Scheduler/API-Server Push (controller-side scheduler push)_ + +- [ ] 5.2 (P) Build the compact API-server push + - Same construction as 5.1, for the API-server push handler and its own existing per-job push + - Observable: a synthetic multi-element notification produces exactly one compact push per connected API server, not N individually-encoded job structs; the single-element branch is unaffected + - _Requirements: 6.2, 6.4_ + - _Boundary: Compact Scheduler/API-Server Push (controller-side apiserver push)_ + +- [ ] 5.3 (P) Scheduler-side local expansion + - Decode the compact push and expand it locally into N ordinary schedulable tasks using the same base-jobid-plus-index arithmetic the controller itself uses + - Deduplicate by element identity so a redelivered push (e.g. after a scheduler reconnect) does not create duplicate tasks + - Log one identifiable confirmation line (base jobid plus element count) once expansion completes + - Observable: feeding the same compact push twice to the receiver produces the task set once, not twice, and exactly one confirmation log line per successful expansion + - _Requirements: 6.3, 6.4, 6.5, 8.4_ + - _Boundary: Compact Scheduler/API-Server Push (scheduler daemon receive path)_ + - _Depends: 5.1_ + +- [ ] 5.4 (P) API-server-side local expansion + - Decode the compact push and expand it locally into N per-element records using the same base-jobid-plus-index arithmetic + - Deduplicate by element identity so a redelivered push does not create duplicate records + - Log one identifiable confirmation line (base jobid plus element count) once expansion completes + - Observable: feeding the same compact push twice to the receiver produces the record set once, not twice, and exactly one confirmation log line per successful expansion + - _Requirements: 6.2, 6.4, 6.5, 8.4_ + - _Boundary: Compact Scheduler/API-Server Push (apiserver receive path)_ + - _Depends: 5.2_ + +- [ ] 5.5 Integration test: end-to-end compact push + - Commit one array record end-to-end and confirm the scheduler and the API server each receive exactly one push and each expand it into N correctly-addressable entries + - Confirm a simulated redelivery (reconnect) to either receiver does not double-expand + - Observable: a passing integration test covering both receivers' one-push/N-entries/no-double-expand behavior, including both receivers' confirmation log lines + - _Requirements: 6.1, 6.2, 6.3, 6.4, 6.5, 8.4_ + - _Boundary: Compact Scheduler/API-Server Push_ + +- [ ] 6. Integration & non-regression soak +- [ ] 6.1 Large-array soak scenario + - Extend the existing HA soak harness with a large-array submission scenario asserting: one WAL record and one commit barrier regardless of array size, failover replay time governed by the record's size rather than element count, and every array element individually addressable (status, cancel, completion) after a failover + - Observable: a soak run reporting the one-record/one-barrier assertion, a replay-time measurement independent of array size, and successful per-element status checks post-failover + - _Requirements: 4.3, 5.2, 5.3, 8.2, 8.3_ + - _Boundary: Non-Regression & Testing_ + +- [ ] 6.2 Chaotic-failover array retry scenario + - Extend the soak harness with a scenario submitting an array immediately before killing the primary, then retrying the identical submission against the new primary + - Assert the retry follows the same cross-primary dedup mechanism as single-job submission, extended to the array's full index range as one unit, and does not double-submit or partially duplicate + - Observable: a soak run showing the retried array resolves to exactly one committed instance of the array, with no partial duplication + - _Requirements: 8.2, 8.3_ + - _Boundary: Non-Regression & Testing_ + +- [ ] 6.3 Full regression pass + - Run the complete existing unit, integration, and soak suite (sequential and chaotic) and confirm zero new job loss, divergence, or liveness stall attributable to this feature + - Confirm the non-array (array size 1) WAL format, commit-barrier count, and reply are byte-identical to the pre-feature baseline + - Observable: a clean full-suite run with the byte-identical single-job baseline check passing alongside it + - _Requirements: 8.1, 8.2, 8.4_ + - _Boundary: Non-Regression & Testing_ diff --git a/src/controller/persist.c b/src/controller/persist.c index 21fe385..dda83ac 100644 --- a/src/controller/persist.c +++ b/src/controller/persist.c @@ -4517,6 +4517,12 @@ wal_commit_mode_t wal_classify(uint32_t wal_type, int detail) /* client job submit — request-shaped; keeps the in-hook sync path * (reference implementation, handle_jobsubmit) */ return WAL_MODE_WAIT; + case WAL_JOB_ARRAY_SUBMIT: + /* client job-array submit (job-array-single-wal-record task 1.1) — + * request-shaped exactly like WAL_JOB_SUBMIT: one requester awaits + * the single commit barrier before reply, regardless of array + * size. */ + return WAL_MODE_WAIT; case WAL_QUEUE_CTRL: /* client queue control — request-shaped; the handler barrier owes * commit completion before the requester reply (tasks 2-3) */ diff --git a/src/controller/testsuite/test_commit_barrier_rollout.c b/src/controller/testsuite/test_commit_barrier_rollout.c index c98f022..34fff02 100644 --- a/src/controller/testsuite/test_commit_barrier_rollout.c +++ b/src/controller/testsuite/test_commit_barrier_rollout.c @@ -171,6 +171,22 @@ static void test_classify_job_submit_is_wait(void) printf("test_classify_job_submit_is_wait: PASS\n"); } +static void test_classify_job_array_submit_is_wait(void) +{ + /* Arrange / Act */ + wal_commit_mode_t mode = wal_classify(WAL_JOB_ARRAY_SUBMIT, + HOOK_DETAIL_JOB_NEW); + + /* Assert: a job-array submission is request-shaped exactly like a + * single-job submission — one requester awaiting the commit barrier + * before reply (job-array-single-wal-record task 1.1) */ + TEST_ASSERT_EQUAL_INT_MESSAGE(WAL_MODE_WAIT, mode, + "WAL_JOB_ARRAY_SUBMIT must classify WAIT (request-shaped, same as " + "WAL_JOB_SUBMIT)"); + + printf("test_classify_job_array_submit_is_wait: PASS\n"); +} + static void test_classify_queue_ctrl_is_wait(void) { /* Arrange / Act */ @@ -266,6 +282,7 @@ static void test_classify_is_total_over_wal_entry_type(void) const char *name; } all_types[] = { { WAL_JOB_SUBMIT, HOOK_DETAIL_JOB_NEW, "WAL_JOB_SUBMIT" }, + { WAL_JOB_ARRAY_SUBMIT, HOOK_DETAIL_JOB_NEW, "WAL_JOB_ARRAY_SUBMIT" }, { WAL_JOB_START, HOOK_DETAIL_JOB_START, "WAL_JOB_START" }, { WAL_JOB_FINISH, HOOK_DETAIL_JOB_EXIT, "WAL_JOB_FINISH" }, { WAL_JOB_FAIL, HOOK_DETAIL_JOB_FAIL, "WAL_JOB_FAIL" }, @@ -1714,6 +1731,7 @@ void test_commit_barrier_rollout(void) UNITY_BEGIN(); /* Task 1.1 — classification totality + RETAIN routing (Req 2.1, 2.2, 3.1) */ RUN_TEST(test_classify_job_submit_is_wait); + RUN_TEST(test_classify_job_array_submit_is_wait); RUN_TEST(test_classify_queue_ctrl_is_wait); RUN_TEST(test_classify_worker_ctrl_detail_split); RUN_TEST(test_classify_all_job_state_types_are_retain); diff --git a/src/include/wal_types.h b/src/include/wal_types.h index 279f918..30dae2d 100644 --- a/src/include/wal_types.h +++ b/src/include/wal_types.h @@ -110,6 +110,10 @@ typedef enum { WAL_JOB_RUSAGE = 8, WAL_JOB_CKPT = 9, WAL_JOB_PREASON = 10, + WAL_JOB_ARRAY_SUBMIT = 11, /* job-array-single-wal-record: one record for + * the whole array (base spec + index range), + * distinct from WAL_JOB_SUBMIT's single-job + * format, which stays unchanged. */ WAL_QUEUE_CTRL = 20, WAL_WORKER_CTRL = 30, WAL_NOOP = 40, /* Internal: empty payload, replay no-op; appended only at promotion */