#define _GNU_SOURCE #include "persist.h" #include "config.h" #include "ctrl_metrics.h" #include "hook.h" #include "xce/cluster_def.h" #include "xce/errnox.h" #include "xce/types.h" #include "controller.capnp.h" #include "typemap.h" /* xce_job_rusages_t -- bug-451 RUSAGE hook arm */ #include "xlist.h" #include "cluster.h" #include "commit.h" /* commit_submit RETAIN registration (barrier-rollout 1.2) */ #include "xstring.h" /* makeString() -- bug-1691 WAL_JOB_CKPT ckptinfo adopt */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* restore_* functions defined in job.c / worker.c / queue.c */ extern int restore_job(cluster_t* c, uint64_t jobid, const void* capnp_payload, size_t len, uint64_t entry_lsn); extern int restore_job_state(cluster_t* c, uint64_t jobid, uint32_t state); /* restore_job_dispatch_state defined in job.c (bug-1678): WAL_JOB_DISPATCH's * own sibling of restore_job_state() above, additionally taking ownership of * an optional decoded xce_job_alloc_t so the dispatch-time resource * allocation survives standby catch-up / replicated-apply the same way * apply_committed_entry's WAL_JOB_DISPATCH case already restores it on the * primary's own boot replay. bug-1697: also takes a wire-carried * `start_time` (0 means "not carried -- leave job.start_time untouched", * the pre-fix payload's behavior) -- job.start_time is now DISPATCH-owned * (see wal_types.h's WAL_JOB_START doc comment for the full rationale), so * this seam must restore it too, not just state/allocs. */ extern int restore_job_dispatch_state(cluster_t* c, uint64_t jobid, uint32_t state, uint64_t start_time, xce_job_alloc_t* alloc); /* restore_job_start_state defined in job.c (bug-1687): WAL_JOB_START's own * sibling of restore_job_state() above, additionally taking ownership of an * optional decoded job_pidinfo_t so the worker-confirmed start survives * standby catch-up / replicated-apply the same way apply_committed_entry's * WAL_JOB_START case already restores it on the primary's own boot replay. * bug-1697: no longer takes/touches start_time -- job.start_time is now * exclusively DISPATCH-owned (restore_job_dispatch_state() above), so this * seam must leave it alone entirely rather than risk clobbering the correct * dispatch-time value with a stale worker-ack-time one. */ extern int restore_job_start_state(cluster_t* c, uint64_t jobid, uint32_t state, job_pidinfo_t* pidinfo); /* restore_job_finish_state defined in job.c (bug-1689): WAL_JOB_FINISH's own * sibling of restore_job_state() above, additionally taking ownership of an * optional decoded job_exitinfo_t and a wire-carried finish_time (0 means * "not carried -- leave job.finish_time/run_time untouched", the pre-fix * payload's behavior) so the worker-confirmed finish survives standby * catch-up / replicated-apply the same way apply_committed_entry's * WAL_JOB_FINISH case already restores it on the primary's own boot * replay. */ extern int restore_job_finish_state(cluster_t* c, uint64_t jobid, uint32_t state, uint64_t finish_time, job_exitinfo_t* exit_info); /* restore_job_rusage_state defined in job.c (bug-1690): WAL_JOB_RUSAGE's own * sibling of restore_job_state() above, additionally taking ownership of an * optional decoded xce_job_runusage_t (NULL for a state-only WAL_JOB_RUSAGE * entry -- either one written before this fix, or one fired while * job.run_usage was still NULL) so the client-reported per-host CPU/mem * usage survives standby catch-up / replicated-apply the same way * apply_committed_entry's WAL_JOB_RUSAGE case already restores it on the * primary's own boot replay. */ extern int restore_job_rusage_state(cluster_t* c, uint64_t jobid, uint32_t state, xce_job_runusage_t* runusage); /* restore_job_ckpt_state defined in job.c (bug-1691): WAL_JOB_CKPT's own * sibling of restore_job_state() above, additionally taking ownership of an * optional decoded ckptinfo string (NULL for a state-only WAL_JOB_CKPT entry * -- either one written before this fix, or one fired while job.ckptinfo was * still NULL) so the checkpoint info survives standby catch-up / replicated- * apply the same way apply_committed_entry's WAL_JOB_CKPT branch already * restores it on the primary's own boot replay. */ extern int restore_job_ckpt_state(cluster_t* c, uint64_t jobid, uint32_t state, char* ckptinfo); /* restore_job_preason_state defined in job.c (bug-1692): WAL_JOB_PREASON's own * sibling of restore_job_state() above. Unlike restore_job_ckpt_state()/ * restore_job_rusage_state(), a NULL decoded pointer is not automatically * "leave untouched" here -- pend_reasons has a genuine CLEAR operation * (do_clr_preason/HOOK_DETAIL_CLRPREASON) that must durably survive replay * too, so has_field disambiguates a pre-fix bare {jobid,state} payload * (has_field=0, pend_reasons left untouched -- full backward compatibility) * from a post-fix payload that explicitly carries the field's new value * (has_field=1): preason non-NULL adopts a freshly-decoded value, preason * NULL means pend_reasons was genuinely cleared and must become NULL too. */ extern int restore_job_preason_state(cluster_t* c, uint64_t jobid, uint32_t state, int has_field, xce_job_preason_t* preason); /* merge_job_runusage defined in job.c (bug-1690): merges a freshly-decoded * xce_job_runusage_t into *dst rather than freeing-and-replacing it. Needed * here too (apply_committed_entry's own WAL_JOB_RUSAGE branch below), not * just by restore_job_rusage_state() above -- *dst can be the job's LIVE * run_usage (update_job_rusages()/update_host_runusage(), misc.c, mutate it * synchronously and in place, with an xstring-backed host_name, BEFORE the * WAL fact is even appended), and persist_apply_pump_run() re-applies that * SAME fact on the SAME live job moments later via apply_committed_entry -- * a plain free()-and-replace (e.g. the capnp-generated free_JobRunusage_ptr) * would free a non-malloc'd, RTTI-managed pointer there. See * merge_job_runusage()'s own doc comment (job.c) for the full rationale. */ extern void merge_job_runusage(xce_job_runusage_t** dst, xce_job_runusage_t* src); extern int worker_from_capnp(cluster_t* c, const void* capnp_payload, size_t len); extern int restore_queue_state(cluster_t* c, const char* name, uint32_t status); /* restore_host_state defined in worker.c (config-upgrade-snapshot task 3, * Req 1.10/1.11/5.8) -- create-if-missing host admin-state restore, called * from load_snapshot_entities' host-frame loop below (task 4). */ extern int restore_host_state(cluster_t* c, const char* name, uint32_t admin_status); /* upgrade_materialization_verify defined in ha.c (config-upgrade-snapshot * task 6.1/6.2, Req 1.8/3.2/3.4/5.2/5.5) -- the Activation Fence's own * verify function. Called unconditionally from apply_committed_entry's * WAL_CONFIG_MARKER case below (task 6.2): checkpoint completion is the * design-intended trigger that re-evaluates this fence (design.md's * Activation Fence component, "naturally re-evaluates against the newer * marker once that checkpoint's write_snapshot call runs"), since it is * the ONLY event that can newly set OR clear it from a running process -- * every other call path (upgrade_materialization_retry_fn and its * callers) is itself gated on the fence already being set, so without * this call site the fence could never transition from its zero- * initialized default. */ extern int upgrade_materialization_verify(cluster_t* c); /* config_activate_committed defined in config_repl.c (config-upgrade-snapshot * task 5/6.2 remediation, bug-1469, Req 1.8/3.2/3.4) -- re-parses the * locally-held config bytes referenced by the latest committed * WAL_CONFIG_MARKER into c->myconf (load_primary_config), hash-verifying * them first. Called unconditionally from apply_committed_entry's * WAL_CONFIG_MARKER case below, BEFORE upgrade_materialization_verify: every * member's own live apply of a marker it just committed must refresh its * own c->myconf, not only members that separately receive an out-of-band * handle_repl_config_push (see the call site's own doc comment for the full * rationale). Declared here via extern, mirroring upgrade_materialization_ * verify just above, rather than pulling in config_repl.h's full include * surface for one symbol. */ extern int config_activate_committed(cluster_t* c); /* raft-apply-decoupling task 3.2: pending command-apply hand-off list node * (producer side; the not-yet-built ordered apply pump, task 6.1/6.2, is * the consumer). Declared here (rather than persist.h) so the list stays * an implementation detail — persist.h exposes only the opaque `void*` * head field plus persist_register_pending_apply/persist_has_pending_apply. * See persist.h's doc comment on persist_register_pending_apply for the * full contract. */ typedef struct pending_command_apply { uint64_t lsn; pending_apply_notify_fn on_applied; void* ctx; struct pending_command_apply* next; } pending_command_apply_t; /* ---- Cap'n Proto serialization helpers (WAL payload encoding) ---------- */ /* Serialize xce_job_t to a malloc'd Cap'n Proto buffer. * Caller must free(*out) on success. */ static int capnp_encode_job(const xce_job_t* j, uint8_t** out, size_t* out_len) { struct capn c; struct capn_segment* cs; struct Job cj; Job_ptr jp; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_Job(cs, &cj, (xce_job_t*)j); jp = new_Job(cs); write_Job(&cj, jp); capn_setp(capn_root(&c), 0, jp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } /* Serialize xce_worker_t to a malloc'd Cap'n Proto buffer. */ static int capnp_encode_worker(const xce_worker_t* w, uint8_t** out, size_t* out_len) { struct capn c; struct capn_segment* cs; struct Worker cw; Worker_ptr wp; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_Worker(cs, &cw, (xce_worker_t*)w); wp = new_Worker(cs); write_Worker(&cw, wp); capn_setp(capn_root(&c), 0, wp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } /* Serialize xce_job_alloc_t to a malloc'd Cap'n Proto buffer (bug-1678): * lets WAL_JOB_DISPATCH carry the scheduler's resource allocation alongside * the {jobid, state} header, so a restart replay can reconstruct it instead * of leaving job.allocs permanently NULL. Mirrors capnp_encode_worker above * exactly, using the JobAlloc struct (already in controller.capnp as the * existing Job.allocs field -- no schema change needed here). */ static int capnp_encode_joballoc(const xce_job_alloc_t* a, uint8_t** out, size_t* out_len) { struct capn c; struct capn_segment* cs; struct JobAlloc cja; JobAlloc_ptr jap; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_JobAlloc(cs, &cja, (xce_job_alloc_t*)a); jap = new_JobAlloc(cs); write_JobAlloc(&cja, jap); capn_setp(capn_root(&c), 0, jap.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } /* Decode a Cap'n Proto JobAlloc buffer (bug-1678) into a freshly heap- * allocated xce_job_alloc_t*, mirroring restore_job()'s Job decode pattern * (job.c) but returning the struct directly instead of constructing a job. * Returns NULL on a NULL/empty payload; caller owns the result (release via * free_JobAlloc_ptr). */ static xce_job_alloc_t* decode_joballoc(const uint8_t* payload, size_t len) { struct capn c_ctx; JobAlloc_ptr jap; struct JobAlloc cja; xce_job_alloc_t* alloc; if (!payload || len == 0) return NULL; capn_init_mem(&c_ctx, payload, len, 0); jap.p = capn_getp(capn_root(&c_ctx), 0, 1); memset(&cja, 0, sizeof(cja)); read_JobAlloc(&cja, jap); alloc = calloc(1, sizeof(*alloc)); if (!alloc) { capn_free(&c_ctx); return NULL; } decode_JobAlloc(alloc, &cja); capn_free(&c_ctx); return alloc; } /* Serialize xce_job_preason_t to a malloc'd Cap'n Proto buffer (bug-1692): * lets WAL_JOB_PREASON carry the scheduler's host/resource-based pend * reasons alongside the {jobid, state} header, so a restart replay can * reconstruct job.pend_reasons instead of leaving it permanently NULL. * Mirrors capnp_encode_joballoc above exactly, using the JobPendReason * struct (already in controller.capnp as the existing Job.pendreason field * -- no schema change needed here). */ static int capnp_encode_jobpreason(const xce_job_preason_t* pr, uint8_t** out, size_t* out_len) { struct capn c; struct capn_segment* cs; struct JobPendReason cjp; JobPendReason_ptr jpp; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_JobPendReason(cs, &cjp, (xce_job_preason_t*)pr); jpp = new_JobPendReason(cs); write_JobPendReason(&cjp, jpp); capn_setp(capn_root(&c), 0, jpp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } /* Decode a Cap'n Proto JobPendReason buffer (bug-1692) into a freshly heap- * allocated xce_job_preason_t*, mirroring decode_joballoc()'s pattern above. * Returns NULL on a NULL/empty payload; caller owns the result. The decoded * tree is plain calloc()/strdup() throughout (capnp-generated * decode_HostBasedReason/decode_ResourceBasedReason use STRING_DUP + calloc, * verified in controller.capnp.c), NOT xstring/RTTI-backed like ckptinfo or * rusage's host_name -- release via free_JobPendReason_ptr (plain free()), * never dereference(). */ static xce_job_preason_t* decode_jobpreason(const uint8_t* payload, size_t len) { struct capn c_ctx; JobPendReason_ptr jpp; struct JobPendReason cjp; xce_job_preason_t* preason; if (!payload || len == 0) return NULL; capn_init_mem(&c_ctx, payload, len, 0); jpp.p = capn_getp(capn_root(&c_ctx), 0, 1); memset(&cjp, 0, sizeof(cjp)); read_JobPendReason(&cjp, jpp); preason = calloc(1, sizeof(*preason)); if (!preason) { capn_free(&c_ctx); return NULL; } decode_JobPendReason(preason, &cjp); capn_free(&c_ctx); return preason; } /* Serialize job_pidinfo_t to a malloc'd Cap'n Proto buffer (bug-1687): * lets WAL_JOB_START carry the worker-reported pids/pgids/exdata alongside * the {jobid, state} header, so a restart replay can reconstruct * job.pidinfo instead of leaving it permanently NULL. Mirrors * capnp_encode_joballoc above exactly, using the JobPIDInfo struct (already * in controller.capnp as the existing Job.pidinfo field -- no schema change * needed here). */ static int capnp_encode_pidinfo(const job_pidinfo_t* info, uint8_t** out, size_t* out_len) { struct capn c; struct capn_segment* cs; struct JobPIDInfo cji; JobPIDInfo_ptr jip; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_JobPIDInfo(cs, &cji, (job_pidinfo_t*)info); jip = new_JobPIDInfo(cs); write_JobPIDInfo(&cji, jip); capn_setp(capn_root(&c), 0, jip.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } /* Decode a Cap'n Proto JobPIDInfo buffer (bug-1687) into a freshly heap- * allocated job_pidinfo_t*, mirroring decode_joballoc()'s pattern above. * Returns NULL on a NULL/empty payload; caller owns the result (release via * free_JobPIDInfo_ptr). */ static job_pidinfo_t* decode_pidinfo(const uint8_t* payload, size_t len) { struct capn c_ctx; JobPIDInfo_ptr jip; struct JobPIDInfo cji; job_pidinfo_t* info; if (!payload || len == 0) return NULL; capn_init_mem(&c_ctx, payload, len, 0); jip.p = capn_getp(capn_root(&c_ctx), 0, 1); memset(&cji, 0, sizeof(cji)); read_JobPIDInfo(&cji, jip); info = calloc(1, sizeof(*info)); if (!info) { capn_free(&c_ctx); return NULL; } decode_JobPIDInfo(info, &cji); capn_free(&c_ctx); return info; } /* Decode a native (non-capnp) WAL_JOB_FINISH exit_code trailer (bug-1689) * into a freshly heap-allocated job_exitinfo_t*. job_exitinfo_t (xce/types.h) * is a single int with NO existing Cap'n Proto mapping (confirmed -- no * JobExitInfo struct in controller.capnp), so this mirrors wal_jobstate_t's * own native-struct style rather than inventing a capnp wrapper for one int * -- unlike decode_joballoc()/decode_pidinfo() above, which decode genuine * capnp payloads. Returns NULL on a short/absent payload; caller owns the * result (release via free()/FREEUP -- no nested pointers). */ static job_exitinfo_t* decode_jobexitinfo(const uint8_t* payload, size_t len) { job_exitinfo_t* info; int32_t exitcode; if (!payload || len < sizeof(exitcode)) return NULL; memcpy(&exitcode, payload, sizeof(exitcode)); info = (job_exitinfo_t*)calloc(1, sizeof(*info)); if (!info) return NULL; info->exitcode = (int)exitcode; return info; } /* Decode a native (non-capnp) WAL_JOB_CKPT trailing string payload (bug-1691) * into a freshly heap-allocated, NUL-terminated char*. job.ckptinfo (xce/ * types.h) is a plain char* -- NOT the richer xce_ckptinfo_t/CheckpointInfo * capnp struct used elsewhere for a different purpose -- so this mirrors * WAL_QUEUE_CTRL's own [header][name NUL-terminated] wire shape * (persist_queue_hook_fun / apply_wal_entry's WAL_QUEUE_CTRL case) rather * than inventing a capnp wrapper for a bare string. Bounds-checked against * `len` (the WAL entry's own payload_len, per the WAL_QUEUE_CTRL precedent) * rather than trusting the bytes are NUL-terminated -- returns NULL if no * NUL exists within the bounded region (a defensively-rejected malformed/ * truncated payload), mirroring decode_jobexitinfo()'s short-payload guard * above. Returns NULL on a NULL/empty payload too; caller owns the result * (release via free()). The returned buffer is a TRANSIENT decode copy, not * itself adopted into job.ckptinfo -- makeString() (xstring.c) copies its * input into its own refcounted allocation, so callers pass this result to * makeString() and then free it (see restore_job_ckpt_state()/job.c and * apply_committed_entry()'s own WAL_JOB_CKPT branch below). */ static char* decode_ckptinfo(const uint8_t* payload, size_t len) { char* copy; if (!payload || len == 0) return NULL; if (memchr(payload, '\0', len) == NULL) return NULL; /* not NUL-terminated within the declared bound -- reject */ copy = (char*)malloc(len); if (!copy) return NULL; memcpy(copy, payload, len); return copy; } /* Serialize xce_job_runusage_t to a malloc'd Cap'n Proto buffer (bug-1690): * lets WAL_JOB_RUSAGE carry the client-reported, accumulated per-host * CPU/mem usage (job->job.run_usage) alongside the {jobid, state} header, so * a restart/catch-up replay can reconstruct it instead of leaving it * permanently NULL. Mirrors capnp_encode_joballoc()/capnp_encode_pidinfo() * above exactly, using the NEW JobRunusage struct (controller.capnp) -- * unlike JobAlloc/JobPIDInfo, xce_job_runusage_t had NO existing Cap'n Proto * mapping at all, so this is a genuine schema addition (see the schema's own * doc comment on JobRunusage for why it is not the same type as * xce_job_rusages_t/JobRusages, a similar-looking but unrelated struct). */ static int capnp_encode_jobrunusage(const xce_job_runusage_t* r, uint8_t** out, size_t* out_len) { struct capn c; struct capn_segment* cs; struct JobRunusage cjr; JobRunusage_ptr jrp; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_JobRunusage(cs, &cjr, (xce_job_runusage_t*)r); jrp = new_JobRunusage(cs); write_JobRunusage(&cjr, jrp); capn_setp(capn_root(&c), 0, jrp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } /* Decode a Cap'n Proto JobRunusage buffer (bug-1690) into a freshly heap- * allocated xce_job_runusage_t*, mirroring decode_joballoc()/decode_pidinfo()'s * pattern above. Returns NULL on a NULL/empty payload; caller owns the * result (release via free_JobRunusage_ptr). */ static xce_job_runusage_t* decode_jobrunusage(const uint8_t* payload, size_t len) { struct capn c_ctx; JobRunusage_ptr jrp; struct JobRunusage cjr; xce_job_runusage_t* r; if (!payload || len == 0) return NULL; capn_init_mem(&c_ctx, payload, len, 0); jrp.p = capn_getp(capn_root(&c_ctx), 0, 1); memset(&cjr, 0, sizeof(cjr)); read_JobRunusage(&cjr, jrp); r = calloc(1, sizeof(*r)); if (!r) { capn_free(&c_ctx); return NULL; } decode_JobRunusage(r, &cjr); capn_free(&c_ctx); return r; } /* Serialize a host admin-state frame (config-upgrade-snapshot task 2.1, Req * 1.6/1.7/1.9) to a malloc'd Cap'n Proto buffer. Mirrors capnp_encode_job * above exactly -- capnp_host_admin_state_t (typemap.h) is a frame-only * (name, status) pair, never the full xce_worker_t (design.md Design * Invariant 5). Caller must free(*out) on success. */ static int capnp_encode_host_admin_state(const capnp_host_admin_state_t* h, uint8_t** out, size_t* out_len) { struct capn c; struct capn_segment* cs; struct HostAdminState ch; HostAdminState_ptr hp; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_HostAdminState(cs, &ch, (capnp_host_admin_state_t*)h); hp = new_HostAdminState(cs); write_HostAdminState(&ch, hp); capn_setp(capn_root(&c), 0, hp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } /* Serialize a queue admin-state frame (config-upgrade-snapshot task 2.2, Req * 1.9/5.7) to a malloc'd Cap'n Proto buffer. Mirrors * capnp_encode_host_admin_state above exactly -- capnp_queue_admin_state_t * (typemap.h) is a frame-only (name, status) pair, matching * restore_queue_state's exact (name, status) contract (queue.c:208), same * field content as the prior raw layout, only the wire encoding changes. * Caller must free(*out) on success. */ static int capnp_encode_queue_admin_state(const capnp_queue_admin_state_t* q, uint8_t** out, size_t* out_len) { struct capn c; struct capn_segment* cs; struct QueueAdminState cq; QueueAdminState_ptr qp; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_QueueAdminState(cs, &cq, (capnp_queue_admin_state_t*)q); qp = new_QueueAdminState(cs); write_QueueAdminState(&cq, qp); capn_setp(capn_root(&c), 0, qp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } /* WAL_WORKER_CTRL payload: [uint32_t detail][capnp bytes...] or [uint32_t detail][name\0] */ #define WORKER_WAL_HDR_SIZE sizeof(uint32_t) /* ------------------------------------------------------------------ */ /* Constants */ /* ------------------------------------------------------------------ */ #define WAL_SEG_FMT "%s/seg-%016" PRIx64 ".wal" /* NFS_SUPER_MAGIC — reject WAL on NFS mounts (unreliable fsync) */ #define NFS_SUPER_MAGIC 0x6969 /* ------------------------------------------------------------------ */ /* Internal helpers */ /* ------------------------------------------------------------------ */ static uint64_t now_us(void) { struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); return (uint64_t)ts.tv_sec * 1000000ULL + (uint64_t)(ts.tv_nsec / 1000); } /* Recursive mkdir — equivalent to mkdir -p */ static int mkdir_p(const char* path, mode_t mode) { char tmp[4096]; char* p; size_t len; snprintf(tmp, sizeof(tmp), "%s", path); len = strlen(tmp); if (len > 0 && tmp[len - 1] == '/') tmp[len - 1] = '\0'; for (p = tmp + 1; *p; p++) { if (*p == '/') { *p = '\0'; if (mkdir(tmp, mode) != 0 && errno != EEXIST) return -1; *p = '/'; } } if (mkdir(tmp, mode) != 0 && errno != EEXIST) return -1; return 0; } /* Open (and pre-allocate) a new WAL segment named after current_lsn */ static int open_new_segment(persist_t* p) { snprintf(p->seg_path, sizeof(p->seg_path), WAL_SEG_FMT, p->wal_path, p->current_lsn); int fd = open(p->seg_path, O_RDWR | O_CREAT | O_TRUNC, 0600); if (fd < 0) return -errno; /* Pre-allocate segment space to avoid fragmentation and metadata updates * on every write. posix_fallocate is best-effort — ignore errors. */ posix_fallocate(fd, 0, (off_t)p->config.wal_segment_bytes); p->wal_fd = fd; p->seg_offset = 0; return 0; } /* Close current segment and open a fresh one */ static int rotate_segment(persist_t* p) { if (p->wal_fd >= 0) { fdatasync(p->wal_fd); close(p->wal_fd); p->wal_fd = -1; } return open_new_segment(p); } /* ------------------------------------------------------------------ */ /* Demand-driven group-commit flusher (ha-commit-durability task 2.1) */ /* */ /* Repurposes the old fixed-interval fsync timer into a flusher driven */ /* by the append-vs-synced gap. On each pass it batches one fdatasync */ /* over all entries appended since the last sync, advances durable_lsn */ /* to the synced tip, and wakes the commit evaluation. --wal-sync-ms */ /* now bounds the MAX batching latency (a no-further-append entry is */ /* still synced within the cadence), not a durability gap. */ /* */ /* LOST-WAKEUP / CORRECTNESS REASONING (gating item): */ /* */ /* The "work pending" condition is LEVEL-triggered on two persistent */ /* variables, NOT an edge: pending ⇔ durable_lsn < current_lsn - 1 */ /* (flusher_work_pending below, evaluated under p->lock). Because it */ /* is a comparison of stored state — not a transient flag consumed on */ /* read — a signal that races the flusher's wait can never be lost: */ /* */ /* - persist_wal_async advances current_lsn under p->lock, THEN */ /* signals fsync_cond. If the flusher is mid-flush when the append */ /* lands, current_lsn has already moved past the tip the flusher */ /* captured, so after that flush durable_lsn < current_lsn - 1 */ /* STILL holds; the flusher re-checks the level under p->lock on */ /* its next loop iteration and flushes again (it does NOT wait */ /* first). The missed signal is therefore harmless — the level */ /* itself carries the pending work forward. */ /* - If the append's signal arrives while the flusher sits in */ /* pthread_cond_wait, it wakes it. If it arrives in the tiny window */ /* after the flusher decided "no work" but before it re-enters the */ /* wait, the flusher re-evaluates the level under p->lock before */ /* waiting (we re-check inside the fsync_mutex-held region by */ /* reading current_lsn/durable_lsn under p->lock first), so it */ /* will not block while work is pending. */ /* */ /* "NEVER durable before its fdatasync": the flusher snapshots */ /* tip = current_lsn - 1 AND calls fdatasync WHILE HOLDING p->lock. */ /* persist_wal_async also pwrites under p->lock, so no append can */ /* extend the segment between the tip snapshot and the fdatasync — by */ /* the time fdatasync returns, every byte for LSNs <= tip is on stable */ /* storage. durable_lsn is only then advanced to tip (monotonic max). */ /* Releasing p->lock and broadcasting commit-eval happens after the */ /* advance, so a WAIT caller that observes durable_lsn >= its LSN is */ /* guaranteed the bytes are synced. */ /* */ /* Lock discipline: p->lock and fsync_mutex are NEVER held together */ /* (fsync_mutex guards only the cond wait; the flush section takes */ /* p->lock alone). This keeps fdatasync off any lock the iomgr / */ /* heartbeat path needs and preserves liveness (Req 1.6). */ /* ------------------------------------------------------------------ */ /* Test-only fault-injection probe (ha-commit-durability task 2.1 error path). * Returns true when the test has forced fdatasync to be treated as failed. * Compiled to a constant `false` in production (no -DXCE_TESTING), so the * fail-loud gate below collapses to the pure rc != 0 check with zero overhead * and no dependency on the test-only struct field. */ static inline bool flusher_fsync_forced_fail(persist_t* p) { #ifdef XCE_TESTING return atomic_load_explicit(&p->test_force_fsync_fail, memory_order_acquire) != 0; #else (void)p; return false; #endif } /* Returns true iff there is un-synced appended data (level predicate). * MUST be called with p->lock held. current_lsn - 1 is the appended tip * (current_lsn is the NEXT lsn to assign); LSN 0 reserved, so a fresh * store has current_lsn == 1 → tip 0 → no pending work. */ static bool flusher_work_pending_locked(persist_t* p) { uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); return dur < tip; } /* Perform one group-commit flush: snapshot the appended tip, fdatasync the * live segment, and advance durable_lsn to the synced tip. Takes p->lock * for the whole snapshot+sync+advance so no concurrent append can extend * the segment between the tip read and the fdatasync. Returns the new * durable_lsn (unchanged if there was nothing to sync). */ static uint64_t flusher_flush_once(persist_t* p) { #ifdef XCE_TESTING atomic_fetch_add_explicit(&p->test_flusher_flush_attempt_count, 1, memory_order_relaxed); #endif pthread_mutex_lock(&p->lock); uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); if (dur >= tip || p->wal_fd < 0) { /* Nothing new to sync (or no segment yet). Still flush the fd for a * clean-shutdown / fallback pass, but do not advance the watermark. * The watermark is already <= tip here, so a failed fallback sync * cannot make it claim more than is durable — but we still surface * the error loudly so a degrading disk is never silent (Req 1.1). */ if (p->wal_fd >= 0) { int rc = fdatasync(p->wal_fd); if (rc != 0 || flusher_fsync_forced_fail(p)) { int e = (rc != 0) ? errno : EIO; ERROR("persist: flusher fallback fdatasync FAILED (errno=%d %s)" " — durable_lsn held at %" PRIu64 "; WAL not advanced", e, strerror(e), dur); } } pthread_mutex_unlock(&p->lock); return dur; } /* fdatasync under p->lock: appends also hold p->lock to pwrite, so the * segment cannot grow between this tip snapshot and the sync. Every byte * for LSNs <= tip is therefore on stable storage when fdatasync returns * SUCCESSFULLY. */ int rc = fdatasync(p->wal_fd); /* FAIL-LOUD GATE (Req 1.1/1.2): advance durable_lsn ONLY on a successful * sync. On EIO/ENOSPC the bytes are NOT on stable storage, so leaving the * watermark unchanged keeps the non-durable LSNs from being counted toward * the commit majority. We do NOT retry into an advanced watermark — the * level predicate stays pending and the next pass retries the sync. */ if (rc != 0 || flusher_fsync_forced_fail(p)) { int e = (rc != 0) ? errno : EIO; ERROR("persist: flusher fdatasync FAILED (errno=%d %s) — durable_lsn " "held at %" PRIu64 " (tip=%" PRIu64 " NOT made durable); commit " "majority will not count these LSNs", e, strerror(e), dur, tip); pthread_mutex_unlock(&p->lock); return dur; } /* Monotonic advance to the synced tip — never past it (the invariant * durable_lsn <= current_lsn - 1 is preserved because tip == current_lsn-1 * was read under this same lock). One fdatasync advances durable_lsn past * however many LSNs were appended since the last sync (batching). */ if (tip > dur) atomic_store_explicit(&p->durable_lsn, tip, memory_order_release); #ifdef XCE_TESTING /* Record which thread ran this successful fdatasync (task 5.1 / Req 1.6): * a liveness test asserts this is the dedicated flusher thread, never the * caller/iomgr/heartbeat thread. Written under p->lock; read after the * flusher has caught up, so no extra synchronization is needed. */ p->test_last_fdatasync_thread = pthread_self(); atomic_store_explicit(&p->test_last_fdatasync_thread_set, 1, memory_order_release); #endif pthread_mutex_unlock(&p->lock); return tip; } static void* fsync_worker(void* arg) { persist_t* p = arg; pthread_mutex_lock(&p->fsync_mutex); while (!atomic_load_explicit(&p->fsync_stop, memory_order_acquire)) { /* Re-evaluate the level predicate BEFORE waiting so a signal that * raced in just before we entered the wait is not needed for * correctness — if work is already pending we skip the wait entirely * and flush. (Reads current_lsn/durable_lsn under p->lock.) */ pthread_mutex_lock(&p->lock); bool pending = flusher_work_pending_locked(p); pthread_mutex_unlock(&p->lock); if (!pending) { /* No pending work: wait for an append-signal OR the cadence * deadline (bounded fallback flush so a lone no-further-append * entry is still synced within wal_sync_ms — Req 3.1). */ long ms = (long)p->config.wal_sync_ms; struct timespec dl; clock_gettime(CLOCK_REALTIME, &dl); dl.tv_sec += ms / 1000; dl.tv_nsec += (ms % 1000) * 1000000L; if (dl.tv_nsec >= 1000000000L) { dl.tv_sec += 1; dl.tv_nsec -= 1000000000L; } pthread_cond_timedwait(&p->fsync_cond, &p->fsync_mutex, &dl); if (atomic_load_explicit(&p->fsync_stop, memory_order_acquire)) break; } /* Flush section: never hold fsync_mutex across the fdatasync (the * flush takes p->lock alone). Release fsync_mutex, flush, re-acquire. * After the flush we loop and re-check the level — if a new append * landed during the flush, the level is still pending and we flush * again without needing its signal (lost-wakeup-safe). */ pthread_mutex_unlock(&p->fsync_mutex); uint64_t before = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); uint64_t after = flusher_flush_once(p); /* COMMIT-EVAL WAKE (task 2.2 consumer wired): durable_lsn is advanced * INSIDE flusher_flush_once (atomic store with release) BEFORE we * return here, and flusher_flush_once has already RELEASED p->lock. * Now — holding NO persist lock — invoke the registered durable-advance * callback so the commit engine re-evaluates its majority and wakes any * COMMIT_MODE_WAIT caller whose entry just became durable locally. * * LOST-WAKEUP SAFETY: the wake fires strictly AFTER the durable_lsn * release-store, and the commit engine's predicate re-reads durable_lsn * via an acquire load under ce->lock — so a WAIT caller either (a) sees * the advanced watermark on its own re-check, or (b) is blocked in * pthread_cond_wait and is woken by this broadcast; it can never both * miss the watermark AND miss the wake. We only fire the callback when * durable_lsn actually advanced (after > before): a no-op flush (or a * failed fdatasync that held the watermark) needs no commit re-eval, * avoiding spurious wakes. * * LOCK ORDERING (cycle-free): the flusher releases p->lock inside * flusher_flush_once, THEN the callback takes ce->lock (commit side). * The commit side only ever takes ce->lock and reads persist atomics — * it never takes p->lock while holding ce->lock. So the only ordering * is p->lock → ce->lock; there is no ce->lock → p->lock edge, hence no * deadlock cycle. The callback explicitly MUST NOT take p->lock. */ if (after > before && p->on_durable_advance != NULL) { p->on_durable_advance(p->durable_cb_ctx); } pthread_mutex_lock(&p->fsync_mutex); /* bug-1563: a sustained fdatasync failure leaves durable_lsn < tip * forever (the fail-loud gate above deliberately never advances it * on failure — that part is correct and unchanged). flusher_work_ * pending_locked() can therefore never distinguish "genuine fresh * work just landed" from "still stuck from the last failed attempt" * — both read as pending=true, so without this check the loop above * would skip the wait on EVERY iteration and spin flusher_flush_once * (and thus fdatasync) as fast as the CPU/lock allow. Live evidence: * a 40-round chaotic /xce-test-ha soak hit 300k+ failed attempts on * one node in well under a minute (one core pinned, log flooded). * Only skip the wait when the flush actually made progress (pending * was true and it succeeded) or there was nothing pending to begin * with (the pre-existing, correct fast path); a flush that was * pending but made no progress (after == before) still waits out the * normal wal_sync_ms cadence before retrying — same bounded-latency * budget the idle path already uses, no new tunable needed. */ if (pending && after == before && !atomic_load_explicit(&p->fsync_stop, memory_order_acquire)) { long ms = (long)p->config.wal_sync_ms; struct timespec dl; clock_gettime(CLOCK_REALTIME, &dl); dl.tv_sec += ms / 1000; dl.tv_nsec += (ms % 1000) * 1000000L; if (dl.tv_nsec >= 1000000000L) { dl.tv_sec += 1; dl.tv_nsec -= 1000000000L; } pthread_cond_timedwait(&p->fsync_cond, &p->fsync_mutex, &dl); } } pthread_mutex_unlock(&p->fsync_mutex); return NULL; } /* ------------------------------------------------------------------ * * WAL term adapter (ha-term-safety task 1.3, R1.1/R1.5/R1.6/R1.7/R1.8) * * ------------------------------------------------------------------ */ /* Sentinel stored in a term_index[] slot that has never been explicitly * written by record_entry_term — distinct from a genuinely-recorded raw term * of 0 (a legacy pre-term-safety entry, which maps to WAL_LEGACY_TERM at * read-back). Reserved so growth (below) can leave a gap slot reading back as * truly ABSENT forever, not just until the NEXT lsn happens to be recorded * (bug-1475 follow-up: a WAL_CHECKPOINT slot is deliberately never recorded — * see persist_write_checkpoint_marker's comment — but before this fix, the * geometric-growth realloc for ANY later lsn zero-filled the checkpoint's own * gap slot, and entry_term_locked's raw==0 rule then read that zero-fill back * as WAL_LEGACY_TERM, not absent — silently and retroactively flipping a * deliberately-unrecorded LSN's term the instant any later entry was * appended. Confirmed via live instrumentation: persist_entry_term(checkpoint * lsn) read back 0 (absent) immediately after the checkpoint, then 1 * (WAL_LEGACY_TERM) once a later real entry was appended and grew the array * past it — a time-of-check/time-of-use divergence between the sender (which * scans/streams AFTER later entries exist) and a receiver that has not yet * appended past the same checkpoint, causing logOk's prev_log_term equality * check to spuriously mismatch (0 != 1) and permanently reject catch-up). */ #define TERM_INDEX_UNSET UINT64_MAX /* Record an entry's RAW hdr.term at its 1-based LSN in the in-memory per-LSN * term index (R1.5). Grows the index on demand; LSNs replay/append contiguously * so the array is dense (slot i == LSN i+1). Any gap left by a non-contiguous * LSN (e.g. a deliberately-skipped WAL_CHECKPOINT slot) is filled with the * TERM_INDEX_UNSET sentinel by realloc-then-memset, which reads back as * "absent" PERMANENTLY — regardless of whether a later LSN is subsequently * recorded and grows term_index_count past it (bug-1475 follow-up; see * TERM_INDEX_UNSET's comment). The raw term (including a legacy 0) is stored * verbatim; the legacy-0 -> WAL_LEGACY_TERM mapping is applied only at * READ-BACK (persist_entry_term), keeping "present legacy-0" distinguishable * from "absent". * * Callers: the live append path (holds p->lock) and the single-threaded * recovery path (replay_wal — no other thread touches the index before the node * participates, so no lock is required there). Best-effort: an allocation * failure is logged loudly and leaves the index unextended (read-back then * reports the LSN absent) — it never aborts and never corrupts existing * entries. */ static void record_entry_term(persist_t* p, uint64_t lsn, uint64_t term) { if (!p || lsn == 0) return; /* LSN 0 is reserved — never an entry */ if (lsn > p->term_index_cap) { /* Grow geometrically to amortize; never below the requested LSN. */ uint64_t new_cap = p->term_index_cap ? p->term_index_cap * 2 : 64; if (new_cap < lsn) new_cap = lsn; uint64_t* grown = realloc(p->term_index, new_cap * sizeof(uint64_t)); if (!grown) { ERROR("persist: term-index realloc failed (lsn=%" PRIu64 ", cap=%" PRIu64 "->%" PRIu64 "); term for this LSN will read " "back as absent", lsn, p->term_index_cap, new_cap); return; } /* Fill the newly grown region with the UNSET sentinel (all-1s byte * pattern, so memset can fill it directly) so any never-explicitly- * written slot reads absent FOREVER, not just until term_index_count * happens to grow past it (bug-1475 follow-up). */ memset(grown + p->term_index_cap, 0xFF, (new_cap - p->term_index_cap) * sizeof(uint64_t)); p->term_index = grown; p->term_index_cap = new_cap; } p->term_index[lsn - 1] = term; if (lsn > p->term_index_count) p->term_index_count = lsn; } /* entry_term_locked: the lock-free core of persist_entry_term — read the * recorded Raft term for `lsn` from the in-memory term index, applying the * legacy-0 rule (R1.8: a present term-0 entry maps to WAL_LEGACY_TERM; an absent * LSN returns 0). Caller MUST hold p->lock. Used by the append path * (persist_wal_async) to source the AppendEntries prev_log_term while already * under the lock — calling persist_entry_term there would re-acquire p->lock * and deadlock. */ static uint64_t entry_term_locked(const persist_t* p, uint64_t lsn) { if (!p || lsn == 0) return 0; if (lsn <= p->term_index_count && p->term_index != NULL) { uint64_t raw = p->term_index[lsn - 1]; /* bug-1475 follow-up: a slot filled by growth but never explicitly * recorded (e.g. a skipped WAL_CHECKPOINT lsn) stays absent forever — * see TERM_INDEX_UNSET. */ if (raw == TERM_INDEX_UNSET) return 0; return (raw == 0) ? WAL_LEGACY_TERM : raw; } return 0; } /* Shrink the in-memory per-LSN term index so that every LSN > `keep_lsn` reads * back as absent (ha-term-safety task 2.2). Called from the truncate path after * the on-disk entries above keep_lsn are removed, keeping the term index * consistent with the WAL: a later persist_entry_term on a truncated LSN must * return 0. The backing allocation is left as-is (only the live count moves); * the TERM_INDEX_UNSET-sentinel-on-grow contract (bug-1475 follow-up — grown * slots are filled with TERM_INDEX_UNSET, not zero, since this fix) means a * later re-append at those LSNs still overwrites cleanly: record_entry_term * always writes the slot explicitly, regardless of what fill value (zero or * sentinel) it previously held. Caller must hold p->lock. */ static void truncate_term_index_locked(persist_t* p, uint64_t keep_lsn) { if (!p) return; if (keep_lsn < p->term_index_count) p->term_index_count = keep_lsn; } void persist_set_term(persist_t* p, uint64_t term) { if (!p) return; /* p->term is read on the append path under p->lock; take the lock so the * stamped value is consistent with the append that reads it. */ pthread_mutex_lock(&p->lock); p->term = term; pthread_mutex_unlock(&p->lock); } uint64_t persist_entry_term(const persist_t* p, uint64_t lsn) { if (!p || lsn == 0) return 0; /* Take p->lock to read consistently against concurrent record_entry_term * on the append path. Cast away const — the lock is logically a mutable * sub-object of an otherwise-const read. */ persist_t* mp = (persist_t*)p; pthread_mutex_lock(&mp->lock); /* Legacy-zero rule (R1.8) lives in entry_term_locked: a present entry * carrying term 0 maps to the single lowest-known real Raft term, applied * uniformly so two legacy-0 entries at the same LSN match; absent LSNs * return 0. */ uint64_t result = entry_term_locked(mp, lsn); pthread_mutex_unlock(&mp->lock); return result; } uint64_t persist_entry_term_locked(const persist_t* p, uint64_t lsn) { return entry_term_locked(p, lsn); } /* persist_reconcile_checkpoint_term — raft-ha-core-safety task 3.2 (Req * 3.1/5.1), Snapshot Boundary Reconcile. See persist.h for the full contract. * Reuses record_entry_term (same translation unit) to authoritatively * overwrite whatever term (if any) the local term index held at * checkpoint_lsn — this is a genuine reconcile-overwrite, not an idempotent * skip, because a pre-snapshot divergent entry may have left a stale local * term at this exact LSN. Only the checkpoint LSN's own term is reconciled: * post-install Raft semantics mean no AppendEntries prevLogIndex/prevLogTerm * check against this member ever references an LSN below the checkpoint * again (the checkpoint is the new log-start boundary), and the WAL bytes * below it were compacted away, so there is no data to reconstruct a full * per-LSN history from even if it were needed. * * bug-1475 Finding 2 (coordinator review, 2026-07-27/28): checkpoint_term==0 * on the wire is UNAMBIGUOUS today, not a legacy default to fall back to * WAL_LEGACY_TERM. ha_snapshot_do_send (ha.c) — the only sender — computes * this value via persist_entry_term(leader, checkpoint_lsn), whose public * contract already collapses "absent" and "present raw-0 legacy" into * DIFFERENT return values (0 vs WAL_LEGACY_TERM respectively — see * entry_term_locked). A wire value of exactly 0 can therefore only mean * "this sender's checkpoint LSN is genuinely, permanently absent" (the * expected, designed state for ANY real checkpoint — persist_write_ * checkpoint_marker never records a term for its own entry). Before this * fix, record_entry_term(p, checkpoint_lsn, 0) stored a RAW 0 — an EXPLICIT * write, not a growth-fill, so it never got the TERM_INDEX_UNSET sentinel — * and entry_term_locked's legacy-0 rule then read it back as * WAL_LEGACY_TERM(1) forever, permanently diverging from the sender's own * permanently-absent(0) value for the SAME LSN: a subsequent AppendEntries * whose prev_log_index==checkpoint_lsn (the entry immediately following it) * then mismatched logOk (0 != 1) forever — the identical wedge class * bug-1475 fixed for WAL-streaming catch-up, reachable instead via * HA_REPL_SEND_SNAPSHOT (dispatched whenever a standby has fallen too far * behind for incremental streaming — a live, common path). Recording the * SAME TERM_INDEX_UNSET sentinel here (instead of raw 0) keeps a genuinely- * absent checkpoint boundary reading absent on the receiver too, matching * the sender — while a genuinely non-zero checkpoint_term (a real historical * term, however it arrived on the wire) is still recorded verbatim and still * correctly overwrites any stale local term at this LSN, unchanged. */ void persist_reconcile_checkpoint_term(persist_t* p, uint64_t checkpoint_lsn, uint64_t checkpoint_term) { if (!p || checkpoint_lsn == 0) return; uint64_t stored_term = (checkpoint_term == 0) ? TERM_INDEX_UNSET : checkpoint_term; pthread_mutex_lock(&p->lock); record_entry_term(p, checkpoint_lsn, stored_term); pthread_mutex_unlock(&p->lock); } /* ------------------------------------------------------------------ */ /* Public API */ /* ------------------------------------------------------------------ */ void persist_set_workdir(persist_t* p, const char* workdir, const char* name) { char effective[4096]; snprintf(effective, sizeof(effective), "%s/%s", workdir, name); snprintf(p->wal_path, sizeof(p->wal_path), "%s/data/wal", effective); snprintf(p->snap_path, sizeof(p->snap_path), "%s/data/snap", effective); } int persist_init(persist_t* p, persist_config_t* cfg, cluster_t* c) { p->cluster = c; p->config = cfg ? *cfg : persist_config_defaults(); p->wal_fd = -1; p->last_replicated_crc = WAL_CRC_NONE; p->last_crc = WAL_CRC_NONE; /* LSN 0 is RESERVED as the "none / empty" sentinel — it is NEVER assigned * to a real WAL entry. A fresh store therefore starts current_lsn at 1, so * the first-ever appended entry (e.g. the promotion WAL_NOOP) is lsn 1 and * lsn 0 unambiguously means "nothing here": * - committed_lsn == 0 → nothing committed (the lsn-1 NOOP is committable * 0 → 1, vs. the old lsn-0 entry which could never advance committed_lsn); * - writable_lsn == 0 → own-term gate off (the gate armed at lsn 1 is * effective, vs. lsn 0 which read as "gate off" on a cold-start leader); * - standby_lsn == 0 → empty standby (the streamer's `hdr.lsn > 0` * filter streams the lsn-1 entry, closing the cold-start WAL hole that * dropped lsn 0 on `0 > 0`). * persist_recover() overwrites current_lsn from the durable snapshot/WAL tip * (load_snapshot / replay_wal), so this fresh default only applies before any * recovery — i.e. a brand-new store with no snapshot and no WAL. */ p->current_lsn = 1; /* Enforce minimum value for max_wal_records to prevent runaway checkpointing */ if (p->config.max_wal_records < 100) { ERROR("persist: max_wal_records=%u too low (minimum 100) — setting to 100", p->config.max_wal_records); p->config.max_wal_records = 100; } /* Enforce minimum value for wal_segment_bytes to prevent a rotation storm * (must comfortably fit at least one max-payload entry: header + up to * MAX_WAL_PAYLOAD_BYTES). */ if (p->config.wal_segment_bytes < 4096) { ERROR("persist: wal_segment_bytes=%u too low (minimum 4096) — setting to 4096", p->config.wal_segment_bytes); p->config.wal_segment_bytes = 4096; } /* Initialise atomic checkpoint-trigger counters */ atomic_store(&p->wal_record_count, 0); atomic_store(&p->checkpoint_in_progress, 0); /* Durable watermark starts at 0 ("nothing fdatasync'd"): mirrors the * LSN-0-reserved committed_lsn convention. It advances only on the * flusher thread after a successful fdatasync (ha-commit-durability 2.1). */ atomic_store(&p->durable_lsn, 0); /* Recovery-integrity verdict defaults to CLEAN (ha-commit-durability 3.1): * a fresh store has no corruption. persist_recover overwrites this with the * replay outcome (RECOVER_TRUNCATED only on a structural log corruption). */ p->recovery_verdict = RECOVER_CLEAN; /* Sticky "unrecoverable, loudly fenced" flag starts clear (Req 2.4): a fresh * store is promotable. Set by persist_recover_loud_fence_if_unrecoverable * on an unrecoverable boot; cleared on catch-up (never self-exits). */ atomic_store(&p->recovery_unrecoverable, false); /* Post-replay apply-correctness verdict defaults to CLEAN (raft-apply- * decoupling task 5.1, Req 8.2): a fresh store has nothing to diverge * from. persist_recover overwrites both after every replay. */ p->apply_verdict = APPLY_CLEAN; atomic_store(&p->last_applied_lsn, 0); /* Live Apply Resource Fence starts clear (job-array-single-wal-record * task 4.2, Req 9.4): a fresh store has nothing stuck failing to apply. * Set by persist_mark_apply_resource_fenced on a resource-exhaustion * apply failure; cleared by persist_clear_apply_resource_fence the * instant a retry of the same LSN succeeds -- see persist_t's field doc * comment for why this is deliberately separate from * recovery_unrecoverable just above. */ atomic_store(&p->apply_resource_fenced_lsn, 0); /* bug-1416: defaults to true (nothing to gate) so every fixture that * calls persist_init() but never persist_recover() -- most of this * spec's own test suite -- sees identical, ungated apply-pump/live- * replication behavior to before this flag existed. Only persist_ * recover() ever sets this false, for its own single-threaded * replay+verify duration (see the field's doc comment in persist.h). */ atomic_store(&p->recovery_complete, true); /* Durable-advance wake hook starts unregistered (ha-commit-durability 2.2): * the commit engine registers it in commit_engine_init. Until then every * flusher durable advance is a no-op wake. */ p->on_durable_advance = NULL; p->durable_cb_ctx = NULL; /* Truncation wake hook starts unregistered (raft-apply-decoupling task * 1.4): the commit engine registers it in commit_engine_init. Until * then every truncation is a no-op wake. */ p->on_truncate = NULL; p->truncate_cb_ctx = NULL; /* Per-LSN term index starts empty (ha-term-safety 1.3, R1.5): no entries * recorded yet. Grown on demand by record_entry_term during replay/append; * freed in persist_fini. */ p->term_index = NULL; p->term_index_count = 0; p->term_index_cap = 0; /* Pending command-apply hand-off list starts empty (raft-apply- * decoupling task 3.2): no committed-outcome continuations have * registered yet. Freed in persist_fini. */ p->pending_applies = NULL; pthread_mutex_init(&p->pending_applies_lock, NULL); /* Ordered apply pump starts idle (raft-apply-decoupling task 6.1): no * walk scheduled or running yet. Freed/destroyed in persist_fini. */ pthread_mutex_init(&p->apply_pump_lock, NULL); atomic_store(&p->apply_pump_scheduled, 0); /* Apply-pump segment-lookup cache starts invalid (raft-apply-decoupling * task 7.4 perf-fix): no segment/fd cached yet. -1 is always a safe * fallback trigger for apply_pump_read_entry's fast path (see persist.h * doc comment). */ p->apply_pump_cache_fd = -1; p->apply_pump_cache_seg_lsn = 0; p->apply_pump_cache_off = 0; p->apply_pump_cache_next_lsn = 0; /* Create WAL and snapshot directories */ if (mkdir_p(p->wal_path, 0700) != 0) return -errno; if (mkdir_p(p->snap_path, 0700) != 0) return -errno; /* Reject NFS mounts — fsync durability is not guaranteed over NFS */ struct statfs sfs; if (statfs(p->wal_path, &sfs) == 0 && (unsigned long)sfs.f_type == (unsigned long)NFS_SUPER_MAGIC) { ERROR("persist: WAL path %s is on NFS — unsafe for fsync durability", p->wal_path); return -EREMOTE; } pthread_mutex_init(&p->lock, NULL); pthread_mutex_init(&p->checkpoint_mutex, NULL); pthread_cond_init(&p->checkpoint_done, NULL); pthread_mutex_init(&p->fsync_mutex, NULL); pthread_cond_init(&p->fsync_cond, NULL); repl_init(&p->repl); /* P2-4: fan-out replicator lifecycle */ WARN("WAL/snapshot stored unencrypted in %s — " "ensure filesystem is encrypted if job payloads contain sensitive data", p->wal_path); return 0; } int persist_start(persist_t* p) { ctrl_metrics_init(); /* Register AFTER_EVENT persistence hooks if we have a cluster back-pointer */ if (p->cluster != NULL) { xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_JOB, (hook_fn_t)persist_job_hook_fun, p); xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_QUEUE, (hook_fn_t)persist_queue_hook_fun, p); xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_WORKER, (hook_fn_t)persist_worker_hook_fun, p); } atomic_store_explicit(&p->fsync_stop, 0, memory_order_release); int rc = pthread_create(&p->fsync_thread, NULL, fsync_worker, p); if (rc == 0) atomic_store_explicit(&p->fsync_thread_running, 1, memory_order_release); return rc; } void persist_stop(persist_t* p) { if (atomic_load_explicit(&p->fsync_thread_running, memory_order_acquire)) { /* Set the stop flag and wake the timer wait under fsync_mutex so the * worker cannot miss the signal between its flag check and the wait. */ pthread_mutex_lock(&p->fsync_mutex); atomic_store_explicit(&p->fsync_stop, 1, memory_order_release); pthread_cond_signal(&p->fsync_cond); pthread_mutex_unlock(&p->fsync_mutex); pthread_join(p->fsync_thread, NULL); atomic_store_explicit(&p->fsync_thread_running, 0, memory_order_release); } pthread_mutex_lock(&p->lock); if (p->wal_fd >= 0) { /* Final synchronous flush-or-exit on stop: fdatasync everything * appended so far and advance durable_lsn to the synced tip so a * post-stop read reflects the durable state (the flusher thread is * already joined and will not advance it again). * * FAIL-LOUD GATE (Req 1.1/1.2): advance the watermark ONLY when the * final sync SUCCEEDS — on EIO/ENOSPC the bytes are not on stable * storage, so durable_lsn must stay where it is and the failure is * surfaced loudly. */ int rc = fdatasync(p->wal_fd); if (rc == 0 && !flusher_fsync_forced_fail(p)) { uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); if (tip > dur) atomic_store_explicit(&p->durable_lsn, tip, memory_order_release); } else { int e = (rc != 0) ? errno : EIO; ERROR("persist: final fdatasync on stop FAILED (errno=%d %s) — " "durable_lsn left unchanged; appended-but-unsynced entries " "are NOT durable", e, strerror(e)); } close(p->wal_fd); p->wal_fd = -1; } pthread_mutex_unlock(&p->lock); } void persist_fini(persist_t* p) { repl_fini(&p->repl); /* P2-4: fan-out replicator lifecycle */ pthread_mutex_destroy(&p->lock); pthread_mutex_destroy(&p->checkpoint_mutex); pthread_cond_destroy(&p->checkpoint_done); pthread_mutex_destroy(&p->fsync_mutex); pthread_cond_destroy(&p->fsync_cond); /* Free the in-memory per-LSN term index (ha-term-safety 1.3). */ free(p->term_index); p->term_index = NULL; p->term_index_count = 0; p->term_index_cap = 0; /* Free any pending command-apply hand-off entries never drained by a * (not-yet-built) task 6 apply pump (raft-apply-decoupling task 3.2). * Each node's `ctx` is opaque to persist.c (owned by the registering * caller, e.g. job.c's jobsubmit_async_ctx_t) — freeing the node here * does NOT free ctx; a caller shutting down with entries still pending * is responsible for its own ctx lifetime (test harnesses that tear * down mid-flight must account for this). */ pending_command_apply_t* pn = (pending_command_apply_t*)p->pending_applies; while (pn != NULL) { pending_command_apply_t* next = pn->next; free(pn); pn = next; } p->pending_applies = NULL; pthread_mutex_destroy(&p->pending_applies_lock); /* raft-apply-decoupling task 6.1: no walk can still be in flight by the * time persist_fini runs (the caller owns quiescing any vtask pool * first, same precondition every other persist_t resource here already * assumes). */ pthread_mutex_destroy(&p->apply_pump_lock); /* raft-apply-decoupling task 7.4 perf-fix: close any cached segment fd * left open by apply_pump_read_entry. No walk can be in flight here * (same precondition as apply_pump_lock's destroy just above), so no * lock is needed. */ if (p->apply_pump_cache_fd >= 0) { close(p->apply_pump_cache_fd); p->apply_pump_cache_fd = -1; } } /* Forward declaration — defined after checkpoint internals */ static void schedule_checkpoint(persist_t* persist); int persist_wal_async(persist_t* p, uint32_t type, const void* payload, uint32_t len) { /* WAL_CHECKPOINT is a local segment marker, never written via this path */ if (type == WAL_CHECKPOINT) return -EINVAL; pthread_mutex_lock(&p->lock); /* Open first segment lazily */ if (p->wal_fd < 0) { if (open_new_segment(p) != 0) { pthread_mutex_unlock(&p->lock); return -EIO; } } /* Build entry header */ wal_entry_hdr_t hdr; memset(&hdr, 0, sizeof(hdr)); hdr.term = p->term; hdr.lsn = p->current_lsn++; hdr.timestamp = now_us(); hdr.type = type; hdr.payload_len = len; hdr.prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc; /* CRC32 over WAL_HDR_CRC_SIZE bytes of header (excludes crc + prev_crc * fields themselves) then over the payload */ uint32_t c = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); if (payload && len > 0) c = crc32(c, (const Bytef*)payload, len); hdr.crc = c; /* Write header */ ssize_t hw = pwrite(p->wal_fd, &hdr, sizeof(hdr), (off_t)p->seg_offset); if (hw != (ssize_t)sizeof(hdr)) { pthread_mutex_unlock(&p->lock); return -EIO; } /* Write payload (if any) */ if (payload && len > 0) { ssize_t pw = pwrite(p->wal_fd, payload, len, (off_t)(p->seg_offset + sizeof(hdr))); if (pw != (ssize_t)len) { pthread_mutex_unlock(&p->lock); return -EIO; } } uint32_t entry_size = (uint32_t)(sizeof(hdr) + len); p->seg_offset += entry_size; p->total_wal_bytes += entry_size; p->last_crc = hdr.crc; /* Record this entry's term at its LSN for recovery read-back (ha-term-safety * 1.3, R1.5). Under p->lock; stamps the same hdr.term written to disk. */ record_entry_term(p, hdr.lsn, hdr.term); /* Replicate to standbys — never replicate WAL_CHECKPOINT entries. * last_replicated_crc is distinct from last_crc: * last_crc includes checkpoint markers; last_replicated_crc skips them. * This maintains a continuous CRC chain for standbys (design doc Part 2.3). * Note: WAL_CHECKPOINT is blocked by the assert at function entry, so * every entry that reaches here is a replicable entry. */ p->last_replicated_crc = hdr.crc; /* ha-term-safety task 3.4: source the AppendEntries prev_log_term (term of * the entry at hdr.lsn-1) under p->lock via the lock-free accessor, so the * live broadcast carries the leader's term + prev tuple and the standby * routes it through the library (raft_on_append_entries_request). 0 when * hdr.lsn == 1 (no predecessor). */ uint64_t prev_log_term = entry_term_locked(p, hdr.lsn - 1); repl_broadcast(&p->repl, &hdr, payload, len, prev_log_term); /* Rotate segment when full */ if (p->seg_offset >= p->config.wal_segment_bytes) { if (rotate_segment(p) != 0) { pthread_mutex_unlock(&p->lock); return -EIO; } } pthread_mutex_unlock(&p->lock); /* Signal the group-commit flusher (ha-commit-durability 2.1): current_lsn * advanced and the entry's bytes are pwritten, so durable_lsn < tip now * holds (the level predicate). The signal is sent AFTER releasing p->lock * and is level-triggered — if it races the flusher and is "lost", the * flusher re-checks durable_lsn < current_lsn-1 under p->lock and flushes * anyway (see fsync_worker). We do NOT fdatasync inline on this path; the * flusher batches the sync across all pending appends. */ pthread_mutex_lock(&p->fsync_mutex); pthread_cond_signal(&p->fsync_cond); pthread_mutex_unlock(&p->fsync_mutex); /* WAL record-count checkpoint trigger: schedule an async checkpoint once * the record count reaches the configured threshold. The fetch_add is * outside the lock — atomic_fetch_add is sufficient for the counter; the * checkpoint itself is guarded by its own CAS inside schedule_checkpoint. */ if (atomic_fetch_add(&p->wal_record_count, 1) + 1 >= p->config.max_wal_records) { schedule_checkpoint(p); } return 0; } /* Lock-held core of persist_truncate_wal_above (defined below): assumes the * caller already holds p->lock and never touches the lock itself. The * replicated-write conflict guard reuses it to repair a divergent entry while * still under p->lock (the follower-AE-conflict truncate contract). */ static int truncate_wal_above_locked(persist_t* p, uint64_t cutoff_lsn); /* read_entry_meta_locked: scan the on-disk WAL for the entry at `lsn` and * return its (payload_len, payload-only CRC, raw stored term). The * replicated-write conflict guard uses this to compare a re-streamed entry * against the copy already on disk WITHOUT trusting the LSN alone. pcrc is * computed as crc32(0, payload, len) — the SAME payload-only fingerprint * walcompare/waldump compare across members (NOT the chained header CRC); an * empty (PLEN=0) entry yields len=0, pcrc=0. Reads through the page cache so * a written-but-not-yet-fdatasync'd entry is still visible. Caller MUST hold * p->lock. Returns 0 if the entry is found, -1 if absent or unreadable. */ static int read_entry_meta_locked(persist_t* p, uint64_t lsn, uint32_t* out_len, uint32_t* out_pcrc, uint64_t* out_term) { if (!p || lsn == 0) return -1; DIR* d = opendir(p->wal_path); if (!d) return -1; uint64_t seg_lsns[4096]; int nseg = 0; struct dirent* e; while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) { if (strncmp(e->d_name, "seg-", 4) != 0) continue; uint64_t sl = 0; if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &sl) != 1) continue; seg_lsns[nseg++] = sl; } closedir(d); if (nseg == 0) return -1; /* Ascending sort by start LSN (segment count is tiny). */ for (int i = 1; i < nseg; i++) { uint64_t key = seg_lsns[i]; int j = i - 1; while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; } seg_lsns[j + 1] = key; } int rc = -1; for (int si = 0; si < nseg && rc != 0; si++) { /* Segments are LSN-ordered; one whose start LSN already exceeds the * target cannot contain it (nor can any later one). */ if (seg_lsns[si] > lsn) break; char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[si]); int fd = open(seg_path, O_RDONLY); if (fd < 0) continue; uint32_t off = 0; for (;;) { wal_entry_hdr_t hdr; ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)off); if (hn < (ssize_t)sizeof(hdr)) break; /* short read / EOF / pre-allocated zeros */ if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break; if (hdr.lsn == lsn) { uint32_t pc = 0; if (hdr.payload_len > 0) { uint8_t* pl = malloc(hdr.payload_len); if (!pl) break; /* leave rc == -1: unreadable → conservative */ ssize_t pn = pread(fd, pl, hdr.payload_len, (off_t)(off + sizeof(hdr))); if (pn == (ssize_t)hdr.payload_len) pc = crc32(0L, (const Bytef*)pl, hdr.payload_len); free(pl); if (pn != (ssize_t)hdr.payload_len) break; /* unreadable body → conservative */ } if (out_len) *out_len = hdr.payload_len; if (out_pcrc) *out_pcrc = pc; if (out_term) *out_term = hdr.term; rc = 0; break; } off += (uint32_t)(sizeof(hdr) + hdr.payload_len); } close(fd); } return rc; } int persist_wal_write_replicated(persist_t* p, uint64_t lsn, uint32_t type, const void* payload, uint32_t len) { if (!p) return -EINVAL; /* WAL_CHECKPOINT is a local-only marker, never streamed to standbys. */ if (type == WAL_CHECKPOINT) return -EINVAL; /* Reject oversized payloads BEFORE touching the segment. */ if (len > MAX_WAL_PAYLOAD_BYTES) return -EMSGSIZE; pthread_mutex_lock(&p->lock); /* Tip-repair bookkeeping (set only on the TIP-conflict repair branch below): * the truncate-and-reappend repair transiently lowers committed_lsn to * lsn-1, so we snapshot the true committed boundary here and restore it * EXACTLY after the re-append re-establishes the entry at `lsn` (never let a * same-(term,lsn) payload repair regress the committed boundary — commit * monotonicity / State Machine Safety). */ int repaired_tip = 0; uint64_t repaired_committed_save = 0; /* Payload-aware conflict guard (raft-ha-core-safety task 2.2): this lsn is * already persisted (current_lsn advanced past it). The legacy guard was * (lsn)-only — it no-op'd ANY re-stream, including a corrective full-payload * write over a previously-stored empty (PLEN=0) body, which is exactly the * cross-term catch-up payload-loss signature (a committed LSN that carries * the right (term,lsn) but an empty payload). Compare (payload-crc, len) * against the copy already on disk and act on the difference: * (c) identical payload → idempotent no-op (keep). * (b) incoming EMPTY over a stored entry → never degrade (keep stored); * an empty body is never the durable form of a known entry. * (a) incoming NON-EMPTY whose payload differs from the stored copy at an * existing (term,lsn) → a genuine payload conflict whose handling * depends on WHERE the conflict sits relative to the stored tip: * · AT THE TIP (lsn == current_lsn - 1, nothing stored above) → * REPAIR: truncate just this one entry and re-append the * authoritative payload at the same (term,lsn). Because nothing * is stored above it, no distinct committed entry is destroyed, * and the committed boundary is restored verbatim afterwards. * · BELOW THE TIP (lsn < current_lsn - 1, committed entries stored * ABOVE the conflict) → MUST NOT truncate: dropping [lsn..tip] * would delete committed entries above the conflict AND roll * committed_lsn/durable_lsn backward — a State Machine Safety / * commit-monotonicity violation. Instead keep the stored entry * untouched, raise a loud non-promotable fence, and let catch-up * reconverge. The real source fix (the streamer must stop * shipping an empty body) is task 2.4; the guard's only job here * is to NOT make things worse (never drop a committed entry, * never regress a boundary) and to surface the corruption loudly. * An unreadable stored entry stays a conservative no-op — never truncate a * committed LSN we cannot read back. */ if (lsn + 1 <= p->current_lsn) { uint32_t inc_pcrc = (payload && len > 0) ? crc32(0L, (const Bytef*)payload, len) : 0; uint32_t st_len = 0; uint32_t st_pcrc = 0; uint64_t st_term = 0; int have = read_entry_meta_locked(p, lsn, &st_len, &st_pcrc, &st_term); int payload_matches = (have == 0 && st_len == len && st_pcrc == inc_pcrc); /* (c) identical payload, (b) empty incoming, or an unreadable stored * entry → leave the stored entry untouched. */ if (payload_matches || len == 0 || have != 0) { pthread_mutex_unlock(&p->lock); return 0; } /* (a) non-empty incoming whose payload differs at an existing (term,lsn) * — a genuine payload conflict. Branch on tip vs below-tip. */ uint64_t tip_lsn = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; if (lsn != tip_lsn) { /* BELOW-TIP conflict: committed entries are stored ABOVE `lsn`. * Truncating here would delete them and roll committed_lsn/ * durable_lsn backward (State Machine Safety / commit-monotonicity * violation — bug-class the round-1 review rejected). Do NOT * truncate, do NOT lower any boundary, do NOT degrade the stored * entry. Surface loudly and raise the sticky non-promotable fence * (reuse the recovery_unrecoverable mechanism: loud, stays ALIVE, * non-promotable, self-heals via persist_clear_recovery_unrecoverable * once catch-up reaches a committed-consistent state — never * self-exits, per the controller invariant). The source fix lives * in task 2.4 (the streamer must stop shipping an empty payload). */ ERROR("persist: persist_wal_write_replicated payload CONFLICT BELOW " "TIP at lsn=%" PRIu64 " (stored_term=%" PRIu64 " stored_len=%u stored_pcrc=%08x vs incoming_len=%u " "incoming_pcrc=%08x); current_lsn=%" PRIu64 " committed_lsn=%" PRIu64 " — refusing to truncate committed entries above the " "conflict (would regress the committed boundary). Keeping the " "stored entry, raising the non-promotable fence and staying " "ALIVE; awaiting a correct re-stream (source fix: task 2.4)", lsn, st_term, st_len, st_pcrc, len, inc_pcrc, p->current_lsn, atomic_load_explicit(&p->committed_lsn, memory_order_acquire)); atomic_store_explicit(&p->recovery_unrecoverable, true, memory_order_release); pthread_mutex_unlock(&p->lock); return 0; /* no-op the write: keep stored, never degrade/regress */ } /* TIP conflict: `lsn` is the highest stored entry, so nothing is stored * above it — truncate-and-reappend replaces EXACTLY this one divergent * entry. truncate_wal_above_locked(lsn-1) drops it and sets * current_lsn = lsn, so control falls through to the append below and * re-persists the authoritative entry at this exact LSN. Snapshot the * committed boundary first: the truncate transiently rewrites * committed_lsn to lsn-1, and we restore it verbatim after the * re-append so a same-(term,lsn) payload repair never regresses (nor * spuriously advances) the committed boundary. */ repaired_committed_save = atomic_load_explicit(&p->committed_lsn, memory_order_acquire); int trc = truncate_wal_above_locked(p, lsn - 1); if (trc != 0) { ERROR("persist: persist_wal_write_replicated repair-truncate FAILED " "for lsn=%" PRIu64 " (rc=%d) — divergent entry NOT repaired", lsn, trc); pthread_mutex_unlock(&p->lock); return trc; } repaired_tip = 1; /* fall through to the contiguity guard + append (lsn == current_lsn). */ } /* Contiguity guard (bug-850): REFUSE a forward GAP. current_lsn is the * next-expected LSN, so a replicated entry with lsn > current_lsn would * leave [current_lsn .. lsn-1] missing AND advance current_lsn past that * hole — after which the idempotency guard above silently DROPS the later * in-order catch-up of the prefix (the failover prefix-loss mechanism). * Append only the contiguous next entry (lsn == current_lsn); reject a gap * without writing or acking so the ACK-driven catch-up continuation * re-streams from current_lsn and the log fills in order. This is Raft * log-matching: append entry N only when the log already holds N-1. A * behind standby that keeps receiving live broadcasts simply drops the * out-of-order ones until catch-up backfills the gap (no loss). -EAGAIN is * the "retry via catch-up" signal the apply path treats as a quiet skip. */ if (lsn > p->current_lsn) { pthread_mutex_unlock(&p->lock); return -EAGAIN; } /* Open first segment lazily. open_new_segment names the file from * current_lsn — for a fresh standby receiving lsn 1 first, current_lsn is * 1 (reserve-lsn-0), so the segment is seg-...001.wal. */ if (p->wal_fd < 0) { if (open_new_segment(p) != 0) { pthread_mutex_unlock(&p->lock); return -EIO; } } /* Build entry header. Unlike persist_wal_async, lsn is GIVEN (not * current_lsn++). term isn't streamed, so use our best-effort p->term. * The CRC chain is SELF-computed (the streamed entry carries no crc): * recovery validates an intra-segment chain (prev_crc == last_crc), so the * standby's own writes must chain to each other — cross-node CRC equality * is neither required nor possible. */ wal_entry_hdr_t hdr; memset(&hdr, 0, sizeof(hdr)); hdr.term = p->term; hdr.lsn = lsn; hdr.timestamp = now_us(); hdr.type = type; hdr.payload_len = len; hdr.prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc; uint32_t c = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); if (payload && len > 0) c = crc32(c, (const Bytef*)payload, len); hdr.crc = c; /* Write header */ ssize_t hw = pwrite(p->wal_fd, &hdr, sizeof(hdr), (off_t)p->seg_offset); if (hw != (ssize_t)sizeof(hdr)) { pthread_mutex_unlock(&p->lock); return -EIO; } /* Write payload (if any) */ if (payload && len > 0) { ssize_t pw = pwrite(p->wal_fd, payload, len, (off_t)(p->seg_offset + sizeof(hdr))); if (pw != (ssize_t)len) { pthread_mutex_unlock(&p->lock); return -EIO; } } uint32_t entry_size = (uint32_t)(sizeof(hdr) + len); p->seg_offset += entry_size; p->total_wal_bytes += entry_size; p->last_crc = hdr.crc; /* Record this entry's term at its LSN for recovery read-back (ha-term-safety * 1.3, R1.5). Stamps the same hdr.term written to disk. NOTE: the streamed * envelope does not yet carry the leader's term, so this records the local * best-effort p->term (the design's known hazard closed by task 2.x, which * carries the leader's term on the envelope; this task only provides the * read-back seam over whatever term was persisted). */ record_entry_term(p, hdr.lsn, hdr.term); /* Advance current_lsn to one past the given lsn (max-idempotent — mirrors * apply_wal_entry's monotonic advance). */ if (lsn + 1 > p->current_lsn) p->current_lsn = lsn + 1; /* Durability: fsync the segment BEFORE returning so the caller's ACK * implies the entry is on stable storage (persist-before-ACK). * persist_wal_async defers fsync to the background thread / persist_flush; * here we sync inline because the entry feeds the primary's commit * majority and must survive a standby restart. fdatasync is sufficient * (entry bytes + segment metadata are pre-allocated by open_new_segment). * * Fail-loud rc-gate (ha-commit-durability Req 1.3, symmetric with the * append-path gate at task 2.1): the ACK is sent by the caller ONLY when * this returns success, and the ACK claims the entry is durable. So if * the fdatasync FAILS (EIO/ENOSPC, or the forced-fail test seam), we must * NOT return success — we log loudly and return -EIO so the caller's * existing error path (ha_apply_single_entry) does NOT apply or ACK. */ int fsrc = fdatasync(p->wal_fd); #ifdef XCE_TESTING atomic_fetch_add_explicit(&p->test_replicated_fdatasync_count, 1, memory_order_relaxed); #endif if (fsrc != 0 || flusher_fsync_forced_fail(p)) { int e = (fsrc != 0) ? errno : EIO; ERROR("persist: persist_wal_write_replicated fdatasync FAILED for " "lsn=%" PRIu64 " (errno=%d %s) — entry NOT durable; returning " "error so the standby does NOT ACK", lsn, e, strerror(e)); pthread_mutex_unlock(&p->lock); return -EIO; } /* bug-828 secondary: the entry is now fdatasync'd durable, so advance * durable_lsn to the synced tip immediately (mirror persist_flush). The * 10ms cadence flusher would otherwise advance it lazily, leaving this * standby's ADVERTISED durable position (persist_durable_floor → * RequestVoteReply.peerLastLogLsn) lagging its own synced tip during the * vote window. A standby that has durably persisted the committed prefix * must advertise it at once so a clean failover's safe-commit-from-votes * computation sees the true majority-held LSN. Monotonic max; preserves * durable_lsn <= current_lsn - 1 (tip read under this same p->lock, after * the current_lsn advance above). Reached ONLY past the fail-loud rc-gate * — never advances on an fdatasync error. */ { uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); if (tip > dur) atomic_store_explicit(&p->durable_lsn, tip, memory_order_release); } /* Tip-repair committed-boundary restore (raft-ha-core-safety task 2.2): a * TIP-conflict repair above truncated [lsn..tip] (rewriting committed_lsn to * lsn-1) and has now re-appended + fdatasync'd the authoritative entry at the * SAME (term,lsn). Restore the committed boundary VERBATIM so the repair is * commit-boundary-neutral: a same-(term,lsn) payload correction must never * regress committed_lsn (State Machine Safety / commit monotonicity) nor * spuriously advance it (truncate would otherwise have raised it to lsn-1). * Reached only past the fail-loud fdatasync gate, with current_lsn re-advanced * to lsn+1 and durable_lsn at the synced tip, so the invariant * committed_lsn <= min(current_lsn - 1, durable_lsn) holds (saved value was * itself <= the pre-repair tip == lsn). The whole repair is serialized under * p->lock, so no external observer ever sees the transient lowered value. */ if (repaired_tip) { atomic_store_explicit(&p->committed_lsn, repaired_committed_save, memory_order_release); } /* Rotate segment when full (same threshold as the primary append). */ if (p->seg_offset >= p->config.wal_segment_bytes) { if (rotate_segment(p) != 0) { pthread_mutex_unlock(&p->lock); return -EIO; } } /* NOTE: no repl_broadcast — standbys never re-replicate. */ pthread_mutex_unlock(&p->lock); return 0; } int persist_flush(persist_t* p) { pthread_mutex_lock(&p->lock); int rc = (p->wal_fd >= 0) ? fdatasync(p->wal_fd) : 0; /* persist_flush is a synchronous durability point (tests, clean shutdown): * it fdatasync'd everything appended so far, so advance durable_lsn to the * synced tip too. Without this, an explicit-flush-only path (slow flusher * cadence) would leave durable_lsn lagging even though the bytes are on * disk. Monotonic max; preserves durable_lsn <= current_lsn - 1 (tip read * under the same p->lock). Gated on rc == 0 (fail-loud, Req 1.1/1.2): a * failed flush must not advance the watermark. */ bool advanced = false; if (rc == 0 && !flusher_fsync_forced_fail(p)) { uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); if (tip > dur) { atomic_store_explicit(&p->durable_lsn, tip, memory_order_release); advanced = true; } } else if (rc != 0 || flusher_fsync_forced_fail(p)) { int e = (rc != 0) ? errno : EIO; ERROR("persist: persist_flush fdatasync FAILED (errno=%d %s) — " "durable_lsn not advanced; flushed bytes are NOT durable", e, strerror(e)); } pthread_mutex_unlock(&p->lock); /* bug-1564: fire the same durable-advance wake fsync_worker fires on its * own successful flush (identical gate: only when durable_lsn actually * advanced; identical lock discipline: strictly after releasing p->lock, * so the callback -> ce->lock ordering has no p->lock held underneath * it, matching fsync_worker's own documented cycle-free reasoning * above). Latent gap unmasked by bug-1563: this synchronous flush path * (tests, clean shutdown) used to advance durable_lsn silently -- any * WAIT-blocked commit caller relying on this callback to know a fresh * durable-majority exists would only ever notice via the ASYNC * fsync_worker's own next successful flush, not this call. Before * bug-1563's fix, fsync_worker's un-throttled retry loop happened to * paper over the gap (it re-checked and re-fired within microseconds of * any state change); bug-1563's backoff correctly removed that spin, * which is what surfaced this as a real, reproducible test timeout * (test_durable_majority_commit_end_to_end: commit_submit(WAIT) timing * out at -110/ETIMEDOUT, deterministic 3/3 direct runs) instead of a * pre-existing gap nothing had ever pushed hard enough to expose. */ if (advanced && p->on_durable_advance != NULL) { p->on_durable_advance(p->durable_cb_ctx); } return rc; } uint64_t persist_durable_lsn(persist_t* p) { return atomic_load_explicit(&p->durable_lsn, memory_order_acquire); } uint64_t persist_durable_floor(persist_t* p) { /* The durable freshness floor (durable last-log index) IS the durable * watermark: persist_recover re-derives it from the recovered WAL tip * (ha-commit-durability task 3.4), and the flusher advances it on every * successful fdatasync — so a thin wrapper avoids a redundant field. NULL * guard keeps the accessor safe for the election consumers (task 4.2). */ return p ? atomic_load_explicit(&p->durable_lsn, memory_order_acquire) : 0; } void persist_set_durable_cb(persist_t* p, void (*fn)(void* ctx), void* ctx) { if (!p) return; /* Set ctx before fn so the flusher (which checks fn != NULL) never sees a * live callback with a stale/uninitialised ctx. Registration happens in * commit_engine_init, before the engine is reachable by any ACK path. */ p->durable_cb_ctx = ctx; p->on_durable_advance = fn; } /* raft-apply-decoupling task 1.4: registration counterpart of * persist_set_durable_cb -- see persist.h for the full contract. */ void persist_set_truncate_cb(persist_t* p, void (*fn)(void* ctx, uint64_t cutoff_lsn), void* ctx) { if (!p) return; p->truncate_cb_ctx = ctx; p->on_truncate = fn; } #ifdef XCE_TESTING void persist_test_set_force_fsync_fail(persist_t* p, int fail) { if (p) atomic_store_explicit(&p->test_force_fsync_fail, fail ? 1 : 0, memory_order_release); } void persist_test_set_force_job_encode_fail_at(persist_t* p, int index) { if (p) atomic_store_explicit(&p->test_force_job_encode_fail_at, index + 1, memory_order_relaxed); } unsigned persist_test_replicated_fdatasync_count(persist_t* p) { if (!p) return 0; return atomic_load_explicit(&p->test_replicated_fdatasync_count, memory_order_relaxed); } pthread_t persist_test_last_fdatasync_thread(persist_t* p) { /* If no flusher fdatasync has advanced durable_lsn yet, return the * caller's own id so an unprimed read never spuriously looks off-thread. * After a flush, return the recorded flusher thread id (Req 1.6). */ if (!p || atomic_load_explicit(&p->test_last_fdatasync_thread_set, memory_order_acquire) == 0) return pthread_self(); return p->test_last_fdatasync_thread; } /* Force a fresh WAL segment at the current LSN (test seam — see persist.h). * Takes p->lock for the same single-writer serialization the append path uses, * so the segment swap cannot race a concurrent write. */ int persist_test_force_new_segment(persist_t* p) { if (!p) return -EINVAL; pthread_mutex_lock(&p->lock); int rc = rotate_segment(p); pthread_mutex_unlock(&p->lock); return rc; } /* ------------------------------------------------------------------ * * Durability-verification test seam (ha-commit-durability Req 3.2/3.3)* * * * Re-read + CRC-validate the WAL segments from stable storage and * * return the highest fully-persisted LSN — independent of the * * in-memory current_lsn / committed_lsn watermarks. Mirrors the * * segment-scan + CRC-chain validation of replay_wal(), but is strictly * * read-only: it applies no entries, fires no hooks, and mutates no * * persist_t state, so it cannot change the externally observable * * acknowledgment semantics (Req 3.3). * * * * Stops at the first corruption / short read / pre-allocated zero * * region (exactly like recovery), so only entries genuinely on stable * * storage are counted. Returns the highest such LSN, or 0 when nothing * * is durable. * * ------------------------------------------------------------------ */ uint64_t persist_test_durable_lsn_on_disk(persist_t* p) { if (!p) return 0; DIR* d = opendir(p->wal_path); if (!d) return 0; /* no WAL dir → nothing durable */ /* Collect segment start LSNs (same bound as replay_wal). */ uint64_t seg_lsns[4096]; int nseg = 0; struct dirent* e; while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) { if (strncmp(e->d_name, "seg-", 4) != 0) continue; uint64_t seg_start = 0; if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start) != 1) continue; seg_lsns[nseg++] = seg_start; } closedir(d); if (nseg == 0) return 0; /* Insertion sort by start LSN (segment count is tiny). */ for (int i = 1; i < nseg; i++) { uint64_t key = seg_lsns[i]; int j = i - 1; while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; } seg_lsns[j + 1] = key; } uint64_t highest_durable = 0; uint32_t last_crc = WAL_CRC_NONE; for (int si = 0; si < nseg; si++) { char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[si]); int fd = open(seg_path, O_RDONLY); if (fd < 0) continue; uint32_t seg_off = 0; for (;;) { wal_entry_hdr_t hdr; ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)seg_off); if (hn == 0) break; /* clean end of segment */ if (hn < (ssize_t)sizeof(hdr)) break; /* truncated / pre-allocated zeros */ /* Pre-allocated zero region: no valid entry is all-zero. */ if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break; /* CRC chain validation — identical rule to replay_wal. */ if (seg_off == 0) { if (hdr.prev_crc != WAL_CRC_NONE && last_crc != WAL_CRC_NONE && hdr.prev_crc != last_crc) break; } else { if (hdr.prev_crc != last_crc) break; } /* Verify entry CRC over header + payload. */ uint32_t expected_crc = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); uint8_t* payload = NULL; if (hdr.payload_len > 0) { payload = malloc(hdr.payload_len); if (!payload) { close(fd); return highest_durable; } ssize_t pn = pread(fd, payload, hdr.payload_len, (off_t)(seg_off + sizeof(hdr))); if (pn != (ssize_t)hdr.payload_len) { free(payload); break; } expected_crc = crc32(expected_crc, (const Bytef*)payload, hdr.payload_len); } if (hdr.crc != expected_crc) { free(payload); break; } free(payload); /* Entry is intact and durable on disk. */ last_crc = hdr.crc; if (hdr.lsn > highest_durable) highest_durable = hdr.lsn; seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); } close(fd); } return highest_durable; } #endif /* XCE_TESTING */ /* ------------------------------------------------------------------ */ /* Checkpoint internals */ /* ------------------------------------------------------------------ */ /* Write a WAL_CHECKPOINT marker to the current segment. * Never calls repl_broadcast() — checkpoint markers are local only. */ static int persist_write_checkpoint_marker(persist_t* p) { pthread_mutex_lock(&p->lock); if (p->wal_fd < 0) { if (open_new_segment(p) != 0) { pthread_mutex_unlock(&p->lock); return -EIO; } } wal_entry_hdr_t hdr; memset(&hdr, 0, sizeof(hdr)); hdr.term = p->term; hdr.lsn = p->current_lsn++; hdr.timestamp = now_us(); hdr.type = WAL_CHECKPOINT; hdr.payload_len = 0; hdr.prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc; uint32_t c = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); hdr.crc = c; ssize_t hw = pwrite(p->wal_fd, &hdr, sizeof(hdr), (off_t)p->seg_offset); if (hw != (ssize_t)sizeof(hdr)) { pthread_mutex_unlock(&p->lock); return -EIO; } fdatasync(p->wal_fd); p->seg_offset += (uint32_t)sizeof(hdr); p->total_wal_bytes += sizeof(hdr); p->last_crc = hdr.crc; /* NOTE: last_replicated_crc is NOT updated — WAL_CHECKPOINT is never replicated */ /* NOTE: the term index is NOT updated for WAL_CHECKPOINT — replay_wal SKIPS * checkpoint markers (local bookkeeping, not a logical Raft entry), so the * recovered index never holds a checkpoint LSN; the live index must match * (ha-term-safety 1.3). persist_entry_term is for (term,lsn) reconcile, which * never compares against a checkpoint marker. */ pthread_mutex_unlock(&p->lock); return 0; } /* Entity-bearing checkpoint payload (ha-entity-snapshots task 2.2/3.2, Req * 1.1-1.4). Physical layout appended after the existing legacy header * (design.md "Checkpoint File Layout"): * [uint32_t format_version] * [uint32_t job_count] * [job_count * { uint32_t payload_len; uint8_t capnp_Job_bytes[payload_len] }] * [uint32_t queue_count] * [queue_count * { uint32_t name_len; char name[name_len]; uint32_t status }] * * config-upgrade-snapshot task 1 (Req 1.9, 5.7): bumped 1->2 to reserve the * version for the HostAdminState/QueueAdminState capnp migration -- the * queue-frame layout above (name_len/name/status raw bytes) is the PRIOR * layout this version now hard-rejects; the byte-layout migration itself * (new host section, queue frames re-encoded as capnp) lands in tasks * 2.1/2.2. A binary built against version 2 must never misinterpret a * checkpoint written under version 1's raw queue-frame layout (Design * Invariant 4: fail loud, never silently reinterpret). */ #define SNAPSHOT_FORMAT_VERSION 2u typedef struct { uint8_t* buf; uint32_t len; } snap_job_frame_t; static void snap_free_job_frames(xlist_t* frames) { for (int i = 0; i < frames->count; i++) { snap_job_frame_t* fr = (snap_job_frame_t*)xlist_get(frames, i); if (fr) { free(fr->buf); free(fr); } } xlist_fini(frames); } /* Collect + encode every retained job into `frames`, one malloc'd * snap_job_frame_t per job, under a point-in-time-consistent walk * (design.md "Existing Architecture Analysis" / Requirement 1.2): the * jobmgr table's own RTTI_LOCK_READ is held for the ENTIRE walk (excludes * concurrent insert/delete), and each retained job's own RTTI_LOCK_READ is * ADDITIONALLY taken nested around just that job's encode call (excludes a * concurrent per-entity writer for exactly the duration of its encode -- * the table lock alone does not do this). No lock is held during the * disk write itself: this function only builds in-memory frames; the * caller writes them to disk after the table walk has fully released both * locks. * * Filters via job_should_retain (Requirement 1.1) so the checkpoint's * notion of "live" matches the existing GC retention policy. * * Returns 0 on success (frames populated; `frames` is always left * xlist_init'd -- even zero results -- so the caller can always * xlist_fini it) or a negative errno on the first encode failure, in which * case every frame collected so far has already been freed and `frames` * is empty (Requirement 1.3: a mid-walk failure must not leave partial * entity content behind). */ static int collect_job_frames(persist_t* p, xlist_t* frames) { cluster_t* c = p->cluster; int rc = 0; int retained_index = 0; xlist_init(frames, 64); /* No live cluster back-pointer (e.g. a bare unit-test persist_t) -- * zero jobs is not a failure. */ if (c == NULL || c->jobmgr == NULL) return 0; lock(c->jobmgr, RTTI_LOCK_READ); iterator_t it; int irc = iter_make(&it, c->jobmgr); if (irc < 0) { unlock(c->jobmgr); return 0; /* nothing to walk -- mirrors collect_jobs' empty-table handling */ } for (;;) { job_t* j = NULL; int nrc = iter_next(&it, (void**)&j); if (nrc < 0 || nrc == RTTI_ITER_NO_DATA) break; if (j == NULL) continue; if (!job_should_retain(j, &p->config)) continue; uint8_t* buf = NULL; size_t sz = 0; int enc_rc; lock(j, RTTI_LOCK_READ); #ifdef XCE_TESTING int forced = atomic_load_explicit(&p->test_force_job_encode_fail_at, memory_order_relaxed); if (forced > 0 && (forced - 1) == retained_index) enc_rc = -EIO; /* simulated mid-loop encode failure */ else enc_rc = capnp_encode_job(&j->job, &buf, &sz); #else enc_rc = capnp_encode_job(&j->job, &buf, &sz); #endif unlock(j); retained_index++; if (enc_rc != 0) { rc = enc_rc; break; } snap_job_frame_t* fr = (snap_job_frame_t*)malloc(sizeof(*fr)); if (fr == NULL) { free(buf); rc = -ENOMEM; break; } fr->buf = buf; fr->len = (uint32_t)sz; if (xlist_store(frames, fr) < 0) { free(buf); free(fr); rc = -ENOMEM; break; } } iter_destroy(&it); unlock(c->jobmgr); if (rc != 0) snap_free_job_frames(frames); /* Requirement 1.3: discard, not partial */ return rc; } /* Queue admin-status frame (config-upgrade-snapshot task 2.2, Req 1.9, 5.7). * Migrated from the prior raw {name_len, name, status} layout to a capnp * QueueAdminState payload -- identical {buf,len} shape to snap_host_frame_t * (both carry a capnp payload), same (name, status) field content as * before, restore_queue_state's contract (queue.c:208) unchanged; only the * wire encoding changes (design.md "Snapshot Entity/Config Payload"). */ typedef struct { uint8_t* buf; uint32_t len; } snap_queue_frame_t; static void snap_free_queue_frames(xlist_t* frames) { for (int i = 0; i < frames->count; i++) { snap_queue_frame_t* fr = (snap_queue_frame_t*)xlist_get(frames, i); if (fr) { free(fr->buf); free(fr); } } xlist_fini(frames); } /* Collect + encode every queue in `frames`, one malloc'd snap_queue_frame_t * per queue, under the same point-in-time-consistent walk discipline as * collect_job_frames above (design.md "Existing Architecture Analysis" / * Requirement 1.2): the qmgr table's own RTTI_LOCK_READ is held for the * ENTIRE walk, and each queue's own RTTI_LOCK_READ is ADDITIONALLY taken * nested around just that queue's (name, status) read and capnp encode -- * mirroring collect_host_frames' identical lock-then-encode-then-unlock * shape. No lock is held during the disk write itself. * * Unlike collect_job_frames, no retention predicate is applied: a queue's * mere presence in the live qmgr table already means config provisioned * it (design.md "write_snapshot (extended)" Responsibilities), so every * queue found is included. * * Returns 0 on success (frames populated; `frames` is always left * xlist_init'd, even for zero results, so the caller can always xlist_fini * it) or a negative errno on the first encode failure, in which case every * frame collected so far has already been freed and `frames` is empty * (Requirement 1.3: a mid-walk failure must not leave partial entity * content behind). */ static int collect_queue_frames(persist_t* p, xlist_t* frames) { cluster_t* c = p->cluster; int rc = 0; xlist_init(frames, 64); /* No live cluster back-pointer (e.g. a bare unit-test persist_t) -- * zero queues is not a failure. */ if (c == NULL || c->qmgr == NULL) return 0; lock(c->qmgr, RTTI_LOCK_READ); iterator_t it; int irc = iter_make(&it, c->qmgr); if (irc < 0) { unlock(c->qmgr); return 0; /* nothing to walk -- mirrors collect_queues' empty-table handling */ } for (;;) { queue_t* q = NULL; int nrc = iter_next(&it, (void**)&q); if (nrc < 0 || nrc == RTTI_ITER_NO_DATA) break; if (q == NULL) continue; capnp_queue_admin_state_t qs; uint8_t* buf = NULL; size_t sz = 0; int enc_rc; lock(q, RTTI_LOCK_READ); qs.name = q->queue.queue; qs.status = q->queue.status; enc_rc = capnp_encode_queue_admin_state(&qs, &buf, &sz); unlock(q); if (enc_rc != 0) { rc = enc_rc; break; } snap_queue_frame_t* fr = (snap_queue_frame_t*)malloc(sizeof(*fr)); if (fr == NULL) { free(buf); rc = -ENOMEM; break; } fr->buf = buf; fr->len = (uint32_t)sz; if (xlist_store(frames, fr) < 0) { free(buf); free(fr); rc = -ENOMEM; break; } } iter_destroy(&it); unlock(c->qmgr); if (rc != 0) snap_free_queue_frames(frames); /* Requirement 1.3: discard, not partial */ return rc; } /* Bits set by admin commands that must survive a checkpoint round-trip. * Mirrors worker.c's and lib/wobj.c's own local XCE_WORKER_ADMIN_MASK * definitions -- both already independently duplicate this exact bit list * (design.md Design Invariant 6 notes this pre-existing duplication); this * is a third, deliberate copy for the same reason (no shared header exists * for it today), not a new layering dependency on worker.c. */ #define PERSIST_WORKER_ADMIN_MASK \ (XCE_WORKER_LOCKEDU | XCE_WORKER_CLOSED | XCE_WORKER_HIDDEN \ | XCE_WORKER_CLOSED_BY_ADMIN) typedef struct { uint8_t* buf; uint32_t len; } snap_host_frame_t; static void snap_free_host_frames(xlist_t* frames) { for (int i = 0; i < frames->count; i++) { snap_host_frame_t* fr = (snap_host_frame_t*)xlist_get(frames, i); if (fr) { free(fr->buf); free(fr); } } xlist_fini(frames); } /* Collect + encode every host in c->workermgr into `frames`, one malloc'd * snap_host_frame_t (identical {buf,len} shape to snap_job_frame_t -- both * carry a capnp payload) per host, under the same point-in-time-consistent * walk discipline as * collect_job_frames/collect_queue_frames above (design.md "Existing * Architecture Analysis" / Requirement 1.2): the workermgr table's own * RTTI_LOCK_READ is held for the ENTIRE walk, and each worker's own * RTTI_LOCK_READ is ADDITIONALLY taken nested around just that host's * (name, status) read and encode. No lock is held during the disk write * itself. * * Like collect_queue_frames and UNLIKE collect_job_frames, no retention * predicate is applied: every host currently in the live workermgr table * gets exactly one frame (mirrors collect_queue_frames's own "mere presence * already means it's included" precedent above), so restore later sees a * frame for every host this member currently knows about, whether or not * any admin bit happens to be set on it right now. * * Each frame carries ONLY the host's admin-issued status bits * (w->host.status & PERSIST_WORKER_ADMIN_MASK) -- never the raw status * (which also carries dynamic/liveness bits like XCE_WORKER_UNAVAIL) and * never any other xce_worker_t field (attrs, resvals, dead_since, ...): * Requirement 1.7's exclusion, design.md Design Invariant 5 (never inherit * a live struct's dynamic fields just because reusing it is convenient -- * capnp_encode_worker/Worker is deliberately NOT used here). `name` is * read directly off w->host.name -- the same field key_find(c->workermgr, * name) looks up elsewhere in this codebase (join.c:91, worker.c:524,645,751, * lib/wobj.c:212), so a later restore's key_find(name) lookup matches * exactly what was collected here. * * Returns 0 on success (frames populated; `frames` is always left * xlist_init'd -- even zero results -- so the caller can always * xlist_fini it) or a negative errno on the first encode failure, in which * case every frame collected so far has already been freed and `frames` * is empty (Requirement 1.3: a mid-walk failure must not leave partial * entity content behind). */ static int collect_host_frames(persist_t* p, xlist_t* frames) { cluster_t* c = p->cluster; int rc = 0; xlist_init(frames, 64); /* No live cluster back-pointer (e.g. a bare unit-test persist_t) -- * zero hosts is not a failure. */ if (c == NULL || c->workermgr == NULL) return 0; lock(c->workermgr, RTTI_LOCK_READ); iterator_t it; int irc = iter_make(&it, c->workermgr); if (irc < 0) { unlock(c->workermgr); return 0; /* nothing to walk -- mirrors collect_job/queue_frames' empty-table handling */ } for (;;) { worker_t* w = NULL; int nrc = iter_next(&it, (void**)&w); if (nrc < 0 || nrc == RTTI_ITER_NO_DATA) break; if (w == NULL) continue; capnp_host_admin_state_t h; uint8_t* buf = NULL; size_t sz = 0; int enc_rc; lock(w, RTTI_LOCK_READ); h.name = w->host.name; h.status = w->host.status & PERSIST_WORKER_ADMIN_MASK; enc_rc = capnp_encode_host_admin_state(&h, &buf, &sz); unlock(w); if (enc_rc != 0) { rc = enc_rc; break; } snap_host_frame_t* fr = (snap_host_frame_t*)malloc(sizeof(*fr)); if (fr == NULL) { free(buf); rc = -ENOMEM; break; } fr->buf = buf; fr->len = (uint32_t)sz; if (xlist_store(frames, fr) < 0) { free(buf); free(fr); rc = -ENOMEM; break; } } iter_destroy(&it); unlock(c->workermgr); if (rc != 0) snap_free_host_frames(frames); /* Requirement 1.3: discard, not partial */ return rc; } /* Bound on the config bytes this file will embed in a checkpoint -- mirrors * config_repl.h's CONF_MAX_SIZE (1 MiB) exactly. Duplicated as its own local * constant rather than including config_repl.h, per the zero-config_repl.c- * dependency decision documented on read_active_config_bytes() below. */ #define PERSIST_CONFIG_BYTES_MAX_SIZE (1u * 1024u * 1024u) /* read_active_config_bytes (config-upgrade-snapshot task 2.3, Req 2.1, 2.3, * 2.4, 5.1): read the locally-held active configuration's raw bytes straight * off disk, for embedding in every checkpoint's trailing config-bytes * section. * * DELIBERATE DESIGN CHOICE -- read this before changing the call site below: * this function duplicates config_repl.c's `conf_read_current_filename` + * `conf_read_raw` path-resolution-and-read logic INLINE, rather than calling * those functions (config_repl.h is not included by this file at all). This * mirrors scan_latest_committed_config_ref()'s own precedent just above, * which independently decodes WAL_CONFIG_MARKER payloads inline "so persist * stays free of a config_repl layering dependency" instead of adding a * persist.c -> config_repl.c function-call edge -- and design.md's * "Snapshot Entity/Config Payload" + File Structure Plan sections direct the * SAME treatment here verbatim ("no new config_repl.c function is added... * this deliberately mirrors scan_latest_committed_config_ref's own existing * precedent of avoiding a persist.c -> config_repl.c function-call * dependency, rather than introducing the layering violation a new accessor * would create"). * * A future reader who spots ha.c's two `extern int conf_read_raw(...)` call * sites (config push to standbys) and wonders why this file doesn't do the * same: ha.c is the intentional CONSUMER of config-bytes sourcing * (design.md's Architecture boundary map: "config replication (config_repl.c) * owns config-bytes sourcing"), so a ha.c -> config_repl.c call is expected, * in-boundary integration. persist.c owns materialization/on-disk-format and * is deliberately kept free of any dependency on config_repl.c's domain -- * the same boundary this file already draws for the WAL-marker case above. * This is a considered choice, not an oversight: duplicating this ~30-line * read-with-validation routine once was judged cheaper than adding this * file's first cross-domain coupling edge into config_repl.c. * * Returns 0 with *out_bytes (caller must free()) / *out_len set on success. * Returns -ENOENT when no local config bytes exist yet: no cluster/myconf * back-pointer (e.g. a bare unit-test persist_t), no workdir/name configured, * no conf/current pointer file yet (this node has not received its first * config push), or the pointer names a versioned file that is not present. * This is a BENIGN, expected state (early boot / not-yet-provisioned), never * corruption -- callers must treat -ENOENT as "no config yet", not failure. * Any OTHER negative errno (-EINVAL malformed pointer/oversized file, -EIO * read failure, -ENOMEM, -EFBIG) indicates genuine on-disk corruption or a * real I/O failure and must propagate as a hard failure, never be silently * downgraded to an empty section. */ static int read_active_config_bytes(persist_t* p, uint8_t** out_bytes, size_t* out_len) { cluster_t* c = p->cluster; if (c == NULL || c->myconf == NULL || c->myconf->workdir == NULL || c->myconf->name == NULL) return -ENOENT; /* effective_dir = workdir/name -- must match conf_save_config's own * path construction (config_repl.c:781) exactly. */ char conf_dir[4096]; snprintf(conf_dir, sizeof(conf_dir), "%s/%s", c->myconf->workdir, c->myconf->name); /* Step 1: resolve the "current" versioned filename -- mirrors * conf_read_current_filename's validation exactly (non-empty, no path * separator, "cluster.bin." prefix + 12 lowercase hex chars) so a * corrupted pointer file is caught here just as loudly as it would be * there. */ char head_path[4160]; snprintf(head_path, sizeof(head_path), "%s/conf/current", conf_dir); FILE* hf = fopen(head_path, "r"); if (hf == NULL) return -ENOENT; /* no config pushed to this node yet */ struct stat hst; if (fstat(fileno(hf), &hst) == 0 && hst.st_size == 0) { fclose(hf); return -ENOENT; /* interrupted-write pointer file -- treat as absent */ } char fname[256]; if (fgets(fname, (int)sizeof(fname), hf) == NULL) { fclose(hf); return -EIO; } fclose(hf); fname[strcspn(fname, "\n")] = '\0'; if (strchr(fname, '/') != NULL) return -EINVAL; /* corrupted pointer file -- embedded path separator */ if (strncmp(fname, "cluster.bin.", 12) != 0) return -EINVAL; const char* hex = fname + 12; if (strlen(hex) != 12 || strspn(hex, "0123456789abcdef") != 12) return -EINVAL; /* Step 2: read the versioned config file's raw bytes. */ char cfg_path[4416]; snprintf(cfg_path, sizeof(cfg_path), "%s/conf/%s", conf_dir, fname); FILE* cf = fopen(cfg_path, "rb"); if (cf == NULL) return -ENOENT; /* pointer references a file that is not there -- * treat as not-yet-ready rather than corruption */ if (fseek(cf, 0, SEEK_END) != 0) { fclose(cf); return -EIO; } long sz = ftell(cf); if (sz < 0) { fclose(cf); return -EIO; } if (fseek(cf, 0, SEEK_SET) != 0) { fclose(cf); return -EIO; } if (sz == 0) { fclose(cf); return -EIO; /* zero-byte versioned config file: genuine corruption */ } if ((size_t)sz > PERSIST_CONFIG_BYTES_MAX_SIZE) { fclose(cf); return -EFBIG; } uint8_t* buf = (uint8_t*)malloc((size_t)sz); if (buf == NULL) { fclose(cf); return -ENOMEM; } if (fread(buf, 1, (size_t)sz, cf) != (size_t)sz) { free(buf); fclose(cf); return -EIO; } fclose(cf); *out_bytes = buf; *out_len = (size_t)sz; return 0; } /* Write an entity-bearing checkpoint: legacy header (unchanged) + entity * payload (ha-entity-snapshots task 2.2/3.2, Req 1.1-1.4, 3.1-3.3). Any * failure at any step -- collect, encode, write, sync -- leaves NO file at * the final path; the tmp file is always unlinked on error. This is a * retriable-next-checkpoint-cycle failure, never a crash/abort (controller * never self-terminates on an operational condition). */ static int write_snapshot(persist_t* p) { char tmp_path[4096]; char final_path[4096]; snprintf(tmp_path, sizeof(tmp_path), "%s/snap-%016" PRIx64 ".tmp", p->snap_path, p->checkpoint_lsn); snprintf(final_path, sizeof(final_path), "%s/snap-%016" PRIx64 ".capnp", p->snap_path, p->checkpoint_lsn); /* Collect + encode entities BEFORE opening the tmp file: a collection * failure (Requirement 1.3) then never creates a tmp file at all. */ xlist_t job_frames; int rc = collect_job_frames(p, &job_frames); if (rc != 0) return rc; xlist_t queue_frames; rc = collect_queue_frames(p, &queue_frames); if (rc != 0) { snap_free_job_frames(&job_frames); return rc; } xlist_t host_frames; rc = collect_host_frames(p, &host_frames); if (rc != 0) { snap_free_job_frames(&job_frames); snap_free_queue_frames(&queue_frames); return rc; } int fd = open(tmp_path, O_CREAT | O_WRONLY | O_TRUNC, 0600); if (fd < 0) { snap_free_job_frames(&job_frames); snap_free_queue_frames(&queue_frames); snap_free_host_frames(&host_frames); return -errno; } /* Sentinel: magic (8) + checkpoint_lsn (8) + committed-config reference * (1 present + 8 tag + 32 sha = 41 bytes; ha-term-safety task 4.2 r1). * The config-ref tail is OPTIONAL on read-back, so an older 16-byte * sentinel still loads (load_snapshot reads the tail best-effort). The * tag is little-endian on this arch, matching the checkpoint_lsn field. */ char header[16 + 1 + 8 + 32]; memset(header, 0, sizeof(header)); memcpy(header, "XCESNAP\0", 8); memcpy(header + 8, &p->checkpoint_lsn, 8); header[16] = (char)(p->cfgref_present ? 1 : 0); memcpy(header + 17, &p->cfgref_tag, 8); memcpy(header + 25, p->cfgref_sha, 32); if (write(fd, header, sizeof(header)) != (ssize_t)sizeof(header)) rc = -EIO; /* Entity payload: format_version, then job frames (design.md "Checkpoint * File Layout"). No table/entity lock is held here -- collect_job_frames * already released both before returning. */ if (rc == 0) { uint32_t fmt = SNAPSHOT_FORMAT_VERSION; if (write(fd, &fmt, sizeof(fmt)) != (ssize_t)sizeof(fmt)) rc = -EIO; } if (rc == 0) { uint32_t job_count = (uint32_t)job_frames.count; if (write(fd, &job_count, sizeof(job_count)) != (ssize_t)sizeof(job_count)) rc = -EIO; } for (int i = 0; rc == 0 && i < job_frames.count; i++) { snap_job_frame_t* fr = (snap_job_frame_t*)xlist_get(&job_frames, i); if (write(fd, &fr->len, sizeof(fr->len)) != (ssize_t)sizeof(fr->len)) { rc = -EIO; break; } if (write(fd, fr->buf, fr->len) != (ssize_t)fr->len) { rc = -EIO; break; } } /* Queue trailer (config-upgrade-snapshot task 2.2, Req 1.9, 5.7): * capnp QueueAdminState frames, same [payload_len][capnp bytes] shape * as the job/host frame loops -- migrated from the prior raw * {name_len, name, status} layout (design.md "Checkpoint File Layout": * same field content, new wire encoding only). */ if (rc == 0) { uint32_t queue_count = (uint32_t)queue_frames.count; if (write(fd, &queue_count, sizeof(queue_count)) != (ssize_t)sizeof(queue_count)) rc = -EIO; } for (int i = 0; rc == 0 && i < queue_frames.count; i++) { snap_queue_frame_t* fr = (snap_queue_frame_t*)xlist_get(&queue_frames, i); if (write(fd, &fr->len, sizeof(fr->len)) != (ssize_t)sizeof(fr->len)) { rc = -EIO; break; } if (write(fd, fr->buf, fr->len) != (ssize_t)fr->len) { rc = -EIO; break; } } /* Host trailer (config-upgrade-snapshot task 2.1, Req 1.6, 1.7, 1.9): * capnp HostAdminState frames, same [payload_len][capnp bytes] shape as * the job frame loop above (design.md "Checkpoint File Layout": host * frames come after queue frames). */ if (rc == 0) { uint32_t host_count = (uint32_t)host_frames.count; if (write(fd, &host_count, sizeof(host_count)) != (ssize_t)sizeof(host_count)) rc = -EIO; } for (int i = 0; rc == 0 && i < host_frames.count; i++) { snap_host_frame_t* fr = (snap_host_frame_t*)xlist_get(&host_frames, i); if (write(fd, &fr->len, sizeof(fr->len)) != (ssize_t)sizeof(fr->len)) { rc = -EIO; break; } if (write(fd, fr->buf, fr->len) != (ssize_t)fr->len) { rc = -EIO; break; } } /* Config-bytes section (config-upgrade-snapshot task 2.3, Req 2.1, 2.3, * 2.4, 5.1): raw active-configuration bytes, appended LAST. This is a * single flat length-prefixed blob for the whole file -- NOT a * [count][frame...] shape like the job/queue/host sections above, since * there is exactly one active configuration, never a set of them. * Populated on EVERY checkpoint (marker-triggered or periodic alike, * never gated on which triggered this call) so a member joining or * catching up after the marker-triggered upgrade snapshot has rotated * away still gets configuration and materialized state together from * whichever checkpoint it installs (Req 2.4). A pure function of * whatever bytes currently sit at the resolved config path -- no * timestamp, no ordering dependence -- so reading the same unchanged * file twice yields byte-identical output (Req 5.1 determinism). */ uint8_t* cfg_bytes = NULL; size_t cfg_len = 0; if (rc == 0) { int cfg_rc = read_active_config_bytes(p, &cfg_bytes, &cfg_len); if (cfg_rc == -ENOENT) { /* No local config bytes yet -- no cluster/myconf back-pointer * (bare unit-test persist_t) or this node has not received its * first config push. Mirrors the "no live cluster back-pointer" * empty-is-not-failure discipline collect_job/queue/host_frames * already apply above: write an explicit zero-length section * rather than failing the whole checkpoint over a benign, * expected early-boot / not-yet-provisioned state. */ cfg_bytes = NULL; cfg_len = 0; } else if (cfg_rc != 0) { /* Any other error means genuinely corrupt/unreadable on-disk * config state, not benign absence -- fail the checkpoint like * any other collect/restore failure (design.md Error Handling: * "Host placeholder creation failure ... treated as any other * collect/restore failure today -- discard partial state, fail * the checkpoint/restore attempt, retriable on the next * cycle"). */ rc = cfg_rc; } } if (rc == 0) { uint32_t cfg_bytes_len = (uint32_t)cfg_len; if (write(fd, &cfg_bytes_len, sizeof(cfg_bytes_len)) != (ssize_t)sizeof(cfg_bytes_len)) rc = -EIO; } if (rc == 0 && cfg_len > 0 && write(fd, cfg_bytes, cfg_len) != (ssize_t)cfg_len) rc = -EIO; free(cfg_bytes); if (rc == 0 && fdatasync(fd) != 0) rc = -errno; close(fd); snap_free_job_frames(&job_frames); snap_free_queue_frames(&queue_frames); snap_free_host_frames(&host_frames); if (rc != 0) { unlink(tmp_path); return rc; } if (rename(tmp_path, final_path) != 0) { int saved = errno; unlink(tmp_path); return -saved; } return 0; } /* scan_latest_committed_config_ref (ha-term-safety task 4.2 remediation r1): * walk the local WAL segments and capture the HIGHEST-LSN WAL_CONFIG_MARKER * whose LSN is <= committed_lsn into p->cfgref_{present,tag,sha}. This is the * SAME committed-prefix marker config_committed_marker derives, cached durably * so a later checkpoint can trim the marker's segment without losing the * derived committed-config reference (compaction safety). The marker payload is * a fixed 40-byte reference (8-byte big-endian version_tag + 32-byte sha256 — * mirrors config_marker_encode; the config BYTES are never in the WAL), decoded * inline here so persist stays free of a config_repl layering dependency. */ static void scan_latest_committed_config_ref(persist_t* p) { uint64_t committed = atomic_load_explicit(&p->committed_lsn, memory_order_acquire); if (committed == 0) return; /* nothing committed → keep any prior cached ref unchanged */ DIR* d = opendir(p->wal_path); if (!d) return; uint64_t seg_lsns[4096]; int nseg = 0; struct dirent* e; while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) { uint64_t sl = 0; if (sscanf(e->d_name, "seg-%016" SCNx64 ".wal", &sl) == 1) seg_lsns[nseg++] = sl; } closedir(d); if (nseg == 0) return; for (int i = 1; i < nseg; i++) { uint64_t key = seg_lsns[i]; int j = i - 1; while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; } seg_lsns[j + 1] = key; } for (int si = 0; si < nseg; si++) { char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[si]); int fd = open(seg_path, O_RDONLY); if (fd < 0) continue; uint32_t off = 0; for (;;) { wal_entry_hdr_t hdr; ssize_t n = pread(fd, &hdr, sizeof(hdr), (off_t)off); if (n < (ssize_t)sizeof(hdr)) break; if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break; if (hdr.lsn > committed) break; /* past committed prefix */ if (hdr.type == WAL_CONFIG_MARKER && hdr.payload_len >= WAL_CONFIG_MARKER_PAYLOAD_LEN) { uint8_t buf[WAL_CONFIG_MARKER_PAYLOAD_LEN]; ssize_t pr = pread(fd, buf, sizeof(buf), (off_t)(off + sizeof(hdr))); if (pr == (ssize_t)sizeof(buf)) { p->cfgref_tag = ((uint64_t)buf[0] << 56) | ((uint64_t)buf[1] << 48) | ((uint64_t)buf[2] << 40) | ((uint64_t)buf[3] << 32) | ((uint64_t)buf[4] << 24) | ((uint64_t)buf[5] << 16) | ((uint64_t)buf[6] << 8) | ((uint64_t)buf[7]); memcpy(p->cfgref_sha, buf + 8, 32); p->cfgref_present = 1; /* later markers overwrite — latest wins */ } } off += (uint32_t)(sizeof(hdr) + hdr.payload_len); } close(fd); } } /* Delete WAL segment files whose start LSN is strictly less than the * active segment's start LSN AND strictly less than the trim floor. * * Bug-1147 (stopgap, raft-drive-async task 3.2): the trim floor is the * minimum of the active-segment LSN, every active standby's last acked LSN, * and every recently-disconnected slot's last_disconnected_lsn (strategy-A * retention discipline). This ensures we NEVER delete a WAL segment that a * live standby still needs for catch-up (the bug-795/828 committed-job-loss * family). It does NOT fix the empty/far-behind join case (cursor=0) — * that requires entity-bearing snapshots (separate spec). * * Lock ordering (verified 2026-06-26): persist_checkpoint takes p->lock only * to snapshot checkpoint_lsn, then RELEASES it before calling * trim_wal_segments — so this function runs with NO p->lock held. * repl_trim_lsn and repl_expire_disconnect_slots take only r->lock * (p->repl.lock); neither ever acquires p->lock or ce->lock, so there is no * p->lock↔repl.lock (or ce->lock↔repl.lock) inversion. */ static void trim_wal_segments(persist_t* p) { /* Extract the start LSN of the currently-open segment from seg_path. * seg_path has the form "/seg-.wal". * If no segment is open yet (wal_fd < 0), skip trimming. */ if (p->wal_fd < 0 || p->seg_path[0] == '\0') return; const char* seg_basename = strrchr(p->seg_path, '/'); if (!seg_basename) return; seg_basename++; /* skip the '/' */ uint64_t active_seg_lsn = 0; if (sscanf(seg_basename, "seg-%" SCNx64 ".wal", &active_seg_lsn) != 1) return; /* Expire stale disconnected slots so their last_disconnected_lsn guards * don't hold WAL segments indefinitely (slot guard expires at * last_disconnect_time + snap_interval_s). */ repl_expire_disconnect_slots(&p->repl, time(NULL), p->config.snap_interval_s); /* Compute the trim floor: min(active_seg_lsn, all active standby * acked_lsn, all valid disconnected-slot last_disconnected_lsn). * Any segment with seg_start_lsn BELOW this floor is disposable. */ uint64_t floor = repl_trim_lsn(&p->repl, active_seg_lsn, p->config.snap_interval_s); /* bug-1426: also cap the floor at the LOCAL apply pump's own progress * (last_applied_lsn), not just remote standby-acked positions. * repl_trim_lsn's floor only reflects what OTHER members have * durably received -- it has no idea whether this member's own live * apply pump has actually caught up locally. If the pump is lagging * (e.g. mid-recovery, or a slow apply burst) when a trim fires, a * segment below the standby-floor but AT OR ABOVE the pump's own * still-unread position could be deleted while apply_pump_read_entry's * cache holds an open fd into it -- POSIX unlink-keeps-fd-alive lets * the pump keep reading (no immediate corruption, no wrong data), but * a FRESH read of the same range (a restart's persist_recover() replay, * or the pump's own cache falling back to its slow path after a miss) * would find the segment already gone -- a genuine, if narrow, apply- * correctness gap: what the live process quietly caught up to and what * a fresh read of disk state can see would diverge. Mirrors the exact * same "floor = min(...)" pattern already used above for the standby/ * disconnect-guard case -- same imprecision (segment-granularity, not * exact-entry-granularity), same established acceptable tradeoff. * * +1, not the raw value: last_applied_lsn == N means the entry AT lsn N * has been fully applied -- unlike repl_trim_lsn's floor (an acked_lsn * where equality means "the standby might still need this segment, * preserve it" -- see the seg_start_lsn < floor check below, which * already treats equality as preserve for that input by construction), * the LOCAL apply cursor's equality case means the opposite: this * process is DONE with that entry and will never re-read it. Feeding * last_applied_lsn+1 into the same shared "< floor" check gets the * correct delete-on-equal semantics for this term specifically, * without needing a second, differently-shaped comparison. */ uint64_t apply_floor = atomic_load_explicit(&p->last_applied_lsn, memory_order_acquire) + 1; if (apply_floor < floor) floor = apply_floor; DIR* d = opendir(p->wal_path); if (!d) return; struct dirent* e; while ((e = readdir(d)) != NULL) { if (strncmp(e->d_name, "seg-", 4) != 0) continue; uint64_t seg_start_lsn = 0; if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start_lsn) != 1) continue; /* Only delete segments below BOTH the active segment AND the * standby-cursor floor (bug-1147 guard). floor <= active_seg_lsn * by construction (repl_trim_lsn caps at checkpoint_lsn=active_seg_lsn), * so this check subsumes the old < active_seg_lsn guard. */ if (seg_start_lsn < floor) { char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), "%s/%s", p->wal_path, e->d_name); unlink(seg_path); } } closedir(d); } /* Snapshot filename comparison helper for qsort — sorts ascending by name * (which sorts ascending by LSN since filenames embed the LSN as hex). */ static int snap_name_cmp(const void* a, const void* b) { return strcmp(*(const char* const*)a, *(const char* const*)b); } /* Keep only the snap_keep_n most recent snapshot files; delete the rest. */ /* True when `lsn` is the checkpoint position an in-flight peer snapshot * transfer currently references (ha-entity-snapshots task 4.1/4.3, Req * 3.1-3.3) — checked directly against snapshot_inflight_lsn[], not gated on * snapshot_inflight[] first: the LSN clears before the flag in every * ha_snapshot_send_dispatch/snapshot_lane_fn path, so a stale flag=1 can * momentarily coexist with an already-cleared LSN=0 (see task 4.1's * Implementation Note). A checkpoint's LSN is never 0, so "nonzero and * matches" is the correct, race-free protection signal on its own. No * cluster (p->cluster == NULL, e.g. standalone lib-level tests) means no HA * layer exists to be mid-transfer — nothing is protected. */ static bool snapshot_lsn_in_flight(persist_t* p, uint64_t lsn) { if (p->cluster == NULL || lsn == 0) return false; for (int i = 0; i < MAX_HA_CANDIDATES; i++) { if (atomic_load_explicit(&p->cluster->ha.snapshot_inflight_lsn[i], memory_order_acquire) == lsn) return true; } return false; } static void rotate_snapshots(persist_t* p) { DIR* d = opendir(p->snap_path); if (!d) return; /* Collect all .capnp snapshot filenames */ char* names[1024]; int n = 0; struct dirent* e; while ((e = readdir(d)) != NULL && n < (int)(sizeof(names) / sizeof(names[0]))) { size_t len = strlen(e->d_name); if (len <= 6 || strcmp(e->d_name + len - 6, ".capnp") != 0) continue; names[n] = strdup(e->d_name); if (names[n]) n++; } closedir(d); if (n == 0) return; /* Sort ascending — oldest first */ qsort(names, (size_t)n, sizeof(names[0]), snap_name_cmp); /* Delete all but the snap_keep_n newest, skipping any file an in-flight * peer transfer is still reading from (task 4.3) — it is retried on the * next rotation pass once that transfer completes. */ int keep = (int)p->config.snap_keep_n; int delete_up_to = n - keep; /* first index to keep */ for (int i = 0; i < delete_up_to && i < n; i++) { uint64_t lsn = 0; if (sscanf(names[i], "snap-%" SCNx64 ".capnp", &lsn) == 1 && snapshot_lsn_in_flight(p, lsn)) continue; char path[4096]; snprintf(path, sizeof(path), "%s/%s", p->snap_path, names[i]); unlink(path); } for (int i = 0; i < n; i++) free(names[i]); } /* ------------------------------------------------------------------ */ /* Async checkpoint scheduling (schedule_checkpoint + task fn) */ /* ------------------------------------------------------------------ */ #ifdef XCE_TESTING /* Test-injectable substitute for the real (blocking) checkpoint work, * mirroring ha.c's ha_repl_send_snapshot_hook idiom exactly: when non-NULL, * checkpoint_lane_fn calls this INSTEAD OF persist_checkpoint() — lets a * test park/gate the lane worker without needing a full disk-backed * checkpoint to actually complete. NULL in every fixture that never installs * it (the overwhelming common case): checkpoint_lane_fn falls through to the * real call. Symbol always compiled so test TUs can `extern` it; unused in * a non-XCE_TESTING (production) build. */ void (*persist_checkpoint_test_hook)(persist_t*) = NULL; #endif /* g_persist_checkpoint_offload: checkpoint slow-lane offload feature flag * (xce-audit-invariants controller pass, dimension 1 async-I/O thread-role * fix, bug-1694). * * checkpoint_task_fn (below) was the last delaymgr-triggered path in this * codebase that still ran its full blocking body directly on the delaymgr * timer thread -- the SAME single thread that drives heartbeats and election * timeouts cluster-wide (ha-async-io.md thread-role contract: delaymgr must * only trigger work, never block). The blocking body is real: a * WAL_CHECKPOINT marker pwrite+fdatasync, write_snapshot()'s unbounded * open/write walk over every job/queue/host frame (scales with cluster state * size, held under apply_pump_lock), then trim_wal_segments + rotate_ * snapshots' unlink. Every sibling long-blocking-I/O path (snapshot send, * config-push, reconnect) was already moved off its liveness thread by * raft-drive-async -- this was the one documented remaining gap. * * DEFAULT ON (1), unlike the still-experimental catchup/demotion offloads * (g_drive_async_catchup_offload / g_drive_async_demotion_offload in * config_repl.c / ha_convergence.c, both default OFF pending an unrelated, * real test-fixture hang -- see their own comments and bug-1429): this * reuses an ALREADY-PROVEN-SAFE call, not new surface. persist_checkpoint() * already runs on this SAME slow lane in production TODAY via * ha_snapshot_do_send() -> snapshot_lane_fn (g_drive_async_snapshot_offload, * ha.c, also default ON) -- every snapshot-send cycle already calls * persist_checkpoint() from a slow-lane worker thread, so apply_pump_lock's * acquisition inside persist_checkpoint() from a slow-lane thread is * exercised in production every day, not a new code path. persist_ * checkpoint()'s own atomic_exchange guard (checkpoint_in_progress) is the * existing, already-relied-upon single-flight barrier that keeps this * offload and the snapshot-send path from ever double-running a checkpoint * concurrently (see g_drive_async_snapshot_offload's own comment: "at most * one persist_checkpoint in flight (existing atomic guard)"). * * Destination is the SLOW LANE (slowlane_submit), deliberately NOT the * vtaskmgr N-thread pool: write_snapshot()'s per-entity walk is genuinely * unbounded, matching ha-async-io.md's own table entry ("vtaskmgr must NOT * do: long/unbounded blocking I/O (snapshot transfer, config-push) -- * starves the pool"). The vtask pool was investigated and ruled out * specifically because persist_apply_pump_run() -- which ALSO acquires * apply_pump_lock, across its entire per-entry walk -- is itself dispatched * onto that SAME pool via persist_apply_pump_kick()'s * vtask_enq_classified(pool, task, VTASK_LIVE_BLOCK). Putting checkpoint's * unbounded, lock-held I/O on the identical N-thread pool would risk a * worker thread parked on apply_pump_lock for the whole snapshot-write * duration, starving the pool's liveness-critical capacity (commit * continuations, apply-pump drains) -- exactly the failure mode the * codebase's own slow-lane/vtask-pool split already exists to prevent. The * slow lane's dedicated, separate worker threads carry no such contention * with the vtask pool's own capacity, and this checkpoint offload does not * introduce any NEW lock-order edge: it is the same apply_pump_lock * acquisition the snapshot-send path already makes from a slow-lane thread. * * No synchronous-wait hazard (the actual mechanism behind the catchup/ * demotion offloads staying OFF -- an uninitialized slow lane silently * swallows a submitted task with zero live workers to ever run it, and * something ELSE downstream then hangs forever waiting for its completion): * checkpoint's trigger (schedule_checkpoint, fired from the WAL-append hot * path when wal_record_count crosses max_wal_records) is pure * fire-and-forget -- nothing waits synchronously for a checkpoint to finish, * so a silently-swallowed submission degrades to "this checkpoint cycle is * skipped, retried on the next WAL threshold crossing" (a liveness/WAL- * growth concern in a hypothetical mis-configured fixture, never a hang). * Verified no existing controller_unit_test fixture depends on this trigger * path completing: test_ha_core_safety.c's live-cluster HA fixtures * deliberately pin max_wal_records to 1u<<30 specifically "so no checkpoint * is [triggered]" / "never auto-checkpoint during a unit test"; every other * max_wal_records reference in the suite is a synthetic/no-cluster check * that never reaches schedule_checkpoint's cluster branch at all. Production * always calls slowlane_init via cluster_init_phase2 before the delaymgr * loop can fire this trigger. * * OFF -> legacy inline blocking path on the delaymgr thread; retained only * so the TDD RED test can demonstrate the pre-fix stall. Compiled in ALL * builds; the setter is the test seam to flip RED<->GREEN. */ static _Atomic int g_persist_checkpoint_offload = 1; void persist_set_checkpoint_offload_for_test(int on) { atomic_store_explicit(&g_persist_checkpoint_offload, on ? 1 : 0, memory_order_release); } int persist_checkpoint_offload_enabled(void) { return atomic_load_explicit(&g_persist_checkpoint_offload, memory_order_acquire); } /* run_checkpoint_body: the actual (blocking) checkpoint work -- persist_ * checkpoint() itself is UNCHANGED. Shared by every call site that can end * up running it (the slow-lane worker AND checkpoint_task_fn's flag-OFF / * no-cluster / submit-failure inline-fallback branches) so the XCE_TESTING * hook seam (persist_checkpoint_test_hook) covers all of them identically -- * a test can park/observe the checkpoint body regardless of which path * dispatched it, which is exactly what proves (or, pre-fix, disproves) the * delaymgr thread never runs it directly. Resets wal_record_count AFTER the * work actually ran (unconditionally, success or failure alike -- matches * the pre-fix ordering exactly: the original inline checkpoint_task_fn also * reset it unconditionally right after persist_checkpoint() returned). */ static void run_checkpoint_body(persist_t *persist) { #ifdef XCE_TESTING if (persist_checkpoint_test_hook != NULL) { persist_checkpoint_test_hook(persist); atomic_store(&persist->wal_record_count, 0); return; } #endif persist_checkpoint(persist); atomic_store(&persist->wal_record_count, 0); } /* checkpoint_lane_fn: slow-lane worker entry (offload flag ON). Runs * run_checkpoint_body() off the delaymgr thread. Mirrors snapshot_lane_fn / * configpush_lane_fn / catchup_lane_fn / demotion_lane_fn exactly. */ static void checkpoint_lane_fn(void *arg) { persist_t *persist = (persist_t *)arg; if (!persist) return; run_checkpoint_body(persist); } /* checkpoint_task_fn: executed by delaymgr worker thread -- TRIGGER ONLY * (ha-async-io.md thread-role contract: delaymgr fires timers and triggers * work, then returns fast -- it never blocks). * * The CAS in schedule_checkpoint already holds the "in_progress" slot (=1). * Release it so persist_checkpoint's own guard (inside checkpoint_lane_fn, * now running off-thread) can acquire it, then hand the real (blocking) * checkpoint work to the slow lane -- see g_persist_checkpoint_offload above * for the flag + full design rationale. * * Inline-fallback + WARN on slowlane_submit failure (mirrors every sibling * offload's own rule): a scheduled checkpoint must never be silently * dropped -- that risks unbounded WAL growth. -EEXIST (a checkpoint already * queued/running under this same lane key) and -EAGAIN (the lane's * admission cap is momentarily full) are NOT failures -- in both cases the * work is already going to run (slowlane_submit's own documented contract: * "In all of {0, -EAGAIN} the work is enqueued and will run exactly once"; * -EEXIST means it is already enqueued/running under this key). Only a * genuine submit failure (alloc / lane stopping) falls back inline. */ static int checkpoint_task_fn(void* arg) { persist_t* persist = (persist_t*)arg; /* Release the "scheduled" slot so persist_checkpoint's guard can acquire */ atomic_store(&persist->checkpoint_in_progress, 0); if (!persist_checkpoint_offload_enabled() || persist->cluster == NULL) { /* Legacy inline path -- blocks the calling (delaymgr) thread. * Retained solely so the RED test can show the pre-change stall; * production runs with the flag ON. */ run_checkpoint_body(persist); return 0; } int srv = slowlane_submit(&persist->cluster->slowlane, /*key=*/1, checkpoint_lane_fn, persist); if (srv < 0 && srv != -EAGAIN && srv != -EEXIST) { WARN("persist: checkpoint slowlane_submit failed (rc=%d) -- running " "inline (never silently drop a scheduled checkpoint -- risk of " "unbounded WAL growth)", srv); run_checkpoint_body(persist); } /* 0 or -EAGAIN: enqueued exactly once; checkpoint_lane_fn resets * wal_record_count when it actually runs. -EEXIST: a checkpoint is * already queued/running -- nothing further to do here. */ return 0; } #ifdef XCE_TESTING /* Test seam: directly drive checkpoint_task_fn (production's only call site * is via delay_task, which a test would otherwise have to wait out — the * real delaymgr fires it 1s after schedule_checkpoint's delay_task call). * Returns checkpoint_task_fn's own return value (always 0). */ int persist_checkpoint_task_fn_for_test(persist_t *p) { return checkpoint_task_fn(p); } #endif /* schedule_checkpoint: idempotent async checkpoint request via delaymgr. * Safe to call from any thread. Uses atomic CAS to ensure at most one * checkpoint is scheduled or running at a time. * * If cluster is NULL (unit tests without a live delaymgr) the function * falls back to a synchronous checkpoint. */ static void schedule_checkpoint(persist_t* persist) { int expected = 0; if (!atomic_compare_exchange_strong(&persist->checkpoint_in_progress, &expected, 1)) return; /* already scheduled or running — idempotency guard */ if (persist->cluster == NULL) { /* Unit-test / no-cluster path: run synchronously and reset. */ atomic_store(&persist->checkpoint_in_progress, 0); persist_checkpoint(persist); atomic_store(&persist->wal_record_count, 0); return; } /* Schedule via delaymgr with 1 s coalescing delay, single execution */ delay_task(&persist->cluster->delaymgr, 1, 1, checkpoint_task_fn, persist); } /* ------------------------------------------------------------------ */ /* Checkpoint core (shared by persist_checkpoint and the marker-apply */ /* trigger, config-upgrade-snapshot task 5) */ /* ------------------------------------------------------------------ */ /* persist_checkpoint_write_locked: pin checkpoint_lsn to the EXPLICIT * `checkpoint_lsn` given (never re-derived internally from * last_applied_lsn), capture the committed-config reference, and serialize * live state to a snapshot file. * * Caller MUST already hold apply_pump_lock (or otherwise guarantee no * concurrent apply-pump walk can be racing this call, e.g. the * single-threaded boot-replay window persist_recover() already documents * elsewhere in this file) -- this function never locks or unlocks * apply_pump_lock itself, so it is safe to call both from a context that * already holds it (the live apply pump) and from a context where no lock * is needed at all (boot replay). * * The explicit-lsn parameter exists because the two callers need two * different values at two different moments: * - persist_checkpoint() (periodic/explicit trigger): last_applied_lsn, * read fresh by the caller immediately before this call. * - the marker-apply trigger (apply_committed_entry's WAL_CONFIG_MARKER * case): the marker's OWN lsn -- last_applied_lsn is NOT yet advanced * to the marker's position at the moment that case runs (both live * apply and replay advance it strictly AFTER dispatching this call), * so reading last_applied_lsn from inside that case would mislabel the * checkpoint with the marker's PREDECESSOR's lsn, not its own. */ static int persist_checkpoint_write_locked(persist_t* p, uint64_t checkpoint_lsn) { p->checkpoint_lsn = checkpoint_lsn; /* Capture the latest committed config-marker reference BEFORE trimming * (ha-term-safety task 4.2 r1): trim_wal_segments (run by both callers, * after this function returns) can unlink the segment holding a * committed WAL_CONFIG_MARKER, so the derived committed config must be * cached durably (into the sentinel) before that happens. */ scan_latest_committed_config_ref(p); /* Serialize in-memory state to a snapshot file (P0: no live kvmgrs — * writes sentinel header + cached config ref). */ return write_snapshot(p); } /* ------------------------------------------------------------------ */ /* Public: persist_checkpoint */ /* ------------------------------------------------------------------ */ int persist_checkpoint(persist_t* p) { /* Guard against concurrent checkpoints */ if (atomic_exchange(&p->checkpoint_in_progress, 1) != 0) return -EBUSY; int rc = 0; /* 1. Write WAL_CHECKPOINT marker (local only — never replicated) */ rc = persist_write_checkpoint_marker(p); if (rc != 0) goto done; /* 2. Record checkpoint_lsn = the APPLIED cursor (last_applied_lsn), not * the physical WAL append tip (current_lsn). write_snapshot() below * serializes live jobmgr state, which only reflects entries the * apply pump has actually dispatched -- NOT every entry durably * appended/committed, since apply is decoupled from commit * (raft-apply-decoupling) and can lag behind current_lsn under load. * Stamping current_lsn here mislabels the snapshot: on restart, * last_applied_lsn is re-seeded FROM checkpoint_lsn and replay * resumes strictly above it, so any committed-but-not-yet-applied * entries between the true apply cursor and current_lsn at * checkpoint time are silently skipped forever, even though jobmgr * never actually contained them (persist.h last_applied_lsn doc). * Held under apply_pump_lock together with 3a/3b so the label and * the jobmgr walk it describes are captured atomically w.r.t. the * live apply pump -- the pump can only ever move last_applied_lsn * and jobmgr forward, never backward, so freezing it here rules out * the label ever claiming more than jobmgr actually holds. */ pthread_mutex_lock(&p->apply_pump_lock); rc = persist_checkpoint_write_locked( p, atomic_load_explicit(&p->last_applied_lsn, memory_order_acquire)); pthread_mutex_unlock(&p->apply_pump_lock); if (rc != 0) goto done; /* 4. Trim old WAL segments before the checkpoint */ trim_wal_segments(p); /* 5. Rotate old snapshots — keep only snap_keep_n most recent */ rotate_snapshots(p); done: atomic_store(&p->checkpoint_in_progress, 0); return rc; } /* persist_checkpoint_marker_triggered (config-upgrade-snapshot task 5, Req * 1.1, 1.5, 1.8, 4.4, 5.2, 5.3): force an immediate checkpoint at exactly * `marker_lsn` -- the configuration-change marker's OWN lsn -- called from * apply_committed_entry's WAL_CONFIG_MARKER case. * * Unlike persist_checkpoint(), this function NEVER locks apply_pump_lock: * it relies entirely on the caller's own locking context, because the two * production call sites of apply_committed_entry() differ in that context * and this function must be correct under BOTH without deadlocking under * either: * - persist_apply_pump_run() (live online-apply, both primary and * standby route through this) holds apply_pump_lock across its ENTIRE * per-LSN dispatch loop, including the call into apply_committed_entry * that reaches this function. Re-locking that same, non-recursive * mutex here would self-deadlock -- so this function must NOT lock it; * it is already held by the caller for the whole duration of this call. * - replay_wal() (boot/restart replay, persist_recover()'s sole caller) * does NOT hold apply_pump_lock during its per-entry dispatch loop -- * by design, since persist_recover() runs before the apply pump is * ever kicked and nothing else can be racing it (see persist_recover's * own doc comment on its brief, unrelated apply_pump_lock use for * cache invalidation). No lock is needed here either, for the same * reason. * Both call sites are internal to persist.c and fully controlled, so this * documented precondition (caller already serializes, one way or another) * is safe to rely on rather than adding a redundant/ambiguous lock here. * * Shares checkpoint_in_progress -- the SAME single-in-flight CAS guard * persist_checkpoint() uses -- so a concurrent WAL-threshold-triggered * periodic checkpoint and this marker-triggered one can never both * proceed at once (Req 5.3). Returns -EBUSY when that guard is already * held by someone else; the caller (apply_committed_entry) treats that as * an expected, self-resolving condition, not a failure (Req 4.4) -- the * in-flight checkpoint (whichever triggered it) still completes and * either it or a later checkpoint picks up this marker's configuration * (every checkpoint embeds the currently active configuration, task 2.3). */ static int persist_checkpoint_marker_triggered(persist_t* p, uint64_t marker_lsn) { if (atomic_exchange(&p->checkpoint_in_progress, 1) != 0) return -EBUSY; int rc = persist_write_checkpoint_marker(p); if (rc == 0) rc = persist_checkpoint_write_locked(p, marker_lsn); if (rc == 0) { trim_wal_segments(p); rotate_snapshots(p); } atomic_store(&p->checkpoint_in_progress, 0); return rc; } /* ------------------------------------------------------------------ */ /* Recovery internals */ /* ------------------------------------------------------------------ */ /* Parse and restore every job frame, then every queue frame, then every host * frame from the entity payload beginning at the current `fd` offset * (design.md "Checkpoint File Layout", SNAPSHOT_FORMAT_VERSION=2): * [job_count][job_count * {payload_len; capnp Job bytes}] * [queue_count][queue_count * {payload_len; capnp QueueAdminState bytes}] * [host_count][host_count * {payload_len; capnp HostAdminState bytes}] * A trailing config-bytes section follows the host section in the file * (config-upgrade-snapshot task 2.3) but is deliberately NOT read here -- * out of this function's scope; any bytes after the host section are simply * left unread, exactly as host-frame bytes were already tolerated as * present-but-unread before this function grew a host section of its own. * * `file_size` is the checkpoint file's total size (from fstat) and * `consumed` is the number of bytes already read from it (legacy header + * format_version) — every length-prefixed frame's length is bounds-checked * against the ACTUAL remaining bytes in the file before being trusted for a * read or allocation, never blindly followed (a corrupt/attacker-controlled * length must fail fast, not drive an oversized read). * * Restores in file order: every job via restore_job, then every queue via * restore_queue_state, then every host via restore_host_state * (config-upgrade-snapshot task 4, Req 1.10 — design.md sequence diagram * and "Referential integrity" note: restore order does not matter * functionally, since re-homing decisions are already baked into each * entity's encoded content at materialization time, never re-derived at * restore time, Design Invariant 2). Returns 0 only if every frame * restored; on the FIRST failure of any kind (short read, oversized * length, decode error, or restore error) returns immediately with a * negative errno — the caller MUST NOT commit any position change on a * non-zero return (Requirement 2.3). * * restore_job's -EEXIST is treated as success, not a failure: it is job.c's * own documented idempotent-replay guard (job already present), needed so a * retried load — after an earlier partial attempt already created some jobs * before failing on a later one — can continue past already-restored entries * rather than aborting on them again. restore_queue_state and * restore_host_state have no such idempotent-duplicate case of their own to * special-case here (re-applying the same status is already a harmless * overwrite/OR, and restore_host_state's create-if-missing placeholder path * is independently idempotent under retry — Requirement 5.8, worker.c), so * any non-zero return from either is a genuine failure. */ static int load_snapshot_entities(int fd, off_t file_size, off_t consumed, persist_t* p, uint64_t stored_lsn) { uint32_t job_count = 0; if (read(fd, &job_count, sizeof(job_count)) != (ssize_t)sizeof(job_count)) return -EIO; consumed += (off_t)sizeof(job_count); if (job_count > 0 && (p->cluster == NULL || p->cluster->jobmgr == NULL)) return -EINVAL; /* entities present but nowhere to restore them into */ for (uint32_t i = 0; i < job_count; i++) { uint32_t payload_len = 0; if (read(fd, &payload_len, sizeof(payload_len)) != (ssize_t)sizeof(payload_len)) return -EIO; consumed += (off_t)sizeof(payload_len); if (payload_len == 0 || (off_t)payload_len > file_size - consumed) return -EIO; uint8_t* buf = (uint8_t*)malloc(payload_len); if (buf == NULL) return -ENOMEM; if (read(fd, buf, payload_len) != (ssize_t)payload_len) { free(buf); return -EIO; } consumed += (off_t)payload_len; int rrc = restore_job(p->cluster, 0, buf, payload_len, stored_lsn); free(buf); if (rrc != 0 && rrc != -EEXIST) return rrc; } uint32_t queue_count = 0; if (read(fd, &queue_count, sizeof(queue_count)) != (ssize_t)sizeof(queue_count)) return -EIO; consumed += (off_t)sizeof(queue_count); if (queue_count > 0 && (p->cluster == NULL || p->cluster->qmgr == NULL)) return -EINVAL; /* entities present but nowhere to restore them into */ for (uint32_t i = 0; i < queue_count; i++) { uint32_t payload_len = 0; if (read(fd, &payload_len, sizeof(payload_len)) != (ssize_t)sizeof(payload_len)) return -EIO; consumed += (off_t)sizeof(payload_len); if (payload_len == 0 || (off_t)payload_len > file_size - consumed) return -EIO; uint8_t* buf = (uint8_t*)malloc(payload_len); if (buf == NULL) return -ENOMEM; if (read(fd, buf, payload_len) != (ssize_t)payload_len) { free(buf); return -EIO; } consumed += (off_t)payload_len; /* Decode the capnp QueueAdminState frame (config-upgrade-snapshot * task 4, Req 1.9/5.7 decode side — migrated off the prior raw * {name_len, name, status} layout collect_queue_frames stopped * writing in task 2.2). Mirrors the WAL_JOB_SUBMIT re-decode block * above in this file (capn_init_mem/capn_getp/read_*) and * test_persist_snapshot_entities.c's own * snapent_read_checkpoint_queues reader exactly. */ struct capn c_ctx; capn_init_mem(&c_ctx, buf, payload_len, 0); QueueAdminState_ptr qp; qp.p = capn_getp(capn_root(&c_ctx), 0, 1); struct QueueAdminState cq; memset(&cq, 0, sizeof(cq)); read_QueueAdminState(&cq, qp); capnp_queue_admin_state_t decoded; memset(&decoded, 0, sizeof(decoded)); decode_QueueAdminState(&decoded, &cq); capn_free(&c_ctx); free(buf); int qrc = restore_queue_state(p->cluster, decoded.name, decoded.status); free_QueueAdminState(&decoded); if (qrc != 0) return qrc; } /* Host section (config-upgrade-snapshot task 4, Req 1.10): new in * SNAPSHOT_FORMAT_VERSION=2, added after the queue section (design.md * "Checkpoint File Layout"). Same [payload_len][capnp bytes] per-frame * shape as the job/queue sections above. Every host frame dispatches * to restore_host_state (worker.c, task 3), which is create-if-missing * (a worker_t is join-provisioned, not config-provisioned like a queue * — Design Invariant 1) and idempotent under retry on its own * (Requirement 5.8), so this loop applies the exact same "any failure * returns immediately" discipline as the job/queue loops above without * needing any extra idempotency handling here. */ uint32_t host_count = 0; if (read(fd, &host_count, sizeof(host_count)) != (ssize_t)sizeof(host_count)) return -EIO; consumed += (off_t)sizeof(host_count); if (host_count > 0 && (p->cluster == NULL || p->cluster->workermgr == NULL)) return -EINVAL; /* entities present but nowhere to restore them into */ for (uint32_t i = 0; i < host_count; i++) { uint32_t payload_len = 0; if (read(fd, &payload_len, sizeof(payload_len)) != (ssize_t)sizeof(payload_len)) return -EIO; consumed += (off_t)sizeof(payload_len); if (payload_len == 0 || (off_t)payload_len > file_size - consumed) return -EIO; uint8_t* buf = (uint8_t*)malloc(payload_len); if (buf == NULL) return -ENOMEM; if (read(fd, buf, payload_len) != (ssize_t)payload_len) { free(buf); return -EIO; } consumed += (off_t)payload_len; /* Decode the capnp HostAdminState frame -- mirrors the queue decode * block immediately above and * test_persist_snapshot_entities.c's snapent_read_checkpoint_hosts * reader exactly. */ struct capn c_ctx; capn_init_mem(&c_ctx, buf, payload_len, 0); HostAdminState_ptr hp; hp.p = capn_getp(capn_root(&c_ctx), 0, 1); struct HostAdminState ch; memset(&ch, 0, sizeof(ch)); read_HostAdminState(&ch, hp); capnp_host_admin_state_t decoded; memset(&decoded, 0, sizeof(decoded)); decode_HostAdminState(&decoded, &ch); capn_free(&c_ctx); free(buf); int hrc = restore_host_state(p->cluster, decoded.name, decoded.status); free_HostAdminState(&decoded); if (hrc != 0) return hrc; } return 0; } /* Load a snapshot: legacy header (magic + checkpoint_lsn + optional * config-ref tail), then — for an entity-bearing checkpoint — every job, * queue, and host frame that follows it (ha-entity-snapshots task 5.2, * config-upgrade-snapshot task 4; Requirements 2.1-2.4, 5.1, 5.2, 1.10). * See design.md "load_snapshot (extended)" and its sequence diagram for the * exact contract this implements. * * Ordering (Requirement 2.1, 2.2, 2.3 — restore FIRST, validate FULLY, THEN * advance position, never the reverse, never partial): the header and any * config-ref tail are parsed into LOCAL variables only. Format is then * detected by whether any bytes remain past the header (design.md's stated * discriminator — not a flag bit). A short file (nothing remains) is a * legacy, pre-feature checkpoint: not an error, so its recorded position is * committed immediately and *out_entities_restored is false. Otherwise every * job, then every queue, then every host frame is restored via * load_snapshot_entities() before ANYTHING is written to * p->checkpoint_lsn/p->current_lsn/p->cfgref_*; on any restore failure this * function returns non-zero with p's fields PROVABLY untouched (the single * commit point below is only ever reached after load_snapshot_entities() * returns 0). *out_entities_restored is only set on a rc==0 return, matching * persist.h's documented contract. * * Returns 0 on success, -EINVAL on bad magic / unrecognized entity format, * -EIO on a short/corrupt read, -ENOMEM on allocation failure, -errno on * open/fstat failure, or a restore_job/restore_queue_state/restore_host_state * error code. */ static int load_snapshot(persist_t* p, const char* path, uint64_t lsn, bool* out_entities_restored) { (void)lsn; /* lsn is encoded in the file itself — we re-read it */ int fd = open(path, O_RDONLY); if (fd < 0) return -errno; struct stat st; if (fstat(fd, &st) != 0) { int saved = errno; close(fd); return -saved; } /* magic(8) + checkpoint_lsn(8) + optional config-ref tail(1+8+32). */ char header[16 + 1 + 8 + 32]; ssize_t n = read(fd, header, sizeof(header)); if (n < 16) { close(fd); return -EIO; } if (memcmp(header, "XCESNAP\0", 8) != 0) { close(fd); return -EINVAL; } /* Extract checkpoint_lsn from bytes 8-15 (little-endian on this arch). * Held LOCALLY -- not yet written to p-> -- until the single commit * point below (legacy short-circuit, or after every entity restores). */ uint64_t stored_lsn; memcpy(&stored_lsn, header + 8, 8); /* Optional committed-config reference (ha-term-safety task 4.2 r1): * present only when the sentinel carries the 41-byte tail. Also held * locally until the same commit point. */ int cfgref_present = 0; uint64_t cfgref_tag = 0; uint8_t cfgref_sha[32]; memset(cfgref_sha, 0, sizeof(cfgref_sha)); if (n >= 16 + 1 + 8 + 32 && header[16] != 0) { cfgref_present = 1; memcpy(&cfgref_tag, header + 17, 8); memcpy(cfgref_sha, header + 25, 32); } off_t consumed = (off_t)n; /* Legacy-format detection (Requirement 2.4, 5.1, 5.2): "does more data * exist" is the discriminator (design.md), not a flag bit -- a file * ending exactly at the legacy header is unambiguously a pre-feature * checkpoint. This is an expected rollout shape, not corruption: commit * the position it records (existing, preserved legacy behavior) and * report entities_restored=false so the caller falls back to normal * catch-up rather than treating this checkpoint as fully complete. */ if (st.st_size <= consumed) { close(fd); p->checkpoint_lsn = stored_lsn; p->current_lsn = stored_lsn; if (cfgref_present) { p->cfgref_present = 1; p->cfgref_tag = cfgref_tag; memcpy(p->cfgref_sha, cfgref_sha, sizeof(p->cfgref_sha)); } if (out_entities_restored) *out_entities_restored = false; return 0; } /* Entity-bearing checkpoint: format_version, then job frames, then queue * frames (design.md "Checkpoint File Layout"). */ uint32_t fmt = 0; if (read(fd, &fmt, sizeof(fmt)) != (ssize_t)sizeof(fmt)) { close(fd); return -EIO; } consumed += (off_t)sizeof(fmt); /* An unrecognized format_version is corruption or a future layout this * binary cannot read -- fail loud rather than silently treating it as * legacy (task 5.1 review note: a version mismatch must never be masked * as an expected legacy shape, or real corruption goes undetected). */ if (fmt != SNAPSHOT_FORMAT_VERSION) { close(fd); return -EINVAL; } int rc = load_snapshot_entities(fd, st.st_size, consumed, p, stored_lsn); close(fd); if (rc != 0) return rc; /* Requirement 2.3: p->*_lsn / p->cfgref_* untouched */ /* Single commit point (Requirement 2.1, 2.2): reached ONLY after every * job and queue frame restored successfully above. */ p->checkpoint_lsn = stored_lsn; p->current_lsn = stored_lsn; if (cfgref_present) { p->cfgref_present = 1; p->cfgref_tag = cfgref_tag; memcpy(p->cfgref_sha, cfgref_sha, sizeof(p->cfgref_sha)); } if (out_entities_restored) *out_entities_restored = true; return 0; } /* Public wrapper for load_snapshot — exposed via persist.h for ha.c (Task * 3.5c). load_snapshot() above now owns the full restore-before-advance * implementation (ha-entity-snapshots task 5.2): this wrapper is a thin * pass-through, since load_snapshot() already gates *out_entities_restored * and the p->checkpoint_lsn/current_lsn commit on entity-restore success * (Requirements 2.1-2.4, 5.1, 5.2). */ int persist_load_snapshot(persist_t *p, const char *path, uint64_t lsn, bool *out_entities_restored) { return load_snapshot(p, path, lsn, out_entities_restored); } /* P2-2: Standby WAL replay — apply a single replicated entry to in-memory state. * * No hooks fire on standbys; the hook system is empty until promotion. * * Returns 0 on success (including skip for checkpoint / unknown types), * negative errno on error (NULL args, oversized payload). */ int apply_wal_entry(persist_t* p, uint32_t type, uint64_t lsn, const void* payload, uint32_t payload_len) { if (!p) return -EINVAL; /* Reject oversized payloads BEFORE any allocation */ if (payload_len > MAX_WAL_PAYLOAD_BYTES) { ERROR("apply_wal_entry: payload_len %u exceeds MAX_WAL_PAYLOAD_BYTES %u", payload_len, MAX_WAL_PAYLOAD_BYTES); return -EMSGSIZE; } /* LOGICAL entity restore ONLY. The on-disk WAL header, the CRC chain * (p->last_crc), and current_lsn are owned exclusively by the physical * layer — persist_wal_write_replicated on the live replicated path, and * replay_wal/persist_recover on recovery. This function MUST NOT touch * them. (bug-778: it previously set p->last_crc from a synthetic crc=0 * header, breaking the on-disk CRC chain and cold-restart recovery.) */ /* WAL_CHECKPOINT (100) is a local-only marker — standbys never receive it * via replication. Skip silently. */ if (type == WAL_CHECKPOINT) return 0; /* Unknown types above WAL_ENTRY_TYPE_MAX: log WARN and skip for forward * compatibility — an older controller can boot on WAL written by a newer * one without hard-failing on unrecognised entry types. */ if (type > WAL_ENTRY_TYPE_MAX) { WARN("apply_wal_entry: unknown type %u > WAL_ENTRY_TYPE_MAX %u" " (lsn=%" PRIu64 ") — skipping for forward compatibility", type, WAL_ENTRY_TYPE_MAX, lsn); return 0; } /* Advance the current_lsn high-water (monotonic, idempotent). This is a * write-cursor high-water that is safe to bump from multiple paths. The * CRC chain link (p->last_crc) is deliberately NOT touched here — it has a * single owner: the physical writer (persist_wal_write_replicated) and * replay (persist_recover). (bug-778) */ pthread_mutex_lock(&p->lock); if (lsn + 1 > p->current_lsn) p->current_lsn = lsn + 1; pthread_mutex_unlock(&p->lock); if (!p->cluster) return 0; /* unit tests without a full cluster — skip entity restore */ switch (type) { case WAL_JOB_SUBMIT: /* Payload: Cap'n Proto serialized xce_job_t. Thread the entry LSN * through so the restored job is tagged for the egress watermark * (ha-convergence-recovery 3.3). */ restore_job(p->cluster, 0, payload, payload_len, lsn); break; case WAL_JOB_ARRAY_SUBMIT: /* job-array-single-wal-record task 3.3 (design.md "Array Apply * Expansion"): the array-record analog of WAL_JOB_SUBMIT's * restore_job() call directly above, dispatched from this SEPARATE, * still-live standby catch-up / replicated-apply seam so it can * never diverge from apply_committed_entry()'s WAL_JOB_ARRAY_SUBMIT * case (Req 3.1, 3.2, 4.1). Mirrors the single-job case's exact * minimalism: no xlist_t, no hook fire — this dispatcher is * deliberately hook-free (standby catch-up does not notify the * local scheduler/API server; that is the primary's job, done via * apply_committed_entry()). out_new_jobs is not needed here, so * NULL is passed (restore_job_array()'s out_new_jobs parameter is * optional — see test_rja_out_new_jobs_is_optional, * test_job_array_apply.c). The return value is intentionally * discarded, matching every other case in this switch today (task * 4.1 — Live Apply Resource Fence — is what later makes this * function's callers respect a negative rc; not this task's * boundary). */ restore_job_array(p->cluster, payload, payload_len, lsn, NULL); break; case WAL_JOB_FAIL: case WAL_JOB_STOP: case WAL_JOB_RESUME: case WAL_JOB_UPDATE: /* Payload: wal_jobstate_t {jobid, state} */ if (payload_len >= sizeof(wal_jobstate_t)) { const wal_jobstate_t* js = (const wal_jobstate_t*)payload; restore_job_state(p->cluster, js->jobid, js->state); } break; case WAL_JOB_PREASON: { /* bug-1692: split out of the shared group above -- mirrors * WAL_JOB_CKPT's/WAL_JOB_RUSAGE's splits (bug-1690/1691). Unlike * those, pend_reasons has a genuine "clear" operation * (do_clr_preason), so the payload beyond the wal_jobstate_t header * always starts with an explicit presence byte (0 = cleared, adopt * NULL; 1 = capnp JobPendReason bytes follow, adopt the decoded * value) on any entry written after this fix. A payload of exactly * sizeof(wal_jobstate_t) (no presence byte -- any WAL_JOB_PREASON * entry written before this fix) leaves pend_reasons untouched, * matching the pre-fix state-only behavior exactly. */ if (payload_len >= sizeof(wal_jobstate_t)) { const wal_jobstate_t* js = (const wal_jobstate_t*)payload; int has_field = 0; xce_job_preason_t* preason = NULL; if (payload_len > sizeof(wal_jobstate_t)) { uint8_t present = *((const uint8_t*)payload + sizeof(wal_jobstate_t)); if (present) { if (payload_len > sizeof(wal_jobstate_t) + 1) { preason = decode_jobpreason( (const uint8_t*)payload + sizeof(wal_jobstate_t) + 1, payload_len - sizeof(wal_jobstate_t) - 1); } /* present-but-failed-decode (corrupt/truncated trailing * bytes) must NOT be treated as a clear -- leave * pend_reasons untouched, matching every other decode * helper's failure fallback in this function. */ has_field = (preason != NULL); } else { has_field = 1; /* explicit clear -- adopt NULL */ } } restore_job_preason_state(p->cluster, js->jobid, js->state, has_field, preason); } break; } case WAL_JOB_CKPT: { /* bug-1691: split out of the shared group above -- mirrors * WAL_JOB_RUSAGE's split just below (bug-1690). This is the one * fact type in that (former) group whose payload can carry a * trailing NUL-terminated ckptinfo string (persist_job_hook_fun's * HOOK_DETAIL_JOB_CKPT payload shape) alongside the {jobid, state} * header, so a promoted standby's catch-up replay recovers the * checkpoint info too, not just state. A payload of exactly * sizeof(wal_jobstate_t) (no trailing bytes -- any WAL_JOB_CKPT * entry written before this fix, or fired while job.ckptinfo was * still NULL at write time) decodes no ckptinfo, matching the * pre-fix state-only behavior exactly. */ if (payload_len >= sizeof(wal_jobstate_t)) { const wal_jobstate_t* js = (const wal_jobstate_t*)payload; char* ckptinfo = NULL; if (payload_len > sizeof(wal_jobstate_t)) { ckptinfo = decode_ckptinfo( (const uint8_t*)payload + sizeof(wal_jobstate_t), payload_len - sizeof(wal_jobstate_t)); } restore_job_ckpt_state(p->cluster, js->jobid, js->state, ckptinfo); } break; } case WAL_JOB_RUSAGE: { /* bug-1690: split out of the shared group above -- mirrors * WAL_JOB_START's/WAL_JOB_FINISH's splits just below (bug-1687/ * bug-1689) and WAL_JOB_DISPATCH's split further below (bug-1678). * This is the one fact type in that group whose payload can carry * trailing Cap'n Proto JobRunusage bytes (persist_job_hook_fun's * WAL_JOB_RUSAGE payload shape, appended only when job.run_usage * was non-NULL at write time) alongside the {jobid, state} header, * so a promoted standby's catch-up replay recovers the * client-reported per-host CPU/mem usage too, not just state. A * payload of exactly sizeof(wal_jobstate_t) (no trailing bytes -- * any WAL_JOB_RUSAGE entry written before this fix, or one fired * while job.run_usage was still NULL) decodes no run_usage, * matching the pre-fix state-only behavior exactly. */ if (payload_len >= sizeof(wal_jobstate_t)) { const wal_jobstate_t* js = (const wal_jobstate_t*)payload; xce_job_runusage_t* runusage = NULL; if (payload_len > sizeof(wal_jobstate_t)) { runusage = decode_jobrunusage( (const uint8_t*)payload + sizeof(wal_jobstate_t), payload_len - sizeof(wal_jobstate_t)); } restore_job_rusage_state(p->cluster, js->jobid, js->state, runusage); } break; } case WAL_JOB_START: { /* bug-1687: split out of the shared group above -- mirrors * WAL_JOB_DISPATCH's split just below (bug-1678). This is the one * fact type in that group whose payload can carry a fixed * start_time PLUS trailing Cap'n Proto JobPIDInfo bytes * (persist_submit_job_start_fact's shape) alongside the {jobid, * state} header, so a promoted standby's catch-up replay recovers * the worker-confirmed pids/pgids/exdata too, not just state. A * payload of exactly sizeof(wal_jobstate_t) (no trailing bytes -- * any WAL_JOB_START entry written before this fix) decodes no * pidinfo, matching the pre-fix state-only behavior exactly. * * bug-1697: the wire-carried start_time field is decoded here only * to locate where the trailing JobPIDInfo bytes start -- it is no * longer passed to restore_job_start_state() at all. job.start_time * is now exclusively DISPATCH-owned (restore_job_dispatch_state() * below); leaving it untouched on this seam too prevents a stale * worker-ack-time value from ever clobbering the correct * dispatch-time one (see wal_types.h's WAL_JOB_START doc comment * for the full rationale). */ if (payload_len >= sizeof(wal_jobstate_t)) { const wal_jobstate_t* js = (const wal_jobstate_t*)payload; job_pidinfo_t* pidinfo = NULL; uint32_t extra = payload_len - (uint32_t)sizeof(wal_jobstate_t); if (extra >= sizeof(uint64_t)) { uint32_t pidinfo_len = extra - (uint32_t)sizeof(uint64_t); if (pidinfo_len > 0) { pidinfo = decode_pidinfo( (const uint8_t*)payload + sizeof(wal_jobstate_t) + sizeof(uint64_t), pidinfo_len); } } restore_job_start_state(p->cluster, js->jobid, js->state, pidinfo); } break; } case WAL_JOB_FINISH: { /* bug-1689: split out of the shared group above -- mirrors * WAL_JOB_START's split just above (bug-1687) / WAL_JOB_DISPATCH's * split just below (bug-1678). This is the one fact type in that * group whose payload can carry a fixed finish_time PLUS a fixed * exit_code (persist_submit_job_finish_fact's shape) alongside the * {jobid, state} header, so a promoted standby's catch-up replay * recovers the worker-confirmed exit_code and the true finish_time * too, not just state. A payload of exactly sizeof(wal_jobstate_t) * (no trailing bytes -- any WAL_JOB_FINISH entry written before this * fix) decodes neither, matching the pre-fix state-only behavior * exactly (finish_time/run_time/exit_info all left untouched on * this seam). */ if (payload_len >= sizeof(wal_jobstate_t)) { const wal_jobstate_t* js = (const wal_jobstate_t*)payload; uint64_t finish_time = 0; job_exitinfo_t* exit_info = NULL; uint32_t extra = payload_len - (uint32_t)sizeof(wal_jobstate_t); if (extra >= sizeof(finish_time)) { memcpy(&finish_time, (const uint8_t*)payload + sizeof(wal_jobstate_t), sizeof(finish_time)); uint32_t exitinfo_len = extra - (uint32_t)sizeof(finish_time); if (exitinfo_len > 0) { exit_info = decode_jobexitinfo( (const uint8_t*)payload + sizeof(wal_jobstate_t) + sizeof(finish_time), exitinfo_len); } } restore_job_finish_state(p->cluster, js->jobid, js->state, finish_time, exit_info); } break; } case WAL_JOB_DISPATCH: { /* bug-1678: split out of the shared group above -- this is the one * fact type whose payload can carry a trailing Cap'n Proto JobAlloc * (persist_submit_job_dispatch_fact's shape) alongside the * {jobid, state} header, so a promoted standby's catch-up replay * recovers the dispatch-time resource allocation too, not just the * state. A payload of exactly sizeof(wal_jobstate_t) (no trailing * bytes -- any WAL_JOB_DISPATCH entry written before this fix) * decodes no allocation, matching the pre-fix state-only behavior * exactly. * * bug-1697: when WAL_JOB_DISPATCH_HAS_START_TIME_FLAG is set in * js->_pad, a fixed uint64_t start_time immediately follows the * header, BEFORE the JobAlloc bytes -- decode and pass it through to * restore_job_dispatch_state() too (job.start_time is now * DISPATCH-owned, see wal_types.h's WAL_JOB_START doc comment for * the full rationale). The flag, not payload length, is the * discriminator -- see that same doc comment for why a length-only * test is ambiguous against JobAlloc's own variable length. An * entry with the flag clear (any WAL_JOB_DISPATCH entry written * before this fix) decodes the trailing bytes as JobAlloc only, * leaving start_time at its "not carried" sentinel (0) -- full * backward compatibility. */ if (payload_len >= sizeof(wal_jobstate_t)) { const wal_jobstate_t* js = (const wal_jobstate_t*)payload; xce_job_alloc_t* alloc = NULL; uint64_t start_time = 0; const uint8_t* alloc_ptr = (const uint8_t*)payload + sizeof(wal_jobstate_t); uint32_t alloc_len = payload_len - (uint32_t)sizeof(wal_jobstate_t); if ((js->_pad & WAL_JOB_DISPATCH_HAS_START_TIME_FLAG) != 0 && alloc_len >= sizeof(start_time)) { memcpy(&start_time, alloc_ptr, sizeof(start_time)); alloc_ptr += sizeof(start_time); alloc_len -= (uint32_t)sizeof(start_time); } if (alloc_len > 0) { alloc = decode_joballoc(alloc_ptr, alloc_len); } restore_job_dispatch_state(p->cluster, js->jobid, js->state, start_time, alloc); } break; } case WAL_WORKER_CTRL: /* Payload: [uint32_t detail][capnp worker] */ if (payload_len > WORKER_WAL_HDR_SIZE) worker_from_capnp(p->cluster, payload, payload_len); break; case WAL_QUEUE_CTRL: /* Payload: [wal_queuehdr_t][name NUL-terminated] */ if (payload_len > sizeof(wal_queuehdr_t)) { const wal_queuehdr_t* qh = (const wal_queuehdr_t*)payload; const char* name = (const char*)(payload + sizeof(wal_queuehdr_t)); restore_queue_state(p->cluster, name, qh->status); } break; case WAL_NOOP: /* Internal promotion no-op: empty payload, no entity restore, no * hooks. current_lsn is advanced by the physical layer (writer / * replay), not here — recognised explicitly rather than via default. */ break; case WAL_CONFIG_MARKER: /* Config-change marker (ha-term-safety task 4.2): a {version_tag, * sha256} reference, NEVER the config bytes (R7.4). It carries NO * entity state, so replay performs no restore here — the committed * configuration is DERIVED from the latest committed marker on demand * (config_committed_marker), and the bytes are obtained + hash-verified * out-of-band before activation (task 4.3). Recognised explicitly so * the per-LSN term index (built above this switch's caller) records its * term and on_get_entry_term / (term,lsn) reconcile treat it as an * ordinary log entry. */ break; default: break; } return 0; } /* raft-apply-decoupling task 2.2 — see persist.h for the full contract doc * (routing table, the job-state-family hook-mapping deviation, and what this * function deliberately does NOT own). Not yet wired into any live call * site — replay (2.3) and the live apply pump (6.1) will call this later. */ /* raft-apply-decoupling task 6.1 (Hazard 1 fix): thread-local "this call * stack is inside apply_committed_entry's own post-mutation notification" * guard. persist_job_hook_fun / persist_queue_hook_fun / persist_worker_hook_fun * are AFTER_EVENT subscribers that (for RETAIN/fact-shaped entry types) * durably append a NEW WAL entry every time they fire — the correct behavior * for their ORIGINAL, still-live purpose (job_ctrl's mark_job_update/ * mark_job_fail synchronous command path; the legacy queue/worker control * command handlers) which mutate live state and rely on this exact hook * firing to durably record that mutation. * * apply_committed_entry() now ALSO fires these same AFTER_EVENT hooks (job * facts always as the generic HOOK_DETAIL_JOB_UPDATE, per task 2.2's * documented wire-format deviation; queue/worker control at their native * detail) to deliver its own post-mutation NOTIFICATION — but that entry is * by definition already durably committed (it just got applied FROM the * WAL), so re-appending it would silently duplicate the entry on every live * apply. This is fixable, correctly, ONLY by provenance (which caller fired * the hook), NOT by hook `detail` value: HOOK_DETAIL_JOB_UPDATE in * particular is genuinely dual-origin (job_ctrl's SIGCONT/SIGSTOP path uses * the identical detail for its own, still-legitimate live append) — an * unconditional detail-based filter would silently break job_ctrl's suspend/ * resume durability instead of fixing anything. A shared persist_t-wide flag * would ALSO be wrong: it would race across threads (e.g. job_ctrl mutating * a DIFFERENT job on the iomgr thread while this thread's apply walk is * mid-hook for an unrelated fact) and could wrongly suppress or wrongly * permit an append depending on scheduling. A thread-local flag, scoped * narrowly around each xce_run_hooks() call this function makes, is race-free * by construction: it is only ever observed by hook subscribers invoked * SYNCHRONOUSLY on this exact call stack, from this exact function. */ static _Thread_local bool t_apply_seam_notifying = false; #ifdef XCE_TESTING /* config-upgrade-snapshot task 8.1 (Req 6.3): deterministic fault-injection * seam at the marker-apply-to-checkpoint-durable boundary, for later use by * a crash/restart regression (task 8.7): "force a controller crash/restart * between a configuration-change marker's commit and that member's own * snapshot write becoming durable, and assert materialization resumes * correctly from the marker on restart." When non-NULL AND it returns * nonzero for the marker's own (p, lsn), the WAL_CONFIG_MARKER case below * skips its call into persist_checkpoint_marker_triggered() entirely -- * simulating "the process crashes here, before the checkpoint write becomes * durable" (a real crash simply stops the process; the checkpoint call * never runs either way). The skip is deliberately routed through the SAME * rc!=0 handling an actual checkpoint failure already takes below (not a * new branch): last_applied_lsn is not advanced past this marker, so the * next kick/retry -- or, after a genuine restart, boot replay's independent * call into this same case (already proven idempotent by * test_marker_apply_retry_is_idempotent) -- resumes materialization from * the marker fresh, exactly matching Requirement 5.2's crash-safety * contract. NULL in production -- no behaviour change. Mirrors * ha_stream_body_read_fault_hook's shape (config_repl.c) and this same * spec's own xce_upgrade_marker_committed_hook placement convention * (defined here at file scope, declared extern in whichever test file * needs it -- never in a header). */ int (*persist_marker_checkpoint_fault_hook)(persist_t *p, uint64_t marker_lsn) = NULL; #endif int apply_committed_entry(persist_t* p, uint32_t wal_type, uint64_t lsn, const void* payload, uint32_t len) { if (!p) return -EINVAL; if (!p->cluster) return 0; /* unit tests without a full cluster — nothing to dispatch into */ cluster_t* c = p->cluster; switch (wal_type) { case WAL_JOB_SUBMIT: { /* Return value distinguishes a genuine creation from an idempotent * no-op (bug-1419): 1 = restore_job() actually constructed a new * job, 0 = the jobid already existed (duplicate WAL entry — see * below), negative = a real error. replay_wal()'s new-job tally * relies on this distinction; the live apply pump discards it. * * A duplicate WAL_JOB_SUBMIT for the same jobid is an EXPECTED, * harmless occurrence, not a bug: a client's local commit-timeout * retry can append a second entry for the same jobid while the * first is still in flight during a quorum-loss window; the WAL * entry that timed out is never retracted, so if it later achieves * majority commit anyway, both entries end up committed. restore_job * ()'s own -EEXIST here is what makes that safe to replay twice. */ int rc = restore_job(c, 0, payload, len, lsn); if (rc == -EEXIST) return 0; /* idempotent re-apply (replay / resync) — no hook */ if (rc < 0) return rc; struct capn jc_ctx; capn_init_mem(&jc_ctx, (const uint8_t*)payload, len, 0); Job_ptr jp; jp.p = capn_getp(capn_root(&jc_ctx), 0, 1); struct Job cj; read_Job(&cj, jp); uint64_t jobid = cj.jobid; capn_free(&jc_ctx); job_t* j = NULL; lock(c->jobmgr, RTTI_LOCK_READ); key_find(c->jobmgr, (void*)jobid, (void**)&j); unlock(c->jobmgr); if (!j) return 1; /* defensive: should not happen after a successful restore */ void* jptr = j; xlist_t jlist = { .count = 1, .caps = 1, .delta = 1, .data = &jptr }; t_apply_seam_notifying = true; xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_NEW, c, &jlist); t_apply_seam_notifying = false; return 1; } case WAL_JOB_ARRAY_SUBMIT: { /* job-array-single-wal-record task 3.2 (design.md "Array Apply * Expansion"): the array-record analog of WAL_JOB_SUBMIT's * restore_job() above, dispatched from this SAME shared seam so live * apply, boot replay (both callers of this function), and (task * 3.3) standby catch-up can never diverge in how they expand a * committed array record (Req 3.1, 3.2). * * restore_job_array()'s return generalizes restore_job()'s * three-way contract to N elements: a positive count = the number * of genuinely NEW elements this call constructed (this is exactly, * and only, what the hook fire below carries — Req 3.4); -EEXIST = * every index in the record already existed (idempotent * whole-record no-op, Req 3.3) — no hook; a negative errno = a * genuine construction/resource-exhaustion failure (Req 9.4) — * propagated so persist_apply_pump_run() (task 4.1) can halt and * fence on it rather than silently skipping it. **Task 7.1 * (bug-1443) update**: a negative return no longer unconditionally * means "no hook" — restore_job_array()'s bulk-insert-phase residual * gap can leave a genuinely, permanently inserted prefix in * out_new_jobs even on failure (job.c's doc comment, "Residual * atomicity gap"); see below. */ xlist_t out_new_jobs; memset(&out_new_jobs, 0, sizeof(out_new_jobs)); int rc = restore_job_array(c, payload, len, lsn, &out_new_jobs); if (rc == -EEXIST) return 0; /* idempotent re-apply (replay / resync) — no hook */ if (rc < 0) { /* job-array-single-wal-record task 7.1 (bug-1443 partial-prefix * fix): out_new_jobs is untouched (count == 0, from the memset * above) for a reservation or scratch-build-phase failure — * nothing was ever inserted, so no hook fires, unchanged from * before this task. But for the narrower bulk-insert-phase * residual gap, out_new_jobs now carries a non-empty, genuinely * and permanently inserted prefix (job.c) that must still get * its one HOOK_DETAIL_JOB_NEW fire — a job that is really in * c->jobmgr must never be silently un-notified. The overall * apply is still, correctly, reported as failed via the * negative rc returned below, so persist_apply_pump_run() halts * the LSN and the Live Apply Resource Fence engages for the * un-constructed remainder (Req 2.3, 9.4, 9.6). */ if (out_new_jobs.count > 0) { t_apply_seam_notifying = true; xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_NEW, c, &out_new_jobs); t_apply_seam_notifying = false; } xlist_fini(&out_new_jobs); return rc; } /* Exactly ONE fire carrying every newly created element in one * list — never once per element (Req 3.4). out_new_jobs is already * the precise newly-created set restore_job_array() built; no * re-derivation via key_find needed. */ t_apply_seam_notifying = true; xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_NEW, c, &out_new_jobs); t_apply_seam_notifying = false; /* Only the list's own backing array is released here — ownership of * every job_t* it holds already transferred to c->jobmgr inside * restore_job_array()'s bulk-insert pass, mirroring * test_job_array_apply.c's own xlist_fini(&out) usage after a * successful call. */ xlist_fini(&out_new_jobs); return rc; } case WAL_JOB_START: case WAL_JOB_FINISH: case WAL_JOB_STOP: case WAL_JOB_RESUME: case WAL_JOB_RUSAGE: case WAL_JOB_CKPT: case WAL_JOB_PREASON: case WAL_JOB_DISPATCH: { /* Every job-state fact type in this group shares one wire shape — * wal_jobstate_t {jobid, state} — and therefore one guarded core. * Their payloads are deterministic/state-independent (a START is * always JOB_STAT_RUN, a FINISH's outcome depends only on * exit_code), so an absolute `state` baked at receipt time never * goes stale — unlike WAL_JOB_FAIL/WAL_JOB_UPDATE below, which have * their own dedicated cases. See persist.h's doc comment on * apply_committed_entry for the full routing table and why this * does not dispatch to the richer type-specific apply_job_{start, * finish}_transition() cores (their parameter structs are not * present on this wire format) — start_time/finish_time/run_time * are instead stamped directly below, keyed on wal_type, since * apply_job_update_transition() only ever touches `state`. * WAL_JOB_DISPATCH joins this group for the same reason: it is * synchronously computed at dispatch-commit time under the job's * own write lock, with no other in-flight fact for the same job * (it is the first transition after PEND) — so it never needs * WAL_JOB_FAIL/WAL_JOB_UPDATE's deferred, apply-time-recomputed * handling. */ if (len < sizeof(wal_jobstate_t)) return 0; /* undersized payload — skip, mirrors apply_wal_entry */ const wal_jobstate_t* js = (const wal_jobstate_t*)payload; job_t* j = NULL; lock(c->jobmgr, RTTI_LOCK_READ); key_find(c->jobmgr, (void*)js->jobid, (void**)&j); unlock(c->jobmgr); if (!j) return -ENOENT; lock(j, RTTI_LOCK_WRITE); job_transition_result_t result = apply_job_update_transition(j, (int)js->state); /* bug-1690: WAL_JOB_RUSAGE carries the client-reported run_usage * data independent of the job's state transition -- a rusage * report almost never changes state (the job stays RUN across * every repeat report), so gating this adoption on * result==JOB_TRANSITION_APPLIED the way START/FINISH/DISPATCH do * below would make it dead code for that realistic, common case: * every report after the first would see JOB_TRANSITION_NOOP_ * DUPLICATE and the decode would never run. Decode/adopt * unconditionally, before the transition-guarded block below, * whether or not the state actually changed. A payload of exactly * sizeof(wal_jobstate_t) (no trailing bytes -- any WAL_JOB_RUSAGE * entry written before this fix, or one fired while job.run_usage * was still NULL at write time) decodes no run_usage, leaving it * untouched, matching the pre-fix behavior exactly. * * MERGE, not free-and-replace: on the primary, this exact branch * re-runs via persist_apply_pump_run()'s live pump moments after * update_job_rusages() already live-mutated j->job.run_usage in * place (with an xstring-backed host_name, makeString(), * src/lib/xstring.c) -- j->job.run_usage here can genuinely be that * SAME live structure, not just a replay artifact. Freeing it via * the capnp-generated free_JobRunusage_ptr() (written for plain * malloc'd, capnp-decoded Text fields) would free a non-malloc'd, * RTTI-managed pointer -- heap corruption. merge_job_runusage() * (job.c) never frees an existing host_name; see its own doc * comment for the full rationale. */ if (wal_type == WAL_JOB_RUSAGE && len > sizeof(wal_jobstate_t)) { xce_job_runusage_t* decoded = decode_jobrunusage( (const uint8_t*)payload + sizeof(wal_jobstate_t), len - sizeof(wal_jobstate_t)); if (decoded != NULL) merge_job_runusage(&(j->job.run_usage), decoded); } /* bug-1691: WAL_JOB_CKPT mirrors WAL_JOB_RUSAGE's unconditional- * adopt pattern immediately above -- see that block's comment for * the full rationale. A checkpoint report never changes job.state * (on_job_ckpt(), job.c, only ever sets j->job.ckptinfo -- js->state * is simply the job's UNCHANGED current state, captured verbatim by * persist_job_hook_fun's HOOK_DETAIL_JOB_CKPT case), so gating this * adoption on result==JOB_TRANSITION_APPLIED the way START/FINISH/ * DISPATCH do below would make it dead code: apply_job_update_ * transition() sees the wire state == the job's current state and * reports JOB_TRANSITION_NOOP_DUPLICATE every time, so the gated * block below would never run. Decode/adopt unconditionally, before * the transition-guarded block. A payload of exactly * sizeof(wal_jobstate_t) (no trailing bytes -- any WAL_JOB_CKPT * entry written before this fix, or one fired while job.ckptinfo * was still NULL at write time) decodes no ckptinfo, matching the * pre-fix behavior exactly. * * j->job.ckptinfo is xstring-backed (makeString(), xstring.c), NOT * a plain malloc'd/capnp-decoded string -- release the prior value * via dereference() (the RTTI verb), never free()/FREEUP, mirroring * on_job_ckpt()'s own dereference(j->job.ckptinfo) call. Unlike * WAL_JOB_RUSAGE's run_usage, a free-and-replace here is safe (no * merge needed): dereference() is the correct RTTI release call * regardless of whether j->job.ckptinfo happens to be the SAME live * xstring the primary's own live apply pump re-applies moments * after on_job_ckpt() already set it -- it decrements a refcount * rather than raw free()ing a non-owned pointer, so re-applying the * identical fact is a harmless extra alloc/dereference cycle, not a * hazard. decode_ckptinfo()'s result is a transient decode buffer * (makeString() copies its input into its own allocation), so it is * always freed here after use, whether or not adoption happened. */ if (wal_type == WAL_JOB_CKPT && len > sizeof(wal_jobstate_t)) { char* decoded = decode_ckptinfo( (const uint8_t*)payload + sizeof(wal_jobstate_t), len - sizeof(wal_jobstate_t)); if (decoded != NULL) { if (j->job.ckptinfo != NULL) dereference(j->job.ckptinfo); j->job.ckptinfo = makeString(decoded); free(decoded); } } /* bug-1692: WAL_JOB_PREASON mirrors WAL_JOB_CKPT's/WAL_JOB_RUSAGE's * unconditional-adopt pattern immediately above -- set_preason()/ * do_clr_preason() (job.c) never touch job.state, so gating * adoption on result==JOB_TRANSITION_APPLIED would make it dead * code: js->state is always the job's unchanged current state * (persist_job_hook_fun's HOOK_DETAIL_SETPREASON/CLRPREASON arm * captures it verbatim), so apply_job_update_transition() always * reports JOB_TRANSITION_NOOP_DUPLICATE here. Decode/adopt * unconditionally, before the transition-guarded block below. * * Unlike CKPT/RUSAGE, a presence byte (not "trailing bytes exist or * not") disambiguates a genuine CLEAR (do_clr_preason) from an * old-format payload -- see decode_jobpreason()'s/ * restore_job_preason_state()'s doc comments and WAL_JOB_PREASON's * own doc comment in wal_types.h for the full rationale. `present == * 0` (explicit clear) or a successful decode both trigger adoption; * a present-but-failed decode (corrupt/truncated trailing bytes) * leaves the existing value untouched, matching every other type's * decode-failure fallback in this function. * * pend_reasons is plain calloc()/strdup()-backed throughout * (verified via erase_job_preason()'s FREEUP-only release and every * add_*_preason() constructor in src/lib/preason.c, plus the * capnp-generated decode_HostBasedReason/decode_ResourceBasedReason * using STRING_DUP+calloc), NOT xstring/RTTI-backed like * ckptinfo/rusage's host_name -- so a straightforward * free-and-replace via free_JobPendReason_ptr() is safe here, no * merge-not-replace treatment needed (unlike bug-1690's * WAL_JOB_RUSAGE). */ if (wal_type == WAL_JOB_PREASON && len > sizeof(wal_jobstate_t)) { uint8_t present = *((const uint8_t*)payload + sizeof(wal_jobstate_t)); xce_job_preason_t* decoded = NULL; if (present && len > sizeof(wal_jobstate_t) + 1) { decoded = decode_jobpreason( (const uint8_t*)payload + sizeof(wal_jobstate_t) + 1, len - sizeof(wal_jobstate_t) - 1); } if (present == 0 || decoded != NULL) { if (j->job.pend_reasons != NULL) free_JobPendReason_ptr(&(j->job.pend_reasons)); j->job.pend_reasons = decoded; /* NULL (cleared) or decoded */ } } /* bug-1697: WAL_JOB_START's pidinfo (worker-reported pids/pgids/ * exdata) must be decoded/adopted UNCONDITIONALLY, mirroring * WAL_JOB_RUSAGE/WAL_JOB_CKPT/WAL_JOB_PREASON immediately above -- * dispatch_a_job_() now sets job.state = JOB_STAT_RUN synchronously * at DISPATCH time (scheduler-dispatch-commit task 3.1), so by the * time this WAL_JOB_START fact applies, apply_job_update_ * transition() above ALWAYS finds state already == RUN and returns * JOB_TRANSITION_NOOP_DUPLICATE -- gating pidinfo adoption on * result==JOB_TRANSITION_APPLIED (as the pre-fix code did, inside * the block below) made it dead code for EVERY job, live, forever. * job.start_time is DELIBERATELY not touched here at all any more: * WAL_JOB_DISPATCH (further below) is now the sole owner of * start_time, and this fact's own wire-carried start_time field * (still written by persist_submit_job_start_fact() for wire-layout * continuity) is intentionally skipped over -- never read, never * adopted -- so it can never clobber the correct dispatch-time * value once WAL_JOB_DISPATCH's own entry has already applied. A * payload of exactly sizeof(wal_jobstate_t) (no trailing bytes at * all -- any WAL_JOB_START entry written before bug-1687) decodes * no pidinfo, matching the pre-1687 state-only behavior exactly. */ /* bug-1731: pidinfo_just_decoded is a call-scoped flag, set ONLY * where the decode-and-adopt block immediately below genuinely * decodes a new job_pidinfo_t THIS call -- deliberately NOT the * same thing as `j->job.pidinfo != NULL` (which could already be * true from an earlier apply of this same job, and would wrongly * re-trigger the HOOK_DETAIL_JOB_START fire below for an unrelated * later transition). This flag gates that fire; see the fire site * near the end of this case for why it must NOT be gated on * `result == JOB_TRANSITION_APPLIED` the way HOOK_DETAIL_JOB_UPDATE * is. */ bool pidinfo_just_decoded = false; if (wal_type == WAL_JOB_START && len >= sizeof(wal_jobstate_t) + sizeof(uint64_t)) { uint32_t pidinfo_len = (uint32_t)len - (uint32_t)sizeof(wal_jobstate_t) - (uint32_t)sizeof(uint64_t); if (pidinfo_len > 0) { job_pidinfo_t* decoded = decode_pidinfo( (const uint8_t*)payload + sizeof(wal_jobstate_t) + sizeof(uint64_t), pidinfo_len); if (decoded != NULL) { if (j->job.pidinfo != NULL) free_JobPIDInfo_ptr(&(j->job.pidinfo)); j->job.pidinfo = decoded; /* bug-1731 belt-and-suspenders: explicitly stamp jobid * on the decoded struct rather than trusting the capnp * decode alone -- cheap and always correct, and this is * exactly the field sched_job_start_fun (sched.c) * unconditionally dereferences (info->jobid, no NULL * guard) the moment HOOK_DETAIL_JOB_START fires below; * see persist.h's apply_committed_entry doc comment for * the full NULL-safety history that made this detail * permanently dead code until now. */ decoded->jobid = js->jobid; pidinfo_just_decoded = true; } } } if (result == JOB_TRANSITION_APPLIED) { /* apply_job_update_transition() only carries {jobid, state} on * this wire shape (see the doc comment above) and never touches * finish_time/run_time itself — stamp them here, keyed on * wal_type, mirroring what the richer apply_job_finish_ * transition() core does for its own (currently unreachable on * this dispatch path) callers. WAL_JOB_START no longer has a * branch here (bug-1697) -- its only apply-time-relevant field, * pidinfo, is decoded/adopted unconditionally above; start_time * is DISPATCH-owned and is never touched by this fact type. */ if (wal_type == WAL_JOB_FINISH) { /* bug-1689: a WAL_JOB_FINISH payload can carry a fixed * finish_time plus a fixed exit_code beyond the * wal_jobstate_t header (persist_submit_job_finish_fact's * shape, mirroring bug-1687's WAL_JOB_START trailing-fields * handling above). Decode and adopt the wire-carried * finish_time/exit_code when present; a payload of exactly * sizeof(wal_jobstate_t) (no trailing bytes -- any * WAL_JOB_FINISH entry written before this fix) * intentionally falls through to the old fabricated- * time(NULL)/exit_info-untouched behavior for full backward * compatibility. run_time is always re-derived from * finish_time - start_time either way. */ if (len >= sizeof(wal_jobstate_t) + sizeof(uint64_t)) { uint64_t wire_finish_time; memcpy(&wire_finish_time, (const uint8_t*)payload + sizeof(wal_jobstate_t), sizeof(wire_finish_time)); j->job.finish_time = (time_t)wire_finish_time; uint32_t exitinfo_len = len - (uint32_t)sizeof(wal_jobstate_t) - (uint32_t)sizeof(uint64_t); if (exitinfo_len > 0) { job_exitinfo_t* decoded = decode_jobexitinfo( (const uint8_t*)payload + sizeof(wal_jobstate_t) + sizeof(uint64_t), exitinfo_len); if (decoded != NULL) { if (j->job.exit_info != NULL) free(j->job.exit_info); j->job.exit_info = decoded; } } } else { j->job.finish_time = time(NULL); } j->job.run_time = (int)(j->job.finish_time - j->job.start_time); } else if (wal_type == WAL_JOB_DISPATCH && len > sizeof(wal_jobstate_t)) { /* bug-1678: trailing bytes beyond the header carry the * scheduler's resource allocation (persist_submit_job_ * dispatch_fact's payload shape) -- decode and adopt it so * a restart replay recovers EXEC_HOST, not just STATUS. A * payload of exactly sizeof(wal_jobstate_t) (no trailing * bytes -- e.g. any WAL_JOB_DISPATCH entry written before * this fix) intentionally falls through to the same * state-only behavior every other type in this group gets. * * bug-1697: when WAL_JOB_DISPATCH_HAS_START_TIME_FLAG is set * in js->_pad, a fixed uint64_t start_time immediately * follows the header, BEFORE the JobAlloc bytes -- decode * and adopt it here too (job.start_time is now DISPATCH- * owned, see wal_types.h's WAL_JOB_START doc comment for the * full rationale). The flag, not payload length, decides * whether the field is present -- a length-only test cannot * distinguish an old-format entry with a large alloc from a * new-format entry with a small one. An entry with the flag * clear (any WAL_JOB_DISPATCH entry written before this fix) * skips straight to decoding the trailing bytes as JobAlloc * only, leaving start_time untouched -- full backward * compatibility. */ const uint8_t* alloc_ptr = (const uint8_t*)payload + sizeof(wal_jobstate_t); size_t alloc_len = len - sizeof(wal_jobstate_t); if ((js->_pad & WAL_JOB_DISPATCH_HAS_START_TIME_FLAG) != 0 && alloc_len >= sizeof(uint64_t)) { uint64_t wire_start_time; memcpy(&wire_start_time, alloc_ptr, sizeof(wire_start_time)); j->job.start_time = (time_t)wire_start_time; alloc_ptr += sizeof(uint64_t); alloc_len -= sizeof(uint64_t); } if (alloc_len > 0) { xce_job_alloc_t* decoded = decode_joballoc(alloc_ptr, alloc_len); if (decoded != NULL) { if (j->job.allocs != NULL) free_JobAlloc_ptr(&(j->job.allocs)); j->job.allocs = decoded; } } } } unlock(j); /* bug-1731: this gate used to be a flat `result != APPLIED` early * return (no hook at all on a rejected-backward/no-op-duplicate * transition). It now also lets a genuinely-just-decoded * WAL_JOB_START pidinfo through even when `result` is a confirmed * NOOP_DUPLICATE -- which, per bug-1697, is what EVERY real job's * WAL_JOB_START fact applies as in production (dispatch_a_job_() * already set state=RUN synchronously before WAL_JOB_START ever * applies). Gating the new HOOK_DETAIL_JOB_START fire on * result==JOB_TRANSITION_APPLIED, like HOOK_DETAIL_JOB_UPDATE below, * would make it dead code for every real job -- exactly the bug * being fixed here for HOOK_DETAIL_JOB_EXIT. A genuine idempotent * re-apply of an already-applied WAL_JOB_FINISH still hits this same * early return and fires NOTHING (pidinfo_just_decoded is only ever * set inside the WAL_JOB_START branch above, so it stays false for * every other wal_type in this group). */ if (result != JOB_TRANSITION_APPLIED && !pidinfo_just_decoded) return 0; /* rejected-backward / no-op-duplicate, and no pidinfo * just decoded either — nothing to notify */ /* Hazard 1 guard (see t_apply_seam_notifying's doc comment above): * HOOK_DETAIL_JOB_UPDATE is dual-origin — job_ctrl's mark_job_update * (SIGCONT/SIGSTOP) fires the identical detail for its own live, * still-legitimate append via persist_job_hook_fun. Without this * guard, EVERY fact this apply pump applies would re-append itself * as a duplicate, mis-typed WAL_JOB_UPDATE entry the moment the live * pump (task 6.1) starts calling this function. Single bracket * wraps ALL THREE possible fires below (bug-1731 adds JOB_START/ * JOB_EXIT alongside the pre-existing JOB_UPDATE) — do not narrow or * duplicate this guard per-fire. */ t_apply_seam_notifying = true; /* bug-1731: HOOK_DETAIL_JOB_START -- fires independent of `result`; * see pidinfo_just_decoded's own doc comment at its declaration * above, and the gate comment immediately above this bracket. Args * mirror job.c's own HOOK_BEFORE_EVENT fire site for this exact * detail (record_job_start_fact, job.c) — (c, info, job) — so every * subscriber's va_arg reads stay consistent regardless of which * situation/call site fired it. */ if (pidinfo_just_decoded) { xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_START, c, j->job.pidinfo, j); } if (result == JOB_TRANSITION_APPLIED) { /* bug-1731: HOOK_DETAIL_JOB_EXIT for WAL_JOB_FINISH -- this is * the PRIMARY fix for bug-1731 (apiserver's OnJobFinished was * permanently starved of data; see persist.h's * apply_committed_entry doc comment for the full history of why * this detail was dead). exit_code is read from j->job.exit_info * (decoded/adopted just above when the wire payload carried * trailing bytes, bug-1689) — an old-format payload with no * trailing bytes leaves exit_info NULL, so exit_code falls back * to 0 rather than reading a dangling/garbage value. Args mirror * job.c's own HOOK_BEFORE_EVENT fire site (record_job_finish_ * fact, job.c) — (c, finish, job). */ if (wal_type == WAL_JOB_FINISH) { xce_jobfinish_t finish_info = { .jobid = js->jobid, .exit_code = (j->job.exit_info != NULL) ? j->job.exit_info->exitcode : 0, }; xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_EXIT, c, &finish_info, j); } xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_UPDATE, c, j); } t_apply_seam_notifying = false; return 0; } case WAL_JOB_FAIL: { /* task 6.1 round 2 (Req 3.3/7.1-7.3, 11.3) fix for a job-state * corruption bug: unlike the deterministic group above, a FAIL's * resulting state depends on the job's CURRENT state (PERR merges * onto whatever terminal state already holds). Dispatch on _pad * (see wal_types.h's doc comment on wal_jobstate_t._pad): * - WAL_JOB_FAIL_STAGE_FLAG set: a deferred fail-fact * (record_job_fail_fact, job.c) — reconstruct a minimal * job_failinfo_t from {jobid, stage} and call the REAL * apply_job_fail_transition() against the job's ACTUAL CURRENT * state at apply time, never a receipt-time snapshot. * - _pad == 0: the legacy live path (job_ctrl's mark_job_fail) — * `state` is already the correct, current, absolute value, * computed synchronously under the same lock as the mutation * (no staleness window) — an absolute overwrite is safe and * unchanged. */ if (len < sizeof(wal_jobstate_t)) return 0; /* undersized payload — skip, mirrors apply_wal_entry */ const wal_jobstate_t* js = (const wal_jobstate_t*)payload; job_t* j = NULL; lock(c->jobmgr, RTTI_LOCK_READ); key_find(c->jobmgr, (void*)js->jobid, (void**)&j); unlock(c->jobmgr); if (!j) return -ENOENT; lock(j, RTTI_LOCK_WRITE); job_transition_result_t result; if (js->_pad & WAL_JOB_FAIL_STAGE_FLAG) { job_failinfo_t failinfo = { .jobid = js->jobid, .stage = js->_pad & ~WAL_JOB_FAIL_STAGE_FLAG, .code = 0, .detail = NULL, }; result = apply_job_fail_transition(j, &failinfo); } else { result = apply_job_update_transition(j, (int)js->state); } unlock(j); if (result != JOB_TRANSITION_APPLIED) return 0; /* rejected-backward / no-op-duplicate — no hook */ /* Hazard 1 guard — see the identical guard above. */ t_apply_seam_notifying = true; xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_UPDATE, c, j); t_apply_seam_notifying = false; return 0; } case WAL_JOB_UPDATE: { /* task 6.1 round 2 (Req 3.3/7.1-7.3, 11.3): same class of fix as * WAL_JOB_FAIL above. Dispatch on _pad: * - WAL_JOB_UPDATE_POSTDONE_MARK: a deferred postdone fact * (record_job_postdone_fact, job.c) — call * apply_job_postdone_transition(), which re-derives the * postdone merge (conditional EXIT backstop + OR JOB_STAT_PDONE) * fresh against the job's ACTUAL CURRENT state at apply time. * - _pad == 0: the legacy live path (job_ctrl's mark_job_update, * SIGCONT/SIGSTOP) — `state` is already correct/absolute, * computed synchronously under the same lock as the mutation — * an absolute overwrite is safe and unchanged. */ if (len < sizeof(wal_jobstate_t)) return 0; /* undersized payload — skip, mirrors apply_wal_entry */ const wal_jobstate_t* js = (const wal_jobstate_t*)payload; job_t* j = NULL; lock(c->jobmgr, RTTI_LOCK_READ); key_find(c->jobmgr, (void*)js->jobid, (void**)&j); unlock(c->jobmgr); if (!j) return -ENOENT; lock(j, RTTI_LOCK_WRITE); job_transition_result_t result = (js->_pad == WAL_JOB_UPDATE_POSTDONE_MARK) ? apply_job_postdone_transition(j) : apply_job_update_transition(j, (int)js->state); unlock(j); if (result != JOB_TRANSITION_APPLIED) return 0; /* rejected-backward / no-op-duplicate — no hook */ /* Hazard 1 guard — see the identical guard above. */ t_apply_seam_notifying = true; xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_UPDATE, c, j); t_apply_seam_notifying = false; return 0; } case WAL_QUEUE_CTRL: { if (len <= sizeof(wal_queuehdr_t)) return 0; /* undersized payload — skip, mirrors apply_wal_entry */ const wal_queuehdr_t* qh = (const wal_queuehdr_t*)payload; const char* name = (const char*)((const uint8_t*)payload + sizeof(wal_queuehdr_t)); /* restore_queue_state's -ENOENT ("queue not yet built from config") * and -EINVAL (bad name) are both benign skips here, matching * apply_wal_entry's existing behavior of never fencing on this * entry type's restore outcome. */ if (restore_queue_state(c, name, qh->status) < 0) return 0; queue_t* q = NULL; lock(c->qmgr, RTTI_LOCK_READ); key_find(c->qmgr, (void*)name, (void**)&q); unlock(c->qmgr); if (!q) return 0; /* Same double-append class as the job-fact guard above: the legacy * queue-control command handler still mutates then relies on this * exact AFTER_EVENT firing to durably append via * persist_queue_hook_fun (WAL_QUEUE_CTRL is unconditionally * appended on AFTER_EVENT there, not gated on WAL_MODE_WAIT like the * job-submit case is) — guard so this live-pump re-notify does not * re-append an already-committed entry. */ t_apply_seam_notifying = true; xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE, HOOK_AFTER_EVENT, HOOK_DETAIL_QUEUE_UPDATE, c, q); t_apply_seam_notifying = false; return 0; } case WAL_WORKER_CTRL: { if (len <= WORKER_WAL_HDR_SIZE) return 0; /* undersized payload — skip, mirrors apply_wal_entry */ uint32_t hook_detail; memcpy(&hook_detail, payload, sizeof(hook_detail)); int rc = worker_from_capnp(c, payload, len); if (rc == -EEXIST) return 0; /* idempotent re-apply (replay / resync) — no hook */ if (rc < 0) return rc; /* Recover the worker's name via a cheap raw capnp field read * (read_Worker's capn_get_text is a borrowed, NUL-terminated pointer * into the decode buffer — no allocation, unlike decode_Worker) so * the just-restored worker can be looked up for the notification * hook without changing worker_from_capnp's own contract. */ const uint8_t* wpayload = (const uint8_t*)payload + WORKER_WAL_HDR_SIZE; size_t wlen = len - WORKER_WAL_HDR_SIZE; struct capn wc_ctx; capn_init_mem(&wc_ctx, wpayload, wlen, 0); Worker_ptr wp; wp.p = capn_getp(capn_root(&wc_ctx), 0, 1); struct Worker cw; read_Worker(&cw, wp); worker_t* w = NULL; lock(c->workermgr, RTTI_LOCK_READ); key_find(c->workermgr, (void*)cw.name.str, (void**)&w); unlock(c->workermgr); capn_free(&wc_ctx); if (!w) return 0; /* defensive: should not happen after a successful restore */ /* Same double-append class as the job-fact/queue guards above: * persist_worker_hook_fun unconditionally appends on AFTER_EVENT for * every worker detail (no WAL_MODE_WAIT gate), and the legacy worker * join/update/dead command paths still rely on that firing for their * own live durability — guard the whole re-notify switch so this * live-pump call never re-appends an already-committed entry. */ t_apply_seam_notifying = true; switch (hook_detail) { case HOOK_DETAIL_WORKER_JOIN: xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER, HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_JOIN, c, w->host.name, w); break; case HOOK_DETAIL_WORKER_UPDATE: xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER, HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_UPDATE, c, w); break; case HOOK_DETAIL_WORKER_DEAD: xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER, HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_DEAD, c, w); break; default: break; /* unrecognized discriminator — mutation already applied, * notification skipped rather than guessed at */ } t_apply_seam_notifying = false; /* bug-1427: propagate rc (1 = genuinely created a new worker via * worker_from_capnp()'s restore_worker() path, 0 = an UPDATE/DEAD * applied to an already-existing worker) instead of the previous * hardcoded 0 — replay_wal()'s new-worker tally below relies on * this distinction, exactly mirroring WAL_JOB_SUBMIT's apply_rc. * Every existing caller only checks sign / -EEXIST, never the * exact non-negative value, so this is a safe, additive change. */ return rc; } case WAL_CONFIG_MARKER: { /* config-upgrade-snapshot task 5 (Req 1.1, 1.5, 1.8): force an * immediate, deterministic checkpoint at exactly this marker's own * (term, lsn) boundary, materializing the upgrade snapshot the * moment every member applies the marker -- primary online-apply, * standby online-apply, and boot/restart replay alike, since this * one unified apply seam is exactly what all three route through * (design.md "Architecture" / "Key simplification found during * design": no new leader-specific "resume" logic is needed either, * since a newly-elected leader satisfies this the same way, by * continuing to apply its own committed log). * * Deliberately does NOT touch entity interpretation/re-homing * (lost_and_found etc.) -- Req 1.8 requires apply and activation to * stay separate steps; this case's only effect is materialization, * identical in kind to any other checkpoint. * * persist_checkpoint_marker_triggered() (not a bare * persist_checkpoint() call) is required here to avoid two real * hazards a naive direct call would hit: * 1. Lock reentrancy: persist_apply_pump_run() (the live pump's * call site for online-apply, both primary and standby) holds * apply_pump_lock across this entire dispatch call; * persist_checkpoint()'s own body re-acquires that same, * non-recursive mutex, which would self-deadlock. * persist_checkpoint_marker_triggered() never locks * apply_pump_lock itself -- see its own doc comment for why * that is safe under both production call sites. * 2. Stale checkpoint_lsn: both live-apply and replay advance * last_applied_lsn AFTER this dispatch call returns, never * before (persist_apply_pump_run / replay_wal), so reading * last_applied_lsn from inside this case would capture the * PRECEDING entry's lsn, not the marker's own, mislabeling the * checkpoint. `lsn` (this function's own parameter) is * already the marker's exact position, so it is passed * through explicitly instead of re-derived. */ /* bug-1469 fix (live HA soak, config-upgrade-snapshot task 8.6): * unconditionally re-run config activation right here, BEFORE the * forced checkpoint below, so EVERY member -- primary, standby, and * any future newly-elected leader -- is guaranteed to have * refreshed c->myconf (and the runtime queue/host tables * load_primary_config provisions from it) from disk BEFORE the * checkpoint writer walks that same runtime state to build the * queue-entity list. Before this call existed at all, * config_activate_committed() was reachable from exactly two * places: handle_repl_config_push (a STANDBY receiving a push) and * the boot/recovery path -- never from a member's own live apply of * the marker it just committed. A live 3-node chaotic-kill soak * caught the gap empirically: the primary that ORIGINATED a config * upgrade (do_upgrade_config_with_bytes, which saves the new bytes * to disk and bumps config_generation but never re-activates them * into c->myconf on itself) forced a checkpoint at the marker's own * LSN whose queue-entity section still reflected the OLD * (pre-upgrade) queue set, then was killed before any LATER, * independent event (restart, or a racing inbound push) happened to * call config_activate_committed -- freezing a stale * queue-entity list into that checkpoint forever (a real * cross-member materialized-state divergence, * .wolf/buglog.json bug-1469's byte-level trace: the checkpoint's * embedded raw config-bytes section correctly carried the new * bytes, but the SEPARATE queue-entity list section -- built by * walking runtime state, not the raw bytes -- did not, because that * runtime state was never refreshed before the write). * * MUST run BEFORE persist_checkpoint_marker_triggered(), not merely * before upgrade_materialization_verify: the checkpoint writer * (write_snapshot -> collect_queue_frames/collect_host_frames) * builds its queue/host-entity sections by walking c->qmgr/ * c->workermgr -- LIVE RUNTIME TABLES, populated at boot from * c->myconf->cluster_config and never otherwise re-synced from a * later config change on this call path -- not by re-parsing * mc->cluster_config directly. So activation must settle BEFORE * persist_checkpoint_marker_triggered() runs, or the checkpoint * this marker forces still captures whatever runtime state existed * prior to activation, regardless of how soon afterward activation * runs. (This ordering constraint is logically necessary given * write_snapshot's own read path, confirmed by reading * collect_queue_frames/write_snapshot directly -- not yet * separately confirmed against a live soak re-run at the time this * comment was written; see the task's own final live-soak * verification step for that.) * * Symmetric with task 6.2's own fix shape below: one unconditional * call at the single apply seam every member's marker-apply is * guaranteed to pass through, rather than patching only the * originating code path. * * Safe to call here for the same reason task 6.2's own call * (upgrade_materialization_verify, below) is safe at this exact * seam: neither config_activate_committed nor anything it * transitively calls (config_committed_marker, local_config_sha256, * conf_read_current_filename, conf_read_raw, load_primary_config) * acquires apply_pump_lock, p->apply_pump_lock, or any other lock * (verified -- config_repl.c's only two mutexes, c->ha.lock and * c->persist.lock, are both taken in unrelated functions never on * this call path) -- so no reentrant-lock hazard even though this * runs while persist_apply_pump_run holds apply_pump_lock across * its entire dispatch. The disk I/O this performs (WAL segment * scan, config file read, Cap'n Proto decode) is the same class * already unconditionally performed at this exact call site by * upgrade_materialization_verify below (config_committed_marker's * own scan + a config-bytes read/hash), so this adds no new * blocking-I/O hazard class. Return value intentionally ignored * (mirrors upgrade_materialization_verify below): a * mismatch/absence return (-EAGAIN/-ENOENT) means this member's own * serve-fence (config_serve_fenced, a SEPARATE mechanism from the * Activation Fence below) is set instead, and that fence already * has its own established retry path (config_schedule_activate_retry) * driven off commit-advance -- this call's only job here is to give * every marker-apply one unconditional attempt so a member never * depends solely on a later, independent event to activate. * * Guarded on c->myconf != NULL: in production myconf is always * wired during cluster init, well before any commit/apply can run, * so this is a no-op check on every real code path -- but * config_activate_committed's own -ENOENT fallback calls * load_primary_config(c) unconditionally on c->myconf without its * own null check (pre-existing in config_repl.c, out of this fix's * scope to touch), and this call site is reachable from a * bare/otherwise-empty test cluster_t that exercises * WAL_CONFIG_MARKER apply mechanics without wiring myconf * (test_upgrade_snapshot_apply.c's tusa_t fixture, * test_marker_apply_via_live_pump_checkpoints_at_marker_own_lsn -- * confirmed via a live SEGV in load_primary_config, address 0x28, * before this guard was added) -- mirrors this same file's existing * tolerance of a bare fixture elsewhere (e.g. collect_queue_frames' * `c->qmgr == NULL` check, read_active_config_bytes' "no live * cluster back-pointer" handling) rather than crashing a test that * predates and is unrelated to config activation. */ if (c->myconf != NULL) (void)config_activate_committed(c); int rc; #ifdef XCE_TESTING if (persist_marker_checkpoint_fault_hook != NULL && persist_marker_checkpoint_fault_hook(p, lsn)) { /* Simulated crash (task 8.1, Req 6.3): skip the checkpoint call * entirely and fall into the SAME genuine-failure handling below * a real checkpoint I/O error already takes -- see the fault * hook's own doc comment above for why that is the correct, * not merely convenient, simulation of "the process never got * to checkpoint". */ rc = -ECANCELED; } else #endif rc = persist_checkpoint_marker_triggered(p, lsn); if (rc == -EBUSY) { /* Expected, self-resolving (Req 4.4/5.3): a concurrent * WAL-threshold-triggered or earlier marker-triggered * checkpoint already holds the shared single-in-flight guard * (checkpoint_in_progress). That in-flight checkpoint * completes on its own; this marker's own configuration is * still picked up by it or a later checkpoint (every * checkpoint embeds the currently active configuration, task * 2.3) -- not an error, no retry loop here. */ TRACE("apply_committed_entry: WAL_CONFIG_MARKER lsn=%" PRIu64 " checkpoint already in progress -- skipping, " "self-resolving", lsn); } else if (rc != 0) { /* A genuine checkpoint failure (I/O error, etc.) -- propagate * so the caller's existing halt-and-retry-from-same-position * handling engages (persist_apply_pump_run / replay_wal), * exactly mirroring every other negative-return case in this * dispatch. last_applied_lsn is NOT advanced past this marker, * so the next kick/retry re-attempts the checkpoint fresh * (Req 5.2 crash-safety: resume materialization from the * marker). upgrade_materialization_verify below still runs * even on this path (it reads its own signals -- the * committed marker + c->persist.cfgref_* -- independently of * this rc; a failed checkpoint attempt here does not by * itself change whether a PRIOR checkpoint already * materialized under the current marker). */ ERROR("apply_committed_entry: WAL_CONFIG_MARKER lsn=%" PRIu64 " forced checkpoint failed (rc=%d)", lsn, rc); } /* config-upgrade-snapshot task 6.2 (Req 1.8, 3.2, 3.4, 5.5): * unconditional Activation Fence re-evaluation right after every * checkpoint attempt tied to this marker, regardless of whether * persist_checkpoint_marker_triggered just above returned 0, * -EBUSY, or a genuine error -- upgrade_materialization_verify * itself already handles "materialization not done yet" correctly * (it fences and returns -EAGAIN rather than erroring), so gating * this call on rc would only reintroduce the closed loop this call * site exists to break (every OTHER caller of * upgrade_materialization_verify is itself gated on the fence * already being set -- see upgrade_materialization_retry_fn and * its callers in ha.c -- so without one unconditional trigger the * fence could never transition from its zero-initialized default). * Checkpoint completion is the design-intended trigger for this * re-evaluation (design.md's Activation Fence component). Safe to * call here: neither upgrade_materialization_verify nor anything * it calls (config_committed_marker, conf_read_current_filename, * conf_read_raw) acquires apply_pump_lock or p->apply_pump_lock -- * they only do atomic reads of c->persist.committed_lsn / * c->persist.cfgref_* and plain file I/O (opendir/pread-style WAL * scan, conf_read_raw) -- so no reentrant-lock hazard even though * this call runs while persist_apply_pump_run holds * apply_pump_lock across this entire dispatch (same hazard class * as task 5's own persist_checkpoint_marker_triggered, avoided the * same way: never touch apply_pump_lock). Return value * intentionally ignored here -- ha_dispatch_gate and the * promotion gate (ha.c) are what actually consume the fence; * this call's only job is to keep it current. */ (void)upgrade_materialization_verify(c); return (rc == -EBUSY) ? 0 : rc; } default: /* WAL_NOOP / WAL_CHECKPOINT / unknown — no entity effect at this * seam, no hook (unchanged from apply_wal_entry's existing * treatment of these types). WAL_CONFIG_MARKER has its own explicit * case above (config-upgrade-snapshot task 5) and no longer falls * through to here. */ return 0; } } /* ============================================================ * Ordered apply pump (raft-apply-decoupling task 6.1) * ============================================================ */ /* Locate and read back the WAL entry at exactly `lsn` from the on-disk * segment files, for the live apply pump to hand to apply_committed_entry() * (mirrors read_entry_meta_locked's segment-scan idiom above, but returns * the entry's type and a malloc'd copy of its payload instead of just * metadata, and re-verifies the entry CRC — this is live-path plumbing, not * a metadata probe). * * Does not take p->lock: every LSN this is ever called with is already * majority-committed (durable and replicated), strictly below the physical * append tip a concurrent persist_wal_async could be extending — so this * never races the single-writer append path for the bytes it actually * reads. * * Returns 0 and populates *out_type/*out_payload (malloc'd if payload_len > * 0, caller frees; NULL when payload_len is 0)/*out_len on success. * Returns -ENOENT if no segment contains this LSN (should not happen for a * genuinely committed entry) or a negative errno on a read/CRC failure. */ static int apply_pump_read_entry(persist_t* p, uint64_t lsn, uint32_t* out_type, uint8_t** out_payload, uint32_t* out_len) { if (!p || lsn == 0 || !out_type || !out_payload || !out_len) return -EINVAL; /* Fast path (raft-apply-decoupling task 7.4 perf-fix): the sole caller * (persist_apply_pump_run) calls this once per LSN in strict ascending * order with no gaps in the common case, and holds apply_pump_lock * across the entire walk -- so the cache fields below need no lock of * their own here (see persist.h doc comment). A cache hit reads the next * header directly at the remembered offset on the already-open fd, * giving amortized O(1) per entry instead of a fresh * opendir/readdir/segment-sort plus a scan-from-zero on every single * call (the O(n^2) regression this fixes). ANY miss below just drops the * cache and falls through to the unchanged slow path -- correctness * never depends on the cache being right, only performance does. */ if (p->apply_pump_cache_fd >= 0 && p->apply_pump_cache_next_lsn == lsn) { wal_entry_hdr_t hdr; ssize_t hn = pread(p->apply_pump_cache_fd, &hdr, sizeof(hdr), (off_t)p->apply_pump_cache_off); bool hit = (hn == (ssize_t)sizeof(hdr)) && (hdr.lsn == lsn); uint8_t* payload = NULL; if (hit && hdr.payload_len > 0) { payload = malloc(hdr.payload_len); if (!payload) { hit = false; } else { ssize_t pn = pread(p->apply_pump_cache_fd, payload, hdr.payload_len, (off_t)(p->apply_pump_cache_off + sizeof(hdr))); if (pn != (ssize_t)hdr.payload_len) { free(payload); payload = NULL; hit = false; } } } if (hit) { uint32_t expected_crc = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); if (hdr.payload_len > 0) expected_crc = crc32(expected_crc, (const Bytef*)payload, hdr.payload_len); if (hdr.crc != expected_crc) { free(payload); payload = NULL; hit = false; } } if (hit) { *out_type = hdr.type; *out_payload = payload; *out_len = hdr.payload_len; p->apply_pump_cache_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); p->apply_pump_cache_next_lsn = lsn + 1; return 0; } /* Miss (short read/EOF meaning the entry rolled into the next * segment, header mismatch, CRC failure, whatever): don't try to be * clever about advancing to the next segment inline. Close and fall * through to the EXISTING slow path unchanged -- it already * correctly finds the right segment from scratch. */ close(p->apply_pump_cache_fd); p->apply_pump_cache_fd = -1; } DIR* d = opendir(p->wal_path); if (!d) return -ENOENT; uint64_t seg_lsns[4096]; int nseg = 0; struct dirent* e; while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) { if (strncmp(e->d_name, "seg-", 4) != 0) continue; uint64_t sl = 0; if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &sl) != 1) continue; seg_lsns[nseg++] = sl; } closedir(d); if (nseg == 0) return -ENOENT; /* Ascending sort by start LSN (segment count is tiny in practice). */ for (int i = 1; i < nseg; i++) { uint64_t key = seg_lsns[i]; int j = i - 1; while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; } seg_lsns[j + 1] = key; } int rc = -ENOENT; for (int si = 0; si < nseg; si++) { /* Segments are LSN-ordered; one whose start LSN already exceeds the * target cannot contain it, nor can any later one. */ if (seg_lsns[si] > lsn) break; char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[si]); int fd = open(seg_path, O_RDONLY); if (fd < 0) continue; uint32_t off = 0; for (;;) { wal_entry_hdr_t hdr; ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)off); if (hn < (ssize_t)sizeof(hdr)) break; /* short read / EOF / pre-allocated zero tail */ if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break; if (hdr.lsn == lsn) { uint8_t* payload = NULL; uint32_t expected_crc = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); if (hdr.payload_len > 0) { payload = malloc(hdr.payload_len); if (!payload) { rc = -ENOMEM; break; } ssize_t pn = pread(fd, payload, hdr.payload_len, (off_t)(off + sizeof(hdr))); if (pn != (ssize_t)hdr.payload_len) { free(payload); rc = -EIO; break; } expected_crc = crc32(expected_crc, (const Bytef*)payload, hdr.payload_len); } if (hdr.crc != expected_crc) { free(payload); rc = -EIO; break; } *out_type = hdr.type; *out_payload = payload; *out_len = hdr.payload_len; rc = 0; /* raft-apply-decoupling task 7.4 perf-fix: populate the * segment-lookup cache with this fd positioned just past the * entry we just returned, BEFORE closing it below, so the * next expected LSN (lsn+1) hits the fast path above instead * of re-scanning from segment start next time. Every OTHER * branch (error paths, not-found-in-this-segment * continuation) closes fd exactly as before -- caching only * ever happens on this success branch. */ p->apply_pump_cache_fd = fd; p->apply_pump_cache_seg_lsn = seg_lsns[si]; p->apply_pump_cache_off = off + (uint32_t)(sizeof(hdr) + hdr.payload_len); p->apply_pump_cache_next_lsn = lsn + 1; fd = -1; /* ownership transferred to the cache -- the * close(fd) below becomes a guarded no-op. */ break; } off += (uint32_t)(sizeof(hdr) + hdr.payload_len); } if (fd >= 0) close(fd); if (rc == 0 || rc == -ENOMEM || rc == -EIO) break; /* found it, or hit a genuine error inside the segment * that should contain it */ } return rc; } int persist_apply_pump_run(persist_t* p) { if (!p) return -EINVAL; pthread_mutex_lock(&p->apply_pump_lock); int applied = 0; for (;;) { uint64_t committed = atomic_load_explicit(&p->committed_lsn, memory_order_acquire); uint64_t next = atomic_load_explicit(&p->last_applied_lsn, memory_order_relaxed) + 1; if (next > committed) break; /* caught up as of this instant */ uint32_t wal_type = 0; uint8_t* payload = NULL; uint32_t len = 0; int rc = apply_pump_read_entry(p, next, &wal_type, &payload, &len); if (rc < 0) { ERROR("persist_apply_pump_run: failed to read committed entry " "lsn=%" PRIu64 " (rc=%d) -- stopping this pass; the entry " "stays committed and will be retried, from the same " "position, on the next kick", next, rc); pthread_mutex_unlock(&p->apply_pump_lock); return rc; } /* Strict order, no skipping or reordering: this is the single * dispatch call every newly committed entry passes through here, * regardless of its commit mode (WAIT command or RETAIN fact) — the * loop only ever advances last_applied_lsn one entry at a time, in * ascending order, so a burst of commands and facts that committed * out of their original submission order still applies in strict * commit (LSN) order (Req 3.3, 7.3). */ int apply_rc = apply_committed_entry(p, wal_type, next, payload, len); if (apply_rc < 0) { /* job-array-single-wal-record task 4.1 (design.md "Live Apply * Resource Fence"): this return value was previously discarded * -- (void)apply_committed_entry(...) -- with last_applied_lsn * advancing unconditionally, silently and permanently losing * any apply failure. A negative rc here is never a content- * derived failure (Req 9.1-9.3's pre-commit validation already * made those impossible post-commit); it is a node-local * resource-exhaustion failure (e.g. OOM mid array expansion, * Req 9.4) that must not be skipped. Mirror the * apply_pump_read_entry failure branch immediately above: stop * the walk exactly at this LSN, do not advance * last_applied_lsn, and do not take this LSN's pending-apply * hand-off (persist_take_pending_apply/on_applied) -- the entry * never actually applied, so that hand-off must not fire for * it. The entry stays committed and will be retried, from the * same position, on the next kick. This is a GENERAL fix: every * WAL type's apply outcome is now respected, not just * WAL_JOB_ARRAY_SUBMIT's. Under NORMAL operation every other * existing type's apply case returns 0 or positive, so this * branch is a behavior no-op for them in practice; the * fact-type cases' pre-existing "-ENOENT: referenced job not * found" branch (persist.h's dispatch table) is one other real, * pre-existing source of a negative return here, and now also * correctly halts the walk here instead of being silently * discarded, exactly as this fix intends. */ ERROR("persist_apply_pump_run: apply_committed_entry failed for " "committed entry lsn=%" PRIu64 " wal_type=%" PRIu32 " (rc=%d) -- stopping this pass at this position; the " "entry stays committed and will be retried, from the same " "position, on the next kick; Live Apply Resource Fence " "ENGAGED (job-array-single-wal-record Req 9.4)", next, wal_type, apply_rc); /* job-array-single-wal-record task 4.2 (design.md "Live Apply * Resource Fence"): engage the NEW, live, self-healing fence -- * distinct from recovery_unrecoverable -- so the promotion gate * (task 4.3) can refuse this node while it stays alive and * keeps retrying. Cleared below, from inside this same walk, * the instant a retry of this exact LSN succeeds (Req 9.5). */ persist_mark_apply_resource_fenced(p, next); free(payload); pthread_mutex_unlock(&p->apply_pump_lock); return apply_rc; } free(payload); atomic_store_explicit(&p->last_applied_lsn, next, memory_order_relaxed); applied++; /* job-array-single-wal-record task 4.2 (design.md "Live Apply * Resource Fence", Req 9.5): self-heal. Called unconditionally on * EVERY successful apply, not just after a known-fenced LSN -- * persist_clear_apply_resource_fence() is a cheap no-op when nothing * is fenced (the overwhelming common case). This is correct and * sufficient because the walk above is strictly sequential and * returns immediately on any failure (see the halt branch just * above): it can never advance PAST a fenced LSN without that exact * LSN applying successfully first, so "the next entry to apply * successfully after a fence engages" and "a retry of the SAME * stuck LSN succeeding" are the same event by construction. No * election, promotion check, or external/admin action is involved -- * the fence clears purely because this walk itself made progress. */ persist_clear_apply_resource_fence(p); /* raft-apply-decoupling task 6.2 (Req 1.1, 2.1-2.3): consume this * LSN's command hand-off, if any (task 3.2's * persist_register_pending_apply(), e.g. handle_jobsubmit's * commit-outcome continuation) -- strictly AFTER the mutation above * and after last_applied_lsn already reflects it, never before or * during. A fact entry never registers a hand-off, so * persist_take_pending_apply() simply returns 0 for it and this is a * correct no-op with no per-entry-type branching needed here. * persist_take_pending_apply() unlinks the registration before * returning it, so re-applying the same lsn (a future retry/replay * landing on this exact walk position again) can never fire the * continuation a second time. */ pending_apply_notify_fn on_applied = NULL; void* apply_ctx = NULL; if (persist_take_pending_apply(p, next, &on_applied, &apply_ctx)) on_applied(apply_ctx, next); } pthread_mutex_unlock(&p->apply_pump_lock); return applied; } /* Heap payload for one dispatched apply-pump walk invocation; frees itself * after use, mirroring commit.c's commit_resolve_dispatch_t convention. */ typedef struct { persist_t* p; } apply_pump_dispatch_t; /* Run walk passes until genuinely caught up, using the same recheck-then- * clear idiom commit.c's sweep thread uses (task 1.3) to avoid a lost-wakeup * race between "persist_apply_pump_run just decided it is caught up" and "a * new commit lands right after". Runs on a pool worker thread (or inline as * persist_apply_pump_kick's documented fallback) — never on the calling * (potentially replication-ACK/iomgr) thread when dispatched normally. * * Does NOT immediately retry a failed pass: persist_apply_pump_run() already * logs loudly and leaves last_applied_lsn unadvanced past a read failure, so * an immediate self-retry here would busy-loop forever on the exact same * unresolved head-of-line entry (a real, reproduced hazard — a persist_t * whose committed_lsn is ever advanced without a corresponding real WAL * entry on disk, e.g. a test fixture exercising commit-engine resolve logic * in isolation, spun this into a multi-million-iteration tight loop before * this guard existed). A future, genuinely NEW kick — triggered by a later * real commit-advance event — retries then, at that event's natural cadence, * never in a busy spin. */ static void apply_pump_drain(persist_t* p) { for (;;) { int rc = persist_apply_pump_run(p); atomic_store_explicit(&p->apply_pump_scheduled, 0, memory_order_release); if (rc < 0) return; /* already logged loudly inside persist_apply_pump_run; * do not busy-retry the same unresolved entry */ uint64_t committed = atomic_load_explicit(&p->committed_lsn, memory_order_acquire); uint64_t next = atomic_load_explicit(&p->last_applied_lsn, memory_order_relaxed) + 1; if (next > committed) return; /* genuinely caught up: no race window survived open */ int expected = 0; if (!atomic_compare_exchange_strong_explicit( &p->apply_pump_scheduled, &expected, 1, memory_order_acq_rel, memory_order_relaxed)) return; /* a concurrent kick already reclaimed the flag; it will * run its own pass and pick up this same new work */ /* else: we reclaimed the flag ourselves -- loop and run another pass */ } } static int apply_pump_vtask_entry(vtask_t* task) { apply_pump_dispatch_t* d = (apply_pump_dispatch_t*)task->data->u.generic; apply_pump_drain(d->p); free(d); return 0; } void persist_apply_pump_kick(persist_t* p, void* pool_arg) { if (!p || !p->cluster) return; /* bug-1416 (raft-apply-decoupling task 7.2 round 2): defer while this * node's own boot-time persist_recover() replay+verification pass is * still running -- last_applied_lsn is being written UNLOCKED by * replay_wal() during that window (single-threaded-recovery * assumption), and persist_verify_apply_correctness() is about to * compare a live re-read of it (and of the live job table) against a * WAL-scan-derived expectation. Letting this walk run concurrently * would race BOTH: it could advance last_applied_lsn out from under * replay_wal's own plain (non-CAS) store, and it mutates the same job * table persist_verify_apply_correctness re-reads as "actual" -- * producing a false APPLY_DIVERGENT verdict on a perfectly healthy * boot (confirmed live). No work is lost: persist_recover() sets this * flag back to true and issues exactly one catch-up kick right before * it returns, which will walk from last_applied_lsn+1 up through * whatever committed_lsn has grown to by then -- a full, ordered * drain, not a partial one. Intentionally checked with NO lock: a * stale `true` read racing the flag's transition to `false` is * harmless (the walk itself still only advances one entry at a time * under apply_pump_lock, so at worst this call proceeds a few * instructions into a window persist_recover() is about to close, and * the deferred re-kick after recover() still runs); a stale `false` * merely defers one no-op-cost kick to the next real trigger. */ if (!atomic_load_explicit(&p->recovery_complete, memory_order_acquire)) return; int expected = 0; if (!atomic_compare_exchange_strong_explicit( &p->apply_pump_scheduled, &expected, 1, memory_order_acq_rel, memory_order_relaxed)) return; /* already scheduled/running -- it will pick up any new work */ vtaskmgr_t* pool = (vtaskmgr_t*)pool_arg; vtask_data_t* data = calloc(1, sizeof(*data)); vtask_t* task = vtask_alloc(); apply_pump_dispatch_t* d = calloc(1, sizeof(*d)); /* Mirrors commit.c's dispatch_continuation fallback exactly: never leave * a scheduled walk silently unrun. pool == NULL is the expected shape of * a commit-only / bare-persist-cluster unit fixture that never calls * commit_set_pool — a real cluster always wires the pool before any * commit activity (cluster_init_phase2), so this fallback path is the * normal, deterministic behavior for that class of test, not a rare * production condition. */ if (pool == NULL || data == NULL || task == NULL || d == NULL) { free(data); free(task); free(d); apply_pump_drain(p); return; } d->p = p; data->vtype = VTASK_GENERIC; data->u.generic = d; vtask_init(task, (vtask_func_t)apply_pump_vtask_entry, data); /* VTASK_LIVE_BLOCK is never admission-gated (same delivery guarantee as * commit.c's own continuation dispatch) — this cannot fast-reject the * way VTASK_REQ_BLOCK could. */ if (vtask_enq_classified(pool, task, VTASK_LIVE_BLOCK) < 0) { WARN("persist_apply_pump_kick: vtask enqueue failed -- running " "inline (stay alive, never leave a scheduled walk unrun)"); free(data); free(task); free(d); apply_pump_drain(p); } } /* persist_apply_pump_run_sync (2026-07-10, chaotic-soak regression fix, * ROUND 2): synchronously drain the apply pump on THIS thread, GUARANTEEING * last_applied_lsn reaches committed_lsn (as observed at entry) before * returning -- regardless of what any concurrent caller is doing. * * Round 1 of this fix CAS-claimed apply_pump_scheduled and, on losing that * claim to a concurrent caller (e.g. a heartbeat-driven kick that fired * moments before promotion), just returned -- betting the OTHER caller's * async vtask-dispatched drain would finish in time. A second live chaotic * soak proved that bet wrong: job loss persisted even with the environmental * fdatasync noise (bug-924) independently confirmed and eliminated, so this * was a genuine, separate bug in the fix itself, not noise. The caller here * (promotion) cannot outsource its guarantee to a drain it doesn't control * the completion of. * * Fix: do NOT use the CAS-coalescing flag as a "someone else has it, I'm * done" signal -- call persist_apply_pump_run() directly in a loop until * THIS call observes last_applied_lsn caught up to committed_lsn. Multiple * concurrent callers (this one and any vtask-dispatched persist_apply_pump_ * kick) safely interleave via persist_apply_pump_run's own internal * apply_pump_lock and apply_committed_entry's per-LSN idempotency -- at * worst a redundant no-op pass, never lost or duplicated work. Stops (does * not busy-loop forever) on a genuine read failure, exactly mirroring * apply_pump_drain's own error handling: persist_apply_pump_run() already * logs loudly and leaves last_applied_lsn unadvanced on that path. */ void persist_apply_pump_run_sync(persist_t* p) { if (!p || !p->cluster) return; if (!atomic_load_explicit(&p->recovery_complete, memory_order_acquire)) return; for (;;) { uint64_t committed = atomic_load_explicit(&p->committed_lsn, memory_order_acquire); uint64_t applied = atomic_load_explicit(&p->last_applied_lsn, memory_order_relaxed); if (applied >= committed) return; /* caught up -- by us or a concurrent drain */ int rc = persist_apply_pump_run(p); if (rc < 0) return; /* already logged loudly; do not busy-loop on the same * unresolved entry (mirrors apply_pump_drain) */ /* Loop: re-check in case committed_lsn advanced further mid-walk, or * a concurrent drain made partial progress this pass didn't see. */ } } /* Read WAL segment files in LSN order; replay entries with lsn > from_lsn. * * Segment filename format: seg-.wal * Entry format: wal_entry_hdr_t (40 bytes) + payload (hdr.payload_len bytes). * * CRC chain: hdr.prev_crc must match last_crc (WAL_CRC_NONE for first entry * in a segment). If prev_crc mismatches, stop at last good entry and log WARN. * * Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1): * *verdict_out (when non-NULL) is set to RECOVER_TRUNCATED iff replay stopped * on a STRUCTURAL corruption — a CRC chain break, a CRC mismatch, or a * short/partial entry read — and RECOVER_CLEAN otherwise (clean EOF, * pre-allocated-zero tail, empty WAL, or all entries replayed). The verdict is * about the STRUCTURE of the local log, not about being behind the cluster: a * clean log that simply ends below the cluster high-water is still CLEAN. * * new_job_submits_out (raft-apply-decoupling task 5.1, when non-NULL) is set * to the number of WAL_JOB_SUBMIT entries this call actually dispatched via * apply_committed_entry() that resulted in a GENUINE new job (that function's * return value == 1) — not merely the number of WAL_JOB_SUBMIT entries above * `from_lsn` present in the log. This is the log-derived half of the * post-replay apply-correctness expectation (see apply_expectation_t). * * A WAL_JOB_SUBMIT entry can legitimately be duplicated for the same jobid * (bug-1419): a client's local commit-timeout retry appends a second entry * while the first is still in flight during a quorum-loss window, and the * timed-out entry is never retracted, so both can end up committed if the * first later achieves majority commit anyway. apply_committed_entry() * dedupes this via restore_job()'s -EEXIST and returns 0 (no-op) rather than * 1 (created) for the duplicate — counting raw WAL_JOB_SUBMIT occurrences * instead would tally that duplicate as a second job and spuriously fail the * post-replay comparison against jobmgr's actual growth (APPLY_DIVERGENT). * * new_worker_joins_out (bug-1427, when non-NULL) is the worker analog of * new_job_submits_out — the number of WAL_WORKER_CTRL entries this call * dispatched that genuinely created a new worker (apply_committed_entry()'s * return value == 1, per worker_from_capnp()'s bug-1427 contract), not raw * WAL_WORKER_CTRL occurrences (which also include UPDATE/DEAD entries and * idempotent JOIN replays, neither of which grows the worker table). * * Returns number of entries replayed (>= 0) or negative errno on I/O error. */ static int replay_wal(persist_t* p, uint64_t from_lsn, recover_verdict_t* verdict_out, uint64_t* new_job_submits_out, uint64_t* new_worker_joins_out) { /* Default to CLEAN; only a structural corruption flips it to TRUNCATED. */ if (verdict_out) *verdict_out = RECOVER_CLEAN; if (new_job_submits_out) *new_job_submits_out = 0; if (new_worker_joins_out) *new_worker_joins_out = 0; /* --- Collect and sort segment filenames by start LSN --- */ DIR* d = opendir(p->wal_path); if (!d) return 0; /* no WAL dir → first boot, nothing to replay (CLEAN) */ /* Collect up to 4096 segment names */ uint64_t seg_lsns[4096]; int nseg = 0; struct dirent* e; while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns)/sizeof(seg_lsns[0]))) { if (strncmp(e->d_name, "seg-", 4) != 0) continue; uint64_t seg_start = 0; if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start) != 1) continue; seg_lsns[nseg++] = seg_start; } closedir(d); if (nseg == 0) return 0; /* Simple insertion sort — segment count is tiny in practice */ for (int i = 1; i < nseg; i++) { uint64_t key = seg_lsns[i]; int j = i - 1; while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; } seg_lsns[j + 1] = key; } /* --- Replay entries from each segment --- */ int replayed = 0; uint32_t last_crc = WAL_CRC_NONE; /* job-array-single-wal-record task 7.2 (design.md "Live Apply Resource * Fence", task 4.2's self-flagged gap): once a negative * apply_committed_entry() return is hit during this replay pass, every * SUBSEQUENT entry is skipped for LOGICAL apply purposes -- mirroring * persist_apply_pump_run()'s halt-and-fence branch, which never * dispatches past a failed LSN either. This preserves the strict * sequential-no-skip apply invariant (task 4.1): a later entry may be * causally dependent on the one that never actually applied (e.g. a * job-state fact for an array element that was never constructed). * The PHYSICAL scan (CRC chain, current_lsn, seg_off) below is * deliberately NOT gated on this flag -- see the apply_rc < 0 branch's * doc comment for why current_lsn/last_crc must stay accurate for the * entire on-disk log regardless of the logical apply outcome. */ bool apply_halted = false; for (int si = 0; si < nseg; si++) { char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[si]); int fd = open(seg_path, O_RDONLY); if (fd < 0) { WARN("persist_recover: cannot open segment %s: %s", seg_path, strerror(errno)); continue; } uint32_t seg_off = 0; for (;;) { /* Read header */ wal_entry_hdr_t hdr; ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)seg_off); if (hn == 0) break; /* clean end of segment (CLEAN) */ if (hn < (ssize_t)sizeof(hdr)) { /* A partial header (or read error) is a SHORT READ — the log * stops mid-entry, so it cannot be reconstructed contiguously * (RECOVER_TRUNCATED, Req 2.1). Distinct from clean EOF above. */ if (hn < 0) WARN("persist_recover: read error in %s at offset %u: %s", seg_path, seg_off, strerror(errno)); else WARN("persist_recover: short header read in %s at offset %u " "(got %zd of %zu bytes); stopping replay", seg_path, seg_off, hn, sizeof(hdr)); if (verdict_out) *verdict_out = RECOVER_TRUNCATED; break; } /* Detect pre-allocated zero region (no valid entry starts with * all-zero header: lsn == 0 && type == 0 && payload_len == 0). * This is the normal fallocate'd tail after the last valid entry — * a clean stop, NOT corruption (stays RECOVER_CLEAN). */ if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break; /* CRC chain validation: prev_crc must match last_crc in segment */ if (seg_off == 0) { /* First entry in this segment: prev_crc must be WAL_CRC_NONE * (or we accept any value if last_crc is also WAL_CRC_NONE, * e.g. across a segment boundary after checkpoint trim). */ if (hdr.prev_crc != WAL_CRC_NONE && last_crc != WAL_CRC_NONE && hdr.prev_crc != last_crc) { WARN("persist_recover: CRC chain break at start of %s " "(prev_crc=%08x expected=%08x); stopping replay", seg_path, hdr.prev_crc, last_crc); if (verdict_out) *verdict_out = RECOVER_TRUNCATED; break; } } else { if (hdr.prev_crc != last_crc) { WARN("persist_recover: CRC chain break in %s at offset %u " "(prev_crc=%08x expected=%08x); stopping replay at last good entry", seg_path, seg_off, hdr.prev_crc, last_crc); if (verdict_out) *verdict_out = RECOVER_TRUNCATED; break; } } /* Verify entry CRC */ uint32_t expected_crc = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); uint8_t* payload = NULL; if (hdr.payload_len > 0) { payload = malloc(hdr.payload_len); if (!payload) { close(fd); return -ENOMEM; } ssize_t pn = pread(fd, payload, hdr.payload_len, (off_t)(seg_off + sizeof(hdr))); if (pn != (ssize_t)hdr.payload_len) { WARN("persist_recover: short payload read in %s at offset %u", seg_path, seg_off); if (verdict_out) *verdict_out = RECOVER_TRUNCATED; free(payload); break; } expected_crc = crc32(expected_crc, (const Bytef*)payload, hdr.payload_len); } if (hdr.crc != expected_crc) { WARN("persist_recover: CRC mismatch in %s at offset %u " "(got=%08x expected=%08x); stopping replay", seg_path, seg_off, hdr.crc, expected_crc); if (verdict_out) *verdict_out = RECOVER_TRUNCATED; free(payload); break; } /* Advance segment-local CRC */ last_crc = hdr.crc; /* Record this entry's term at its LSN for recovery read-back * (ha-term-safety 1.3, R1.5). Record EVERY CRC-validated logical * entry — including snapshot-covered ones below from_lsn — so the * index spans the full LSN range the reconcile/promotion logic may * query. EXCLUDE WAL_CHECKPOINT markers (type == WAL_CHECKPOINT) and * unknown future types (type > WAL_CHECKPOINT): they are skipped * below and are never logical Raft entries, so the live append path * does not record them either (kept symmetric). The raw hdr.term is * stored verbatim (legacy 0 included); the legacy-0 -> WAL_LEGACY_TERM * mapping is applied only at read-back. */ if (hdr.type < WAL_CHECKPOINT) record_entry_term(p, hdr.lsn, hdr.term); /* Skip entries already covered by the snapshot */ if (hdr.lsn <= from_lsn) { free(payload); seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); continue; } /* Skip WAL_CHECKPOINT markers (local bookkeeping only) */ if (hdr.type == WAL_CHECKPOINT) { free(payload); seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); /* Advance current_lsn past the marker */ if (hdr.lsn + 1 > p->current_lsn) p->current_lsn = hdr.lsn + 1; continue; } /* Unknown future entry types: log and skip */ if (hdr.type > WAL_CHECKPOINT) { WARN("persist_recover: unknown WAL entry type %u at LSN %lu; skipping", hdr.type, (unsigned long)hdr.lsn); free(payload); seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); continue; } /* Logical entity restore via the shared apply seam * (raft-apply-decoupling task 2.3, design.md "Apply Seam"): the * SAME apply_committed_entry() dispatch the live commit-advance * apply pump will use (task 6.1), so a node's materialized state * is the deterministic result of its committed log regardless of * whether it got there via restart replay or live apply * (Req 3.1-3.4). apply_wal_entry() (this replay loop's previous * per-entry dispatch) is NOT retired by this change — it remains * the live standby catch-up/replicated-apply dispatch used * elsewhere (ha.c); it is simply no longer this loop's call. The * CRC chain + current_lsn remain owned by this replay loop * (current_lsn advanced below; last_crc at line ~1601) — neither * dispatch function ever touched them (bug-778). * * KNOWN HAZARD (tracked in tasks.md Implementation Notes, task * 2.3): this replays the FULL post-checkpoint WAL range on every * restart (last_applied_lsn is not persisted, deliberately * recomputed here), so apply_committed_entry()'s AFTER_EVENT * hooks fire again for entries already applied+notified in a * prior process life. Confirmed inert today: scheduler hooks * aren't registered until a post-boot network JOIN * (on_scheduler_join, sched.c), and apiserver hooks — though * registered before this runs — gate their notify on a * connected endpoint that is empty at first-boot replay time. * MUST be re-verified before tasks 4.1/6.1/6.2/6.3 make this * path broadly live. */ int apply_rc = 0; if (!apply_halted) { apply_rc = apply_committed_entry(p, hdr.type, hdr.lsn, payload, hdr.payload_len); } free(payload); if (apply_halted) { /* job-array-single-wal-record task 7.2: an earlier entry in * THIS replay pass already fenced (see the apply_rc < 0 * branch below) -- this entry and every later one is * skipped for logical apply, exactly like the * WAL_CHECKPOINT/unknown-type skip branches above. Physical * bookkeeping (current_lsn/seg_off) still runs * unconditionally below. */ } else if (apply_rc < 0) { /* job-array-single-wal-record task 7.2 (design.md "Live * Apply Resource Fence", task 4.2's self-flagged residual * gap): mirror persist_apply_pump_run()'s halt-and-fence * branch exactly. This is never a content-derived failure * (Req 9.1-9.3's pre-commit validation already made those * impossible post-commit) -- it is the SAME node-local * resource-exhaustion class (Req 9.4) the live pump already * guards against, just hit during boot/restart replay * instead of live apply. Do NOT advance last_applied_lsn * past this LSN (skip that store entirely -- last_applied_lsn * stays parked at this entry's predecessor) and do NOT tally * this entry's job count; engage the SAME * persist_mark_apply_resource_fenced() primitive the live * path uses -- reused verbatim, no parallel mechanism. This * closes task 4.2's documented residual gap: without this, a * node that crashes while fenced and re-hits the identical * resource-exhaustion condition on restart would silently * discard the failure here (as this branch previously did, * unconditionally advancing last_applied_lsn regardless of * apply_rc's sign) and boot up looking fully promotable * while actually divergent/behind. * * Engaging synchronously HERE, inside this boot-time call * (persist_recover(), called synchronously from * cluster_init_phase2() before it returns), matters: * persist_apply_pump_kick()'s post-recovery catch-up kick * dispatches ASYNCHRONOUSLY onto a pool worker vtask (or * only runs inline as its own documented fallback) -- relying * on that kick alone to eventually re-fail and set the fence * would leave a real window, between persist_recover() * returning and that async retry actually running, where a * promotion-gate check could observe "not fenced" and * wrongly treat this node as eligible. * * KNOWN BOUNDARY (task 7.2 adversarial review, NOT fixed by * this task -- ha.c change required, out of this function's * scope): this fence, once engaged here, protects every * SUBSEQUENT promotion evaluation in this process's life -- * a later election's raft_cb_become_leader() gate (ha.c:1337 * reads persist_apply_resource_fenced(), inside the * eligibility block at ha.c:1303-1408) and the periodic * re-alert timer (promotion_fence_timeout_fn, ha.c ~876) * both correctly observe it. It does NOT protect this SAME * boot's own first-ever promotion evaluation: on a node's * first-ever election evaluation in its process's life (i.e. * BEFORE cluster_init_phase2() has ever run), ha.c's * raft_cb_become_leader() runs its eligibility gate at * ha.c:1303-1408 strictly BEFORE the is_first_boot block at * ha.c:1427-1439 that calls cluster_init_phase2() -- which is * what calls persist_recover()/replay_wal() (cluster.c:1482) * and is what would engage this fence for the first time. * There is no re-check of persist_apply_resource_fenced() * after cluster_init_phase2() returns; the code proceeds * straight to CTRL_PRIMARY. So a fence engaged by THIS SAME * first replay is invisible to the one promotion check * gating that specific boot -- exactly the "all nodes * restart simultaneously" race this whole task exists to * guard against. Closing this requires an ha.c change (a * re-check of persist_apply_resource_fenced() after * cluster_init_phase2() returns, or reordering recovery * ahead of the promotion decision), which is out of this * spec's current scope -- see tasks.md's task 7.2 * Implementation Notes for the full finding. */ ERROR("replay_wal: apply_committed_entry failed for " "committed entry lsn=%" PRIu64 " wal_type=%" PRIu32 " (rc=%d) during boot/restart replay -- halting this " "replay's LOGICAL apply at this position (the entry " "stays committed and will be retried by the live apply " "pump's post-recovery kick, from the same position); " "Live Apply Resource Fence ENGAGED " "(job-array-single-wal-record Req 9.4/9.5)", hdr.lsn, hdr.type, apply_rc); persist_mark_apply_resource_fenced(p, hdr.lsn); apply_halted = true; } else { /* raft-apply-decoupling task 5.1: track the replay-derived * LOGICAL apply cursor separately from current_lsn (the * physical WAL position owned by this loop, bug-778) -- * last_applied_lsn is the "highest entry actually dispatched * via apply_committed_entry()" position the post-replay * verification compares against. Segments are scanned in * ascending LSN order (sorted above) and entries within a * segment are sequential, so a plain store (no monotonic max * needed) is correct here. */ atomic_store_explicit(&p->last_applied_lsn, hdr.lsn, memory_order_relaxed); /* Log-derived half of the post-replay apply-correctness * expectation (Req 8.1): every WAL_JOB_SUBMIT that reaches * this point is, by construction, above the loaded * snapshot's checkpoint_lsn (the from_lsn skip above already * filtered out anything the snapshot covers) — so it names a * job that must now exist in the live job table that did not * exist before this replay pass began. A duplicate * WAL_JOB_SUBMIT for an already-existing jobid (bug-1419: * timeout-then-late-commit retry) is a legitimate no-op — * apply_committed_entry() returns 0 for it, not 1 — so it * must NOT be tallied here or the post-replay comparison * against jobmgr's real growth spuriously fails * (APPLY_DIVERGENT). */ if (hdr.type == WAL_JOB_SUBMIT && new_job_submits_out && apply_rc == 1) (*new_job_submits_out)++; /* job-array-single-wal-record task 6.0a: the array analog of * the WAL_JOB_SUBMIT tally immediately above. * apply_committed_entry()'s WAL_JOB_ARRAY_SUBMIT case * (restore_job_array()'s return, generalized to N elements) * returns apply_rc == the genuine count of NEWLY created * elements (1..array_size) on a real apply, or 0 for an * idempotent whole-record re-apply (every index already * existed -- restore_job_array() returned -EEXIST, which * this same dispatch maps to 0, mirroring WAL_JOB_SUBMIT's * own dedup return). Tallying apply_rc (not a flat +1) is * required here, unlike the single-job case above, because * one WAL_JOB_ARRAY_SUBMIT entry can materialize many jobmgr * entries at once; guarding on apply_rc > 0 excludes the * idempotent 0 case from double-counting a re-apply, exactly * as apply_rc == 1 excludes WAL_JOB_SUBMIT's duplicate-jobid * case (bug-1419) above. Without this branch, replay_wal() * silently undercounts every array record's real N-element * jobmgr growth to 0, and persist_verify_apply_correctness() * then reports a false APPLY_DIVERGENT verdict that engages * the PERMANENT, non-self-healing * persist_recover_loud_fence_if_unrecoverable() fence * (live-reproduced task 6.1 soak, 2026-07-15). */ if (hdr.type == WAL_JOB_ARRAY_SUBMIT && new_job_submits_out && apply_rc > 0) (*new_job_submits_out) += (uint64_t)apply_rc; /* bug-1427: worker analog of the WAL_JOB_SUBMIT tally above * — apply_rc == 1 means apply_committed_entry()'s * WAL_WORKER_CTRL case genuinely created a new worker (a * JOIN against a jobmgr slot that didn't exist yet); * apply_rc == 0 means an UPDATE/DEAD applied to an * already-existing worker (or a no-hook idempotent case * already excluded above via the apply_rc < 0 branch and * worker_from_capnp's -EEXIST-maps-to-0 path) — neither * grows the worker table, so only == 1 is tallied. */ if (hdr.type == WAL_WORKER_CTRL && new_worker_joins_out && apply_rc == 1) (*new_worker_joins_out)++; replayed++; } /* Advance current_lsn to one past this entry -- unconditional, * regardless of the LOGICAL apply outcome above (mirrors the * WAL_CHECKPOINT/unknown-type skip branches earlier in this * loop, which also advance current_lsn without ever dispatching * through apply_committed_entry()). job-array-single-wal-record * task 7.2: the physical WAL scan must account for every * on-disk entry so current_lsn/last_crc stay correct for any * future append this process makes (as a promoted primary, or * as a replication standby receiving new entries) -- halting * this advance too (treating a fenced entry as physically * absent) would understate the true on-disk tail and risk a * later append colliding with/overwriting already-durable * entries beyond the fenced position (persist_recover()'s own * "Round-2 fix" comment above, for the durable_lsn/apply_cursor * split, already establishes that current_lsn and * last_applied_lsn are expected to diverge -- this is the same * principle, applied to a halted apply instead of a skipped * checkpoint marker). */ if (hdr.lsn + 1 > p->current_lsn) p->current_lsn = hdr.lsn + 1; seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); } close(fd); } /* Update last_crc so the next WAL write chains correctly */ pthread_mutex_lock(&p->lock); p->last_crc = last_crc; pthread_mutex_unlock(&p->lock); return replayed; } /* raft-apply-decoupling task 5.1 / bug-1427: cheap read-lock count of a * manager's live entries, mirroring collect_job_frames' iteration idiom * (this file, above) without collecting or filtering anything. `mgr` is an * rtti `var` handle (c->jobmgr or c->workermgr) — lock/iter_* are generic * over the stored element type, so one counter serves both tables. Used only * to derive the post-replay materialized-entity expectation (never on a hot * path). Safe on a NULL mgr — returns 0. */ static uint64_t count_manager_entries(var mgr) { if (!mgr) return 0; lock(mgr, RTTI_LOCK_READ); iterator_t it; int irc = iter_make(&it, mgr); if (irc < 0) { unlock(mgr); return 0; /* empty table */ } uint64_t n = 0; for (;;) { void* e = NULL; int nrc = iter_next(&it, &e); if (nrc < 0 || nrc == RTTI_ITER_NO_DATA) break; if (e == NULL) continue; n++; } iter_destroy(&it); unlock(mgr); return n; } static uint64_t count_jobmgr_entries(cluster_t* c) { return (!c) ? 0 : count_manager_entries(c->jobmgr); } /* bug-1427: worker analog of count_jobmgr_entries above — same NULL-safety * contract, over c->workermgr instead of c->jobmgr. */ static uint64_t count_workermgr_entries(cluster_t* c) { return (!c) ? 0 : count_manager_entries(c->workermgr); } /* persist_verify_apply_correctness — see persist.h for the full contract * doc (raft-apply-decoupling task 5.1, Req 8.1/8.2; worker half bug-1427). * Pure comparison: no logging, no fence side effects. */ apply_verdict_t persist_verify_apply_correctness( persist_t* p, const apply_expectation_t* exp, uint64_t jobmgr_count_before_replay, uint64_t workermgr_count_before_replay) { if (!p || !exp) return APPLY_CLEAN; /* defensive: nothing to verify */ uint64_t actual_last_applied = atomic_load_explicit(&p->last_applied_lsn, memory_order_acquire); if (actual_last_applied != exp->expected_last_applied_lsn) return APPLY_DIVERGENT; /* Entity-materialization half of the check: skip gracefully when there * is no live cluster to query — mirrors apply_committed_entry()'s own * "nothing to dispatch into" convention for a bare unit-test persist_t. * Only the position check above applies in that case. */ if (!p->cluster || !p->cluster->jobmgr) return APPLY_CLEAN; uint64_t jobmgr_count_after = count_jobmgr_entries(p->cluster); if (jobmgr_count_after < jobmgr_count_before_replay) return APPLY_DIVERGENT; /* table shrank — impossible on a clean replay */ uint64_t actual_new_jobs = jobmgr_count_after - jobmgr_count_before_replay; if (actual_new_jobs != exp->expected_new_job_count) return APPLY_DIVERGENT; /* bug-1427: worker-count check, mirroring the job-count check above * exactly. Independently skipped (not just via the shared p->cluster * check above) when workermgr itself is unset, mirroring the jobmgr * NULL-safety convention. */ if (!p->cluster->workermgr) return APPLY_CLEAN; uint64_t workermgr_count_after = count_workermgr_entries(p->cluster); if (workermgr_count_after < workermgr_count_before_replay) return APPLY_DIVERGENT; /* table shrank — impossible on a clean replay */ uint64_t actual_new_workers = workermgr_count_after - workermgr_count_before_replay; if (actual_new_workers != exp->expected_new_worker_count) return APPLY_DIVERGENT; return APPLY_CLEAN; } apply_verdict_t persist_apply_verdict(persist_t* p) { return p ? p->apply_verdict : APPLY_CLEAN; } uint64_t persist_last_applied_lsn(persist_t* p) { return p ? atomic_load_explicit(&p->last_applied_lsn, memory_order_acquire) : 0; } /* bug-1416 (raft-apply-decoupling task 7.2 round 2) -- see persist.h's * doc comment on the recovery_complete field for the full contract. */ bool persist_recovery_complete(persist_t* p) { return !p || atomic_load_explicit(&p->recovery_complete, memory_order_acquire); } /* bug-1416 round 2 -- see persist.h's doc comment on this function for the * full rationale (closes the gap between persist_init() and the FIRST * persist_recover() call, which persist_recover()'s own false/true bracket * cannot see). */ void persist_mark_recovery_pending(persist_t* p) { if (!p) return; atomic_store_explicit(&p->recovery_complete, false, memory_order_release); } /* ------------------------------------------------------------------ */ /* Public: persist_recover */ /* ------------------------------------------------------------------ */ int persist_recover(persist_t* p) { /* bug-1416 (raft-apply-decoupling task 7.2 round 2): close the boot-time * race window for the FULL duration of this call, not just the position * check inside persist_verify_apply_correctness -- see the * recovery_complete field's doc comment (persist.h) for the complete * rationale. Held false until the natural (success) return at the * bottom of this function, which restores it and issues one deliberate * catch-up kick. The two hard-I/O-error early returns below * (load_snapshot / replay_wal failure) deliberately do NOT restore it: * those are pre-existing, rare disk-I/O-level failures the caller * already treats as fatal to this boot attempt (cluster_init_phase2 * tears down persist_t on a nonzero return) -- leaving the gate closed * is the conservative, fail-static choice (never let the live apply * pump or live-replication path mutate state on top of a persist_t * whose own recovery never finished), not a new hazard. */ atomic_store_explicit(&p->recovery_complete, false, memory_order_release); /* raft-apply-decoupling task 7.4 perf-fix (round 2): invalidate the apply * pump's segment-lookup cache before replay -- the cached fd/offset are * only valid relative to the pre-recovery on-disk state, and replay below * can change checkpoint_lsn/current_lsn. Doing this first covers every * return path below (including the early I/O-failure returns), so the * first post-recovery pump kick always falls back to full segment * rediscovery from the correct, recovered position -- never trusting a * stale pre-recovery cursor. Wrapped in apply_pump_lock (matching * persist_truncate_wal_above's identical invalidation site below) so the * cache fields' documented invariant -- "guarded ENTIRELY by * apply_pump_lock, never p->lock" (persist.h) -- holds unconditionally * here too, rather than resting on the implicit fact that no live pump * walk can be racing this boot-time call. No caller of persist_recover() * holds apply_pump_lock (its sole production call site, cluster.c boot * sequence, runs before the pump is ever kicked), so this cannot * deadlock. */ pthread_mutex_lock(&p->apply_pump_lock); if (p->apply_pump_cache_fd >= 0) { close(p->apply_pump_cache_fd); p->apply_pump_cache_fd = -1; } pthread_mutex_unlock(&p->apply_pump_lock); /* --- Step 1: find latest snapshot (highest LSN filename) --- */ char snap_file[4096] = ""; uint64_t snap_lsn = 0; DIR* d = opendir(p->snap_path); if (d) { struct dirent* e; while ((e = readdir(d)) != NULL) { size_t len = strlen(e->d_name); if (len < 6 || strcmp(e->d_name + len - 6, ".capnp") != 0) continue; uint64_t lsn = 0; if (sscanf(e->d_name, "snap-%" SCNx64 ".capnp", &lsn) != 1) continue; if (lsn > snap_lsn) { snap_lsn = lsn; snprintf(snap_file, sizeof(snap_file), "%s/%s", p->snap_path, e->d_name); } } closedir(d); } /* --- Step 2: load snapshot (if any) --- */ if (snap_lsn > 0 && snap_file[0] != '\0') { /* Local recovery doesn't need the entities-restored indicator to * decide behavior: the WAL replay in Step 3 applies everything * above checkpoint_lsn regardless, unchanged by ha-entity-snapshots. * It IS consumed for observability parity with the wire receive * path (ha_handle_snapshot_chunk, task 6.1) — a rolling-upgrade * restart against a pre-feature (legacy, position-only) local * checkpoint is an expected rollout state, not an error. */ bool entities_restored = false; int rc = load_snapshot(p, snap_file, snap_lsn, &entities_restored); if (rc != 0) { ERROR("persist_recover: failed to load snapshot %s: %s", snap_file, strerror(-rc)); return rc; } /* checkpoint_lsn and current_lsn set by load_snapshot */ if (entities_restored) { INFO("persist_recover: snapshot %s loaded lsn=%" PRIu64 " with full job/queue entity restoration", snap_file, p->checkpoint_lsn); } else { INFO("persist_recover: snapshot %s lsn=%" PRIu64 " is a legacy " "position-only checkpoint (no entity content) — continuing " "via normal WAL replay", snap_file, p->checkpoint_lsn); } } /* raft-apply-decoupling task 5.1 (Req 8.1): seed the replay-derived * apply cursor from the loaded baseline (checkpoint_lsn stays 0 when no * snapshot was found) so that, on a CLEAN replay with no new entries * beyond the snapshot, the cursor already sits at the recovered tip — * mirroring how current_lsn is seeded by load_snapshot. Step 3's * replay_wal advances it further for every entry actually dispatched * above this baseline. Also capture the live job table's size BEFORE * replay runs: the post-replay verification below needs this baseline to * isolate the jobs replay itself materializes from anything the snapshot * already restored. */ atomic_store_explicit(&p->last_applied_lsn, p->checkpoint_lsn, memory_order_release); uint64_t jobmgr_count_before_replay = count_jobmgr_entries(p->cluster); uint64_t workermgr_count_before_replay = count_workermgr_entries(p->cluster); /* bug-1427 */ /* --- Step 3: replay WAL entries after checkpoint_lsn --- */ /* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1): * replay distinguishes a clean log end from a corruption-truncated end. * We CLASSIFY + record the verdict here; we do NOT FATAL (task 3.3) and do * NOT fence promotion (task 4.3 consumes p->recovery_verdict). A clean * replay over a loaded snapshot-sentinel (no entries below the sentinel) is * CLEAN — the sentinel established the committed position; replay only adds * entries above it. */ recover_verdict_t verdict = RECOVER_CLEAN; uint64_t new_job_submits = 0; uint64_t new_worker_joins = 0; /* bug-1427 */ int replayed = replay_wal(p, p->checkpoint_lsn, &verdict, &new_job_submits, &new_worker_joins); if (replayed < 0) return replayed; p->recovery_verdict = verdict; if (verdict == RECOVER_TRUNCATED) WARN("persist_recover: RECOVER_TRUNCATED — local WAL is structurally " "corrupt (CRC break / mismatch / short read); member marked " "recovery-truncated (non-promotable until caught up)"); /* --- Step 3b: re-derive the durable floor from the recovered WAL tip * (ha-commit-durability task 3.4, Req 2.3) --- * * Publish the member's durable floor (the durable last-log index consumed * by the Raft freshness gate, task 4.2) as the highest INTACT replayed LSN. * replay_wal advanced current_lsn to one past the last intact entry — incl. * stopping at a corruption boundary on TRUNCATED — so current_lsn - 1 is * exactly that intact WAL tip, and a snapshot-sentinel load leaves it at the * sentinel/replayed tip. We store it into durable_lsn because the recovered * on-disk entries are GENUINELY durable (each was CRC-validated and read * from stable storage by replay_wal), so the freshness advertisement is now * backed by durable state rather than a volatile in-memory position. * * Crux (Req 2.3 — a truncated member cannot over-advertise): this is an * ASSIGN to the recovered tip, NOT a max() with whatever durable_lsn held. * A fresh recover starts from durable_lsn=0 (persist_init), so on a * truncating restart the floor lands at the LOWER intact tip — never a * stale higher in-memory value and never the commit position * (committed_lsn). The recovered tip is, by construction, <= current_lsn-1, * preserving the durable_lsn <= current_lsn-1 invariant. The post-election * safe-commit point continues to come from voter replies, not this floor * (CLAUDE.md). */ uint64_t recovered_tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; atomic_store_explicit(&p->durable_lsn, recovered_tip, memory_order_release); /* raft-apply-decoupling task 5.1 (Req 8.1/8.2): post-replay apply- * correctness verification, reported ALONGSIDE the structural * recovery_verdict set above — neither implies the other. See * persist_verify_apply_correctness's doc comment for exactly what is * compared. Routing an APPLY_DIVERGENT verdict into the existing * non-promotable loud-fence mechanism is task 5.2; this only classifies * and logs. * * Round-2 fix: the expected position MUST be the LOGICAL apply cursor * replay_wal() already maintains in p->last_applied_lsn — NOT * recovered_tip (current_lsn - 1, the PHYSICAL WAL-scan tip used above * for durable_lsn). current_lsn advances for every WAL entry replay_wal * scans, including WAL_CHECKPOINT markers and unknown future types, * neither of which is ever routed through apply_committed_entry() (see * the skip branches in replay_wal). last_applied_lsn only advances when * a real dispatch happens. If the highest-LSN entry above checkpoint_lsn * is a trailing WAL_CHECKPOINT marker — the exact state left behind by a * crash between persist_write_checkpoint_marker() (step 1, durably * fsynced) and write_snapshot() (step 3b, persists the advanced * checkpoint_lsn) — recovered_tip over-counts past it while * last_applied_lsn correctly does not, and comparing against * recovered_tip previously produced a false APPLY_DIVERGENT for a * perfectly clean recovery (violates Req 8.2). */ uint64_t apply_cursor = atomic_load_explicit(&p->last_applied_lsn, memory_order_acquire); apply_expectation_t apply_exp = { .expected_last_applied_lsn = apply_cursor, .expected_new_job_count = new_job_submits, .expected_new_worker_count = new_worker_joins, /* bug-1427 */ }; p->apply_verdict = persist_verify_apply_correctness( p, &apply_exp, jobmgr_count_before_replay, workermgr_count_before_replay); if (p->apply_verdict == APPLY_DIVERGENT) { ERROR("persist_recover: APPLY_DIVERGENT — materialized state does " "not match the committed log tip (expected last_applied_lsn=%" PRIu64 " new_job_count=%" PRIu64 " new_worker_count=%" PRIu64 "); a hidden apply defect is present (ha-commit-durability " "recover_verdict=%s)", apply_exp.expected_last_applied_lsn, apply_exp.expected_new_job_count, apply_exp.expected_new_worker_count, (verdict == RECOVER_TRUNCATED) ? "RECOVER_TRUNCATED" : "RECOVER_CLEAN"); /* raft-apply-decoupling task 5.2 (Req 8.3/8.4): route the DIVERGENT * verdict through the EXISTING non-promotable loud-fence mechanism * (persist_recover_loud_fence_if_unrecoverable + the sticky * recovery_unrecoverable flag) instead of a new fence path. * apply_divergent=true fences UNCONDITIONALLY (see * persist_recover_decision), so the peer/HWM fields below are * inert placeholders here -- they are not yet known at this * boot-time replay call site (unlike the structural verdict's own * fence check, which stays deferred to promotion time in ha.c, * where live peer/HWM context exists). verdict/recovered_tip are * passed through only for accurate, consistent logging. The member * stays ALIVE, non-promotable, and keeps retrying catch-up -- it * never exits (controller-never-self-terminate invariant, * CLAUDE.md). */ recover_policy_inputs_t apply_fence_in = { .verdict = verdict, .durable_below_hwm = false, .healthy_peer_reachable = false, .catch_up_succeeded = false, .durable_floor_lsn = recovered_tip, .cluster_hwm_lsn = 0, .apply_divergent = true, }; persist_recover_loud_fence_if_unrecoverable(p, &apply_fence_in); } else INFO("persist_recover: APPLY_CLEAN — materialized state matches the " "committed log tip (last_applied_lsn=%" PRIu64 ")", apply_cursor); /* --- Step 4: retention pass (TTL check) — no-op in P0 (no entities) --- */ /* bug-1416: replay + verification is fully complete -- safe to let the * live apply pump and the standby live-replication apply path resume * mutating state (see the recovery_complete field's doc comment). * Restored unconditionally here regardless of the verdict computed * above: a genuine APPLY_DIVERGENT already fenced this member via * persist_recover_loud_fence_if_unrecoverable (non-promotable, stays * alive, keeps retrying) -- that sticky fence and its clearing logic * (task 5.3) are untouched by this flag, which governs ONLY whether * in-memory apply may proceed, not whether this member may lead. Issue * one deliberate catch-up kick for anything that committed locally (or * arrived via live replication and was durably persisted but had its * entity-apply deferred, see ha_apply_single_entry / ha_apply_term_ * entry_via_library) while this pass ran -- persist_apply_pump_run's * own walk drains fully from last_applied_lsn+1 through the current * committed_lsn, so this is a complete, ordered catch-up, not a partial * one; a harmless no-op if nothing queued. */ atomic_store_explicit(&p->recovery_complete, true, memory_order_release); /* n_vcpus==0 is this codebase's own established "not yet initialized" * sentinel for vtaskmgr_t (mirrors cluster_init_phase2's own * `if (c->vtaskmgr.n_vcpus == 0) vtaskmgr_init(...)` guard) -- a real * cluster always has n_vcpus > 0 by the time persist_recover() runs * (vtaskmgr_init happens earlier in myapp_run()/phase2), while a bare * recovery-focused unit-test cluster_t (none of which call * vtaskmgr_init, confirmed by inspection) leaves it zeroed. Passing a * pointer to an uninitialized vtaskmgr_t into vtask_enq_classified * would be unsafe, so fall back to persist_apply_pump_kick's own * existing, already-exercised NULL-pool inline path in that case -- * identical to how every apply-pump test fixture in this spec already * exercises the pump. */ void* pool = (p->cluster && p->cluster->vtaskmgr.n_vcpus > 0) ? &p->cluster->vtaskmgr : NULL; persist_apply_pump_kick(p, pool); return 0; } /* Recovery-integrity accessors (ha-commit-durability task 3.1, Req 2.1). */ recover_verdict_t persist_recovery_verdict(persist_t* p) { return p ? p->recovery_verdict : RECOVER_CLEAN; } bool persist_recovery_truncated(persist_t* p) { return p && p->recovery_verdict == RECOVER_TRUNCATED; } /* ------------------------------------------------------------------ * * Boot-policy loud-fence decision (ha-commit-durability task 3.3, Req 2.4; * never-self-exit rework 2026-06-18). * * Implements the FENCE_UNRECOVERABLE terminal of the Boot Policy Decision Table * (design.md). The decision is a PURE function of its inputs; the action (loud * operator-visible log + set the sticky non-promotable flag) is separated into * persist_recover_loud_fence_if_unrecoverable so the decision can be unit-tested * in isolation. A reachable healthy peer NEVER yields FENCE_UNRECOVERABLE * (rows 3, 5); it is reserved for a member that cannot recover locally AND * cannot catch up (no healthy peer, rows 4, 4b). * * CRITICAL: this terminal is a LOUD, PERMANENT, NON-promotable fence — the * process STAYS ALIVE and keeps retrying catch-up. There is NO abort()/exit(): * a controller MUST NEVER self-terminate as a designed response to an * operational condition (user invariant, 2026-06-18). A self-exited controller * needs manual admin recovery and degrades quorum meanwhile; one that stays * alive and keeps retrying self-heals when a peer returns (Req 2.6). * ------------------------------------------------------------------ */ recover_decision_t persist_recover_decision(const recover_policy_inputs_t* in) { if (in == NULL) return RECOVER_DECISION_ELIGIBLE; /* defensive: nothing to fence */ /* raft-apply-decoupling task 5.2 (Req 8.3/8.4): an APPLY_DIVERGENT * post-replay verdict is a NEW INPUT to this same decision table, not a * new fence mechanism (design.md "Restart Replay & Verification"). It * fences UNCONDITIONALLY -- every other field below is irrelevant once * this is true. Unlike a structurally truncated/behind log (rows 2/3/5), * where a healthy peer's missing entries can repair the gap and legitimately * downgrade the outcome to a plain FENCE-then-catch-up, an apply-time * defect means the LOCAL apply path itself produced the wrong * materialized state from bytes it already had -- streaming more entries * from a peer does not prove that path is now trustworthy, so this is * never downgraded by peer reachability or catch-up success. */ if (in->apply_divergent) return RECOVER_DECISION_FENCE_UNRECOVERABLE; bool truncated = (in->verdict == RECOVER_TRUNCATED); /* Rows 1 & 6: a structurally CLEAN log that is NOT behind the cluster * high-water mark is a normal promotable member — no fence. */ if (!truncated && !in->durable_below_hwm) return RECOVER_DECISION_ELIGIBLE; /* From here the member is truncated OR behind the HWM — it must NOT lead * as-is. A truncated log is by definition not provably at/above the HWM * (design.md note), so it is treated as behind for fence purposes. */ /* Rows 2, 3, 5: a healthy peer is reachable. Catch up from it and regain * eligibility (task 4.3). A reachable peer NEVER yields the unrecoverable * fence — even for a truncated log or a legitimate snapshot-sentinel. If * catch-up has already succeeded the member is eligible; otherwise it stays * fenced (4.3 drives the catch-up + release). */ if (in->healthy_peer_reachable) { if (in->catch_up_succeeded) return RECOVER_DECISION_ELIGIBLE; return RECOVER_DECISION_FENCE; } /* No healthy peer. If catch-up did not reach a committed-consistent state, * the member can neither recover locally nor catch up right now — rows 4 & * 4b: a LOUD, PERMANENT, NON-promotable fence (NOT process exit). It stays * alive and keeps retrying; the fence releases when a peer returns * (Req 2.4/2.6). */ if (!in->catch_up_succeeded) return RECOVER_DECISION_FENCE_UNRECOVERABLE; /* No peer but catch-up is reported succeeded (e.g. already at/above after a * local catch-up): eligible. */ return RECOVER_DECISION_ELIGIBLE; } bool persist_recovery_unrecoverable(persist_t* p) { return p && atomic_load_explicit(&p->recovery_unrecoverable, memory_order_acquire); } void persist_clear_recovery_unrecoverable(persist_t* p) { if (!p) return; /* Clear only on a genuine transition so the loud "fence released" log fires * once, not on every catch-up tick. */ if (atomic_exchange_explicit(&p->recovery_unrecoverable, false, memory_order_acq_rel)) { INFO("persist_recover: unrecoverable fence RELEASED — member reached a " "committed-consistent state via catch-up; promotable again " "(ha-commit-durability Req 2.6)"); } } /* job-array-single-wal-record task 4.2 (design.md "Live Apply Resource * Fence", Req 9.4/9.5). See the apply_resource_fenced_lsn field's doc * comment (persist.h) and this trio's own doc comment (persist.h) for the * full rationale and the deliberate distinction from * persist_recovery_unrecoverable/persist_clear_recovery_unrecoverable just * above -- that boot-only mechanism is NOT reused here on purpose. */ bool persist_apply_resource_fenced(persist_t* p) { return p && atomic_load_explicit(&p->apply_resource_fenced_lsn, memory_order_acquire) != 0; } void persist_mark_apply_resource_fenced(persist_t* p, uint64_t lsn) { if (!p) return; atomic_store_explicit(&p->apply_resource_fenced_lsn, lsn, memory_order_release); } void persist_clear_apply_resource_fence(persist_t* p) { if (!p) return; /* Clear only on a genuine transition so the self-heal log fires once per * episode, not on every ordinary successful apply (the overwhelming * common case: nothing was fenced, so this is a cheap no-op). */ uint64_t prev = atomic_exchange_explicit(&p->apply_resource_fenced_lsn, 0, memory_order_acq_rel); if (prev != 0) { INFO("persist_apply_pump_run: Live Apply Resource Fence RELEASED — " "lsn=%" PRIu64 " applied successfully on retry; promotable " "again (job-array-single-wal-record Req 9.5)", prev); } } recover_decision_t persist_recover_loud_fence_if_unrecoverable(persist_t* p, const recover_policy_inputs_t* in) { recover_decision_t decision = persist_recover_decision(in); if (decision != RECOVER_DECISION_FENCE_UNRECOVERABLE) return decision; /* Loud, operator-visible alert. The two triggers reaching this terminal * are NOT logged with the same clause (round-2 fix, raft-apply- * decoupling task 5.2 remediation): * * - apply_divergent: fences UNCONDITIONALLY (persist_recover_decision), * evaluated at boot-replay time BEFORE peer/HWM context exists (see * the call site in persist_recover()). `healthy_peer_reachable` and * `cluster_hwm_lsn` are inert placeholders there, not real facts — * asserting "no healthy peer reachable" or a durable_floor_lsn < hwm * comparison for this trigger would report unevaluated-or-false * information to the operator (round-1 review: the placeholders can * make the comparison numerically impossible, e.g. "2 < 0"). This * branch names the real reason (apply-correctness divergence) only. * - structural (truncated / behind-HWM): reaches this terminal only * via the pre-existing "no healthy peer reachable" branch in * persist_recover_decision, so the peer-reachability and * durable_floor_lsn/hwm comparison ARE guaranteed true here — that * clause is unchanged from before this task. */ if (in->apply_divergent) { ERROR("persist_recover: UNRECOVERABLE — post-replay apply-correctness " "verification is DIVERGENT (materialized state does not match " "the committed log tip, raft-apply-decoupling Req 8.3); a local " "apply defect cannot be proven fixed by peer reachability or " "catch-up alone. Staying ALIVE and PERMANENTLY non-promotable " "(refusing to serve divergent state) and RETRYING catch-up " "indefinitely — operator attention is needed to diagnose the " "apply path (raft-apply-decoupling Req 8.3/8.4)"); } else { const char* why = (in->verdict == RECOVER_TRUNCATED) ? "recovery-truncated log (structural corruption)" : "log behind the cluster committed high-water mark"; ERROR("persist_recover: UNRECOVERABLE — %s and no healthy peer reachable to " "catch up; truncation boundary durable_floor_lsn=%" PRIu64 " < cluster committed high-water hwm=%" PRIu64 ". Staying ALIVE and " "PERMANENTLY non-promotable (refusing to serve divergent state) and " "RETRYING catch-up indefinitely — will self-heal when a healthy peer " "returns; operator attention may be needed " "(ha-commit-durability Req 2.4)", why, in->durable_floor_lsn, in->cluster_hwm_lsn); } /* Set the sticky non-promotable flag so the promotion gate refuses * leadership as belt-and-suspenders until catch-up clears it (Req 2.4/2.6). * p may be NULL for pure-classification callers that only want the loud log. */ if (p != NULL) atomic_store_explicit(&p->recovery_unrecoverable, true, memory_order_release); return RECOVER_DECISION_FENCE_UNRECOVERABLE; } /* Advance committed_lsn to `written_lsn` on a majority ACK — but ONLY if that * entry is already durable on local stable storage (ha-commit-durability task * 2.4, Req 1.1/1.2). This is the legacy (engine-less / LSN-0) sibling of the * commit engine's durable self-cap (commit.c majority_candidate_locked): a * majority ACK no longer implies the local copy is fsync'd, because group- * commit (task 2.1) deferred the append fsync to the demand-driven flusher. So * advancing committed_lsn here on the ACK alone would reintroduce the bug-661 * ACK-without-durability hazard — committed could be durable on a single * member and lost across failover. We gate the store on written_lsn <= * durable_lsn so committed_lsn ⇒ durable-on-this-member on every path. * committed_lsn is monotonic, so a max() store; the durable cap can only delay * the advance until the flusher syncs the entry (the durable-advance commit * wake re-drives the engine path; the legacy path is re-evaluated on the next * sync_append). */ static void sync_append_legacy_commit_store(persist_t* p, uint64_t written_lsn) { uint64_t durable = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); /* DURABILITY PRECONDITION: never advance committed past what is durable * locally. The LSN-0 first-entry edge stays valid (0 <= 0). */ if (written_lsn > durable) return; uint64_t cur = atomic_load_explicit(&p->committed_lsn, memory_order_acquire); if (written_lsn <= cur) return; /* monotonic: committed_lsn never regresses */ atomic_store_explicit(&p->committed_lsn, written_lsn, memory_order_release); #ifdef XCE_TESTING /* INVARIANT (debug guard): committed_lsn must never exceed durable_lsn on * any commit advance. A future caller reintroducing ACK-without-durability * trips this. */ assert(atomic_load_explicit(&p->committed_lsn, memory_order_acquire) <= atomic_load_explicit(&p->durable_lsn, memory_order_acquire)); #endif } int persist_wal_sync_append(persist_t* p, uint32_t type, const void* payload, uint32_t len, int* sync_rc_out) { /* ha-term-safety task 3.1: the primary's WAL_JOB_SUBMIT append is routed * through the raft library (ha_primary_log_append → raft_log_append) so the * library stamps the term + assigns the LSN (firing on_persist_entry → the * WAL write), keeping p->current_lsn co-tracked with s->log_len; the existing * majority-commit barrier below (repl_wait_majority_ack) is UNCHANGED and runs * after the append. Non-raft road / engine-less tests (library not LEADER) * fall back inside ha_primary_log_append to persist_wal_async. Standalone * persist (p->cluster == NULL, lib-level tests) keeps the direct append. */ int rc; if (p->cluster != NULL) rc = ha_primary_log_append(p->cluster, type, payload, len, NULL); else rc = persist_wal_async(p, type, payload, len); if (rc != 0) { if (sync_rc_out) *sync_rc_out = rc; return rc; } /* Majority-commit replication (Req 5.1–5.3; ha-commit-engine Req 1.1–1.3). * persist_wal_async appends to the page cache and signals the demand-driven * flusher; it no longer fsyncs inline (group-commit, task 2.1). Durability * is therefore established by the flusher advancing durable_lsn, and a * commit position advances only once the entry is durable locally — the * engine path via the durable self-cap (commit.c), the legacy path below * via sync_append_legacy_commit_store's written_lsn <= durable_lsn gate. * We use current_lsn - 1 because persist_wal_async already incremented it. */ uint64_t written_lsn; pthread_mutex_lock(&p->lock); written_lsn = p->current_lsn - 1; pthread_mutex_unlock(&p->lock); int sync_err; /* Engine-wired path (ha-commit-engine task 5.3): delegate the commit * decision to the commit engine through the walAck adapter. The engine * is the single committed_lsn writer (no store here), and its * n_peers<=1 self-quorum replaces the old has_standbys → -ENODEV * degraded special case (Req 6.3). * * LSN-0 edge (ACCEPTED interim behavior — spec Implementation Notes): * written_lsn is passed unconditionally, but the FIRST-EVER WAL entry * of a fresh cluster is LSN 0 and commit_submit rejects lsn==0, so the * adapter's entry_lsn != 0 guard routes that one entry to the legacy * branch below. This happens once per cluster lifetime; * ha-single-leader-role's promotion no-op entry eliminates it in * practice. */ if (p->repl.ce != NULL && written_lsn != 0) { sync_err = repl_wait_majority_ack(&p->repl, written_lsn, p->config.sync_timeout_ms, (uint8_t)OPCode_walAck); if (sync_err != 0) CTRL_METRIC_INC(ctrl_repl_degraded_count); } else { /* Residual non-engine branch: engine-less (ce == NULL) unit-test * contexts plus the LSN-0 first entry. The legacy connectivity- * based counting was retired by ha-commit-barrier-rollout task 2.2: * repl_wait_majority_ack now confirms LSN 0 immediately (documented * once-per-lifetime edge) and surfaces engine-less walAck waits as * -EINVAL via commit_submit, so the committed_lsn store below only * fires for the LSN-0 case (a no-op store of 0). */ mylock_lock(&p->repl.lock); int has_standbys = (p->repl.nstandbys > 0); mylock_unlock(&p->repl.lock); if (!has_standbys) { /* No standbys registered → degraded mode (no quorum to wait for) */ sync_err = -ENODEV; } else { sync_err = repl_wait_majority_ack(&p->repl, written_lsn, p->config.sync_timeout_ms, (uint8_t)OPCode_walAck); if (sync_err == 0) /* Durable-gated commit advance (task 2.4): advance committed_lsn * only if written_lsn is durable locally — never on the ACK * alone (group-commit removed the inline append fsync). */ sync_append_legacy_commit_store(p, written_lsn); else CTRL_METRIC_INC(ctrl_repl_degraded_count); } } if (sync_rc_out) *sync_rc_out = sync_err; /* sync_err < 0 — replication failed. * -ENODEV: legacy degraded mode (no standbys, engine-less) — acceptable * for forward progress. * -ETIMEDOUT: no per-entry majority within the deadline — the write * must be rejected to preserve the majority-consistency * invariant. * -XCE_ERR_NOT_READY: engine admission cap exhausted — retriable * rejection. * Timeout and admission rejection propagate to the job-submit error * path (ha-commit-engine Req 1.1, 4.4). */ if (sync_err == -ETIMEDOUT || sync_err == -XCE_ERR_NOT_READY) return sync_err; return 0; } #ifdef XCE_TESTING /* Test seam (task 2.4): invoke the EXACT durable-gated commit store that the * legacy sync_append branch uses on a majority ACK, so a test can prove the * ACK-without-durability hazard is closed without standing up the replication * wire. Pre-fix this branch stored committed_lsn = written_lsn unconditionally * (over-advancing past durable_lsn). */ void persist_test_sync_append_legacy_commit_store(persist_t* p, uint64_t written_lsn) { sync_append_legacy_commit_store(p, written_lsn); } #endif /* XCE_TESTING */ void persist_log_entry_metadata(const wal_entry_hdr_t* hdr) { TRACE("WAL entry: type=%u lsn=%lu term=%lu payload_len=%u crc=%08x", hdr->type, (unsigned long)hdr->lsn, (unsigned long)hdr->term, hdr->payload_len, hdr->crc); } /* ------------------------------------------------------------------ */ /* Hook functions — AFTER_EVENT handlers that write WAL entries */ /* ------------------------------------------------------------------ */ /* Map HOOK_DETAIL_* values to WAL entry types. * Returns 0 for details that have no WAL representation (skip). */ static uint32_t detail_to_wal_type(int detail) { switch (detail) { case HOOK_DETAIL_JOB_NEW: return WAL_JOB_SUBMIT; case HOOK_DETAIL_JOB_START: return WAL_JOB_START; case HOOK_DETAIL_JOB_EXIT: return WAL_JOB_FINISH; case HOOK_DETAIL_JOB_FAIL: return WAL_JOB_FAIL; case HOOK_DETAIL_JOB_STOP: return WAL_JOB_STOP; case HOOK_DETAIL_JOB_RESUME: return WAL_JOB_RESUME; case HOOK_DETAIL_JOB_UPDATE: return WAL_JOB_UPDATE; case HOOK_DETAIL_JOB_CKPT: return WAL_JOB_CKPT; case HOOK_DETAIL_JOB_RUSAGE_UPDATE: return WAL_JOB_RUSAGE; case HOOK_DETAIL_SETPREASON: return WAL_JOB_PREASON; case HOOK_DETAIL_CLRPREASON: return WAL_JOB_PREASON; case HOOK_DETAIL_QUEUE_UPDATE: return WAL_QUEUE_CTRL; case HOOK_DETAIL_WORKER_JOIN: return WAL_WORKER_CTRL; case HOOK_DETAIL_WORKER_UPDATE: return WAL_WORKER_CTRL; case HOOK_DETAIL_WORKER_DEAD: return WAL_WORKER_CTRL; default: return 0; /* unknown — skip */ } } /* Write-path commit classification (ha-commit-barrier-rollout task 1.2). * Pure lookup over the VERIFIED table (design.md §Write-Path * Classification); `detail` disambiguates WAL_WORKER_CTRL only. * Unclassified types fail fast under XCE_TESTING (abort) and fall back to * RETAIN in production — never dropped, never falsely confirmed (Req 2.2). */ wal_commit_mode_t wal_classify(uint32_t wal_type, int detail) { switch (wal_type) { case WAL_JOB_SUBMIT: /* client job submit — request-shaped; keeps the in-hook sync path * (reference implementation, handle_jobsubmit) */ return WAL_MODE_WAIT; case WAL_JOB_ARRAY_SUBMIT: /* client job-array submit (job-array-single-wal-record task 1.1) — * request-shaped exactly like WAL_JOB_SUBMIT: one requester awaits * the single commit barrier before reply, regardless of array * size. */ return WAL_MODE_WAIT; case WAL_QUEUE_CTRL: /* client queue control — request-shaped; the handler barrier owes * commit completion before the requester reply (tasks 2-3) */ return WAL_MODE_WAIT; case WAL_WORKER_CTRL: /* the only detail-disambiguated type: JOIN/UPDATE have a requester * awaiting confirmation (WAIT); DEAD is a controller-detected fact. * Unexpected details take the fact-safe RETAIN side. */ if (detail == HOOK_DETAIL_WORKER_JOIN || detail == HOOK_DETAIL_WORKER_UPDATE) return WAL_MODE_WAIT; return WAL_MODE_RETAIN; case WAL_JOB_START: case WAL_JOB_FINISH: case WAL_JOB_FAIL: case WAL_JOB_STOP: case WAL_JOB_RESUME: case WAL_JOB_UPDATE: case WAL_JOB_RUSAGE: case WAL_JOB_CKPT: case WAL_JOB_PREASON: /* worker/scheduler-reported facts — retained until committed. * FAIL/UPDATE are dual-origin; their request-shaped side is * realized by the handler barrier in do_jobcontrol_op * (research.md §2 call-site verification). */ return WAL_MODE_RETAIN; case WAL_NOOP: /* internal promotion no-op — replicated like any committed entry, * retained until committed (ha-commit-barrier-rollout pre-declared * row: promotion | internal | RETAIN; guard forces classification) */ return WAL_MODE_RETAIN; case WAL_CONFIG_MARKER: /* config-change marker (ha-term-safety task 4.2): a committable * (term,lsn) log entry like a job-state fact — replicated and retained * until committed; it commits through the SAME barrier as job state * (R7.1/R7.11), so it is RETAIN, NOT a no-op or a LOCAL marker. The * requester barrier on a config upgrade is owned by the upgrade path * (do_upgrade_config_with_bytes), not the per-entry WAIT hook. */ return WAL_MODE_RETAIN; case WAL_CHECKPOINT: /* local segment marker — never replicated, by design */ return WAL_MODE_LOCAL; default: ERROR("wal_classify: unclassified WAL type %u (detail %d) — " "RETAIN fallback; add it to the classification table", wal_type, detail); #ifdef XCE_TESTING assert(0 && "unclassified WAL type — add it to wal_classify"); #endif return WAL_MODE_RETAIN; } } /* Commit barrier helper (ha-commit-barrier-rollout task 2.2, design.md * §Commit Barrier Helper; Req 1.1, 1.2, 1.3). * * Handler-callable WAIT completion on the appended watermark: snapshot the * last-appended LSN, return 0 when the commit index already covers it, * else block in the commit engine until coverage or deadline. The engine * result is returned UNMODIFIED (0 / -ETIMEDOUT / -XCE_ERR_NOT_READY / * -EINVAL) so callers map it to their own error replies. * * Lock discipline: p->lock is held only for the current_lsn snapshot and * is released BEFORE commit_submit(WAIT) — the engine blocks on ce->lock * + condvar, and p->lock must never be held across it (engine * lock-ordering rule). */ int persist_commit_barrier(persist_t* p, uint32_t timeout_ms) { uint64_t watermark; if (p == NULL) return -EINVAL; /* Snapshot the last-appended watermark under p->lock. current_lsn is * the NEXT LSN to assign (persist.h invariant: committed_lsn <= * current_lsn - 1), so the highest appended entry is current_lsn - 1 * (resolved contract, spec Implementation Notes 2.1). current_lsn == * 0 means nothing was ever appended — vacuously satisfied. */ pthread_mutex_lock(&p->lock); if (p->current_lsn == 0) { pthread_mutex_unlock(&p->lock); return 0; } watermark = p->current_lsn - 1; pthread_mutex_unlock(&p->lock); /* Already covered — committed_lsn is monotone, so coverage observed * here can never regress. This check also absorbs the LSN-0 first * entry (watermark 0 vs committed_lsn 0): that once-per-lifetime entry * has no quorum contract (ha-commit-engine Implementation Notes) and * commit_submit would reject lsn == 0 anyway. */ if (atomic_load_explicit(&p->committed_lsn, memory_order_acquire) >= watermark) return 0; /* Engine not wired: legacy/bootstrap contexts have no engine commit * contract — there is nothing to wait on, so the barrier is a no-op. * Production always wires the engine (cluster_init_phase2). */ if (p->repl.ce == NULL) return 0; /* timeout_ms == 0 borrows the configured sync deadline (Req 1.2). */ return commit_submit(p->repl.ce, watermark, COMMIT_MODE_WAIT, timeout_ms != 0 ? timeout_ms : p->config.sync_timeout_ms); } /* Classified append at the hook seam (barrier-rollout task 1.2). * * WAIT (queue/worker control): the append stays async here — the handler * barrier (tasks 2-3) owes commit completion before the requester reply. * WAL_JOB_SUBMIT never reaches this helper (it keeps the in-hook sync * path in persist_job_hook_fun). * * RETAIN (facts): append + broadcast, then register the appended LSN with * the commit engine. commit_submit(COMMIT_MODE_RETAIN) is registration- * free bookkeeping (commit.c — returns 0 immediately; commit state is * derived from registry watermarks at ACK time), so the call is contract * documentation: the entry stays pending purely by lsn > committed_lsn * and commits later via covering ACKs. Commit-progress results are NEVER * propagated to hook callers — only the local append rc is (Req 3.1). * * Lock context: persist_wal_async takes/releases p->lock internally; the * appended-LSN read below re-takes p->lock briefly (same discipline as * persist_wal_sync_append) and releases it BEFORE commit_submit, so no * p->lock is ever held across ce->lock (engine lock-ordering rule). */ static int wal_hook_append_classified(persist_t* p, uint32_t wal_type, wal_commit_mode_t mode, const void* payload, uint32_t len) { uint64_t appended_lsn; int rc; /* ha-term-safety task 3.1: route the primary's local append through the raft * library (ha_primary_log_append → raft_log_append) so the LIBRARY stamps the * live election term and assigns the LSN (firing on_persist_entry → the WAL * write), keeping p->current_lsn co-tracked with s->log_len. The library is * the SINGLE primary append entry — no append bypasses it on the raft road. * On the non-raft road (library not LEADER) ha_primary_log_append falls back * to persist_wal_async, so engine-less / init-time contexts are unchanged. * When p->cluster is NULL (persist used standalone in lib tests) keep the * direct append (no cluster → no raft state to drive). */ if (p->cluster != NULL) rc = ha_primary_log_append(p->cluster, wal_type, payload, len, NULL); else rc = persist_wal_async(p, wal_type, payload, len); if (rc != 0 || mode != WAL_MODE_RETAIN) return rc; if (p->repl.ce == NULL) return rc; /* engine not wired (bootstrap / engine-less contexts) */ /* current_lsn - 1 is the LSN persist_wal_async just assigned (it * increments under p->lock). A concurrent append may bump it further; * that is benign — contiguous advancement means the higher watermark * still covers this entry. LSN-0 edge: commit_submit rejects lsn == 0 * (first-ever entry of a fresh cluster — accepted interim behavior, * spec Implementation Notes). */ pthread_mutex_lock(&p->lock); appended_lsn = p->current_lsn - 1; pthread_mutex_unlock(&p->lock); if (appended_lsn != 0) (void)commit_submit(p->repl.ce, appended_lsn, COMMIT_MODE_RETAIN, 0); return rc; /* local append rc only — facts have no commit timeout */ } /* Shared step for a single WAL_JOB_SUBMIT entry (raft-apply-decoupling task * 3.1): encode `job`, append, and block on the majority-commit barrier. * Returns 0 when the entry committed on a majority; a negative rc otherwise * (-ETIMEDOUT: no majority within the deadline; -XCE_ERR_NOT_READY: admission * cap; -ENOMEM: encode failure; or a local-write error). Whenever the LOCAL * WAL write succeeded (rc == 0, -ETIMEDOUT, or -XCE_ERR_NOT_READY — the entry * was appended locally and the LSN advanced even though replication is * degraded), *out_lsn is set to the entry's assigned LSN so a caller can tag * it for egress-watermark purposes even on a rejected submission. */ static int submit_job_wal_entry_and_commit(persist_t* p, const xce_job_t* job, uint64_t* out_lsn) { uint8_t* buf = NULL; size_t sz = 0; int rc; if (capnp_encode_job(job, &buf, &sz) != 0) return -ENOMEM; /* ha-convergence-recovery (3.3): snapshot the LSN this submit entry * will receive BEFORE the append. persist_wal_async assigns * hdr.lsn = current_lsn then increments, so the pre-append current_lsn * is exactly the written entry's LSN. */ pthread_mutex_lock(&p->lock); uint64_t submit_lsn = p->current_lsn; pthread_mutex_unlock(&p->lock); int sync_rc = 0; rc = persist_wal_sync_append(p, WAL_JOB_SUBMIT, buf, (uint32_t)sz, &sync_rc); free(buf); if (rc == 0 || rc == -ETIMEDOUT || rc == -XCE_ERR_NOT_READY) *out_lsn = submit_lsn; return rc; } /* Explicit controller-side raft step for a job submission (raft-apply-decoupling * item #1 / bug-1306): WAL-append each job in `jlist` and block on the * majority-commit barrier, returning the commit verdict DIRECTLY to the caller * (handle_jobsubmit) instead of through the xce_run_hooks fan-out (which returns * only the last hook's rc and so clobbered this -ETIMEDOUT with a later notify * hook's 0 — the committed-job-loss root cause). * * Returns 0 when every job's entry committed on a majority; the FIRST negative * rc otherwise (-ETIMEDOUT: no majority within the deadline; -XCE_ERR_NOT_READY: * admission cap; -ENOMEM: encode failure; or a local-write error). The caller * MUST reject the submission on a negative rc (never reply success) to preserve * the Commit Durability invariant. On -ETIMEDOUT / -XCE_ERR_NOT_READY the entry * WAS appended locally and the LSN advanced, so j->submit_lsn is still tagged — * the entry remains a pending, uncommitted tail a new-term leader may discard. */ int persist_submit_wal_and_commit(persist_t* p, xlist_t* jlist) { int rc = 0; if (p == NULL || jlist == NULL) return -EINVAL; for (int i = 0; i < jlist->count && rc == 0; i++) { job_t* j = (job_t*)xlist_get(jlist, i); uint64_t lsn = 0; rc = submit_job_wal_entry_and_commit(p, &j->job, &lsn); if (rc == 0 || rc == -ETIMEDOUT || rc == -XCE_ERR_NOT_READY) j->submit_lsn = lsn; } return rc; } /* Sibling of persist_submit_wal_and_commit for a job submission whose job * object(s) do not exist yet (raft-apply-decoupling task 3.1 / Req 2.1, 2.3): * `payloads` is an xlist_t of xce_job_t* that have never been registered in * any job table. On full success (return 0), out_lsns[i] holds payloads[i]'s * committed LSN — the caller constructs and tags the job object AFTER this * call returns, so a failed/timed-out commit never leaves a job object * behind (there is nothing to construct until this succeeds). Returns the * first negative rc on failure. */ int persist_submit_job_payloads_and_commit(persist_t* p, xlist_t* payloads, uint64_t* out_lsns) { int rc = 0; if (p == NULL || payloads == NULL || out_lsns == NULL) return -EINVAL; for (int i = 0; i < payloads->count && rc == 0; i++) { xce_job_t* j = (xce_job_t*)xlist_get(payloads, i); rc = submit_job_wal_entry_and_commit(p, j, &out_lsns[i]); } return rc; } /* raft-apply-decoupling task 3.2: append-only counterpart of * submit_job_wal_entry_and_commit — writes the entry via the SAME * primary-append primitive wal_hook_append_classified already uses for * RETAIN-mode facts (ha_primary_log_append / persist_wal_async), WITHOUT * waiting on the majority-commit barrier. Deliberately does NOT reuse * submit_job_wal_entry_and_commit itself: that helper is shared with * persist_submit_wal_and_commit's still-synchronous, still-blocking * contract (relied on directly by test_egress_filter.c / test_e2e_hooks.c / * test_persist_hooks.c), which this task must not disturb. */ static int append_job_wal_entry_local(persist_t* p, const xce_job_t* job, uint64_t* out_lsn) { uint8_t* buf = NULL; size_t sz = 0; int rc; uint64_t lsn; if (capnp_encode_job(job, &buf, &sz) != 0) return -ENOMEM; /* Snapshot the LSN this entry will receive BEFORE the append (same * convention as submit_job_wal_entry_and_commit / wal_hook_append_ * classified: persist_wal_async / ha_primary_log_append assigns * hdr.lsn = current_lsn then increments). */ pthread_mutex_lock(&p->lock); lsn = p->current_lsn; pthread_mutex_unlock(&p->lock); if (p->cluster != NULL) rc = ha_primary_log_append(p->cluster, WAL_JOB_SUBMIT, buf, (uint32_t)sz, NULL); else rc = persist_wal_async(p, WAL_JOB_SUBMIT, buf, (uint32_t)sz); free(buf); if (rc == 0) *out_lsn = lsn; return rc; } /* raft-apply-decoupling task 3.2: bridges commit_submit_async's * commit_continuation_fn shape (commit_outcome_t + rc) to * persist_job_commit_notify_fn's shape (a single rc) so persist.h's public * declaration never needs commit.h's commit_outcome_t (see persist.h's * doc comment on persist_job_commit_notify_fn). One adapter is heap- * allocated per registered lsn and frees itself after its single * invocation — safe because commit_resolve guarantees exactly-once * delivery. */ typedef struct { persist_job_commit_notify_fn fn; void* ctx; } job_commit_adapter_ctx_t; static void job_commit_resolve_adapter(void* adapter_ctx, uint64_t lsn, commit_outcome_t outcome, int rc) { job_commit_adapter_ctx_t* a = (job_commit_adapter_ctx_t*)adapter_ctx; int final_rc = (outcome == COMMIT_OUTCOME_COMMITTED) ? 0 : rc; a->fn(a->ctx, lsn, final_rc); free(a); } int persist_submit_job_payloads_async(persist_t* p, xlist_t* payloads, uint64_t* out_lsns, persist_job_commit_notify_fn fn, void* ctx) { int first_rc = 0; if (p == NULL || payloads == NULL || out_lsns == NULL || fn == NULL) return -EINVAL; for (int i = 0; i < payloads->count; i++) { xce_job_t* j = (xce_job_t*)xlist_get(payloads, i); uint64_t lsn = 0; int rc; if (first_rc != 0) { /* A prior payload in this batch already failed locally: every * remaining payload resolves with that SAME failure now, * synchronously — so `fn` is invoked exactly once per payload * regardless of where the failure occurred, and the caller * never has to separately track "never going to be called" * entries. */ fn(ctx, 0, first_rc); continue; } rc = append_job_wal_entry_local(p, j, &lsn); if (rc != 0) { first_rc = rc; fn(ctx, 0, rc); continue; } out_lsns[i] = lsn; /* LSN-0 edge (documented once-per-cluster-lifetime interim * behavior — ha-commit-engine Implementation Notes, mirrored by * repl_wait_majority_ack's walAck branch): the first-ever WAL * entry of a fresh cluster is structurally un-commitable and * commit_submit_async rejects lsn == 0, so resolve it inline. * Engine-not-wired (bootstrap / engine-less contexts) mirrors * persist_commit_barrier's own "no engine -> no-op success" * convention — there is no async completion authority to * register with. */ if (lsn == 0 || p->repl.ce == NULL) { fn(ctx, lsn, 0); continue; } job_commit_adapter_ctx_t* a = calloc(1, sizeof(*a)); if (a == NULL) { first_rc = -ENOMEM; fn(ctx, lsn, -ENOMEM); continue; } a->fn = fn; a->ctx = ctx; int reg_rc = commit_submit_async(p->repl.ce, lsn, job_commit_resolve_adapter, a); if (reg_rc != 0) { free(a); first_rc = reg_rc; ERROR("persist_submit_job_payloads_async: lsn=%lu appended but " "commit_submit_async registration failed (rc=%d) -- the " "entry remains durable and will still be caught up " "eventually via replay, but this caller will not be told " "its outcome via this path", (unsigned long)lsn, reg_rc); fn(ctx, lsn, reg_rc); continue; } } return first_rc; } /* job-array-single-wal-record task 2.1: serialize a job-array submission * (base jobid + array size + one shared job spec) to a malloc'd Cap'n Proto * buffer, reusing the SAME capnp struct (JobSubmit) the client's own submit * request uses -- mirrors capnp_encode_job above, which mirrors encode_Job/ * new_Job/write_Job for a single job, except here via encode_JobSubmit/ * new_JobSubmit/write_JobSubmit. The encoded payload is O(1) in array_size: * arraySize is a fixed-width uint64 field, not a per-element list -- so this * function's output size never grows with array_size (Req 7.1). Caller must * free(*out) on success. */ static int capnp_encode_job_array(const xce_job_submit_t* s, uint8_t** out, size_t* out_len) { struct capn c; struct capn_segment* cs; struct JobSubmit cjs; JobSubmit_ptr jp; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_JobSubmit(cs, &cjs, (xce_job_submit_t*)s); jp = new_JobSubmit(cs); write_JobSubmit(&cjs, jp); capn_setp(capn_root(&c), 0, jp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } /* job-array-single-wal-record task 2.1: append-only, single-record * counterpart of append_job_wal_entry_local for a WHOLE job array. Encodes * ONE JobSubmit-shaped payload (base_jobid, array_size, base_spec->submit) * and appends it as exactly one WAL_JOB_ARRAY_SUBMIT entry via the SAME * primary-append primitive append_job_wal_entry_local already uses * (ha_primary_log_append / persist_wal_async) -- never one entry per array * element, regardless of array_size (Req 1.1, 1.2, 1.3). base_jobid/ * array_size are taken as explicit parameters rather than read from * base_spec->jobid/base_spec->array_size, so the caller's own * already-range-claimed values are always what gets committed, independent * of whatever same-shaped fields the client's own wire request happened to * carry. */ static int append_job_array_wal_entry_local(persist_t* p, uint64_t base_jobid, uint64_t array_size, const xce_job_submit_t* base_spec, uint64_t* out_lsn) { xce_job_submit_t wire = { .jobid = base_jobid, .array_size = array_size, .submit = base_spec->submit }; uint8_t* buf = NULL; size_t sz = 0; int rc; uint64_t lsn; if (capnp_encode_job_array(&wire, &buf, &sz) != 0) return -ENOMEM; /* Snapshot the LSN this entry will receive BEFORE the append (same * convention as append_job_wal_entry_local / submit_job_wal_entry_and_ * commit: persist_wal_async / ha_primary_log_append assigns * hdr.lsn = current_lsn then increments). */ pthread_mutex_lock(&p->lock); lsn = p->current_lsn; pthread_mutex_unlock(&p->lock); if (p->cluster != NULL) rc = ha_primary_log_append(p->cluster, WAL_JOB_ARRAY_SUBMIT, buf, (uint32_t)sz, NULL); else rc = persist_wal_async(p, WAL_JOB_ARRAY_SUBMIT, buf, (uint32_t)sz); free(buf); if (rc == 0) *out_lsn = lsn; return rc; } /* job-array-single-wal-record task 2.1 (design.md §WAL Array Submission, * Service Interface): encodes and appends ONE array-shaped WAL entry * (WAL_JOB_ARRAY_SUBMIT) and registers ONE commit continuation for it, * regardless of array_size -- the single-record sibling of * persist_submit_job_payloads_async's per-payload loop, specialized to * exactly one payload. No job_t/xce_job_t object is constructed here or * anywhere on this call path -- array expansion into individual job * entries happens later, in the shared apply seam (task 3.x's * restore_job_array), never here. * * Mirrors persist_submit_job_payloads_async's contract exactly: * - a local encode/append failure resolves `fn` synchronously with that * failure and returns the same negative rc; * - the LSN-0 / engine-not-wired edge cases (documented on * persist_submit_job_payloads_async above) resolve `fn` inline with * rc=0; * - otherwise `fn` is registered via commit_submit_async and invoked * exactly once, asynchronously, once the entry's commit outcome is * known. */ int persist_submit_job_array_async(persist_t* p, uint64_t base_jobid, uint64_t array_size, const xce_job_submit_t* base_spec, uint64_t* out_lsn, persist_job_commit_notify_fn fn, void* ctx) { uint64_t lsn = 0; int rc; if (p == NULL || base_spec == NULL || base_spec->submit == NULL || out_lsn == NULL || fn == NULL || array_size <= 1) return -EINVAL; rc = append_job_array_wal_entry_local(p, base_jobid, array_size, base_spec, &lsn); if (rc != 0) { fn(ctx, 0, rc); return rc; } *out_lsn = lsn; /* LSN-0 / engine-not-wired edge (see persist_submit_job_payloads_async's * doc comment above for the full rationale -- identical here, * specialized to one entry instead of a loop). */ if (lsn == 0 || p->repl.ce == NULL) { fn(ctx, lsn, 0); return 0; } job_commit_adapter_ctx_t* a = calloc(1, sizeof(*a)); if (a == NULL) { fn(ctx, lsn, -ENOMEM); return -ENOMEM; } a->fn = fn; a->ctx = ctx; int reg_rc = commit_submit_async(p->repl.ce, lsn, job_commit_resolve_adapter, a); if (reg_rc != 0) { free(a); ERROR("persist_submit_job_array_async: lsn=%lu appended but " "commit_submit_async registration failed (rc=%d) -- the " "entry remains durable and will still be caught up " "eventually via replay, but this caller will not be told " "its outcome via this path", (unsigned long)lsn, reg_rc); fn(ctx, lsn, reg_rc); return reg_rc; } return 0; } void persist_register_pending_apply(persist_t* p, uint64_t lsn, pending_apply_notify_fn on_applied, void* ctx) { pending_command_apply_t* node; if (p == NULL || on_applied == NULL) return; /* bug-1415 self-healing (round 2): commit.c's continuation dispatch and * the apply pump's walk (persist_apply_pump_run) are two independent * vtasks with no completion-order guarantee -- see this function's doc * comment in persist.h for the full race. If the walk has already * applied `lsn` (last_applied_lsn already at or past it) by the time we * get here, linking this registration onto pending_applies would orphan * it forever: the walk only ever visits each lsn once, and it already * called persist_take_pending_apply() for this lsn and found nothing. * Fire the hand-off immediately, synchronously, on THIS thread instead * -- never leave a continuation unresolved (controllers never * self-terminate; the same "fail loud and complete the work" posture * commit.c's dispatch_continuation fallback uses). * * bug-1428: this self-heal check MUST run under pending_applies_lock, * not before acquiring it. The window this closes: this thread reads * last_applied_lsn (sees "not yet applied"), then -- BEFORE it acquires * the lock below to insert -- the apply pump's thread advances * last_applied_lsn past `lsn` AND completes its own * persist_take_pending_apply(lsn) call (finding nothing, since this * thread hasn't inserted yet). This thread then inserts a node the walk * will never revisit -- a permanently orphaned hand-off, the exact * class of hang the round-2 self-heal above was meant to eliminate, * reintroduced by the gap between the unlocked check and the locked * insert. Holding pending_applies_lock across BOTH the check and the * insert makes the two mutually exclusive with * persist_take_pending_apply's own locked scan: either this thread's * check runs before the pump's take (sees "not yet applied", inserts, * and the pump's later take -- now blocked on this same lock until this * critical section ends -- finds the freshly-inserted node), or it runs * after (sees "already applied", self-heals here instead of inserting, * and the pump's take correctly finds nothing) -- never the interleaved * gap that orphans it. */ pthread_mutex_lock(&p->pending_applies_lock); if (lsn <= persist_last_applied_lsn(p)) { pthread_mutex_unlock(&p->pending_applies_lock); on_applied(ctx, lsn); return; } node = calloc(1, sizeof(*node)); if (node == NULL) { pthread_mutex_unlock(&p->pending_applies_lock); ERROR("persist_register_pending_apply: OOM registering the apply " "hand-off for lsn=%lu -- the entry is still durably committed " "and will still be applied by a future restart replay, but no " "continuation will fire for it via this path", (unsigned long)lsn); return; } node->lsn = lsn; node->on_applied = on_applied; node->ctx = ctx; node->next = (pending_command_apply_t*)p->pending_applies; p->pending_applies = node; pthread_mutex_unlock(&p->pending_applies_lock); } int persist_has_pending_apply(persist_t* p, uint64_t lsn) { pending_command_apply_t* n; int found = 0; if (p == NULL) return 0; pthread_mutex_lock(&p->pending_applies_lock); for (n = (pending_command_apply_t*)p->pending_applies; n != NULL; n = n->next) { if (n->lsn == lsn) { found = 1; break; } } pthread_mutex_unlock(&p->pending_applies_lock); return found; } int persist_take_pending_apply(persist_t* p, uint64_t lsn, pending_apply_notify_fn* out_fn, void** out_ctx) { pending_command_apply_t* prev = NULL; pending_command_apply_t* n; int found = 0; if (p == NULL) return 0; pthread_mutex_lock(&p->pending_applies_lock); for (n = (pending_command_apply_t*)p->pending_applies; n != NULL; n = n->next) { if (n->lsn == lsn) { if (prev == NULL) p->pending_applies = n->next; else prev->next = n->next; if (out_fn) *out_fn = n->on_applied; if (out_ctx) *out_ctx = n->ctx; found = 1; break; } prev = n; } pthread_mutex_unlock(&p->pending_applies_lock); free(n); /* NULL when not found -- free(NULL) is a no-op */ return found; } /* raft-apply-decoupling task 4.1 (Req 5.1): the receipt-time durable-append * primitive for a worker-reported job-state fact. Unlike persist_job_hook_fun * below (an AFTER_EVENT hook that fires only once a mutation has already * happened), this is a direct, hook-fan-out-free call: job.c's fact receipt * handlers (record_job_start_fact/record_job_finish_fact/record_job_fail_fact/ * record_job_postdone_fact) call this (or persist_submit_job_fact_ex below) * INSTEAD of mutating and firing AFTER_EVENT, so no notification (scheduler * dispatch, slot release, apiserver push) can ever fire for an uncommitted * fact (Req 5.3). `state` is always WAL_MODE_RETAIN for every job-state fact * type (verified against wal_classify's table) — hardcoded here rather than * routed through wal_classify since none of these types are detail- * disambiguated the way WAL_WORKER_CTRL is. * * Thin wrapper over persist_submit_job_fact_ex() with pad = 0 — the "legacy * absolute state" discriminator (see wal_types.h's doc comment on * wal_jobstate_t._pad). Unchanged contract/signature for every existing * caller. */ int persist_submit_job_fact(persist_t* p, uint32_t wal_type, uint64_t jobid, int state) { return persist_submit_job_fact_ex(p, wal_type, jobid, state, 0); } /* raft-apply-decoupling task 6.1 round 2 (Req 3.3/7.1-7.3): full-control * sibling of persist_submit_job_fact() above, letting the caller set * wal_jobstate_t's `_pad` discriminator explicitly. Used by * record_job_fail_fact (WAL_JOB_FAIL_STAGE_FLAG | stage) and * record_job_postdone_fact (WAL_JOB_UPDATE_POSTDONE_MARK) to mark a fact * whose real mutation must be re-derived at apply time against the job's * CURRENT state, rather than trusting a receipt-time snapshot baked into * `state` — see wal_types.h's doc comment on wal_jobstate_t._pad for the * full rationale. */ int persist_submit_job_fact_ex(persist_t* p, uint32_t wal_type, uint64_t jobid, int state, uint32_t pad) { wal_jobstate_t js = { .jobid = jobid, .state = (uint32_t)state, ._pad = pad }; return wal_hook_append_classified(p, wal_type, WAL_MODE_RETAIN, &js, sizeof(js)); } /* bug-1678: WAL_JOB_DISPATCH's own richer sibling of persist_submit_job_fact() * -- the ONLY job-state fact type that needs to carry a resource allocation * alongside {jobid, state}, since it is the sole durable record of the * dispatch-time PEND->RUN transition (design.md "WAL Dispatch Fact"). Payload * shape (bug-1697 update): wal_jobstate_t header, with * WAL_JOB_DISPATCH_HAS_START_TIME_FLAG set in _pad, immediately followed by * a fixed uint64_t start_time (the dispatch-time clock, captured ONCE by the * caller under the job's write lock -- job.start_time is now DISPATCH-owned, * see wal_types.h's WAL_JOB_START doc comment for the full rationale), * immediately followed by the Cap'n Proto-encoded JobAlloc bytes. * apply_committed_entry()'s and apply_wal_entry()'s WAL_JOB_DISPATCH cases * decode exactly this shape, keyed on the _pad flag (not payload length, * which is ambiguous against JobAlloc's own variable length) -- an entry * with the flag clear (any WAL_JOB_DISPATCH entry written before this fix) * decodes the pre-fix shape (header + JobAlloc bytes only, start_time left * untouched) for full backward compatibility. */ int persist_submit_job_dispatch_fact(persist_t* p, uint64_t jobid, int state, uint64_t start_time, const xce_job_alloc_t* alloc) { wal_jobstate_t js = { .jobid = jobid, .state = (uint32_t)state, ._pad = WAL_JOB_DISPATCH_HAS_START_TIME_FLAG }; uint8_t* allocbuf = NULL; size_t alloclen = 0; uint8_t* payload; size_t total; int rc; rc = capnp_encode_joballoc(alloc, &allocbuf, &alloclen); if (rc != 0) return rc; total = sizeof(js) + sizeof(start_time) + alloclen; payload = malloc(total); if (!payload) { free(allocbuf); return -ENOMEM; } memcpy(payload, &js, sizeof(js)); memcpy(payload + sizeof(js), &start_time, sizeof(start_time)); memcpy(payload + sizeof(js) + sizeof(start_time), allocbuf, alloclen); free(allocbuf); rc = wal_hook_append_classified(p, WAL_JOB_DISPATCH, WAL_MODE_RETAIN, payload, (uint32_t)total); free(payload); return rc; } /* bug-1687: WAL_JOB_START's own richer sibling of persist_submit_job_fact() * -- mirrors persist_submit_job_dispatch_fact() above (bug-1678) exactly, * but for the worker-confirmed-start fact: the sole durable record of the * job's real job_pidinfo_t (pids/pgids/exdata a worker reported) and its * true start_time. Payload shape: wal_jobstate_t header, immediately * followed by a fixed uint64_t start_time (the receipt-time clock, captured * ONCE by the caller rather than re-derived at each apply seam's own * apply-time moment), immediately followed by the Cap'n Proto-encoded * JobPIDInfo bytes -- apply_committed_entry()'s and apply_wal_entry()'s * WAL_JOB_START cases decode exactly this shape, and treat a payload of * exactly sizeof(wal_jobstate_t) (no trailing bytes) as "state-only, * fabricate start_time and leave pidinfo untouched" for backward * compatibility with any WAL_JOB_START entry written before this fix. */ int persist_submit_job_start_fact(persist_t* p, uint64_t jobid, int state, uint64_t start_time, const job_pidinfo_t* info) { wal_jobstate_t js = { .jobid = jobid, .state = (uint32_t)state, ._pad = 0 }; uint8_t* infobuf = NULL; size_t infolen = 0; uint8_t* payload; size_t total; int rc; rc = capnp_encode_pidinfo(info, &infobuf, &infolen); if (rc != 0) return rc; total = sizeof(js) + sizeof(start_time) + infolen; payload = malloc(total); if (!payload) { free(infobuf); return -ENOMEM; } memcpy(payload, &js, sizeof(js)); memcpy(payload + sizeof(js), &start_time, sizeof(start_time)); memcpy(payload + sizeof(js) + sizeof(start_time), infobuf, infolen); free(infobuf); rc = wal_hook_append_classified(p, WAL_JOB_START, WAL_MODE_RETAIN, payload, (uint32_t)total); free(payload); return rc; } /* bug-1689: WAL_JOB_FINISH's own richer sibling of persist_submit_job_fact() * -- mirrors persist_submit_job_start_fact() (bug-1687) exactly, but for the * worker-confirmed-finish fact: the sole durable record of the job's real * exit_code and its true finish_time. Payload shape: wal_jobstate_t header, * immediately followed by a fixed uint64_t finish_time (the receipt-time * clock, captured ONCE by the caller rather than re-derived at each apply * seam's own apply-time moment), immediately followed by a fixed int32_t * exit_code -- a small NATIVE trailer, not capnp (job_exitinfo_t is a single * int with no existing capnp mapping, unlike WAL_JOB_START's variable-length * JobPIDInfo, so no malloc is needed here -- the whole payload is a fixed * size). apply_committed_entry()'s and apply_wal_entry()'s WAL_JOB_FINISH * cases decode exactly this shape, and treat a payload of exactly * sizeof(wal_jobstate_t) (no trailing bytes) as "state-only, fabricate * finish_time and leave exit_info untouched" for backward compatibility * with any WAL_JOB_FINISH entry written before this fix. run_time is NOT * carried -- it is re-derived (finish_time - start_time) at each apply seam * once both are known. */ int persist_submit_job_finish_fact(persist_t* p, uint64_t jobid, int state, uint64_t finish_time, int exit_code) { wal_jobstate_t js = { .jobid = jobid, .state = (uint32_t)state, ._pad = 0 }; int32_t ec = (int32_t)exit_code; uint8_t payload[sizeof(js) + sizeof(finish_time) + sizeof(ec)]; memcpy(payload, &js, sizeof(js)); memcpy(payload + sizeof(js), &finish_time, sizeof(finish_time)); memcpy(payload + sizeof(js) + sizeof(finish_time), &ec, sizeof(ec)); return wal_hook_append_classified(p, WAL_JOB_FINISH, WAL_MODE_RETAIN, payload, (uint32_t)sizeof(payload)); } /* bug-1679: see persist.h's doc comment for the full rationale. Mirrors * apply_committed_entry()'s own identical Hazard-1-guarded hook fire * (its shared WAL_JOB_START/.../DISPATCH case, right after its * `if (result != JOB_TRANSITION_APPLIED) return 0;` line) exactly -- * same guard, same hook, same detail. The only difference is WHO is * calling: there it is the apply seam notifying after a REPLAYED * transition; here it is a live caller (dispatch_a_job_()) notifying * after a transition it already durably persisted itself. */ void persist_notify_job_updated_already_persisted(cluster_t* c, job_t* j) { t_apply_seam_notifying = true; xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_UPDATE, c, j); t_apply_seam_notifying = false; } /* P0 payload: just the detail value (4 bytes). * Full Cap'n Proto serialization is wired in Task 15. */ int persist_job_hook_fun(int type, int situation, int detail, void* cls, va_list ap) { (void)type; /* Write to WAL only after the state change is complete. * BEFORE_EVENT has a different va_list layout and must be ignored. */ if (situation != HOOK_AFTER_EVENT) return 0; /* raft-apply-decoupling task 6.1 (Hazard 1): this AFTER_EVENT firing * originated from apply_committed_entry()'s own post-mutation notify — * the entry is already durably committed (that is why it is being * applied at all), so appending it again here would duplicate it. See * t_apply_seam_notifying's doc comment (above apply_committed_entry) for * why this MUST be a call-provenance guard, not a `detail` filter: * job_ctrl's mark_job_update/mark_job_fail still fire this exact same * hook for their own live, still-legitimate append. */ if (t_apply_seam_notifying) return 0; persist_t* p = cls; uint32_t wal_type = detail_to_wal_type(detail); wal_commit_mode_t mode; if (wal_type == 0) return 0; mode = wal_classify(wal_type, detail); if (mode == WAL_MODE_WAIT) { /* WAL_JOB_SUBMIT durability (WAL append + majority-commit barrier) is * an explicit step invoked directly by handle_jobsubmit — * persist_submit_job_payloads_and_commit(), called strictly before * any job object is constructed (raft-apply-decoupling task 3.1 / * bug-1306) — never from this hook. This hook fires only after that * commit has already succeeded, purely as a notification, so there * is nothing left for it to do here. */ return 0; } /* All other job events: payload = wal_jobstate_t {jobid, state}. * State is read from the job_t after the AFTER_EVENT hook updates it. */ wal_jobstate_t js = {0}; /* bug-1690: WAL_JOB_RUSAGE's own optional trailing Cap'n Proto * JobRunusage bytes -- populated only by the HOOK_DETAIL_JOB_RUSAGE_UPDATE * arm below, consumed by the shared append at the bottom of this * function. Every other detail leaves this NULL, so their payload stays * the plain wal_jobstate_t it always was -- zero behavior change for * them. */ uint8_t* rusage_buf = NULL; size_t rusage_len = 0; /* bug-1691: WAL_JOB_CKPT's own optional trailing NUL-terminated * ckptinfo string -- populated only by the HOOK_DETAIL_JOB_CKPT arm * below, consumed by the shared append at the bottom of this function. * Every other detail leaves this NULL, so their payload stays the plain * wal_jobstate_t it always was -- zero behavior change for them. */ uint8_t* ckpt_buf = NULL; size_t ckpt_len = 0; /* bug-1692: WAL_JOB_PREASON's own optional trailing payload -- populated * only by the HOOK_DETAIL_SETPREASON/HOOK_DETAIL_CLRPREASON arm below, * consumed by the shared append at the bottom of this function. * pend_present stays -1 for every OTHER detail (unchanged, plain * wal_jobstate_t payload). Unlike rusage_buf/ckpt_buf above, * pend_reasons has a genuine CLEAR operation (do_clr_preason) that must * be distinguishable on replay from "old-format payload, leave * untouched" -- so this fact type always appends an explicit presence * byte (0 or 1) once pend_present >= 0, even when pend_buf stays NULL * (the clear case). */ uint8_t* pend_buf = NULL; size_t pend_len = 0; int pend_present = -1; switch (detail) { case HOOK_DETAIL_JOB_START: { /* args: cluster_t*, job_pidinfo_t*, job_t* */ (void)va_arg(ap, cluster_t*); (void)va_arg(ap, void*); job_t* j = va_arg(ap, job_t*); js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state; break; } case HOOK_DETAIL_JOB_EXIT: { /* args: cluster_t*, xce_jobfinish_t*, job_t* */ (void)va_arg(ap, cluster_t*); (void)va_arg(ap, void*); job_t* j = va_arg(ap, job_t*); js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state; break; } case HOOK_DETAIL_JOB_FAIL: { /* args: cluster_t*, job_failinfo_t*, job_t* */ (void)va_arg(ap, cluster_t*); (void)va_arg(ap, void*); job_t* j = va_arg(ap, job_t*); js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state; break; } case HOOK_DETAIL_JOB_RUSAGE_UPDATE: { /* args: cluster_t*, xce_job_rusages_t* -- bug-451: a DIFFERENT * va_list shape than every other case below. job.c's * update_job_rusages() fires this detail with the raw rusages * payload it received (not the job_t* it already holds * write-locked), because this hook's other two listeners * (sched_job_rusage_update_fun / apiserver_job_rusage_update_fun) * genuinely need the full payload (pids/pgids/usages), not just a * job_t*. Reading the second vararg as job_t* here (as the * default arm below does) would misinterpret an * xce_job_rusages_t* as a job_t* and read state from a byte * offset past the smaller struct's end. Look the job up by * rusages->jobid via the cluster_t* this call already carries -- * the same jobmgr lookup job.c's own caller already did moments * earlier -- to recover the real state for this generic * wal_jobstate_t fact. */ cluster_t* c = va_arg(ap, cluster_t*); xce_job_rusages_t* rusages = va_arg(ap, xce_job_rusages_t*); job_t* j = NULL; lock(c->jobmgr, RTTI_LOCK_READ); key_find(c->jobmgr, (void*)rusages->jobid, (void**)&j); unlock(c->jobmgr); if (j == NULL) return 0; /* job gone by the time this fired -- nothing to log */ js.jobid = rusages->jobid; js.state = (uint32_t)j->job.state; /* bug-1690: WAL_JOB_RUSAGE previously carried only {jobid, state} * -- the same silent-field-drop bug class as bug-1678/1687/1689 -- * the client-reported per-host CPU/mem usage (j->job.run_usage, * just mutated in-place by update_job_rusages() moments ago, under * the same write lock this AFTER_EVENT fires inside) was never in * the log at all, so a restart replay left job.run_usage * permanently NULL. Encode it as a trailing Cap'n Proto JobRunusage * blob when present -- it is NULL only in test doubles that stub * out update_host_runusage (e.g. test_commit_barrier_reply.c); the * real production caller always has one by the time this fires. A * NULL run_usage falls back to the legacy state-only payload at * this function's shared tail, unchanged. */ if (j->job.run_usage != NULL) (void)capnp_encode_jobrunusage(j->job.run_usage, &rusage_buf, &rusage_len); break; } case HOOK_DETAIL_JOB_CKPT: { /* args: cluster_t*, job_t* -- same shape as the default arm below, * split out to also capture j->job.ckptinfo (bug-1691: same * silent-field-drop bug class as bug-1678/1687/1689/1690 -- * WAL_JOB_CKPT previously carried only {jobid, state}, so a restart * replay left job.ckptinfo permanently NULL even though * on_job_ckpt() (job.c) had just set it, under the same write lock * this AFTER_EVENT fires inside). j->job.ckptinfo is a plain * NUL-terminated char* -- xstring-backed (makeString(), xstring.c), * NOT the richer xce_ckptinfo_t/CheckpointInfo capnp struct used * elsewhere for a different purpose -- so no capnp wrapper is * needed; append it directly as a NUL-terminated string trailing * the header, mirroring WAL_QUEUE_CTRL's own [header][name * NUL-terminated] wire shape (persist_queue_hook_fun below) * exactly. NULL only in test doubles that never set job.ckptinfo -- * the real production caller (on_job_ckpt) always sets it via * makeString() before this hook fires; a NULL ckptinfo falls back * to the legacy state-only payload at this function's shared tail, * unchanged. */ (void)va_arg(ap, cluster_t*); job_t* j = va_arg(ap, job_t*); js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state; if (j->job.ckptinfo != NULL) { ckpt_len = strlen(j->job.ckptinfo) + 1; ckpt_buf = malloc(ckpt_len); if (ckpt_buf != NULL) memcpy(ckpt_buf, j->job.ckptinfo, ckpt_len); else ckpt_len = 0; /* OOM -- fall back to the state-only payload */ } break; } case HOOK_DETAIL_SETPREASON: case HOOK_DETAIL_CLRPREASON: { /* args: cluster_t*, job_t* -- same shape as the default arm below, * split out to also capture j->job.pend_reasons (bug-1692: same * silent-field-drop bug class as bug-1678/1687/1689/1690/1691 -- * WAL_JOB_PREASON previously carried only {jobid, state}, so a * restart replay left job.pend_reasons permanently NULL/stale even * though set_preason()/do_clr_preason() (job.c) had just set/cleared * it, under the same write lock this AFTER_EVENT fires inside). * pend_present distinguishes "cleared" (0, do_clr_preason has just * FREEUP'd job.pend_reasons back to NULL) from "set" (1, capnp * JobPendReason bytes follow) -- both HOOK_DETAIL_SETPREASON and * HOOK_DETAIL_CLRPREASON share this arm since the shared append * tail below decides purely from j->job.pend_reasons' CURRENT value, * not from which detail fired. */ (void)va_arg(ap, cluster_t*); job_t* j = va_arg(ap, job_t*); js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state; if (j->job.pend_reasons != NULL) { pend_present = 1; (void)capnp_encode_jobpreason(j->job.pend_reasons, &pend_buf, &pend_len); } else { pend_present = 0; } break; } default: { /* STOP, RESUME, UPDATE — args: cluster_t*, job_t* */ (void)va_arg(ap, cluster_t*); job_t* j = va_arg(ap, job_t*); js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state; break; } } /* mode == WAL_MODE_RETAIN — fact-shaped job state transition */ if (rusage_buf != NULL) { /* bug-1690: append the header + trailing JobRunusage bytes as one * contiguous payload -- persist_submit_job_dispatch_fact()'s/ * persist_submit_job_start_fact()'s own header+trailer construction * (bug-1678/1687), inlined here since WAL_JOB_RUSAGE is hook-driven * (from the raw rusages payload update_job_rusages() fires, not a * direct persist_submit_job_*_fact() call site job.c could own). */ size_t total = sizeof(js) + rusage_len; uint8_t* payload = malloc(total); int rc; if (!payload) { free(rusage_buf); return -ENOMEM; } memcpy(payload, &js, sizeof(js)); memcpy(payload + sizeof(js), rusage_buf, rusage_len); free(rusage_buf); rc = wal_hook_append_classified(p, wal_type, mode, payload, (uint32_t)total); free(payload); return rc; } if (ckpt_buf != NULL) { /* bug-1691: append the header + trailing ckptinfo bytes as one * contiguous payload -- mirrors the rusage_buf construction just * above (bug-1690) and persist_queue_hook_fun's own header+trailer * construction for WAL_QUEUE_CTRL (this function's own file, just * below). */ size_t total = sizeof(js) + ckpt_len; uint8_t* payload = malloc(total); int rc; if (!payload) { free(ckpt_buf); return -ENOMEM; } memcpy(payload, &js, sizeof(js)); memcpy(payload + sizeof(js), ckpt_buf, ckpt_len); free(ckpt_buf); rc = wal_hook_append_classified(p, wal_type, mode, payload, (uint32_t)total); free(payload); return rc; } if (pend_present >= 0) { /* bug-1692: append [header][presence byte][optional capnp * JobPendReason bytes] as one contiguous payload -- mirrors the * rusage_buf/ckpt_buf constructions just above (bug-1690/1691), but * ALWAYS appends (even when pend_buf is NULL, the clear case) since * the presence byte itself is the signal both apply seams need to * distinguish "cleared" from "old-format, don't touch". */ size_t total = sizeof(js) + 1 + pend_len; uint8_t* payload = malloc(total); int rc; if (!payload) { free(pend_buf); return -ENOMEM; } memcpy(payload, &js, sizeof(js)); payload[sizeof(js)] = (uint8_t)pend_present; if (pend_len > 0) memcpy(payload + sizeof(js) + 1, pend_buf, pend_len); free(pend_buf); rc = wal_hook_append_classified(p, wal_type, mode, payload, (uint32_t)total); free(payload); return rc; } return wal_hook_append_classified(p, wal_type, mode, &js, sizeof(js)); } int persist_queue_hook_fun(int type, int situation, int detail, void* cls, va_list ap) { (void)type; if (situation != HOOK_AFTER_EVENT) return 0; /* raft-apply-decoupling task 6.1 (Hazard 1, queue analog): see * persist_job_hook_fun's identical guard above for the full rationale — * apply_committed_entry()'s WAL_QUEUE_CTRL branch fires this same * AFTER_EVENT to notify, and this hook appends unconditionally * (WAL_QUEUE_CTRL is WAL_MODE_WAIT with no early-return here), so * without this guard a live-applied queue entry would re-append and * re-trigger a nested majority-commit wait every time the pump walks * it. */ if (t_apply_seam_notifying) return 0; persist_t* p = cls; uint32_t wal_type = detail_to_wal_type(detail); wal_commit_mode_t mode; if (wal_type == 0) return 0; mode = wal_classify(wal_type, detail); /* args: cluster_t*, queue_t* */ (void)va_arg(ap, cluster_t*); queue_t* q = va_arg(ap, queue_t*); /* Payload: [wal_queuehdr_t][queue name NUL-terminated] */ const char* name = q->queue.queue ? q->queue.queue : ""; size_t name_len = strlen(name) + 1; size_t total = sizeof(wal_queuehdr_t) + name_len; uint8_t* pkt = malloc(total); if (!pkt) return -ENOMEM; wal_queuehdr_t hdr = { .status = q->queue.status }; memcpy(pkt, &hdr, sizeof(hdr)); memcpy(pkt + sizeof(hdr), name, name_len); /* mode == WAL_MODE_WAIT — the append stays async; the handler barrier * owes commit completion before the requester reply (task 3) */ int rc = wal_hook_append_classified(p, wal_type, mode, pkt, (uint32_t)total); free(pkt); return rc; } int persist_worker_hook_fun(int type, int situation, int detail, void* cls, va_list ap) { (void)type; if (situation != HOOK_AFTER_EVENT) return 0; /* raft-apply-decoupling task 6.1 (Hazard 1, worker analog): see * persist_job_hook_fun's identical guard above for the full rationale. */ if (t_apply_seam_notifying) return 0; persist_t* p = cls; uint32_t wal_type = detail_to_wal_type(detail); wal_commit_mode_t mode; if (wal_type == 0) return 0; mode = wal_classify(wal_type, detail); /* WAL_WORKER_CTRL payload layout: [uint32_t detail][capnp worker OR name] */ uint32_t disc = (uint32_t)detail; uint8_t* buf = NULL; size_t capnp_sz = 0; int rc = 0; /* All worker detail types: args are cluster_t* then worker_t*. * JOIN uses a unified signature with an extra char* name between them; * DEAD and UPDATE use (cluster_t*, worker_t*) directly. */ (void)va_arg(ap, cluster_t*); if (detail == HOOK_DETAIL_WORKER_JOIN) (void)va_arg(ap, char*); /* skip name — present only for JOIN */ worker_t* w = va_arg(ap, worker_t*); if (!w) return 0; /* guard: failed join produced no worker slot */ rc = capnp_encode_worker(&w->host, &buf, &capnp_sz); if (rc != 0) return rc; /* Build [disc | capnp bytes] in one allocation */ size_t total = WORKER_WAL_HDR_SIZE + capnp_sz; uint8_t* pkt = malloc(total); if (!pkt) { free(buf); return -ENOMEM; } memcpy(pkt, &disc, WORKER_WAL_HDR_SIZE); memcpy(pkt + WORKER_WAL_HDR_SIZE, buf, capnp_sz); free(buf); /* JOIN/UPDATE → WAIT (handler barrier owes completion, task 3); * DEAD → RETAIN (controller-detected fact, registered with the engine) */ rc = wal_hook_append_classified(p, wal_type, mode, pkt, (uint32_t)total); free(pkt); return rc; } /* ============================================================ * GC retention predicates (P0-5a / P0-8 / P0-13) * * Placed in persist.c so the unit test binary (which links only * persist.c from the controller sources) can access them without * pulling in RTTI-dependent job.c / worker.c. * ============================================================ */ bool job_is_terminal(const job_t* j) { return (j->job.state & (JOB_STAT_EXIT | JOB_STAT_DONE | JOB_STAT_PERR)) != 0; } bool job_should_retain(const job_t* j, const persist_config_t* cfg) { time_t now = time(NULL); /* TRANSIENT jobs: auto-terminate when TTL elapses after failover. * transient_since == 0 means the timestamp was never set — do not GC. */ if (j->job.state & JOB_STAT_TRANSIENT) { if (j->job.transient_since > 0 && now > (time_t)j->job.transient_since + (time_t)cfg->transient_job_timeout_s) { return false; } return true; } /* Non-terminal jobs are always retained */ if (!job_is_terminal(j)) return true; /* Terminal jobs: retain until finished_retain_s expires */ return (j->job.finish_time + (time_t)cfg->finished_retain_s >= now); } bool worker_should_retain(const xce_worker_t* w, const persist_config_t* cfg) { time_t now = time(NULL); /* HIDDEN workers: admin explicitly removed — clean up at next checkpoint */ if (w->status & XCE_WORKER_HIDDEN) return false; /* UNAVAILABLE workers: retain until worker_retain_s expires. * dead_since == 0 means not yet set — do not GC. */ if (w->status & XCE_WORKER_UNAVAIL) { if (w->dead_since > 0 && now > w->dead_since + (time_t)cfg->worker_retain_s) { return false; } } return true; } /* ============================================================ * WAL truncation (task 7.5) * * Remove all WAL entries with LSN > cutoff_lsn. * Idempotent: calling twice with same cutoff_lsn returns 0 both times. * ============================================================ */ /* Clamp durable_lsn down to `tip` when the log tip recedes below it (truncate * / divergent-suffix discard). Without this, durable_lsn could exceed * current_lsn - 1 after a truncate, violating its invariant and letting the * commit majority count self at an LSN whose bytes were just removed. MUST be * called with p->lock held (mirrors the committed_lsn clamp in the truncate * paths). durable_lsn only ever decreases here — the sole exception to its * monotonic-non-decreasing rule (ha-commit-durability 2.1 State Management). */ static void clamp_durable_lsn_locked(persist_t* p, uint64_t tip) { uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); if (dur > tip) atomic_store_explicit(&p->durable_lsn, tip, memory_order_release); } int persist_truncate_wal_above(persist_t* p, uint64_t cutoff_lsn) { pthread_mutex_lock(&p->lock); uint64_t before = p->current_lsn; int rc = truncate_wal_above_locked(p, cutoff_lsn); uint64_t after = p->current_lsn; pthread_mutex_unlock(&p->lock); /* raft-apply-decoupling task 1.4: fire the truncation wake only when a * truncation actually removed entries above cutoff_lsn (current_lsn * strictly decreased -- both truncate_wal_above_locked branches set * current_lsn = cutoff_lsn + 1 on a real truncation) -- mirrors the * durable-advance wake's own before/after guard (see fsync_worker * above) so a no-op truncate (nothing above cutoff_lsn, or an I/O * error before current_lsn is touched) never spuriously fails a * continuation whose entry is still safely in the log. Called AFTER * releasing p->lock: the callback (the commit engine) takes only * ce->lock, preserving the cycle-free p->lock -> ce->lock edge already * established by the durable-advance wake. */ if (rc == 0 && after < before) { /* raft-apply-decoupling task 7.4 perf-fix: a real truncation may * ftruncate()/unlink() segment files the apply-pump's segment-lookup * cache still points into -- invalidate it so a subsequent * apply_pump_read_entry can never read truncated-away data. p->lock * is already released above and apply_pump_lock is never held by * this call path (persist_apply_pump_run is the only place that * holds it, during its own walk), so no lock-order hazard here. */ pthread_mutex_lock(&p->apply_pump_lock); if (p->apply_pump_cache_fd >= 0) { close(p->apply_pump_cache_fd); p->apply_pump_cache_fd = -1; } p->apply_pump_cache_next_lsn = 0; pthread_mutex_unlock(&p->apply_pump_lock); if (p->on_truncate != NULL) { p->on_truncate(p->truncate_cb_ctx, cutoff_lsn); } } return rc; } /* persist_committed_lsn_advance_atomic (2026-07-10, controller-invariant-audit * finding): race-safe monotonic-max advance of p->committed_lsn via a CAS * retry loop. Replaces the naive "load; if target>cur store" pattern, which * can lose an update if two writers race: persist_promotion_advance_committed * (below) and ha_standby_advance_committed_and_apply (ha.c) can both run * during the promotion window before role flips to CTRL_PRIMARY (a * still-in-flight WAL catch-up entry processed on another thread at the same * instant as this node's own election completing). No lock required -- CAS * only succeeds if the value hasn't changed since the read, so a losing * racer retries with the freshly-observed value and can never clobber a * larger value with a smaller one. */ uint64_t persist_committed_lsn_advance_atomic(persist_t* p, uint64_t target) { uint64_t cur = atomic_load_explicit(&p->committed_lsn, memory_order_acquire); while (target > cur) { if (atomic_compare_exchange_weak_explicit(&p->committed_lsn, &cur, target, memory_order_release, memory_order_acquire)) return target; /* cur was refreshed to the current value by the failed CAS; retry. */ } return cur; } /* persist_promotion_advance_committed (2026-07-10, promotion apply-lag fix): * unconditionally advance p->committed_lsn toward `safe_lsn` at promotion * time. persist_truncate_wal_above only sets committed_lsn on the branches * of truncate_wal_above_locked that perform a REAL truncation (partial * truncate, or the full-segment-purge demotion-convergence case); its no-op * early return (nothing above cutoff_lsn to remove -- the common case for a * newly-elected leader whose own log already covers safe_lsn) leaves * committed_lsn untouched at its pre-election value. Without this call, * committed_lsn -- and therefore apply, which is commit-gated * (raft-apply-decoupling) -- stalls until the normal per-entry ack-driven * commit path organically catches it up, which can take seconds while * surviving standbys reconnect and re-ACK. During that gap a job already * committed on a majority is invisible via expect-job/joblist on the very * primary that is authoritative for it (bug-828 class). * * DESIGN NOTE -- why this function never forces a synchronous flush (round 3, * 2026-07-10, controller-invariant-audit): an earlier round of this fix * called flusher_flush_once(p) here when durable_lsn lagged safe_lsn, to * force this node's own fsync to catch up immediately rather than waiting on * fsync_worker's cadence. That was a real bug-1198-CLASS thread-role * violation: the real multi-node winning-vote-reply path reaches * raft_cb_become_leader (and so this function) via peer_ctrl_classify's * VTASK_LIVENESS dispatch (ha.c ~5935-5940) -- a task class the design * DOCUMENTS as "never admission-gated" / never expected to block (see the * bug-1198 hazard box a few hundred lines above peer_ctrl_classify's * definition, which closed the EXACT same class of bug for * HA_REPL_SEND_SNAPSHOT's persist_checkpoint). A blocking fdatasync here * would reintroduce that closed bug class on the pool's liveness-reserved * workers. The visibility gap the force-flush was trying to close is instead * closed non-blockingly by ha_start_primary_heartbeat sending its first * heartbeat synchronously at promotion (bug-1432) -- that lets a reconnecting * standby's real position get learned, and the entry gets committed via the * normal ack-driven commit_advance_locked path, within one network RTT * instead of waiting a full heartbeat interval. A live chaotic soak (10 * rounds) confirmed durable-capped-only + immediate-heartbeat together reach * committed_job_loss=0 with zero blocking calls added to this function. * * Durable-capped (never advances committed_lsn past this node's own * durable_lsn -- see persistence.md's "acknowledged implies durable on a * majority" core invariant) and monotonic + race-safe via * persist_committed_lsn_advance_atomic. */ void persist_promotion_advance_committed(persist_t* p, uint64_t safe_lsn) { if (!p) return; uint64_t durable = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); uint64_t target = safe_lsn < durable ? safe_lsn : durable; persist_committed_lsn_advance_atomic(p, target); } /* Lock-held core of persist_truncate_wal_above (see the forward declaration and * the conflict-guard reuse near persist_wal_write_replicated). Caller MUST hold * p->lock; this function never touches the lock so it is reusable from another * already-locked path. */ static int truncate_wal_above_locked(persist_t* p, uint64_t cutoff_lsn) { /* No-op: nothing written yet, or cutoff covers all written entries */ if (p->current_lsn == 0 || cutoff_lsn + 1 >= p->current_lsn) { return 0; } /* --- Collect segment filenames and sort by start LSN --- */ uint64_t seg_lsns[4096]; int nseg = 0; DIR* d = opendir(p->wal_path); if (d) { struct dirent* e; while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) { if (strncmp(e->d_name, "seg-", 4) != 0) continue; uint64_t sl = 0; if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &sl) != 1) continue; seg_lsns[nseg++] = sl; } closedir(d); } if (nseg == 0) { return 0; } /* Insertion sort ascending by start LSN */ for (int i = 1; i < nseg; i++) { uint64_t key = seg_lsns[i]; int j = i - 1; while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; } seg_lsns[j + 1] = key; } /* --- Find segment containing cutoff_lsn --- */ int cutoff_seg_idx = -1; for (int i = 0; i < nseg; i++) { if (seg_lsns[i] <= cutoff_lsn) cutoff_seg_idx = i; else break; } if (cutoff_seg_idx < 0) { /* cutoff_lsn is below the oldest retained segment's start LSN, so EVERY * on-disk entry has LSN > cutoff_lsn and must be removed (the function's * contract: "remove all entries with LSN > cutoff_lsn"). This is the * demotion-convergence case (ha-convergence-recovery Req 1.2): a demoted * ex-primary whose committed boundary sits below its entire retained log * must discard all of it and collapse the tip to cutoff_lsn + 1. Delete * every segment, close the active fd, and reset the in-memory tip. */ for (int i = 0; i < nseg; i++) { char del_path[4096]; snprintf(del_path, sizeof(del_path), WAL_SEG_FMT, p->wal_path, seg_lsns[i]); unlink(del_path); } if (p->wal_fd >= 0) { fdatasync(p->wal_fd); close(p->wal_fd); p->wal_fd = -1; } p->seg_offset = 0; p->seg_path[0] = '\0'; p->current_lsn = cutoff_lsn + 1; p->last_crc = WAL_CRC_NONE; atomic_store_explicit(&p->committed_lsn, cutoff_lsn, memory_order_release); clamp_durable_lsn_locked(p, cutoff_lsn); /* Keep the per-LSN term index consistent: drop terms for removed LSNs. */ truncate_term_index_locked(p, cutoff_lsn); return 0; } /* --- Scan segment to find byte offset of first entry with lsn > cutoff --- */ char cutoff_seg_path[4096]; snprintf(cutoff_seg_path, sizeof(cutoff_seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[cutoff_seg_idx]); int scan_fd = open(cutoff_seg_path, O_RDONLY); if (scan_fd < 0) { return -errno; } uint32_t scan_off = 0; uint32_t trunc_off = 0; uint32_t new_last_crc = WAL_CRC_NONE; bool found_cut = false; for (;;) { wal_entry_hdr_t hdr; ssize_t n = pread(scan_fd, &hdr, sizeof(hdr), (off_t)scan_off); if (n < (ssize_t)sizeof(hdr)) break; /* short read or EOF */ /* Pre-allocated zero region: type==0 is not a valid WAL type (min=1) */ if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break; if (hdr.lsn > cutoff_lsn) { trunc_off = scan_off; found_cut = true; break; } new_last_crc = hdr.crc; scan_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); } close(scan_fd); if (!found_cut) trunc_off = scan_off; /* end of valid data in segment */ /* --- Close the current wal_fd --- */ if (p->wal_fd >= 0) { fdatasync(p->wal_fd); close(p->wal_fd); p->wal_fd = -1; } /* --- Truncate the cutoff segment at the computed offset --- */ int new_fd = open(cutoff_seg_path, O_RDWR); if (new_fd < 0) { return -errno; } if (ftruncate(new_fd, (off_t)trunc_off) != 0) { int saved = errno; close(new_fd); return -saved; } fdatasync(new_fd); /* --- Delete all segments after the cutoff segment --- */ for (int i = cutoff_seg_idx + 1; i < nseg; i++) { char del_path[4096]; snprintf(del_path, sizeof(del_path), WAL_SEG_FMT, p->wal_path, seg_lsns[i]); unlink(del_path); } /* --- Update persist_t state under lock --- */ p->wal_fd = new_fd; p->seg_offset = trunc_off; snprintf(p->seg_path, sizeof(p->seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[cutoff_seg_idx]); p->current_lsn = cutoff_lsn + 1; p->last_crc = new_last_crc; atomic_store_explicit(&p->committed_lsn, cutoff_lsn, memory_order_release); clamp_durable_lsn_locked(p, cutoff_lsn); /* Keep the per-LSN term index consistent: drop terms for removed LSNs. */ truncate_term_index_locked(p, cutoff_lsn); return 0; }