Rongsong Shen
f7b732a300
Same class as bug-1678/1687/1689: the WAL fact for WAL_JOB_RUSAGE only
carried {jobid, state}, so job.run_usage (client-reported per-host
CPU/mem usage) was never restored on either apply seam after a restart.
Unlike the prior three, xce_job_runusage_t/xce_host_runusage_t had no
existing Cap'n Proto mapping -- a genuine new schema addition
(HostRunusage/JobRunusage in controller.capnp, mirroring JobPendReason's
List(NestedStruct)+$C.maplistcount style), verified against the real
struct definitions in types.h, codegen confirmed via nm on the built
binary rather than assumed.
Two deliberate deviations from the established precedent, found by
reading the real code rather than copying the pattern blindly:
1. Adoption is unconditional, not gated on a state transition -- a
rusage report almost never changes job.state (repeat reports keep
state at RUN), so gating on JOB_TRANSITION_APPLIED like START/FINISH/
DISPATCH do would make the fix dead code for the realistic case.
2. Merge, not free-and-replace. update_host_runusage() builds run_usage
with host_name sourced from makeString() -- an RTTI-managed pointer,
not malloc() -- and the primary's own live apply pump re-applies every
fact it just wrote. A naive free_JobRunusage_ptr()-then-adopt (matching
the prior three fixes exactly) would free() a non-malloc'd pointer on
the very first live rusage report for a new host -- guaranteed heap
corruption. Fixed with merge_job_runusage(): existing hosts get
scalar-only in-place updates (host_name pointer never touched), only
genuinely new hosts adopt their freshly-decoded (safe, malloc'd)
host_name. Proved via a real makeString()-backed test asserting
pointer identity survives, not just value equality. job_destroy()
(src/lib/jobj.c) was found to have this identical latent mismatch,
pre-existing and independent of this fix -- flagged, not fixed here.
Found by an /xce-audit-invariants controller pass (field-complete apply
parity dimension). 8 new tests in test_wal_job_rusage_fact.c; RED/GREEN
independently verified. Full controller suite: 1160 tests, 2 failures --
both the pre-existing, unrelated bug-1561 overlayfs sandbox flake. No
ASAN/heap-corruption findings. Backward compatibility confirmed for
old-format bare-payload entries.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AgYRU3zDTqNGV5Jk8aiKLg
7946 lines
360 KiB
C
7946 lines
360 KiB
C
#define _GNU_SOURCE
|
||
#include "persist.h"
|
||
#include "config.h"
|
||
#include "ctrl_metrics.h"
|
||
#include "hook.h"
|
||
#include "xce/cluster_def.h"
|
||
#include "xce/errnox.h"
|
||
#include "xce/types.h"
|
||
#include "controller.capnp.h"
|
||
#include "typemap.h" /* xce_job_rusages_t -- bug-451 RUSAGE hook arm */
|
||
#include "xlist.h"
|
||
#include "cluster.h"
|
||
#include "commit.h" /* commit_submit RETAIN registration (barrier-rollout 1.2) */
|
||
|
||
#include <assert.h>
|
||
#include <stdbool.h>
|
||
#include <dirent.h>
|
||
#include <errno.h>
|
||
#include <fcntl.h>
|
||
#include <inttypes.h>
|
||
#include <pthread.h>
|
||
#include <stdatomic.h>
|
||
#include <stdint.h>
|
||
#include <stdio.h>
|
||
#include <stdlib.h>
|
||
#include <string.h>
|
||
#include <sys/stat.h>
|
||
#include <sys/statfs.h>
|
||
#include <time.h>
|
||
#include <unistd.h>
|
||
#include <zlib.h>
|
||
|
||
/* restore_* functions defined in job.c / worker.c / queue.c */
|
||
extern int restore_job(cluster_t* c, uint64_t jobid, const void* capnp_payload,
|
||
size_t len, uint64_t entry_lsn);
|
||
extern int restore_job_state(cluster_t* c, uint64_t jobid, uint32_t state);
|
||
/* restore_job_dispatch_state defined in job.c (bug-1678): WAL_JOB_DISPATCH's
|
||
* own sibling of restore_job_state() above, additionally taking ownership of
|
||
* an optional decoded xce_job_alloc_t so the dispatch-time resource
|
||
* allocation survives standby catch-up / replicated-apply the same way
|
||
* apply_committed_entry's WAL_JOB_DISPATCH case already restores it on the
|
||
* primary's own boot replay. */
|
||
extern int restore_job_dispatch_state(cluster_t* c, uint64_t jobid,
|
||
uint32_t state, xce_job_alloc_t* alloc);
|
||
/* restore_job_start_state defined in job.c (bug-1687): WAL_JOB_START's own
|
||
* sibling of restore_job_state() above, additionally taking ownership of an
|
||
* optional decoded job_pidinfo_t and a wire-carried start_time (0 means
|
||
* "not carried -- leave job.start_time untouched", the pre-fix payload's
|
||
* behavior) so the worker-confirmed start survives standby catch-up /
|
||
* replicated-apply the same way apply_committed_entry's WAL_JOB_START case
|
||
* already restores it on the primary's own boot replay. */
|
||
extern int restore_job_start_state(cluster_t* c, uint64_t jobid,
|
||
uint32_t state, uint64_t start_time,
|
||
job_pidinfo_t* pidinfo);
|
||
/* restore_job_finish_state defined in job.c (bug-1689): WAL_JOB_FINISH's own
|
||
* sibling of restore_job_state() above, additionally taking ownership of an
|
||
* optional decoded job_exitinfo_t and a wire-carried finish_time (0 means
|
||
* "not carried -- leave job.finish_time/run_time untouched", the pre-fix
|
||
* payload's behavior) so the worker-confirmed finish survives standby
|
||
* catch-up / replicated-apply the same way apply_committed_entry's
|
||
* WAL_JOB_FINISH case already restores it on the primary's own boot
|
||
* replay. */
|
||
extern int restore_job_finish_state(cluster_t* c, uint64_t jobid,
|
||
uint32_t state, uint64_t finish_time,
|
||
job_exitinfo_t* exit_info);
|
||
/* restore_job_rusage_state defined in job.c (bug-1690): WAL_JOB_RUSAGE's own
|
||
* sibling of restore_job_state() above, additionally taking ownership of an
|
||
* optional decoded xce_job_runusage_t (NULL for a state-only WAL_JOB_RUSAGE
|
||
* entry -- either one written before this fix, or one fired while
|
||
* job.run_usage was still NULL) so the client-reported per-host CPU/mem
|
||
* usage survives standby catch-up / replicated-apply the same way
|
||
* apply_committed_entry's WAL_JOB_RUSAGE case already restores it on the
|
||
* primary's own boot replay. */
|
||
extern int restore_job_rusage_state(cluster_t* c, uint64_t jobid,
|
||
uint32_t state,
|
||
xce_job_runusage_t* runusage);
|
||
/* merge_job_runusage defined in job.c (bug-1690): merges a freshly-decoded
|
||
* xce_job_runusage_t into *dst rather than freeing-and-replacing it. Needed
|
||
* here too (apply_committed_entry's own WAL_JOB_RUSAGE branch below), not
|
||
* just by restore_job_rusage_state() above -- *dst can be the job's LIVE
|
||
* run_usage (update_job_rusages()/update_host_runusage(), misc.c, mutate it
|
||
* synchronously and in place, with an xstring-backed host_name, BEFORE the
|
||
* WAL fact is even appended), and persist_apply_pump_run() re-applies that
|
||
* SAME fact on the SAME live job moments later via apply_committed_entry --
|
||
* a plain free()-and-replace (e.g. the capnp-generated free_JobRunusage_ptr)
|
||
* would free a non-malloc'd, RTTI-managed pointer there. See
|
||
* merge_job_runusage()'s own doc comment (job.c) for the full rationale. */
|
||
extern void merge_job_runusage(xce_job_runusage_t** dst,
|
||
xce_job_runusage_t* src);
|
||
extern int worker_from_capnp(cluster_t* c,
|
||
const void* capnp_payload, size_t len);
|
||
extern int restore_queue_state(cluster_t* c,
|
||
const char* name, uint32_t status);
|
||
/* restore_host_state defined in worker.c (config-upgrade-snapshot task 3,
|
||
* Req 1.10/1.11/5.8) -- create-if-missing host admin-state restore, called
|
||
* from load_snapshot_entities' host-frame loop below (task 4). */
|
||
extern int restore_host_state(cluster_t* c, const char* name,
|
||
uint32_t admin_status);
|
||
/* upgrade_materialization_verify defined in ha.c (config-upgrade-snapshot
|
||
* task 6.1/6.2, Req 1.8/3.2/3.4/5.2/5.5) -- the Activation Fence's own
|
||
* verify function. Called unconditionally from apply_committed_entry's
|
||
* WAL_CONFIG_MARKER case below (task 6.2): checkpoint completion is the
|
||
* design-intended trigger that re-evaluates this fence (design.md's
|
||
* Activation Fence component, "naturally re-evaluates against the newer
|
||
* marker once that checkpoint's write_snapshot call runs"), since it is
|
||
* the ONLY event that can newly set OR clear it from a running process --
|
||
* every other call path (upgrade_materialization_retry_fn and its
|
||
* callers) is itself gated on the fence already being set, so without
|
||
* this call site the fence could never transition from its zero-
|
||
* initialized default. */
|
||
extern int upgrade_materialization_verify(cluster_t* c);
|
||
|
||
/* config_activate_committed defined in config_repl.c (config-upgrade-snapshot
|
||
* task 5/6.2 remediation, bug-1469, Req 1.8/3.2/3.4) -- re-parses the
|
||
* locally-held config bytes referenced by the latest committed
|
||
* WAL_CONFIG_MARKER into c->myconf (load_primary_config), hash-verifying
|
||
* them first. Called unconditionally from apply_committed_entry's
|
||
* WAL_CONFIG_MARKER case below, BEFORE upgrade_materialization_verify: every
|
||
* member's own live apply of a marker it just committed must refresh its
|
||
* own c->myconf, not only members that separately receive an out-of-band
|
||
* handle_repl_config_push (see the call site's own doc comment for the full
|
||
* rationale). Declared here via extern, mirroring upgrade_materialization_
|
||
* verify just above, rather than pulling in config_repl.h's full include
|
||
* surface for one symbol. */
|
||
extern int config_activate_committed(cluster_t* c);
|
||
|
||
/* raft-apply-decoupling task 3.2: pending command-apply hand-off list node
|
||
* (producer side; the not-yet-built ordered apply pump, task 6.1/6.2, is
|
||
* the consumer). Declared here (rather than persist.h) so the list stays
|
||
* an implementation detail — persist.h exposes only the opaque `void*`
|
||
* head field plus persist_register_pending_apply/persist_has_pending_apply.
|
||
* See persist.h's doc comment on persist_register_pending_apply for the
|
||
* full contract. */
|
||
typedef struct pending_command_apply {
|
||
uint64_t lsn;
|
||
pending_apply_notify_fn on_applied;
|
||
void* ctx;
|
||
struct pending_command_apply* next;
|
||
} pending_command_apply_t;
|
||
|
||
/* ---- Cap'n Proto serialization helpers (WAL payload encoding) ---------- */
|
||
|
||
/* Serialize xce_job_t to a malloc'd Cap'n Proto buffer.
|
||
* Caller must free(*out) on success. */
|
||
static int capnp_encode_job(const xce_job_t* j, uint8_t** out, size_t* out_len)
|
||
{
|
||
struct capn c;
|
||
struct capn_segment* cs;
|
||
struct Job cj;
|
||
Job_ptr jp;
|
||
|
||
capn_init_malloc(&c);
|
||
cs = capn_root(&c).seg;
|
||
encode_Job(cs, &cj, (xce_job_t*)j);
|
||
jp = new_Job(cs);
|
||
write_Job(&cj, jp);
|
||
capn_setp(capn_root(&c), 0, jp.p);
|
||
|
||
*out_len = capn_size(&c);
|
||
*out = malloc(*out_len);
|
||
if (!*out) { capn_free(&c); return -ENOMEM; }
|
||
capn_write_mem(&c, *out, *out_len, 0);
|
||
capn_free(&c);
|
||
return 0;
|
||
}
|
||
|
||
/* Serialize xce_worker_t to a malloc'd Cap'n Proto buffer. */
|
||
static int capnp_encode_worker(const xce_worker_t* w, uint8_t** out, size_t* out_len)
|
||
{
|
||
struct capn c;
|
||
struct capn_segment* cs;
|
||
struct Worker cw;
|
||
Worker_ptr wp;
|
||
|
||
capn_init_malloc(&c);
|
||
cs = capn_root(&c).seg;
|
||
encode_Worker(cs, &cw, (xce_worker_t*)w);
|
||
wp = new_Worker(cs);
|
||
write_Worker(&cw, wp);
|
||
capn_setp(capn_root(&c), 0, wp.p);
|
||
|
||
*out_len = capn_size(&c);
|
||
*out = malloc(*out_len);
|
||
if (!*out) { capn_free(&c); return -ENOMEM; }
|
||
capn_write_mem(&c, *out, *out_len, 0);
|
||
capn_free(&c);
|
||
return 0;
|
||
}
|
||
|
||
/* Serialize xce_job_alloc_t to a malloc'd Cap'n Proto buffer (bug-1678):
|
||
* lets WAL_JOB_DISPATCH carry the scheduler's resource allocation alongside
|
||
* the {jobid, state} header, so a restart replay can reconstruct it instead
|
||
* of leaving job.allocs permanently NULL. Mirrors capnp_encode_worker above
|
||
* exactly, using the JobAlloc struct (already in controller.capnp as the
|
||
* existing Job.allocs field -- no schema change needed here). */
|
||
static int capnp_encode_joballoc(const xce_job_alloc_t* a, uint8_t** out,
|
||
size_t* out_len)
|
||
{
|
||
struct capn c;
|
||
struct capn_segment* cs;
|
||
struct JobAlloc cja;
|
||
JobAlloc_ptr jap;
|
||
|
||
capn_init_malloc(&c);
|
||
cs = capn_root(&c).seg;
|
||
encode_JobAlloc(cs, &cja, (xce_job_alloc_t*)a);
|
||
jap = new_JobAlloc(cs);
|
||
write_JobAlloc(&cja, jap);
|
||
capn_setp(capn_root(&c), 0, jap.p);
|
||
|
||
*out_len = capn_size(&c);
|
||
*out = malloc(*out_len);
|
||
if (!*out) { capn_free(&c); return -ENOMEM; }
|
||
capn_write_mem(&c, *out, *out_len, 0);
|
||
capn_free(&c);
|
||
return 0;
|
||
}
|
||
|
||
/* Decode a Cap'n Proto JobAlloc buffer (bug-1678) into a freshly heap-
|
||
* allocated xce_job_alloc_t*, mirroring restore_job()'s Job decode pattern
|
||
* (job.c) but returning the struct directly instead of constructing a job.
|
||
* Returns NULL on a NULL/empty payload; caller owns the result (release via
|
||
* free_JobAlloc_ptr). */
|
||
static xce_job_alloc_t* decode_joballoc(const uint8_t* payload, size_t len)
|
||
{
|
||
struct capn c_ctx;
|
||
JobAlloc_ptr jap;
|
||
struct JobAlloc cja;
|
||
xce_job_alloc_t* alloc;
|
||
|
||
if (!payload || len == 0)
|
||
return NULL;
|
||
|
||
capn_init_mem(&c_ctx, payload, len, 0);
|
||
jap.p = capn_getp(capn_root(&c_ctx), 0, 1);
|
||
memset(&cja, 0, sizeof(cja));
|
||
read_JobAlloc(&cja, jap);
|
||
|
||
alloc = calloc(1, sizeof(*alloc));
|
||
if (!alloc) { capn_free(&c_ctx); return NULL; }
|
||
decode_JobAlloc(alloc, &cja);
|
||
capn_free(&c_ctx);
|
||
return alloc;
|
||
}
|
||
|
||
/* Serialize job_pidinfo_t to a malloc'd Cap'n Proto buffer (bug-1687):
|
||
* lets WAL_JOB_START carry the worker-reported pids/pgids/exdata alongside
|
||
* the {jobid, state} header, so a restart replay can reconstruct
|
||
* job.pidinfo instead of leaving it permanently NULL. Mirrors
|
||
* capnp_encode_joballoc above exactly, using the JobPIDInfo struct (already
|
||
* in controller.capnp as the existing Job.pidinfo field -- no schema change
|
||
* needed here). */
|
||
static int capnp_encode_pidinfo(const job_pidinfo_t* info, uint8_t** out,
|
||
size_t* out_len)
|
||
{
|
||
struct capn c;
|
||
struct capn_segment* cs;
|
||
struct JobPIDInfo cji;
|
||
JobPIDInfo_ptr jip;
|
||
|
||
capn_init_malloc(&c);
|
||
cs = capn_root(&c).seg;
|
||
encode_JobPIDInfo(cs, &cji, (job_pidinfo_t*)info);
|
||
jip = new_JobPIDInfo(cs);
|
||
write_JobPIDInfo(&cji, jip);
|
||
capn_setp(capn_root(&c), 0, jip.p);
|
||
|
||
*out_len = capn_size(&c);
|
||
*out = malloc(*out_len);
|
||
if (!*out) { capn_free(&c); return -ENOMEM; }
|
||
capn_write_mem(&c, *out, *out_len, 0);
|
||
capn_free(&c);
|
||
return 0;
|
||
}
|
||
|
||
/* Decode a Cap'n Proto JobPIDInfo buffer (bug-1687) into a freshly heap-
|
||
* allocated job_pidinfo_t*, mirroring decode_joballoc()'s pattern above.
|
||
* Returns NULL on a NULL/empty payload; caller owns the result (release via
|
||
* free_JobPIDInfo_ptr). */
|
||
static job_pidinfo_t* decode_pidinfo(const uint8_t* payload, size_t len)
|
||
{
|
||
struct capn c_ctx;
|
||
JobPIDInfo_ptr jip;
|
||
struct JobPIDInfo cji;
|
||
job_pidinfo_t* info;
|
||
|
||
if (!payload || len == 0)
|
||
return NULL;
|
||
|
||
capn_init_mem(&c_ctx, payload, len, 0);
|
||
jip.p = capn_getp(capn_root(&c_ctx), 0, 1);
|
||
memset(&cji, 0, sizeof(cji));
|
||
read_JobPIDInfo(&cji, jip);
|
||
|
||
info = calloc(1, sizeof(*info));
|
||
if (!info) { capn_free(&c_ctx); return NULL; }
|
||
decode_JobPIDInfo(info, &cji);
|
||
capn_free(&c_ctx);
|
||
return info;
|
||
}
|
||
|
||
/* Decode a native (non-capnp) WAL_JOB_FINISH exit_code trailer (bug-1689)
|
||
* into a freshly heap-allocated job_exitinfo_t*. job_exitinfo_t (xce/types.h)
|
||
* is a single int with NO existing Cap'n Proto mapping (confirmed -- no
|
||
* JobExitInfo struct in controller.capnp), so this mirrors wal_jobstate_t's
|
||
* own native-struct style rather than inventing a capnp wrapper for one int
|
||
* -- unlike decode_joballoc()/decode_pidinfo() above, which decode genuine
|
||
* capnp payloads. Returns NULL on a short/absent payload; caller owns the
|
||
* result (release via free()/FREEUP -- no nested pointers). */
|
||
static job_exitinfo_t* decode_jobexitinfo(const uint8_t* payload, size_t len)
|
||
{
|
||
job_exitinfo_t* info;
|
||
int32_t exitcode;
|
||
|
||
if (!payload || len < sizeof(exitcode))
|
||
return NULL;
|
||
|
||
memcpy(&exitcode, payload, sizeof(exitcode));
|
||
|
||
info = (job_exitinfo_t*)calloc(1, sizeof(*info));
|
||
if (!info)
|
||
return NULL;
|
||
info->exitcode = (int)exitcode;
|
||
return info;
|
||
}
|
||
|
||
/* Serialize xce_job_runusage_t to a malloc'd Cap'n Proto buffer (bug-1690):
|
||
* lets WAL_JOB_RUSAGE carry the client-reported, accumulated per-host
|
||
* CPU/mem usage (job->job.run_usage) alongside the {jobid, state} header, so
|
||
* a restart/catch-up replay can reconstruct it instead of leaving it
|
||
* permanently NULL. Mirrors capnp_encode_joballoc()/capnp_encode_pidinfo()
|
||
* above exactly, using the NEW JobRunusage struct (controller.capnp) --
|
||
* unlike JobAlloc/JobPIDInfo, xce_job_runusage_t had NO existing Cap'n Proto
|
||
* mapping at all, so this is a genuine schema addition (see the schema's own
|
||
* doc comment on JobRunusage for why it is not the same type as
|
||
* xce_job_rusages_t/JobRusages, a similar-looking but unrelated struct). */
|
||
static int capnp_encode_jobrunusage(const xce_job_runusage_t* r, uint8_t** out,
|
||
size_t* out_len)
|
||
{
|
||
struct capn c;
|
||
struct capn_segment* cs;
|
||
struct JobRunusage cjr;
|
||
JobRunusage_ptr jrp;
|
||
|
||
capn_init_malloc(&c);
|
||
cs = capn_root(&c).seg;
|
||
encode_JobRunusage(cs, &cjr, (xce_job_runusage_t*)r);
|
||
jrp = new_JobRunusage(cs);
|
||
write_JobRunusage(&cjr, jrp);
|
||
capn_setp(capn_root(&c), 0, jrp.p);
|
||
|
||
*out_len = capn_size(&c);
|
||
*out = malloc(*out_len);
|
||
if (!*out) { capn_free(&c); return -ENOMEM; }
|
||
capn_write_mem(&c, *out, *out_len, 0);
|
||
capn_free(&c);
|
||
return 0;
|
||
}
|
||
|
||
/* Decode a Cap'n Proto JobRunusage buffer (bug-1690) into a freshly heap-
|
||
* allocated xce_job_runusage_t*, mirroring decode_joballoc()/decode_pidinfo()'s
|
||
* pattern above. Returns NULL on a NULL/empty payload; caller owns the
|
||
* result (release via free_JobRunusage_ptr). */
|
||
static xce_job_runusage_t* decode_jobrunusage(const uint8_t* payload,
|
||
size_t len)
|
||
{
|
||
struct capn c_ctx;
|
||
JobRunusage_ptr jrp;
|
||
struct JobRunusage cjr;
|
||
xce_job_runusage_t* r;
|
||
|
||
if (!payload || len == 0)
|
||
return NULL;
|
||
|
||
capn_init_mem(&c_ctx, payload, len, 0);
|
||
jrp.p = capn_getp(capn_root(&c_ctx), 0, 1);
|
||
memset(&cjr, 0, sizeof(cjr));
|
||
read_JobRunusage(&cjr, jrp);
|
||
|
||
r = calloc(1, sizeof(*r));
|
||
if (!r) { capn_free(&c_ctx); return NULL; }
|
||
decode_JobRunusage(r, &cjr);
|
||
capn_free(&c_ctx);
|
||
return r;
|
||
}
|
||
|
||
/* Serialize a host admin-state frame (config-upgrade-snapshot task 2.1, Req
|
||
* 1.6/1.7/1.9) to a malloc'd Cap'n Proto buffer. Mirrors capnp_encode_job
|
||
* above exactly -- capnp_host_admin_state_t (typemap.h) is a frame-only
|
||
* (name, status) pair, never the full xce_worker_t (design.md Design
|
||
* Invariant 5). Caller must free(*out) on success. */
|
||
static int capnp_encode_host_admin_state(const capnp_host_admin_state_t* h,
|
||
uint8_t** out, size_t* out_len)
|
||
{
|
||
struct capn c;
|
||
struct capn_segment* cs;
|
||
struct HostAdminState ch;
|
||
HostAdminState_ptr hp;
|
||
|
||
capn_init_malloc(&c);
|
||
cs = capn_root(&c).seg;
|
||
encode_HostAdminState(cs, &ch, (capnp_host_admin_state_t*)h);
|
||
hp = new_HostAdminState(cs);
|
||
write_HostAdminState(&ch, hp);
|
||
capn_setp(capn_root(&c), 0, hp.p);
|
||
|
||
*out_len = capn_size(&c);
|
||
*out = malloc(*out_len);
|
||
if (!*out) { capn_free(&c); return -ENOMEM; }
|
||
capn_write_mem(&c, *out, *out_len, 0);
|
||
capn_free(&c);
|
||
return 0;
|
||
}
|
||
|
||
/* Serialize a queue admin-state frame (config-upgrade-snapshot task 2.2, Req
|
||
* 1.9/5.7) to a malloc'd Cap'n Proto buffer. Mirrors
|
||
* capnp_encode_host_admin_state above exactly -- capnp_queue_admin_state_t
|
||
* (typemap.h) is a frame-only (name, status) pair, matching
|
||
* restore_queue_state's exact (name, status) contract (queue.c:208), same
|
||
* field content as the prior raw layout, only the wire encoding changes.
|
||
* Caller must free(*out) on success. */
|
||
static int capnp_encode_queue_admin_state(const capnp_queue_admin_state_t* q,
|
||
uint8_t** out, size_t* out_len)
|
||
{
|
||
struct capn c;
|
||
struct capn_segment* cs;
|
||
struct QueueAdminState cq;
|
||
QueueAdminState_ptr qp;
|
||
|
||
capn_init_malloc(&c);
|
||
cs = capn_root(&c).seg;
|
||
encode_QueueAdminState(cs, &cq, (capnp_queue_admin_state_t*)q);
|
||
qp = new_QueueAdminState(cs);
|
||
write_QueueAdminState(&cq, qp);
|
||
capn_setp(capn_root(&c), 0, qp.p);
|
||
|
||
*out_len = capn_size(&c);
|
||
*out = malloc(*out_len);
|
||
if (!*out) { capn_free(&c); return -ENOMEM; }
|
||
capn_write_mem(&c, *out, *out_len, 0);
|
||
capn_free(&c);
|
||
return 0;
|
||
}
|
||
|
||
/* WAL_WORKER_CTRL payload: [uint32_t detail][capnp bytes...] or [uint32_t detail][name\0] */
|
||
#define WORKER_WAL_HDR_SIZE sizeof(uint32_t)
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Constants */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
#define WAL_SEG_FMT "%s/seg-%016" PRIx64 ".wal"
|
||
|
||
/* NFS_SUPER_MAGIC — reject WAL on NFS mounts (unreliable fsync) */
|
||
#define NFS_SUPER_MAGIC 0x6969
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Internal helpers */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
static uint64_t now_us(void)
|
||
{
|
||
struct timespec ts;
|
||
clock_gettime(CLOCK_REALTIME, &ts);
|
||
return (uint64_t)ts.tv_sec * 1000000ULL + (uint64_t)(ts.tv_nsec / 1000);
|
||
}
|
||
|
||
/* Recursive mkdir — equivalent to mkdir -p */
|
||
static int mkdir_p(const char* path, mode_t mode)
|
||
{
|
||
char tmp[4096];
|
||
char* p;
|
||
size_t len;
|
||
|
||
snprintf(tmp, sizeof(tmp), "%s", path);
|
||
len = strlen(tmp);
|
||
if (len > 0 && tmp[len - 1] == '/')
|
||
tmp[len - 1] = '\0';
|
||
|
||
for (p = tmp + 1; *p; p++) {
|
||
if (*p == '/') {
|
||
*p = '\0';
|
||
if (mkdir(tmp, mode) != 0 && errno != EEXIST)
|
||
return -1;
|
||
*p = '/';
|
||
}
|
||
}
|
||
if (mkdir(tmp, mode) != 0 && errno != EEXIST)
|
||
return -1;
|
||
return 0;
|
||
}
|
||
|
||
/* Open (and pre-allocate) a new WAL segment named after current_lsn */
|
||
static int open_new_segment(persist_t* p)
|
||
{
|
||
snprintf(p->seg_path, sizeof(p->seg_path),
|
||
WAL_SEG_FMT, p->wal_path, p->current_lsn);
|
||
|
||
int fd = open(p->seg_path, O_RDWR | O_CREAT | O_TRUNC, 0600);
|
||
if (fd < 0)
|
||
return -errno;
|
||
|
||
/* Pre-allocate segment space to avoid fragmentation and metadata updates
|
||
* on every write. posix_fallocate is best-effort — ignore errors. */
|
||
posix_fallocate(fd, 0, (off_t)p->config.wal_segment_bytes);
|
||
|
||
p->wal_fd = fd;
|
||
p->seg_offset = 0;
|
||
return 0;
|
||
}
|
||
|
||
/* Close current segment and open a fresh one */
|
||
static int rotate_segment(persist_t* p)
|
||
{
|
||
if (p->wal_fd >= 0) {
|
||
fdatasync(p->wal_fd);
|
||
close(p->wal_fd);
|
||
p->wal_fd = -1;
|
||
}
|
||
return open_new_segment(p);
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Demand-driven group-commit flusher (ha-commit-durability task 2.1) */
|
||
/* */
|
||
/* Repurposes the old fixed-interval fsync timer into a flusher driven */
|
||
/* by the append-vs-synced gap. On each pass it batches one fdatasync */
|
||
/* over all entries appended since the last sync, advances durable_lsn */
|
||
/* to the synced tip, and wakes the commit evaluation. --wal-sync-ms */
|
||
/* now bounds the MAX batching latency (a no-further-append entry is */
|
||
/* still synced within the cadence), not a durability gap. */
|
||
/* */
|
||
/* LOST-WAKEUP / CORRECTNESS REASONING (gating item): */
|
||
/* */
|
||
/* The "work pending" condition is LEVEL-triggered on two persistent */
|
||
/* variables, NOT an edge: pending ⇔ durable_lsn < current_lsn - 1 */
|
||
/* (flusher_work_pending below, evaluated under p->lock). Because it */
|
||
/* is a comparison of stored state — not a transient flag consumed on */
|
||
/* read — a signal that races the flusher's wait can never be lost: */
|
||
/* */
|
||
/* - persist_wal_async advances current_lsn under p->lock, THEN */
|
||
/* signals fsync_cond. If the flusher is mid-flush when the append */
|
||
/* lands, current_lsn has already moved past the tip the flusher */
|
||
/* captured, so after that flush durable_lsn < current_lsn - 1 */
|
||
/* STILL holds; the flusher re-checks the level under p->lock on */
|
||
/* its next loop iteration and flushes again (it does NOT wait */
|
||
/* first). The missed signal is therefore harmless — the level */
|
||
/* itself carries the pending work forward. */
|
||
/* - If the append's signal arrives while the flusher sits in */
|
||
/* pthread_cond_wait, it wakes it. If it arrives in the tiny window */
|
||
/* after the flusher decided "no work" but before it re-enters the */
|
||
/* wait, the flusher re-evaluates the level under p->lock before */
|
||
/* waiting (we re-check inside the fsync_mutex-held region by */
|
||
/* reading current_lsn/durable_lsn under p->lock first), so it */
|
||
/* will not block while work is pending. */
|
||
/* */
|
||
/* "NEVER durable before its fdatasync": the flusher snapshots */
|
||
/* tip = current_lsn - 1 AND calls fdatasync WHILE HOLDING p->lock. */
|
||
/* persist_wal_async also pwrites under p->lock, so no append can */
|
||
/* extend the segment between the tip snapshot and the fdatasync — by */
|
||
/* the time fdatasync returns, every byte for LSNs <= tip is on stable */
|
||
/* storage. durable_lsn is only then advanced to tip (monotonic max). */
|
||
/* Releasing p->lock and broadcasting commit-eval happens after the */
|
||
/* advance, so a WAIT caller that observes durable_lsn >= its LSN is */
|
||
/* guaranteed the bytes are synced. */
|
||
/* */
|
||
/* Lock discipline: p->lock and fsync_mutex are NEVER held together */
|
||
/* (fsync_mutex guards only the cond wait; the flush section takes */
|
||
/* p->lock alone). This keeps fdatasync off any lock the iomgr / */
|
||
/* heartbeat path needs and preserves liveness (Req 1.6). */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
/* Test-only fault-injection probe (ha-commit-durability task 2.1 error path).
|
||
* Returns true when the test has forced fdatasync to be treated as failed.
|
||
* Compiled to a constant `false` in production (no -DXCE_TESTING), so the
|
||
* fail-loud gate below collapses to the pure rc != 0 check with zero overhead
|
||
* and no dependency on the test-only struct field. */
|
||
static inline bool flusher_fsync_forced_fail(persist_t* p)
|
||
{
|
||
#ifdef XCE_TESTING
|
||
return atomic_load_explicit(&p->test_force_fsync_fail,
|
||
memory_order_acquire) != 0;
|
||
#else
|
||
(void)p;
|
||
return false;
|
||
#endif
|
||
}
|
||
|
||
/* Returns true iff there is un-synced appended data (level predicate).
|
||
* MUST be called with p->lock held. current_lsn - 1 is the appended tip
|
||
* (current_lsn is the NEXT lsn to assign); LSN 0 reserved, so a fresh
|
||
* store has current_lsn == 1 → tip 0 → no pending work. */
|
||
static bool flusher_work_pending_locked(persist_t* p)
|
||
{
|
||
uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
|
||
uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire);
|
||
return dur < tip;
|
||
}
|
||
|
||
/* Perform one group-commit flush: snapshot the appended tip, fdatasync the
|
||
* live segment, and advance durable_lsn to the synced tip. Takes p->lock
|
||
* for the whole snapshot+sync+advance so no concurrent append can extend
|
||
* the segment between the tip read and the fdatasync. Returns the new
|
||
* durable_lsn (unchanged if there was nothing to sync). */
|
||
static uint64_t flusher_flush_once(persist_t* p)
|
||
{
|
||
#ifdef XCE_TESTING
|
||
atomic_fetch_add_explicit(&p->test_flusher_flush_attempt_count, 1,
|
||
memory_order_relaxed);
|
||
#endif
|
||
pthread_mutex_lock(&p->lock);
|
||
|
||
uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
|
||
uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire);
|
||
|
||
if (dur >= tip || p->wal_fd < 0) {
|
||
/* Nothing new to sync (or no segment yet). Still flush the fd for a
|
||
* clean-shutdown / fallback pass, but do not advance the watermark.
|
||
* The watermark is already <= tip here, so a failed fallback sync
|
||
* cannot make it claim more than is durable — but we still surface
|
||
* the error loudly so a degrading disk is never silent (Req 1.1). */
|
||
if (p->wal_fd >= 0) {
|
||
int rc = fdatasync(p->wal_fd);
|
||
if (rc != 0 || flusher_fsync_forced_fail(p)) {
|
||
int e = (rc != 0) ? errno : EIO;
|
||
ERROR("persist: flusher fallback fdatasync FAILED (errno=%d %s)"
|
||
" — durable_lsn held at %" PRIu64 "; WAL not advanced",
|
||
e, strerror(e), dur);
|
||
}
|
||
}
|
||
pthread_mutex_unlock(&p->lock);
|
||
return dur;
|
||
}
|
||
|
||
/* fdatasync under p->lock: appends also hold p->lock to pwrite, so the
|
||
* segment cannot grow between this tip snapshot and the sync. Every byte
|
||
* for LSNs <= tip is therefore on stable storage when fdatasync returns
|
||
* SUCCESSFULLY. */
|
||
int rc = fdatasync(p->wal_fd);
|
||
|
||
/* FAIL-LOUD GATE (Req 1.1/1.2): advance durable_lsn ONLY on a successful
|
||
* sync. On EIO/ENOSPC the bytes are NOT on stable storage, so leaving the
|
||
* watermark unchanged keeps the non-durable LSNs from being counted toward
|
||
* the commit majority. We do NOT retry into an advanced watermark — the
|
||
* level predicate stays pending and the next pass retries the sync. */
|
||
if (rc != 0 || flusher_fsync_forced_fail(p)) {
|
||
int e = (rc != 0) ? errno : EIO;
|
||
ERROR("persist: flusher fdatasync FAILED (errno=%d %s) — durable_lsn "
|
||
"held at %" PRIu64 " (tip=%" PRIu64 " NOT made durable); commit "
|
||
"majority will not count these LSNs",
|
||
e, strerror(e), dur, tip);
|
||
pthread_mutex_unlock(&p->lock);
|
||
return dur;
|
||
}
|
||
|
||
/* Monotonic advance to the synced tip — never past it (the invariant
|
||
* durable_lsn <= current_lsn - 1 is preserved because tip == current_lsn-1
|
||
* was read under this same lock). One fdatasync advances durable_lsn past
|
||
* however many LSNs were appended since the last sync (batching). */
|
||
if (tip > dur)
|
||
atomic_store_explicit(&p->durable_lsn, tip, memory_order_release);
|
||
|
||
#ifdef XCE_TESTING
|
||
/* Record which thread ran this successful fdatasync (task 5.1 / Req 1.6):
|
||
* a liveness test asserts this is the dedicated flusher thread, never the
|
||
* caller/iomgr/heartbeat thread. Written under p->lock; read after the
|
||
* flusher has caught up, so no extra synchronization is needed. */
|
||
p->test_last_fdatasync_thread = pthread_self();
|
||
atomic_store_explicit(&p->test_last_fdatasync_thread_set, 1,
|
||
memory_order_release);
|
||
#endif
|
||
|
||
pthread_mutex_unlock(&p->lock);
|
||
return tip;
|
||
}
|
||
|
||
static void* fsync_worker(void* arg)
|
||
{
|
||
persist_t* p = arg;
|
||
|
||
pthread_mutex_lock(&p->fsync_mutex);
|
||
while (!atomic_load_explicit(&p->fsync_stop, memory_order_acquire)) {
|
||
/* Re-evaluate the level predicate BEFORE waiting so a signal that
|
||
* raced in just before we entered the wait is not needed for
|
||
* correctness — if work is already pending we skip the wait entirely
|
||
* and flush. (Reads current_lsn/durable_lsn under p->lock.) */
|
||
pthread_mutex_lock(&p->lock);
|
||
bool pending = flusher_work_pending_locked(p);
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
if (!pending) {
|
||
/* No pending work: wait for an append-signal OR the cadence
|
||
* deadline (bounded fallback flush so a lone no-further-append
|
||
* entry is still synced within wal_sync_ms — Req 3.1). */
|
||
long ms = (long)p->config.wal_sync_ms;
|
||
struct timespec dl;
|
||
clock_gettime(CLOCK_REALTIME, &dl);
|
||
dl.tv_sec += ms / 1000;
|
||
dl.tv_nsec += (ms % 1000) * 1000000L;
|
||
if (dl.tv_nsec >= 1000000000L) {
|
||
dl.tv_sec += 1;
|
||
dl.tv_nsec -= 1000000000L;
|
||
}
|
||
pthread_cond_timedwait(&p->fsync_cond, &p->fsync_mutex, &dl);
|
||
|
||
if (atomic_load_explicit(&p->fsync_stop, memory_order_acquire))
|
||
break;
|
||
}
|
||
|
||
/* Flush section: never hold fsync_mutex across the fdatasync (the
|
||
* flush takes p->lock alone). Release fsync_mutex, flush, re-acquire.
|
||
* After the flush we loop and re-check the level — if a new append
|
||
* landed during the flush, the level is still pending and we flush
|
||
* again without needing its signal (lost-wakeup-safe). */
|
||
pthread_mutex_unlock(&p->fsync_mutex);
|
||
uint64_t before = atomic_load_explicit(&p->durable_lsn,
|
||
memory_order_acquire);
|
||
uint64_t after = flusher_flush_once(p);
|
||
|
||
/* COMMIT-EVAL WAKE (task 2.2 consumer wired): durable_lsn is advanced
|
||
* INSIDE flusher_flush_once (atomic store with release) BEFORE we
|
||
* return here, and flusher_flush_once has already RELEASED p->lock.
|
||
* Now — holding NO persist lock — invoke the registered durable-advance
|
||
* callback so the commit engine re-evaluates its majority and wakes any
|
||
* COMMIT_MODE_WAIT caller whose entry just became durable locally.
|
||
*
|
||
* LOST-WAKEUP SAFETY: the wake fires strictly AFTER the durable_lsn
|
||
* release-store, and the commit engine's predicate re-reads durable_lsn
|
||
* via an acquire load under ce->lock — so a WAIT caller either (a) sees
|
||
* the advanced watermark on its own re-check, or (b) is blocked in
|
||
* pthread_cond_wait and is woken by this broadcast; it can never both
|
||
* miss the watermark AND miss the wake. We only fire the callback when
|
||
* durable_lsn actually advanced (after > before): a no-op flush (or a
|
||
* failed fdatasync that held the watermark) needs no commit re-eval,
|
||
* avoiding spurious wakes.
|
||
*
|
||
* LOCK ORDERING (cycle-free): the flusher releases p->lock inside
|
||
* flusher_flush_once, THEN the callback takes ce->lock (commit side).
|
||
* The commit side only ever takes ce->lock and reads persist atomics —
|
||
* it never takes p->lock while holding ce->lock. So the only ordering
|
||
* is p->lock → ce->lock; there is no ce->lock → p->lock edge, hence no
|
||
* deadlock cycle. The callback explicitly MUST NOT take p->lock. */
|
||
if (after > before && p->on_durable_advance != NULL) {
|
||
p->on_durable_advance(p->durable_cb_ctx);
|
||
}
|
||
|
||
pthread_mutex_lock(&p->fsync_mutex);
|
||
|
||
/* bug-1563: a sustained fdatasync failure leaves durable_lsn < tip
|
||
* forever (the fail-loud gate above deliberately never advances it
|
||
* on failure — that part is correct and unchanged). flusher_work_
|
||
* pending_locked() can therefore never distinguish "genuine fresh
|
||
* work just landed" from "still stuck from the last failed attempt"
|
||
* — both read as pending=true, so without this check the loop above
|
||
* would skip the wait on EVERY iteration and spin flusher_flush_once
|
||
* (and thus fdatasync) as fast as the CPU/lock allow. Live evidence:
|
||
* a 40-round chaotic /xce-test-ha soak hit 300k+ failed attempts on
|
||
* one node in well under a minute (one core pinned, log flooded).
|
||
* Only skip the wait when the flush actually made progress (pending
|
||
* was true and it succeeded) or there was nothing pending to begin
|
||
* with (the pre-existing, correct fast path); a flush that was
|
||
* pending but made no progress (after == before) still waits out the
|
||
* normal wal_sync_ms cadence before retrying — same bounded-latency
|
||
* budget the idle path already uses, no new tunable needed. */
|
||
if (pending && after == before &&
|
||
!atomic_load_explicit(&p->fsync_stop, memory_order_acquire)) {
|
||
long ms = (long)p->config.wal_sync_ms;
|
||
struct timespec dl;
|
||
clock_gettime(CLOCK_REALTIME, &dl);
|
||
dl.tv_sec += ms / 1000;
|
||
dl.tv_nsec += (ms % 1000) * 1000000L;
|
||
if (dl.tv_nsec >= 1000000000L) {
|
||
dl.tv_sec += 1;
|
||
dl.tv_nsec -= 1000000000L;
|
||
}
|
||
pthread_cond_timedwait(&p->fsync_cond, &p->fsync_mutex, &dl);
|
||
}
|
||
}
|
||
pthread_mutex_unlock(&p->fsync_mutex);
|
||
return NULL;
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ *
|
||
* WAL term adapter (ha-term-safety task 1.3, R1.1/R1.5/R1.6/R1.7/R1.8) *
|
||
* ------------------------------------------------------------------ */
|
||
|
||
/* Sentinel stored in a term_index[] slot that has never been explicitly
|
||
* written by record_entry_term — distinct from a genuinely-recorded raw term
|
||
* of 0 (a legacy pre-term-safety entry, which maps to WAL_LEGACY_TERM at
|
||
* read-back). Reserved so growth (below) can leave a gap slot reading back as
|
||
* truly ABSENT forever, not just until the NEXT lsn happens to be recorded
|
||
* (bug-1475 follow-up: a WAL_CHECKPOINT slot is deliberately never recorded —
|
||
* see persist_write_checkpoint_marker's comment — but before this fix, the
|
||
* geometric-growth realloc for ANY later lsn zero-filled the checkpoint's own
|
||
* gap slot, and entry_term_locked's raw==0 rule then read that zero-fill back
|
||
* as WAL_LEGACY_TERM, not absent — silently and retroactively flipping a
|
||
* deliberately-unrecorded LSN's term the instant any later entry was
|
||
* appended. Confirmed via live instrumentation: persist_entry_term(checkpoint
|
||
* lsn) read back 0 (absent) immediately after the checkpoint, then 1
|
||
* (WAL_LEGACY_TERM) once a later real entry was appended and grew the array
|
||
* past it — a time-of-check/time-of-use divergence between the sender (which
|
||
* scans/streams AFTER later entries exist) and a receiver that has not yet
|
||
* appended past the same checkpoint, causing logOk's prev_log_term equality
|
||
* check to spuriously mismatch (0 != 1) and permanently reject catch-up). */
|
||
#define TERM_INDEX_UNSET UINT64_MAX
|
||
|
||
/* Record an entry's RAW hdr.term at its 1-based LSN in the in-memory per-LSN
|
||
* term index (R1.5). Grows the index on demand; LSNs replay/append contiguously
|
||
* so the array is dense (slot i == LSN i+1). Any gap left by a non-contiguous
|
||
* LSN (e.g. a deliberately-skipped WAL_CHECKPOINT slot) is filled with the
|
||
* TERM_INDEX_UNSET sentinel by realloc-then-memset, which reads back as
|
||
* "absent" PERMANENTLY — regardless of whether a later LSN is subsequently
|
||
* recorded and grows term_index_count past it (bug-1475 follow-up; see
|
||
* TERM_INDEX_UNSET's comment). The raw term (including a legacy 0) is stored
|
||
* verbatim; the legacy-0 -> WAL_LEGACY_TERM mapping is applied only at
|
||
* READ-BACK (persist_entry_term), keeping "present legacy-0" distinguishable
|
||
* from "absent".
|
||
*
|
||
* Callers: the live append path (holds p->lock) and the single-threaded
|
||
* recovery path (replay_wal — no other thread touches the index before the node
|
||
* participates, so no lock is required there). Best-effort: an allocation
|
||
* failure is logged loudly and leaves the index unextended (read-back then
|
||
* reports the LSN absent) — it never aborts and never corrupts existing
|
||
* entries. */
|
||
static void record_entry_term(persist_t* p, uint64_t lsn, uint64_t term)
|
||
{
|
||
if (!p || lsn == 0)
|
||
return; /* LSN 0 is reserved — never an entry */
|
||
|
||
if (lsn > p->term_index_cap) {
|
||
/* Grow geometrically to amortize; never below the requested LSN. */
|
||
uint64_t new_cap = p->term_index_cap ? p->term_index_cap * 2 : 64;
|
||
if (new_cap < lsn)
|
||
new_cap = lsn;
|
||
uint64_t* grown = realloc(p->term_index, new_cap * sizeof(uint64_t));
|
||
if (!grown) {
|
||
ERROR("persist: term-index realloc failed (lsn=%" PRIu64
|
||
", cap=%" PRIu64 "->%" PRIu64 "); term for this LSN will read "
|
||
"back as absent", lsn, p->term_index_cap, new_cap);
|
||
return;
|
||
}
|
||
/* Fill the newly grown region with the UNSET sentinel (all-1s byte
|
||
* pattern, so memset can fill it directly) so any never-explicitly-
|
||
* written slot reads absent FOREVER, not just until term_index_count
|
||
* happens to grow past it (bug-1475 follow-up). */
|
||
memset(grown + p->term_index_cap, 0xFF,
|
||
(new_cap - p->term_index_cap) * sizeof(uint64_t));
|
||
p->term_index = grown;
|
||
p->term_index_cap = new_cap;
|
||
}
|
||
|
||
p->term_index[lsn - 1] = term;
|
||
if (lsn > p->term_index_count)
|
||
p->term_index_count = lsn;
|
||
}
|
||
|
||
/* entry_term_locked: the lock-free core of persist_entry_term — read the
|
||
* recorded Raft term for `lsn` from the in-memory term index, applying the
|
||
* legacy-0 rule (R1.8: a present term-0 entry maps to WAL_LEGACY_TERM; an absent
|
||
* LSN returns 0). Caller MUST hold p->lock. Used by the append path
|
||
* (persist_wal_async) to source the AppendEntries prev_log_term while already
|
||
* under the lock — calling persist_entry_term there would re-acquire p->lock
|
||
* and deadlock. */
|
||
static uint64_t entry_term_locked(const persist_t* p, uint64_t lsn)
|
||
{
|
||
if (!p || lsn == 0)
|
||
return 0;
|
||
if (lsn <= p->term_index_count && p->term_index != NULL) {
|
||
uint64_t raw = p->term_index[lsn - 1];
|
||
/* bug-1475 follow-up: a slot filled by growth but never explicitly
|
||
* recorded (e.g. a skipped WAL_CHECKPOINT lsn) stays absent forever —
|
||
* see TERM_INDEX_UNSET. */
|
||
if (raw == TERM_INDEX_UNSET)
|
||
return 0;
|
||
return (raw == 0) ? WAL_LEGACY_TERM : raw;
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
/* Shrink the in-memory per-LSN term index so that every LSN > `keep_lsn` reads
|
||
* back as absent (ha-term-safety task 2.2). Called from the truncate path after
|
||
* the on-disk entries above keep_lsn are removed, keeping the term index
|
||
* consistent with the WAL: a later persist_entry_term on a truncated LSN must
|
||
* return 0. The backing allocation is left as-is (only the live count moves);
|
||
* the TERM_INDEX_UNSET-sentinel-on-grow contract (bug-1475 follow-up — grown
|
||
* slots are filled with TERM_INDEX_UNSET, not zero, since this fix) means a
|
||
* later re-append at those LSNs still overwrites cleanly: record_entry_term
|
||
* always writes the slot explicitly, regardless of what fill value (zero or
|
||
* sentinel) it previously held. Caller must hold p->lock. */
|
||
static void truncate_term_index_locked(persist_t* p, uint64_t keep_lsn)
|
||
{
|
||
if (!p)
|
||
return;
|
||
if (keep_lsn < p->term_index_count)
|
||
p->term_index_count = keep_lsn;
|
||
}
|
||
|
||
void persist_set_term(persist_t* p, uint64_t term)
|
||
{
|
||
if (!p)
|
||
return;
|
||
/* p->term is read on the append path under p->lock; take the lock so the
|
||
* stamped value is consistent with the append that reads it. */
|
||
pthread_mutex_lock(&p->lock);
|
||
p->term = term;
|
||
pthread_mutex_unlock(&p->lock);
|
||
}
|
||
|
||
uint64_t persist_entry_term(const persist_t* p, uint64_t lsn)
|
||
{
|
||
if (!p || lsn == 0)
|
||
return 0;
|
||
|
||
/* Take p->lock to read consistently against concurrent record_entry_term
|
||
* on the append path. Cast away const — the lock is logically a mutable
|
||
* sub-object of an otherwise-const read. */
|
||
persist_t* mp = (persist_t*)p;
|
||
pthread_mutex_lock(&mp->lock);
|
||
|
||
/* Legacy-zero rule (R1.8) lives in entry_term_locked: a present entry
|
||
* carrying term 0 maps to the single lowest-known real Raft term, applied
|
||
* uniformly so two legacy-0 entries at the same LSN match; absent LSNs
|
||
* return 0. */
|
||
uint64_t result = entry_term_locked(mp, lsn);
|
||
|
||
pthread_mutex_unlock(&mp->lock);
|
||
return result;
|
||
}
|
||
|
||
uint64_t persist_entry_term_locked(const persist_t* p, uint64_t lsn)
|
||
{
|
||
return entry_term_locked(p, lsn);
|
||
}
|
||
|
||
/* persist_reconcile_checkpoint_term — raft-ha-core-safety task 3.2 (Req
|
||
* 3.1/5.1), Snapshot Boundary Reconcile. See persist.h for the full contract.
|
||
* Reuses record_entry_term (same translation unit) to authoritatively
|
||
* overwrite whatever term (if any) the local term index held at
|
||
* checkpoint_lsn — this is a genuine reconcile-overwrite, not an idempotent
|
||
* skip, because a pre-snapshot divergent entry may have left a stale local
|
||
* term at this exact LSN. Only the checkpoint LSN's own term is reconciled:
|
||
* post-install Raft semantics mean no AppendEntries prevLogIndex/prevLogTerm
|
||
* check against this member ever references an LSN below the checkpoint
|
||
* again (the checkpoint is the new log-start boundary), and the WAL bytes
|
||
* below it were compacted away, so there is no data to reconstruct a full
|
||
* per-LSN history from even if it were needed.
|
||
*
|
||
* bug-1475 Finding 2 (coordinator review, 2026-07-27/28): checkpoint_term==0
|
||
* on the wire is UNAMBIGUOUS today, not a legacy default to fall back to
|
||
* WAL_LEGACY_TERM. ha_snapshot_do_send (ha.c) — the only sender — computes
|
||
* this value via persist_entry_term(leader, checkpoint_lsn), whose public
|
||
* contract already collapses "absent" and "present raw-0 legacy" into
|
||
* DIFFERENT return values (0 vs WAL_LEGACY_TERM respectively — see
|
||
* entry_term_locked). A wire value of exactly 0 can therefore only mean
|
||
* "this sender's checkpoint LSN is genuinely, permanently absent" (the
|
||
* expected, designed state for ANY real checkpoint — persist_write_
|
||
* checkpoint_marker never records a term for its own entry). Before this
|
||
* fix, record_entry_term(p, checkpoint_lsn, 0) stored a RAW 0 — an EXPLICIT
|
||
* write, not a growth-fill, so it never got the TERM_INDEX_UNSET sentinel —
|
||
* and entry_term_locked's legacy-0 rule then read it back as
|
||
* WAL_LEGACY_TERM(1) forever, permanently diverging from the sender's own
|
||
* permanently-absent(0) value for the SAME LSN: a subsequent AppendEntries
|
||
* whose prev_log_index==checkpoint_lsn (the entry immediately following it)
|
||
* then mismatched logOk (0 != 1) forever — the identical wedge class
|
||
* bug-1475 fixed for WAL-streaming catch-up, reachable instead via
|
||
* HA_REPL_SEND_SNAPSHOT (dispatched whenever a standby has fallen too far
|
||
* behind for incremental streaming — a live, common path). Recording the
|
||
* SAME TERM_INDEX_UNSET sentinel here (instead of raw 0) keeps a genuinely-
|
||
* absent checkpoint boundary reading absent on the receiver too, matching
|
||
* the sender — while a genuinely non-zero checkpoint_term (a real historical
|
||
* term, however it arrived on the wire) is still recorded verbatim and still
|
||
* correctly overwrites any stale local term at this LSN, unchanged. */
|
||
void persist_reconcile_checkpoint_term(persist_t* p, uint64_t checkpoint_lsn,
|
||
uint64_t checkpoint_term)
|
||
{
|
||
if (!p || checkpoint_lsn == 0)
|
||
return;
|
||
|
||
uint64_t stored_term =
|
||
(checkpoint_term == 0) ? TERM_INDEX_UNSET : checkpoint_term;
|
||
|
||
pthread_mutex_lock(&p->lock);
|
||
record_entry_term(p, checkpoint_lsn, stored_term);
|
||
pthread_mutex_unlock(&p->lock);
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Public API */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
void persist_set_workdir(persist_t* p, const char* workdir, const char* name)
|
||
{
|
||
char effective[4096];
|
||
snprintf(effective, sizeof(effective), "%s/%s", workdir, name);
|
||
snprintf(p->wal_path, sizeof(p->wal_path), "%s/data/wal", effective);
|
||
snprintf(p->snap_path, sizeof(p->snap_path), "%s/data/snap", effective);
|
||
}
|
||
|
||
int persist_init(persist_t* p, persist_config_t* cfg, cluster_t* c)
|
||
{
|
||
p->cluster = c;
|
||
p->config = cfg ? *cfg : persist_config_defaults();
|
||
p->wal_fd = -1;
|
||
p->last_replicated_crc = WAL_CRC_NONE;
|
||
p->last_crc = WAL_CRC_NONE;
|
||
|
||
/* LSN 0 is RESERVED as the "none / empty" sentinel — it is NEVER assigned
|
||
* to a real WAL entry. A fresh store therefore starts current_lsn at 1, so
|
||
* the first-ever appended entry (e.g. the promotion WAL_NOOP) is lsn 1 and
|
||
* lsn 0 unambiguously means "nothing here":
|
||
* - committed_lsn == 0 → nothing committed (the lsn-1 NOOP is committable
|
||
* 0 → 1, vs. the old lsn-0 entry which could never advance committed_lsn);
|
||
* - writable_lsn == 0 → own-term gate off (the gate armed at lsn 1 is
|
||
* effective, vs. lsn 0 which read as "gate off" on a cold-start leader);
|
||
* - standby_lsn == 0 → empty standby (the streamer's `hdr.lsn > 0`
|
||
* filter streams the lsn-1 entry, closing the cold-start WAL hole that
|
||
* dropped lsn 0 on `0 > 0`).
|
||
* persist_recover() overwrites current_lsn from the durable snapshot/WAL tip
|
||
* (load_snapshot / replay_wal), so this fresh default only applies before any
|
||
* recovery — i.e. a brand-new store with no snapshot and no WAL. */
|
||
p->current_lsn = 1;
|
||
|
||
/* Enforce minimum value for max_wal_records to prevent runaway checkpointing */
|
||
if (p->config.max_wal_records < 100) {
|
||
ERROR("persist: max_wal_records=%u too low (minimum 100) — setting to 100",
|
||
p->config.max_wal_records);
|
||
p->config.max_wal_records = 100;
|
||
}
|
||
|
||
/* Enforce minimum value for wal_segment_bytes to prevent a rotation storm
|
||
* (must comfortably fit at least one max-payload entry: header + up to
|
||
* MAX_WAL_PAYLOAD_BYTES). */
|
||
if (p->config.wal_segment_bytes < 4096) {
|
||
ERROR("persist: wal_segment_bytes=%u too low (minimum 4096) — setting to 4096",
|
||
p->config.wal_segment_bytes);
|
||
p->config.wal_segment_bytes = 4096;
|
||
}
|
||
|
||
/* Initialise atomic checkpoint-trigger counters */
|
||
atomic_store(&p->wal_record_count, 0);
|
||
atomic_store(&p->checkpoint_in_progress, 0);
|
||
|
||
/* Durable watermark starts at 0 ("nothing fdatasync'd"): mirrors the
|
||
* LSN-0-reserved committed_lsn convention. It advances only on the
|
||
* flusher thread after a successful fdatasync (ha-commit-durability 2.1). */
|
||
atomic_store(&p->durable_lsn, 0);
|
||
|
||
/* Recovery-integrity verdict defaults to CLEAN (ha-commit-durability 3.1):
|
||
* a fresh store has no corruption. persist_recover overwrites this with the
|
||
* replay outcome (RECOVER_TRUNCATED only on a structural log corruption). */
|
||
p->recovery_verdict = RECOVER_CLEAN;
|
||
|
||
/* Sticky "unrecoverable, loudly fenced" flag starts clear (Req 2.4): a fresh
|
||
* store is promotable. Set by persist_recover_loud_fence_if_unrecoverable
|
||
* on an unrecoverable boot; cleared on catch-up (never self-exits). */
|
||
atomic_store(&p->recovery_unrecoverable, false);
|
||
|
||
/* Post-replay apply-correctness verdict defaults to CLEAN (raft-apply-
|
||
* decoupling task 5.1, Req 8.2): a fresh store has nothing to diverge
|
||
* from. persist_recover overwrites both after every replay. */
|
||
p->apply_verdict = APPLY_CLEAN;
|
||
atomic_store(&p->last_applied_lsn, 0);
|
||
|
||
/* Live Apply Resource Fence starts clear (job-array-single-wal-record
|
||
* task 4.2, Req 9.4): a fresh store has nothing stuck failing to apply.
|
||
* Set by persist_mark_apply_resource_fenced on a resource-exhaustion
|
||
* apply failure; cleared by persist_clear_apply_resource_fence the
|
||
* instant a retry of the same LSN succeeds -- see persist_t's field doc
|
||
* comment for why this is deliberately separate from
|
||
* recovery_unrecoverable just above. */
|
||
atomic_store(&p->apply_resource_fenced_lsn, 0);
|
||
|
||
/* bug-1416: defaults to true (nothing to gate) so every fixture that
|
||
* calls persist_init() but never persist_recover() -- most of this
|
||
* spec's own test suite -- sees identical, ungated apply-pump/live-
|
||
* replication behavior to before this flag existed. Only persist_
|
||
* recover() ever sets this false, for its own single-threaded
|
||
* replay+verify duration (see the field's doc comment in persist.h). */
|
||
atomic_store(&p->recovery_complete, true);
|
||
|
||
/* Durable-advance wake hook starts unregistered (ha-commit-durability 2.2):
|
||
* the commit engine registers it in commit_engine_init. Until then every
|
||
* flusher durable advance is a no-op wake. */
|
||
p->on_durable_advance = NULL;
|
||
p->durable_cb_ctx = NULL;
|
||
|
||
/* Truncation wake hook starts unregistered (raft-apply-decoupling task
|
||
* 1.4): the commit engine registers it in commit_engine_init. Until
|
||
* then every truncation is a no-op wake. */
|
||
p->on_truncate = NULL;
|
||
p->truncate_cb_ctx = NULL;
|
||
|
||
/* Per-LSN term index starts empty (ha-term-safety 1.3, R1.5): no entries
|
||
* recorded yet. Grown on demand by record_entry_term during replay/append;
|
||
* freed in persist_fini. */
|
||
p->term_index = NULL;
|
||
p->term_index_count = 0;
|
||
p->term_index_cap = 0;
|
||
|
||
/* Pending command-apply hand-off list starts empty (raft-apply-
|
||
* decoupling task 3.2): no committed-outcome continuations have
|
||
* registered yet. Freed in persist_fini. */
|
||
p->pending_applies = NULL;
|
||
pthread_mutex_init(&p->pending_applies_lock, NULL);
|
||
|
||
/* Ordered apply pump starts idle (raft-apply-decoupling task 6.1): no
|
||
* walk scheduled or running yet. Freed/destroyed in persist_fini. */
|
||
pthread_mutex_init(&p->apply_pump_lock, NULL);
|
||
atomic_store(&p->apply_pump_scheduled, 0);
|
||
|
||
/* Apply-pump segment-lookup cache starts invalid (raft-apply-decoupling
|
||
* task 7.4 perf-fix): no segment/fd cached yet. -1 is always a safe
|
||
* fallback trigger for apply_pump_read_entry's fast path (see persist.h
|
||
* doc comment). */
|
||
p->apply_pump_cache_fd = -1;
|
||
p->apply_pump_cache_seg_lsn = 0;
|
||
p->apply_pump_cache_off = 0;
|
||
p->apply_pump_cache_next_lsn = 0;
|
||
|
||
/* Create WAL and snapshot directories */
|
||
if (mkdir_p(p->wal_path, 0700) != 0) return -errno;
|
||
if (mkdir_p(p->snap_path, 0700) != 0) return -errno;
|
||
|
||
/* Reject NFS mounts — fsync durability is not guaranteed over NFS */
|
||
struct statfs sfs;
|
||
if (statfs(p->wal_path, &sfs) == 0 &&
|
||
(unsigned long)sfs.f_type == (unsigned long)NFS_SUPER_MAGIC) {
|
||
ERROR("persist: WAL path %s is on NFS — unsafe for fsync durability",
|
||
p->wal_path);
|
||
return -EREMOTE;
|
||
}
|
||
|
||
pthread_mutex_init(&p->lock, NULL);
|
||
pthread_mutex_init(&p->checkpoint_mutex, NULL);
|
||
pthread_cond_init(&p->checkpoint_done, NULL);
|
||
pthread_mutex_init(&p->fsync_mutex, NULL);
|
||
pthread_cond_init(&p->fsync_cond, NULL);
|
||
repl_init(&p->repl); /* P2-4: fan-out replicator lifecycle */
|
||
|
||
WARN("WAL/snapshot stored unencrypted in %s — "
|
||
"ensure filesystem is encrypted if job payloads contain sensitive data",
|
||
p->wal_path);
|
||
return 0;
|
||
}
|
||
|
||
int persist_start(persist_t* p)
|
||
{
|
||
ctrl_metrics_init();
|
||
|
||
/* Register AFTER_EVENT persistence hooks if we have a cluster back-pointer */
|
||
if (p->cluster != NULL) {
|
||
xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_JOB,
|
||
(hook_fn_t)persist_job_hook_fun, p);
|
||
xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_QUEUE,
|
||
(hook_fn_t)persist_queue_hook_fun, p);
|
||
xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_WORKER,
|
||
(hook_fn_t)persist_worker_hook_fun, p);
|
||
}
|
||
|
||
atomic_store_explicit(&p->fsync_stop, 0, memory_order_release);
|
||
int rc = pthread_create(&p->fsync_thread, NULL, fsync_worker, p);
|
||
if (rc == 0)
|
||
atomic_store_explicit(&p->fsync_thread_running, 1, memory_order_release);
|
||
return rc;
|
||
}
|
||
|
||
void persist_stop(persist_t* p)
|
||
{
|
||
if (atomic_load_explicit(&p->fsync_thread_running, memory_order_acquire)) {
|
||
/* Set the stop flag and wake the timer wait under fsync_mutex so the
|
||
* worker cannot miss the signal between its flag check and the wait. */
|
||
pthread_mutex_lock(&p->fsync_mutex);
|
||
atomic_store_explicit(&p->fsync_stop, 1, memory_order_release);
|
||
pthread_cond_signal(&p->fsync_cond);
|
||
pthread_mutex_unlock(&p->fsync_mutex);
|
||
pthread_join(p->fsync_thread, NULL);
|
||
atomic_store_explicit(&p->fsync_thread_running, 0, memory_order_release);
|
||
}
|
||
|
||
pthread_mutex_lock(&p->lock);
|
||
if (p->wal_fd >= 0) {
|
||
/* Final synchronous flush-or-exit on stop: fdatasync everything
|
||
* appended so far and advance durable_lsn to the synced tip so a
|
||
* post-stop read reflects the durable state (the flusher thread is
|
||
* already joined and will not advance it again).
|
||
*
|
||
* FAIL-LOUD GATE (Req 1.1/1.2): advance the watermark ONLY when the
|
||
* final sync SUCCEEDS — on EIO/ENOSPC the bytes are not on stable
|
||
* storage, so durable_lsn must stay where it is and the failure is
|
||
* surfaced loudly. */
|
||
int rc = fdatasync(p->wal_fd);
|
||
if (rc == 0 && !flusher_fsync_forced_fail(p)) {
|
||
uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
|
||
uint64_t dur = atomic_load_explicit(&p->durable_lsn,
|
||
memory_order_acquire);
|
||
if (tip > dur)
|
||
atomic_store_explicit(&p->durable_lsn, tip,
|
||
memory_order_release);
|
||
} else {
|
||
int e = (rc != 0) ? errno : EIO;
|
||
ERROR("persist: final fdatasync on stop FAILED (errno=%d %s) — "
|
||
"durable_lsn left unchanged; appended-but-unsynced entries "
|
||
"are NOT durable",
|
||
e, strerror(e));
|
||
}
|
||
close(p->wal_fd);
|
||
p->wal_fd = -1;
|
||
}
|
||
pthread_mutex_unlock(&p->lock);
|
||
}
|
||
|
||
void persist_fini(persist_t* p)
|
||
{
|
||
repl_fini(&p->repl); /* P2-4: fan-out replicator lifecycle */
|
||
pthread_mutex_destroy(&p->lock);
|
||
pthread_mutex_destroy(&p->checkpoint_mutex);
|
||
pthread_cond_destroy(&p->checkpoint_done);
|
||
pthread_mutex_destroy(&p->fsync_mutex);
|
||
pthread_cond_destroy(&p->fsync_cond);
|
||
|
||
/* Free the in-memory per-LSN term index (ha-term-safety 1.3). */
|
||
free(p->term_index);
|
||
p->term_index = NULL;
|
||
p->term_index_count = 0;
|
||
p->term_index_cap = 0;
|
||
|
||
/* Free any pending command-apply hand-off entries never drained by a
|
||
* (not-yet-built) task 6 apply pump (raft-apply-decoupling task 3.2).
|
||
* Each node's `ctx` is opaque to persist.c (owned by the registering
|
||
* caller, e.g. job.c's jobsubmit_async_ctx_t) — freeing the node here
|
||
* does NOT free ctx; a caller shutting down with entries still pending
|
||
* is responsible for its own ctx lifetime (test harnesses that tear
|
||
* down mid-flight must account for this). */
|
||
pending_command_apply_t* pn =
|
||
(pending_command_apply_t*)p->pending_applies;
|
||
while (pn != NULL) {
|
||
pending_command_apply_t* next = pn->next;
|
||
free(pn);
|
||
pn = next;
|
||
}
|
||
p->pending_applies = NULL;
|
||
pthread_mutex_destroy(&p->pending_applies_lock);
|
||
|
||
/* raft-apply-decoupling task 6.1: no walk can still be in flight by the
|
||
* time persist_fini runs (the caller owns quiescing any vtask pool
|
||
* first, same precondition every other persist_t resource here already
|
||
* assumes). */
|
||
pthread_mutex_destroy(&p->apply_pump_lock);
|
||
|
||
/* raft-apply-decoupling task 7.4 perf-fix: close any cached segment fd
|
||
* left open by apply_pump_read_entry. No walk can be in flight here
|
||
* (same precondition as apply_pump_lock's destroy just above), so no
|
||
* lock is needed. */
|
||
if (p->apply_pump_cache_fd >= 0) {
|
||
close(p->apply_pump_cache_fd);
|
||
p->apply_pump_cache_fd = -1;
|
||
}
|
||
}
|
||
|
||
/* Forward declaration — defined after checkpoint internals */
|
||
static void schedule_checkpoint(persist_t* persist);
|
||
|
||
int persist_wal_async(persist_t* p, uint32_t type,
|
||
const void* payload, uint32_t len)
|
||
{
|
||
/* WAL_CHECKPOINT is a local segment marker, never written via this path */
|
||
if (type == WAL_CHECKPOINT)
|
||
return -EINVAL;
|
||
|
||
pthread_mutex_lock(&p->lock);
|
||
|
||
/* Open first segment lazily */
|
||
if (p->wal_fd < 0) {
|
||
if (open_new_segment(p) != 0) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
}
|
||
|
||
/* Build entry header */
|
||
wal_entry_hdr_t hdr;
|
||
memset(&hdr, 0, sizeof(hdr));
|
||
hdr.term = p->term;
|
||
hdr.lsn = p->current_lsn++;
|
||
hdr.timestamp = now_us();
|
||
hdr.type = type;
|
||
hdr.payload_len = len;
|
||
hdr.prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc;
|
||
|
||
/* CRC32 over WAL_HDR_CRC_SIZE bytes of header (excludes crc + prev_crc
|
||
* fields themselves) then over the payload */
|
||
uint32_t c = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
|
||
if (payload && len > 0)
|
||
c = crc32(c, (const Bytef*)payload, len);
|
||
hdr.crc = c;
|
||
|
||
/* Write header */
|
||
ssize_t hw = pwrite(p->wal_fd, &hdr, sizeof(hdr), (off_t)p->seg_offset);
|
||
if (hw != (ssize_t)sizeof(hdr)) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
|
||
/* Write payload (if any) */
|
||
if (payload && len > 0) {
|
||
ssize_t pw = pwrite(p->wal_fd, payload, len,
|
||
(off_t)(p->seg_offset + sizeof(hdr)));
|
||
if (pw != (ssize_t)len) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
}
|
||
|
||
uint32_t entry_size = (uint32_t)(sizeof(hdr) + len);
|
||
p->seg_offset += entry_size;
|
||
p->total_wal_bytes += entry_size;
|
||
p->last_crc = hdr.crc;
|
||
|
||
/* Record this entry's term at its LSN for recovery read-back (ha-term-safety
|
||
* 1.3, R1.5). Under p->lock; stamps the same hdr.term written to disk. */
|
||
record_entry_term(p, hdr.lsn, hdr.term);
|
||
|
||
/* Replicate to standbys — never replicate WAL_CHECKPOINT entries.
|
||
* last_replicated_crc is distinct from last_crc:
|
||
* last_crc includes checkpoint markers; last_replicated_crc skips them.
|
||
* This maintains a continuous CRC chain for standbys (design doc Part 2.3).
|
||
* Note: WAL_CHECKPOINT is blocked by the assert at function entry, so
|
||
* every entry that reaches here is a replicable entry. */
|
||
p->last_replicated_crc = hdr.crc;
|
||
/* ha-term-safety task 3.4: source the AppendEntries prev_log_term (term of
|
||
* the entry at hdr.lsn-1) under p->lock via the lock-free accessor, so the
|
||
* live broadcast carries the leader's term + prev tuple and the standby
|
||
* routes it through the library (raft_on_append_entries_request). 0 when
|
||
* hdr.lsn == 1 (no predecessor). */
|
||
uint64_t prev_log_term = entry_term_locked(p, hdr.lsn - 1);
|
||
repl_broadcast(&p->repl, &hdr, payload, len, prev_log_term);
|
||
|
||
/* Rotate segment when full */
|
||
if (p->seg_offset >= p->config.wal_segment_bytes) {
|
||
if (rotate_segment(p) != 0) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
}
|
||
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
/* Signal the group-commit flusher (ha-commit-durability 2.1): current_lsn
|
||
* advanced and the entry's bytes are pwritten, so durable_lsn < tip now
|
||
* holds (the level predicate). The signal is sent AFTER releasing p->lock
|
||
* and is level-triggered — if it races the flusher and is "lost", the
|
||
* flusher re-checks durable_lsn < current_lsn-1 under p->lock and flushes
|
||
* anyway (see fsync_worker). We do NOT fdatasync inline on this path; the
|
||
* flusher batches the sync across all pending appends. */
|
||
pthread_mutex_lock(&p->fsync_mutex);
|
||
pthread_cond_signal(&p->fsync_cond);
|
||
pthread_mutex_unlock(&p->fsync_mutex);
|
||
|
||
/* WAL record-count checkpoint trigger: schedule an async checkpoint once
|
||
* the record count reaches the configured threshold. The fetch_add is
|
||
* outside the lock — atomic_fetch_add is sufficient for the counter; the
|
||
* checkpoint itself is guarded by its own CAS inside schedule_checkpoint. */
|
||
if (atomic_fetch_add(&p->wal_record_count, 1) + 1 >=
|
||
p->config.max_wal_records) {
|
||
schedule_checkpoint(p);
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
/* Lock-held core of persist_truncate_wal_above (defined below): assumes the
|
||
* caller already holds p->lock and never touches the lock itself. The
|
||
* replicated-write conflict guard reuses it to repair a divergent entry while
|
||
* still under p->lock (the follower-AE-conflict truncate contract). */
|
||
static int truncate_wal_above_locked(persist_t* p, uint64_t cutoff_lsn);
|
||
|
||
/* read_entry_meta_locked: scan the on-disk WAL for the entry at `lsn` and
|
||
* return its (payload_len, payload-only CRC, raw stored term). The
|
||
* replicated-write conflict guard uses this to compare a re-streamed entry
|
||
* against the copy already on disk WITHOUT trusting the LSN alone. pcrc is
|
||
* computed as crc32(0, payload, len) — the SAME payload-only fingerprint
|
||
* walcompare/waldump compare across members (NOT the chained header CRC); an
|
||
* empty (PLEN=0) entry yields len=0, pcrc=0. Reads through the page cache so
|
||
* a written-but-not-yet-fdatasync'd entry is still visible. Caller MUST hold
|
||
* p->lock. Returns 0 if the entry is found, -1 if absent or unreadable. */
|
||
static int read_entry_meta_locked(persist_t* p, uint64_t lsn,
|
||
uint32_t* out_len, uint32_t* out_pcrc,
|
||
uint64_t* out_term)
|
||
{
|
||
if (!p || lsn == 0)
|
||
return -1;
|
||
|
||
DIR* d = opendir(p->wal_path);
|
||
if (!d)
|
||
return -1;
|
||
|
||
uint64_t seg_lsns[4096];
|
||
int nseg = 0;
|
||
struct dirent* e;
|
||
while ((e = readdir(d)) != NULL &&
|
||
nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) {
|
||
if (strncmp(e->d_name, "seg-", 4) != 0)
|
||
continue;
|
||
uint64_t sl = 0;
|
||
if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &sl) != 1)
|
||
continue;
|
||
seg_lsns[nseg++] = sl;
|
||
}
|
||
closedir(d);
|
||
if (nseg == 0)
|
||
return -1;
|
||
|
||
/* Ascending sort by start LSN (segment count is tiny). */
|
||
for (int i = 1; i < nseg; i++) {
|
||
uint64_t key = seg_lsns[i];
|
||
int j = i - 1;
|
||
while (j >= 0 && seg_lsns[j] > key) {
|
||
seg_lsns[j + 1] = seg_lsns[j];
|
||
j--;
|
||
}
|
||
seg_lsns[j + 1] = key;
|
||
}
|
||
|
||
int rc = -1;
|
||
for (int si = 0; si < nseg && rc != 0; si++) {
|
||
/* Segments are LSN-ordered; one whose start LSN already exceeds the
|
||
* target cannot contain it (nor can any later one). */
|
||
if (seg_lsns[si] > lsn)
|
||
break;
|
||
|
||
char seg_path[4096];
|
||
snprintf(seg_path, sizeof(seg_path),
|
||
WAL_SEG_FMT, p->wal_path, seg_lsns[si]);
|
||
int fd = open(seg_path, O_RDONLY);
|
||
if (fd < 0)
|
||
continue;
|
||
|
||
uint32_t off = 0;
|
||
for (;;) {
|
||
wal_entry_hdr_t hdr;
|
||
ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)off);
|
||
if (hn < (ssize_t)sizeof(hdr))
|
||
break; /* short read / EOF / pre-allocated zeros */
|
||
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
|
||
break;
|
||
|
||
if (hdr.lsn == lsn) {
|
||
uint32_t pc = 0;
|
||
if (hdr.payload_len > 0) {
|
||
uint8_t* pl = malloc(hdr.payload_len);
|
||
if (!pl)
|
||
break; /* leave rc == -1: unreadable → conservative */
|
||
ssize_t pn = pread(fd, pl, hdr.payload_len,
|
||
(off_t)(off + sizeof(hdr)));
|
||
if (pn == (ssize_t)hdr.payload_len)
|
||
pc = crc32(0L, (const Bytef*)pl, hdr.payload_len);
|
||
free(pl);
|
||
if (pn != (ssize_t)hdr.payload_len)
|
||
break; /* unreadable body → conservative */
|
||
}
|
||
if (out_len)
|
||
*out_len = hdr.payload_len;
|
||
if (out_pcrc)
|
||
*out_pcrc = pc;
|
||
if (out_term)
|
||
*out_term = hdr.term;
|
||
rc = 0;
|
||
break;
|
||
}
|
||
|
||
off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
}
|
||
close(fd);
|
||
}
|
||
return rc;
|
||
}
|
||
|
||
int persist_wal_write_replicated(persist_t* p, uint64_t lsn, uint32_t type,
|
||
const void* payload, uint32_t len)
|
||
{
|
||
if (!p)
|
||
return -EINVAL;
|
||
|
||
/* WAL_CHECKPOINT is a local-only marker, never streamed to standbys. */
|
||
if (type == WAL_CHECKPOINT)
|
||
return -EINVAL;
|
||
|
||
/* Reject oversized payloads BEFORE touching the segment. */
|
||
if (len > MAX_WAL_PAYLOAD_BYTES)
|
||
return -EMSGSIZE;
|
||
|
||
pthread_mutex_lock(&p->lock);
|
||
|
||
/* Tip-repair bookkeeping (set only on the TIP-conflict repair branch below):
|
||
* the truncate-and-reappend repair transiently lowers committed_lsn to
|
||
* lsn-1, so we snapshot the true committed boundary here and restore it
|
||
* EXACTLY after the re-append re-establishes the entry at `lsn` (never let a
|
||
* same-(term,lsn) payload repair regress the committed boundary — commit
|
||
* monotonicity / State Machine Safety). */
|
||
int repaired_tip = 0;
|
||
uint64_t repaired_committed_save = 0;
|
||
|
||
/* Payload-aware conflict guard (raft-ha-core-safety task 2.2): this lsn is
|
||
* already persisted (current_lsn advanced past it). The legacy guard was
|
||
* (lsn)-only — it no-op'd ANY re-stream, including a corrective full-payload
|
||
* write over a previously-stored empty (PLEN=0) body, which is exactly the
|
||
* cross-term catch-up payload-loss signature (a committed LSN that carries
|
||
* the right (term,lsn) but an empty payload). Compare (payload-crc, len)
|
||
* against the copy already on disk and act on the difference:
|
||
* (c) identical payload → idempotent no-op (keep).
|
||
* (b) incoming EMPTY over a stored entry → never degrade (keep stored);
|
||
* an empty body is never the durable form of a known entry.
|
||
* (a) incoming NON-EMPTY whose payload differs from the stored copy at an
|
||
* existing (term,lsn) → a genuine payload conflict whose handling
|
||
* depends on WHERE the conflict sits relative to the stored tip:
|
||
* · AT THE TIP (lsn == current_lsn - 1, nothing stored above) →
|
||
* REPAIR: truncate just this one entry and re-append the
|
||
* authoritative payload at the same (term,lsn). Because nothing
|
||
* is stored above it, no distinct committed entry is destroyed,
|
||
* and the committed boundary is restored verbatim afterwards.
|
||
* · BELOW THE TIP (lsn < current_lsn - 1, committed entries stored
|
||
* ABOVE the conflict) → MUST NOT truncate: dropping [lsn..tip]
|
||
* would delete committed entries above the conflict AND roll
|
||
* committed_lsn/durable_lsn backward — a State Machine Safety /
|
||
* commit-monotonicity violation. Instead keep the stored entry
|
||
* untouched, raise a loud non-promotable fence, and let catch-up
|
||
* reconverge. The real source fix (the streamer must stop
|
||
* shipping an empty body) is task 2.4; the guard's only job here
|
||
* is to NOT make things worse (never drop a committed entry,
|
||
* never regress a boundary) and to surface the corruption loudly.
|
||
* An unreadable stored entry stays a conservative no-op — never truncate a
|
||
* committed LSN we cannot read back. */
|
||
if (lsn + 1 <= p->current_lsn) {
|
||
uint32_t inc_pcrc =
|
||
(payload && len > 0) ? crc32(0L, (const Bytef*)payload, len) : 0;
|
||
uint32_t st_len = 0;
|
||
uint32_t st_pcrc = 0;
|
||
uint64_t st_term = 0;
|
||
int have = read_entry_meta_locked(p, lsn, &st_len, &st_pcrc,
|
||
&st_term);
|
||
|
||
int payload_matches =
|
||
(have == 0 && st_len == len && st_pcrc == inc_pcrc);
|
||
|
||
/* (c) identical payload, (b) empty incoming, or an unreadable stored
|
||
* entry → leave the stored entry untouched. */
|
||
if (payload_matches || len == 0 || have != 0) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return 0;
|
||
}
|
||
|
||
/* (a) non-empty incoming whose payload differs at an existing (term,lsn)
|
||
* — a genuine payload conflict. Branch on tip vs below-tip. */
|
||
uint64_t tip_lsn = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
|
||
|
||
if (lsn != tip_lsn) {
|
||
/* BELOW-TIP conflict: committed entries are stored ABOVE `lsn`.
|
||
* Truncating here would delete them and roll committed_lsn/
|
||
* durable_lsn backward (State Machine Safety / commit-monotonicity
|
||
* violation — bug-class the round-1 review rejected). Do NOT
|
||
* truncate, do NOT lower any boundary, do NOT degrade the stored
|
||
* entry. Surface loudly and raise the sticky non-promotable fence
|
||
* (reuse the recovery_unrecoverable mechanism: loud, stays ALIVE,
|
||
* non-promotable, self-heals via persist_clear_recovery_unrecoverable
|
||
* once catch-up reaches a committed-consistent state — never
|
||
* self-exits, per the controller invariant). The source fix lives
|
||
* in task 2.4 (the streamer must stop shipping an empty payload). */
|
||
ERROR("persist: persist_wal_write_replicated payload CONFLICT BELOW "
|
||
"TIP at lsn=%" PRIu64 " (stored_term=%" PRIu64
|
||
" stored_len=%u stored_pcrc=%08x vs incoming_len=%u "
|
||
"incoming_pcrc=%08x); current_lsn=%" PRIu64 " committed_lsn=%"
|
||
PRIu64 " — refusing to truncate committed entries above the "
|
||
"conflict (would regress the committed boundary). Keeping the "
|
||
"stored entry, raising the non-promotable fence and staying "
|
||
"ALIVE; awaiting a correct re-stream (source fix: task 2.4)",
|
||
lsn, st_term, st_len, st_pcrc, len, inc_pcrc, p->current_lsn,
|
||
atomic_load_explicit(&p->committed_lsn, memory_order_acquire));
|
||
atomic_store_explicit(&p->recovery_unrecoverable, true,
|
||
memory_order_release);
|
||
pthread_mutex_unlock(&p->lock);
|
||
return 0; /* no-op the write: keep stored, never degrade/regress */
|
||
}
|
||
|
||
/* TIP conflict: `lsn` is the highest stored entry, so nothing is stored
|
||
* above it — truncate-and-reappend replaces EXACTLY this one divergent
|
||
* entry. truncate_wal_above_locked(lsn-1) drops it and sets
|
||
* current_lsn = lsn, so control falls through to the append below and
|
||
* re-persists the authoritative entry at this exact LSN. Snapshot the
|
||
* committed boundary first: the truncate transiently rewrites
|
||
* committed_lsn to lsn-1, and we restore it verbatim after the
|
||
* re-append so a same-(term,lsn) payload repair never regresses (nor
|
||
* spuriously advances) the committed boundary. */
|
||
repaired_committed_save =
|
||
atomic_load_explicit(&p->committed_lsn, memory_order_acquire);
|
||
int trc = truncate_wal_above_locked(p, lsn - 1);
|
||
if (trc != 0) {
|
||
ERROR("persist: persist_wal_write_replicated repair-truncate FAILED "
|
||
"for lsn=%" PRIu64 " (rc=%d) — divergent entry NOT repaired",
|
||
lsn, trc);
|
||
pthread_mutex_unlock(&p->lock);
|
||
return trc;
|
||
}
|
||
repaired_tip = 1;
|
||
/* fall through to the contiguity guard + append (lsn == current_lsn). */
|
||
}
|
||
|
||
/* Contiguity guard (bug-850): REFUSE a forward GAP. current_lsn is the
|
||
* next-expected LSN, so a replicated entry with lsn > current_lsn would
|
||
* leave [current_lsn .. lsn-1] missing AND advance current_lsn past that
|
||
* hole — after which the idempotency guard above silently DROPS the later
|
||
* in-order catch-up of the prefix (the failover prefix-loss mechanism).
|
||
* Append only the contiguous next entry (lsn == current_lsn); reject a gap
|
||
* without writing or acking so the ACK-driven catch-up continuation
|
||
* re-streams from current_lsn and the log fills in order. This is Raft
|
||
* log-matching: append entry N only when the log already holds N-1. A
|
||
* behind standby that keeps receiving live broadcasts simply drops the
|
||
* out-of-order ones until catch-up backfills the gap (no loss). -EAGAIN is
|
||
* the "retry via catch-up" signal the apply path treats as a quiet skip. */
|
||
if (lsn > p->current_lsn) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EAGAIN;
|
||
}
|
||
|
||
/* Open first segment lazily. open_new_segment names the file from
|
||
* current_lsn — for a fresh standby receiving lsn 1 first, current_lsn is
|
||
* 1 (reserve-lsn-0), so the segment is seg-...001.wal. */
|
||
if (p->wal_fd < 0) {
|
||
if (open_new_segment(p) != 0) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
}
|
||
|
||
/* Build entry header. Unlike persist_wal_async, lsn is GIVEN (not
|
||
* current_lsn++). term isn't streamed, so use our best-effort p->term.
|
||
* The CRC chain is SELF-computed (the streamed entry carries no crc):
|
||
* recovery validates an intra-segment chain (prev_crc == last_crc), so the
|
||
* standby's own writes must chain to each other — cross-node CRC equality
|
||
* is neither required nor possible. */
|
||
wal_entry_hdr_t hdr;
|
||
memset(&hdr, 0, sizeof(hdr));
|
||
hdr.term = p->term;
|
||
hdr.lsn = lsn;
|
||
hdr.timestamp = now_us();
|
||
hdr.type = type;
|
||
hdr.payload_len = len;
|
||
hdr.prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc;
|
||
|
||
uint32_t c = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
|
||
if (payload && len > 0)
|
||
c = crc32(c, (const Bytef*)payload, len);
|
||
hdr.crc = c;
|
||
|
||
/* Write header */
|
||
ssize_t hw = pwrite(p->wal_fd, &hdr, sizeof(hdr), (off_t)p->seg_offset);
|
||
if (hw != (ssize_t)sizeof(hdr)) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
|
||
/* Write payload (if any) */
|
||
if (payload && len > 0) {
|
||
ssize_t pw = pwrite(p->wal_fd, payload, len,
|
||
(off_t)(p->seg_offset + sizeof(hdr)));
|
||
if (pw != (ssize_t)len) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
}
|
||
|
||
uint32_t entry_size = (uint32_t)(sizeof(hdr) + len);
|
||
p->seg_offset += entry_size;
|
||
p->total_wal_bytes += entry_size;
|
||
p->last_crc = hdr.crc;
|
||
|
||
/* Record this entry's term at its LSN for recovery read-back (ha-term-safety
|
||
* 1.3, R1.5). Stamps the same hdr.term written to disk. NOTE: the streamed
|
||
* envelope does not yet carry the leader's term, so this records the local
|
||
* best-effort p->term (the design's known hazard closed by task 2.x, which
|
||
* carries the leader's term on the envelope; this task only provides the
|
||
* read-back seam over whatever term was persisted). */
|
||
record_entry_term(p, hdr.lsn, hdr.term);
|
||
|
||
/* Advance current_lsn to one past the given lsn (max-idempotent — mirrors
|
||
* apply_wal_entry's monotonic advance). */
|
||
if (lsn + 1 > p->current_lsn)
|
||
p->current_lsn = lsn + 1;
|
||
|
||
/* Durability: fsync the segment BEFORE returning so the caller's ACK
|
||
* implies the entry is on stable storage (persist-before-ACK).
|
||
* persist_wal_async defers fsync to the background thread / persist_flush;
|
||
* here we sync inline because the entry feeds the primary's commit
|
||
* majority and must survive a standby restart. fdatasync is sufficient
|
||
* (entry bytes + segment metadata are pre-allocated by open_new_segment).
|
||
*
|
||
* Fail-loud rc-gate (ha-commit-durability Req 1.3, symmetric with the
|
||
* append-path gate at task 2.1): the ACK is sent by the caller ONLY when
|
||
* this returns success, and the ACK claims the entry is durable. So if
|
||
* the fdatasync FAILS (EIO/ENOSPC, or the forced-fail test seam), we must
|
||
* NOT return success — we log loudly and return -EIO so the caller's
|
||
* existing error path (ha_apply_single_entry) does NOT apply or ACK. */
|
||
int fsrc = fdatasync(p->wal_fd);
|
||
#ifdef XCE_TESTING
|
||
atomic_fetch_add_explicit(&p->test_replicated_fdatasync_count, 1,
|
||
memory_order_relaxed);
|
||
#endif
|
||
if (fsrc != 0 || flusher_fsync_forced_fail(p)) {
|
||
int e = (fsrc != 0) ? errno : EIO;
|
||
ERROR("persist: persist_wal_write_replicated fdatasync FAILED for "
|
||
"lsn=%" PRIu64 " (errno=%d %s) — entry NOT durable; returning "
|
||
"error so the standby does NOT ACK", lsn, e, strerror(e));
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
|
||
/* bug-828 secondary: the entry is now fdatasync'd durable, so advance
|
||
* durable_lsn to the synced tip immediately (mirror persist_flush). The
|
||
* 10ms cadence flusher would otherwise advance it lazily, leaving this
|
||
* standby's ADVERTISED durable position (persist_durable_floor →
|
||
* RequestVoteReply.peerLastLogLsn) lagging its own synced tip during the
|
||
* vote window. A standby that has durably persisted the committed prefix
|
||
* must advertise it at once so a clean failover's safe-commit-from-votes
|
||
* computation sees the true majority-held LSN. Monotonic max; preserves
|
||
* durable_lsn <= current_lsn - 1 (tip read under this same p->lock, after
|
||
* the current_lsn advance above). Reached ONLY past the fail-loud rc-gate
|
||
* — never advances on an fdatasync error. */
|
||
{
|
||
uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
|
||
uint64_t dur = atomic_load_explicit(&p->durable_lsn,
|
||
memory_order_acquire);
|
||
if (tip > dur)
|
||
atomic_store_explicit(&p->durable_lsn, tip, memory_order_release);
|
||
}
|
||
|
||
/* Tip-repair committed-boundary restore (raft-ha-core-safety task 2.2): a
|
||
* TIP-conflict repair above truncated [lsn..tip] (rewriting committed_lsn to
|
||
* lsn-1) and has now re-appended + fdatasync'd the authoritative entry at the
|
||
* SAME (term,lsn). Restore the committed boundary VERBATIM so the repair is
|
||
* commit-boundary-neutral: a same-(term,lsn) payload correction must never
|
||
* regress committed_lsn (State Machine Safety / commit monotonicity) nor
|
||
* spuriously advance it (truncate would otherwise have raised it to lsn-1).
|
||
* Reached only past the fail-loud fdatasync gate, with current_lsn re-advanced
|
||
* to lsn+1 and durable_lsn at the synced tip, so the invariant
|
||
* committed_lsn <= min(current_lsn - 1, durable_lsn) holds (saved value was
|
||
* itself <= the pre-repair tip == lsn). The whole repair is serialized under
|
||
* p->lock, so no external observer ever sees the transient lowered value. */
|
||
if (repaired_tip) {
|
||
atomic_store_explicit(&p->committed_lsn, repaired_committed_save,
|
||
memory_order_release);
|
||
}
|
||
|
||
/* Rotate segment when full (same threshold as the primary append). */
|
||
if (p->seg_offset >= p->config.wal_segment_bytes) {
|
||
if (rotate_segment(p) != 0) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
}
|
||
|
||
/* NOTE: no repl_broadcast — standbys never re-replicate. */
|
||
|
||
pthread_mutex_unlock(&p->lock);
|
||
return 0;
|
||
}
|
||
|
||
int persist_flush(persist_t* p)
|
||
{
|
||
pthread_mutex_lock(&p->lock);
|
||
int rc = (p->wal_fd >= 0) ? fdatasync(p->wal_fd) : 0;
|
||
|
||
/* persist_flush is a synchronous durability point (tests, clean shutdown):
|
||
* it fdatasync'd everything appended so far, so advance durable_lsn to the
|
||
* synced tip too. Without this, an explicit-flush-only path (slow flusher
|
||
* cadence) would leave durable_lsn lagging even though the bytes are on
|
||
* disk. Monotonic max; preserves durable_lsn <= current_lsn - 1 (tip read
|
||
* under the same p->lock). Gated on rc == 0 (fail-loud, Req 1.1/1.2): a
|
||
* failed flush must not advance the watermark. */
|
||
bool advanced = false;
|
||
if (rc == 0 && !flusher_fsync_forced_fail(p)) {
|
||
uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
|
||
uint64_t dur = atomic_load_explicit(&p->durable_lsn,
|
||
memory_order_acquire);
|
||
if (tip > dur) {
|
||
atomic_store_explicit(&p->durable_lsn, tip, memory_order_release);
|
||
advanced = true;
|
||
}
|
||
} else if (rc != 0 || flusher_fsync_forced_fail(p)) {
|
||
int e = (rc != 0) ? errno : EIO;
|
||
ERROR("persist: persist_flush fdatasync FAILED (errno=%d %s) — "
|
||
"durable_lsn not advanced; flushed bytes are NOT durable",
|
||
e, strerror(e));
|
||
}
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
/* bug-1564: fire the same durable-advance wake fsync_worker fires on its
|
||
* own successful flush (identical gate: only when durable_lsn actually
|
||
* advanced; identical lock discipline: strictly after releasing p->lock,
|
||
* so the callback -> ce->lock ordering has no p->lock held underneath
|
||
* it, matching fsync_worker's own documented cycle-free reasoning
|
||
* above). Latent gap unmasked by bug-1563: this synchronous flush path
|
||
* (tests, clean shutdown) used to advance durable_lsn silently -- any
|
||
* WAIT-blocked commit caller relying on this callback to know a fresh
|
||
* durable-majority exists would only ever notice via the ASYNC
|
||
* fsync_worker's own next successful flush, not this call. Before
|
||
* bug-1563's fix, fsync_worker's un-throttled retry loop happened to
|
||
* paper over the gap (it re-checked and re-fired within microseconds of
|
||
* any state change); bug-1563's backoff correctly removed that spin,
|
||
* which is what surfaced this as a real, reproducible test timeout
|
||
* (test_durable_majority_commit_end_to_end: commit_submit(WAIT) timing
|
||
* out at -110/ETIMEDOUT, deterministic 3/3 direct runs) instead of a
|
||
* pre-existing gap nothing had ever pushed hard enough to expose. */
|
||
if (advanced && p->on_durable_advance != NULL) {
|
||
p->on_durable_advance(p->durable_cb_ctx);
|
||
}
|
||
|
||
return rc;
|
||
}
|
||
|
||
uint64_t persist_durable_lsn(persist_t* p)
|
||
{
|
||
return atomic_load_explicit(&p->durable_lsn, memory_order_acquire);
|
||
}
|
||
|
||
uint64_t persist_durable_floor(persist_t* p)
|
||
{
|
||
/* The durable freshness floor (durable last-log index) IS the durable
|
||
* watermark: persist_recover re-derives it from the recovered WAL tip
|
||
* (ha-commit-durability task 3.4), and the flusher advances it on every
|
||
* successful fdatasync — so a thin wrapper avoids a redundant field. NULL
|
||
* guard keeps the accessor safe for the election consumers (task 4.2). */
|
||
return p ? atomic_load_explicit(&p->durable_lsn, memory_order_acquire) : 0;
|
||
}
|
||
|
||
void persist_set_durable_cb(persist_t* p, void (*fn)(void* ctx), void* ctx)
|
||
{
|
||
if (!p)
|
||
return;
|
||
/* Set ctx before fn so the flusher (which checks fn != NULL) never sees a
|
||
* live callback with a stale/uninitialised ctx. Registration happens in
|
||
* commit_engine_init, before the engine is reachable by any ACK path. */
|
||
p->durable_cb_ctx = ctx;
|
||
p->on_durable_advance = fn;
|
||
}
|
||
|
||
/* raft-apply-decoupling task 1.4: registration counterpart of
|
||
* persist_set_durable_cb -- see persist.h for the full contract. */
|
||
void persist_set_truncate_cb(persist_t* p, void (*fn)(void* ctx, uint64_t cutoff_lsn),
|
||
void* ctx)
|
||
{
|
||
if (!p)
|
||
return;
|
||
p->truncate_cb_ctx = ctx;
|
||
p->on_truncate = fn;
|
||
}
|
||
|
||
#ifdef XCE_TESTING
|
||
void persist_test_set_force_fsync_fail(persist_t* p, int fail)
|
||
{
|
||
if (p)
|
||
atomic_store_explicit(&p->test_force_fsync_fail, fail ? 1 : 0,
|
||
memory_order_release);
|
||
}
|
||
|
||
void persist_test_set_force_job_encode_fail_at(persist_t* p, int index)
|
||
{
|
||
if (p)
|
||
atomic_store_explicit(&p->test_force_job_encode_fail_at, index + 1,
|
||
memory_order_relaxed);
|
||
}
|
||
|
||
unsigned persist_test_replicated_fdatasync_count(persist_t* p)
|
||
{
|
||
if (!p)
|
||
return 0;
|
||
return atomic_load_explicit(&p->test_replicated_fdatasync_count,
|
||
memory_order_relaxed);
|
||
}
|
||
|
||
pthread_t persist_test_last_fdatasync_thread(persist_t* p)
|
||
{
|
||
/* If no flusher fdatasync has advanced durable_lsn yet, return the
|
||
* caller's own id so an unprimed read never spuriously looks off-thread.
|
||
* After a flush, return the recorded flusher thread id (Req 1.6). */
|
||
if (!p ||
|
||
atomic_load_explicit(&p->test_last_fdatasync_thread_set,
|
||
memory_order_acquire) == 0)
|
||
return pthread_self();
|
||
return p->test_last_fdatasync_thread;
|
||
}
|
||
|
||
/* Force a fresh WAL segment at the current LSN (test seam — see persist.h).
|
||
* Takes p->lock for the same single-writer serialization the append path uses,
|
||
* so the segment swap cannot race a concurrent write. */
|
||
int persist_test_force_new_segment(persist_t* p)
|
||
{
|
||
if (!p)
|
||
return -EINVAL;
|
||
pthread_mutex_lock(&p->lock);
|
||
int rc = rotate_segment(p);
|
||
pthread_mutex_unlock(&p->lock);
|
||
return rc;
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ *
|
||
* Durability-verification test seam (ha-commit-durability Req 3.2/3.3)*
|
||
* *
|
||
* Re-read + CRC-validate the WAL segments from stable storage and *
|
||
* return the highest fully-persisted LSN — independent of the *
|
||
* in-memory current_lsn / committed_lsn watermarks. Mirrors the *
|
||
* segment-scan + CRC-chain validation of replay_wal(), but is strictly *
|
||
* read-only: it applies no entries, fires no hooks, and mutates no *
|
||
* persist_t state, so it cannot change the externally observable *
|
||
* acknowledgment semantics (Req 3.3). *
|
||
* *
|
||
* Stops at the first corruption / short read / pre-allocated zero *
|
||
* region (exactly like recovery), so only entries genuinely on stable *
|
||
* storage are counted. Returns the highest such LSN, or 0 when nothing *
|
||
* is durable. *
|
||
* ------------------------------------------------------------------ */
|
||
uint64_t persist_test_durable_lsn_on_disk(persist_t* p)
|
||
{
|
||
if (!p)
|
||
return 0;
|
||
|
||
DIR* d = opendir(p->wal_path);
|
||
if (!d)
|
||
return 0; /* no WAL dir → nothing durable */
|
||
|
||
/* Collect segment start LSNs (same bound as replay_wal). */
|
||
uint64_t seg_lsns[4096];
|
||
int nseg = 0;
|
||
|
||
struct dirent* e;
|
||
while ((e = readdir(d)) != NULL &&
|
||
nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) {
|
||
if (strncmp(e->d_name, "seg-", 4) != 0)
|
||
continue;
|
||
uint64_t seg_start = 0;
|
||
if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start) != 1)
|
||
continue;
|
||
seg_lsns[nseg++] = seg_start;
|
||
}
|
||
closedir(d);
|
||
|
||
if (nseg == 0)
|
||
return 0;
|
||
|
||
/* Insertion sort by start LSN (segment count is tiny). */
|
||
for (int i = 1; i < nseg; i++) {
|
||
uint64_t key = seg_lsns[i];
|
||
int j = i - 1;
|
||
while (j >= 0 && seg_lsns[j] > key) {
|
||
seg_lsns[j + 1] = seg_lsns[j];
|
||
j--;
|
||
}
|
||
seg_lsns[j + 1] = key;
|
||
}
|
||
|
||
uint64_t highest_durable = 0;
|
||
uint32_t last_crc = WAL_CRC_NONE;
|
||
|
||
for (int si = 0; si < nseg; si++) {
|
||
char seg_path[4096];
|
||
snprintf(seg_path, sizeof(seg_path),
|
||
WAL_SEG_FMT, p->wal_path, seg_lsns[si]);
|
||
|
||
int fd = open(seg_path, O_RDONLY);
|
||
if (fd < 0)
|
||
continue;
|
||
|
||
uint32_t seg_off = 0;
|
||
|
||
for (;;) {
|
||
wal_entry_hdr_t hdr;
|
||
ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)seg_off);
|
||
if (hn == 0)
|
||
break; /* clean end of segment */
|
||
if (hn < (ssize_t)sizeof(hdr))
|
||
break; /* truncated / pre-allocated zeros */
|
||
|
||
/* Pre-allocated zero region: no valid entry is all-zero. */
|
||
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
|
||
break;
|
||
|
||
/* CRC chain validation — identical rule to replay_wal. */
|
||
if (seg_off == 0) {
|
||
if (hdr.prev_crc != WAL_CRC_NONE && last_crc != WAL_CRC_NONE &&
|
||
hdr.prev_crc != last_crc)
|
||
break;
|
||
} else {
|
||
if (hdr.prev_crc != last_crc)
|
||
break;
|
||
}
|
||
|
||
/* Verify entry CRC over header + payload. */
|
||
uint32_t expected_crc =
|
||
crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
|
||
|
||
uint8_t* payload = NULL;
|
||
if (hdr.payload_len > 0) {
|
||
payload = malloc(hdr.payload_len);
|
||
if (!payload) {
|
||
close(fd);
|
||
return highest_durable;
|
||
}
|
||
ssize_t pn = pread(fd, payload, hdr.payload_len,
|
||
(off_t)(seg_off + sizeof(hdr)));
|
||
if (pn != (ssize_t)hdr.payload_len) {
|
||
free(payload);
|
||
break;
|
||
}
|
||
expected_crc =
|
||
crc32(expected_crc, (const Bytef*)payload, hdr.payload_len);
|
||
}
|
||
|
||
if (hdr.crc != expected_crc) {
|
||
free(payload);
|
||
break;
|
||
}
|
||
free(payload);
|
||
|
||
/* Entry is intact and durable on disk. */
|
||
last_crc = hdr.crc;
|
||
if (hdr.lsn > highest_durable)
|
||
highest_durable = hdr.lsn;
|
||
|
||
seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
}
|
||
|
||
close(fd);
|
||
}
|
||
|
||
return highest_durable;
|
||
}
|
||
#endif /* XCE_TESTING */
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Checkpoint internals */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
/* Write a WAL_CHECKPOINT marker to the current segment.
|
||
* Never calls repl_broadcast() — checkpoint markers are local only. */
|
||
static int persist_write_checkpoint_marker(persist_t* p)
|
||
{
|
||
pthread_mutex_lock(&p->lock);
|
||
|
||
if (p->wal_fd < 0) {
|
||
if (open_new_segment(p) != 0) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
}
|
||
|
||
wal_entry_hdr_t hdr;
|
||
memset(&hdr, 0, sizeof(hdr));
|
||
hdr.term = p->term;
|
||
hdr.lsn = p->current_lsn++;
|
||
hdr.timestamp = now_us();
|
||
hdr.type = WAL_CHECKPOINT;
|
||
hdr.payload_len = 0;
|
||
hdr.prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc;
|
||
|
||
uint32_t c = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
|
||
hdr.crc = c;
|
||
|
||
ssize_t hw = pwrite(p->wal_fd, &hdr, sizeof(hdr), (off_t)p->seg_offset);
|
||
if (hw != (ssize_t)sizeof(hdr)) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
fdatasync(p->wal_fd);
|
||
|
||
p->seg_offset += (uint32_t)sizeof(hdr);
|
||
p->total_wal_bytes += sizeof(hdr);
|
||
p->last_crc = hdr.crc;
|
||
/* NOTE: last_replicated_crc is NOT updated — WAL_CHECKPOINT is never replicated */
|
||
/* NOTE: the term index is NOT updated for WAL_CHECKPOINT — replay_wal SKIPS
|
||
* checkpoint markers (local bookkeeping, not a logical Raft entry), so the
|
||
* recovered index never holds a checkpoint LSN; the live index must match
|
||
* (ha-term-safety 1.3). persist_entry_term is for (term,lsn) reconcile, which
|
||
* never compares against a checkpoint marker. */
|
||
|
||
pthread_mutex_unlock(&p->lock);
|
||
return 0;
|
||
}
|
||
|
||
/* Entity-bearing checkpoint payload (ha-entity-snapshots task 2.2/3.2, Req
|
||
* 1.1-1.4). Physical layout appended after the existing legacy header
|
||
* (design.md "Checkpoint File Layout"):
|
||
* [uint32_t format_version]
|
||
* [uint32_t job_count]
|
||
* [job_count * { uint32_t payload_len; uint8_t capnp_Job_bytes[payload_len] }]
|
||
* [uint32_t queue_count]
|
||
* [queue_count * { uint32_t name_len; char name[name_len]; uint32_t status }]
|
||
*
|
||
* config-upgrade-snapshot task 1 (Req 1.9, 5.7): bumped 1->2 to reserve the
|
||
* version for the HostAdminState/QueueAdminState capnp migration -- the
|
||
* queue-frame layout above (name_len/name/status raw bytes) is the PRIOR
|
||
* layout this version now hard-rejects; the byte-layout migration itself
|
||
* (new host section, queue frames re-encoded as capnp) lands in tasks
|
||
* 2.1/2.2. A binary built against version 2 must never misinterpret a
|
||
* checkpoint written under version 1's raw queue-frame layout (Design
|
||
* Invariant 4: fail loud, never silently reinterpret). */
|
||
#define SNAPSHOT_FORMAT_VERSION 2u
|
||
|
||
typedef struct {
|
||
uint8_t* buf;
|
||
uint32_t len;
|
||
} snap_job_frame_t;
|
||
|
||
static void snap_free_job_frames(xlist_t* frames)
|
||
{
|
||
for (int i = 0; i < frames->count; i++) {
|
||
snap_job_frame_t* fr = (snap_job_frame_t*)xlist_get(frames, i);
|
||
if (fr) {
|
||
free(fr->buf);
|
||
free(fr);
|
||
}
|
||
}
|
||
xlist_fini(frames);
|
||
}
|
||
|
||
/* Collect + encode every retained job into `frames`, one malloc'd
|
||
* snap_job_frame_t per job, under a point-in-time-consistent walk
|
||
* (design.md "Existing Architecture Analysis" / Requirement 1.2): the
|
||
* jobmgr table's own RTTI_LOCK_READ is held for the ENTIRE walk (excludes
|
||
* concurrent insert/delete), and each retained job's own RTTI_LOCK_READ is
|
||
* ADDITIONALLY taken nested around just that job's encode call (excludes a
|
||
* concurrent per-entity writer for exactly the duration of its encode --
|
||
* the table lock alone does not do this). No lock is held during the
|
||
* disk write itself: this function only builds in-memory frames; the
|
||
* caller writes them to disk after the table walk has fully released both
|
||
* locks.
|
||
*
|
||
* Filters via job_should_retain (Requirement 1.1) so the checkpoint's
|
||
* notion of "live" matches the existing GC retention policy.
|
||
*
|
||
* Returns 0 on success (frames populated; `frames` is always left
|
||
* xlist_init'd -- even zero results -- so the caller can always
|
||
* xlist_fini it) or a negative errno on the first encode failure, in which
|
||
* case every frame collected so far has already been freed and `frames`
|
||
* is empty (Requirement 1.3: a mid-walk failure must not leave partial
|
||
* entity content behind). */
|
||
static int collect_job_frames(persist_t* p, xlist_t* frames)
|
||
{
|
||
cluster_t* c = p->cluster;
|
||
int rc = 0;
|
||
int retained_index = 0;
|
||
|
||
xlist_init(frames, 64);
|
||
|
||
/* No live cluster back-pointer (e.g. a bare unit-test persist_t) --
|
||
* zero jobs is not a failure. */
|
||
if (c == NULL || c->jobmgr == NULL)
|
||
return 0;
|
||
|
||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||
|
||
iterator_t it;
|
||
int irc = iter_make(&it, c->jobmgr);
|
||
if (irc < 0) {
|
||
unlock(c->jobmgr);
|
||
return 0; /* nothing to walk -- mirrors collect_jobs' empty-table handling */
|
||
}
|
||
|
||
for (;;) {
|
||
job_t* j = NULL;
|
||
int nrc = iter_next(&it, (void**)&j);
|
||
if (nrc < 0 || nrc == RTTI_ITER_NO_DATA)
|
||
break;
|
||
if (j == NULL)
|
||
continue;
|
||
if (!job_should_retain(j, &p->config))
|
||
continue;
|
||
|
||
uint8_t* buf = NULL;
|
||
size_t sz = 0;
|
||
int enc_rc;
|
||
|
||
lock(j, RTTI_LOCK_READ);
|
||
#ifdef XCE_TESTING
|
||
int forced = atomic_load_explicit(&p->test_force_job_encode_fail_at,
|
||
memory_order_relaxed);
|
||
if (forced > 0 && (forced - 1) == retained_index)
|
||
enc_rc = -EIO; /* simulated mid-loop encode failure */
|
||
else
|
||
enc_rc = capnp_encode_job(&j->job, &buf, &sz);
|
||
#else
|
||
enc_rc = capnp_encode_job(&j->job, &buf, &sz);
|
||
#endif
|
||
unlock(j);
|
||
|
||
retained_index++;
|
||
|
||
if (enc_rc != 0) {
|
||
rc = enc_rc;
|
||
break;
|
||
}
|
||
|
||
snap_job_frame_t* fr = (snap_job_frame_t*)malloc(sizeof(*fr));
|
||
if (fr == NULL) {
|
||
free(buf);
|
||
rc = -ENOMEM;
|
||
break;
|
||
}
|
||
fr->buf = buf;
|
||
fr->len = (uint32_t)sz;
|
||
if (xlist_store(frames, fr) < 0) {
|
||
free(buf);
|
||
free(fr);
|
||
rc = -ENOMEM;
|
||
break;
|
||
}
|
||
}
|
||
|
||
iter_destroy(&it);
|
||
unlock(c->jobmgr);
|
||
|
||
if (rc != 0)
|
||
snap_free_job_frames(frames); /* Requirement 1.3: discard, not partial */
|
||
|
||
return rc;
|
||
}
|
||
|
||
/* Queue admin-status frame (config-upgrade-snapshot task 2.2, Req 1.9, 5.7).
|
||
* Migrated from the prior raw {name_len, name, status} layout to a capnp
|
||
* QueueAdminState payload -- identical {buf,len} shape to snap_host_frame_t
|
||
* (both carry a capnp payload), same (name, status) field content as
|
||
* before, restore_queue_state's contract (queue.c:208) unchanged; only the
|
||
* wire encoding changes (design.md "Snapshot Entity/Config Payload"). */
|
||
typedef struct {
|
||
uint8_t* buf;
|
||
uint32_t len;
|
||
} snap_queue_frame_t;
|
||
|
||
static void snap_free_queue_frames(xlist_t* frames)
|
||
{
|
||
for (int i = 0; i < frames->count; i++) {
|
||
snap_queue_frame_t* fr = (snap_queue_frame_t*)xlist_get(frames, i);
|
||
if (fr) {
|
||
free(fr->buf);
|
||
free(fr);
|
||
}
|
||
}
|
||
xlist_fini(frames);
|
||
}
|
||
|
||
/* Collect + encode every queue in `frames`, one malloc'd snap_queue_frame_t
|
||
* per queue, under the same point-in-time-consistent walk discipline as
|
||
* collect_job_frames above (design.md "Existing Architecture Analysis" /
|
||
* Requirement 1.2): the qmgr table's own RTTI_LOCK_READ is held for the
|
||
* ENTIRE walk, and each queue's own RTTI_LOCK_READ is ADDITIONALLY taken
|
||
* nested around just that queue's (name, status) read and capnp encode --
|
||
* mirroring collect_host_frames' identical lock-then-encode-then-unlock
|
||
* shape. No lock is held during the disk write itself.
|
||
*
|
||
* Unlike collect_job_frames, no retention predicate is applied: a queue's
|
||
* mere presence in the live qmgr table already means config provisioned
|
||
* it (design.md "write_snapshot (extended)" Responsibilities), so every
|
||
* queue found is included.
|
||
*
|
||
* Returns 0 on success (frames populated; `frames` is always left
|
||
* xlist_init'd, even for zero results, so the caller can always xlist_fini
|
||
* it) or a negative errno on the first encode failure, in which case every
|
||
* frame collected so far has already been freed and `frames` is empty
|
||
* (Requirement 1.3: a mid-walk failure must not leave partial entity
|
||
* content behind). */
|
||
static int collect_queue_frames(persist_t* p, xlist_t* frames)
|
||
{
|
||
cluster_t* c = p->cluster;
|
||
int rc = 0;
|
||
|
||
xlist_init(frames, 64);
|
||
|
||
/* No live cluster back-pointer (e.g. a bare unit-test persist_t) --
|
||
* zero queues is not a failure. */
|
||
if (c == NULL || c->qmgr == NULL)
|
||
return 0;
|
||
|
||
lock(c->qmgr, RTTI_LOCK_READ);
|
||
|
||
iterator_t it;
|
||
int irc = iter_make(&it, c->qmgr);
|
||
if (irc < 0) {
|
||
unlock(c->qmgr);
|
||
return 0; /* nothing to walk -- mirrors collect_queues' empty-table handling */
|
||
}
|
||
|
||
for (;;) {
|
||
queue_t* q = NULL;
|
||
int nrc = iter_next(&it, (void**)&q);
|
||
if (nrc < 0 || nrc == RTTI_ITER_NO_DATA)
|
||
break;
|
||
if (q == NULL)
|
||
continue;
|
||
|
||
capnp_queue_admin_state_t qs;
|
||
uint8_t* buf = NULL;
|
||
size_t sz = 0;
|
||
int enc_rc;
|
||
|
||
lock(q, RTTI_LOCK_READ);
|
||
qs.name = q->queue.queue;
|
||
qs.status = q->queue.status;
|
||
enc_rc = capnp_encode_queue_admin_state(&qs, &buf, &sz);
|
||
unlock(q);
|
||
|
||
if (enc_rc != 0) {
|
||
rc = enc_rc;
|
||
break;
|
||
}
|
||
|
||
snap_queue_frame_t* fr = (snap_queue_frame_t*)malloc(sizeof(*fr));
|
||
if (fr == NULL) {
|
||
free(buf);
|
||
rc = -ENOMEM;
|
||
break;
|
||
}
|
||
fr->buf = buf;
|
||
fr->len = (uint32_t)sz;
|
||
if (xlist_store(frames, fr) < 0) {
|
||
free(buf);
|
||
free(fr);
|
||
rc = -ENOMEM;
|
||
break;
|
||
}
|
||
}
|
||
|
||
iter_destroy(&it);
|
||
unlock(c->qmgr);
|
||
|
||
if (rc != 0)
|
||
snap_free_queue_frames(frames); /* Requirement 1.3: discard, not partial */
|
||
|
||
return rc;
|
||
}
|
||
|
||
/* Bits set by admin commands that must survive a checkpoint round-trip.
|
||
* Mirrors worker.c's and lib/wobj.c's own local XCE_WORKER_ADMIN_MASK
|
||
* definitions -- both already independently duplicate this exact bit list
|
||
* (design.md Design Invariant 6 notes this pre-existing duplication); this
|
||
* is a third, deliberate copy for the same reason (no shared header exists
|
||
* for it today), not a new layering dependency on worker.c. */
|
||
#define PERSIST_WORKER_ADMIN_MASK \
|
||
(XCE_WORKER_LOCKEDU | XCE_WORKER_CLOSED | XCE_WORKER_HIDDEN \
|
||
| XCE_WORKER_CLOSED_BY_ADMIN)
|
||
|
||
typedef struct {
|
||
uint8_t* buf;
|
||
uint32_t len;
|
||
} snap_host_frame_t;
|
||
|
||
static void snap_free_host_frames(xlist_t* frames)
|
||
{
|
||
for (int i = 0; i < frames->count; i++) {
|
||
snap_host_frame_t* fr = (snap_host_frame_t*)xlist_get(frames, i);
|
||
if (fr) {
|
||
free(fr->buf);
|
||
free(fr);
|
||
}
|
||
}
|
||
xlist_fini(frames);
|
||
}
|
||
|
||
/* Collect + encode every host in c->workermgr into `frames`, one malloc'd
|
||
* snap_host_frame_t (identical {buf,len} shape to snap_job_frame_t -- both
|
||
* carry a capnp payload) per host, under the same point-in-time-consistent
|
||
* walk discipline as
|
||
* collect_job_frames/collect_queue_frames above (design.md "Existing
|
||
* Architecture Analysis" / Requirement 1.2): the workermgr table's own
|
||
* RTTI_LOCK_READ is held for the ENTIRE walk, and each worker's own
|
||
* RTTI_LOCK_READ is ADDITIONALLY taken nested around just that host's
|
||
* (name, status) read and encode. No lock is held during the disk write
|
||
* itself.
|
||
*
|
||
* Like collect_queue_frames and UNLIKE collect_job_frames, no retention
|
||
* predicate is applied: every host currently in the live workermgr table
|
||
* gets exactly one frame (mirrors collect_queue_frames's own "mere presence
|
||
* already means it's included" precedent above), so restore later sees a
|
||
* frame for every host this member currently knows about, whether or not
|
||
* any admin bit happens to be set on it right now.
|
||
*
|
||
* Each frame carries ONLY the host's admin-issued status bits
|
||
* (w->host.status & PERSIST_WORKER_ADMIN_MASK) -- never the raw status
|
||
* (which also carries dynamic/liveness bits like XCE_WORKER_UNAVAIL) and
|
||
* never any other xce_worker_t field (attrs, resvals, dead_since, ...):
|
||
* Requirement 1.7's exclusion, design.md Design Invariant 5 (never inherit
|
||
* a live struct's dynamic fields just because reusing it is convenient --
|
||
* capnp_encode_worker/Worker is deliberately NOT used here). `name` is
|
||
* read directly off w->host.name -- the same field key_find(c->workermgr,
|
||
* name) looks up elsewhere in this codebase (join.c:91, worker.c:524,645,751,
|
||
* lib/wobj.c:212), so a later restore's key_find(name) lookup matches
|
||
* exactly what was collected here.
|
||
*
|
||
* Returns 0 on success (frames populated; `frames` is always left
|
||
* xlist_init'd -- even zero results -- so the caller can always
|
||
* xlist_fini it) or a negative errno on the first encode failure, in which
|
||
* case every frame collected so far has already been freed and `frames`
|
||
* is empty (Requirement 1.3: a mid-walk failure must not leave partial
|
||
* entity content behind). */
|
||
static int collect_host_frames(persist_t* p, xlist_t* frames)
|
||
{
|
||
cluster_t* c = p->cluster;
|
||
int rc = 0;
|
||
|
||
xlist_init(frames, 64);
|
||
|
||
/* No live cluster back-pointer (e.g. a bare unit-test persist_t) --
|
||
* zero hosts is not a failure. */
|
||
if (c == NULL || c->workermgr == NULL)
|
||
return 0;
|
||
|
||
lock(c->workermgr, RTTI_LOCK_READ);
|
||
|
||
iterator_t it;
|
||
int irc = iter_make(&it, c->workermgr);
|
||
if (irc < 0) {
|
||
unlock(c->workermgr);
|
||
return 0; /* nothing to walk -- mirrors collect_job/queue_frames' empty-table handling */
|
||
}
|
||
|
||
for (;;) {
|
||
worker_t* w = NULL;
|
||
int nrc = iter_next(&it, (void**)&w);
|
||
if (nrc < 0 || nrc == RTTI_ITER_NO_DATA)
|
||
break;
|
||
if (w == NULL)
|
||
continue;
|
||
|
||
capnp_host_admin_state_t h;
|
||
uint8_t* buf = NULL;
|
||
size_t sz = 0;
|
||
int enc_rc;
|
||
|
||
lock(w, RTTI_LOCK_READ);
|
||
h.name = w->host.name;
|
||
h.status = w->host.status & PERSIST_WORKER_ADMIN_MASK;
|
||
enc_rc = capnp_encode_host_admin_state(&h, &buf, &sz);
|
||
unlock(w);
|
||
|
||
if (enc_rc != 0) {
|
||
rc = enc_rc;
|
||
break;
|
||
}
|
||
|
||
snap_host_frame_t* fr = (snap_host_frame_t*)malloc(sizeof(*fr));
|
||
if (fr == NULL) {
|
||
free(buf);
|
||
rc = -ENOMEM;
|
||
break;
|
||
}
|
||
fr->buf = buf;
|
||
fr->len = (uint32_t)sz;
|
||
if (xlist_store(frames, fr) < 0) {
|
||
free(buf);
|
||
free(fr);
|
||
rc = -ENOMEM;
|
||
break;
|
||
}
|
||
}
|
||
|
||
iter_destroy(&it);
|
||
unlock(c->workermgr);
|
||
|
||
if (rc != 0)
|
||
snap_free_host_frames(frames); /* Requirement 1.3: discard, not partial */
|
||
|
||
return rc;
|
||
}
|
||
|
||
/* Bound on the config bytes this file will embed in a checkpoint -- mirrors
|
||
* config_repl.h's CONF_MAX_SIZE (1 MiB) exactly. Duplicated as its own local
|
||
* constant rather than including config_repl.h, per the zero-config_repl.c-
|
||
* dependency decision documented on read_active_config_bytes() below. */
|
||
#define PERSIST_CONFIG_BYTES_MAX_SIZE (1u * 1024u * 1024u)
|
||
|
||
/* read_active_config_bytes (config-upgrade-snapshot task 2.3, Req 2.1, 2.3,
|
||
* 2.4, 5.1): read the locally-held active configuration's raw bytes straight
|
||
* off disk, for embedding in every checkpoint's trailing config-bytes
|
||
* section.
|
||
*
|
||
* DELIBERATE DESIGN CHOICE -- read this before changing the call site below:
|
||
* this function duplicates config_repl.c's `conf_read_current_filename` +
|
||
* `conf_read_raw` path-resolution-and-read logic INLINE, rather than calling
|
||
* those functions (config_repl.h is not included by this file at all). This
|
||
* mirrors scan_latest_committed_config_ref()'s own precedent just above,
|
||
* which independently decodes WAL_CONFIG_MARKER payloads inline "so persist
|
||
* stays free of a config_repl layering dependency" instead of adding a
|
||
* persist.c -> config_repl.c function-call edge -- and design.md's
|
||
* "Snapshot Entity/Config Payload" + File Structure Plan sections direct the
|
||
* SAME treatment here verbatim ("no new config_repl.c function is added...
|
||
* this deliberately mirrors scan_latest_committed_config_ref's own existing
|
||
* precedent of avoiding a persist.c -> config_repl.c function-call
|
||
* dependency, rather than introducing the layering violation a new accessor
|
||
* would create").
|
||
*
|
||
* A future reader who spots ha.c's two `extern int conf_read_raw(...)` call
|
||
* sites (config push to standbys) and wonders why this file doesn't do the
|
||
* same: ha.c is the intentional CONSUMER of config-bytes sourcing
|
||
* (design.md's Architecture boundary map: "config replication (config_repl.c)
|
||
* owns config-bytes sourcing"), so a ha.c -> config_repl.c call is expected,
|
||
* in-boundary integration. persist.c owns materialization/on-disk-format and
|
||
* is deliberately kept free of any dependency on config_repl.c's domain --
|
||
* the same boundary this file already draws for the WAL-marker case above.
|
||
* This is a considered choice, not an oversight: duplicating this ~30-line
|
||
* read-with-validation routine once was judged cheaper than adding this
|
||
* file's first cross-domain coupling edge into config_repl.c.
|
||
*
|
||
* Returns 0 with *out_bytes (caller must free()) / *out_len set on success.
|
||
* Returns -ENOENT when no local config bytes exist yet: no cluster/myconf
|
||
* back-pointer (e.g. a bare unit-test persist_t), no workdir/name configured,
|
||
* no conf/current pointer file yet (this node has not received its first
|
||
* config push), or the pointer names a versioned file that is not present.
|
||
* This is a BENIGN, expected state (early boot / not-yet-provisioned), never
|
||
* corruption -- callers must treat -ENOENT as "no config yet", not failure.
|
||
* Any OTHER negative errno (-EINVAL malformed pointer/oversized file, -EIO
|
||
* read failure, -ENOMEM, -EFBIG) indicates genuine on-disk corruption or a
|
||
* real I/O failure and must propagate as a hard failure, never be silently
|
||
* downgraded to an empty section. */
|
||
static int read_active_config_bytes(persist_t* p, uint8_t** out_bytes,
|
||
size_t* out_len)
|
||
{
|
||
cluster_t* c = p->cluster;
|
||
if (c == NULL || c->myconf == NULL ||
|
||
c->myconf->workdir == NULL || c->myconf->name == NULL)
|
||
return -ENOENT;
|
||
|
||
/* effective_dir = workdir/name -- must match conf_save_config's own
|
||
* path construction (config_repl.c:781) exactly. */
|
||
char conf_dir[4096];
|
||
snprintf(conf_dir, sizeof(conf_dir), "%s/%s", c->myconf->workdir,
|
||
c->myconf->name);
|
||
|
||
/* Step 1: resolve the "current" versioned filename -- mirrors
|
||
* conf_read_current_filename's validation exactly (non-empty, no path
|
||
* separator, "cluster.bin." prefix + 12 lowercase hex chars) so a
|
||
* corrupted pointer file is caught here just as loudly as it would be
|
||
* there. */
|
||
char head_path[4160];
|
||
snprintf(head_path, sizeof(head_path), "%s/conf/current", conf_dir);
|
||
|
||
FILE* hf = fopen(head_path, "r");
|
||
if (hf == NULL)
|
||
return -ENOENT; /* no config pushed to this node yet */
|
||
|
||
struct stat hst;
|
||
if (fstat(fileno(hf), &hst) == 0 && hst.st_size == 0) {
|
||
fclose(hf);
|
||
return -ENOENT; /* interrupted-write pointer file -- treat as absent */
|
||
}
|
||
|
||
char fname[256];
|
||
if (fgets(fname, (int)sizeof(fname), hf) == NULL) {
|
||
fclose(hf);
|
||
return -EIO;
|
||
}
|
||
fclose(hf);
|
||
fname[strcspn(fname, "\n")] = '\0';
|
||
|
||
if (strchr(fname, '/') != NULL)
|
||
return -EINVAL; /* corrupted pointer file -- embedded path separator */
|
||
if (strncmp(fname, "cluster.bin.", 12) != 0)
|
||
return -EINVAL;
|
||
const char* hex = fname + 12;
|
||
if (strlen(hex) != 12 || strspn(hex, "0123456789abcdef") != 12)
|
||
return -EINVAL;
|
||
|
||
/* Step 2: read the versioned config file's raw bytes. */
|
||
char cfg_path[4416];
|
||
snprintf(cfg_path, sizeof(cfg_path), "%s/conf/%s", conf_dir, fname);
|
||
|
||
FILE* cf = fopen(cfg_path, "rb");
|
||
if (cf == NULL)
|
||
return -ENOENT; /* pointer references a file that is not there --
|
||
* treat as not-yet-ready rather than corruption */
|
||
|
||
if (fseek(cf, 0, SEEK_END) != 0) { fclose(cf); return -EIO; }
|
||
long sz = ftell(cf);
|
||
if (sz < 0) { fclose(cf); return -EIO; }
|
||
if (fseek(cf, 0, SEEK_SET) != 0) { fclose(cf); return -EIO; }
|
||
|
||
if (sz == 0) {
|
||
fclose(cf);
|
||
return -EIO; /* zero-byte versioned config file: genuine corruption */
|
||
}
|
||
if ((size_t)sz > PERSIST_CONFIG_BYTES_MAX_SIZE) {
|
||
fclose(cf);
|
||
return -EFBIG;
|
||
}
|
||
|
||
uint8_t* buf = (uint8_t*)malloc((size_t)sz);
|
||
if (buf == NULL) {
|
||
fclose(cf);
|
||
return -ENOMEM;
|
||
}
|
||
if (fread(buf, 1, (size_t)sz, cf) != (size_t)sz) {
|
||
free(buf);
|
||
fclose(cf);
|
||
return -EIO;
|
||
}
|
||
fclose(cf);
|
||
|
||
*out_bytes = buf;
|
||
*out_len = (size_t)sz;
|
||
return 0;
|
||
}
|
||
|
||
/* Write an entity-bearing checkpoint: legacy header (unchanged) + entity
|
||
* payload (ha-entity-snapshots task 2.2/3.2, Req 1.1-1.4, 3.1-3.3). Any
|
||
* failure at any step -- collect, encode, write, sync -- leaves NO file at
|
||
* the final path; the tmp file is always unlinked on error. This is a
|
||
* retriable-next-checkpoint-cycle failure, never a crash/abort (controller
|
||
* never self-terminates on an operational condition). */
|
||
static int write_snapshot(persist_t* p)
|
||
{
|
||
char tmp_path[4096];
|
||
char final_path[4096];
|
||
|
||
snprintf(tmp_path, sizeof(tmp_path),
|
||
"%s/snap-%016" PRIx64 ".tmp", p->snap_path, p->checkpoint_lsn);
|
||
snprintf(final_path, sizeof(final_path),
|
||
"%s/snap-%016" PRIx64 ".capnp", p->snap_path, p->checkpoint_lsn);
|
||
|
||
/* Collect + encode entities BEFORE opening the tmp file: a collection
|
||
* failure (Requirement 1.3) then never creates a tmp file at all. */
|
||
xlist_t job_frames;
|
||
int rc = collect_job_frames(p, &job_frames);
|
||
if (rc != 0)
|
||
return rc;
|
||
|
||
xlist_t queue_frames;
|
||
rc = collect_queue_frames(p, &queue_frames);
|
||
if (rc != 0) {
|
||
snap_free_job_frames(&job_frames);
|
||
return rc;
|
||
}
|
||
|
||
xlist_t host_frames;
|
||
rc = collect_host_frames(p, &host_frames);
|
||
if (rc != 0) {
|
||
snap_free_job_frames(&job_frames);
|
||
snap_free_queue_frames(&queue_frames);
|
||
return rc;
|
||
}
|
||
|
||
int fd = open(tmp_path, O_CREAT | O_WRONLY | O_TRUNC, 0600);
|
||
if (fd < 0) {
|
||
snap_free_job_frames(&job_frames);
|
||
snap_free_queue_frames(&queue_frames);
|
||
snap_free_host_frames(&host_frames);
|
||
return -errno;
|
||
}
|
||
|
||
/* Sentinel: magic (8) + checkpoint_lsn (8) + committed-config reference
|
||
* (1 present + 8 tag + 32 sha = 41 bytes; ha-term-safety task 4.2 r1).
|
||
* The config-ref tail is OPTIONAL on read-back, so an older 16-byte
|
||
* sentinel still loads (load_snapshot reads the tail best-effort). The
|
||
* tag is little-endian on this arch, matching the checkpoint_lsn field. */
|
||
char header[16 + 1 + 8 + 32];
|
||
memset(header, 0, sizeof(header));
|
||
memcpy(header, "XCESNAP\0", 8);
|
||
memcpy(header + 8, &p->checkpoint_lsn, 8);
|
||
header[16] = (char)(p->cfgref_present ? 1 : 0);
|
||
memcpy(header + 17, &p->cfgref_tag, 8);
|
||
memcpy(header + 25, p->cfgref_sha, 32);
|
||
|
||
if (write(fd, header, sizeof(header)) != (ssize_t)sizeof(header))
|
||
rc = -EIO;
|
||
|
||
/* Entity payload: format_version, then job frames (design.md "Checkpoint
|
||
* File Layout"). No table/entity lock is held here -- collect_job_frames
|
||
* already released both before returning. */
|
||
if (rc == 0) {
|
||
uint32_t fmt = SNAPSHOT_FORMAT_VERSION;
|
||
if (write(fd, &fmt, sizeof(fmt)) != (ssize_t)sizeof(fmt))
|
||
rc = -EIO;
|
||
}
|
||
|
||
if (rc == 0) {
|
||
uint32_t job_count = (uint32_t)job_frames.count;
|
||
if (write(fd, &job_count, sizeof(job_count)) != (ssize_t)sizeof(job_count))
|
||
rc = -EIO;
|
||
}
|
||
|
||
for (int i = 0; rc == 0 && i < job_frames.count; i++) {
|
||
snap_job_frame_t* fr = (snap_job_frame_t*)xlist_get(&job_frames, i);
|
||
if (write(fd, &fr->len, sizeof(fr->len)) != (ssize_t)sizeof(fr->len)) {
|
||
rc = -EIO;
|
||
break;
|
||
}
|
||
if (write(fd, fr->buf, fr->len) != (ssize_t)fr->len) {
|
||
rc = -EIO;
|
||
break;
|
||
}
|
||
}
|
||
|
||
/* Queue trailer (config-upgrade-snapshot task 2.2, Req 1.9, 5.7):
|
||
* capnp QueueAdminState frames, same [payload_len][capnp bytes] shape
|
||
* as the job/host frame loops -- migrated from the prior raw
|
||
* {name_len, name, status} layout (design.md "Checkpoint File Layout":
|
||
* same field content, new wire encoding only). */
|
||
if (rc == 0) {
|
||
uint32_t queue_count = (uint32_t)queue_frames.count;
|
||
if (write(fd, &queue_count, sizeof(queue_count)) != (ssize_t)sizeof(queue_count))
|
||
rc = -EIO;
|
||
}
|
||
|
||
for (int i = 0; rc == 0 && i < queue_frames.count; i++) {
|
||
snap_queue_frame_t* fr = (snap_queue_frame_t*)xlist_get(&queue_frames, i);
|
||
if (write(fd, &fr->len, sizeof(fr->len)) != (ssize_t)sizeof(fr->len)) {
|
||
rc = -EIO;
|
||
break;
|
||
}
|
||
if (write(fd, fr->buf, fr->len) != (ssize_t)fr->len) {
|
||
rc = -EIO;
|
||
break;
|
||
}
|
||
}
|
||
|
||
/* Host trailer (config-upgrade-snapshot task 2.1, Req 1.6, 1.7, 1.9):
|
||
* capnp HostAdminState frames, same [payload_len][capnp bytes] shape as
|
||
* the job frame loop above (design.md "Checkpoint File Layout": host
|
||
* frames come after queue frames). */
|
||
if (rc == 0) {
|
||
uint32_t host_count = (uint32_t)host_frames.count;
|
||
if (write(fd, &host_count, sizeof(host_count)) != (ssize_t)sizeof(host_count))
|
||
rc = -EIO;
|
||
}
|
||
|
||
for (int i = 0; rc == 0 && i < host_frames.count; i++) {
|
||
snap_host_frame_t* fr = (snap_host_frame_t*)xlist_get(&host_frames, i);
|
||
if (write(fd, &fr->len, sizeof(fr->len)) != (ssize_t)sizeof(fr->len)) {
|
||
rc = -EIO;
|
||
break;
|
||
}
|
||
if (write(fd, fr->buf, fr->len) != (ssize_t)fr->len) {
|
||
rc = -EIO;
|
||
break;
|
||
}
|
||
}
|
||
|
||
/* Config-bytes section (config-upgrade-snapshot task 2.3, Req 2.1, 2.3,
|
||
* 2.4, 5.1): raw active-configuration bytes, appended LAST. This is a
|
||
* single flat length-prefixed blob for the whole file -- NOT a
|
||
* [count][frame...] shape like the job/queue/host sections above, since
|
||
* there is exactly one active configuration, never a set of them.
|
||
* Populated on EVERY checkpoint (marker-triggered or periodic alike,
|
||
* never gated on which triggered this call) so a member joining or
|
||
* catching up after the marker-triggered upgrade snapshot has rotated
|
||
* away still gets configuration and materialized state together from
|
||
* whichever checkpoint it installs (Req 2.4). A pure function of
|
||
* whatever bytes currently sit at the resolved config path -- no
|
||
* timestamp, no ordering dependence -- so reading the same unchanged
|
||
* file twice yields byte-identical output (Req 5.1 determinism). */
|
||
uint8_t* cfg_bytes = NULL;
|
||
size_t cfg_len = 0;
|
||
if (rc == 0) {
|
||
int cfg_rc = read_active_config_bytes(p, &cfg_bytes, &cfg_len);
|
||
if (cfg_rc == -ENOENT) {
|
||
/* No local config bytes yet -- no cluster/myconf back-pointer
|
||
* (bare unit-test persist_t) or this node has not received its
|
||
* first config push. Mirrors the "no live cluster back-pointer"
|
||
* empty-is-not-failure discipline collect_job/queue/host_frames
|
||
* already apply above: write an explicit zero-length section
|
||
* rather than failing the whole checkpoint over a benign,
|
||
* expected early-boot / not-yet-provisioned state. */
|
||
cfg_bytes = NULL;
|
||
cfg_len = 0;
|
||
} else if (cfg_rc != 0) {
|
||
/* Any other error means genuinely corrupt/unreadable on-disk
|
||
* config state, not benign absence -- fail the checkpoint like
|
||
* any other collect/restore failure (design.md Error Handling:
|
||
* "Host placeholder creation failure ... treated as any other
|
||
* collect/restore failure today -- discard partial state, fail
|
||
* the checkpoint/restore attempt, retriable on the next
|
||
* cycle"). */
|
||
rc = cfg_rc;
|
||
}
|
||
}
|
||
|
||
if (rc == 0) {
|
||
uint32_t cfg_bytes_len = (uint32_t)cfg_len;
|
||
if (write(fd, &cfg_bytes_len, sizeof(cfg_bytes_len)) !=
|
||
(ssize_t)sizeof(cfg_bytes_len))
|
||
rc = -EIO;
|
||
}
|
||
if (rc == 0 && cfg_len > 0 &&
|
||
write(fd, cfg_bytes, cfg_len) != (ssize_t)cfg_len)
|
||
rc = -EIO;
|
||
free(cfg_bytes);
|
||
|
||
if (rc == 0 && fdatasync(fd) != 0)
|
||
rc = -errno;
|
||
|
||
close(fd);
|
||
snap_free_job_frames(&job_frames);
|
||
snap_free_queue_frames(&queue_frames);
|
||
snap_free_host_frames(&host_frames);
|
||
|
||
if (rc != 0) {
|
||
unlink(tmp_path);
|
||
return rc;
|
||
}
|
||
|
||
if (rename(tmp_path, final_path) != 0) {
|
||
int saved = errno;
|
||
unlink(tmp_path);
|
||
return -saved;
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
/* scan_latest_committed_config_ref (ha-term-safety task 4.2 remediation r1):
|
||
* walk the local WAL segments and capture the HIGHEST-LSN WAL_CONFIG_MARKER
|
||
* whose LSN is <= committed_lsn into p->cfgref_{present,tag,sha}. This is the
|
||
* SAME committed-prefix marker config_committed_marker derives, cached durably
|
||
* so a later checkpoint can trim the marker's segment without losing the
|
||
* derived committed-config reference (compaction safety). The marker payload is
|
||
* a fixed 40-byte reference (8-byte big-endian version_tag + 32-byte sha256 —
|
||
* mirrors config_marker_encode; the config BYTES are never in the WAL), decoded
|
||
* inline here so persist stays free of a config_repl layering dependency. */
|
||
static void scan_latest_committed_config_ref(persist_t* p)
|
||
{
|
||
uint64_t committed = atomic_load_explicit(&p->committed_lsn,
|
||
memory_order_acquire);
|
||
if (committed == 0)
|
||
return; /* nothing committed → keep any prior cached ref unchanged */
|
||
|
||
DIR* d = opendir(p->wal_path);
|
||
if (!d)
|
||
return;
|
||
|
||
uint64_t seg_lsns[4096];
|
||
int nseg = 0;
|
||
struct dirent* e;
|
||
while ((e = readdir(d)) != NULL &&
|
||
nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) {
|
||
uint64_t sl = 0;
|
||
if (sscanf(e->d_name, "seg-%016" SCNx64 ".wal", &sl) == 1)
|
||
seg_lsns[nseg++] = sl;
|
||
}
|
||
closedir(d);
|
||
if (nseg == 0)
|
||
return;
|
||
|
||
for (int i = 1; i < nseg; i++) {
|
||
uint64_t key = seg_lsns[i];
|
||
int j = i - 1;
|
||
while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; }
|
||
seg_lsns[j + 1] = key;
|
||
}
|
||
|
||
for (int si = 0; si < nseg; si++) {
|
||
char seg_path[4096];
|
||
snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, p->wal_path,
|
||
seg_lsns[si]);
|
||
int fd = open(seg_path, O_RDONLY);
|
||
if (fd < 0)
|
||
continue;
|
||
|
||
uint32_t off = 0;
|
||
for (;;) {
|
||
wal_entry_hdr_t hdr;
|
||
ssize_t n = pread(fd, &hdr, sizeof(hdr), (off_t)off);
|
||
if (n < (ssize_t)sizeof(hdr)) break;
|
||
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break;
|
||
if (hdr.lsn > committed) break; /* past committed prefix */
|
||
|
||
if (hdr.type == WAL_CONFIG_MARKER &&
|
||
hdr.payload_len >= WAL_CONFIG_MARKER_PAYLOAD_LEN) {
|
||
uint8_t buf[WAL_CONFIG_MARKER_PAYLOAD_LEN];
|
||
ssize_t pr = pread(fd, buf, sizeof(buf),
|
||
(off_t)(off + sizeof(hdr)));
|
||
if (pr == (ssize_t)sizeof(buf)) {
|
||
p->cfgref_tag =
|
||
((uint64_t)buf[0] << 56) | ((uint64_t)buf[1] << 48) |
|
||
((uint64_t)buf[2] << 40) | ((uint64_t)buf[3] << 32) |
|
||
((uint64_t)buf[4] << 24) | ((uint64_t)buf[5] << 16) |
|
||
((uint64_t)buf[6] << 8) | ((uint64_t)buf[7]);
|
||
memcpy(p->cfgref_sha, buf + 8, 32);
|
||
p->cfgref_present = 1; /* later markers overwrite — latest wins */
|
||
}
|
||
}
|
||
off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
}
|
||
close(fd);
|
||
}
|
||
}
|
||
|
||
/* Delete WAL segment files whose start LSN is strictly less than the
|
||
* active segment's start LSN AND strictly less than the trim floor.
|
||
*
|
||
* Bug-1147 (stopgap, raft-drive-async task 3.2): the trim floor is the
|
||
* minimum of the active-segment LSN, every active standby's last acked LSN,
|
||
* and every recently-disconnected slot's last_disconnected_lsn (strategy-A
|
||
* retention discipline). This ensures we NEVER delete a WAL segment that a
|
||
* live standby still needs for catch-up (the bug-795/828 committed-job-loss
|
||
* family). It does NOT fix the empty/far-behind join case (cursor=0) —
|
||
* that requires entity-bearing snapshots (separate spec).
|
||
*
|
||
* Lock ordering (verified 2026-06-26): persist_checkpoint takes p->lock only
|
||
* to snapshot checkpoint_lsn, then RELEASES it before calling
|
||
* trim_wal_segments — so this function runs with NO p->lock held.
|
||
* repl_trim_lsn and repl_expire_disconnect_slots take only r->lock
|
||
* (p->repl.lock); neither ever acquires p->lock or ce->lock, so there is no
|
||
* p->lock↔repl.lock (or ce->lock↔repl.lock) inversion. */
|
||
static void trim_wal_segments(persist_t* p)
|
||
{
|
||
/* Extract the start LSN of the currently-open segment from seg_path.
|
||
* seg_path has the form "<wal_path>/seg-<hex16>.wal".
|
||
* If no segment is open yet (wal_fd < 0), skip trimming. */
|
||
if (p->wal_fd < 0 || p->seg_path[0] == '\0')
|
||
return;
|
||
|
||
const char* seg_basename = strrchr(p->seg_path, '/');
|
||
if (!seg_basename) return;
|
||
seg_basename++; /* skip the '/' */
|
||
|
||
uint64_t active_seg_lsn = 0;
|
||
if (sscanf(seg_basename, "seg-%" SCNx64 ".wal", &active_seg_lsn) != 1)
|
||
return;
|
||
|
||
/* Expire stale disconnected slots so their last_disconnected_lsn guards
|
||
* don't hold WAL segments indefinitely (slot guard expires at
|
||
* last_disconnect_time + snap_interval_s). */
|
||
repl_expire_disconnect_slots(&p->repl, time(NULL),
|
||
p->config.snap_interval_s);
|
||
|
||
/* Compute the trim floor: min(active_seg_lsn, all active standby
|
||
* acked_lsn, all valid disconnected-slot last_disconnected_lsn).
|
||
* Any segment with seg_start_lsn BELOW this floor is disposable. */
|
||
uint64_t floor = repl_trim_lsn(&p->repl, active_seg_lsn,
|
||
p->config.snap_interval_s);
|
||
|
||
/* bug-1426: also cap the floor at the LOCAL apply pump's own progress
|
||
* (last_applied_lsn), not just remote standby-acked positions.
|
||
* repl_trim_lsn's floor only reflects what OTHER members have
|
||
* durably received -- it has no idea whether this member's own live
|
||
* apply pump has actually caught up locally. If the pump is lagging
|
||
* (e.g. mid-recovery, or a slow apply burst) when a trim fires, a
|
||
* segment below the standby-floor but AT OR ABOVE the pump's own
|
||
* still-unread position could be deleted while apply_pump_read_entry's
|
||
* cache holds an open fd into it -- POSIX unlink-keeps-fd-alive lets
|
||
* the pump keep reading (no immediate corruption, no wrong data), but
|
||
* a FRESH read of the same range (a restart's persist_recover() replay,
|
||
* or the pump's own cache falling back to its slow path after a miss)
|
||
* would find the segment already gone -- a genuine, if narrow, apply-
|
||
* correctness gap: what the live process quietly caught up to and what
|
||
* a fresh read of disk state can see would diverge. Mirrors the exact
|
||
* same "floor = min(...)" pattern already used above for the standby/
|
||
* disconnect-guard case -- same imprecision (segment-granularity, not
|
||
* exact-entry-granularity), same established acceptable tradeoff.
|
||
*
|
||
* +1, not the raw value: last_applied_lsn == N means the entry AT lsn N
|
||
* has been fully applied -- unlike repl_trim_lsn's floor (an acked_lsn
|
||
* where equality means "the standby might still need this segment,
|
||
* preserve it" -- see the seg_start_lsn < floor check below, which
|
||
* already treats equality as preserve for that input by construction),
|
||
* the LOCAL apply cursor's equality case means the opposite: this
|
||
* process is DONE with that entry and will never re-read it. Feeding
|
||
* last_applied_lsn+1 into the same shared "< floor" check gets the
|
||
* correct delete-on-equal semantics for this term specifically,
|
||
* without needing a second, differently-shaped comparison. */
|
||
uint64_t apply_floor =
|
||
atomic_load_explicit(&p->last_applied_lsn, memory_order_acquire) + 1;
|
||
if (apply_floor < floor)
|
||
floor = apply_floor;
|
||
|
||
DIR* d = opendir(p->wal_path);
|
||
if (!d) return;
|
||
|
||
struct dirent* e;
|
||
while ((e = readdir(d)) != NULL) {
|
||
if (strncmp(e->d_name, "seg-", 4) != 0) continue;
|
||
uint64_t seg_start_lsn = 0;
|
||
if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start_lsn) != 1) continue;
|
||
/* Only delete segments below BOTH the active segment AND the
|
||
* standby-cursor floor (bug-1147 guard). floor <= active_seg_lsn
|
||
* by construction (repl_trim_lsn caps at checkpoint_lsn=active_seg_lsn),
|
||
* so this check subsumes the old < active_seg_lsn guard. */
|
||
if (seg_start_lsn < floor) {
|
||
char seg_path[4096];
|
||
snprintf(seg_path, sizeof(seg_path), "%s/%s", p->wal_path, e->d_name);
|
||
unlink(seg_path);
|
||
}
|
||
}
|
||
closedir(d);
|
||
}
|
||
|
||
/* Snapshot filename comparison helper for qsort — sorts ascending by name
|
||
* (which sorts ascending by LSN since filenames embed the LSN as hex). */
|
||
static int snap_name_cmp(const void* a, const void* b)
|
||
{
|
||
return strcmp(*(const char* const*)a, *(const char* const*)b);
|
||
}
|
||
|
||
/* Keep only the snap_keep_n most recent snapshot files; delete the rest. */
|
||
/* True when `lsn` is the checkpoint position an in-flight peer snapshot
|
||
* transfer currently references (ha-entity-snapshots task 4.1/4.3, Req
|
||
* 3.1-3.3) — checked directly against snapshot_inflight_lsn[], not gated on
|
||
* snapshot_inflight[] first: the LSN clears before the flag in every
|
||
* ha_snapshot_send_dispatch/snapshot_lane_fn path, so a stale flag=1 can
|
||
* momentarily coexist with an already-cleared LSN=0 (see task 4.1's
|
||
* Implementation Note). A checkpoint's LSN is never 0, so "nonzero and
|
||
* matches" is the correct, race-free protection signal on its own. No
|
||
* cluster (p->cluster == NULL, e.g. standalone lib-level tests) means no HA
|
||
* layer exists to be mid-transfer — nothing is protected. */
|
||
static bool snapshot_lsn_in_flight(persist_t* p, uint64_t lsn)
|
||
{
|
||
if (p->cluster == NULL || lsn == 0)
|
||
return false;
|
||
|
||
for (int i = 0; i < MAX_HA_CANDIDATES; i++) {
|
||
if (atomic_load_explicit(&p->cluster->ha.snapshot_inflight_lsn[i],
|
||
memory_order_acquire) == lsn)
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
static void rotate_snapshots(persist_t* p)
|
||
{
|
||
DIR* d = opendir(p->snap_path);
|
||
if (!d) return;
|
||
|
||
/* Collect all .capnp snapshot filenames */
|
||
char* names[1024];
|
||
int n = 0;
|
||
struct dirent* e;
|
||
while ((e = readdir(d)) != NULL && n < (int)(sizeof(names) / sizeof(names[0]))) {
|
||
size_t len = strlen(e->d_name);
|
||
if (len <= 6 || strcmp(e->d_name + len - 6, ".capnp") != 0) continue;
|
||
names[n] = strdup(e->d_name);
|
||
if (names[n]) n++;
|
||
}
|
||
closedir(d);
|
||
|
||
if (n == 0) return;
|
||
|
||
/* Sort ascending — oldest first */
|
||
qsort(names, (size_t)n, sizeof(names[0]), snap_name_cmp);
|
||
|
||
/* Delete all but the snap_keep_n newest, skipping any file an in-flight
|
||
* peer transfer is still reading from (task 4.3) — it is retried on the
|
||
* next rotation pass once that transfer completes. */
|
||
int keep = (int)p->config.snap_keep_n;
|
||
int delete_up_to = n - keep; /* first index to keep */
|
||
|
||
for (int i = 0; i < delete_up_to && i < n; i++) {
|
||
uint64_t lsn = 0;
|
||
if (sscanf(names[i], "snap-%" SCNx64 ".capnp", &lsn) == 1
|
||
&& snapshot_lsn_in_flight(p, lsn))
|
||
continue;
|
||
|
||
char path[4096];
|
||
snprintf(path, sizeof(path), "%s/%s", p->snap_path, names[i]);
|
||
unlink(path);
|
||
}
|
||
|
||
for (int i = 0; i < n; i++)
|
||
free(names[i]);
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Async checkpoint scheduling (schedule_checkpoint + task fn) */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
/* checkpoint_task_fn: executed by delaymgr worker thread.
|
||
*
|
||
* The CAS in schedule_checkpoint already holds the "in_progress" slot (=1).
|
||
* We must clear it before calling persist_checkpoint(), which has its own
|
||
* atomic_exchange guard. After persist_checkpoint() returns (it resets the
|
||
* flag to 0), we also reset wal_record_count so the counter starts fresh. */
|
||
static int checkpoint_task_fn(void* arg)
|
||
{
|
||
persist_t* persist = (persist_t*)arg;
|
||
|
||
/* Release the "scheduled" slot so persist_checkpoint's guard can acquire */
|
||
atomic_store(&persist->checkpoint_in_progress, 0);
|
||
|
||
persist_checkpoint(persist);
|
||
|
||
/* Reset the record counter so the next checkpoint window starts clean.
|
||
* Done after persist_checkpoint returns — the flag is already 0 by now. */
|
||
atomic_store(&persist->wal_record_count, 0);
|
||
return 0;
|
||
}
|
||
|
||
/* schedule_checkpoint: idempotent async checkpoint request via delaymgr.
|
||
* Safe to call from any thread. Uses atomic CAS to ensure at most one
|
||
* checkpoint is scheduled or running at a time.
|
||
*
|
||
* If cluster is NULL (unit tests without a live delaymgr) the function
|
||
* falls back to a synchronous checkpoint. */
|
||
static void schedule_checkpoint(persist_t* persist)
|
||
{
|
||
int expected = 0;
|
||
if (!atomic_compare_exchange_strong(&persist->checkpoint_in_progress,
|
||
&expected, 1))
|
||
return; /* already scheduled or running — idempotency guard */
|
||
|
||
if (persist->cluster == NULL) {
|
||
/* Unit-test / no-cluster path: run synchronously and reset. */
|
||
atomic_store(&persist->checkpoint_in_progress, 0);
|
||
persist_checkpoint(persist);
|
||
atomic_store(&persist->wal_record_count, 0);
|
||
return;
|
||
}
|
||
|
||
/* Schedule via delaymgr with 1 s coalescing delay, single execution */
|
||
delay_task(&persist->cluster->delaymgr, 1, 1,
|
||
checkpoint_task_fn, persist);
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Checkpoint core (shared by persist_checkpoint and the marker-apply */
|
||
/* trigger, config-upgrade-snapshot task 5) */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
/* persist_checkpoint_write_locked: pin checkpoint_lsn to the EXPLICIT
|
||
* `checkpoint_lsn` given (never re-derived internally from
|
||
* last_applied_lsn), capture the committed-config reference, and serialize
|
||
* live state to a snapshot file.
|
||
*
|
||
* Caller MUST already hold apply_pump_lock (or otherwise guarantee no
|
||
* concurrent apply-pump walk can be racing this call, e.g. the
|
||
* single-threaded boot-replay window persist_recover() already documents
|
||
* elsewhere in this file) -- this function never locks or unlocks
|
||
* apply_pump_lock itself, so it is safe to call both from a context that
|
||
* already holds it (the live apply pump) and from a context where no lock
|
||
* is needed at all (boot replay).
|
||
*
|
||
* The explicit-lsn parameter exists because the two callers need two
|
||
* different values at two different moments:
|
||
* - persist_checkpoint() (periodic/explicit trigger): last_applied_lsn,
|
||
* read fresh by the caller immediately before this call.
|
||
* - the marker-apply trigger (apply_committed_entry's WAL_CONFIG_MARKER
|
||
* case): the marker's OWN lsn -- last_applied_lsn is NOT yet advanced
|
||
* to the marker's position at the moment that case runs (both live
|
||
* apply and replay advance it strictly AFTER dispatching this call),
|
||
* so reading last_applied_lsn from inside that case would mislabel the
|
||
* checkpoint with the marker's PREDECESSOR's lsn, not its own. */
|
||
static int persist_checkpoint_write_locked(persist_t* p,
|
||
uint64_t checkpoint_lsn)
|
||
{
|
||
p->checkpoint_lsn = checkpoint_lsn;
|
||
|
||
/* Capture the latest committed config-marker reference BEFORE trimming
|
||
* (ha-term-safety task 4.2 r1): trim_wal_segments (run by both callers,
|
||
* after this function returns) can unlink the segment holding a
|
||
* committed WAL_CONFIG_MARKER, so the derived committed config must be
|
||
* cached durably (into the sentinel) before that happens. */
|
||
scan_latest_committed_config_ref(p);
|
||
|
||
/* Serialize in-memory state to a snapshot file (P0: no live kvmgrs —
|
||
* writes sentinel header + cached config ref). */
|
||
return write_snapshot(p);
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Public: persist_checkpoint */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
int persist_checkpoint(persist_t* p)
|
||
{
|
||
/* Guard against concurrent checkpoints */
|
||
if (atomic_exchange(&p->checkpoint_in_progress, 1) != 0)
|
||
return -EBUSY;
|
||
|
||
int rc = 0;
|
||
|
||
/* 1. Write WAL_CHECKPOINT marker (local only — never replicated) */
|
||
rc = persist_write_checkpoint_marker(p);
|
||
if (rc != 0)
|
||
goto done;
|
||
|
||
/* 2. Record checkpoint_lsn = the APPLIED cursor (last_applied_lsn), not
|
||
* the physical WAL append tip (current_lsn). write_snapshot() below
|
||
* serializes live jobmgr state, which only reflects entries the
|
||
* apply pump has actually dispatched -- NOT every entry durably
|
||
* appended/committed, since apply is decoupled from commit
|
||
* (raft-apply-decoupling) and can lag behind current_lsn under load.
|
||
* Stamping current_lsn here mislabels the snapshot: on restart,
|
||
* last_applied_lsn is re-seeded FROM checkpoint_lsn and replay
|
||
* resumes strictly above it, so any committed-but-not-yet-applied
|
||
* entries between the true apply cursor and current_lsn at
|
||
* checkpoint time are silently skipped forever, even though jobmgr
|
||
* never actually contained them (persist.h last_applied_lsn doc).
|
||
* Held under apply_pump_lock together with 3a/3b so the label and
|
||
* the jobmgr walk it describes are captured atomically w.r.t. the
|
||
* live apply pump -- the pump can only ever move last_applied_lsn
|
||
* and jobmgr forward, never backward, so freezing it here rules out
|
||
* the label ever claiming more than jobmgr actually holds. */
|
||
pthread_mutex_lock(&p->apply_pump_lock);
|
||
|
||
rc = persist_checkpoint_write_locked(
|
||
p, atomic_load_explicit(&p->last_applied_lsn, memory_order_acquire));
|
||
|
||
pthread_mutex_unlock(&p->apply_pump_lock);
|
||
|
||
if (rc != 0)
|
||
goto done;
|
||
|
||
/* 4. Trim old WAL segments before the checkpoint */
|
||
trim_wal_segments(p);
|
||
|
||
/* 5. Rotate old snapshots — keep only snap_keep_n most recent */
|
||
rotate_snapshots(p);
|
||
|
||
done:
|
||
atomic_store(&p->checkpoint_in_progress, 0);
|
||
return rc;
|
||
}
|
||
|
||
/* persist_checkpoint_marker_triggered (config-upgrade-snapshot task 5, Req
|
||
* 1.1, 1.5, 1.8, 4.4, 5.2, 5.3): force an immediate checkpoint at exactly
|
||
* `marker_lsn` -- the configuration-change marker's OWN lsn -- called from
|
||
* apply_committed_entry's WAL_CONFIG_MARKER case.
|
||
*
|
||
* Unlike persist_checkpoint(), this function NEVER locks apply_pump_lock:
|
||
* it relies entirely on the caller's own locking context, because the two
|
||
* production call sites of apply_committed_entry() differ in that context
|
||
* and this function must be correct under BOTH without deadlocking under
|
||
* either:
|
||
* - persist_apply_pump_run() (live online-apply, both primary and
|
||
* standby route through this) holds apply_pump_lock across its ENTIRE
|
||
* per-LSN dispatch loop, including the call into apply_committed_entry
|
||
* that reaches this function. Re-locking that same, non-recursive
|
||
* mutex here would self-deadlock -- so this function must NOT lock it;
|
||
* it is already held by the caller for the whole duration of this call.
|
||
* - replay_wal() (boot/restart replay, persist_recover()'s sole caller)
|
||
* does NOT hold apply_pump_lock during its per-entry dispatch loop --
|
||
* by design, since persist_recover() runs before the apply pump is
|
||
* ever kicked and nothing else can be racing it (see persist_recover's
|
||
* own doc comment on its brief, unrelated apply_pump_lock use for
|
||
* cache invalidation). No lock is needed here either, for the same
|
||
* reason.
|
||
* Both call sites are internal to persist.c and fully controlled, so this
|
||
* documented precondition (caller already serializes, one way or another)
|
||
* is safe to rely on rather than adding a redundant/ambiguous lock here.
|
||
*
|
||
* Shares checkpoint_in_progress -- the SAME single-in-flight CAS guard
|
||
* persist_checkpoint() uses -- so a concurrent WAL-threshold-triggered
|
||
* periodic checkpoint and this marker-triggered one can never both
|
||
* proceed at once (Req 5.3). Returns -EBUSY when that guard is already
|
||
* held by someone else; the caller (apply_committed_entry) treats that as
|
||
* an expected, self-resolving condition, not a failure (Req 4.4) -- the
|
||
* in-flight checkpoint (whichever triggered it) still completes and
|
||
* either it or a later checkpoint picks up this marker's configuration
|
||
* (every checkpoint embeds the currently active configuration, task 2.3). */
|
||
static int persist_checkpoint_marker_triggered(persist_t* p,
|
||
uint64_t marker_lsn)
|
||
{
|
||
if (atomic_exchange(&p->checkpoint_in_progress, 1) != 0)
|
||
return -EBUSY;
|
||
|
||
int rc = persist_write_checkpoint_marker(p);
|
||
if (rc == 0)
|
||
rc = persist_checkpoint_write_locked(p, marker_lsn);
|
||
|
||
if (rc == 0) {
|
||
trim_wal_segments(p);
|
||
rotate_snapshots(p);
|
||
}
|
||
|
||
atomic_store(&p->checkpoint_in_progress, 0);
|
||
return rc;
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Recovery internals */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
/* Parse and restore every job frame, then every queue frame, then every host
|
||
* frame from the entity payload beginning at the current `fd` offset
|
||
* (design.md "Checkpoint File Layout", SNAPSHOT_FORMAT_VERSION=2):
|
||
* [job_count][job_count * {payload_len; capnp Job bytes}]
|
||
* [queue_count][queue_count * {payload_len; capnp QueueAdminState bytes}]
|
||
* [host_count][host_count * {payload_len; capnp HostAdminState bytes}]
|
||
* A trailing config-bytes section follows the host section in the file
|
||
* (config-upgrade-snapshot task 2.3) but is deliberately NOT read here --
|
||
* out of this function's scope; any bytes after the host section are simply
|
||
* left unread, exactly as host-frame bytes were already tolerated as
|
||
* present-but-unread before this function grew a host section of its own.
|
||
*
|
||
* `file_size` is the checkpoint file's total size (from fstat) and
|
||
* `consumed` is the number of bytes already read from it (legacy header +
|
||
* format_version) — every length-prefixed frame's length is bounds-checked
|
||
* against the ACTUAL remaining bytes in the file before being trusted for a
|
||
* read or allocation, never blindly followed (a corrupt/attacker-controlled
|
||
* length must fail fast, not drive an oversized read).
|
||
*
|
||
* Restores in file order: every job via restore_job, then every queue via
|
||
* restore_queue_state, then every host via restore_host_state
|
||
* (config-upgrade-snapshot task 4, Req 1.10 — design.md sequence diagram
|
||
* and "Referential integrity" note: restore order does not matter
|
||
* functionally, since re-homing decisions are already baked into each
|
||
* entity's encoded content at materialization time, never re-derived at
|
||
* restore time, Design Invariant 2). Returns 0 only if every frame
|
||
* restored; on the FIRST failure of any kind (short read, oversized
|
||
* length, decode error, or restore error) returns immediately with a
|
||
* negative errno — the caller MUST NOT commit any position change on a
|
||
* non-zero return (Requirement 2.3).
|
||
*
|
||
* restore_job's -EEXIST is treated as success, not a failure: it is job.c's
|
||
* own documented idempotent-replay guard (job already present), needed so a
|
||
* retried load — after an earlier partial attempt already created some jobs
|
||
* before failing on a later one — can continue past already-restored entries
|
||
* rather than aborting on them again. restore_queue_state and
|
||
* restore_host_state have no such idempotent-duplicate case of their own to
|
||
* special-case here (re-applying the same status is already a harmless
|
||
* overwrite/OR, and restore_host_state's create-if-missing placeholder path
|
||
* is independently idempotent under retry — Requirement 5.8, worker.c), so
|
||
* any non-zero return from either is a genuine failure. */
|
||
static int load_snapshot_entities(int fd, off_t file_size, off_t consumed,
|
||
persist_t* p, uint64_t stored_lsn)
|
||
{
|
||
uint32_t job_count = 0;
|
||
if (read(fd, &job_count, sizeof(job_count)) != (ssize_t)sizeof(job_count))
|
||
return -EIO;
|
||
consumed += (off_t)sizeof(job_count);
|
||
|
||
if (job_count > 0 && (p->cluster == NULL || p->cluster->jobmgr == NULL))
|
||
return -EINVAL; /* entities present but nowhere to restore them into */
|
||
|
||
for (uint32_t i = 0; i < job_count; i++) {
|
||
uint32_t payload_len = 0;
|
||
if (read(fd, &payload_len, sizeof(payload_len)) !=
|
||
(ssize_t)sizeof(payload_len))
|
||
return -EIO;
|
||
consumed += (off_t)sizeof(payload_len);
|
||
|
||
if (payload_len == 0 || (off_t)payload_len > file_size - consumed)
|
||
return -EIO;
|
||
|
||
uint8_t* buf = (uint8_t*)malloc(payload_len);
|
||
if (buf == NULL)
|
||
return -ENOMEM;
|
||
if (read(fd, buf, payload_len) != (ssize_t)payload_len) {
|
||
free(buf);
|
||
return -EIO;
|
||
}
|
||
consumed += (off_t)payload_len;
|
||
|
||
int rrc = restore_job(p->cluster, 0, buf, payload_len, stored_lsn);
|
||
free(buf);
|
||
if (rrc != 0 && rrc != -EEXIST)
|
||
return rrc;
|
||
}
|
||
|
||
uint32_t queue_count = 0;
|
||
if (read(fd, &queue_count, sizeof(queue_count)) !=
|
||
(ssize_t)sizeof(queue_count))
|
||
return -EIO;
|
||
consumed += (off_t)sizeof(queue_count);
|
||
|
||
if (queue_count > 0 && (p->cluster == NULL || p->cluster->qmgr == NULL))
|
||
return -EINVAL; /* entities present but nowhere to restore them into */
|
||
|
||
for (uint32_t i = 0; i < queue_count; i++) {
|
||
uint32_t payload_len = 0;
|
||
if (read(fd, &payload_len, sizeof(payload_len)) !=
|
||
(ssize_t)sizeof(payload_len))
|
||
return -EIO;
|
||
consumed += (off_t)sizeof(payload_len);
|
||
|
||
if (payload_len == 0 || (off_t)payload_len > file_size - consumed)
|
||
return -EIO;
|
||
|
||
uint8_t* buf = (uint8_t*)malloc(payload_len);
|
||
if (buf == NULL)
|
||
return -ENOMEM;
|
||
if (read(fd, buf, payload_len) != (ssize_t)payload_len) {
|
||
free(buf);
|
||
return -EIO;
|
||
}
|
||
consumed += (off_t)payload_len;
|
||
|
||
/* Decode the capnp QueueAdminState frame (config-upgrade-snapshot
|
||
* task 4, Req 1.9/5.7 decode side — migrated off the prior raw
|
||
* {name_len, name, status} layout collect_queue_frames stopped
|
||
* writing in task 2.2). Mirrors the WAL_JOB_SUBMIT re-decode block
|
||
* above in this file (capn_init_mem/capn_getp/read_*) and
|
||
* test_persist_snapshot_entities.c's own
|
||
* snapent_read_checkpoint_queues reader exactly. */
|
||
struct capn c_ctx;
|
||
capn_init_mem(&c_ctx, buf, payload_len, 0);
|
||
QueueAdminState_ptr qp;
|
||
qp.p = capn_getp(capn_root(&c_ctx), 0, 1);
|
||
struct QueueAdminState cq;
|
||
memset(&cq, 0, sizeof(cq));
|
||
read_QueueAdminState(&cq, qp);
|
||
|
||
capnp_queue_admin_state_t decoded;
|
||
memset(&decoded, 0, sizeof(decoded));
|
||
decode_QueueAdminState(&decoded, &cq);
|
||
capn_free(&c_ctx);
|
||
free(buf);
|
||
|
||
int qrc = restore_queue_state(p->cluster, decoded.name, decoded.status);
|
||
free_QueueAdminState(&decoded);
|
||
if (qrc != 0)
|
||
return qrc;
|
||
}
|
||
|
||
/* Host section (config-upgrade-snapshot task 4, Req 1.10): new in
|
||
* SNAPSHOT_FORMAT_VERSION=2, added after the queue section (design.md
|
||
* "Checkpoint File Layout"). Same [payload_len][capnp bytes] per-frame
|
||
* shape as the job/queue sections above. Every host frame dispatches
|
||
* to restore_host_state (worker.c, task 3), which is create-if-missing
|
||
* (a worker_t is join-provisioned, not config-provisioned like a queue
|
||
* — Design Invariant 1) and idempotent under retry on its own
|
||
* (Requirement 5.8), so this loop applies the exact same "any failure
|
||
* returns immediately" discipline as the job/queue loops above without
|
||
* needing any extra idempotency handling here. */
|
||
uint32_t host_count = 0;
|
||
if (read(fd, &host_count, sizeof(host_count)) !=
|
||
(ssize_t)sizeof(host_count))
|
||
return -EIO;
|
||
consumed += (off_t)sizeof(host_count);
|
||
|
||
if (host_count > 0 &&
|
||
(p->cluster == NULL || p->cluster->workermgr == NULL))
|
||
return -EINVAL; /* entities present but nowhere to restore them into */
|
||
|
||
for (uint32_t i = 0; i < host_count; i++) {
|
||
uint32_t payload_len = 0;
|
||
if (read(fd, &payload_len, sizeof(payload_len)) !=
|
||
(ssize_t)sizeof(payload_len))
|
||
return -EIO;
|
||
consumed += (off_t)sizeof(payload_len);
|
||
|
||
if (payload_len == 0 || (off_t)payload_len > file_size - consumed)
|
||
return -EIO;
|
||
|
||
uint8_t* buf = (uint8_t*)malloc(payload_len);
|
||
if (buf == NULL)
|
||
return -ENOMEM;
|
||
if (read(fd, buf, payload_len) != (ssize_t)payload_len) {
|
||
free(buf);
|
||
return -EIO;
|
||
}
|
||
consumed += (off_t)payload_len;
|
||
|
||
/* Decode the capnp HostAdminState frame -- mirrors the queue decode
|
||
* block immediately above and
|
||
* test_persist_snapshot_entities.c's snapent_read_checkpoint_hosts
|
||
* reader exactly. */
|
||
struct capn c_ctx;
|
||
capn_init_mem(&c_ctx, buf, payload_len, 0);
|
||
HostAdminState_ptr hp;
|
||
hp.p = capn_getp(capn_root(&c_ctx), 0, 1);
|
||
struct HostAdminState ch;
|
||
memset(&ch, 0, sizeof(ch));
|
||
read_HostAdminState(&ch, hp);
|
||
|
||
capnp_host_admin_state_t decoded;
|
||
memset(&decoded, 0, sizeof(decoded));
|
||
decode_HostAdminState(&decoded, &ch);
|
||
capn_free(&c_ctx);
|
||
free(buf);
|
||
|
||
int hrc = restore_host_state(p->cluster, decoded.name, decoded.status);
|
||
free_HostAdminState(&decoded);
|
||
if (hrc != 0)
|
||
return hrc;
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
/* Load a snapshot: legacy header (magic + checkpoint_lsn + optional
|
||
* config-ref tail), then — for an entity-bearing checkpoint — every job,
|
||
* queue, and host frame that follows it (ha-entity-snapshots task 5.2,
|
||
* config-upgrade-snapshot task 4; Requirements 2.1-2.4, 5.1, 5.2, 1.10).
|
||
* See design.md "load_snapshot (extended)" and its sequence diagram for the
|
||
* exact contract this implements.
|
||
*
|
||
* Ordering (Requirement 2.1, 2.2, 2.3 — restore FIRST, validate FULLY, THEN
|
||
* advance position, never the reverse, never partial): the header and any
|
||
* config-ref tail are parsed into LOCAL variables only. Format is then
|
||
* detected by whether any bytes remain past the header (design.md's stated
|
||
* discriminator — not a flag bit). A short file (nothing remains) is a
|
||
* legacy, pre-feature checkpoint: not an error, so its recorded position is
|
||
* committed immediately and *out_entities_restored is false. Otherwise every
|
||
* job, then every queue, then every host frame is restored via
|
||
* load_snapshot_entities() before ANYTHING is written to
|
||
* p->checkpoint_lsn/p->current_lsn/p->cfgref_*; on any restore failure this
|
||
* function returns non-zero with p's fields PROVABLY untouched (the single
|
||
* commit point below is only ever reached after load_snapshot_entities()
|
||
* returns 0). *out_entities_restored is only set on a rc==0 return, matching
|
||
* persist.h's documented contract.
|
||
*
|
||
* Returns 0 on success, -EINVAL on bad magic / unrecognized entity format,
|
||
* -EIO on a short/corrupt read, -ENOMEM on allocation failure, -errno on
|
||
* open/fstat failure, or a restore_job/restore_queue_state/restore_host_state
|
||
* error code. */
|
||
static int load_snapshot(persist_t* p, const char* path, uint64_t lsn,
|
||
bool* out_entities_restored)
|
||
{
|
||
(void)lsn; /* lsn is encoded in the file itself — we re-read it */
|
||
|
||
int fd = open(path, O_RDONLY);
|
||
if (fd < 0)
|
||
return -errno;
|
||
|
||
struct stat st;
|
||
if (fstat(fd, &st) != 0) {
|
||
int saved = errno;
|
||
close(fd);
|
||
return -saved;
|
||
}
|
||
|
||
/* magic(8) + checkpoint_lsn(8) + optional config-ref tail(1+8+32). */
|
||
char header[16 + 1 + 8 + 32];
|
||
ssize_t n = read(fd, header, sizeof(header));
|
||
|
||
if (n < 16) {
|
||
close(fd);
|
||
return -EIO;
|
||
}
|
||
|
||
if (memcmp(header, "XCESNAP\0", 8) != 0) {
|
||
close(fd);
|
||
return -EINVAL;
|
||
}
|
||
|
||
/* Extract checkpoint_lsn from bytes 8-15 (little-endian on this arch).
|
||
* Held LOCALLY -- not yet written to p-> -- until the single commit
|
||
* point below (legacy short-circuit, or after every entity restores). */
|
||
uint64_t stored_lsn;
|
||
memcpy(&stored_lsn, header + 8, 8);
|
||
|
||
/* Optional committed-config reference (ha-term-safety task 4.2 r1):
|
||
* present only when the sentinel carries the 41-byte tail. Also held
|
||
* locally until the same commit point. */
|
||
int cfgref_present = 0;
|
||
uint64_t cfgref_tag = 0;
|
||
uint8_t cfgref_sha[32];
|
||
memset(cfgref_sha, 0, sizeof(cfgref_sha));
|
||
if (n >= 16 + 1 + 8 + 32 && header[16] != 0) {
|
||
cfgref_present = 1;
|
||
memcpy(&cfgref_tag, header + 17, 8);
|
||
memcpy(cfgref_sha, header + 25, 32);
|
||
}
|
||
|
||
off_t consumed = (off_t)n;
|
||
|
||
/* Legacy-format detection (Requirement 2.4, 5.1, 5.2): "does more data
|
||
* exist" is the discriminator (design.md), not a flag bit -- a file
|
||
* ending exactly at the legacy header is unambiguously a pre-feature
|
||
* checkpoint. This is an expected rollout shape, not corruption: commit
|
||
* the position it records (existing, preserved legacy behavior) and
|
||
* report entities_restored=false so the caller falls back to normal
|
||
* catch-up rather than treating this checkpoint as fully complete. */
|
||
if (st.st_size <= consumed) {
|
||
close(fd);
|
||
p->checkpoint_lsn = stored_lsn;
|
||
p->current_lsn = stored_lsn;
|
||
if (cfgref_present) {
|
||
p->cfgref_present = 1;
|
||
p->cfgref_tag = cfgref_tag;
|
||
memcpy(p->cfgref_sha, cfgref_sha, sizeof(p->cfgref_sha));
|
||
}
|
||
if (out_entities_restored)
|
||
*out_entities_restored = false;
|
||
return 0;
|
||
}
|
||
|
||
/* Entity-bearing checkpoint: format_version, then job frames, then queue
|
||
* frames (design.md "Checkpoint File Layout"). */
|
||
uint32_t fmt = 0;
|
||
if (read(fd, &fmt, sizeof(fmt)) != (ssize_t)sizeof(fmt)) {
|
||
close(fd);
|
||
return -EIO;
|
||
}
|
||
consumed += (off_t)sizeof(fmt);
|
||
|
||
/* An unrecognized format_version is corruption or a future layout this
|
||
* binary cannot read -- fail loud rather than silently treating it as
|
||
* legacy (task 5.1 review note: a version mismatch must never be masked
|
||
* as an expected legacy shape, or real corruption goes undetected). */
|
||
if (fmt != SNAPSHOT_FORMAT_VERSION) {
|
||
close(fd);
|
||
return -EINVAL;
|
||
}
|
||
|
||
int rc = load_snapshot_entities(fd, st.st_size, consumed, p, stored_lsn);
|
||
close(fd);
|
||
if (rc != 0)
|
||
return rc; /* Requirement 2.3: p->*_lsn / p->cfgref_* untouched */
|
||
|
||
/* Single commit point (Requirement 2.1, 2.2): reached ONLY after every
|
||
* job and queue frame restored successfully above. */
|
||
p->checkpoint_lsn = stored_lsn;
|
||
p->current_lsn = stored_lsn;
|
||
if (cfgref_present) {
|
||
p->cfgref_present = 1;
|
||
p->cfgref_tag = cfgref_tag;
|
||
memcpy(p->cfgref_sha, cfgref_sha, sizeof(p->cfgref_sha));
|
||
}
|
||
if (out_entities_restored)
|
||
*out_entities_restored = true;
|
||
return 0;
|
||
}
|
||
|
||
/* Public wrapper for load_snapshot — exposed via persist.h for ha.c (Task
|
||
* 3.5c). load_snapshot() above now owns the full restore-before-advance
|
||
* implementation (ha-entity-snapshots task 5.2): this wrapper is a thin
|
||
* pass-through, since load_snapshot() already gates *out_entities_restored
|
||
* and the p->checkpoint_lsn/current_lsn commit on entity-restore success
|
||
* (Requirements 2.1-2.4, 5.1, 5.2). */
|
||
int persist_load_snapshot(persist_t *p, const char *path, uint64_t lsn,
|
||
bool *out_entities_restored)
|
||
{
|
||
return load_snapshot(p, path, lsn, out_entities_restored);
|
||
}
|
||
|
||
/* P2-2: Standby WAL replay — apply a single replicated entry to in-memory state.
|
||
*
|
||
* No hooks fire on standbys; the hook system is empty until promotion.
|
||
*
|
||
* Returns 0 on success (including skip for checkpoint / unknown types),
|
||
* negative errno on error (NULL args, oversized payload). */
|
||
int apply_wal_entry(persist_t* p, uint32_t type, uint64_t lsn,
|
||
const void* payload, uint32_t payload_len)
|
||
{
|
||
if (!p)
|
||
return -EINVAL;
|
||
|
||
/* Reject oversized payloads BEFORE any allocation */
|
||
if (payload_len > MAX_WAL_PAYLOAD_BYTES) {
|
||
ERROR("apply_wal_entry: payload_len %u exceeds MAX_WAL_PAYLOAD_BYTES %u",
|
||
payload_len, MAX_WAL_PAYLOAD_BYTES);
|
||
return -EMSGSIZE;
|
||
}
|
||
|
||
/* LOGICAL entity restore ONLY. The on-disk WAL header, the CRC chain
|
||
* (p->last_crc), and current_lsn are owned exclusively by the physical
|
||
* layer — persist_wal_write_replicated on the live replicated path, and
|
||
* replay_wal/persist_recover on recovery. This function MUST NOT touch
|
||
* them. (bug-778: it previously set p->last_crc from a synthetic crc=0
|
||
* header, breaking the on-disk CRC chain and cold-restart recovery.) */
|
||
|
||
/* WAL_CHECKPOINT (100) is a local-only marker — standbys never receive it
|
||
* via replication. Skip silently. */
|
||
if (type == WAL_CHECKPOINT)
|
||
return 0;
|
||
|
||
/* Unknown types above WAL_ENTRY_TYPE_MAX: log WARN and skip for forward
|
||
* compatibility — an older controller can boot on WAL written by a newer
|
||
* one without hard-failing on unrecognised entry types. */
|
||
if (type > WAL_ENTRY_TYPE_MAX) {
|
||
WARN("apply_wal_entry: unknown type %u > WAL_ENTRY_TYPE_MAX %u"
|
||
" (lsn=%" PRIu64 ") — skipping for forward compatibility",
|
||
type, WAL_ENTRY_TYPE_MAX, lsn);
|
||
return 0;
|
||
}
|
||
|
||
/* Advance the current_lsn high-water (monotonic, idempotent). This is a
|
||
* write-cursor high-water that is safe to bump from multiple paths. The
|
||
* CRC chain link (p->last_crc) is deliberately NOT touched here — it has a
|
||
* single owner: the physical writer (persist_wal_write_replicated) and
|
||
* replay (persist_recover). (bug-778) */
|
||
pthread_mutex_lock(&p->lock);
|
||
if (lsn + 1 > p->current_lsn)
|
||
p->current_lsn = lsn + 1;
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
if (!p->cluster)
|
||
return 0; /* unit tests without a full cluster — skip entity restore */
|
||
|
||
switch (type) {
|
||
case WAL_JOB_SUBMIT:
|
||
/* Payload: Cap'n Proto serialized xce_job_t. Thread the entry LSN
|
||
* through so the restored job is tagged for the egress watermark
|
||
* (ha-convergence-recovery 3.3). */
|
||
restore_job(p->cluster, 0, payload, payload_len, lsn);
|
||
break;
|
||
|
||
case WAL_JOB_ARRAY_SUBMIT:
|
||
/* job-array-single-wal-record task 3.3 (design.md "Array Apply
|
||
* Expansion"): the array-record analog of WAL_JOB_SUBMIT's
|
||
* restore_job() call directly above, dispatched from this SEPARATE,
|
||
* still-live standby catch-up / replicated-apply seam so it can
|
||
* never diverge from apply_committed_entry()'s WAL_JOB_ARRAY_SUBMIT
|
||
* case (Req 3.1, 3.2, 4.1). Mirrors the single-job case's exact
|
||
* minimalism: no xlist_t, no hook fire — this dispatcher is
|
||
* deliberately hook-free (standby catch-up does not notify the
|
||
* local scheduler/API server; that is the primary's job, done via
|
||
* apply_committed_entry()). out_new_jobs is not needed here, so
|
||
* NULL is passed (restore_job_array()'s out_new_jobs parameter is
|
||
* optional — see test_rja_out_new_jobs_is_optional,
|
||
* test_job_array_apply.c). The return value is intentionally
|
||
* discarded, matching every other case in this switch today (task
|
||
* 4.1 — Live Apply Resource Fence — is what later makes this
|
||
* function's callers respect a negative rc; not this task's
|
||
* boundary). */
|
||
restore_job_array(p->cluster, payload, payload_len, lsn, NULL);
|
||
break;
|
||
|
||
case WAL_JOB_FAIL:
|
||
case WAL_JOB_STOP:
|
||
case WAL_JOB_RESUME:
|
||
case WAL_JOB_UPDATE:
|
||
case WAL_JOB_CKPT:
|
||
case WAL_JOB_PREASON:
|
||
/* Payload: wal_jobstate_t {jobid, state} */
|
||
if (payload_len >= sizeof(wal_jobstate_t)) {
|
||
const wal_jobstate_t* js = (const wal_jobstate_t*)payload;
|
||
restore_job_state(p->cluster, js->jobid, js->state);
|
||
}
|
||
break;
|
||
|
||
case WAL_JOB_RUSAGE: {
|
||
/* bug-1690: split out of the shared group above -- mirrors
|
||
* WAL_JOB_START's/WAL_JOB_FINISH's splits just below (bug-1687/
|
||
* bug-1689) and WAL_JOB_DISPATCH's split further below (bug-1678).
|
||
* This is the one fact type in that group whose payload can carry
|
||
* trailing Cap'n Proto JobRunusage bytes (persist_job_hook_fun's
|
||
* WAL_JOB_RUSAGE payload shape, appended only when job.run_usage
|
||
* was non-NULL at write time) alongside the {jobid, state} header,
|
||
* so a promoted standby's catch-up replay recovers the
|
||
* client-reported per-host CPU/mem usage too, not just state. A
|
||
* payload of exactly sizeof(wal_jobstate_t) (no trailing bytes --
|
||
* any WAL_JOB_RUSAGE entry written before this fix, or one fired
|
||
* while job.run_usage was still NULL) decodes no run_usage,
|
||
* matching the pre-fix state-only behavior exactly. */
|
||
if (payload_len >= sizeof(wal_jobstate_t)) {
|
||
const wal_jobstate_t* js = (const wal_jobstate_t*)payload;
|
||
xce_job_runusage_t* runusage = NULL;
|
||
if (payload_len > sizeof(wal_jobstate_t)) {
|
||
runusage = decode_jobrunusage(
|
||
(const uint8_t*)payload + sizeof(wal_jobstate_t),
|
||
payload_len - sizeof(wal_jobstate_t));
|
||
}
|
||
restore_job_rusage_state(p->cluster, js->jobid, js->state,
|
||
runusage);
|
||
}
|
||
break;
|
||
}
|
||
|
||
case WAL_JOB_START: {
|
||
/* bug-1687: split out of the shared group above -- mirrors
|
||
* WAL_JOB_DISPATCH's split just below (bug-1678). This is the one
|
||
* fact type in that group whose payload can carry a fixed
|
||
* start_time PLUS trailing Cap'n Proto JobPIDInfo bytes
|
||
* (persist_submit_job_start_fact's shape) alongside the {jobid,
|
||
* state} header, so a promoted standby's catch-up replay recovers
|
||
* the worker-confirmed pids/pgids/exdata and the true start_time
|
||
* too, not just state. A payload of exactly sizeof(wal_jobstate_t)
|
||
* (no trailing bytes -- any WAL_JOB_START entry written before this
|
||
* fix) decodes neither, matching the pre-fix state-only behavior
|
||
* exactly. */
|
||
if (payload_len >= sizeof(wal_jobstate_t)) {
|
||
const wal_jobstate_t* js = (const wal_jobstate_t*)payload;
|
||
uint64_t start_time = 0;
|
||
job_pidinfo_t* pidinfo = NULL;
|
||
uint32_t extra = payload_len - (uint32_t)sizeof(wal_jobstate_t);
|
||
|
||
if (extra >= sizeof(start_time)) {
|
||
memcpy(&start_time,
|
||
(const uint8_t*)payload + sizeof(wal_jobstate_t),
|
||
sizeof(start_time));
|
||
|
||
uint32_t pidinfo_len = extra - (uint32_t)sizeof(start_time);
|
||
if (pidinfo_len > 0) {
|
||
pidinfo = decode_pidinfo(
|
||
(const uint8_t*)payload + sizeof(wal_jobstate_t)
|
||
+ sizeof(start_time),
|
||
pidinfo_len);
|
||
}
|
||
}
|
||
restore_job_start_state(p->cluster, js->jobid, js->state,
|
||
start_time, pidinfo);
|
||
}
|
||
break;
|
||
}
|
||
|
||
case WAL_JOB_FINISH: {
|
||
/* bug-1689: split out of the shared group above -- mirrors
|
||
* WAL_JOB_START's split just above (bug-1687) / WAL_JOB_DISPATCH's
|
||
* split just below (bug-1678). This is the one fact type in that
|
||
* group whose payload can carry a fixed finish_time PLUS a fixed
|
||
* exit_code (persist_submit_job_finish_fact's shape) alongside the
|
||
* {jobid, state} header, so a promoted standby's catch-up replay
|
||
* recovers the worker-confirmed exit_code and the true finish_time
|
||
* too, not just state. A payload of exactly sizeof(wal_jobstate_t)
|
||
* (no trailing bytes -- any WAL_JOB_FINISH entry written before this
|
||
* fix) decodes neither, matching the pre-fix state-only behavior
|
||
* exactly (finish_time/run_time/exit_info all left untouched on
|
||
* this seam). */
|
||
if (payload_len >= sizeof(wal_jobstate_t)) {
|
||
const wal_jobstate_t* js = (const wal_jobstate_t*)payload;
|
||
uint64_t finish_time = 0;
|
||
job_exitinfo_t* exit_info = NULL;
|
||
uint32_t extra = payload_len - (uint32_t)sizeof(wal_jobstate_t);
|
||
|
||
if (extra >= sizeof(finish_time)) {
|
||
memcpy(&finish_time,
|
||
(const uint8_t*)payload + sizeof(wal_jobstate_t),
|
||
sizeof(finish_time));
|
||
|
||
uint32_t exitinfo_len = extra - (uint32_t)sizeof(finish_time);
|
||
if (exitinfo_len > 0) {
|
||
exit_info = decode_jobexitinfo(
|
||
(const uint8_t*)payload + sizeof(wal_jobstate_t)
|
||
+ sizeof(finish_time),
|
||
exitinfo_len);
|
||
}
|
||
}
|
||
restore_job_finish_state(p->cluster, js->jobid, js->state,
|
||
finish_time, exit_info);
|
||
}
|
||
break;
|
||
}
|
||
|
||
case WAL_JOB_DISPATCH: {
|
||
/* bug-1678: split out of the shared group above -- this is the one
|
||
* fact type whose payload can carry a trailing Cap'n Proto JobAlloc
|
||
* (persist_submit_job_dispatch_fact's shape) alongside the
|
||
* {jobid, state} header, so a promoted standby's catch-up replay
|
||
* recovers the dispatch-time resource allocation too, not just the
|
||
* state. A payload of exactly sizeof(wal_jobstate_t) (no trailing
|
||
* bytes -- any WAL_JOB_DISPATCH entry written before this fix)
|
||
* decodes no allocation, matching the pre-fix state-only behavior
|
||
* exactly. */
|
||
if (payload_len >= sizeof(wal_jobstate_t)) {
|
||
const wal_jobstate_t* js = (const wal_jobstate_t*)payload;
|
||
xce_job_alloc_t* alloc = NULL;
|
||
if (payload_len > sizeof(wal_jobstate_t)) {
|
||
alloc = decode_joballoc(
|
||
(const uint8_t*)payload + sizeof(wal_jobstate_t),
|
||
payload_len - sizeof(wal_jobstate_t));
|
||
}
|
||
restore_job_dispatch_state(p->cluster, js->jobid, js->state,
|
||
alloc);
|
||
}
|
||
break;
|
||
}
|
||
|
||
case WAL_WORKER_CTRL:
|
||
/* Payload: [uint32_t detail][capnp worker] */
|
||
if (payload_len > WORKER_WAL_HDR_SIZE)
|
||
worker_from_capnp(p->cluster, payload, payload_len);
|
||
break;
|
||
|
||
case WAL_QUEUE_CTRL:
|
||
/* Payload: [wal_queuehdr_t][name NUL-terminated] */
|
||
if (payload_len > sizeof(wal_queuehdr_t)) {
|
||
const wal_queuehdr_t* qh = (const wal_queuehdr_t*)payload;
|
||
const char* name = (const char*)(payload + sizeof(wal_queuehdr_t));
|
||
restore_queue_state(p->cluster, name, qh->status);
|
||
}
|
||
break;
|
||
|
||
case WAL_NOOP:
|
||
/* Internal promotion no-op: empty payload, no entity restore, no
|
||
* hooks. current_lsn is advanced by the physical layer (writer /
|
||
* replay), not here — recognised explicitly rather than via default. */
|
||
break;
|
||
|
||
case WAL_CONFIG_MARKER:
|
||
/* Config-change marker (ha-term-safety task 4.2): a {version_tag,
|
||
* sha256} reference, NEVER the config bytes (R7.4). It carries NO
|
||
* entity state, so replay performs no restore here — the committed
|
||
* configuration is DERIVED from the latest committed marker on demand
|
||
* (config_committed_marker), and the bytes are obtained + hash-verified
|
||
* out-of-band before activation (task 4.3). Recognised explicitly so
|
||
* the per-LSN term index (built above this switch's caller) records its
|
||
* term and on_get_entry_term / (term,lsn) reconcile treat it as an
|
||
* ordinary log entry. */
|
||
break;
|
||
|
||
default:
|
||
break;
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
/* raft-apply-decoupling task 2.2 — see persist.h for the full contract doc
|
||
* (routing table, the job-state-family hook-mapping deviation, and what this
|
||
* function deliberately does NOT own). Not yet wired into any live call
|
||
* site — replay (2.3) and the live apply pump (6.1) will call this later. */
|
||
/* raft-apply-decoupling task 6.1 (Hazard 1 fix): thread-local "this call
|
||
* stack is inside apply_committed_entry's own post-mutation notification"
|
||
* guard. persist_job_hook_fun / persist_queue_hook_fun / persist_worker_hook_fun
|
||
* are AFTER_EVENT subscribers that (for RETAIN/fact-shaped entry types)
|
||
* durably append a NEW WAL entry every time they fire — the correct behavior
|
||
* for their ORIGINAL, still-live purpose (job_ctrl's mark_job_update/
|
||
* mark_job_fail synchronous command path; the legacy queue/worker control
|
||
* command handlers) which mutate live state and rely on this exact hook
|
||
* firing to durably record that mutation.
|
||
*
|
||
* apply_committed_entry() now ALSO fires these same AFTER_EVENT hooks (job
|
||
* facts always as the generic HOOK_DETAIL_JOB_UPDATE, per task 2.2's
|
||
* documented wire-format deviation; queue/worker control at their native
|
||
* detail) to deliver its own post-mutation NOTIFICATION — but that entry is
|
||
* by definition already durably committed (it just got applied FROM the
|
||
* WAL), so re-appending it would silently duplicate the entry on every live
|
||
* apply. This is fixable, correctly, ONLY by provenance (which caller fired
|
||
* the hook), NOT by hook `detail` value: HOOK_DETAIL_JOB_UPDATE in
|
||
* particular is genuinely dual-origin (job_ctrl's SIGCONT/SIGSTOP path uses
|
||
* the identical detail for its own, still-legitimate live append) — an
|
||
* unconditional detail-based filter would silently break job_ctrl's suspend/
|
||
* resume durability instead of fixing anything. A shared persist_t-wide flag
|
||
* would ALSO be wrong: it would race across threads (e.g. job_ctrl mutating
|
||
* a DIFFERENT job on the iomgr thread while this thread's apply walk is
|
||
* mid-hook for an unrelated fact) and could wrongly suppress or wrongly
|
||
* permit an append depending on scheduling. A thread-local flag, scoped
|
||
* narrowly around each xce_run_hooks() call this function makes, is race-free
|
||
* by construction: it is only ever observed by hook subscribers invoked
|
||
* SYNCHRONOUSLY on this exact call stack, from this exact function. */
|
||
static _Thread_local bool t_apply_seam_notifying = false;
|
||
|
||
#ifdef XCE_TESTING
|
||
/* config-upgrade-snapshot task 8.1 (Req 6.3): deterministic fault-injection
|
||
* seam at the marker-apply-to-checkpoint-durable boundary, for later use by
|
||
* a crash/restart regression (task 8.7): "force a controller crash/restart
|
||
* between a configuration-change marker's commit and that member's own
|
||
* snapshot write becoming durable, and assert materialization resumes
|
||
* correctly from the marker on restart." When non-NULL AND it returns
|
||
* nonzero for the marker's own (p, lsn), the WAL_CONFIG_MARKER case below
|
||
* skips its call into persist_checkpoint_marker_triggered() entirely --
|
||
* simulating "the process crashes here, before the checkpoint write becomes
|
||
* durable" (a real crash simply stops the process; the checkpoint call
|
||
* never runs either way). The skip is deliberately routed through the SAME
|
||
* rc!=0 handling an actual checkpoint failure already takes below (not a
|
||
* new branch): last_applied_lsn is not advanced past this marker, so the
|
||
* next kick/retry -- or, after a genuine restart, boot replay's independent
|
||
* call into this same case (already proven idempotent by
|
||
* test_marker_apply_retry_is_idempotent) -- resumes materialization from
|
||
* the marker fresh, exactly matching Requirement 5.2's crash-safety
|
||
* contract. NULL in production -- no behaviour change. Mirrors
|
||
* ha_stream_body_read_fault_hook's shape (config_repl.c) and this same
|
||
* spec's own xce_upgrade_marker_committed_hook placement convention
|
||
* (defined here at file scope, declared extern in whichever test file
|
||
* needs it -- never in a header). */
|
||
int (*persist_marker_checkpoint_fault_hook)(persist_t *p,
|
||
uint64_t marker_lsn) = NULL;
|
||
#endif
|
||
|
||
int apply_committed_entry(persist_t* p, uint32_t wal_type, uint64_t lsn,
|
||
const void* payload, uint32_t len)
|
||
{
|
||
if (!p)
|
||
return -EINVAL;
|
||
|
||
if (!p->cluster)
|
||
return 0; /* unit tests without a full cluster — nothing to dispatch into */
|
||
|
||
cluster_t* c = p->cluster;
|
||
|
||
switch (wal_type) {
|
||
case WAL_JOB_SUBMIT: {
|
||
/* Return value distinguishes a genuine creation from an idempotent
|
||
* no-op (bug-1419): 1 = restore_job() actually constructed a new
|
||
* job, 0 = the jobid already existed (duplicate WAL entry — see
|
||
* below), negative = a real error. replay_wal()'s new-job tally
|
||
* relies on this distinction; the live apply pump discards it.
|
||
*
|
||
* A duplicate WAL_JOB_SUBMIT for the same jobid is an EXPECTED,
|
||
* harmless occurrence, not a bug: a client's local commit-timeout
|
||
* retry can append a second entry for the same jobid while the
|
||
* first is still in flight during a quorum-loss window; the WAL
|
||
* entry that timed out is never retracted, so if it later achieves
|
||
* majority commit anyway, both entries end up committed. restore_job
|
||
* ()'s own -EEXIST here is what makes that safe to replay twice. */
|
||
int rc = restore_job(c, 0, payload, len, lsn);
|
||
if (rc == -EEXIST)
|
||
return 0; /* idempotent re-apply (replay / resync) — no hook */
|
||
if (rc < 0)
|
||
return rc;
|
||
|
||
struct capn jc_ctx;
|
||
capn_init_mem(&jc_ctx, (const uint8_t*)payload, len, 0);
|
||
Job_ptr jp;
|
||
jp.p = capn_getp(capn_root(&jc_ctx), 0, 1);
|
||
struct Job cj;
|
||
read_Job(&cj, jp);
|
||
uint64_t jobid = cj.jobid;
|
||
capn_free(&jc_ctx);
|
||
|
||
job_t* j = NULL;
|
||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||
key_find(c->jobmgr, (void*)jobid, (void**)&j);
|
||
unlock(c->jobmgr);
|
||
if (!j)
|
||
return 1; /* defensive: should not happen after a successful restore */
|
||
|
||
void* jptr = j;
|
||
xlist_t jlist = { .count = 1, .caps = 1, .delta = 1, .data = &jptr };
|
||
t_apply_seam_notifying = true;
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||
HOOK_DETAIL_JOB_NEW, c, &jlist);
|
||
t_apply_seam_notifying = false;
|
||
|
||
return 1;
|
||
}
|
||
|
||
case WAL_JOB_ARRAY_SUBMIT: {
|
||
/* job-array-single-wal-record task 3.2 (design.md "Array Apply
|
||
* Expansion"): the array-record analog of WAL_JOB_SUBMIT's
|
||
* restore_job() above, dispatched from this SAME shared seam so live
|
||
* apply, boot replay (both callers of this function), and (task
|
||
* 3.3) standby catch-up can never diverge in how they expand a
|
||
* committed array record (Req 3.1, 3.2).
|
||
*
|
||
* restore_job_array()'s return generalizes restore_job()'s
|
||
* three-way contract to N elements: a positive count = the number
|
||
* of genuinely NEW elements this call constructed (this is exactly,
|
||
* and only, what the hook fire below carries — Req 3.4); -EEXIST =
|
||
* every index in the record already existed (idempotent
|
||
* whole-record no-op, Req 3.3) — no hook; a negative errno = a
|
||
* genuine construction/resource-exhaustion failure (Req 9.4) —
|
||
* propagated so persist_apply_pump_run() (task 4.1) can halt and
|
||
* fence on it rather than silently skipping it. **Task 7.1
|
||
* (bug-1443) update**: a negative return no longer unconditionally
|
||
* means "no hook" — restore_job_array()'s bulk-insert-phase residual
|
||
* gap can leave a genuinely, permanently inserted prefix in
|
||
* out_new_jobs even on failure (job.c's doc comment, "Residual
|
||
* atomicity gap"); see below. */
|
||
xlist_t out_new_jobs;
|
||
memset(&out_new_jobs, 0, sizeof(out_new_jobs));
|
||
int rc = restore_job_array(c, payload, len, lsn, &out_new_jobs);
|
||
if (rc == -EEXIST)
|
||
return 0; /* idempotent re-apply (replay / resync) — no hook */
|
||
|
||
if (rc < 0) {
|
||
/* job-array-single-wal-record task 7.1 (bug-1443 partial-prefix
|
||
* fix): out_new_jobs is untouched (count == 0, from the memset
|
||
* above) for a reservation or scratch-build-phase failure —
|
||
* nothing was ever inserted, so no hook fires, unchanged from
|
||
* before this task. But for the narrower bulk-insert-phase
|
||
* residual gap, out_new_jobs now carries a non-empty, genuinely
|
||
* and permanently inserted prefix (job.c) that must still get
|
||
* its one HOOK_DETAIL_JOB_NEW fire — a job that is really in
|
||
* c->jobmgr must never be silently un-notified. The overall
|
||
* apply is still, correctly, reported as failed via the
|
||
* negative rc returned below, so persist_apply_pump_run() halts
|
||
* the LSN and the Live Apply Resource Fence engages for the
|
||
* un-constructed remainder (Req 2.3, 9.4, 9.6). */
|
||
if (out_new_jobs.count > 0) {
|
||
t_apply_seam_notifying = true;
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB,
|
||
HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_NEW, c,
|
||
&out_new_jobs);
|
||
t_apply_seam_notifying = false;
|
||
}
|
||
xlist_fini(&out_new_jobs);
|
||
return rc;
|
||
}
|
||
|
||
/* Exactly ONE fire carrying every newly created element in one
|
||
* list — never once per element (Req 3.4). out_new_jobs is already
|
||
* the precise newly-created set restore_job_array() built; no
|
||
* re-derivation via key_find needed. */
|
||
t_apply_seam_notifying = true;
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||
HOOK_DETAIL_JOB_NEW, c, &out_new_jobs);
|
||
t_apply_seam_notifying = false;
|
||
|
||
/* Only the list's own backing array is released here — ownership of
|
||
* every job_t* it holds already transferred to c->jobmgr inside
|
||
* restore_job_array()'s bulk-insert pass, mirroring
|
||
* test_job_array_apply.c's own xlist_fini(&out) usage after a
|
||
* successful call. */
|
||
xlist_fini(&out_new_jobs);
|
||
|
||
return rc;
|
||
}
|
||
|
||
case WAL_JOB_START:
|
||
case WAL_JOB_FINISH:
|
||
case WAL_JOB_STOP:
|
||
case WAL_JOB_RESUME:
|
||
case WAL_JOB_RUSAGE:
|
||
case WAL_JOB_CKPT:
|
||
case WAL_JOB_PREASON:
|
||
case WAL_JOB_DISPATCH: {
|
||
/* Every job-state fact type in this group shares one wire shape —
|
||
* wal_jobstate_t {jobid, state} — and therefore one guarded core.
|
||
* Their payloads are deterministic/state-independent (a START is
|
||
* always JOB_STAT_RUN, a FINISH's outcome depends only on
|
||
* exit_code), so an absolute `state` baked at receipt time never
|
||
* goes stale — unlike WAL_JOB_FAIL/WAL_JOB_UPDATE below, which have
|
||
* their own dedicated cases. See persist.h's doc comment on
|
||
* apply_committed_entry for the full routing table and why this
|
||
* does not dispatch to the richer type-specific apply_job_{start,
|
||
* finish}_transition() cores (their parameter structs are not
|
||
* present on this wire format) — start_time/finish_time/run_time
|
||
* are instead stamped directly below, keyed on wal_type, since
|
||
* apply_job_update_transition() only ever touches `state`.
|
||
* WAL_JOB_DISPATCH joins this group for the same reason: it is
|
||
* synchronously computed at dispatch-commit time under the job's
|
||
* own write lock, with no other in-flight fact for the same job
|
||
* (it is the first transition after PEND) — so it never needs
|
||
* WAL_JOB_FAIL/WAL_JOB_UPDATE's deferred, apply-time-recomputed
|
||
* handling. */
|
||
if (len < sizeof(wal_jobstate_t))
|
||
return 0; /* undersized payload — skip, mirrors apply_wal_entry */
|
||
|
||
const wal_jobstate_t* js = (const wal_jobstate_t*)payload;
|
||
job_t* j = NULL;
|
||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||
key_find(c->jobmgr, (void*)js->jobid, (void**)&j);
|
||
unlock(c->jobmgr);
|
||
if (!j)
|
||
return -ENOENT;
|
||
|
||
lock(j, RTTI_LOCK_WRITE);
|
||
job_transition_result_t result =
|
||
apply_job_update_transition(j, (int)js->state);
|
||
|
||
/* bug-1690: WAL_JOB_RUSAGE carries the client-reported run_usage
|
||
* data independent of the job's state transition -- a rusage
|
||
* report almost never changes state (the job stays RUN across
|
||
* every repeat report), so gating this adoption on
|
||
* result==JOB_TRANSITION_APPLIED the way START/FINISH/DISPATCH do
|
||
* below would make it dead code for that realistic, common case:
|
||
* every report after the first would see JOB_TRANSITION_NOOP_
|
||
* DUPLICATE and the decode would never run. Decode/adopt
|
||
* unconditionally, before the transition-guarded block below,
|
||
* whether or not the state actually changed. A payload of exactly
|
||
* sizeof(wal_jobstate_t) (no trailing bytes -- any WAL_JOB_RUSAGE
|
||
* entry written before this fix, or one fired while job.run_usage
|
||
* was still NULL at write time) decodes no run_usage, leaving it
|
||
* untouched, matching the pre-fix behavior exactly.
|
||
*
|
||
* MERGE, not free-and-replace: on the primary, this exact branch
|
||
* re-runs via persist_apply_pump_run()'s live pump moments after
|
||
* update_job_rusages() already live-mutated j->job.run_usage in
|
||
* place (with an xstring-backed host_name, makeString(),
|
||
* src/lib/xstring.c) -- j->job.run_usage here can genuinely be that
|
||
* SAME live structure, not just a replay artifact. Freeing it via
|
||
* the capnp-generated free_JobRunusage_ptr() (written for plain
|
||
* malloc'd, capnp-decoded Text fields) would free a non-malloc'd,
|
||
* RTTI-managed pointer -- heap corruption. merge_job_runusage()
|
||
* (job.c) never frees an existing host_name; see its own doc
|
||
* comment for the full rationale. */
|
||
if (wal_type == WAL_JOB_RUSAGE && len > sizeof(wal_jobstate_t)) {
|
||
xce_job_runusage_t* decoded = decode_jobrunusage(
|
||
(const uint8_t*)payload + sizeof(wal_jobstate_t),
|
||
len - sizeof(wal_jobstate_t));
|
||
if (decoded != NULL)
|
||
merge_job_runusage(&(j->job.run_usage), decoded);
|
||
}
|
||
|
||
if (result == JOB_TRANSITION_APPLIED) {
|
||
/* apply_job_update_transition() only carries {jobid, state} on
|
||
* this wire shape (see the doc comment above) and never touches
|
||
* start_time/finish_time/run_time itself — stamp them here,
|
||
* keyed on wal_type, mirroring what the richer
|
||
* apply_job_start_transition()/apply_job_finish_transition()
|
||
* cores do for their own (currently unreachable on this dispatch
|
||
* path) callers. */
|
||
if (wal_type == WAL_JOB_START) {
|
||
/* bug-1687: a WAL_JOB_START payload can carry a fixed
|
||
* start_time plus trailing Cap'n Proto JobPIDInfo bytes
|
||
* beyond the wal_jobstate_t header (persist_submit_job_
|
||
* start_fact's shape, mirroring bug-1678's WAL_JOB_DISPATCH
|
||
* trailing-JobAlloc-bytes handling below). Decode and adopt
|
||
* the wire-carried start_time/pidinfo when present; a
|
||
* payload of exactly sizeof(wal_jobstate_t) (no trailing
|
||
* bytes -- any WAL_JOB_START entry written before this fix)
|
||
* intentionally falls through to the old fabricated-
|
||
* time(NULL)/pidinfo-untouched behavior for full backward
|
||
* compatibility. */
|
||
if (len >= sizeof(wal_jobstate_t) + sizeof(uint64_t)) {
|
||
uint64_t wire_start_time;
|
||
memcpy(&wire_start_time,
|
||
(const uint8_t*)payload + sizeof(wal_jobstate_t),
|
||
sizeof(wire_start_time));
|
||
j->job.start_time = (time_t)wire_start_time;
|
||
|
||
uint32_t pidinfo_len = len
|
||
- (uint32_t)sizeof(wal_jobstate_t)
|
||
- (uint32_t)sizeof(uint64_t);
|
||
if (pidinfo_len > 0) {
|
||
job_pidinfo_t* decoded = decode_pidinfo(
|
||
(const uint8_t*)payload + sizeof(wal_jobstate_t)
|
||
+ sizeof(uint64_t),
|
||
pidinfo_len);
|
||
if (decoded != NULL) {
|
||
if (j->job.pidinfo != NULL)
|
||
free_JobPIDInfo_ptr(&(j->job.pidinfo));
|
||
j->job.pidinfo = decoded;
|
||
}
|
||
}
|
||
} else {
|
||
j->job.start_time = time(NULL);
|
||
}
|
||
} else if (wal_type == WAL_JOB_FINISH) {
|
||
/* bug-1689: a WAL_JOB_FINISH payload can carry a fixed
|
||
* finish_time plus a fixed exit_code beyond the
|
||
* wal_jobstate_t header (persist_submit_job_finish_fact's
|
||
* shape, mirroring bug-1687's WAL_JOB_START trailing-fields
|
||
* handling above). Decode and adopt the wire-carried
|
||
* finish_time/exit_code when present; a payload of exactly
|
||
* sizeof(wal_jobstate_t) (no trailing bytes -- any
|
||
* WAL_JOB_FINISH entry written before this fix)
|
||
* intentionally falls through to the old fabricated-
|
||
* time(NULL)/exit_info-untouched behavior for full backward
|
||
* compatibility. run_time is always re-derived from
|
||
* finish_time - start_time either way. */
|
||
if (len >= sizeof(wal_jobstate_t) + sizeof(uint64_t)) {
|
||
uint64_t wire_finish_time;
|
||
memcpy(&wire_finish_time,
|
||
(const uint8_t*)payload + sizeof(wal_jobstate_t),
|
||
sizeof(wire_finish_time));
|
||
j->job.finish_time = (time_t)wire_finish_time;
|
||
|
||
uint32_t exitinfo_len = len
|
||
- (uint32_t)sizeof(wal_jobstate_t)
|
||
- (uint32_t)sizeof(uint64_t);
|
||
if (exitinfo_len > 0) {
|
||
job_exitinfo_t* decoded = decode_jobexitinfo(
|
||
(const uint8_t*)payload + sizeof(wal_jobstate_t)
|
||
+ sizeof(uint64_t),
|
||
exitinfo_len);
|
||
if (decoded != NULL) {
|
||
if (j->job.exit_info != NULL)
|
||
free(j->job.exit_info);
|
||
j->job.exit_info = decoded;
|
||
}
|
||
}
|
||
} else {
|
||
j->job.finish_time = time(NULL);
|
||
}
|
||
j->job.run_time = (int)(j->job.finish_time - j->job.start_time);
|
||
} else if (wal_type == WAL_JOB_DISPATCH
|
||
&& len > sizeof(wal_jobstate_t)) {
|
||
/* bug-1678: trailing bytes beyond the header carry the
|
||
* scheduler's resource allocation (persist_submit_job_
|
||
* dispatch_fact's payload shape) -- decode and adopt it so
|
||
* a restart replay recovers EXEC_HOST, not just STATUS. A
|
||
* payload of exactly sizeof(wal_jobstate_t) (no trailing
|
||
* bytes -- e.g. any WAL_JOB_DISPATCH entry written before
|
||
* this fix) intentionally falls through to the same
|
||
* state-only behavior every other type in this group gets. */
|
||
xce_job_alloc_t* decoded = decode_joballoc(
|
||
(const uint8_t*)payload + sizeof(wal_jobstate_t),
|
||
len - sizeof(wal_jobstate_t));
|
||
if (decoded != NULL) {
|
||
if (j->job.allocs != NULL)
|
||
free_JobAlloc_ptr(&(j->job.allocs));
|
||
j->job.allocs = decoded;
|
||
}
|
||
}
|
||
}
|
||
unlock(j);
|
||
|
||
if (result != JOB_TRANSITION_APPLIED)
|
||
return 0; /* rejected-backward / no-op-duplicate — no hook */
|
||
|
||
/* Hazard 1 guard (see t_apply_seam_notifying's doc comment above):
|
||
* HOOK_DETAIL_JOB_UPDATE is dual-origin — job_ctrl's mark_job_update
|
||
* (SIGCONT/SIGSTOP) fires the identical detail for its own live,
|
||
* still-legitimate append via persist_job_hook_fun. Without this
|
||
* guard, EVERY fact this apply pump applies would re-append itself
|
||
* as a duplicate, mis-typed WAL_JOB_UPDATE entry the moment the live
|
||
* pump (task 6.1) starts calling this function. */
|
||
t_apply_seam_notifying = true;
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||
HOOK_DETAIL_JOB_UPDATE, c, j);
|
||
t_apply_seam_notifying = false;
|
||
return 0;
|
||
}
|
||
|
||
case WAL_JOB_FAIL: {
|
||
/* task 6.1 round 2 (Req 3.3/7.1-7.3, 11.3) fix for a job-state
|
||
* corruption bug: unlike the deterministic group above, a FAIL's
|
||
* resulting state depends on the job's CURRENT state (PERR merges
|
||
* onto whatever terminal state already holds). Dispatch on _pad
|
||
* (see wal_types.h's doc comment on wal_jobstate_t._pad):
|
||
* - WAL_JOB_FAIL_STAGE_FLAG set: a deferred fail-fact
|
||
* (record_job_fail_fact, job.c) — reconstruct a minimal
|
||
* job_failinfo_t from {jobid, stage} and call the REAL
|
||
* apply_job_fail_transition() against the job's ACTUAL CURRENT
|
||
* state at apply time, never a receipt-time snapshot.
|
||
* - _pad == 0: the legacy live path (job_ctrl's mark_job_fail) —
|
||
* `state` is already the correct, current, absolute value,
|
||
* computed synchronously under the same lock as the mutation
|
||
* (no staleness window) — an absolute overwrite is safe and
|
||
* unchanged. */
|
||
if (len < sizeof(wal_jobstate_t))
|
||
return 0; /* undersized payload — skip, mirrors apply_wal_entry */
|
||
|
||
const wal_jobstate_t* js = (const wal_jobstate_t*)payload;
|
||
job_t* j = NULL;
|
||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||
key_find(c->jobmgr, (void*)js->jobid, (void**)&j);
|
||
unlock(c->jobmgr);
|
||
if (!j)
|
||
return -ENOENT;
|
||
|
||
lock(j, RTTI_LOCK_WRITE);
|
||
job_transition_result_t result;
|
||
if (js->_pad & WAL_JOB_FAIL_STAGE_FLAG) {
|
||
job_failinfo_t failinfo = {
|
||
.jobid = js->jobid,
|
||
.stage = js->_pad & ~WAL_JOB_FAIL_STAGE_FLAG,
|
||
.code = 0,
|
||
.detail = NULL,
|
||
};
|
||
result = apply_job_fail_transition(j, &failinfo);
|
||
} else {
|
||
result = apply_job_update_transition(j, (int)js->state);
|
||
}
|
||
unlock(j);
|
||
|
||
if (result != JOB_TRANSITION_APPLIED)
|
||
return 0; /* rejected-backward / no-op-duplicate — no hook */
|
||
|
||
/* Hazard 1 guard — see the identical guard above. */
|
||
t_apply_seam_notifying = true;
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||
HOOK_DETAIL_JOB_UPDATE, c, j);
|
||
t_apply_seam_notifying = false;
|
||
return 0;
|
||
}
|
||
|
||
case WAL_JOB_UPDATE: {
|
||
/* task 6.1 round 2 (Req 3.3/7.1-7.3, 11.3): same class of fix as
|
||
* WAL_JOB_FAIL above. Dispatch on _pad:
|
||
* - WAL_JOB_UPDATE_POSTDONE_MARK: a deferred postdone fact
|
||
* (record_job_postdone_fact, job.c) — call
|
||
* apply_job_postdone_transition(), which re-derives the
|
||
* postdone merge (conditional EXIT backstop + OR JOB_STAT_PDONE)
|
||
* fresh against the job's ACTUAL CURRENT state at apply time.
|
||
* - _pad == 0: the legacy live path (job_ctrl's mark_job_update,
|
||
* SIGCONT/SIGSTOP) — `state` is already correct/absolute,
|
||
* computed synchronously under the same lock as the mutation —
|
||
* an absolute overwrite is safe and unchanged. */
|
||
if (len < sizeof(wal_jobstate_t))
|
||
return 0; /* undersized payload — skip, mirrors apply_wal_entry */
|
||
|
||
const wal_jobstate_t* js = (const wal_jobstate_t*)payload;
|
||
job_t* j = NULL;
|
||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||
key_find(c->jobmgr, (void*)js->jobid, (void**)&j);
|
||
unlock(c->jobmgr);
|
||
if (!j)
|
||
return -ENOENT;
|
||
|
||
lock(j, RTTI_LOCK_WRITE);
|
||
job_transition_result_t result =
|
||
(js->_pad == WAL_JOB_UPDATE_POSTDONE_MARK)
|
||
? apply_job_postdone_transition(j)
|
||
: apply_job_update_transition(j, (int)js->state);
|
||
unlock(j);
|
||
|
||
if (result != JOB_TRANSITION_APPLIED)
|
||
return 0; /* rejected-backward / no-op-duplicate — no hook */
|
||
|
||
/* Hazard 1 guard — see the identical guard above. */
|
||
t_apply_seam_notifying = true;
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||
HOOK_DETAIL_JOB_UPDATE, c, j);
|
||
t_apply_seam_notifying = false;
|
||
return 0;
|
||
}
|
||
|
||
case WAL_QUEUE_CTRL: {
|
||
if (len <= sizeof(wal_queuehdr_t))
|
||
return 0; /* undersized payload — skip, mirrors apply_wal_entry */
|
||
|
||
const wal_queuehdr_t* qh = (const wal_queuehdr_t*)payload;
|
||
const char* name =
|
||
(const char*)((const uint8_t*)payload + sizeof(wal_queuehdr_t));
|
||
|
||
/* restore_queue_state's -ENOENT ("queue not yet built from config")
|
||
* and -EINVAL (bad name) are both benign skips here, matching
|
||
* apply_wal_entry's existing behavior of never fencing on this
|
||
* entry type's restore outcome. */
|
||
if (restore_queue_state(c, name, qh->status) < 0)
|
||
return 0;
|
||
|
||
queue_t* q = NULL;
|
||
lock(c->qmgr, RTTI_LOCK_READ);
|
||
key_find(c->qmgr, (void*)name, (void**)&q);
|
||
unlock(c->qmgr);
|
||
if (!q)
|
||
return 0;
|
||
|
||
/* Same double-append class as the job-fact guard above: the legacy
|
||
* queue-control command handler still mutates then relies on this
|
||
* exact AFTER_EVENT firing to durably append via
|
||
* persist_queue_hook_fun (WAL_QUEUE_CTRL is unconditionally
|
||
* appended on AFTER_EVENT there, not gated on WAL_MODE_WAIT like the
|
||
* job-submit case is) — guard so this live-pump re-notify does not
|
||
* re-append an already-committed entry. */
|
||
t_apply_seam_notifying = true;
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE, HOOK_AFTER_EVENT,
|
||
HOOK_DETAIL_QUEUE_UPDATE, c, q);
|
||
t_apply_seam_notifying = false;
|
||
return 0;
|
||
}
|
||
|
||
case WAL_WORKER_CTRL: {
|
||
if (len <= WORKER_WAL_HDR_SIZE)
|
||
return 0; /* undersized payload — skip, mirrors apply_wal_entry */
|
||
|
||
uint32_t hook_detail;
|
||
memcpy(&hook_detail, payload, sizeof(hook_detail));
|
||
|
||
int rc = worker_from_capnp(c, payload, len);
|
||
if (rc == -EEXIST)
|
||
return 0; /* idempotent re-apply (replay / resync) — no hook */
|
||
if (rc < 0)
|
||
return rc;
|
||
|
||
/* Recover the worker's name via a cheap raw capnp field read
|
||
* (read_Worker's capn_get_text is a borrowed, NUL-terminated pointer
|
||
* into the decode buffer — no allocation, unlike decode_Worker) so
|
||
* the just-restored worker can be looked up for the notification
|
||
* hook without changing worker_from_capnp's own contract. */
|
||
const uint8_t* wpayload = (const uint8_t*)payload + WORKER_WAL_HDR_SIZE;
|
||
size_t wlen = len - WORKER_WAL_HDR_SIZE;
|
||
struct capn wc_ctx;
|
||
capn_init_mem(&wc_ctx, wpayload, wlen, 0);
|
||
Worker_ptr wp;
|
||
wp.p = capn_getp(capn_root(&wc_ctx), 0, 1);
|
||
struct Worker cw;
|
||
read_Worker(&cw, wp);
|
||
|
||
worker_t* w = NULL;
|
||
lock(c->workermgr, RTTI_LOCK_READ);
|
||
key_find(c->workermgr, (void*)cw.name.str, (void**)&w);
|
||
unlock(c->workermgr);
|
||
capn_free(&wc_ctx);
|
||
if (!w)
|
||
return 0; /* defensive: should not happen after a successful restore */
|
||
|
||
/* Same double-append class as the job-fact/queue guards above:
|
||
* persist_worker_hook_fun unconditionally appends on AFTER_EVENT for
|
||
* every worker detail (no WAL_MODE_WAIT gate), and the legacy worker
|
||
* join/update/dead command paths still rely on that firing for their
|
||
* own live durability — guard the whole re-notify switch so this
|
||
* live-pump call never re-appends an already-committed entry. */
|
||
t_apply_seam_notifying = true;
|
||
switch (hook_detail) {
|
||
case HOOK_DETAIL_WORKER_JOIN:
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER,
|
||
HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_JOIN, c,
|
||
w->host.name, w);
|
||
break;
|
||
case HOOK_DETAIL_WORKER_UPDATE:
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER,
|
||
HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_UPDATE, c, w);
|
||
break;
|
||
case HOOK_DETAIL_WORKER_DEAD:
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER,
|
||
HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_DEAD, c, w);
|
||
break;
|
||
default:
|
||
break; /* unrecognized discriminator — mutation already applied,
|
||
* notification skipped rather than guessed at */
|
||
}
|
||
t_apply_seam_notifying = false;
|
||
/* bug-1427: propagate rc (1 = genuinely created a new worker via
|
||
* worker_from_capnp()'s restore_worker() path, 0 = an UPDATE/DEAD
|
||
* applied to an already-existing worker) instead of the previous
|
||
* hardcoded 0 — replay_wal()'s new-worker tally below relies on
|
||
* this distinction, exactly mirroring WAL_JOB_SUBMIT's apply_rc.
|
||
* Every existing caller only checks sign / -EEXIST, never the
|
||
* exact non-negative value, so this is a safe, additive change. */
|
||
return rc;
|
||
}
|
||
|
||
case WAL_CONFIG_MARKER: {
|
||
/* config-upgrade-snapshot task 5 (Req 1.1, 1.5, 1.8): force an
|
||
* immediate, deterministic checkpoint at exactly this marker's own
|
||
* (term, lsn) boundary, materializing the upgrade snapshot the
|
||
* moment every member applies the marker -- primary online-apply,
|
||
* standby online-apply, and boot/restart replay alike, since this
|
||
* one unified apply seam is exactly what all three route through
|
||
* (design.md "Architecture" / "Key simplification found during
|
||
* design": no new leader-specific "resume" logic is needed either,
|
||
* since a newly-elected leader satisfies this the same way, by
|
||
* continuing to apply its own committed log).
|
||
*
|
||
* Deliberately does NOT touch entity interpretation/re-homing
|
||
* (lost_and_found etc.) -- Req 1.8 requires apply and activation to
|
||
* stay separate steps; this case's only effect is materialization,
|
||
* identical in kind to any other checkpoint.
|
||
*
|
||
* persist_checkpoint_marker_triggered() (not a bare
|
||
* persist_checkpoint() call) is required here to avoid two real
|
||
* hazards a naive direct call would hit:
|
||
* 1. Lock reentrancy: persist_apply_pump_run() (the live pump's
|
||
* call site for online-apply, both primary and standby) holds
|
||
* apply_pump_lock across this entire dispatch call;
|
||
* persist_checkpoint()'s own body re-acquires that same,
|
||
* non-recursive mutex, which would self-deadlock.
|
||
* persist_checkpoint_marker_triggered() never locks
|
||
* apply_pump_lock itself -- see its own doc comment for why
|
||
* that is safe under both production call sites.
|
||
* 2. Stale checkpoint_lsn: both live-apply and replay advance
|
||
* last_applied_lsn AFTER this dispatch call returns, never
|
||
* before (persist_apply_pump_run / replay_wal), so reading
|
||
* last_applied_lsn from inside this case would capture the
|
||
* PRECEDING entry's lsn, not the marker's own, mislabeling the
|
||
* checkpoint. `lsn` (this function's own parameter) is
|
||
* already the marker's exact position, so it is passed
|
||
* through explicitly instead of re-derived. */
|
||
/* bug-1469 fix (live HA soak, config-upgrade-snapshot task 8.6):
|
||
* unconditionally re-run config activation right here, BEFORE the
|
||
* forced checkpoint below, so EVERY member -- primary, standby, and
|
||
* any future newly-elected leader -- is guaranteed to have
|
||
* refreshed c->myconf (and the runtime queue/host tables
|
||
* load_primary_config provisions from it) from disk BEFORE the
|
||
* checkpoint writer walks that same runtime state to build the
|
||
* queue-entity list. Before this call existed at all,
|
||
* config_activate_committed() was reachable from exactly two
|
||
* places: handle_repl_config_push (a STANDBY receiving a push) and
|
||
* the boot/recovery path -- never from a member's own live apply of
|
||
* the marker it just committed. A live 3-node chaotic-kill soak
|
||
* caught the gap empirically: the primary that ORIGINATED a config
|
||
* upgrade (do_upgrade_config_with_bytes, which saves the new bytes
|
||
* to disk and bumps config_generation but never re-activates them
|
||
* into c->myconf on itself) forced a checkpoint at the marker's own
|
||
* LSN whose queue-entity section still reflected the OLD
|
||
* (pre-upgrade) queue set, then was killed before any LATER,
|
||
* independent event (restart, or a racing inbound push) happened to
|
||
* call config_activate_committed -- freezing a stale
|
||
* queue-entity list into that checkpoint forever (a real
|
||
* cross-member materialized-state divergence,
|
||
* .wolf/buglog.json bug-1469's byte-level trace: the checkpoint's
|
||
* embedded raw config-bytes section correctly carried the new
|
||
* bytes, but the SEPARATE queue-entity list section -- built by
|
||
* walking runtime state, not the raw bytes -- did not, because that
|
||
* runtime state was never refreshed before the write).
|
||
*
|
||
* MUST run BEFORE persist_checkpoint_marker_triggered(), not merely
|
||
* before upgrade_materialization_verify: the checkpoint writer
|
||
* (write_snapshot -> collect_queue_frames/collect_host_frames)
|
||
* builds its queue/host-entity sections by walking c->qmgr/
|
||
* c->workermgr -- LIVE RUNTIME TABLES, populated at boot from
|
||
* c->myconf->cluster_config and never otherwise re-synced from a
|
||
* later config change on this call path -- not by re-parsing
|
||
* mc->cluster_config directly. So activation must settle BEFORE
|
||
* persist_checkpoint_marker_triggered() runs, or the checkpoint
|
||
* this marker forces still captures whatever runtime state existed
|
||
* prior to activation, regardless of how soon afterward activation
|
||
* runs. (This ordering constraint is logically necessary given
|
||
* write_snapshot's own read path, confirmed by reading
|
||
* collect_queue_frames/write_snapshot directly -- not yet
|
||
* separately confirmed against a live soak re-run at the time this
|
||
* comment was written; see the task's own final live-soak
|
||
* verification step for that.)
|
||
*
|
||
* Symmetric with task 6.2's own fix shape below: one unconditional
|
||
* call at the single apply seam every member's marker-apply is
|
||
* guaranteed to pass through, rather than patching only the
|
||
* originating code path.
|
||
*
|
||
* Safe to call here for the same reason task 6.2's own call
|
||
* (upgrade_materialization_verify, below) is safe at this exact
|
||
* seam: neither config_activate_committed nor anything it
|
||
* transitively calls (config_committed_marker, local_config_sha256,
|
||
* conf_read_current_filename, conf_read_raw, load_primary_config)
|
||
* acquires apply_pump_lock, p->apply_pump_lock, or any other lock
|
||
* (verified -- config_repl.c's only two mutexes, c->ha.lock and
|
||
* c->persist.lock, are both taken in unrelated functions never on
|
||
* this call path) -- so no reentrant-lock hazard even though this
|
||
* runs while persist_apply_pump_run holds apply_pump_lock across
|
||
* its entire dispatch. The disk I/O this performs (WAL segment
|
||
* scan, config file read, Cap'n Proto decode) is the same class
|
||
* already unconditionally performed at this exact call site by
|
||
* upgrade_materialization_verify below (config_committed_marker's
|
||
* own scan + a config-bytes read/hash), so this adds no new
|
||
* blocking-I/O hazard class. Return value intentionally ignored
|
||
* (mirrors upgrade_materialization_verify below): a
|
||
* mismatch/absence return (-EAGAIN/-ENOENT) means this member's own
|
||
* serve-fence (config_serve_fenced, a SEPARATE mechanism from the
|
||
* Activation Fence below) is set instead, and that fence already
|
||
* has its own established retry path (config_schedule_activate_retry)
|
||
* driven off commit-advance -- this call's only job here is to give
|
||
* every marker-apply one unconditional attempt so a member never
|
||
* depends solely on a later, independent event to activate.
|
||
*
|
||
* Guarded on c->myconf != NULL: in production myconf is always
|
||
* wired during cluster init, well before any commit/apply can run,
|
||
* so this is a no-op check on every real code path -- but
|
||
* config_activate_committed's own -ENOENT fallback calls
|
||
* load_primary_config(c) unconditionally on c->myconf without its
|
||
* own null check (pre-existing in config_repl.c, out of this fix's
|
||
* scope to touch), and this call site is reachable from a
|
||
* bare/otherwise-empty test cluster_t that exercises
|
||
* WAL_CONFIG_MARKER apply mechanics without wiring myconf
|
||
* (test_upgrade_snapshot_apply.c's tusa_t fixture,
|
||
* test_marker_apply_via_live_pump_checkpoints_at_marker_own_lsn --
|
||
* confirmed via a live SEGV in load_primary_config, address 0x28,
|
||
* before this guard was added) -- mirrors this same file's existing
|
||
* tolerance of a bare fixture elsewhere (e.g. collect_queue_frames'
|
||
* `c->qmgr == NULL` check, read_active_config_bytes' "no live
|
||
* cluster back-pointer" handling) rather than crashing a test that
|
||
* predates and is unrelated to config activation. */
|
||
if (c->myconf != NULL)
|
||
(void)config_activate_committed(c);
|
||
|
||
int rc;
|
||
#ifdef XCE_TESTING
|
||
if (persist_marker_checkpoint_fault_hook != NULL &&
|
||
persist_marker_checkpoint_fault_hook(p, lsn)) {
|
||
/* Simulated crash (task 8.1, Req 6.3): skip the checkpoint call
|
||
* entirely and fall into the SAME genuine-failure handling below
|
||
* a real checkpoint I/O error already takes -- see the fault
|
||
* hook's own doc comment above for why that is the correct,
|
||
* not merely convenient, simulation of "the process never got
|
||
* to checkpoint". */
|
||
rc = -ECANCELED;
|
||
} else
|
||
#endif
|
||
rc = persist_checkpoint_marker_triggered(p, lsn);
|
||
if (rc == -EBUSY) {
|
||
/* Expected, self-resolving (Req 4.4/5.3): a concurrent
|
||
* WAL-threshold-triggered or earlier marker-triggered
|
||
* checkpoint already holds the shared single-in-flight guard
|
||
* (checkpoint_in_progress). That in-flight checkpoint
|
||
* completes on its own; this marker's own configuration is
|
||
* still picked up by it or a later checkpoint (every
|
||
* checkpoint embeds the currently active configuration, task
|
||
* 2.3) -- not an error, no retry loop here. */
|
||
TRACE("apply_committed_entry: WAL_CONFIG_MARKER lsn=%" PRIu64
|
||
" checkpoint already in progress -- skipping, "
|
||
"self-resolving", lsn);
|
||
} else if (rc != 0) {
|
||
/* A genuine checkpoint failure (I/O error, etc.) -- propagate
|
||
* so the caller's existing halt-and-retry-from-same-position
|
||
* handling engages (persist_apply_pump_run / replay_wal),
|
||
* exactly mirroring every other negative-return case in this
|
||
* dispatch. last_applied_lsn is NOT advanced past this marker,
|
||
* so the next kick/retry re-attempts the checkpoint fresh
|
||
* (Req 5.2 crash-safety: resume materialization from the
|
||
* marker). upgrade_materialization_verify below still runs
|
||
* even on this path (it reads its own signals -- the
|
||
* committed marker + c->persist.cfgref_* -- independently of
|
||
* this rc; a failed checkpoint attempt here does not by
|
||
* itself change whether a PRIOR checkpoint already
|
||
* materialized under the current marker). */
|
||
ERROR("apply_committed_entry: WAL_CONFIG_MARKER lsn=%" PRIu64
|
||
" forced checkpoint failed (rc=%d)", lsn, rc);
|
||
}
|
||
|
||
/* config-upgrade-snapshot task 6.2 (Req 1.8, 3.2, 3.4, 5.5):
|
||
* unconditional Activation Fence re-evaluation right after every
|
||
* checkpoint attempt tied to this marker, regardless of whether
|
||
* persist_checkpoint_marker_triggered just above returned 0,
|
||
* -EBUSY, or a genuine error -- upgrade_materialization_verify
|
||
* itself already handles "materialization not done yet" correctly
|
||
* (it fences and returns -EAGAIN rather than erroring), so gating
|
||
* this call on rc would only reintroduce the closed loop this call
|
||
* site exists to break (every OTHER caller of
|
||
* upgrade_materialization_verify is itself gated on the fence
|
||
* already being set -- see upgrade_materialization_retry_fn and
|
||
* its callers in ha.c -- so without one unconditional trigger the
|
||
* fence could never transition from its zero-initialized default).
|
||
* Checkpoint completion is the design-intended trigger for this
|
||
* re-evaluation (design.md's Activation Fence component). Safe to
|
||
* call here: neither upgrade_materialization_verify nor anything
|
||
* it calls (config_committed_marker, conf_read_current_filename,
|
||
* conf_read_raw) acquires apply_pump_lock or p->apply_pump_lock --
|
||
* they only do atomic reads of c->persist.committed_lsn /
|
||
* c->persist.cfgref_* and plain file I/O (opendir/pread-style WAL
|
||
* scan, conf_read_raw) -- so no reentrant-lock hazard even though
|
||
* this call runs while persist_apply_pump_run holds
|
||
* apply_pump_lock across this entire dispatch (same hazard class
|
||
* as task 5's own persist_checkpoint_marker_triggered, avoided the
|
||
* same way: never touch apply_pump_lock). Return value
|
||
* intentionally ignored here -- ha_dispatch_gate and the
|
||
* promotion gate (ha.c) are what actually consume the fence;
|
||
* this call's only job is to keep it current. */
|
||
(void)upgrade_materialization_verify(c);
|
||
|
||
return (rc == -EBUSY) ? 0 : rc;
|
||
}
|
||
|
||
default:
|
||
/* WAL_NOOP / WAL_CHECKPOINT / unknown — no entity effect at this
|
||
* seam, no hook (unchanged from apply_wal_entry's existing
|
||
* treatment of these types). WAL_CONFIG_MARKER has its own explicit
|
||
* case above (config-upgrade-snapshot task 5) and no longer falls
|
||
* through to here. */
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/* ============================================================
|
||
* Ordered apply pump (raft-apply-decoupling task 6.1)
|
||
* ============================================================ */
|
||
|
||
/* Locate and read back the WAL entry at exactly `lsn` from the on-disk
|
||
* segment files, for the live apply pump to hand to apply_committed_entry()
|
||
* (mirrors read_entry_meta_locked's segment-scan idiom above, but returns
|
||
* the entry's type and a malloc'd copy of its payload instead of just
|
||
* metadata, and re-verifies the entry CRC — this is live-path plumbing, not
|
||
* a metadata probe).
|
||
*
|
||
* Does not take p->lock: every LSN this is ever called with is already
|
||
* majority-committed (durable and replicated), strictly below the physical
|
||
* append tip a concurrent persist_wal_async could be extending — so this
|
||
* never races the single-writer append path for the bytes it actually
|
||
* reads.
|
||
*
|
||
* Returns 0 and populates *out_type/*out_payload (malloc'd if payload_len >
|
||
* 0, caller frees; NULL when payload_len is 0)/*out_len on success.
|
||
* Returns -ENOENT if no segment contains this LSN (should not happen for a
|
||
* genuinely committed entry) or a negative errno on a read/CRC failure. */
|
||
static int apply_pump_read_entry(persist_t* p, uint64_t lsn,
|
||
uint32_t* out_type, uint8_t** out_payload,
|
||
uint32_t* out_len)
|
||
{
|
||
if (!p || lsn == 0 || !out_type || !out_payload || !out_len)
|
||
return -EINVAL;
|
||
|
||
/* Fast path (raft-apply-decoupling task 7.4 perf-fix): the sole caller
|
||
* (persist_apply_pump_run) calls this once per LSN in strict ascending
|
||
* order with no gaps in the common case, and holds apply_pump_lock
|
||
* across the entire walk -- so the cache fields below need no lock of
|
||
* their own here (see persist.h doc comment). A cache hit reads the next
|
||
* header directly at the remembered offset on the already-open fd,
|
||
* giving amortized O(1) per entry instead of a fresh
|
||
* opendir/readdir/segment-sort plus a scan-from-zero on every single
|
||
* call (the O(n^2) regression this fixes). ANY miss below just drops the
|
||
* cache and falls through to the unchanged slow path -- correctness
|
||
* never depends on the cache being right, only performance does. */
|
||
if (p->apply_pump_cache_fd >= 0 && p->apply_pump_cache_next_lsn == lsn) {
|
||
wal_entry_hdr_t hdr;
|
||
ssize_t hn = pread(p->apply_pump_cache_fd, &hdr, sizeof(hdr),
|
||
(off_t)p->apply_pump_cache_off);
|
||
bool hit = (hn == (ssize_t)sizeof(hdr)) && (hdr.lsn == lsn);
|
||
uint8_t* payload = NULL;
|
||
|
||
if (hit && hdr.payload_len > 0) {
|
||
payload = malloc(hdr.payload_len);
|
||
if (!payload) {
|
||
hit = false;
|
||
} else {
|
||
ssize_t pn = pread(p->apply_pump_cache_fd, payload,
|
||
hdr.payload_len,
|
||
(off_t)(p->apply_pump_cache_off +
|
||
sizeof(hdr)));
|
||
if (pn != (ssize_t)hdr.payload_len) {
|
||
free(payload);
|
||
payload = NULL;
|
||
hit = false;
|
||
}
|
||
}
|
||
}
|
||
if (hit) {
|
||
uint32_t expected_crc =
|
||
crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
|
||
if (hdr.payload_len > 0)
|
||
expected_crc = crc32(expected_crc, (const Bytef*)payload,
|
||
hdr.payload_len);
|
||
if (hdr.crc != expected_crc) {
|
||
free(payload);
|
||
payload = NULL;
|
||
hit = false;
|
||
}
|
||
}
|
||
|
||
if (hit) {
|
||
*out_type = hdr.type;
|
||
*out_payload = payload;
|
||
*out_len = hdr.payload_len;
|
||
p->apply_pump_cache_off +=
|
||
(uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
p->apply_pump_cache_next_lsn = lsn + 1;
|
||
return 0;
|
||
}
|
||
|
||
/* Miss (short read/EOF meaning the entry rolled into the next
|
||
* segment, header mismatch, CRC failure, whatever): don't try to be
|
||
* clever about advancing to the next segment inline. Close and fall
|
||
* through to the EXISTING slow path unchanged -- it already
|
||
* correctly finds the right segment from scratch. */
|
||
close(p->apply_pump_cache_fd);
|
||
p->apply_pump_cache_fd = -1;
|
||
}
|
||
|
||
DIR* d = opendir(p->wal_path);
|
||
if (!d)
|
||
return -ENOENT;
|
||
|
||
uint64_t seg_lsns[4096];
|
||
int nseg = 0;
|
||
struct dirent* e;
|
||
while ((e = readdir(d)) != NULL &&
|
||
nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) {
|
||
if (strncmp(e->d_name, "seg-", 4) != 0)
|
||
continue;
|
||
uint64_t sl = 0;
|
||
if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &sl) != 1)
|
||
continue;
|
||
seg_lsns[nseg++] = sl;
|
||
}
|
||
closedir(d);
|
||
if (nseg == 0)
|
||
return -ENOENT;
|
||
|
||
/* Ascending sort by start LSN (segment count is tiny in practice). */
|
||
for (int i = 1; i < nseg; i++) {
|
||
uint64_t key = seg_lsns[i];
|
||
int j = i - 1;
|
||
while (j >= 0 && seg_lsns[j] > key) {
|
||
seg_lsns[j + 1] = seg_lsns[j];
|
||
j--;
|
||
}
|
||
seg_lsns[j + 1] = key;
|
||
}
|
||
|
||
int rc = -ENOENT;
|
||
for (int si = 0; si < nseg; si++) {
|
||
/* Segments are LSN-ordered; one whose start LSN already exceeds the
|
||
* target cannot contain it, nor can any later one. */
|
||
if (seg_lsns[si] > lsn)
|
||
break;
|
||
|
||
char seg_path[4096];
|
||
snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, p->wal_path,
|
||
seg_lsns[si]);
|
||
int fd = open(seg_path, O_RDONLY);
|
||
if (fd < 0)
|
||
continue;
|
||
|
||
uint32_t off = 0;
|
||
for (;;) {
|
||
wal_entry_hdr_t hdr;
|
||
ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)off);
|
||
if (hn < (ssize_t)sizeof(hdr))
|
||
break; /* short read / EOF / pre-allocated zero tail */
|
||
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
|
||
break;
|
||
|
||
if (hdr.lsn == lsn) {
|
||
uint8_t* payload = NULL;
|
||
uint32_t expected_crc =
|
||
crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
|
||
|
||
if (hdr.payload_len > 0) {
|
||
payload = malloc(hdr.payload_len);
|
||
if (!payload) {
|
||
rc = -ENOMEM;
|
||
break;
|
||
}
|
||
ssize_t pn = pread(fd, payload, hdr.payload_len,
|
||
(off_t)(off + sizeof(hdr)));
|
||
if (pn != (ssize_t)hdr.payload_len) {
|
||
free(payload);
|
||
rc = -EIO;
|
||
break;
|
||
}
|
||
expected_crc = crc32(expected_crc, (const Bytef*)payload,
|
||
hdr.payload_len);
|
||
}
|
||
|
||
if (hdr.crc != expected_crc) {
|
||
free(payload);
|
||
rc = -EIO;
|
||
break;
|
||
}
|
||
|
||
*out_type = hdr.type;
|
||
*out_payload = payload;
|
||
*out_len = hdr.payload_len;
|
||
rc = 0;
|
||
|
||
/* raft-apply-decoupling task 7.4 perf-fix: populate the
|
||
* segment-lookup cache with this fd positioned just past the
|
||
* entry we just returned, BEFORE closing it below, so the
|
||
* next expected LSN (lsn+1) hits the fast path above instead
|
||
* of re-scanning from segment start next time. Every OTHER
|
||
* branch (error paths, not-found-in-this-segment
|
||
* continuation) closes fd exactly as before -- caching only
|
||
* ever happens on this success branch. */
|
||
p->apply_pump_cache_fd = fd;
|
||
p->apply_pump_cache_seg_lsn = seg_lsns[si];
|
||
p->apply_pump_cache_off =
|
||
off + (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
p->apply_pump_cache_next_lsn = lsn + 1;
|
||
fd = -1; /* ownership transferred to the cache -- the
|
||
* close(fd) below becomes a guarded no-op. */
|
||
break;
|
||
}
|
||
|
||
off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
}
|
||
if (fd >= 0)
|
||
close(fd);
|
||
|
||
if (rc == 0 || rc == -ENOMEM || rc == -EIO)
|
||
break; /* found it, or hit a genuine error inside the segment
|
||
* that should contain it */
|
||
}
|
||
|
||
return rc;
|
||
}
|
||
|
||
int persist_apply_pump_run(persist_t* p)
|
||
{
|
||
if (!p)
|
||
return -EINVAL;
|
||
|
||
pthread_mutex_lock(&p->apply_pump_lock);
|
||
|
||
int applied = 0;
|
||
for (;;) {
|
||
uint64_t committed = atomic_load_explicit(&p->committed_lsn,
|
||
memory_order_acquire);
|
||
uint64_t next = atomic_load_explicit(&p->last_applied_lsn,
|
||
memory_order_relaxed) + 1;
|
||
if (next > committed)
|
||
break; /* caught up as of this instant */
|
||
|
||
uint32_t wal_type = 0;
|
||
uint8_t* payload = NULL;
|
||
uint32_t len = 0;
|
||
int rc = apply_pump_read_entry(p, next, &wal_type, &payload, &len);
|
||
if (rc < 0) {
|
||
ERROR("persist_apply_pump_run: failed to read committed entry "
|
||
"lsn=%" PRIu64 " (rc=%d) -- stopping this pass; the entry "
|
||
"stays committed and will be retried, from the same "
|
||
"position, on the next kick",
|
||
next, rc);
|
||
pthread_mutex_unlock(&p->apply_pump_lock);
|
||
return rc;
|
||
}
|
||
|
||
/* Strict order, no skipping or reordering: this is the single
|
||
* dispatch call every newly committed entry passes through here,
|
||
* regardless of its commit mode (WAIT command or RETAIN fact) — the
|
||
* loop only ever advances last_applied_lsn one entry at a time, in
|
||
* ascending order, so a burst of commands and facts that committed
|
||
* out of their original submission order still applies in strict
|
||
* commit (LSN) order (Req 3.3, 7.3). */
|
||
int apply_rc = apply_committed_entry(p, wal_type, next, payload, len);
|
||
if (apply_rc < 0) {
|
||
/* job-array-single-wal-record task 4.1 (design.md "Live Apply
|
||
* Resource Fence"): this return value was previously discarded
|
||
* -- (void)apply_committed_entry(...) -- with last_applied_lsn
|
||
* advancing unconditionally, silently and permanently losing
|
||
* any apply failure. A negative rc here is never a content-
|
||
* derived failure (Req 9.1-9.3's pre-commit validation already
|
||
* made those impossible post-commit); it is a node-local
|
||
* resource-exhaustion failure (e.g. OOM mid array expansion,
|
||
* Req 9.4) that must not be skipped. Mirror the
|
||
* apply_pump_read_entry failure branch immediately above: stop
|
||
* the walk exactly at this LSN, do not advance
|
||
* last_applied_lsn, and do not take this LSN's pending-apply
|
||
* hand-off (persist_take_pending_apply/on_applied) -- the entry
|
||
* never actually applied, so that hand-off must not fire for
|
||
* it. The entry stays committed and will be retried, from the
|
||
* same position, on the next kick. This is a GENERAL fix: every
|
||
* WAL type's apply outcome is now respected, not just
|
||
* WAL_JOB_ARRAY_SUBMIT's. Under NORMAL operation every other
|
||
* existing type's apply case returns 0 or positive, so this
|
||
* branch is a behavior no-op for them in practice; the
|
||
* fact-type cases' pre-existing "-ENOENT: referenced job not
|
||
* found" branch (persist.h's dispatch table) is one other real,
|
||
* pre-existing source of a negative return here, and now also
|
||
* correctly halts the walk here instead of being silently
|
||
* discarded, exactly as this fix intends. */
|
||
ERROR("persist_apply_pump_run: apply_committed_entry failed for "
|
||
"committed entry lsn=%" PRIu64 " wal_type=%" PRIu32
|
||
" (rc=%d) -- stopping this pass at this position; the "
|
||
"entry stays committed and will be retried, from the same "
|
||
"position, on the next kick; Live Apply Resource Fence "
|
||
"ENGAGED (job-array-single-wal-record Req 9.4)",
|
||
next, wal_type, apply_rc);
|
||
/* job-array-single-wal-record task 4.2 (design.md "Live Apply
|
||
* Resource Fence"): engage the NEW, live, self-healing fence --
|
||
* distinct from recovery_unrecoverable -- so the promotion gate
|
||
* (task 4.3) can refuse this node while it stays alive and
|
||
* keeps retrying. Cleared below, from inside this same walk,
|
||
* the instant a retry of this exact LSN succeeds (Req 9.5). */
|
||
persist_mark_apply_resource_fenced(p, next);
|
||
free(payload);
|
||
pthread_mutex_unlock(&p->apply_pump_lock);
|
||
return apply_rc;
|
||
}
|
||
free(payload);
|
||
|
||
atomic_store_explicit(&p->last_applied_lsn, next, memory_order_relaxed);
|
||
applied++;
|
||
|
||
/* job-array-single-wal-record task 4.2 (design.md "Live Apply
|
||
* Resource Fence", Req 9.5): self-heal. Called unconditionally on
|
||
* EVERY successful apply, not just after a known-fenced LSN --
|
||
* persist_clear_apply_resource_fence() is a cheap no-op when nothing
|
||
* is fenced (the overwhelming common case). This is correct and
|
||
* sufficient because the walk above is strictly sequential and
|
||
* returns immediately on any failure (see the halt branch just
|
||
* above): it can never advance PAST a fenced LSN without that exact
|
||
* LSN applying successfully first, so "the next entry to apply
|
||
* successfully after a fence engages" and "a retry of the SAME
|
||
* stuck LSN succeeding" are the same event by construction. No
|
||
* election, promotion check, or external/admin action is involved --
|
||
* the fence clears purely because this walk itself made progress. */
|
||
persist_clear_apply_resource_fence(p);
|
||
|
||
/* raft-apply-decoupling task 6.2 (Req 1.1, 2.1-2.3): consume this
|
||
* LSN's command hand-off, if any (task 3.2's
|
||
* persist_register_pending_apply(), e.g. handle_jobsubmit's
|
||
* commit-outcome continuation) -- strictly AFTER the mutation above
|
||
* and after last_applied_lsn already reflects it, never before or
|
||
* during. A fact entry never registers a hand-off, so
|
||
* persist_take_pending_apply() simply returns 0 for it and this is a
|
||
* correct no-op with no per-entry-type branching needed here.
|
||
* persist_take_pending_apply() unlinks the registration before
|
||
* returning it, so re-applying the same lsn (a future retry/replay
|
||
* landing on this exact walk position again) can never fire the
|
||
* continuation a second time. */
|
||
pending_apply_notify_fn on_applied = NULL;
|
||
void* apply_ctx = NULL;
|
||
|
||
if (persist_take_pending_apply(p, next, &on_applied, &apply_ctx))
|
||
on_applied(apply_ctx, next);
|
||
}
|
||
|
||
pthread_mutex_unlock(&p->apply_pump_lock);
|
||
return applied;
|
||
}
|
||
|
||
/* Heap payload for one dispatched apply-pump walk invocation; frees itself
|
||
* after use, mirroring commit.c's commit_resolve_dispatch_t convention. */
|
||
typedef struct {
|
||
persist_t* p;
|
||
} apply_pump_dispatch_t;
|
||
|
||
/* Run walk passes until genuinely caught up, using the same recheck-then-
|
||
* clear idiom commit.c's sweep thread uses (task 1.3) to avoid a lost-wakeup
|
||
* race between "persist_apply_pump_run just decided it is caught up" and "a
|
||
* new commit lands right after". Runs on a pool worker thread (or inline as
|
||
* persist_apply_pump_kick's documented fallback) — never on the calling
|
||
* (potentially replication-ACK/iomgr) thread when dispatched normally.
|
||
*
|
||
* Does NOT immediately retry a failed pass: persist_apply_pump_run() already
|
||
* logs loudly and leaves last_applied_lsn unadvanced past a read failure, so
|
||
* an immediate self-retry here would busy-loop forever on the exact same
|
||
* unresolved head-of-line entry (a real, reproduced hazard — a persist_t
|
||
* whose committed_lsn is ever advanced without a corresponding real WAL
|
||
* entry on disk, e.g. a test fixture exercising commit-engine resolve logic
|
||
* in isolation, spun this into a multi-million-iteration tight loop before
|
||
* this guard existed). A future, genuinely NEW kick — triggered by a later
|
||
* real commit-advance event — retries then, at that event's natural cadence,
|
||
* never in a busy spin. */
|
||
static void apply_pump_drain(persist_t* p)
|
||
{
|
||
for (;;) {
|
||
int rc = persist_apply_pump_run(p);
|
||
|
||
atomic_store_explicit(&p->apply_pump_scheduled, 0, memory_order_release);
|
||
|
||
if (rc < 0)
|
||
return; /* already logged loudly inside persist_apply_pump_run;
|
||
* do not busy-retry the same unresolved entry */
|
||
|
||
uint64_t committed = atomic_load_explicit(&p->committed_lsn,
|
||
memory_order_acquire);
|
||
uint64_t next = atomic_load_explicit(&p->last_applied_lsn,
|
||
memory_order_relaxed) + 1;
|
||
if (next > committed)
|
||
return; /* genuinely caught up: no race window survived open */
|
||
|
||
int expected = 0;
|
||
if (!atomic_compare_exchange_strong_explicit(
|
||
&p->apply_pump_scheduled, &expected, 1, memory_order_acq_rel,
|
||
memory_order_relaxed))
|
||
return; /* a concurrent kick already reclaimed the flag; it will
|
||
* run its own pass and pick up this same new work */
|
||
/* else: we reclaimed the flag ourselves -- loop and run another pass */
|
||
}
|
||
}
|
||
|
||
static int apply_pump_vtask_entry(vtask_t* task)
|
||
{
|
||
apply_pump_dispatch_t* d = (apply_pump_dispatch_t*)task->data->u.generic;
|
||
|
||
apply_pump_drain(d->p);
|
||
free(d);
|
||
return 0;
|
||
}
|
||
|
||
void persist_apply_pump_kick(persist_t* p, void* pool_arg)
|
||
{
|
||
if (!p || !p->cluster)
|
||
return;
|
||
|
||
/* bug-1416 (raft-apply-decoupling task 7.2 round 2): defer while this
|
||
* node's own boot-time persist_recover() replay+verification pass is
|
||
* still running -- last_applied_lsn is being written UNLOCKED by
|
||
* replay_wal() during that window (single-threaded-recovery
|
||
* assumption), and persist_verify_apply_correctness() is about to
|
||
* compare a live re-read of it (and of the live job table) against a
|
||
* WAL-scan-derived expectation. Letting this walk run concurrently
|
||
* would race BOTH: it could advance last_applied_lsn out from under
|
||
* replay_wal's own plain (non-CAS) store, and it mutates the same job
|
||
* table persist_verify_apply_correctness re-reads as "actual" --
|
||
* producing a false APPLY_DIVERGENT verdict on a perfectly healthy
|
||
* boot (confirmed live). No work is lost: persist_recover() sets this
|
||
* flag back to true and issues exactly one catch-up kick right before
|
||
* it returns, which will walk from last_applied_lsn+1 up through
|
||
* whatever committed_lsn has grown to by then -- a full, ordered
|
||
* drain, not a partial one. Intentionally checked with NO lock: a
|
||
* stale `true` read racing the flag's transition to `false` is
|
||
* harmless (the walk itself still only advances one entry at a time
|
||
* under apply_pump_lock, so at worst this call proceeds a few
|
||
* instructions into a window persist_recover() is about to close, and
|
||
* the deferred re-kick after recover() still runs); a stale `false`
|
||
* merely defers one no-op-cost kick to the next real trigger. */
|
||
if (!atomic_load_explicit(&p->recovery_complete, memory_order_acquire))
|
||
return;
|
||
|
||
int expected = 0;
|
||
if (!atomic_compare_exchange_strong_explicit(
|
||
&p->apply_pump_scheduled, &expected, 1, memory_order_acq_rel,
|
||
memory_order_relaxed))
|
||
return; /* already scheduled/running -- it will pick up any new work */
|
||
|
||
vtaskmgr_t* pool = (vtaskmgr_t*)pool_arg;
|
||
vtask_data_t* data = calloc(1, sizeof(*data));
|
||
vtask_t* task = vtask_alloc();
|
||
apply_pump_dispatch_t* d = calloc(1, sizeof(*d));
|
||
|
||
/* Mirrors commit.c's dispatch_continuation fallback exactly: never leave
|
||
* a scheduled walk silently unrun. pool == NULL is the expected shape of
|
||
* a commit-only / bare-persist-cluster unit fixture that never calls
|
||
* commit_set_pool — a real cluster always wires the pool before any
|
||
* commit activity (cluster_init_phase2), so this fallback path is the
|
||
* normal, deterministic behavior for that class of test, not a rare
|
||
* production condition. */
|
||
if (pool == NULL || data == NULL || task == NULL || d == NULL) {
|
||
free(data);
|
||
free(task);
|
||
free(d);
|
||
apply_pump_drain(p);
|
||
return;
|
||
}
|
||
|
||
d->p = p;
|
||
data->vtype = VTASK_GENERIC;
|
||
data->u.generic = d;
|
||
vtask_init(task, (vtask_func_t)apply_pump_vtask_entry, data);
|
||
|
||
/* VTASK_LIVE_BLOCK is never admission-gated (same delivery guarantee as
|
||
* commit.c's own continuation dispatch) — this cannot fast-reject the
|
||
* way VTASK_REQ_BLOCK could. */
|
||
if (vtask_enq_classified(pool, task, VTASK_LIVE_BLOCK) < 0) {
|
||
WARN("persist_apply_pump_kick: vtask enqueue failed -- running "
|
||
"inline (stay alive, never leave a scheduled walk unrun)");
|
||
free(data);
|
||
free(task);
|
||
free(d);
|
||
apply_pump_drain(p);
|
||
}
|
||
}
|
||
|
||
/* persist_apply_pump_run_sync (2026-07-10, chaotic-soak regression fix,
|
||
* ROUND 2): synchronously drain the apply pump on THIS thread, GUARANTEEING
|
||
* last_applied_lsn reaches committed_lsn (as observed at entry) before
|
||
* returning -- regardless of what any concurrent caller is doing.
|
||
*
|
||
* Round 1 of this fix CAS-claimed apply_pump_scheduled and, on losing that
|
||
* claim to a concurrent caller (e.g. a heartbeat-driven kick that fired
|
||
* moments before promotion), just returned -- betting the OTHER caller's
|
||
* async vtask-dispatched drain would finish in time. A second live chaotic
|
||
* soak proved that bet wrong: job loss persisted even with the environmental
|
||
* fdatasync noise (bug-924) independently confirmed and eliminated, so this
|
||
* was a genuine, separate bug in the fix itself, not noise. The caller here
|
||
* (promotion) cannot outsource its guarantee to a drain it doesn't control
|
||
* the completion of.
|
||
*
|
||
* Fix: do NOT use the CAS-coalescing flag as a "someone else has it, I'm
|
||
* done" signal -- call persist_apply_pump_run() directly in a loop until
|
||
* THIS call observes last_applied_lsn caught up to committed_lsn. Multiple
|
||
* concurrent callers (this one and any vtask-dispatched persist_apply_pump_
|
||
* kick) safely interleave via persist_apply_pump_run's own internal
|
||
* apply_pump_lock and apply_committed_entry's per-LSN idempotency -- at
|
||
* worst a redundant no-op pass, never lost or duplicated work. Stops (does
|
||
* not busy-loop forever) on a genuine read failure, exactly mirroring
|
||
* apply_pump_drain's own error handling: persist_apply_pump_run() already
|
||
* logs loudly and leaves last_applied_lsn unadvanced on that path. */
|
||
void persist_apply_pump_run_sync(persist_t* p)
|
||
{
|
||
if (!p || !p->cluster)
|
||
return;
|
||
if (!atomic_load_explicit(&p->recovery_complete, memory_order_acquire))
|
||
return;
|
||
|
||
for (;;) {
|
||
uint64_t committed = atomic_load_explicit(&p->committed_lsn,
|
||
memory_order_acquire);
|
||
uint64_t applied = atomic_load_explicit(&p->last_applied_lsn,
|
||
memory_order_relaxed);
|
||
if (applied >= committed)
|
||
return; /* caught up -- by us or a concurrent drain */
|
||
|
||
int rc = persist_apply_pump_run(p);
|
||
if (rc < 0)
|
||
return; /* already logged loudly; do not busy-loop on the same
|
||
* unresolved entry (mirrors apply_pump_drain) */
|
||
/* Loop: re-check in case committed_lsn advanced further mid-walk, or
|
||
* a concurrent drain made partial progress this pass didn't see. */
|
||
}
|
||
}
|
||
|
||
/* Read WAL segment files in LSN order; replay entries with lsn > from_lsn.
|
||
*
|
||
* Segment filename format: seg-<start_lsn_hex16>.wal
|
||
* Entry format: wal_entry_hdr_t (40 bytes) + payload (hdr.payload_len bytes).
|
||
*
|
||
* CRC chain: hdr.prev_crc must match last_crc (WAL_CRC_NONE for first entry
|
||
* in a segment). If prev_crc mismatches, stop at last good entry and log WARN.
|
||
*
|
||
* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1):
|
||
* *verdict_out (when non-NULL) is set to RECOVER_TRUNCATED iff replay stopped
|
||
* on a STRUCTURAL corruption — a CRC chain break, a CRC mismatch, or a
|
||
* short/partial entry read — and RECOVER_CLEAN otherwise (clean EOF,
|
||
* pre-allocated-zero tail, empty WAL, or all entries replayed). The verdict is
|
||
* about the STRUCTURE of the local log, not about being behind the cluster: a
|
||
* clean log that simply ends below the cluster high-water is still CLEAN.
|
||
*
|
||
* new_job_submits_out (raft-apply-decoupling task 5.1, when non-NULL) is set
|
||
* to the number of WAL_JOB_SUBMIT entries this call actually dispatched via
|
||
* apply_committed_entry() that resulted in a GENUINE new job (that function's
|
||
* return value == 1) — not merely the number of WAL_JOB_SUBMIT entries above
|
||
* `from_lsn` present in the log. This is the log-derived half of the
|
||
* post-replay apply-correctness expectation (see apply_expectation_t).
|
||
*
|
||
* A WAL_JOB_SUBMIT entry can legitimately be duplicated for the same jobid
|
||
* (bug-1419): a client's local commit-timeout retry appends a second entry
|
||
* while the first is still in flight during a quorum-loss window, and the
|
||
* timed-out entry is never retracted, so both can end up committed if the
|
||
* first later achieves majority commit anyway. apply_committed_entry()
|
||
* dedupes this via restore_job()'s -EEXIST and returns 0 (no-op) rather than
|
||
* 1 (created) for the duplicate — counting raw WAL_JOB_SUBMIT occurrences
|
||
* instead would tally that duplicate as a second job and spuriously fail the
|
||
* post-replay comparison against jobmgr's actual growth (APPLY_DIVERGENT).
|
||
*
|
||
* new_worker_joins_out (bug-1427, when non-NULL) is the worker analog of
|
||
* new_job_submits_out — the number of WAL_WORKER_CTRL entries this call
|
||
* dispatched that genuinely created a new worker (apply_committed_entry()'s
|
||
* return value == 1, per worker_from_capnp()'s bug-1427 contract), not raw
|
||
* WAL_WORKER_CTRL occurrences (which also include UPDATE/DEAD entries and
|
||
* idempotent JOIN replays, neither of which grows the worker table).
|
||
*
|
||
* Returns number of entries replayed (>= 0) or negative errno on I/O error. */
|
||
static int replay_wal(persist_t* p, uint64_t from_lsn,
|
||
recover_verdict_t* verdict_out,
|
||
uint64_t* new_job_submits_out,
|
||
uint64_t* new_worker_joins_out)
|
||
{
|
||
/* Default to CLEAN; only a structural corruption flips it to TRUNCATED. */
|
||
if (verdict_out)
|
||
*verdict_out = RECOVER_CLEAN;
|
||
if (new_job_submits_out)
|
||
*new_job_submits_out = 0;
|
||
if (new_worker_joins_out)
|
||
*new_worker_joins_out = 0;
|
||
|
||
/* --- Collect and sort segment filenames by start LSN --- */
|
||
DIR* d = opendir(p->wal_path);
|
||
if (!d)
|
||
return 0; /* no WAL dir → first boot, nothing to replay (CLEAN) */
|
||
|
||
/* Collect up to 4096 segment names */
|
||
uint64_t seg_lsns[4096];
|
||
int nseg = 0;
|
||
|
||
struct dirent* e;
|
||
while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns)/sizeof(seg_lsns[0]))) {
|
||
if (strncmp(e->d_name, "seg-", 4) != 0) continue;
|
||
uint64_t seg_start = 0;
|
||
if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start) != 1) continue;
|
||
seg_lsns[nseg++] = seg_start;
|
||
}
|
||
closedir(d);
|
||
|
||
if (nseg == 0)
|
||
return 0;
|
||
|
||
/* Simple insertion sort — segment count is tiny in practice */
|
||
for (int i = 1; i < nseg; i++) {
|
||
uint64_t key = seg_lsns[i];
|
||
int j = i - 1;
|
||
while (j >= 0 && seg_lsns[j] > key) {
|
||
seg_lsns[j + 1] = seg_lsns[j];
|
||
j--;
|
||
}
|
||
seg_lsns[j + 1] = key;
|
||
}
|
||
|
||
/* --- Replay entries from each segment --- */
|
||
int replayed = 0;
|
||
uint32_t last_crc = WAL_CRC_NONE;
|
||
|
||
/* job-array-single-wal-record task 7.2 (design.md "Live Apply Resource
|
||
* Fence", task 4.2's self-flagged gap): once a negative
|
||
* apply_committed_entry() return is hit during this replay pass, every
|
||
* SUBSEQUENT entry is skipped for LOGICAL apply purposes -- mirroring
|
||
* persist_apply_pump_run()'s halt-and-fence branch, which never
|
||
* dispatches past a failed LSN either. This preserves the strict
|
||
* sequential-no-skip apply invariant (task 4.1): a later entry may be
|
||
* causally dependent on the one that never actually applied (e.g. a
|
||
* job-state fact for an array element that was never constructed).
|
||
* The PHYSICAL scan (CRC chain, current_lsn, seg_off) below is
|
||
* deliberately NOT gated on this flag -- see the apply_rc < 0 branch's
|
||
* doc comment for why current_lsn/last_crc must stay accurate for the
|
||
* entire on-disk log regardless of the logical apply outcome. */
|
||
bool apply_halted = false;
|
||
|
||
for (int si = 0; si < nseg; si++) {
|
||
char seg_path[4096];
|
||
snprintf(seg_path, sizeof(seg_path),
|
||
WAL_SEG_FMT, p->wal_path, seg_lsns[si]);
|
||
|
||
int fd = open(seg_path, O_RDONLY);
|
||
if (fd < 0) {
|
||
WARN("persist_recover: cannot open segment %s: %s",
|
||
seg_path, strerror(errno));
|
||
continue;
|
||
}
|
||
|
||
uint32_t seg_off = 0;
|
||
|
||
for (;;) {
|
||
/* Read header */
|
||
wal_entry_hdr_t hdr;
|
||
ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)seg_off);
|
||
if (hn == 0)
|
||
break; /* clean end of segment (CLEAN) */
|
||
if (hn < (ssize_t)sizeof(hdr)) {
|
||
/* A partial header (or read error) is a SHORT READ — the log
|
||
* stops mid-entry, so it cannot be reconstructed contiguously
|
||
* (RECOVER_TRUNCATED, Req 2.1). Distinct from clean EOF above. */
|
||
if (hn < 0)
|
||
WARN("persist_recover: read error in %s at offset %u: %s",
|
||
seg_path, seg_off, strerror(errno));
|
||
else
|
||
WARN("persist_recover: short header read in %s at offset %u "
|
||
"(got %zd of %zu bytes); stopping replay",
|
||
seg_path, seg_off, hn, sizeof(hdr));
|
||
if (verdict_out)
|
||
*verdict_out = RECOVER_TRUNCATED;
|
||
break;
|
||
}
|
||
|
||
/* Detect pre-allocated zero region (no valid entry starts with
|
||
* all-zero header: lsn == 0 && type == 0 && payload_len == 0).
|
||
* This is the normal fallocate'd tail after the last valid entry —
|
||
* a clean stop, NOT corruption (stays RECOVER_CLEAN). */
|
||
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
|
||
break;
|
||
|
||
/* CRC chain validation: prev_crc must match last_crc in segment */
|
||
if (seg_off == 0) {
|
||
/* First entry in this segment: prev_crc must be WAL_CRC_NONE
|
||
* (or we accept any value if last_crc is also WAL_CRC_NONE,
|
||
* e.g. across a segment boundary after checkpoint trim). */
|
||
if (hdr.prev_crc != WAL_CRC_NONE && last_crc != WAL_CRC_NONE &&
|
||
hdr.prev_crc != last_crc) {
|
||
WARN("persist_recover: CRC chain break at start of %s "
|
||
"(prev_crc=%08x expected=%08x); stopping replay",
|
||
seg_path, hdr.prev_crc, last_crc);
|
||
if (verdict_out)
|
||
*verdict_out = RECOVER_TRUNCATED;
|
||
break;
|
||
}
|
||
} else {
|
||
if (hdr.prev_crc != last_crc) {
|
||
WARN("persist_recover: CRC chain break in %s at offset %u "
|
||
"(prev_crc=%08x expected=%08x); stopping replay at last good entry",
|
||
seg_path, seg_off, hdr.prev_crc, last_crc);
|
||
if (verdict_out)
|
||
*verdict_out = RECOVER_TRUNCATED;
|
||
break;
|
||
}
|
||
}
|
||
|
||
/* Verify entry CRC */
|
||
uint32_t expected_crc = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
|
||
|
||
uint8_t* payload = NULL;
|
||
if (hdr.payload_len > 0) {
|
||
payload = malloc(hdr.payload_len);
|
||
if (!payload) {
|
||
close(fd);
|
||
return -ENOMEM;
|
||
}
|
||
ssize_t pn = pread(fd, payload, hdr.payload_len,
|
||
(off_t)(seg_off + sizeof(hdr)));
|
||
if (pn != (ssize_t)hdr.payload_len) {
|
||
WARN("persist_recover: short payload read in %s at offset %u",
|
||
seg_path, seg_off);
|
||
if (verdict_out)
|
||
*verdict_out = RECOVER_TRUNCATED;
|
||
free(payload);
|
||
break;
|
||
}
|
||
expected_crc = crc32(expected_crc, (const Bytef*)payload, hdr.payload_len);
|
||
}
|
||
|
||
if (hdr.crc != expected_crc) {
|
||
WARN("persist_recover: CRC mismatch in %s at offset %u "
|
||
"(got=%08x expected=%08x); stopping replay",
|
||
seg_path, seg_off, hdr.crc, expected_crc);
|
||
if (verdict_out)
|
||
*verdict_out = RECOVER_TRUNCATED;
|
||
free(payload);
|
||
break;
|
||
}
|
||
|
||
/* Advance segment-local CRC */
|
||
last_crc = hdr.crc;
|
||
|
||
/* Record this entry's term at its LSN for recovery read-back
|
||
* (ha-term-safety 1.3, R1.5). Record EVERY CRC-validated logical
|
||
* entry — including snapshot-covered ones below from_lsn — so the
|
||
* index spans the full LSN range the reconcile/promotion logic may
|
||
* query. EXCLUDE WAL_CHECKPOINT markers (type == WAL_CHECKPOINT) and
|
||
* unknown future types (type > WAL_CHECKPOINT): they are skipped
|
||
* below and are never logical Raft entries, so the live append path
|
||
* does not record them either (kept symmetric). The raw hdr.term is
|
||
* stored verbatim (legacy 0 included); the legacy-0 -> WAL_LEGACY_TERM
|
||
* mapping is applied only at read-back. */
|
||
if (hdr.type < WAL_CHECKPOINT)
|
||
record_entry_term(p, hdr.lsn, hdr.term);
|
||
|
||
/* Skip entries already covered by the snapshot */
|
||
if (hdr.lsn <= from_lsn) {
|
||
free(payload);
|
||
seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
continue;
|
||
}
|
||
|
||
/* Skip WAL_CHECKPOINT markers (local bookkeeping only) */
|
||
if (hdr.type == WAL_CHECKPOINT) {
|
||
free(payload);
|
||
seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
/* Advance current_lsn past the marker */
|
||
if (hdr.lsn + 1 > p->current_lsn)
|
||
p->current_lsn = hdr.lsn + 1;
|
||
continue;
|
||
}
|
||
|
||
/* Unknown future entry types: log and skip */
|
||
if (hdr.type > WAL_CHECKPOINT) {
|
||
WARN("persist_recover: unknown WAL entry type %u at LSN %lu; skipping",
|
||
hdr.type, (unsigned long)hdr.lsn);
|
||
free(payload);
|
||
seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
continue;
|
||
}
|
||
|
||
/* Logical entity restore via the shared apply seam
|
||
* (raft-apply-decoupling task 2.3, design.md "Apply Seam"): the
|
||
* SAME apply_committed_entry() dispatch the live commit-advance
|
||
* apply pump will use (task 6.1), so a node's materialized state
|
||
* is the deterministic result of its committed log regardless of
|
||
* whether it got there via restart replay or live apply
|
||
* (Req 3.1-3.4). apply_wal_entry() (this replay loop's previous
|
||
* per-entry dispatch) is NOT retired by this change — it remains
|
||
* the live standby catch-up/replicated-apply dispatch used
|
||
* elsewhere (ha.c); it is simply no longer this loop's call. The
|
||
* CRC chain + current_lsn remain owned by this replay loop
|
||
* (current_lsn advanced below; last_crc at line ~1601) — neither
|
||
* dispatch function ever touched them (bug-778).
|
||
*
|
||
* KNOWN HAZARD (tracked in tasks.md Implementation Notes, task
|
||
* 2.3): this replays the FULL post-checkpoint WAL range on every
|
||
* restart (last_applied_lsn is not persisted, deliberately
|
||
* recomputed here), so apply_committed_entry()'s AFTER_EVENT
|
||
* hooks fire again for entries already applied+notified in a
|
||
* prior process life. Confirmed inert today: scheduler hooks
|
||
* aren't registered until a post-boot network JOIN
|
||
* (on_scheduler_join, sched.c), and apiserver hooks — though
|
||
* registered before this runs — gate their notify on a
|
||
* connected endpoint that is empty at first-boot replay time.
|
||
* MUST be re-verified before tasks 4.1/6.1/6.2/6.3 make this
|
||
* path broadly live. */
|
||
int apply_rc = 0;
|
||
if (!apply_halted) {
|
||
apply_rc = apply_committed_entry(p, hdr.type, hdr.lsn,
|
||
payload, hdr.payload_len);
|
||
}
|
||
free(payload);
|
||
|
||
if (apply_halted) {
|
||
/* job-array-single-wal-record task 7.2: an earlier entry in
|
||
* THIS replay pass already fenced (see the apply_rc < 0
|
||
* branch below) -- this entry and every later one is
|
||
* skipped for logical apply, exactly like the
|
||
* WAL_CHECKPOINT/unknown-type skip branches above. Physical
|
||
* bookkeeping (current_lsn/seg_off) still runs
|
||
* unconditionally below. */
|
||
} else if (apply_rc < 0) {
|
||
/* job-array-single-wal-record task 7.2 (design.md "Live
|
||
* Apply Resource Fence", task 4.2's self-flagged residual
|
||
* gap): mirror persist_apply_pump_run()'s halt-and-fence
|
||
* branch exactly. This is never a content-derived failure
|
||
* (Req 9.1-9.3's pre-commit validation already made those
|
||
* impossible post-commit) -- it is the SAME node-local
|
||
* resource-exhaustion class (Req 9.4) the live pump already
|
||
* guards against, just hit during boot/restart replay
|
||
* instead of live apply. Do NOT advance last_applied_lsn
|
||
* past this LSN (skip that store entirely -- last_applied_lsn
|
||
* stays parked at this entry's predecessor) and do NOT tally
|
||
* this entry's job count; engage the SAME
|
||
* persist_mark_apply_resource_fenced() primitive the live
|
||
* path uses -- reused verbatim, no parallel mechanism. This
|
||
* closes task 4.2's documented residual gap: without this, a
|
||
* node that crashes while fenced and re-hits the identical
|
||
* resource-exhaustion condition on restart would silently
|
||
* discard the failure here (as this branch previously did,
|
||
* unconditionally advancing last_applied_lsn regardless of
|
||
* apply_rc's sign) and boot up looking fully promotable
|
||
* while actually divergent/behind.
|
||
*
|
||
* Engaging synchronously HERE, inside this boot-time call
|
||
* (persist_recover(), called synchronously from
|
||
* cluster_init_phase2() before it returns), matters:
|
||
* persist_apply_pump_kick()'s post-recovery catch-up kick
|
||
* dispatches ASYNCHRONOUSLY onto a pool worker vtask (or
|
||
* only runs inline as its own documented fallback) -- relying
|
||
* on that kick alone to eventually re-fail and set the fence
|
||
* would leave a real window, between persist_recover()
|
||
* returning and that async retry actually running, where a
|
||
* promotion-gate check could observe "not fenced" and
|
||
* wrongly treat this node as eligible.
|
||
*
|
||
* KNOWN BOUNDARY (task 7.2 adversarial review, NOT fixed by
|
||
* this task -- ha.c change required, out of this function's
|
||
* scope): this fence, once engaged here, protects every
|
||
* SUBSEQUENT promotion evaluation in this process's life --
|
||
* a later election's raft_cb_become_leader() gate (ha.c:1337
|
||
* reads persist_apply_resource_fenced(), inside the
|
||
* eligibility block at ha.c:1303-1408) and the periodic
|
||
* re-alert timer (promotion_fence_timeout_fn, ha.c ~876)
|
||
* both correctly observe it. It does NOT protect this SAME
|
||
* boot's own first-ever promotion evaluation: on a node's
|
||
* first-ever election evaluation in its process's life (i.e.
|
||
* BEFORE cluster_init_phase2() has ever run), ha.c's
|
||
* raft_cb_become_leader() runs its eligibility gate at
|
||
* ha.c:1303-1408 strictly BEFORE the is_first_boot block at
|
||
* ha.c:1427-1439 that calls cluster_init_phase2() -- which is
|
||
* what calls persist_recover()/replay_wal() (cluster.c:1482)
|
||
* and is what would engage this fence for the first time.
|
||
* There is no re-check of persist_apply_resource_fenced()
|
||
* after cluster_init_phase2() returns; the code proceeds
|
||
* straight to CTRL_PRIMARY. So a fence engaged by THIS SAME
|
||
* first replay is invisible to the one promotion check
|
||
* gating that specific boot -- exactly the "all nodes
|
||
* restart simultaneously" race this whole task exists to
|
||
* guard against. Closing this requires an ha.c change (a
|
||
* re-check of persist_apply_resource_fenced() after
|
||
* cluster_init_phase2() returns, or reordering recovery
|
||
* ahead of the promotion decision), which is out of this
|
||
* spec's current scope -- see tasks.md's task 7.2
|
||
* Implementation Notes for the full finding. */
|
||
ERROR("replay_wal: apply_committed_entry failed for "
|
||
"committed entry lsn=%" PRIu64 " wal_type=%" PRIu32
|
||
" (rc=%d) during boot/restart replay -- halting this "
|
||
"replay's LOGICAL apply at this position (the entry "
|
||
"stays committed and will be retried by the live apply "
|
||
"pump's post-recovery kick, from the same position); "
|
||
"Live Apply Resource Fence ENGAGED "
|
||
"(job-array-single-wal-record Req 9.4/9.5)",
|
||
hdr.lsn, hdr.type, apply_rc);
|
||
persist_mark_apply_resource_fenced(p, hdr.lsn);
|
||
apply_halted = true;
|
||
} else {
|
||
/* raft-apply-decoupling task 5.1: track the replay-derived
|
||
* LOGICAL apply cursor separately from current_lsn (the
|
||
* physical WAL position owned by this loop, bug-778) --
|
||
* last_applied_lsn is the "highest entry actually dispatched
|
||
* via apply_committed_entry()" position the post-replay
|
||
* verification compares against. Segments are scanned in
|
||
* ascending LSN order (sorted above) and entries within a
|
||
* segment are sequential, so a plain store (no monotonic max
|
||
* needed) is correct here. */
|
||
atomic_store_explicit(&p->last_applied_lsn, hdr.lsn,
|
||
memory_order_relaxed);
|
||
|
||
/* Log-derived half of the post-replay apply-correctness
|
||
* expectation (Req 8.1): every WAL_JOB_SUBMIT that reaches
|
||
* this point is, by construction, above the loaded
|
||
* snapshot's checkpoint_lsn (the from_lsn skip above already
|
||
* filtered out anything the snapshot covers) — so it names a
|
||
* job that must now exist in the live job table that did not
|
||
* exist before this replay pass began. A duplicate
|
||
* WAL_JOB_SUBMIT for an already-existing jobid (bug-1419:
|
||
* timeout-then-late-commit retry) is a legitimate no-op —
|
||
* apply_committed_entry() returns 0 for it, not 1 — so it
|
||
* must NOT be tallied here or the post-replay comparison
|
||
* against jobmgr's real growth spuriously fails
|
||
* (APPLY_DIVERGENT). */
|
||
if (hdr.type == WAL_JOB_SUBMIT && new_job_submits_out &&
|
||
apply_rc == 1)
|
||
(*new_job_submits_out)++;
|
||
|
||
/* job-array-single-wal-record task 6.0a: the array analog of
|
||
* the WAL_JOB_SUBMIT tally immediately above.
|
||
* apply_committed_entry()'s WAL_JOB_ARRAY_SUBMIT case
|
||
* (restore_job_array()'s return, generalized to N elements)
|
||
* returns apply_rc == the genuine count of NEWLY created
|
||
* elements (1..array_size) on a real apply, or 0 for an
|
||
* idempotent whole-record re-apply (every index already
|
||
* existed -- restore_job_array() returned -EEXIST, which
|
||
* this same dispatch maps to 0, mirroring WAL_JOB_SUBMIT's
|
||
* own dedup return). Tallying apply_rc (not a flat +1) is
|
||
* required here, unlike the single-job case above, because
|
||
* one WAL_JOB_ARRAY_SUBMIT entry can materialize many jobmgr
|
||
* entries at once; guarding on apply_rc > 0 excludes the
|
||
* idempotent 0 case from double-counting a re-apply, exactly
|
||
* as apply_rc == 1 excludes WAL_JOB_SUBMIT's duplicate-jobid
|
||
* case (bug-1419) above. Without this branch, replay_wal()
|
||
* silently undercounts every array record's real N-element
|
||
* jobmgr growth to 0, and persist_verify_apply_correctness()
|
||
* then reports a false APPLY_DIVERGENT verdict that engages
|
||
* the PERMANENT, non-self-healing
|
||
* persist_recover_loud_fence_if_unrecoverable() fence
|
||
* (live-reproduced task 6.1 soak, 2026-07-15). */
|
||
if (hdr.type == WAL_JOB_ARRAY_SUBMIT && new_job_submits_out &&
|
||
apply_rc > 0)
|
||
(*new_job_submits_out) += (uint64_t)apply_rc;
|
||
|
||
/* bug-1427: worker analog of the WAL_JOB_SUBMIT tally above
|
||
* — apply_rc == 1 means apply_committed_entry()'s
|
||
* WAL_WORKER_CTRL case genuinely created a new worker (a
|
||
* JOIN against a jobmgr slot that didn't exist yet);
|
||
* apply_rc == 0 means an UPDATE/DEAD applied to an
|
||
* already-existing worker (or a no-hook idempotent case
|
||
* already excluded above via the apply_rc < 0 branch and
|
||
* worker_from_capnp's -EEXIST-maps-to-0 path) — neither
|
||
* grows the worker table, so only == 1 is tallied. */
|
||
if (hdr.type == WAL_WORKER_CTRL && new_worker_joins_out &&
|
||
apply_rc == 1)
|
||
(*new_worker_joins_out)++;
|
||
|
||
replayed++;
|
||
}
|
||
|
||
/* Advance current_lsn to one past this entry -- unconditional,
|
||
* regardless of the LOGICAL apply outcome above (mirrors the
|
||
* WAL_CHECKPOINT/unknown-type skip branches earlier in this
|
||
* loop, which also advance current_lsn without ever dispatching
|
||
* through apply_committed_entry()). job-array-single-wal-record
|
||
* task 7.2: the physical WAL scan must account for every
|
||
* on-disk entry so current_lsn/last_crc stay correct for any
|
||
* future append this process makes (as a promoted primary, or
|
||
* as a replication standby receiving new entries) -- halting
|
||
* this advance too (treating a fenced entry as physically
|
||
* absent) would understate the true on-disk tail and risk a
|
||
* later append colliding with/overwriting already-durable
|
||
* entries beyond the fenced position (persist_recover()'s own
|
||
* "Round-2 fix" comment above, for the durable_lsn/apply_cursor
|
||
* split, already establishes that current_lsn and
|
||
* last_applied_lsn are expected to diverge -- this is the same
|
||
* principle, applied to a halted apply instead of a skipped
|
||
* checkpoint marker). */
|
||
if (hdr.lsn + 1 > p->current_lsn)
|
||
p->current_lsn = hdr.lsn + 1;
|
||
|
||
seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
}
|
||
|
||
close(fd);
|
||
}
|
||
|
||
/* Update last_crc so the next WAL write chains correctly */
|
||
pthread_mutex_lock(&p->lock);
|
||
p->last_crc = last_crc;
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
return replayed;
|
||
}
|
||
|
||
/* raft-apply-decoupling task 5.1 / bug-1427: cheap read-lock count of a
|
||
* manager's live entries, mirroring collect_job_frames' iteration idiom
|
||
* (this file, above) without collecting or filtering anything. `mgr` is an
|
||
* rtti `var` handle (c->jobmgr or c->workermgr) — lock/iter_* are generic
|
||
* over the stored element type, so one counter serves both tables. Used only
|
||
* to derive the post-replay materialized-entity expectation (never on a hot
|
||
* path). Safe on a NULL mgr — returns 0. */
|
||
static uint64_t count_manager_entries(var mgr)
|
||
{
|
||
if (!mgr)
|
||
return 0;
|
||
|
||
lock(mgr, RTTI_LOCK_READ);
|
||
|
||
iterator_t it;
|
||
int irc = iter_make(&it, mgr);
|
||
if (irc < 0) {
|
||
unlock(mgr);
|
||
return 0; /* empty table */
|
||
}
|
||
|
||
uint64_t n = 0;
|
||
for (;;) {
|
||
void* e = NULL;
|
||
int nrc = iter_next(&it, &e);
|
||
if (nrc < 0 || nrc == RTTI_ITER_NO_DATA)
|
||
break;
|
||
if (e == NULL)
|
||
continue;
|
||
n++;
|
||
}
|
||
|
||
iter_destroy(&it);
|
||
unlock(mgr);
|
||
return n;
|
||
}
|
||
|
||
static uint64_t count_jobmgr_entries(cluster_t* c)
|
||
{
|
||
return (!c) ? 0 : count_manager_entries(c->jobmgr);
|
||
}
|
||
|
||
/* bug-1427: worker analog of count_jobmgr_entries above — same NULL-safety
|
||
* contract, over c->workermgr instead of c->jobmgr. */
|
||
static uint64_t count_workermgr_entries(cluster_t* c)
|
||
{
|
||
return (!c) ? 0 : count_manager_entries(c->workermgr);
|
||
}
|
||
|
||
/* persist_verify_apply_correctness — see persist.h for the full contract
|
||
* doc (raft-apply-decoupling task 5.1, Req 8.1/8.2; worker half bug-1427).
|
||
* Pure comparison: no logging, no fence side effects. */
|
||
apply_verdict_t persist_verify_apply_correctness(
|
||
persist_t* p, const apply_expectation_t* exp,
|
||
uint64_t jobmgr_count_before_replay,
|
||
uint64_t workermgr_count_before_replay)
|
||
{
|
||
if (!p || !exp)
|
||
return APPLY_CLEAN; /* defensive: nothing to verify */
|
||
|
||
uint64_t actual_last_applied =
|
||
atomic_load_explicit(&p->last_applied_lsn, memory_order_acquire);
|
||
if (actual_last_applied != exp->expected_last_applied_lsn)
|
||
return APPLY_DIVERGENT;
|
||
|
||
/* Entity-materialization half of the check: skip gracefully when there
|
||
* is no live cluster to query — mirrors apply_committed_entry()'s own
|
||
* "nothing to dispatch into" convention for a bare unit-test persist_t.
|
||
* Only the position check above applies in that case. */
|
||
if (!p->cluster || !p->cluster->jobmgr)
|
||
return APPLY_CLEAN;
|
||
|
||
uint64_t jobmgr_count_after = count_jobmgr_entries(p->cluster);
|
||
if (jobmgr_count_after < jobmgr_count_before_replay)
|
||
return APPLY_DIVERGENT; /* table shrank — impossible on a clean replay */
|
||
|
||
uint64_t actual_new_jobs = jobmgr_count_after - jobmgr_count_before_replay;
|
||
if (actual_new_jobs != exp->expected_new_job_count)
|
||
return APPLY_DIVERGENT;
|
||
|
||
/* bug-1427: worker-count check, mirroring the job-count check above
|
||
* exactly. Independently skipped (not just via the shared p->cluster
|
||
* check above) when workermgr itself is unset, mirroring the jobmgr
|
||
* NULL-safety convention. */
|
||
if (!p->cluster->workermgr)
|
||
return APPLY_CLEAN;
|
||
|
||
uint64_t workermgr_count_after = count_workermgr_entries(p->cluster);
|
||
if (workermgr_count_after < workermgr_count_before_replay)
|
||
return APPLY_DIVERGENT; /* table shrank — impossible on a clean replay */
|
||
|
||
uint64_t actual_new_workers =
|
||
workermgr_count_after - workermgr_count_before_replay;
|
||
if (actual_new_workers != exp->expected_new_worker_count)
|
||
return APPLY_DIVERGENT;
|
||
|
||
return APPLY_CLEAN;
|
||
}
|
||
|
||
apply_verdict_t persist_apply_verdict(persist_t* p)
|
||
{
|
||
return p ? p->apply_verdict : APPLY_CLEAN;
|
||
}
|
||
|
||
uint64_t persist_last_applied_lsn(persist_t* p)
|
||
{
|
||
return p ? atomic_load_explicit(&p->last_applied_lsn, memory_order_acquire)
|
||
: 0;
|
||
}
|
||
|
||
/* bug-1416 (raft-apply-decoupling task 7.2 round 2) -- see persist.h's
|
||
* doc comment on the recovery_complete field for the full contract. */
|
||
bool persist_recovery_complete(persist_t* p)
|
||
{
|
||
return !p || atomic_load_explicit(&p->recovery_complete,
|
||
memory_order_acquire);
|
||
}
|
||
|
||
/* bug-1416 round 2 -- see persist.h's doc comment on this function for the
|
||
* full rationale (closes the gap between persist_init() and the FIRST
|
||
* persist_recover() call, which persist_recover()'s own false/true bracket
|
||
* cannot see). */
|
||
void persist_mark_recovery_pending(persist_t* p)
|
||
{
|
||
if (!p)
|
||
return;
|
||
atomic_store_explicit(&p->recovery_complete, false, memory_order_release);
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Public: persist_recover */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
int persist_recover(persist_t* p)
|
||
{
|
||
/* bug-1416 (raft-apply-decoupling task 7.2 round 2): close the boot-time
|
||
* race window for the FULL duration of this call, not just the position
|
||
* check inside persist_verify_apply_correctness -- see the
|
||
* recovery_complete field's doc comment (persist.h) for the complete
|
||
* rationale. Held false until the natural (success) return at the
|
||
* bottom of this function, which restores it and issues one deliberate
|
||
* catch-up kick. The two hard-I/O-error early returns below
|
||
* (load_snapshot / replay_wal failure) deliberately do NOT restore it:
|
||
* those are pre-existing, rare disk-I/O-level failures the caller
|
||
* already treats as fatal to this boot attempt (cluster_init_phase2
|
||
* tears down persist_t on a nonzero return) -- leaving the gate closed
|
||
* is the conservative, fail-static choice (never let the live apply
|
||
* pump or live-replication path mutate state on top of a persist_t
|
||
* whose own recovery never finished), not a new hazard. */
|
||
atomic_store_explicit(&p->recovery_complete, false, memory_order_release);
|
||
|
||
/* raft-apply-decoupling task 7.4 perf-fix (round 2): invalidate the apply
|
||
* pump's segment-lookup cache before replay -- the cached fd/offset are
|
||
* only valid relative to the pre-recovery on-disk state, and replay below
|
||
* can change checkpoint_lsn/current_lsn. Doing this first covers every
|
||
* return path below (including the early I/O-failure returns), so the
|
||
* first post-recovery pump kick always falls back to full segment
|
||
* rediscovery from the correct, recovered position -- never trusting a
|
||
* stale pre-recovery cursor. Wrapped in apply_pump_lock (matching
|
||
* persist_truncate_wal_above's identical invalidation site below) so the
|
||
* cache fields' documented invariant -- "guarded ENTIRELY by
|
||
* apply_pump_lock, never p->lock" (persist.h) -- holds unconditionally
|
||
* here too, rather than resting on the implicit fact that no live pump
|
||
* walk can be racing this boot-time call. No caller of persist_recover()
|
||
* holds apply_pump_lock (its sole production call site, cluster.c boot
|
||
* sequence, runs before the pump is ever kicked), so this cannot
|
||
* deadlock. */
|
||
pthread_mutex_lock(&p->apply_pump_lock);
|
||
if (p->apply_pump_cache_fd >= 0) {
|
||
close(p->apply_pump_cache_fd);
|
||
p->apply_pump_cache_fd = -1;
|
||
}
|
||
pthread_mutex_unlock(&p->apply_pump_lock);
|
||
|
||
/* --- Step 1: find latest snapshot (highest LSN filename) --- */
|
||
char snap_file[4096] = "";
|
||
uint64_t snap_lsn = 0;
|
||
|
||
DIR* d = opendir(p->snap_path);
|
||
if (d) {
|
||
struct dirent* e;
|
||
while ((e = readdir(d)) != NULL) {
|
||
size_t len = strlen(e->d_name);
|
||
if (len < 6 || strcmp(e->d_name + len - 6, ".capnp") != 0)
|
||
continue;
|
||
uint64_t lsn = 0;
|
||
if (sscanf(e->d_name, "snap-%" SCNx64 ".capnp", &lsn) != 1)
|
||
continue;
|
||
if (lsn > snap_lsn) {
|
||
snap_lsn = lsn;
|
||
snprintf(snap_file, sizeof(snap_file),
|
||
"%s/%s", p->snap_path, e->d_name);
|
||
}
|
||
}
|
||
closedir(d);
|
||
}
|
||
|
||
/* --- Step 2: load snapshot (if any) --- */
|
||
if (snap_lsn > 0 && snap_file[0] != '\0') {
|
||
/* Local recovery doesn't need the entities-restored indicator to
|
||
* decide behavior: the WAL replay in Step 3 applies everything
|
||
* above checkpoint_lsn regardless, unchanged by ha-entity-snapshots.
|
||
* It IS consumed for observability parity with the wire receive
|
||
* path (ha_handle_snapshot_chunk, task 6.1) — a rolling-upgrade
|
||
* restart against a pre-feature (legacy, position-only) local
|
||
* checkpoint is an expected rollout state, not an error. */
|
||
bool entities_restored = false;
|
||
int rc = load_snapshot(p, snap_file, snap_lsn, &entities_restored);
|
||
if (rc != 0) {
|
||
ERROR("persist_recover: failed to load snapshot %s: %s",
|
||
snap_file, strerror(-rc));
|
||
return rc;
|
||
}
|
||
/* checkpoint_lsn and current_lsn set by load_snapshot */
|
||
if (entities_restored) {
|
||
INFO("persist_recover: snapshot %s loaded lsn=%" PRIu64
|
||
" with full job/queue entity restoration", snap_file,
|
||
p->checkpoint_lsn);
|
||
} else {
|
||
INFO("persist_recover: snapshot %s lsn=%" PRIu64 " is a legacy "
|
||
"position-only checkpoint (no entity content) — continuing "
|
||
"via normal WAL replay", snap_file, p->checkpoint_lsn);
|
||
}
|
||
}
|
||
|
||
/* raft-apply-decoupling task 5.1 (Req 8.1): seed the replay-derived
|
||
* apply cursor from the loaded baseline (checkpoint_lsn stays 0 when no
|
||
* snapshot was found) so that, on a CLEAN replay with no new entries
|
||
* beyond the snapshot, the cursor already sits at the recovered tip —
|
||
* mirroring how current_lsn is seeded by load_snapshot. Step 3's
|
||
* replay_wal advances it further for every entry actually dispatched
|
||
* above this baseline. Also capture the live job table's size BEFORE
|
||
* replay runs: the post-replay verification below needs this baseline to
|
||
* isolate the jobs replay itself materializes from anything the snapshot
|
||
* already restored. */
|
||
atomic_store_explicit(&p->last_applied_lsn, p->checkpoint_lsn,
|
||
memory_order_release);
|
||
uint64_t jobmgr_count_before_replay = count_jobmgr_entries(p->cluster);
|
||
uint64_t workermgr_count_before_replay =
|
||
count_workermgr_entries(p->cluster); /* bug-1427 */
|
||
|
||
/* --- Step 3: replay WAL entries after checkpoint_lsn --- */
|
||
/* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1):
|
||
* replay distinguishes a clean log end from a corruption-truncated end.
|
||
* We CLASSIFY + record the verdict here; we do NOT FATAL (task 3.3) and do
|
||
* NOT fence promotion (task 4.3 consumes p->recovery_verdict). A clean
|
||
* replay over a loaded snapshot-sentinel (no entries below the sentinel) is
|
||
* CLEAN — the sentinel established the committed position; replay only adds
|
||
* entries above it. */
|
||
recover_verdict_t verdict = RECOVER_CLEAN;
|
||
uint64_t new_job_submits = 0;
|
||
uint64_t new_worker_joins = 0; /* bug-1427 */
|
||
int replayed = replay_wal(p, p->checkpoint_lsn, &verdict, &new_job_submits,
|
||
&new_worker_joins);
|
||
if (replayed < 0)
|
||
return replayed;
|
||
|
||
p->recovery_verdict = verdict;
|
||
if (verdict == RECOVER_TRUNCATED)
|
||
WARN("persist_recover: RECOVER_TRUNCATED — local WAL is structurally "
|
||
"corrupt (CRC break / mismatch / short read); member marked "
|
||
"recovery-truncated (non-promotable until caught up)");
|
||
|
||
/* --- Step 3b: re-derive the durable floor from the recovered WAL tip
|
||
* (ha-commit-durability task 3.4, Req 2.3) ---
|
||
*
|
||
* Publish the member's durable floor (the durable last-log index consumed
|
||
* by the Raft freshness gate, task 4.2) as the highest INTACT replayed LSN.
|
||
* replay_wal advanced current_lsn to one past the last intact entry — incl.
|
||
* stopping at a corruption boundary on TRUNCATED — so current_lsn - 1 is
|
||
* exactly that intact WAL tip, and a snapshot-sentinel load leaves it at the
|
||
* sentinel/replayed tip. We store it into durable_lsn because the recovered
|
||
* on-disk entries are GENUINELY durable (each was CRC-validated and read
|
||
* from stable storage by replay_wal), so the freshness advertisement is now
|
||
* backed by durable state rather than a volatile in-memory position.
|
||
*
|
||
* Crux (Req 2.3 — a truncated member cannot over-advertise): this is an
|
||
* ASSIGN to the recovered tip, NOT a max() with whatever durable_lsn held.
|
||
* A fresh recover starts from durable_lsn=0 (persist_init), so on a
|
||
* truncating restart the floor lands at the LOWER intact tip — never a
|
||
* stale higher in-memory value and never the commit position
|
||
* (committed_lsn). The recovered tip is, by construction, <= current_lsn-1,
|
||
* preserving the durable_lsn <= current_lsn-1 invariant. The post-election
|
||
* safe-commit point continues to come from voter replies, not this floor
|
||
* (CLAUDE.md). */
|
||
uint64_t recovered_tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
|
||
atomic_store_explicit(&p->durable_lsn, recovered_tip, memory_order_release);
|
||
|
||
/* raft-apply-decoupling task 5.1 (Req 8.1/8.2): post-replay apply-
|
||
* correctness verification, reported ALONGSIDE the structural
|
||
* recovery_verdict set above — neither implies the other. See
|
||
* persist_verify_apply_correctness's doc comment for exactly what is
|
||
* compared. Routing an APPLY_DIVERGENT verdict into the existing
|
||
* non-promotable loud-fence mechanism is task 5.2; this only classifies
|
||
* and logs.
|
||
*
|
||
* Round-2 fix: the expected position MUST be the LOGICAL apply cursor
|
||
* replay_wal() already maintains in p->last_applied_lsn — NOT
|
||
* recovered_tip (current_lsn - 1, the PHYSICAL WAL-scan tip used above
|
||
* for durable_lsn). current_lsn advances for every WAL entry replay_wal
|
||
* scans, including WAL_CHECKPOINT markers and unknown future types,
|
||
* neither of which is ever routed through apply_committed_entry() (see
|
||
* the skip branches in replay_wal). last_applied_lsn only advances when
|
||
* a real dispatch happens. If the highest-LSN entry above checkpoint_lsn
|
||
* is a trailing WAL_CHECKPOINT marker — the exact state left behind by a
|
||
* crash between persist_write_checkpoint_marker() (step 1, durably
|
||
* fsynced) and write_snapshot() (step 3b, persists the advanced
|
||
* checkpoint_lsn) — recovered_tip over-counts past it while
|
||
* last_applied_lsn correctly does not, and comparing against
|
||
* recovered_tip previously produced a false APPLY_DIVERGENT for a
|
||
* perfectly clean recovery (violates Req 8.2). */
|
||
uint64_t apply_cursor =
|
||
atomic_load_explicit(&p->last_applied_lsn, memory_order_acquire);
|
||
apply_expectation_t apply_exp = {
|
||
.expected_last_applied_lsn = apply_cursor,
|
||
.expected_new_job_count = new_job_submits,
|
||
.expected_new_worker_count = new_worker_joins, /* bug-1427 */
|
||
};
|
||
p->apply_verdict = persist_verify_apply_correctness(
|
||
p, &apply_exp, jobmgr_count_before_replay, workermgr_count_before_replay);
|
||
if (p->apply_verdict == APPLY_DIVERGENT) {
|
||
ERROR("persist_recover: APPLY_DIVERGENT — materialized state does "
|
||
"not match the committed log tip (expected last_applied_lsn=%"
|
||
PRIu64 " new_job_count=%" PRIu64 " new_worker_count=%" PRIu64
|
||
"); a hidden apply defect is present (ha-commit-durability "
|
||
"recover_verdict=%s)",
|
||
apply_exp.expected_last_applied_lsn,
|
||
apply_exp.expected_new_job_count,
|
||
apply_exp.expected_new_worker_count,
|
||
(verdict == RECOVER_TRUNCATED) ? "RECOVER_TRUNCATED"
|
||
: "RECOVER_CLEAN");
|
||
|
||
/* raft-apply-decoupling task 5.2 (Req 8.3/8.4): route the DIVERGENT
|
||
* verdict through the EXISTING non-promotable loud-fence mechanism
|
||
* (persist_recover_loud_fence_if_unrecoverable + the sticky
|
||
* recovery_unrecoverable flag) instead of a new fence path.
|
||
* apply_divergent=true fences UNCONDITIONALLY (see
|
||
* persist_recover_decision), so the peer/HWM fields below are
|
||
* inert placeholders here -- they are not yet known at this
|
||
* boot-time replay call site (unlike the structural verdict's own
|
||
* fence check, which stays deferred to promotion time in ha.c,
|
||
* where live peer/HWM context exists). verdict/recovered_tip are
|
||
* passed through only for accurate, consistent logging. The member
|
||
* stays ALIVE, non-promotable, and keeps retrying catch-up -- it
|
||
* never exits (controller-never-self-terminate invariant,
|
||
* CLAUDE.md). */
|
||
recover_policy_inputs_t apply_fence_in = {
|
||
.verdict = verdict,
|
||
.durable_below_hwm = false,
|
||
.healthy_peer_reachable = false,
|
||
.catch_up_succeeded = false,
|
||
.durable_floor_lsn = recovered_tip,
|
||
.cluster_hwm_lsn = 0,
|
||
.apply_divergent = true,
|
||
};
|
||
persist_recover_loud_fence_if_unrecoverable(p, &apply_fence_in);
|
||
} else
|
||
INFO("persist_recover: APPLY_CLEAN — materialized state matches the "
|
||
"committed log tip (last_applied_lsn=%" PRIu64 ")",
|
||
apply_cursor);
|
||
|
||
/* --- Step 4: retention pass (TTL check) — no-op in P0 (no entities) --- */
|
||
|
||
/* bug-1416: replay + verification is fully complete -- safe to let the
|
||
* live apply pump and the standby live-replication apply path resume
|
||
* mutating state (see the recovery_complete field's doc comment).
|
||
* Restored unconditionally here regardless of the verdict computed
|
||
* above: a genuine APPLY_DIVERGENT already fenced this member via
|
||
* persist_recover_loud_fence_if_unrecoverable (non-promotable, stays
|
||
* alive, keeps retrying) -- that sticky fence and its clearing logic
|
||
* (task 5.3) are untouched by this flag, which governs ONLY whether
|
||
* in-memory apply may proceed, not whether this member may lead. Issue
|
||
* one deliberate catch-up kick for anything that committed locally (or
|
||
* arrived via live replication and was durably persisted but had its
|
||
* entity-apply deferred, see ha_apply_single_entry / ha_apply_term_
|
||
* entry_via_library) while this pass ran -- persist_apply_pump_run's
|
||
* own walk drains fully from last_applied_lsn+1 through the current
|
||
* committed_lsn, so this is a complete, ordered catch-up, not a partial
|
||
* one; a harmless no-op if nothing queued. */
|
||
atomic_store_explicit(&p->recovery_complete, true, memory_order_release);
|
||
/* n_vcpus==0 is this codebase's own established "not yet initialized"
|
||
* sentinel for vtaskmgr_t (mirrors cluster_init_phase2's own
|
||
* `if (c->vtaskmgr.n_vcpus == 0) vtaskmgr_init(...)` guard) -- a real
|
||
* cluster always has n_vcpus > 0 by the time persist_recover() runs
|
||
* (vtaskmgr_init happens earlier in myapp_run()/phase2), while a bare
|
||
* recovery-focused unit-test cluster_t (none of which call
|
||
* vtaskmgr_init, confirmed by inspection) leaves it zeroed. Passing a
|
||
* pointer to an uninitialized vtaskmgr_t into vtask_enq_classified
|
||
* would be unsafe, so fall back to persist_apply_pump_kick's own
|
||
* existing, already-exercised NULL-pool inline path in that case --
|
||
* identical to how every apply-pump test fixture in this spec already
|
||
* exercises the pump. */
|
||
void* pool = (p->cluster && p->cluster->vtaskmgr.n_vcpus > 0)
|
||
? &p->cluster->vtaskmgr
|
||
: NULL;
|
||
persist_apply_pump_kick(p, pool);
|
||
|
||
return 0;
|
||
}
|
||
|
||
/* Recovery-integrity accessors (ha-commit-durability task 3.1, Req 2.1). */
|
||
recover_verdict_t persist_recovery_verdict(persist_t* p)
|
||
{
|
||
return p ? p->recovery_verdict : RECOVER_CLEAN;
|
||
}
|
||
|
||
bool persist_recovery_truncated(persist_t* p)
|
||
{
|
||
return p && p->recovery_verdict == RECOVER_TRUNCATED;
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ *
|
||
* Boot-policy loud-fence decision (ha-commit-durability task 3.3, Req 2.4;
|
||
* never-self-exit rework 2026-06-18).
|
||
*
|
||
* Implements the FENCE_UNRECOVERABLE terminal of the Boot Policy Decision Table
|
||
* (design.md). The decision is a PURE function of its inputs; the action (loud
|
||
* operator-visible log + set the sticky non-promotable flag) is separated into
|
||
* persist_recover_loud_fence_if_unrecoverable so the decision can be unit-tested
|
||
* in isolation. A reachable healthy peer NEVER yields FENCE_UNRECOVERABLE
|
||
* (rows 3, 5); it is reserved for a member that cannot recover locally AND
|
||
* cannot catch up (no healthy peer, rows 4, 4b).
|
||
*
|
||
* CRITICAL: this terminal is a LOUD, PERMANENT, NON-promotable fence — the
|
||
* process STAYS ALIVE and keeps retrying catch-up. There is NO abort()/exit():
|
||
* a controller MUST NEVER self-terminate as a designed response to an
|
||
* operational condition (user invariant, 2026-06-18). A self-exited controller
|
||
* needs manual admin recovery and degrades quorum meanwhile; one that stays
|
||
* alive and keeps retrying self-heals when a peer returns (Req 2.6).
|
||
* ------------------------------------------------------------------ */
|
||
|
||
recover_decision_t persist_recover_decision(const recover_policy_inputs_t* in)
|
||
{
|
||
if (in == NULL)
|
||
return RECOVER_DECISION_ELIGIBLE; /* defensive: nothing to fence */
|
||
|
||
/* raft-apply-decoupling task 5.2 (Req 8.3/8.4): an APPLY_DIVERGENT
|
||
* post-replay verdict is a NEW INPUT to this same decision table, not a
|
||
* new fence mechanism (design.md "Restart Replay & Verification"). It
|
||
* fences UNCONDITIONALLY -- every other field below is irrelevant once
|
||
* this is true. Unlike a structurally truncated/behind log (rows 2/3/5),
|
||
* where a healthy peer's missing entries can repair the gap and legitimately
|
||
* downgrade the outcome to a plain FENCE-then-catch-up, an apply-time
|
||
* defect means the LOCAL apply path itself produced the wrong
|
||
* materialized state from bytes it already had -- streaming more entries
|
||
* from a peer does not prove that path is now trustworthy, so this is
|
||
* never downgraded by peer reachability or catch-up success. */
|
||
if (in->apply_divergent)
|
||
return RECOVER_DECISION_FENCE_UNRECOVERABLE;
|
||
|
||
bool truncated = (in->verdict == RECOVER_TRUNCATED);
|
||
|
||
/* Rows 1 & 6: a structurally CLEAN log that is NOT behind the cluster
|
||
* high-water mark is a normal promotable member — no fence. */
|
||
if (!truncated && !in->durable_below_hwm)
|
||
return RECOVER_DECISION_ELIGIBLE;
|
||
|
||
/* From here the member is truncated OR behind the HWM — it must NOT lead
|
||
* as-is. A truncated log is by definition not provably at/above the HWM
|
||
* (design.md note), so it is treated as behind for fence purposes. */
|
||
|
||
/* Rows 2, 3, 5: a healthy peer is reachable. Catch up from it and regain
|
||
* eligibility (task 4.3). A reachable peer NEVER yields the unrecoverable
|
||
* fence — even for a truncated log or a legitimate snapshot-sentinel. If
|
||
* catch-up has already succeeded the member is eligible; otherwise it stays
|
||
* fenced (4.3 drives the catch-up + release). */
|
||
if (in->healthy_peer_reachable) {
|
||
if (in->catch_up_succeeded)
|
||
return RECOVER_DECISION_ELIGIBLE;
|
||
return RECOVER_DECISION_FENCE;
|
||
}
|
||
|
||
/* No healthy peer. If catch-up did not reach a committed-consistent state,
|
||
* the member can neither recover locally nor catch up right now — rows 4 &
|
||
* 4b: a LOUD, PERMANENT, NON-promotable fence (NOT process exit). It stays
|
||
* alive and keeps retrying; the fence releases when a peer returns
|
||
* (Req 2.4/2.6). */
|
||
if (!in->catch_up_succeeded)
|
||
return RECOVER_DECISION_FENCE_UNRECOVERABLE;
|
||
|
||
/* No peer but catch-up is reported succeeded (e.g. already at/above after a
|
||
* local catch-up): eligible. */
|
||
return RECOVER_DECISION_ELIGIBLE;
|
||
}
|
||
|
||
bool persist_recovery_unrecoverable(persist_t* p)
|
||
{
|
||
return p && atomic_load_explicit(&p->recovery_unrecoverable,
|
||
memory_order_acquire);
|
||
}
|
||
|
||
void persist_clear_recovery_unrecoverable(persist_t* p)
|
||
{
|
||
if (!p)
|
||
return;
|
||
/* Clear only on a genuine transition so the loud "fence released" log fires
|
||
* once, not on every catch-up tick. */
|
||
if (atomic_exchange_explicit(&p->recovery_unrecoverable, false,
|
||
memory_order_acq_rel)) {
|
||
INFO("persist_recover: unrecoverable fence RELEASED — member reached a "
|
||
"committed-consistent state via catch-up; promotable again "
|
||
"(ha-commit-durability Req 2.6)");
|
||
}
|
||
}
|
||
|
||
/* job-array-single-wal-record task 4.2 (design.md "Live Apply Resource
|
||
* Fence", Req 9.4/9.5). See the apply_resource_fenced_lsn field's doc
|
||
* comment (persist.h) and this trio's own doc comment (persist.h) for the
|
||
* full rationale and the deliberate distinction from
|
||
* persist_recovery_unrecoverable/persist_clear_recovery_unrecoverable just
|
||
* above -- that boot-only mechanism is NOT reused here on purpose. */
|
||
bool persist_apply_resource_fenced(persist_t* p)
|
||
{
|
||
return p && atomic_load_explicit(&p->apply_resource_fenced_lsn,
|
||
memory_order_acquire) != 0;
|
||
}
|
||
|
||
void persist_mark_apply_resource_fenced(persist_t* p, uint64_t lsn)
|
||
{
|
||
if (!p)
|
||
return;
|
||
atomic_store_explicit(&p->apply_resource_fenced_lsn, lsn,
|
||
memory_order_release);
|
||
}
|
||
|
||
void persist_clear_apply_resource_fence(persist_t* p)
|
||
{
|
||
if (!p)
|
||
return;
|
||
/* Clear only on a genuine transition so the self-heal log fires once per
|
||
* episode, not on every ordinary successful apply (the overwhelming
|
||
* common case: nothing was fenced, so this is a cheap no-op). */
|
||
uint64_t prev = atomic_exchange_explicit(&p->apply_resource_fenced_lsn, 0,
|
||
memory_order_acq_rel);
|
||
if (prev != 0) {
|
||
INFO("persist_apply_pump_run: Live Apply Resource Fence RELEASED — "
|
||
"lsn=%" PRIu64 " applied successfully on retry; promotable "
|
||
"again (job-array-single-wal-record Req 9.5)",
|
||
prev);
|
||
}
|
||
}
|
||
|
||
recover_decision_t
|
||
persist_recover_loud_fence_if_unrecoverable(persist_t* p,
|
||
const recover_policy_inputs_t* in)
|
||
{
|
||
recover_decision_t decision = persist_recover_decision(in);
|
||
|
||
if (decision != RECOVER_DECISION_FENCE_UNRECOVERABLE)
|
||
return decision;
|
||
|
||
/* Loud, operator-visible alert. The two triggers reaching this terminal
|
||
* are NOT logged with the same clause (round-2 fix, raft-apply-
|
||
* decoupling task 5.2 remediation):
|
||
*
|
||
* - apply_divergent: fences UNCONDITIONALLY (persist_recover_decision),
|
||
* evaluated at boot-replay time BEFORE peer/HWM context exists (see
|
||
* the call site in persist_recover()). `healthy_peer_reachable` and
|
||
* `cluster_hwm_lsn` are inert placeholders there, not real facts —
|
||
* asserting "no healthy peer reachable" or a durable_floor_lsn < hwm
|
||
* comparison for this trigger would report unevaluated-or-false
|
||
* information to the operator (round-1 review: the placeholders can
|
||
* make the comparison numerically impossible, e.g. "2 < 0"). This
|
||
* branch names the real reason (apply-correctness divergence) only.
|
||
* - structural (truncated / behind-HWM): reaches this terminal only
|
||
* via the pre-existing "no healthy peer reachable" branch in
|
||
* persist_recover_decision, so the peer-reachability and
|
||
* durable_floor_lsn/hwm comparison ARE guaranteed true here — that
|
||
* clause is unchanged from before this task.
|
||
*/
|
||
if (in->apply_divergent) {
|
||
ERROR("persist_recover: UNRECOVERABLE — post-replay apply-correctness "
|
||
"verification is DIVERGENT (materialized state does not match "
|
||
"the committed log tip, raft-apply-decoupling Req 8.3); a local "
|
||
"apply defect cannot be proven fixed by peer reachability or "
|
||
"catch-up alone. Staying ALIVE and PERMANENTLY non-promotable "
|
||
"(refusing to serve divergent state) and RETRYING catch-up "
|
||
"indefinitely — operator attention is needed to diagnose the "
|
||
"apply path (raft-apply-decoupling Req 8.3/8.4)");
|
||
} else {
|
||
const char* why = (in->verdict == RECOVER_TRUNCATED)
|
||
? "recovery-truncated log (structural corruption)"
|
||
: "log behind the cluster committed high-water mark";
|
||
ERROR("persist_recover: UNRECOVERABLE — %s and no healthy peer reachable to "
|
||
"catch up; truncation boundary durable_floor_lsn=%" PRIu64
|
||
" < cluster committed high-water hwm=%" PRIu64 ". Staying ALIVE and "
|
||
"PERMANENTLY non-promotable (refusing to serve divergent state) and "
|
||
"RETRYING catch-up indefinitely — will self-heal when a healthy peer "
|
||
"returns; operator attention may be needed "
|
||
"(ha-commit-durability Req 2.4)",
|
||
why, in->durable_floor_lsn, in->cluster_hwm_lsn);
|
||
}
|
||
|
||
/* Set the sticky non-promotable flag so the promotion gate refuses
|
||
* leadership as belt-and-suspenders until catch-up clears it (Req 2.4/2.6).
|
||
* p may be NULL for pure-classification callers that only want the loud log. */
|
||
if (p != NULL)
|
||
atomic_store_explicit(&p->recovery_unrecoverable, true,
|
||
memory_order_release);
|
||
|
||
return RECOVER_DECISION_FENCE_UNRECOVERABLE;
|
||
}
|
||
|
||
/* Advance committed_lsn to `written_lsn` on a majority ACK — but ONLY if that
|
||
* entry is already durable on local stable storage (ha-commit-durability task
|
||
* 2.4, Req 1.1/1.2). This is the legacy (engine-less / LSN-0) sibling of the
|
||
* commit engine's durable self-cap (commit.c majority_candidate_locked): a
|
||
* majority ACK no longer implies the local copy is fsync'd, because group-
|
||
* commit (task 2.1) deferred the append fsync to the demand-driven flusher. So
|
||
* advancing committed_lsn here on the ACK alone would reintroduce the bug-661
|
||
* ACK-without-durability hazard — committed could be durable on a single
|
||
* member and lost across failover. We gate the store on written_lsn <=
|
||
* durable_lsn so committed_lsn ⇒ durable-on-this-member on every path.
|
||
* committed_lsn is monotonic, so a max() store; the durable cap can only delay
|
||
* the advance until the flusher syncs the entry (the durable-advance commit
|
||
* wake re-drives the engine path; the legacy path is re-evaluated on the next
|
||
* sync_append). */
|
||
static void sync_append_legacy_commit_store(persist_t* p, uint64_t written_lsn)
|
||
{
|
||
uint64_t durable = atomic_load_explicit(&p->durable_lsn,
|
||
memory_order_acquire);
|
||
/* DURABILITY PRECONDITION: never advance committed past what is durable
|
||
* locally. The LSN-0 first-entry edge stays valid (0 <= 0). */
|
||
if (written_lsn > durable)
|
||
return;
|
||
uint64_t cur = atomic_load_explicit(&p->committed_lsn,
|
||
memory_order_acquire);
|
||
if (written_lsn <= cur)
|
||
return; /* monotonic: committed_lsn never regresses */
|
||
atomic_store_explicit(&p->committed_lsn, written_lsn, memory_order_release);
|
||
#ifdef XCE_TESTING
|
||
/* INVARIANT (debug guard): committed_lsn must never exceed durable_lsn on
|
||
* any commit advance. A future caller reintroducing ACK-without-durability
|
||
* trips this. */
|
||
assert(atomic_load_explicit(&p->committed_lsn, memory_order_acquire)
|
||
<= atomic_load_explicit(&p->durable_lsn, memory_order_acquire));
|
||
#endif
|
||
}
|
||
|
||
int persist_wal_sync_append(persist_t* p, uint32_t type,
|
||
const void* payload, uint32_t len,
|
||
int* sync_rc_out)
|
||
{
|
||
/* ha-term-safety task 3.1: the primary's WAL_JOB_SUBMIT append is routed
|
||
* through the raft library (ha_primary_log_append → raft_log_append) so the
|
||
* library stamps the term + assigns the LSN (firing on_persist_entry → the
|
||
* WAL write), keeping p->current_lsn co-tracked with s->log_len; the existing
|
||
* majority-commit barrier below (repl_wait_majority_ack) is UNCHANGED and runs
|
||
* after the append. Non-raft road / engine-less tests (library not LEADER)
|
||
* fall back inside ha_primary_log_append to persist_wal_async. Standalone
|
||
* persist (p->cluster == NULL, lib-level tests) keeps the direct append. */
|
||
int rc;
|
||
if (p->cluster != NULL)
|
||
rc = ha_primary_log_append(p->cluster, type, payload, len, NULL);
|
||
else
|
||
rc = persist_wal_async(p, type, payload, len);
|
||
if (rc != 0) {
|
||
if (sync_rc_out)
|
||
*sync_rc_out = rc;
|
||
return rc;
|
||
}
|
||
|
||
/* Majority-commit replication (Req 5.1–5.3; ha-commit-engine Req 1.1–1.3).
|
||
* persist_wal_async appends to the page cache and signals the demand-driven
|
||
* flusher; it no longer fsyncs inline (group-commit, task 2.1). Durability
|
||
* is therefore established by the flusher advancing durable_lsn, and a
|
||
* commit position advances only once the entry is durable locally — the
|
||
* engine path via the durable self-cap (commit.c), the legacy path below
|
||
* via sync_append_legacy_commit_store's written_lsn <= durable_lsn gate.
|
||
* We use current_lsn - 1 because persist_wal_async already incremented it. */
|
||
uint64_t written_lsn;
|
||
pthread_mutex_lock(&p->lock);
|
||
written_lsn = p->current_lsn - 1;
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
int sync_err;
|
||
/* Engine-wired path (ha-commit-engine task 5.3): delegate the commit
|
||
* decision to the commit engine through the walAck adapter. The engine
|
||
* is the single committed_lsn writer (no store here), and its
|
||
* n_peers<=1 self-quorum replaces the old has_standbys → -ENODEV
|
||
* degraded special case (Req 6.3).
|
||
*
|
||
* LSN-0 edge (ACCEPTED interim behavior — spec Implementation Notes):
|
||
* written_lsn is passed unconditionally, but the FIRST-EVER WAL entry
|
||
* of a fresh cluster is LSN 0 and commit_submit rejects lsn==0, so the
|
||
* adapter's entry_lsn != 0 guard routes that one entry to the legacy
|
||
* branch below. This happens once per cluster lifetime;
|
||
* ha-single-leader-role's promotion no-op entry eliminates it in
|
||
* practice. */
|
||
if (p->repl.ce != NULL && written_lsn != 0) {
|
||
sync_err = repl_wait_majority_ack(&p->repl, written_lsn,
|
||
p->config.sync_timeout_ms,
|
||
(uint8_t)OPCode_walAck);
|
||
if (sync_err != 0)
|
||
CTRL_METRIC_INC(ctrl_repl_degraded_count);
|
||
} else {
|
||
/* Residual non-engine branch: engine-less (ce == NULL) unit-test
|
||
* contexts plus the LSN-0 first entry. The legacy connectivity-
|
||
* based counting was retired by ha-commit-barrier-rollout task 2.2:
|
||
* repl_wait_majority_ack now confirms LSN 0 immediately (documented
|
||
* once-per-lifetime edge) and surfaces engine-less walAck waits as
|
||
* -EINVAL via commit_submit, so the committed_lsn store below only
|
||
* fires for the LSN-0 case (a no-op store of 0). */
|
||
mylock_lock(&p->repl.lock);
|
||
int has_standbys = (p->repl.nstandbys > 0);
|
||
mylock_unlock(&p->repl.lock);
|
||
|
||
if (!has_standbys) {
|
||
/* No standbys registered → degraded mode (no quorum to wait for) */
|
||
sync_err = -ENODEV;
|
||
} else {
|
||
sync_err = repl_wait_majority_ack(&p->repl, written_lsn,
|
||
p->config.sync_timeout_ms,
|
||
(uint8_t)OPCode_walAck);
|
||
if (sync_err == 0)
|
||
/* Durable-gated commit advance (task 2.4): advance committed_lsn
|
||
* only if written_lsn is durable locally — never on the ACK
|
||
* alone (group-commit removed the inline append fsync). */
|
||
sync_append_legacy_commit_store(p, written_lsn);
|
||
else
|
||
CTRL_METRIC_INC(ctrl_repl_degraded_count);
|
||
}
|
||
}
|
||
if (sync_rc_out)
|
||
*sync_rc_out = sync_err;
|
||
/* sync_err < 0 — replication failed.
|
||
* -ENODEV: legacy degraded mode (no standbys, engine-less) — acceptable
|
||
* for forward progress.
|
||
* -ETIMEDOUT: no per-entry majority within the deadline — the write
|
||
* must be rejected to preserve the majority-consistency
|
||
* invariant.
|
||
* -XCE_ERR_NOT_READY: engine admission cap exhausted — retriable
|
||
* rejection.
|
||
* Timeout and admission rejection propagate to the job-submit error
|
||
* path (ha-commit-engine Req 1.1, 4.4). */
|
||
if (sync_err == -ETIMEDOUT || sync_err == -XCE_ERR_NOT_READY)
|
||
return sync_err;
|
||
return 0;
|
||
}
|
||
|
||
#ifdef XCE_TESTING
|
||
/* Test seam (task 2.4): invoke the EXACT durable-gated commit store that the
|
||
* legacy sync_append branch uses on a majority ACK, so a test can prove the
|
||
* ACK-without-durability hazard is closed without standing up the replication
|
||
* wire. Pre-fix this branch stored committed_lsn = written_lsn unconditionally
|
||
* (over-advancing past durable_lsn). */
|
||
void persist_test_sync_append_legacy_commit_store(persist_t* p,
|
||
uint64_t written_lsn)
|
||
{
|
||
sync_append_legacy_commit_store(p, written_lsn);
|
||
}
|
||
#endif /* XCE_TESTING */
|
||
|
||
void persist_log_entry_metadata(const wal_entry_hdr_t* hdr)
|
||
{
|
||
TRACE("WAL entry: type=%u lsn=%lu term=%lu payload_len=%u crc=%08x",
|
||
hdr->type,
|
||
(unsigned long)hdr->lsn,
|
||
(unsigned long)hdr->term,
|
||
hdr->payload_len,
|
||
hdr->crc);
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Hook functions — AFTER_EVENT handlers that write WAL entries */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
/* Map HOOK_DETAIL_* values to WAL entry types.
|
||
* Returns 0 for details that have no WAL representation (skip). */
|
||
static uint32_t detail_to_wal_type(int detail)
|
||
{
|
||
switch (detail) {
|
||
case HOOK_DETAIL_JOB_NEW: return WAL_JOB_SUBMIT;
|
||
case HOOK_DETAIL_JOB_START: return WAL_JOB_START;
|
||
case HOOK_DETAIL_JOB_EXIT: return WAL_JOB_FINISH;
|
||
case HOOK_DETAIL_JOB_FAIL: return WAL_JOB_FAIL;
|
||
case HOOK_DETAIL_JOB_STOP: return WAL_JOB_STOP;
|
||
case HOOK_DETAIL_JOB_RESUME: return WAL_JOB_RESUME;
|
||
case HOOK_DETAIL_JOB_UPDATE: return WAL_JOB_UPDATE;
|
||
case HOOK_DETAIL_JOB_CKPT: return WAL_JOB_CKPT;
|
||
case HOOK_DETAIL_JOB_RUSAGE_UPDATE: return WAL_JOB_RUSAGE;
|
||
case HOOK_DETAIL_SETPREASON: return WAL_JOB_PREASON;
|
||
case HOOK_DETAIL_CLRPREASON: return WAL_JOB_PREASON;
|
||
case HOOK_DETAIL_QUEUE_UPDATE: return WAL_QUEUE_CTRL;
|
||
case HOOK_DETAIL_WORKER_JOIN: return WAL_WORKER_CTRL;
|
||
case HOOK_DETAIL_WORKER_UPDATE: return WAL_WORKER_CTRL;
|
||
case HOOK_DETAIL_WORKER_DEAD: return WAL_WORKER_CTRL;
|
||
default: return 0; /* unknown — skip */
|
||
}
|
||
}
|
||
|
||
/* Write-path commit classification (ha-commit-barrier-rollout task 1.2).
|
||
* Pure lookup over the VERIFIED table (design.md §Write-Path
|
||
* Classification); `detail` disambiguates WAL_WORKER_CTRL only.
|
||
* Unclassified types fail fast under XCE_TESTING (abort) and fall back to
|
||
* RETAIN in production — never dropped, never falsely confirmed (Req 2.2). */
|
||
wal_commit_mode_t wal_classify(uint32_t wal_type, int detail)
|
||
{
|
||
switch (wal_type) {
|
||
case WAL_JOB_SUBMIT:
|
||
/* client job submit — request-shaped; keeps the in-hook sync path
|
||
* (reference implementation, handle_jobsubmit) */
|
||
return WAL_MODE_WAIT;
|
||
case WAL_JOB_ARRAY_SUBMIT:
|
||
/* client job-array submit (job-array-single-wal-record task 1.1) —
|
||
* request-shaped exactly like WAL_JOB_SUBMIT: one requester awaits
|
||
* the single commit barrier before reply, regardless of array
|
||
* size. */
|
||
return WAL_MODE_WAIT;
|
||
case WAL_QUEUE_CTRL:
|
||
/* client queue control — request-shaped; the handler barrier owes
|
||
* commit completion before the requester reply (tasks 2-3) */
|
||
return WAL_MODE_WAIT;
|
||
case WAL_WORKER_CTRL:
|
||
/* the only detail-disambiguated type: JOIN/UPDATE have a requester
|
||
* awaiting confirmation (WAIT); DEAD is a controller-detected fact.
|
||
* Unexpected details take the fact-safe RETAIN side. */
|
||
if (detail == HOOK_DETAIL_WORKER_JOIN ||
|
||
detail == HOOK_DETAIL_WORKER_UPDATE)
|
||
return WAL_MODE_WAIT;
|
||
return WAL_MODE_RETAIN;
|
||
case WAL_JOB_START:
|
||
case WAL_JOB_FINISH:
|
||
case WAL_JOB_FAIL:
|
||
case WAL_JOB_STOP:
|
||
case WAL_JOB_RESUME:
|
||
case WAL_JOB_UPDATE:
|
||
case WAL_JOB_RUSAGE:
|
||
case WAL_JOB_CKPT:
|
||
case WAL_JOB_PREASON:
|
||
/* worker/scheduler-reported facts — retained until committed.
|
||
* FAIL/UPDATE are dual-origin; their request-shaped side is
|
||
* realized by the handler barrier in do_jobcontrol_op
|
||
* (research.md §2 call-site verification). */
|
||
return WAL_MODE_RETAIN;
|
||
case WAL_NOOP:
|
||
/* internal promotion no-op — replicated like any committed entry,
|
||
* retained until committed (ha-commit-barrier-rollout pre-declared
|
||
* row: promotion | internal | RETAIN; guard forces classification) */
|
||
return WAL_MODE_RETAIN;
|
||
case WAL_CONFIG_MARKER:
|
||
/* config-change marker (ha-term-safety task 4.2): a committable
|
||
* (term,lsn) log entry like a job-state fact — replicated and retained
|
||
* until committed; it commits through the SAME barrier as job state
|
||
* (R7.1/R7.11), so it is RETAIN, NOT a no-op or a LOCAL marker. The
|
||
* requester barrier on a config upgrade is owned by the upgrade path
|
||
* (do_upgrade_config_with_bytes), not the per-entry WAIT hook. */
|
||
return WAL_MODE_RETAIN;
|
||
case WAL_CHECKPOINT:
|
||
/* local segment marker — never replicated, by design */
|
||
return WAL_MODE_LOCAL;
|
||
default:
|
||
ERROR("wal_classify: unclassified WAL type %u (detail %d) — "
|
||
"RETAIN fallback; add it to the classification table",
|
||
wal_type, detail);
|
||
#ifdef XCE_TESTING
|
||
assert(0 && "unclassified WAL type — add it to wal_classify");
|
||
#endif
|
||
return WAL_MODE_RETAIN;
|
||
}
|
||
}
|
||
|
||
/* Commit barrier helper (ha-commit-barrier-rollout task 2.2, design.md
|
||
* §Commit Barrier Helper; Req 1.1, 1.2, 1.3).
|
||
*
|
||
* Handler-callable WAIT completion on the appended watermark: snapshot the
|
||
* last-appended LSN, return 0 when the commit index already covers it,
|
||
* else block in the commit engine until coverage or deadline. The engine
|
||
* result is returned UNMODIFIED (0 / -ETIMEDOUT / -XCE_ERR_NOT_READY /
|
||
* -EINVAL) so callers map it to their own error replies.
|
||
*
|
||
* Lock discipline: p->lock is held only for the current_lsn snapshot and
|
||
* is released BEFORE commit_submit(WAIT) — the engine blocks on ce->lock
|
||
* + condvar, and p->lock must never be held across it (engine
|
||
* lock-ordering rule). */
|
||
int persist_commit_barrier(persist_t* p, uint32_t timeout_ms)
|
||
{
|
||
uint64_t watermark;
|
||
|
||
if (p == NULL)
|
||
return -EINVAL;
|
||
|
||
/* Snapshot the last-appended watermark under p->lock. current_lsn is
|
||
* the NEXT LSN to assign (persist.h invariant: committed_lsn <=
|
||
* current_lsn - 1), so the highest appended entry is current_lsn - 1
|
||
* (resolved contract, spec Implementation Notes 2.1). current_lsn ==
|
||
* 0 means nothing was ever appended — vacuously satisfied. */
|
||
pthread_mutex_lock(&p->lock);
|
||
if (p->current_lsn == 0) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return 0;
|
||
}
|
||
watermark = p->current_lsn - 1;
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
/* Already covered — committed_lsn is monotone, so coverage observed
|
||
* here can never regress. This check also absorbs the LSN-0 first
|
||
* entry (watermark 0 vs committed_lsn 0): that once-per-lifetime entry
|
||
* has no quorum contract (ha-commit-engine Implementation Notes) and
|
||
* commit_submit would reject lsn == 0 anyway. */
|
||
if (atomic_load_explicit(&p->committed_lsn, memory_order_acquire) >=
|
||
watermark)
|
||
return 0;
|
||
|
||
/* Engine not wired: legacy/bootstrap contexts have no engine commit
|
||
* contract — there is nothing to wait on, so the barrier is a no-op.
|
||
* Production always wires the engine (cluster_init_phase2). */
|
||
if (p->repl.ce == NULL)
|
||
return 0;
|
||
|
||
/* timeout_ms == 0 borrows the configured sync deadline (Req 1.2). */
|
||
return commit_submit(p->repl.ce, watermark, COMMIT_MODE_WAIT,
|
||
timeout_ms != 0 ? timeout_ms
|
||
: p->config.sync_timeout_ms);
|
||
}
|
||
|
||
/* Classified append at the hook seam (barrier-rollout task 1.2).
|
||
*
|
||
* WAIT (queue/worker control): the append stays async here — the handler
|
||
* barrier (tasks 2-3) owes commit completion before the requester reply.
|
||
* WAL_JOB_SUBMIT never reaches this helper (it keeps the in-hook sync
|
||
* path in persist_job_hook_fun).
|
||
*
|
||
* RETAIN (facts): append + broadcast, then register the appended LSN with
|
||
* the commit engine. commit_submit(COMMIT_MODE_RETAIN) is registration-
|
||
* free bookkeeping (commit.c — returns 0 immediately; commit state is
|
||
* derived from registry watermarks at ACK time), so the call is contract
|
||
* documentation: the entry stays pending purely by lsn > committed_lsn
|
||
* and commits later via covering ACKs. Commit-progress results are NEVER
|
||
* propagated to hook callers — only the local append rc is (Req 3.1).
|
||
*
|
||
* Lock context: persist_wal_async takes/releases p->lock internally; the
|
||
* appended-LSN read below re-takes p->lock briefly (same discipline as
|
||
* persist_wal_sync_append) and releases it BEFORE commit_submit, so no
|
||
* p->lock is ever held across ce->lock (engine lock-ordering rule). */
|
||
static int wal_hook_append_classified(persist_t* p, uint32_t wal_type,
|
||
wal_commit_mode_t mode,
|
||
const void* payload, uint32_t len)
|
||
{
|
||
uint64_t appended_lsn;
|
||
int rc;
|
||
|
||
/* ha-term-safety task 3.1: route the primary's local append through the raft
|
||
* library (ha_primary_log_append → raft_log_append) so the LIBRARY stamps the
|
||
* live election term and assigns the LSN (firing on_persist_entry → the WAL
|
||
* write), keeping p->current_lsn co-tracked with s->log_len. The library is
|
||
* the SINGLE primary append entry — no append bypasses it on the raft road.
|
||
* On the non-raft road (library not LEADER) ha_primary_log_append falls back
|
||
* to persist_wal_async, so engine-less / init-time contexts are unchanged.
|
||
* When p->cluster is NULL (persist used standalone in lib tests) keep the
|
||
* direct append (no cluster → no raft state to drive). */
|
||
if (p->cluster != NULL)
|
||
rc = ha_primary_log_append(p->cluster, wal_type, payload, len, NULL);
|
||
else
|
||
rc = persist_wal_async(p, wal_type, payload, len);
|
||
if (rc != 0 || mode != WAL_MODE_RETAIN)
|
||
return rc;
|
||
|
||
if (p->repl.ce == NULL)
|
||
return rc; /* engine not wired (bootstrap / engine-less contexts) */
|
||
|
||
/* current_lsn - 1 is the LSN persist_wal_async just assigned (it
|
||
* increments under p->lock). A concurrent append may bump it further;
|
||
* that is benign — contiguous advancement means the higher watermark
|
||
* still covers this entry. LSN-0 edge: commit_submit rejects lsn == 0
|
||
* (first-ever entry of a fresh cluster — accepted interim behavior,
|
||
* spec Implementation Notes). */
|
||
pthread_mutex_lock(&p->lock);
|
||
appended_lsn = p->current_lsn - 1;
|
||
pthread_mutex_unlock(&p->lock);
|
||
if (appended_lsn != 0)
|
||
(void)commit_submit(p->repl.ce, appended_lsn, COMMIT_MODE_RETAIN, 0);
|
||
|
||
return rc; /* local append rc only — facts have no commit timeout */
|
||
}
|
||
|
||
/* Shared step for a single WAL_JOB_SUBMIT entry (raft-apply-decoupling task
|
||
* 3.1): encode `job`, append, and block on the majority-commit barrier.
|
||
* Returns 0 when the entry committed on a majority; a negative rc otherwise
|
||
* (-ETIMEDOUT: no majority within the deadline; -XCE_ERR_NOT_READY: admission
|
||
* cap; -ENOMEM: encode failure; or a local-write error). Whenever the LOCAL
|
||
* WAL write succeeded (rc == 0, -ETIMEDOUT, or -XCE_ERR_NOT_READY — the entry
|
||
* was appended locally and the LSN advanced even though replication is
|
||
* degraded), *out_lsn is set to the entry's assigned LSN so a caller can tag
|
||
* it for egress-watermark purposes even on a rejected submission. */
|
||
static int submit_job_wal_entry_and_commit(persist_t* p, const xce_job_t* job,
|
||
uint64_t* out_lsn)
|
||
{
|
||
uint8_t* buf = NULL;
|
||
size_t sz = 0;
|
||
int rc;
|
||
|
||
if (capnp_encode_job(job, &buf, &sz) != 0)
|
||
return -ENOMEM;
|
||
|
||
/* ha-convergence-recovery (3.3): snapshot the LSN this submit entry
|
||
* will receive BEFORE the append. persist_wal_async assigns
|
||
* hdr.lsn = current_lsn then increments, so the pre-append current_lsn
|
||
* is exactly the written entry's LSN. */
|
||
pthread_mutex_lock(&p->lock);
|
||
uint64_t submit_lsn = p->current_lsn;
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
int sync_rc = 0;
|
||
rc = persist_wal_sync_append(p, WAL_JOB_SUBMIT, buf, (uint32_t)sz,
|
||
&sync_rc);
|
||
free(buf);
|
||
|
||
if (rc == 0 || rc == -ETIMEDOUT || rc == -XCE_ERR_NOT_READY)
|
||
*out_lsn = submit_lsn;
|
||
|
||
return rc;
|
||
}
|
||
|
||
/* Explicit controller-side raft step for a job submission (raft-apply-decoupling
|
||
* item #1 / bug-1306): WAL-append each job in `jlist` and block on the
|
||
* majority-commit barrier, returning the commit verdict DIRECTLY to the caller
|
||
* (handle_jobsubmit) instead of through the xce_run_hooks fan-out (which returns
|
||
* only the last hook's rc and so clobbered this -ETIMEDOUT with a later notify
|
||
* hook's 0 — the committed-job-loss root cause).
|
||
*
|
||
* Returns 0 when every job's entry committed on a majority; the FIRST negative
|
||
* rc otherwise (-ETIMEDOUT: no majority within the deadline; -XCE_ERR_NOT_READY:
|
||
* admission cap; -ENOMEM: encode failure; or a local-write error). The caller
|
||
* MUST reject the submission on a negative rc (never reply success) to preserve
|
||
* the Commit Durability invariant. On -ETIMEDOUT / -XCE_ERR_NOT_READY the entry
|
||
* WAS appended locally and the LSN advanced, so j->submit_lsn is still tagged —
|
||
* the entry remains a pending, uncommitted tail a new-term leader may discard. */
|
||
int persist_submit_wal_and_commit(persist_t* p, xlist_t* jlist)
|
||
{
|
||
int rc = 0;
|
||
|
||
if (p == NULL || jlist == NULL)
|
||
return -EINVAL;
|
||
|
||
for (int i = 0; i < jlist->count && rc == 0; i++) {
|
||
job_t* j = (job_t*)xlist_get(jlist, i);
|
||
uint64_t lsn = 0;
|
||
|
||
rc = submit_job_wal_entry_and_commit(p, &j->job, &lsn);
|
||
if (rc == 0 || rc == -ETIMEDOUT || rc == -XCE_ERR_NOT_READY)
|
||
j->submit_lsn = lsn;
|
||
}
|
||
|
||
return rc;
|
||
}
|
||
|
||
/* Sibling of persist_submit_wal_and_commit for a job submission whose job
|
||
* object(s) do not exist yet (raft-apply-decoupling task 3.1 / Req 2.1, 2.3):
|
||
* `payloads` is an xlist_t of xce_job_t* that have never been registered in
|
||
* any job table. On full success (return 0), out_lsns[i] holds payloads[i]'s
|
||
* committed LSN — the caller constructs and tags the job object AFTER this
|
||
* call returns, so a failed/timed-out commit never leaves a job object
|
||
* behind (there is nothing to construct until this succeeds). Returns the
|
||
* first negative rc on failure. */
|
||
int persist_submit_job_payloads_and_commit(persist_t* p, xlist_t* payloads,
|
||
uint64_t* out_lsns)
|
||
{
|
||
int rc = 0;
|
||
|
||
if (p == NULL || payloads == NULL || out_lsns == NULL)
|
||
return -EINVAL;
|
||
|
||
for (int i = 0; i < payloads->count && rc == 0; i++) {
|
||
xce_job_t* j = (xce_job_t*)xlist_get(payloads, i);
|
||
|
||
rc = submit_job_wal_entry_and_commit(p, j, &out_lsns[i]);
|
||
}
|
||
|
||
return rc;
|
||
}
|
||
|
||
/* raft-apply-decoupling task 3.2: append-only counterpart of
|
||
* submit_job_wal_entry_and_commit — writes the entry via the SAME
|
||
* primary-append primitive wal_hook_append_classified already uses for
|
||
* RETAIN-mode facts (ha_primary_log_append / persist_wal_async), WITHOUT
|
||
* waiting on the majority-commit barrier. Deliberately does NOT reuse
|
||
* submit_job_wal_entry_and_commit itself: that helper is shared with
|
||
* persist_submit_wal_and_commit's still-synchronous, still-blocking
|
||
* contract (relied on directly by test_egress_filter.c / test_e2e_hooks.c /
|
||
* test_persist_hooks.c), which this task must not disturb. */
|
||
static int append_job_wal_entry_local(persist_t* p, const xce_job_t* job,
|
||
uint64_t* out_lsn)
|
||
{
|
||
uint8_t* buf = NULL;
|
||
size_t sz = 0;
|
||
int rc;
|
||
uint64_t lsn;
|
||
|
||
if (capnp_encode_job(job, &buf, &sz) != 0)
|
||
return -ENOMEM;
|
||
|
||
/* Snapshot the LSN this entry will receive BEFORE the append (same
|
||
* convention as submit_job_wal_entry_and_commit / wal_hook_append_
|
||
* classified: persist_wal_async / ha_primary_log_append assigns
|
||
* hdr.lsn = current_lsn then increments). */
|
||
pthread_mutex_lock(&p->lock);
|
||
lsn = p->current_lsn;
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
if (p->cluster != NULL)
|
||
rc = ha_primary_log_append(p->cluster, WAL_JOB_SUBMIT, buf,
|
||
(uint32_t)sz, NULL);
|
||
else
|
||
rc = persist_wal_async(p, WAL_JOB_SUBMIT, buf, (uint32_t)sz);
|
||
free(buf);
|
||
|
||
if (rc == 0)
|
||
*out_lsn = lsn;
|
||
return rc;
|
||
}
|
||
|
||
/* raft-apply-decoupling task 3.2: bridges commit_submit_async's
|
||
* commit_continuation_fn shape (commit_outcome_t + rc) to
|
||
* persist_job_commit_notify_fn's shape (a single rc) so persist.h's public
|
||
* declaration never needs commit.h's commit_outcome_t (see persist.h's
|
||
* doc comment on persist_job_commit_notify_fn). One adapter is heap-
|
||
* allocated per registered lsn and frees itself after its single
|
||
* invocation — safe because commit_resolve guarantees exactly-once
|
||
* delivery. */
|
||
typedef struct {
|
||
persist_job_commit_notify_fn fn;
|
||
void* ctx;
|
||
} job_commit_adapter_ctx_t;
|
||
|
||
static void job_commit_resolve_adapter(void* adapter_ctx, uint64_t lsn,
|
||
commit_outcome_t outcome, int rc)
|
||
{
|
||
job_commit_adapter_ctx_t* a = (job_commit_adapter_ctx_t*)adapter_ctx;
|
||
int final_rc = (outcome == COMMIT_OUTCOME_COMMITTED) ? 0 : rc;
|
||
|
||
a->fn(a->ctx, lsn, final_rc);
|
||
free(a);
|
||
}
|
||
|
||
int persist_submit_job_payloads_async(persist_t* p, xlist_t* payloads,
|
||
uint64_t* out_lsns,
|
||
persist_job_commit_notify_fn fn,
|
||
void* ctx)
|
||
{
|
||
int first_rc = 0;
|
||
|
||
if (p == NULL || payloads == NULL || out_lsns == NULL || fn == NULL)
|
||
return -EINVAL;
|
||
|
||
for (int i = 0; i < payloads->count; i++) {
|
||
xce_job_t* j = (xce_job_t*)xlist_get(payloads, i);
|
||
uint64_t lsn = 0;
|
||
int rc;
|
||
|
||
if (first_rc != 0) {
|
||
/* A prior payload in this batch already failed locally: every
|
||
* remaining payload resolves with that SAME failure now,
|
||
* synchronously — so `fn` is invoked exactly once per payload
|
||
* regardless of where the failure occurred, and the caller
|
||
* never has to separately track "never going to be called"
|
||
* entries. */
|
||
fn(ctx, 0, first_rc);
|
||
continue;
|
||
}
|
||
|
||
rc = append_job_wal_entry_local(p, j, &lsn);
|
||
if (rc != 0) {
|
||
first_rc = rc;
|
||
fn(ctx, 0, rc);
|
||
continue;
|
||
}
|
||
out_lsns[i] = lsn;
|
||
|
||
/* LSN-0 edge (documented once-per-cluster-lifetime interim
|
||
* behavior — ha-commit-engine Implementation Notes, mirrored by
|
||
* repl_wait_majority_ack's walAck branch): the first-ever WAL
|
||
* entry of a fresh cluster is structurally un-commitable and
|
||
* commit_submit_async rejects lsn == 0, so resolve it inline.
|
||
* Engine-not-wired (bootstrap / engine-less contexts) mirrors
|
||
* persist_commit_barrier's own "no engine -> no-op success"
|
||
* convention — there is no async completion authority to
|
||
* register with. */
|
||
if (lsn == 0 || p->repl.ce == NULL) {
|
||
fn(ctx, lsn, 0);
|
||
continue;
|
||
}
|
||
|
||
job_commit_adapter_ctx_t* a = calloc(1, sizeof(*a));
|
||
if (a == NULL) {
|
||
first_rc = -ENOMEM;
|
||
fn(ctx, lsn, -ENOMEM);
|
||
continue;
|
||
}
|
||
a->fn = fn;
|
||
a->ctx = ctx;
|
||
|
||
int reg_rc = commit_submit_async(p->repl.ce, lsn,
|
||
job_commit_resolve_adapter, a);
|
||
if (reg_rc != 0) {
|
||
free(a);
|
||
first_rc = reg_rc;
|
||
ERROR("persist_submit_job_payloads_async: lsn=%lu appended but "
|
||
"commit_submit_async registration failed (rc=%d) -- the "
|
||
"entry remains durable and will still be caught up "
|
||
"eventually via replay, but this caller will not be told "
|
||
"its outcome via this path",
|
||
(unsigned long)lsn, reg_rc);
|
||
fn(ctx, lsn, reg_rc);
|
||
continue;
|
||
}
|
||
}
|
||
|
||
return first_rc;
|
||
}
|
||
|
||
/* job-array-single-wal-record task 2.1: serialize a job-array submission
|
||
* (base jobid + array size + one shared job spec) to a malloc'd Cap'n Proto
|
||
* buffer, reusing the SAME capnp struct (JobSubmit) the client's own submit
|
||
* request uses -- mirrors capnp_encode_job above, which mirrors encode_Job/
|
||
* new_Job/write_Job for a single job, except here via encode_JobSubmit/
|
||
* new_JobSubmit/write_JobSubmit. The encoded payload is O(1) in array_size:
|
||
* arraySize is a fixed-width uint64 field, not a per-element list -- so this
|
||
* function's output size never grows with array_size (Req 7.1). Caller must
|
||
* free(*out) on success. */
|
||
static int capnp_encode_job_array(const xce_job_submit_t* s, uint8_t** out,
|
||
size_t* out_len)
|
||
{
|
||
struct capn c;
|
||
struct capn_segment* cs;
|
||
struct JobSubmit cjs;
|
||
JobSubmit_ptr jp;
|
||
|
||
capn_init_malloc(&c);
|
||
cs = capn_root(&c).seg;
|
||
encode_JobSubmit(cs, &cjs, (xce_job_submit_t*)s);
|
||
jp = new_JobSubmit(cs);
|
||
write_JobSubmit(&cjs, jp);
|
||
capn_setp(capn_root(&c), 0, jp.p);
|
||
|
||
*out_len = capn_size(&c);
|
||
*out = malloc(*out_len);
|
||
if (!*out) { capn_free(&c); return -ENOMEM; }
|
||
capn_write_mem(&c, *out, *out_len, 0);
|
||
capn_free(&c);
|
||
return 0;
|
||
}
|
||
|
||
/* job-array-single-wal-record task 2.1: append-only, single-record
|
||
* counterpart of append_job_wal_entry_local for a WHOLE job array. Encodes
|
||
* ONE JobSubmit-shaped payload (base_jobid, array_size, base_spec->submit)
|
||
* and appends it as exactly one WAL_JOB_ARRAY_SUBMIT entry via the SAME
|
||
* primary-append primitive append_job_wal_entry_local already uses
|
||
* (ha_primary_log_append / persist_wal_async) -- never one entry per array
|
||
* element, regardless of array_size (Req 1.1, 1.2, 1.3). base_jobid/
|
||
* array_size are taken as explicit parameters rather than read from
|
||
* base_spec->jobid/base_spec->array_size, so the caller's own
|
||
* already-range-claimed values are always what gets committed, independent
|
||
* of whatever same-shaped fields the client's own wire request happened to
|
||
* carry. */
|
||
static int append_job_array_wal_entry_local(persist_t* p, uint64_t base_jobid,
|
||
uint64_t array_size,
|
||
const xce_job_submit_t* base_spec,
|
||
uint64_t* out_lsn)
|
||
{
|
||
xce_job_submit_t wire = { .jobid = base_jobid,
|
||
.array_size = array_size,
|
||
.submit = base_spec->submit };
|
||
uint8_t* buf = NULL;
|
||
size_t sz = 0;
|
||
int rc;
|
||
uint64_t lsn;
|
||
|
||
if (capnp_encode_job_array(&wire, &buf, &sz) != 0)
|
||
return -ENOMEM;
|
||
|
||
/* Snapshot the LSN this entry will receive BEFORE the append (same
|
||
* convention as append_job_wal_entry_local / submit_job_wal_entry_and_
|
||
* commit: persist_wal_async / ha_primary_log_append assigns
|
||
* hdr.lsn = current_lsn then increments). */
|
||
pthread_mutex_lock(&p->lock);
|
||
lsn = p->current_lsn;
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
if (p->cluster != NULL)
|
||
rc = ha_primary_log_append(p->cluster, WAL_JOB_ARRAY_SUBMIT, buf,
|
||
(uint32_t)sz, NULL);
|
||
else
|
||
rc = persist_wal_async(p, WAL_JOB_ARRAY_SUBMIT, buf, (uint32_t)sz);
|
||
free(buf);
|
||
|
||
if (rc == 0)
|
||
*out_lsn = lsn;
|
||
return rc;
|
||
}
|
||
|
||
/* job-array-single-wal-record task 2.1 (design.md §WAL Array Submission,
|
||
* Service Interface): encodes and appends ONE array-shaped WAL entry
|
||
* (WAL_JOB_ARRAY_SUBMIT) and registers ONE commit continuation for it,
|
||
* regardless of array_size -- the single-record sibling of
|
||
* persist_submit_job_payloads_async's per-payload loop, specialized to
|
||
* exactly one payload. No job_t/xce_job_t object is constructed here or
|
||
* anywhere on this call path -- array expansion into individual job
|
||
* entries happens later, in the shared apply seam (task 3.x's
|
||
* restore_job_array), never here.
|
||
*
|
||
* Mirrors persist_submit_job_payloads_async's contract exactly:
|
||
* - a local encode/append failure resolves `fn` synchronously with that
|
||
* failure and returns the same negative rc;
|
||
* - the LSN-0 / engine-not-wired edge cases (documented on
|
||
* persist_submit_job_payloads_async above) resolve `fn` inline with
|
||
* rc=0;
|
||
* - otherwise `fn` is registered via commit_submit_async and invoked
|
||
* exactly once, asynchronously, once the entry's commit outcome is
|
||
* known. */
|
||
int persist_submit_job_array_async(persist_t* p, uint64_t base_jobid,
|
||
uint64_t array_size,
|
||
const xce_job_submit_t* base_spec,
|
||
uint64_t* out_lsn,
|
||
persist_job_commit_notify_fn fn, void* ctx)
|
||
{
|
||
uint64_t lsn = 0;
|
||
int rc;
|
||
|
||
if (p == NULL || base_spec == NULL || base_spec->submit == NULL ||
|
||
out_lsn == NULL || fn == NULL || array_size <= 1)
|
||
return -EINVAL;
|
||
|
||
rc = append_job_array_wal_entry_local(p, base_jobid, array_size,
|
||
base_spec, &lsn);
|
||
if (rc != 0) {
|
||
fn(ctx, 0, rc);
|
||
return rc;
|
||
}
|
||
*out_lsn = lsn;
|
||
|
||
/* LSN-0 / engine-not-wired edge (see persist_submit_job_payloads_async's
|
||
* doc comment above for the full rationale -- identical here,
|
||
* specialized to one entry instead of a loop). */
|
||
if (lsn == 0 || p->repl.ce == NULL) {
|
||
fn(ctx, lsn, 0);
|
||
return 0;
|
||
}
|
||
|
||
job_commit_adapter_ctx_t* a = calloc(1, sizeof(*a));
|
||
if (a == NULL) {
|
||
fn(ctx, lsn, -ENOMEM);
|
||
return -ENOMEM;
|
||
}
|
||
a->fn = fn;
|
||
a->ctx = ctx;
|
||
|
||
int reg_rc = commit_submit_async(p->repl.ce, lsn,
|
||
job_commit_resolve_adapter, a);
|
||
if (reg_rc != 0) {
|
||
free(a);
|
||
ERROR("persist_submit_job_array_async: lsn=%lu appended but "
|
||
"commit_submit_async registration failed (rc=%d) -- the "
|
||
"entry remains durable and will still be caught up "
|
||
"eventually via replay, but this caller will not be told "
|
||
"its outcome via this path",
|
||
(unsigned long)lsn, reg_rc);
|
||
fn(ctx, lsn, reg_rc);
|
||
return reg_rc;
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
void persist_register_pending_apply(persist_t* p, uint64_t lsn,
|
||
pending_apply_notify_fn on_applied,
|
||
void* ctx)
|
||
{
|
||
pending_command_apply_t* node;
|
||
|
||
if (p == NULL || on_applied == NULL)
|
||
return;
|
||
|
||
/* bug-1415 self-healing (round 2): commit.c's continuation dispatch and
|
||
* the apply pump's walk (persist_apply_pump_run) are two independent
|
||
* vtasks with no completion-order guarantee -- see this function's doc
|
||
* comment in persist.h for the full race. If the walk has already
|
||
* applied `lsn` (last_applied_lsn already at or past it) by the time we
|
||
* get here, linking this registration onto pending_applies would orphan
|
||
* it forever: the walk only ever visits each lsn once, and it already
|
||
* called persist_take_pending_apply() for this lsn and found nothing.
|
||
* Fire the hand-off immediately, synchronously, on THIS thread instead
|
||
* -- never leave a continuation unresolved (controllers never
|
||
* self-terminate; the same "fail loud and complete the work" posture
|
||
* commit.c's dispatch_continuation fallback uses).
|
||
*
|
||
* bug-1428: this self-heal check MUST run under pending_applies_lock,
|
||
* not before acquiring it. The window this closes: this thread reads
|
||
* last_applied_lsn (sees "not yet applied"), then -- BEFORE it acquires
|
||
* the lock below to insert -- the apply pump's thread advances
|
||
* last_applied_lsn past `lsn` AND completes its own
|
||
* persist_take_pending_apply(lsn) call (finding nothing, since this
|
||
* thread hasn't inserted yet). This thread then inserts a node the walk
|
||
* will never revisit -- a permanently orphaned hand-off, the exact
|
||
* class of hang the round-2 self-heal above was meant to eliminate,
|
||
* reintroduced by the gap between the unlocked check and the locked
|
||
* insert. Holding pending_applies_lock across BOTH the check and the
|
||
* insert makes the two mutually exclusive with
|
||
* persist_take_pending_apply's own locked scan: either this thread's
|
||
* check runs before the pump's take (sees "not yet applied", inserts,
|
||
* and the pump's later take -- now blocked on this same lock until this
|
||
* critical section ends -- finds the freshly-inserted node), or it runs
|
||
* after (sees "already applied", self-heals here instead of inserting,
|
||
* and the pump's take correctly finds nothing) -- never the interleaved
|
||
* gap that orphans it. */
|
||
pthread_mutex_lock(&p->pending_applies_lock);
|
||
if (lsn <= persist_last_applied_lsn(p)) {
|
||
pthread_mutex_unlock(&p->pending_applies_lock);
|
||
on_applied(ctx, lsn);
|
||
return;
|
||
}
|
||
|
||
node = calloc(1, sizeof(*node));
|
||
if (node == NULL) {
|
||
pthread_mutex_unlock(&p->pending_applies_lock);
|
||
ERROR("persist_register_pending_apply: OOM registering the apply "
|
||
"hand-off for lsn=%lu -- the entry is still durably committed "
|
||
"and will still be applied by a future restart replay, but no "
|
||
"continuation will fire for it via this path",
|
||
(unsigned long)lsn);
|
||
return;
|
||
}
|
||
node->lsn = lsn;
|
||
node->on_applied = on_applied;
|
||
node->ctx = ctx;
|
||
|
||
node->next = (pending_command_apply_t*)p->pending_applies;
|
||
p->pending_applies = node;
|
||
pthread_mutex_unlock(&p->pending_applies_lock);
|
||
}
|
||
|
||
int persist_has_pending_apply(persist_t* p, uint64_t lsn)
|
||
{
|
||
pending_command_apply_t* n;
|
||
int found = 0;
|
||
|
||
if (p == NULL)
|
||
return 0;
|
||
|
||
pthread_mutex_lock(&p->pending_applies_lock);
|
||
for (n = (pending_command_apply_t*)p->pending_applies; n != NULL;
|
||
n = n->next) {
|
||
if (n->lsn == lsn) {
|
||
found = 1;
|
||
break;
|
||
}
|
||
}
|
||
pthread_mutex_unlock(&p->pending_applies_lock);
|
||
|
||
return found;
|
||
}
|
||
|
||
int persist_take_pending_apply(persist_t* p, uint64_t lsn,
|
||
pending_apply_notify_fn* out_fn, void** out_ctx)
|
||
{
|
||
pending_command_apply_t* prev = NULL;
|
||
pending_command_apply_t* n;
|
||
int found = 0;
|
||
|
||
if (p == NULL)
|
||
return 0;
|
||
|
||
pthread_mutex_lock(&p->pending_applies_lock);
|
||
for (n = (pending_command_apply_t*)p->pending_applies; n != NULL;
|
||
n = n->next) {
|
||
if (n->lsn == lsn) {
|
||
if (prev == NULL)
|
||
p->pending_applies = n->next;
|
||
else
|
||
prev->next = n->next;
|
||
if (out_fn)
|
||
*out_fn = n->on_applied;
|
||
if (out_ctx)
|
||
*out_ctx = n->ctx;
|
||
found = 1;
|
||
break;
|
||
}
|
||
prev = n;
|
||
}
|
||
pthread_mutex_unlock(&p->pending_applies_lock);
|
||
|
||
free(n); /* NULL when not found -- free(NULL) is a no-op */
|
||
|
||
return found;
|
||
}
|
||
|
||
/* raft-apply-decoupling task 4.1 (Req 5.1): the receipt-time durable-append
|
||
* primitive for a worker-reported job-state fact. Unlike persist_job_hook_fun
|
||
* below (an AFTER_EVENT hook that fires only once a mutation has already
|
||
* happened), this is a direct, hook-fan-out-free call: job.c's fact receipt
|
||
* handlers (record_job_start_fact/record_job_finish_fact/record_job_fail_fact/
|
||
* record_job_postdone_fact) call this (or persist_submit_job_fact_ex below)
|
||
* INSTEAD of mutating and firing AFTER_EVENT, so no notification (scheduler
|
||
* dispatch, slot release, apiserver push) can ever fire for an uncommitted
|
||
* fact (Req 5.3). `state` is always WAL_MODE_RETAIN for every job-state fact
|
||
* type (verified against wal_classify's table) — hardcoded here rather than
|
||
* routed through wal_classify since none of these types are detail-
|
||
* disambiguated the way WAL_WORKER_CTRL is.
|
||
*
|
||
* Thin wrapper over persist_submit_job_fact_ex() with pad = 0 — the "legacy
|
||
* absolute state" discriminator (see wal_types.h's doc comment on
|
||
* wal_jobstate_t._pad). Unchanged contract/signature for every existing
|
||
* caller. */
|
||
int persist_submit_job_fact(persist_t* p, uint32_t wal_type, uint64_t jobid,
|
||
int state)
|
||
{
|
||
return persist_submit_job_fact_ex(p, wal_type, jobid, state, 0);
|
||
}
|
||
|
||
/* raft-apply-decoupling task 6.1 round 2 (Req 3.3/7.1-7.3): full-control
|
||
* sibling of persist_submit_job_fact() above, letting the caller set
|
||
* wal_jobstate_t's `_pad` discriminator explicitly. Used by
|
||
* record_job_fail_fact (WAL_JOB_FAIL_STAGE_FLAG | stage) and
|
||
* record_job_postdone_fact (WAL_JOB_UPDATE_POSTDONE_MARK) to mark a fact
|
||
* whose real mutation must be re-derived at apply time against the job's
|
||
* CURRENT state, rather than trusting a receipt-time snapshot baked into
|
||
* `state` — see wal_types.h's doc comment on wal_jobstate_t._pad for the
|
||
* full rationale. */
|
||
int persist_submit_job_fact_ex(persist_t* p, uint32_t wal_type,
|
||
uint64_t jobid, int state, uint32_t pad)
|
||
{
|
||
wal_jobstate_t js = { .jobid = jobid, .state = (uint32_t)state,
|
||
._pad = pad };
|
||
|
||
return wal_hook_append_classified(p, wal_type, WAL_MODE_RETAIN, &js,
|
||
sizeof(js));
|
||
}
|
||
|
||
/* bug-1678: WAL_JOB_DISPATCH's own richer sibling of persist_submit_job_fact()
|
||
* -- the ONLY job-state fact type that needs to carry a resource allocation
|
||
* alongside {jobid, state}, since it is the sole durable record of the
|
||
* dispatch-time PEND->RUN transition (design.md "WAL Dispatch Fact"). Payload
|
||
* shape: wal_jobstate_t header immediately followed by the Cap'n Proto-
|
||
* encoded JobAlloc bytes -- apply_committed_entry()'s and apply_wal_entry()'s
|
||
* WAL_JOB_DISPATCH cases decode exactly this shape, and treat a payload of
|
||
* exactly sizeof(wal_jobstate_t) (no trailing bytes) as "state-only, leave
|
||
* allocs untouched" for backward compatibility with any WAL_JOB_DISPATCH
|
||
* entry written before this fix. */
|
||
int persist_submit_job_dispatch_fact(persist_t* p, uint64_t jobid, int state,
|
||
const xce_job_alloc_t* alloc)
|
||
{
|
||
wal_jobstate_t js = { .jobid = jobid, .state = (uint32_t)state,
|
||
._pad = 0 };
|
||
uint8_t* allocbuf = NULL;
|
||
size_t alloclen = 0;
|
||
uint8_t* payload;
|
||
size_t total;
|
||
int rc;
|
||
|
||
rc = capnp_encode_joballoc(alloc, &allocbuf, &alloclen);
|
||
if (rc != 0)
|
||
return rc;
|
||
|
||
total = sizeof(js) + alloclen;
|
||
payload = malloc(total);
|
||
if (!payload) { free(allocbuf); return -ENOMEM; }
|
||
memcpy(payload, &js, sizeof(js));
|
||
memcpy(payload + sizeof(js), allocbuf, alloclen);
|
||
free(allocbuf);
|
||
|
||
rc = wal_hook_append_classified(p, WAL_JOB_DISPATCH, WAL_MODE_RETAIN,
|
||
payload, (uint32_t)total);
|
||
free(payload);
|
||
return rc;
|
||
}
|
||
|
||
/* bug-1687: WAL_JOB_START's own richer sibling of persist_submit_job_fact()
|
||
* -- mirrors persist_submit_job_dispatch_fact() above (bug-1678) exactly,
|
||
* but for the worker-confirmed-start fact: the sole durable record of the
|
||
* job's real job_pidinfo_t (pids/pgids/exdata a worker reported) and its
|
||
* true start_time. Payload shape: wal_jobstate_t header, immediately
|
||
* followed by a fixed uint64_t start_time (the receipt-time clock, captured
|
||
* ONCE by the caller rather than re-derived at each apply seam's own
|
||
* apply-time moment), immediately followed by the Cap'n Proto-encoded
|
||
* JobPIDInfo bytes -- apply_committed_entry()'s and apply_wal_entry()'s
|
||
* WAL_JOB_START cases decode exactly this shape, and treat a payload of
|
||
* exactly sizeof(wal_jobstate_t) (no trailing bytes) as "state-only,
|
||
* fabricate start_time and leave pidinfo untouched" for backward
|
||
* compatibility with any WAL_JOB_START entry written before this fix. */
|
||
int persist_submit_job_start_fact(persist_t* p, uint64_t jobid, int state,
|
||
uint64_t start_time,
|
||
const job_pidinfo_t* info)
|
||
{
|
||
wal_jobstate_t js = { .jobid = jobid, .state = (uint32_t)state,
|
||
._pad = 0 };
|
||
uint8_t* infobuf = NULL;
|
||
size_t infolen = 0;
|
||
uint8_t* payload;
|
||
size_t total;
|
||
int rc;
|
||
|
||
rc = capnp_encode_pidinfo(info, &infobuf, &infolen);
|
||
if (rc != 0)
|
||
return rc;
|
||
|
||
total = sizeof(js) + sizeof(start_time) + infolen;
|
||
payload = malloc(total);
|
||
if (!payload) { free(infobuf); return -ENOMEM; }
|
||
memcpy(payload, &js, sizeof(js));
|
||
memcpy(payload + sizeof(js), &start_time, sizeof(start_time));
|
||
memcpy(payload + sizeof(js) + sizeof(start_time), infobuf, infolen);
|
||
free(infobuf);
|
||
|
||
rc = wal_hook_append_classified(p, WAL_JOB_START, WAL_MODE_RETAIN,
|
||
payload, (uint32_t)total);
|
||
free(payload);
|
||
return rc;
|
||
}
|
||
|
||
/* bug-1689: WAL_JOB_FINISH's own richer sibling of persist_submit_job_fact()
|
||
* -- mirrors persist_submit_job_start_fact() (bug-1687) exactly, but for the
|
||
* worker-confirmed-finish fact: the sole durable record of the job's real
|
||
* exit_code and its true finish_time. Payload shape: wal_jobstate_t header,
|
||
* immediately followed by a fixed uint64_t finish_time (the receipt-time
|
||
* clock, captured ONCE by the caller rather than re-derived at each apply
|
||
* seam's own apply-time moment), immediately followed by a fixed int32_t
|
||
* exit_code -- a small NATIVE trailer, not capnp (job_exitinfo_t is a single
|
||
* int with no existing capnp mapping, unlike WAL_JOB_START's variable-length
|
||
* JobPIDInfo, so no malloc is needed here -- the whole payload is a fixed
|
||
* size). apply_committed_entry()'s and apply_wal_entry()'s WAL_JOB_FINISH
|
||
* cases decode exactly this shape, and treat a payload of exactly
|
||
* sizeof(wal_jobstate_t) (no trailing bytes) as "state-only, fabricate
|
||
* finish_time and leave exit_info untouched" for backward compatibility
|
||
* with any WAL_JOB_FINISH entry written before this fix. run_time is NOT
|
||
* carried -- it is re-derived (finish_time - start_time) at each apply seam
|
||
* once both are known. */
|
||
int persist_submit_job_finish_fact(persist_t* p, uint64_t jobid, int state,
|
||
uint64_t finish_time, int exit_code)
|
||
{
|
||
wal_jobstate_t js = { .jobid = jobid, .state = (uint32_t)state,
|
||
._pad = 0 };
|
||
int32_t ec = (int32_t)exit_code;
|
||
uint8_t payload[sizeof(js) + sizeof(finish_time) + sizeof(ec)];
|
||
|
||
memcpy(payload, &js, sizeof(js));
|
||
memcpy(payload + sizeof(js), &finish_time, sizeof(finish_time));
|
||
memcpy(payload + sizeof(js) + sizeof(finish_time), &ec, sizeof(ec));
|
||
|
||
return wal_hook_append_classified(p, WAL_JOB_FINISH, WAL_MODE_RETAIN,
|
||
payload, (uint32_t)sizeof(payload));
|
||
}
|
||
|
||
/* bug-1679: see persist.h's doc comment for the full rationale. Mirrors
|
||
* apply_committed_entry()'s own identical Hazard-1-guarded hook fire
|
||
* (its shared WAL_JOB_START/.../DISPATCH case, right after its
|
||
* `if (result != JOB_TRANSITION_APPLIED) return 0;` line) exactly --
|
||
* same guard, same hook, same detail. The only difference is WHO is
|
||
* calling: there it is the apply seam notifying after a REPLAYED
|
||
* transition; here it is a live caller (dispatch_a_job_()) notifying
|
||
* after a transition it already durably persisted itself. */
|
||
void persist_notify_job_updated_already_persisted(cluster_t* c, job_t* j)
|
||
{
|
||
t_apply_seam_notifying = true;
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||
HOOK_DETAIL_JOB_UPDATE, c, j);
|
||
t_apply_seam_notifying = false;
|
||
}
|
||
|
||
/* P0 payload: just the detail value (4 bytes).
|
||
* Full Cap'n Proto serialization is wired in Task 15. */
|
||
|
||
int persist_job_hook_fun(int type, int situation, int detail,
|
||
void* cls, va_list ap)
|
||
{
|
||
(void)type;
|
||
/* Write to WAL only after the state change is complete.
|
||
* BEFORE_EVENT has a different va_list layout and must be ignored. */
|
||
if (situation != HOOK_AFTER_EVENT)
|
||
return 0;
|
||
|
||
/* raft-apply-decoupling task 6.1 (Hazard 1): this AFTER_EVENT firing
|
||
* originated from apply_committed_entry()'s own post-mutation notify —
|
||
* the entry is already durably committed (that is why it is being
|
||
* applied at all), so appending it again here would duplicate it. See
|
||
* t_apply_seam_notifying's doc comment (above apply_committed_entry) for
|
||
* why this MUST be a call-provenance guard, not a `detail` filter:
|
||
* job_ctrl's mark_job_update/mark_job_fail still fire this exact same
|
||
* hook for their own live, still-legitimate append. */
|
||
if (t_apply_seam_notifying)
|
||
return 0;
|
||
|
||
persist_t* p = cls;
|
||
uint32_t wal_type = detail_to_wal_type(detail);
|
||
wal_commit_mode_t mode;
|
||
|
||
if (wal_type == 0)
|
||
return 0;
|
||
mode = wal_classify(wal_type, detail);
|
||
|
||
if (mode == WAL_MODE_WAIT) {
|
||
/* WAL_JOB_SUBMIT durability (WAL append + majority-commit barrier) is
|
||
* an explicit step invoked directly by handle_jobsubmit —
|
||
* persist_submit_job_payloads_and_commit(), called strictly before
|
||
* any job object is constructed (raft-apply-decoupling task 3.1 /
|
||
* bug-1306) — never from this hook. This hook fires only after that
|
||
* commit has already succeeded, purely as a notification, so there
|
||
* is nothing left for it to do here. */
|
||
return 0;
|
||
}
|
||
|
||
/* All other job events: payload = wal_jobstate_t {jobid, state}.
|
||
* State is read from the job_t after the AFTER_EVENT hook updates it. */
|
||
wal_jobstate_t js = {0};
|
||
/* bug-1690: WAL_JOB_RUSAGE's own optional trailing Cap'n Proto
|
||
* JobRunusage bytes -- populated only by the HOOK_DETAIL_JOB_RUSAGE_UPDATE
|
||
* arm below, consumed by the shared append at the bottom of this
|
||
* function. Every other detail leaves this NULL, so their payload stays
|
||
* the plain wal_jobstate_t it always was -- zero behavior change for
|
||
* them. */
|
||
uint8_t* rusage_buf = NULL;
|
||
size_t rusage_len = 0;
|
||
switch (detail) {
|
||
case HOOK_DETAIL_JOB_START: {
|
||
/* args: cluster_t*, job_pidinfo_t*, job_t* */
|
||
(void)va_arg(ap, cluster_t*);
|
||
(void)va_arg(ap, void*);
|
||
job_t* j = va_arg(ap, job_t*);
|
||
js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state;
|
||
break;
|
||
}
|
||
case HOOK_DETAIL_JOB_EXIT: {
|
||
/* args: cluster_t*, xce_jobfinish_t*, job_t* */
|
||
(void)va_arg(ap, cluster_t*);
|
||
(void)va_arg(ap, void*);
|
||
job_t* j = va_arg(ap, job_t*);
|
||
js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state;
|
||
break;
|
||
}
|
||
case HOOK_DETAIL_JOB_FAIL: {
|
||
/* args: cluster_t*, job_failinfo_t*, job_t* */
|
||
(void)va_arg(ap, cluster_t*);
|
||
(void)va_arg(ap, void*);
|
||
job_t* j = va_arg(ap, job_t*);
|
||
js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state;
|
||
break;
|
||
}
|
||
case HOOK_DETAIL_JOB_RUSAGE_UPDATE: {
|
||
/* args: cluster_t*, xce_job_rusages_t* -- bug-451: a DIFFERENT
|
||
* va_list shape than every other case below. job.c's
|
||
* update_job_rusages() fires this detail with the raw rusages
|
||
* payload it received (not the job_t* it already holds
|
||
* write-locked), because this hook's other two listeners
|
||
* (sched_job_rusage_update_fun / apiserver_job_rusage_update_fun)
|
||
* genuinely need the full payload (pids/pgids/usages), not just a
|
||
* job_t*. Reading the second vararg as job_t* here (as the
|
||
* default arm below does) would misinterpret an
|
||
* xce_job_rusages_t* as a job_t* and read state from a byte
|
||
* offset past the smaller struct's end. Look the job up by
|
||
* rusages->jobid via the cluster_t* this call already carries --
|
||
* the same jobmgr lookup job.c's own caller already did moments
|
||
* earlier -- to recover the real state for this generic
|
||
* wal_jobstate_t fact. */
|
||
cluster_t* c = va_arg(ap, cluster_t*);
|
||
xce_job_rusages_t* rusages = va_arg(ap, xce_job_rusages_t*);
|
||
job_t* j = NULL;
|
||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||
key_find(c->jobmgr, (void*)rusages->jobid, (void**)&j);
|
||
unlock(c->jobmgr);
|
||
if (j == NULL)
|
||
return 0; /* job gone by the time this fired -- nothing to log */
|
||
js.jobid = rusages->jobid; js.state = (uint32_t)j->job.state;
|
||
|
||
/* bug-1690: WAL_JOB_RUSAGE previously carried only {jobid, state}
|
||
* -- the same silent-field-drop bug class as bug-1678/1687/1689 --
|
||
* the client-reported per-host CPU/mem usage (j->job.run_usage,
|
||
* just mutated in-place by update_job_rusages() moments ago, under
|
||
* the same write lock this AFTER_EVENT fires inside) was never in
|
||
* the log at all, so a restart replay left job.run_usage
|
||
* permanently NULL. Encode it as a trailing Cap'n Proto JobRunusage
|
||
* blob when present -- it is NULL only in test doubles that stub
|
||
* out update_host_runusage (e.g. test_commit_barrier_reply.c); the
|
||
* real production caller always has one by the time this fires. A
|
||
* NULL run_usage falls back to the legacy state-only payload at
|
||
* this function's shared tail, unchanged. */
|
||
if (j->job.run_usage != NULL)
|
||
(void)capnp_encode_jobrunusage(j->job.run_usage, &rusage_buf,
|
||
&rusage_len);
|
||
break;
|
||
}
|
||
default: {
|
||
/* STOP, RESUME, UPDATE, CKPT, PREASON — args: cluster_t*, job_t* */
|
||
(void)va_arg(ap, cluster_t*);
|
||
job_t* j = va_arg(ap, job_t*);
|
||
js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state;
|
||
break;
|
||
}
|
||
}
|
||
|
||
/* mode == WAL_MODE_RETAIN — fact-shaped job state transition */
|
||
if (rusage_buf != NULL) {
|
||
/* bug-1690: append the header + trailing JobRunusage bytes as one
|
||
* contiguous payload -- persist_submit_job_dispatch_fact()'s/
|
||
* persist_submit_job_start_fact()'s own header+trailer construction
|
||
* (bug-1678/1687), inlined here since WAL_JOB_RUSAGE is hook-driven
|
||
* (from the raw rusages payload update_job_rusages() fires, not a
|
||
* direct persist_submit_job_*_fact() call site job.c could own). */
|
||
size_t total = sizeof(js) + rusage_len;
|
||
uint8_t* payload = malloc(total);
|
||
int rc;
|
||
if (!payload) { free(rusage_buf); return -ENOMEM; }
|
||
memcpy(payload, &js, sizeof(js));
|
||
memcpy(payload + sizeof(js), rusage_buf, rusage_len);
|
||
free(rusage_buf);
|
||
rc = wal_hook_append_classified(p, wal_type, mode, payload,
|
||
(uint32_t)total);
|
||
free(payload);
|
||
return rc;
|
||
}
|
||
return wal_hook_append_classified(p, wal_type, mode, &js, sizeof(js));
|
||
}
|
||
|
||
int persist_queue_hook_fun(int type, int situation, int detail,
|
||
void* cls, va_list ap)
|
||
{
|
||
(void)type;
|
||
if (situation != HOOK_AFTER_EVENT)
|
||
return 0;
|
||
|
||
/* raft-apply-decoupling task 6.1 (Hazard 1, queue analog): see
|
||
* persist_job_hook_fun's identical guard above for the full rationale —
|
||
* apply_committed_entry()'s WAL_QUEUE_CTRL branch fires this same
|
||
* AFTER_EVENT to notify, and this hook appends unconditionally
|
||
* (WAL_QUEUE_CTRL is WAL_MODE_WAIT with no early-return here), so
|
||
* without this guard a live-applied queue entry would re-append and
|
||
* re-trigger a nested majority-commit wait every time the pump walks
|
||
* it. */
|
||
if (t_apply_seam_notifying)
|
||
return 0;
|
||
|
||
persist_t* p = cls;
|
||
uint32_t wal_type = detail_to_wal_type(detail);
|
||
wal_commit_mode_t mode;
|
||
|
||
if (wal_type == 0)
|
||
return 0;
|
||
mode = wal_classify(wal_type, detail);
|
||
|
||
/* args: cluster_t*, queue_t* */
|
||
(void)va_arg(ap, cluster_t*);
|
||
queue_t* q = va_arg(ap, queue_t*);
|
||
|
||
/* Payload: [wal_queuehdr_t][queue name NUL-terminated] */
|
||
const char* name = q->queue.queue ? q->queue.queue : "";
|
||
size_t name_len = strlen(name) + 1;
|
||
size_t total = sizeof(wal_queuehdr_t) + name_len;
|
||
uint8_t* pkt = malloc(total);
|
||
if (!pkt) return -ENOMEM;
|
||
|
||
wal_queuehdr_t hdr = { .status = q->queue.status };
|
||
memcpy(pkt, &hdr, sizeof(hdr));
|
||
memcpy(pkt + sizeof(hdr), name, name_len);
|
||
|
||
/* mode == WAL_MODE_WAIT — the append stays async; the handler barrier
|
||
* owes commit completion before the requester reply (task 3) */
|
||
int rc = wal_hook_append_classified(p, wal_type, mode, pkt,
|
||
(uint32_t)total);
|
||
free(pkt);
|
||
return rc;
|
||
}
|
||
|
||
int persist_worker_hook_fun(int type, int situation, int detail,
|
||
void* cls, va_list ap)
|
||
{
|
||
(void)type;
|
||
if (situation != HOOK_AFTER_EVENT)
|
||
return 0;
|
||
|
||
/* raft-apply-decoupling task 6.1 (Hazard 1, worker analog): see
|
||
* persist_job_hook_fun's identical guard above for the full rationale. */
|
||
if (t_apply_seam_notifying)
|
||
return 0;
|
||
|
||
persist_t* p = cls;
|
||
uint32_t wal_type = detail_to_wal_type(detail);
|
||
wal_commit_mode_t mode;
|
||
|
||
if (wal_type == 0)
|
||
return 0;
|
||
mode = wal_classify(wal_type, detail);
|
||
|
||
/* WAL_WORKER_CTRL payload layout: [uint32_t detail][capnp worker OR name] */
|
||
uint32_t disc = (uint32_t)detail;
|
||
uint8_t* buf = NULL;
|
||
size_t capnp_sz = 0;
|
||
int rc = 0;
|
||
|
||
/* All worker detail types: args are cluster_t* then worker_t*.
|
||
* JOIN uses a unified signature with an extra char* name between them;
|
||
* DEAD and UPDATE use (cluster_t*, worker_t*) directly. */
|
||
(void)va_arg(ap, cluster_t*);
|
||
if (detail == HOOK_DETAIL_WORKER_JOIN)
|
||
(void)va_arg(ap, char*); /* skip name — present only for JOIN */
|
||
worker_t* w = va_arg(ap, worker_t*);
|
||
if (!w) return 0; /* guard: failed join produced no worker slot */
|
||
rc = capnp_encode_worker(&w->host, &buf, &capnp_sz);
|
||
if (rc != 0) return rc;
|
||
|
||
/* Build [disc | capnp bytes] in one allocation */
|
||
size_t total = WORKER_WAL_HDR_SIZE + capnp_sz;
|
||
uint8_t* pkt = malloc(total);
|
||
if (!pkt) { free(buf); return -ENOMEM; }
|
||
memcpy(pkt, &disc, WORKER_WAL_HDR_SIZE);
|
||
memcpy(pkt + WORKER_WAL_HDR_SIZE, buf, capnp_sz);
|
||
free(buf);
|
||
|
||
/* JOIN/UPDATE → WAIT (handler barrier owes completion, task 3);
|
||
* DEAD → RETAIN (controller-detected fact, registered with the engine) */
|
||
rc = wal_hook_append_classified(p, wal_type, mode, pkt, (uint32_t)total);
|
||
free(pkt);
|
||
return rc;
|
||
}
|
||
|
||
/* ============================================================
|
||
* GC retention predicates (P0-5a / P0-8 / P0-13)
|
||
*
|
||
* Placed in persist.c so the unit test binary (which links only
|
||
* persist.c from the controller sources) can access them without
|
||
* pulling in RTTI-dependent job.c / worker.c.
|
||
* ============================================================ */
|
||
|
||
bool job_is_terminal(const job_t* j)
|
||
{
|
||
return (j->job.state & (JOB_STAT_EXIT | JOB_STAT_DONE | JOB_STAT_PERR)) != 0;
|
||
}
|
||
|
||
bool job_should_retain(const job_t* j, const persist_config_t* cfg)
|
||
{
|
||
time_t now = time(NULL);
|
||
|
||
/* TRANSIENT jobs: auto-terminate when TTL elapses after failover.
|
||
* transient_since == 0 means the timestamp was never set — do not GC. */
|
||
if (j->job.state & JOB_STAT_TRANSIENT) {
|
||
if (j->job.transient_since > 0 &&
|
||
now > (time_t)j->job.transient_since + (time_t)cfg->transient_job_timeout_s) {
|
||
return false;
|
||
}
|
||
return true;
|
||
}
|
||
|
||
/* Non-terminal jobs are always retained */
|
||
if (!job_is_terminal(j)) return true;
|
||
|
||
/* Terminal jobs: retain until finished_retain_s expires */
|
||
return (j->job.finish_time + (time_t)cfg->finished_retain_s >= now);
|
||
}
|
||
|
||
bool worker_should_retain(const xce_worker_t* w, const persist_config_t* cfg)
|
||
{
|
||
time_t now = time(NULL);
|
||
|
||
/* HIDDEN workers: admin explicitly removed — clean up at next checkpoint */
|
||
if (w->status & XCE_WORKER_HIDDEN) return false;
|
||
|
||
/* UNAVAILABLE workers: retain until worker_retain_s expires.
|
||
* dead_since == 0 means not yet set — do not GC. */
|
||
if (w->status & XCE_WORKER_UNAVAIL) {
|
||
if (w->dead_since > 0 &&
|
||
now > w->dead_since + (time_t)cfg->worker_retain_s) {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
/* ============================================================
|
||
* WAL truncation (task 7.5)
|
||
*
|
||
* Remove all WAL entries with LSN > cutoff_lsn.
|
||
* Idempotent: calling twice with same cutoff_lsn returns 0 both times.
|
||
* ============================================================ */
|
||
|
||
/* Clamp durable_lsn down to `tip` when the log tip recedes below it (truncate
|
||
* / divergent-suffix discard). Without this, durable_lsn could exceed
|
||
* current_lsn - 1 after a truncate, violating its invariant and letting the
|
||
* commit majority count self at an LSN whose bytes were just removed. MUST be
|
||
* called with p->lock held (mirrors the committed_lsn clamp in the truncate
|
||
* paths). durable_lsn only ever decreases here — the sole exception to its
|
||
* monotonic-non-decreasing rule (ha-commit-durability 2.1 State Management). */
|
||
static void clamp_durable_lsn_locked(persist_t* p, uint64_t tip)
|
||
{
|
||
uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire);
|
||
if (dur > tip)
|
||
atomic_store_explicit(&p->durable_lsn, tip, memory_order_release);
|
||
}
|
||
|
||
int persist_truncate_wal_above(persist_t* p, uint64_t cutoff_lsn)
|
||
{
|
||
pthread_mutex_lock(&p->lock);
|
||
uint64_t before = p->current_lsn;
|
||
int rc = truncate_wal_above_locked(p, cutoff_lsn);
|
||
uint64_t after = p->current_lsn;
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
/* raft-apply-decoupling task 1.4: fire the truncation wake only when a
|
||
* truncation actually removed entries above cutoff_lsn (current_lsn
|
||
* strictly decreased -- both truncate_wal_above_locked branches set
|
||
* current_lsn = cutoff_lsn + 1 on a real truncation) -- mirrors the
|
||
* durable-advance wake's own before/after guard (see fsync_worker
|
||
* above) so a no-op truncate (nothing above cutoff_lsn, or an I/O
|
||
* error before current_lsn is touched) never spuriously fails a
|
||
* continuation whose entry is still safely in the log. Called AFTER
|
||
* releasing p->lock: the callback (the commit engine) takes only
|
||
* ce->lock, preserving the cycle-free p->lock -> ce->lock edge already
|
||
* established by the durable-advance wake. */
|
||
if (rc == 0 && after < before) {
|
||
/* raft-apply-decoupling task 7.4 perf-fix: a real truncation may
|
||
* ftruncate()/unlink() segment files the apply-pump's segment-lookup
|
||
* cache still points into -- invalidate it so a subsequent
|
||
* apply_pump_read_entry can never read truncated-away data. p->lock
|
||
* is already released above and apply_pump_lock is never held by
|
||
* this call path (persist_apply_pump_run is the only place that
|
||
* holds it, during its own walk), so no lock-order hazard here. */
|
||
pthread_mutex_lock(&p->apply_pump_lock);
|
||
if (p->apply_pump_cache_fd >= 0) {
|
||
close(p->apply_pump_cache_fd);
|
||
p->apply_pump_cache_fd = -1;
|
||
}
|
||
p->apply_pump_cache_next_lsn = 0;
|
||
pthread_mutex_unlock(&p->apply_pump_lock);
|
||
|
||
if (p->on_truncate != NULL) {
|
||
p->on_truncate(p->truncate_cb_ctx, cutoff_lsn);
|
||
}
|
||
}
|
||
return rc;
|
||
}
|
||
|
||
/* persist_committed_lsn_advance_atomic (2026-07-10, controller-invariant-audit
|
||
* finding): race-safe monotonic-max advance of p->committed_lsn via a CAS
|
||
* retry loop. Replaces the naive "load; if target>cur store" pattern, which
|
||
* can lose an update if two writers race: persist_promotion_advance_committed
|
||
* (below) and ha_standby_advance_committed_and_apply (ha.c) can both run
|
||
* during the promotion window before role flips to CTRL_PRIMARY (a
|
||
* still-in-flight WAL catch-up entry processed on another thread at the same
|
||
* instant as this node's own election completing). No lock required -- CAS
|
||
* only succeeds if the value hasn't changed since the read, so a losing
|
||
* racer retries with the freshly-observed value and can never clobber a
|
||
* larger value with a smaller one. */
|
||
uint64_t persist_committed_lsn_advance_atomic(persist_t* p, uint64_t target)
|
||
{
|
||
uint64_t cur = atomic_load_explicit(&p->committed_lsn, memory_order_acquire);
|
||
while (target > cur) {
|
||
if (atomic_compare_exchange_weak_explicit(&p->committed_lsn, &cur, target,
|
||
memory_order_release,
|
||
memory_order_acquire))
|
||
return target;
|
||
/* cur was refreshed to the current value by the failed CAS; retry. */
|
||
}
|
||
return cur;
|
||
}
|
||
|
||
/* persist_promotion_advance_committed (2026-07-10, promotion apply-lag fix):
|
||
* unconditionally advance p->committed_lsn toward `safe_lsn` at promotion
|
||
* time. persist_truncate_wal_above only sets committed_lsn on the branches
|
||
* of truncate_wal_above_locked that perform a REAL truncation (partial
|
||
* truncate, or the full-segment-purge demotion-convergence case); its no-op
|
||
* early return (nothing above cutoff_lsn to remove -- the common case for a
|
||
* newly-elected leader whose own log already covers safe_lsn) leaves
|
||
* committed_lsn untouched at its pre-election value. Without this call,
|
||
* committed_lsn -- and therefore apply, which is commit-gated
|
||
* (raft-apply-decoupling) -- stalls until the normal per-entry ack-driven
|
||
* commit path organically catches it up, which can take seconds while
|
||
* surviving standbys reconnect and re-ACK. During that gap a job already
|
||
* committed on a majority is invisible via expect-job/joblist on the very
|
||
* primary that is authoritative for it (bug-828 class).
|
||
*
|
||
* DESIGN NOTE -- why this function never forces a synchronous flush (round 3,
|
||
* 2026-07-10, controller-invariant-audit): an earlier round of this fix
|
||
* called flusher_flush_once(p) here when durable_lsn lagged safe_lsn, to
|
||
* force this node's own fsync to catch up immediately rather than waiting on
|
||
* fsync_worker's cadence. That was a real bug-1198-CLASS thread-role
|
||
* violation: the real multi-node winning-vote-reply path reaches
|
||
* raft_cb_become_leader (and so this function) via peer_ctrl_classify's
|
||
* VTASK_LIVENESS dispatch (ha.c ~5935-5940) -- a task class the design
|
||
* DOCUMENTS as "never admission-gated" / never expected to block (see the
|
||
* bug-1198 hazard box a few hundred lines above peer_ctrl_classify's
|
||
* definition, which closed the EXACT same class of bug for
|
||
* HA_REPL_SEND_SNAPSHOT's persist_checkpoint). A blocking fdatasync here
|
||
* would reintroduce that closed bug class on the pool's liveness-reserved
|
||
* workers. The visibility gap the force-flush was trying to close is instead
|
||
* closed non-blockingly by ha_start_primary_heartbeat sending its first
|
||
* heartbeat synchronously at promotion (bug-1432) -- that lets a reconnecting
|
||
* standby's real position get learned, and the entry gets committed via the
|
||
* normal ack-driven commit_advance_locked path, within one network RTT
|
||
* instead of waiting a full heartbeat interval. A live chaotic soak (10
|
||
* rounds) confirmed durable-capped-only + immediate-heartbeat together reach
|
||
* committed_job_loss=0 with zero blocking calls added to this function.
|
||
*
|
||
* Durable-capped (never advances committed_lsn past this node's own
|
||
* durable_lsn -- see persistence.md's "acknowledged implies durable on a
|
||
* majority" core invariant) and monotonic + race-safe via
|
||
* persist_committed_lsn_advance_atomic. */
|
||
void persist_promotion_advance_committed(persist_t* p, uint64_t safe_lsn)
|
||
{
|
||
if (!p)
|
||
return;
|
||
uint64_t durable = atomic_load_explicit(&p->durable_lsn,
|
||
memory_order_acquire);
|
||
uint64_t target = safe_lsn < durable ? safe_lsn : durable;
|
||
persist_committed_lsn_advance_atomic(p, target);
|
||
}
|
||
|
||
/* Lock-held core of persist_truncate_wal_above (see the forward declaration and
|
||
* the conflict-guard reuse near persist_wal_write_replicated). Caller MUST hold
|
||
* p->lock; this function never touches the lock so it is reusable from another
|
||
* already-locked path. */
|
||
static int truncate_wal_above_locked(persist_t* p, uint64_t cutoff_lsn)
|
||
{
|
||
/* No-op: nothing written yet, or cutoff covers all written entries */
|
||
if (p->current_lsn == 0 || cutoff_lsn + 1 >= p->current_lsn) {
|
||
return 0;
|
||
}
|
||
|
||
/* --- Collect segment filenames and sort by start LSN --- */
|
||
uint64_t seg_lsns[4096];
|
||
int nseg = 0;
|
||
|
||
DIR* d = opendir(p->wal_path);
|
||
if (d) {
|
||
struct dirent* e;
|
||
while ((e = readdir(d)) != NULL &&
|
||
nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) {
|
||
if (strncmp(e->d_name, "seg-", 4) != 0)
|
||
continue;
|
||
uint64_t sl = 0;
|
||
if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &sl) != 1)
|
||
continue;
|
||
seg_lsns[nseg++] = sl;
|
||
}
|
||
closedir(d);
|
||
}
|
||
|
||
if (nseg == 0) {
|
||
return 0;
|
||
}
|
||
|
||
/* Insertion sort ascending by start LSN */
|
||
for (int i = 1; i < nseg; i++) {
|
||
uint64_t key = seg_lsns[i];
|
||
int j = i - 1;
|
||
while (j >= 0 && seg_lsns[j] > key) {
|
||
seg_lsns[j + 1] = seg_lsns[j];
|
||
j--;
|
||
}
|
||
seg_lsns[j + 1] = key;
|
||
}
|
||
|
||
/* --- Find segment containing cutoff_lsn --- */
|
||
int cutoff_seg_idx = -1;
|
||
for (int i = 0; i < nseg; i++) {
|
||
if (seg_lsns[i] <= cutoff_lsn)
|
||
cutoff_seg_idx = i;
|
||
else
|
||
break;
|
||
}
|
||
|
||
if (cutoff_seg_idx < 0) {
|
||
/* cutoff_lsn is below the oldest retained segment's start LSN, so EVERY
|
||
* on-disk entry has LSN > cutoff_lsn and must be removed (the function's
|
||
* contract: "remove all entries with LSN > cutoff_lsn"). This is the
|
||
* demotion-convergence case (ha-convergence-recovery Req 1.2): a demoted
|
||
* ex-primary whose committed boundary sits below its entire retained log
|
||
* must discard all of it and collapse the tip to cutoff_lsn + 1. Delete
|
||
* every segment, close the active fd, and reset the in-memory tip. */
|
||
for (int i = 0; i < nseg; i++) {
|
||
char del_path[4096];
|
||
snprintf(del_path, sizeof(del_path),
|
||
WAL_SEG_FMT, p->wal_path, seg_lsns[i]);
|
||
unlink(del_path);
|
||
}
|
||
if (p->wal_fd >= 0) {
|
||
fdatasync(p->wal_fd);
|
||
close(p->wal_fd);
|
||
p->wal_fd = -1;
|
||
}
|
||
p->seg_offset = 0;
|
||
p->seg_path[0] = '\0';
|
||
p->current_lsn = cutoff_lsn + 1;
|
||
p->last_crc = WAL_CRC_NONE;
|
||
atomic_store_explicit(&p->committed_lsn, cutoff_lsn,
|
||
memory_order_release);
|
||
clamp_durable_lsn_locked(p, cutoff_lsn);
|
||
/* Keep the per-LSN term index consistent: drop terms for removed LSNs. */
|
||
truncate_term_index_locked(p, cutoff_lsn);
|
||
return 0;
|
||
}
|
||
|
||
/* --- Scan segment to find byte offset of first entry with lsn > cutoff --- */
|
||
char cutoff_seg_path[4096];
|
||
snprintf(cutoff_seg_path, sizeof(cutoff_seg_path),
|
||
WAL_SEG_FMT, p->wal_path, seg_lsns[cutoff_seg_idx]);
|
||
|
||
int scan_fd = open(cutoff_seg_path, O_RDONLY);
|
||
if (scan_fd < 0) {
|
||
return -errno;
|
||
}
|
||
|
||
uint32_t scan_off = 0;
|
||
uint32_t trunc_off = 0;
|
||
uint32_t new_last_crc = WAL_CRC_NONE;
|
||
bool found_cut = false;
|
||
|
||
for (;;) {
|
||
wal_entry_hdr_t hdr;
|
||
ssize_t n = pread(scan_fd, &hdr, sizeof(hdr), (off_t)scan_off);
|
||
if (n < (ssize_t)sizeof(hdr))
|
||
break; /* short read or EOF */
|
||
/* Pre-allocated zero region: type==0 is not a valid WAL type (min=1) */
|
||
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
|
||
break;
|
||
if (hdr.lsn > cutoff_lsn) {
|
||
trunc_off = scan_off;
|
||
found_cut = true;
|
||
break;
|
||
}
|
||
new_last_crc = hdr.crc;
|
||
scan_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
}
|
||
close(scan_fd);
|
||
|
||
if (!found_cut)
|
||
trunc_off = scan_off; /* end of valid data in segment */
|
||
|
||
/* --- Close the current wal_fd --- */
|
||
if (p->wal_fd >= 0) {
|
||
fdatasync(p->wal_fd);
|
||
close(p->wal_fd);
|
||
p->wal_fd = -1;
|
||
}
|
||
|
||
/* --- Truncate the cutoff segment at the computed offset --- */
|
||
int new_fd = open(cutoff_seg_path, O_RDWR);
|
||
if (new_fd < 0) {
|
||
return -errno;
|
||
}
|
||
if (ftruncate(new_fd, (off_t)trunc_off) != 0) {
|
||
int saved = errno;
|
||
close(new_fd);
|
||
return -saved;
|
||
}
|
||
fdatasync(new_fd);
|
||
|
||
/* --- Delete all segments after the cutoff segment --- */
|
||
for (int i = cutoff_seg_idx + 1; i < nseg; i++) {
|
||
char del_path[4096];
|
||
snprintf(del_path, sizeof(del_path),
|
||
WAL_SEG_FMT, p->wal_path, seg_lsns[i]);
|
||
unlink(del_path);
|
||
}
|
||
|
||
/* --- Update persist_t state under lock --- */
|
||
p->wal_fd = new_fd;
|
||
p->seg_offset = trunc_off;
|
||
snprintf(p->seg_path, sizeof(p->seg_path),
|
||
WAL_SEG_FMT, p->wal_path, seg_lsns[cutoff_seg_idx]);
|
||
p->current_lsn = cutoff_lsn + 1;
|
||
p->last_crc = new_last_crc;
|
||
atomic_store_explicit(&p->committed_lsn, cutoff_lsn, memory_order_release);
|
||
clamp_durable_lsn_locked(p, cutoff_lsn);
|
||
/* Keep the per-LSN term index consistent: drop terms for removed LSNs. */
|
||
truncate_term_index_locked(p, cutoff_lsn);
|
||
|
||
return 0;
|
||
}
|