lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-entity-snapshots): entity-bearing checkpoint write path for jobs (task 2.2)

Browse source 
write_snapshot now Cap'n-Proto-encodes every retained job under nested
table+per-entity RTTI_LOCK_READ (table lock alone does not exclude a
concurrent per-entity writer -- design.md's central correctness finding),
appends length-prefixed job frames after a new format_version marker, and
fixes a previously-unchecked fdatasync return. Any failure anywhere in the
path (encode, write, sync) discards the whole checkpoint -- no partial
file at the final path, proven by a real mid-encode-loop fault-injection
test (forces one specific job's encode to fail among N, asserts nothing
survives).

Also fixed two pre-existing bugs in task 1.1's test fixture: terminal-state
seeded jobs need a non-zero finish_time or job_should_retain's retention
window silently excludes them from any checkpoint; the fixture needs
xstring_table_init or every seeded job's submit/start_info silently
decodes as NULL.

Round 1 of this task also touched load_snapshot/persist.h to make the
round-trip tests pass end-to-end, but review caught a live hazard: it
advanced current_lsn/checkpoint_lsn BEFORE the entity-decode loop ran, so
a decode failure still left the position advanced -- reachable today
through the unmodified ha_handle_snapshot_chunk production path, not
hypothetical. Remediated by fully reverting load_snapshot/persist.h to
byte-identical pre-task-2.2 state and rewriting the round-trip tests to
parse write_snapshot's raw on-disk bytes directly (read_Job/decode_Job)
instead of round-tripping through persist_load_snapshot -- proves the
write side without depending on load_snapshot, which stays task 5.2's
scope entirely, not started here.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JDYFbrgyDHbvNYcxApKhrX
This commit is contained in:
Rongsong Shen 2026-07-01 06:30:28 +00:00
parent 0f344da89a
commit 5929af1078
4 changed files with 453 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

209
src/controller/persist.c
View file

@ -1256,6 +1256,13 @@ void persist_test_set_force_fsync_fail(persist_t* p, int fail)
memory_order_release);
}
void persist_test_set_force_job_encode_fail_at(persist_t* p, int index)
{
if (p)
atomic_store_explicit(&p->test_force_job_encode_fail_at, index + 1,
memory_order_relaxed);
}
unsigned persist_test_replicated_fdatasync_count(persist_t* p)
{
if (!p)
@ -1472,8 +1479,142 @@ static int persist_write_checkpoint_marker(persist_t* p)
return 0;
}
/* Write a P0 sentinel snapshot: "XCESNAP\0" + checkpoint_lsn (16 bytes).
* Full Cap'n Proto serialization is Task 11 when recovery needs to read it back. */
/* Entity-bearing checkpoint payload (ha-entity-snapshots task 2.2, Req
* 1.1-1.4). Physical layout appended after the existing legacy header
* (design.md "Checkpoint File Layout"):
* [uint32_t format_version]
* [uint32_t job_count]
* [job_count * { uint32_t payload_len; uint8_t capnp_Job_bytes[payload_len] }]
* [uint32_t queue_count]
* [queue_count * { ... }] -- written as 0 for now; task 3.2 fills this in
* without needing to touch the job path above. */
#define SNAPSHOT_FORMAT_VERSION 1u
typedef struct {
uint8_t* buf;
uint32_t len;
} snap_job_frame_t;
static void snap_free_job_frames(xlist_t* frames)
{
for (int i = 0; i < frames->count; i++) {
snap_job_frame_t* fr = (snap_job_frame_t*)xlist_get(frames, i);
if (fr) {
free(fr->buf);
free(fr);
}
}
xlist_fini(frames);
}
/* Collect + encode every retained job into `frames`, one malloc'd
* snap_job_frame_t per job, under a point-in-time-consistent walk
* (design.md "Existing Architecture Analysis" / Requirement 1.2): the
* jobmgr table's own RTTI_LOCK_READ is held for the ENTIRE walk (excludes
* concurrent insert/delete), and each retained job's own RTTI_LOCK_READ is
* ADDITIONALLY taken nested around just that job's encode call (excludes a
* concurrent per-entity writer for exactly the duration of its encode --
* the table lock alone does not do this). No lock is held during the
* disk write itself: this function only builds in-memory frames; the
* caller writes them to disk after the table walk has fully released both
* locks.
*
* Filters via job_should_retain (Requirement 1.1) so the checkpoint's
* notion of "live" matches the existing GC retention policy.
*
* Returns 0 on success (frames populated; `frames` is always left
* xlist_init'd -- even zero results -- so the caller can always
* xlist_fini it) or a negative errno on the first encode failure, in which
* case every frame collected so far has already been freed and `frames`
* is empty (Requirement 1.3: a mid-walk failure must not leave partial
* entity content behind). */
static int collect_job_frames(persist_t* p, xlist_t* frames)
{
cluster_t* c = p->cluster;
int rc = 0;
int retained_index = 0;
xlist_init(frames, 64);
/* No live cluster back-pointer (e.g. a bare unit-test persist_t) --
* zero jobs is not a failure. */
if (c == NULL || c->jobmgr == NULL)
return 0;
lock(c->jobmgr, RTTI_LOCK_READ);
iterator_t it;
int irc = iter_make(&it, c->jobmgr);
if (irc < 0) {
unlock(c->jobmgr);
return 0; /* nothing to walk -- mirrors collect_jobs' empty-table handling */
}
for (;;) {
job_t* j = NULL;
int nrc = iter_next(&it, (void**)&j);
if (nrc < 0 || nrc == RTTI_ITER_NO_DATA)
break;
if (j == NULL)
continue;
if (!job_should_retain(j, &p->config))
continue;
uint8_t* buf = NULL;
size_t sz = 0;
int enc_rc;
lock(j, RTTI_LOCK_READ);
#ifdef XCE_TESTING
int forced = atomic_load_explicit(&p->test_force_job_encode_fail_at,
memory_order_relaxed);
if (forced > 0 && (forced - 1) == retained_index)
enc_rc = -EIO; /* simulated mid-loop encode failure */
else
enc_rc = capnp_encode_job(&j->job, &buf, &sz);
#else
enc_rc = capnp_encode_job(&j->job, &buf, &sz);
#endif
unlock(j);
retained_index++;
if (enc_rc != 0) {
rc = enc_rc;
break;
}
snap_job_frame_t* fr = (snap_job_frame_t*)malloc(sizeof(*fr));
if (fr == NULL) {
free(buf);
rc = -ENOMEM;
break;
}
fr->buf = buf;
fr->len = (uint32_t)sz;
if (xlist_store(frames, fr) < 0) {
free(buf);
free(fr);
rc = -ENOMEM;
break;
}
}
iter_destroy(&it);
unlock(c->jobmgr);
if (rc != 0)
snap_free_job_frames(frames); /* Requirement 1.3: discard, not partial */
return rc;
}
/* Write an entity-bearing checkpoint: legacy header (unchanged) + entity
* payload (ha-entity-snapshots task 2.2/3.2, Req 1.1-1.4, 3.1-3.3). Any
* failure at any step -- collect, encode, write, sync -- leaves NO file at
* the final path; the tmp file is always unlinked on error. This is a
* retriable-next-checkpoint-cycle failure, never a crash/abort (controller
* never self-terminates on an operational condition). */
static int write_snapshot(persist_t* p)
{
char tmp_path[4096];
@ -1484,9 +1625,18 @@ static int write_snapshot(persist_t* p)
snprintf(final_path, sizeof(final_path),
"%s/snap-%016" PRIx64 ".capnp", p->snap_path, p->checkpoint_lsn);
/* Collect + encode entities BEFORE opening the tmp file: a collection
* failure (Requirement 1.3) then never creates a tmp file at all. */
xlist_t job_frames;
int rc = collect_job_frames(p, &job_frames);
if (rc != 0)
return rc;
int fd = open(tmp_path, O_CREAT | O_WRONLY | O_TRUNC, 0600);
if (fd < 0)
if (fd < 0) {
snap_free_job_frames(&job_frames);
return -errno;
}
/* Sentinel: magic (8) + checkpoint_lsn (8) + committed-config reference
* (1 present + 8 tag + 32 sha = 41 bytes; ha-term-safety task 4.2 r1).
@ -1501,13 +1651,56 @@ static int write_snapshot(persist_t* p)
memcpy(header + 17, &p->cfgref_tag, 8);
memcpy(header + 25, p->cfgref_sha, 32);
ssize_t nw = write(fd, header, sizeof(header));
fdatasync(fd);
close(fd);
if (write(fd, header, sizeof(header)) != (ssize_t)sizeof(header))
rc = -EIO;
if (nw != (ssize_t)sizeof(header)) {
/* Entity payload: format_version, then job frames (design.md "Checkpoint
* File Layout"). No table/entity lock is held here -- collect_job_frames
* already released both before returning. */
if (rc == 0) {
uint32_t fmt = SNAPSHOT_FORMAT_VERSION;
if (write(fd, &fmt, sizeof(fmt)) != (ssize_t)sizeof(fmt))
rc = -EIO;
}
if (rc == 0) {
uint32_t job_count = (uint32_t)job_frames.count;
if (write(fd, &job_count, sizeof(job_count)) != (ssize_t)sizeof(job_count))
rc = -EIO;
}
for (int i = 0; rc == 0 && i < job_frames.count; i++) {
snap_job_frame_t* fr = (snap_job_frame_t*)xlist_get(&job_frames, i);
if (write(fd, &fr->len, sizeof(fr->len)) != (ssize_t)sizeof(fr->len)) {
rc = -EIO;
break;
}
if (write(fd, fr->buf, fr->len) != (ssize_t)fr->len) {
rc = -EIO;
break;
}
}
/* Queue trailer: task 3.2 populates this with the real collected count
* and frames without needing to touch anything above. Writing an
* explicit 0 now (rather than omitting the section) keeps the on-disk
* layout complete per design.md, so a reader can always expect a
* queue_count word after the job frames. */
if (rc == 0) {
uint32_t queue_count = 0;
if (write(fd, &queue_count, sizeof(queue_count)) != (ssize_t)sizeof(queue_count))
rc = -EIO;
}
if (rc == 0 && fdatasync(fd) != 0)
rc = -errno;
close(fd);
snap_free_job_frames(&job_frames);
if (rc != 0) {
unlink(tmp_path);
return -EIO;
return rc;
}
if (rename(tmp_path, final_path) != 0) {