lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(config-upgrade-snapshot): collect host admin-state frames into checkpoint

Browse source 
Adds collect_host_frames + capnp_encode_host_admin_state, walking
c->workermgr the same way collect_job_frames/collect_queue_frames do.
Every host currently in the table gets one HostAdminState frame
(status masked to XCE_WORKER_ADMIN_MASK only, no dynamic/liveness
fields), written into write_snapshot after the queue trailer. Task 2.1
of 19.

Requirements: 1.6, 1.7, 1.9
This commit is contained in:
Rongsong Shen 2026-07-24 09:17:46 +00:00
parent 9e68335aee
commit a71aa3f8a1
3 changed files with 594 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

193
src/controller/persist.c
View file

@ -101,6 +101,34 @@ static int capnp_encode_worker(const xce_worker_t* w, uint8_t** out, size_t* out
return 0;
}
/* Serialize a host admin-state frame (config-upgrade-snapshot task 2.1, Req
* 1.6/1.7/1.9) to a malloc'd Cap'n Proto buffer. Mirrors capnp_encode_job
* above exactly -- capnp_host_admin_state_t (typemap.h) is a frame-only
* (name, status) pair, never the full xce_worker_t (design.md Design
* Invariant 5). Caller must free(*out) on success. */
static int capnp_encode_host_admin_state(const capnp_host_admin_state_t* h,
uint8_t** out, size_t* out_len)
{
struct capn c;
struct capn_segment* cs;
struct HostAdminState ch;
HostAdminState_ptr hp;
capn_init_malloc(&c);
cs = capn_root(&c).seg;
encode_HostAdminState(cs, &ch, (capnp_host_admin_state_t*)h);
hp = new_HostAdminState(cs);
write_HostAdminState(&ch, hp);
capn_setp(capn_root(&c), 0, hp.p);
*out_len = capn_size(&c);
*out = malloc(*out_len);
if (!*out) { capn_free(&c); return -ENOMEM; }
capn_write_mem(&c, *out, *out_len, 0);
capn_free(&c);
return 0;
}
/* WAL_WORKER_CTRL payload: [uint32_t detail][capnp bytes...] or [uint32_t detail][name\0] */
#define WORKER_WAL_HDR_SIZE sizeof(uint32_t)
@ -1869,6 +1897,139 @@ static int collect_queue_frames(persist_t* p, xlist_t* frames)
return rc;
}
/* Bits set by admin commands that must survive a checkpoint round-trip.
* Mirrors worker.c's and lib/wobj.c's own local XCE_WORKER_ADMIN_MASK
* definitions -- both already independently duplicate this exact bit list
* (design.md Design Invariant 6 notes this pre-existing duplication); this
* is a third, deliberate copy for the same reason (no shared header exists
* for it today), not a new layering dependency on worker.c. */
#define PERSIST_WORKER_ADMIN_MASK \
(XCE_WORKER_LOCKEDU | XCE_WORKER_CLOSED | XCE_WORKER_HIDDEN \
| XCE_WORKER_CLOSED_BY_ADMIN)
typedef struct {
uint8_t* buf;
uint32_t len;
} snap_host_frame_t;
static void snap_free_host_frames(xlist_t* frames)
{
for (int i = 0; i < frames->count; i++) {
snap_host_frame_t* fr = (snap_host_frame_t*)xlist_get(frames, i);
if (fr) {
free(fr->buf);
free(fr);
}
}
xlist_fini(frames);
}
/* Collect + encode every host in c->workermgr into `frames`, one malloc'd
* snap_host_frame_t (identical {buf,len} shape to snap_job_frame_t -- both
* carry a capnp payload) per host, under the same point-in-time-consistent
* walk discipline as
* collect_job_frames/collect_queue_frames above (design.md "Existing
* Architecture Analysis" / Requirement 1.2): the workermgr table's own
* RTTI_LOCK_READ is held for the ENTIRE walk, and each worker's own
* RTTI_LOCK_READ is ADDITIONALLY taken nested around just that host's
* (name, status) read and encode. No lock is held during the disk write
* itself.
*
* Like collect_queue_frames and UNLIKE collect_job_frames, no retention
* predicate is applied: every host currently in the live workermgr table
* gets exactly one frame (mirrors collect_queue_frames's own "mere presence
* already means it's included" precedent above), so restore later sees a
* frame for every host this member currently knows about, whether or not
* any admin bit happens to be set on it right now.
*
* Each frame carries ONLY the host's admin-issued status bits
* (w->host.status & PERSIST_WORKER_ADMIN_MASK) -- never the raw status
* (which also carries dynamic/liveness bits like XCE_WORKER_UNAVAIL) and
* never any other xce_worker_t field (attrs, resvals, dead_since, ...):
* Requirement 1.7's exclusion, design.md Design Invariant 5 (never inherit
* a live struct's dynamic fields just because reusing it is convenient --
* capnp_encode_worker/Worker is deliberately NOT used here). `name` is
* read directly off w->host.name -- the same field key_find(c->workermgr,
* name) looks up elsewhere in this codebase (join.c:91, worker.c:524,645,751,
* lib/wobj.c:212), so a later restore's key_find(name) lookup matches
* exactly what was collected here.
*
* Returns 0 on success (frames populated; `frames` is always left
* xlist_init'd -- even zero results -- so the caller can always
* xlist_fini it) or a negative errno on the first encode failure, in which
* case every frame collected so far has already been freed and `frames`
* is empty (Requirement 1.3: a mid-walk failure must not leave partial
* entity content behind). */
static int collect_host_frames(persist_t* p, xlist_t* frames)
{
cluster_t* c = p->cluster;
int rc = 0;
xlist_init(frames, 64);
/* No live cluster back-pointer (e.g. a bare unit-test persist_t) --
* zero hosts is not a failure. */
if (c == NULL || c->workermgr == NULL)
return 0;
lock(c->workermgr, RTTI_LOCK_READ);
iterator_t it;
int irc = iter_make(&it, c->workermgr);
if (irc < 0) {
unlock(c->workermgr);
return 0; /* nothing to walk -- mirrors collect_job/queue_frames' empty-table handling */
}
for (;;) {
worker_t* w = NULL;
int nrc = iter_next(&it, (void**)&w);
if (nrc < 0 || nrc == RTTI_ITER_NO_DATA)
break;
if (w == NULL)
continue;
capnp_host_admin_state_t h;
uint8_t* buf = NULL;
size_t sz = 0;
int enc_rc;
lock(w, RTTI_LOCK_READ);
h.name = w->host.name;
h.status = w->host.status & PERSIST_WORKER_ADMIN_MASK;
enc_rc = capnp_encode_host_admin_state(&h, &buf, &sz);
unlock(w);
if (enc_rc != 0) {
rc = enc_rc;
break;
}
snap_host_frame_t* fr = (snap_host_frame_t*)malloc(sizeof(*fr));
if (fr == NULL) {
free(buf);
rc = -ENOMEM;
break;
}
fr->buf = buf;
fr->len = (uint32_t)sz;
if (xlist_store(frames, fr) < 0) {
free(buf);
free(fr);
rc = -ENOMEM;
break;
}
}
iter_destroy(&it);
unlock(c->workermgr);
if (rc != 0)
snap_free_host_frames(frames); /* Requirement 1.3: discard, not partial */
return rc;
}
/* Write an entity-bearing checkpoint: legacy header (unchanged) + entity
* payload (ha-entity-snapshots task 2.2/3.2, Req 1.1-1.4, 3.1-3.3). Any
* failure at any step -- collect, encode, write, sync -- leaves NO file at
@ -1899,10 +2060,19 @@ static int write_snapshot(persist_t* p)
return rc;
}
xlist_t host_frames;
rc = collect_host_frames(p, &host_frames);
if (rc != 0) {
snap_free_job_frames(&job_frames);
snap_free_queue_frames(&queue_frames);
return rc;
}
int fd = open(tmp_path, O_CREAT | O_WRONLY | O_TRUNC, 0600);
if (fd < 0) {
snap_free_job_frames(&job_frames);
snap_free_queue_frames(&queue_frames);
snap_free_host_frames(&host_frames);
return -errno;
}
@ -1976,12 +2146,35 @@ static int write_snapshot(persist_t* p)
}
}
/* Host trailer (config-upgrade-snapshot task 2.1, Req 1.6, 1.7, 1.9):
* capnp HostAdminState frames, same [payload_len][capnp bytes] shape as
* the job frame loop above (design.md "Checkpoint File Layout": host
* frames come after queue frames). */
if (rc == 0) {
uint32_t host_count = (uint32_t)host_frames.count;
if (write(fd, &host_count, sizeof(host_count)) != (ssize_t)sizeof(host_count))
rc = -EIO;
}
for (int i = 0; rc == 0 && i < host_frames.count; i++) {
snap_host_frame_t* fr = (snap_host_frame_t*)xlist_get(&host_frames, i);
if (write(fd, &fr->len, sizeof(fr->len)) != (ssize_t)sizeof(fr->len)) {
rc = -EIO;
break;
}
if (write(fd, fr->buf, fr->len) != (ssize_t)fr->len) {
rc = -EIO;
break;
}
}
if (rc == 0 && fdatasync(fd) != 0)
rc = -errno;
close(fd);
snap_free_job_frames(&job_frames);
snap_free_queue_frames(&queue_frames);
snap_free_host_frames(&host_frames);
if (rc != 0) {
unlink(tmp_path);