lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(config-upgrade-snapshot): embed active configuration bytes into every checkpoint

Browse source 
write_snapshot now appends a flat [config_bytes_len][raw bytes]
section after the host frames, reading the locally-held active config
straight off disk (read_active_config_bytes, persist.c) rather than
calling into config_repl.c -- mirrors scan_latest_committed_config_ref's
existing precedent of avoiding a persist.c -> config_repl.c
function-call dependency. Populated on every checkpoint, not only
marker-triggered ones (Req 2.4). No local config yet (early boot)
degrades to an explicit empty section rather than failing the
checkpoint; genuine corruption still fails loud. Task 2.3 of 19.

Requirements: 2.1, 2.3, 2.4, 5.1
This commit is contained in:
Rongsong Shen 2026-07-24 11:56:12 +00:00
parent 70ff3e5b80
commit 387ea35c9f
3 changed files with 494 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

304
src/controller/testsuite/test_persist_snapshot_entities.c
View file

@ -52,6 +52,7 @@
#include <errno.h>
#include <fcntl.h>
#include <openssl/sha.h>
#include <pthread.h>
#include <stdatomic.h>
#include <stdbool.h>
@ -66,8 +67,10 @@
#include <unity/unity.h>
#include "cluster.h"
#include "config_repl.h"
#include "controller.capnp.h"
#include "jobj.h"
#include "myconf.h"
#include "qobj.h"
#include "wobj.h"
#include "xce/cluster_def.h"
@ -174,6 +177,11 @@ static void snapent_job_cmdline(int i, char* buf, size_t buf_sz)
typedef struct {
cluster_t* c;
char tmpdir[256];
/* Only populated/wired (f->c->myconf = &f->myconf) by tests that call
* snapent_wire_myconf -- task 2.3's config-bytes-section tests. Every
* other existing test leaves f->c->myconf NULL, exactly as before this
* field was added (zero footprint on already-passing tests). */
myconf_t myconf;
} snapent_fixture_t;
/* jobj.c's jobsub/startinfo tables are process-global and idempotently
@ -338,6 +346,29 @@ static snapent_fixture_t* snapent_make(int n_jobs, int m_queues)
return f;
}
/* Wire a real myconf_t onto the fixture's cluster (config-upgrade-snapshot
* task 2.3): f->myconf.workdir/name are set to the SAME (f->tmpdir,
* "snapent") pair persist_set_workdir already used above -- exactly mirroring
* production wiring, where a single controller instance's persist_set_workdir
* and c->myconf->workdir/name are literally the same values (cluster.c:1408-
* 1441). effective_dir = workdir/name is therefore shared between persist's
* OWN data/{wal,snap} subtree and myconf's conf/ subtree with no collision
* (different subdirectories under the same instance root), the same layout a
* real controller process has on disk.
*
* Does NOT write any config bytes itself -- callers that need real bytes on
* disk call conf_save_config() afterward (config_repl.h); a caller that
* wires myconf but never saves anything is deliberately exercising the
* "live cluster, no config pushed yet" -ENOENT path in
* read_active_config_bytes (persist.c). */
static void snapent_wire_myconf(snapent_fixture_t* f)
{
f->myconf.workdir = f->tmpdir;
f->myconf.name = (char*)"snapent";
f->myconf.conf_keep_n = 2;
f->c->myconf = &f->myconf;
}
static void snapent_teardown(snapent_fixture_t* f)
{
if (f == NULL)
@ -352,6 +383,11 @@ static void snapent_teardown(snapent_fixture_t* f)
if (f->c->workermgr) dereference(f->c->workermgr);
xce_hook_fini(&f->c->hooksys);
/* free(NULL) is safe -- conffile is only heap-allocated (via
* conf_save_config's asprintf) when a test actually wired+saved a real
* config; every other test leaves it at its calloc'd NULL. */
free(f->myconf.conffile);
free(f->c);
free(f);
}
@ -1217,6 +1253,106 @@ static int snapent_read_checkpoint_hosts(const char* path,
return n;
}
/* Reads the trailing config-bytes section a checkpoint file carries
* (config-upgrade-snapshot task 2.3, Req 2.1, 2.3, 2.4, 5.1): skips past the
* legacy header + format_version + job/queue/host sections exactly like
* snapent_read_checkpoint_hosts above, then reads the final
* [config_bytes_len][raw config bytes] blob -- the single flat section that
* comes after every other section, never a [count][frame...] shape.
*
* On success returns 0 with *out_bytes (caller must free(), NULL when
* out_len is 0) / *out_len set. Returns -1 on any parse/read failure
* (malformed header, truncated section, etc.) -- mirrors every other
* snapent_read_checkpoint_* reader's -1-on-failure convention in this file. */
static int snapent_read_checkpoint_config_bytes(const char* path,
uint8_t** out_bytes,
size_t* out_len)
{
int fd = open(path, O_RDONLY);
if (fd < 0)
return -1;
char header[SNAPENT_LEGACY_HEADER_BYTES];
if (read(fd, header, sizeof(header)) != (ssize_t)sizeof(header) ||
memcmp(header, "XCESNAP\0", 8) != 0) {
close(fd);
return -1;
}
uint32_t format_version = 0;
if (read(fd, &format_version, sizeof(format_version)) !=
(ssize_t)sizeof(format_version) ||
format_version != SNAPENT_TEST_FORMAT_VERSION) {
close(fd);
return -1;
}
/* Skip job, queue, and host sections -- all three share the same
* [count][payload_len][bytes]* shape by this point (task 2.2 migrated
* queue frames to capnp, matching job/host). */
for (int section = 0; section < 3; section++) {
uint32_t count = 0;
if (read(fd, &count, sizeof(count)) != (ssize_t)sizeof(count)) {
close(fd);
return -1;
}
for (uint32_t i = 0; i < count; i++) {
uint32_t payload_len = 0;
if (read(fd, &payload_len, sizeof(payload_len)) !=
(ssize_t)sizeof(payload_len)) {
close(fd);
return -1;
}
if (payload_len == 0 || payload_len > SNAPENT_MAX_DECODE_PAYLOAD) {
close(fd);
return -1;
}
uint8_t* discard = (uint8_t*)malloc(payload_len);
if (discard == NULL) {
close(fd);
return -1;
}
ssize_t got = read(fd, discard, payload_len);
free(discard);
if (got != (ssize_t)payload_len) {
close(fd);
return -1;
}
}
}
/* Config-bytes section: config_bytes_len + raw bytes (no per-item
* framing -- exactly one blob for the whole file). */
uint32_t cfg_len = 0;
if (read(fd, &cfg_len, sizeof(cfg_len)) != (ssize_t)sizeof(cfg_len)) {
close(fd);
return -1;
}
if (cfg_len > SNAPENT_MAX_DECODE_PAYLOAD) {
close(fd);
return -1;
}
uint8_t* buf = NULL;
if (cfg_len > 0) {
buf = (uint8_t*)malloc(cfg_len);
if (buf == NULL) {
close(fd);
return -1;
}
if (read(fd, buf, cfg_len) != (ssize_t)cfg_len) {
free(buf);
close(fd);
return -1;
}
}
close(fd);
*out_bytes = buf;
*out_len = (size_t)cfg_len;
return 0;
}
static snapent_decoded_host_t* snapent_find_decoded_host(
snapent_decoded_host_t* hosts, int count, const char* name)
{
@ -1360,6 +1496,171 @@ static void test_snapent_checkpoint_no_hosts_yields_empty_host_section(void)
printf("test_snapent_checkpoint_no_hosts_yields_empty_host_section: PASS\n");
}
/* ───────────────────────────────────────────────────────────────────────
* config-upgrade-snapshot Task 2.3 (Req 2.1, 2.3, 2.4, 5.1): config-bytes
* section content, absence, and determinism.
* ─────────────────────────────────────────────────────────────────────── */
/* A config file test payload -- content is irrelevant to write_snapshot
* (it reads raw bytes, never parses/interprets them), so any distinctive
* byte string proves the round-trip without needing a real ClusterConfig
* capnp blob. */
static const char SNAPENT_CONFIG_TEST_PAYLOAD[] =
"snapent-config-upgrade-snapshot-task-2.3-test-payload";
/* Saves SNAPENT_CONFIG_TEST_PAYLOAD as f's currently-active config via the
* real conf_save_config path (config_repl.c) -- the exact mechanism
* read_active_config_bytes (persist.c) resolves against (conf/current ->
* conf/cluster.bin.<hex>). Requires snapent_wire_myconf to have been called
* on f first. */
static void snapent_save_test_config(snapent_fixture_t* f)
{
uint8_t sha[32];
SHA256((const unsigned char*)SNAPENT_CONFIG_TEST_PAYLOAD,
sizeof(SNAPENT_CONFIG_TEST_PAYLOAD) - 1, sha);
TEST_ASSERT_EQUAL_INT(0,
conf_save_config(&f->myconf, (const uint8_t*)SNAPENT_CONFIG_TEST_PAYLOAD,
sizeof(SNAPENT_CONFIG_TEST_PAYLOAD) - 1, sha));
}
/* Task 2.3's own literal Observable: a checkpoint's config-bytes section
* content hash matches the currently active configuration -- checked here
* via an exact byte-for-byte memcmp against the source file's own bytes,
* which subsumes a hash match (identical bytes trivially hash identically,
* and is a strictly stronger, more direct assertion). */
static void test_snapent_checkpoint_config_bytes_section_matches_source_file(void)
{
snapent_fixture_t* f = snapent_make(0, 0);
TEST_ASSERT_NOT_NULL(f);
snapent_wire_myconf(f);
snapent_save_test_config(f);
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(&f->c->persist));
char path[4096];
snprintf(path, sizeof(path), "%s/snap-%016" PRIx64 ".capnp",
f->c->persist.snap_path, f->c->persist.checkpoint_lsn);
uint8_t* section_bytes = NULL;
size_t section_len = 0;
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
snapent_read_checkpoint_config_bytes(path, &section_bytes, &section_len),
"checkpoint file must parse as a well-formed entity-bearing payload "
"through the trailing config-bytes section");
TEST_ASSERT_EQUAL_size_t_MESSAGE(sizeof(SNAPENT_CONFIG_TEST_PAYLOAD) - 1,
section_len,
"config-bytes section length must equal the source config file's "
"exact byte length");
TEST_ASSERT_EQUAL_MEMORY_MESSAGE(SNAPENT_CONFIG_TEST_PAYLOAD, section_bytes,
section_len,
"config-bytes section content must be byte-for-byte identical to "
"the currently active configuration on disk (Requirement 2.1) -- "
"an exact match trivially implies the content hash also matches");
free(section_bytes);
snapent_teardown(f);
printf("test_snapent_checkpoint_config_bytes_section_matches_source_file: PASS\n");
}
/* Requirement 2.4's "no config yet" edge: a live cluster (real myconf wired,
* unlike every other test in this file which leaves f->c->myconf NULL) that
* has never received a config push must still produce a valid checkpoint --
* a zero-length config-bytes section, not a failed checkpoint. Mirrors
* test_snapent_checkpoint_no_hosts_yields_empty_host_section's "absence is
* not failure" shape, one level up (persist.c's read_active_config_bytes
* -ENOENT path). */
static void test_snapent_checkpoint_config_bytes_section_empty_when_no_local_config(void)
{
snapent_fixture_t* f = snapent_make(0, 0);
TEST_ASSERT_NOT_NULL(f);
snapent_wire_myconf(f); /* real myconf, but conf_save_config never called */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, persist_checkpoint(&f->c->persist),
"a checkpoint on a live cluster that has not yet received its first "
"config push must still succeed -- absence of local config bytes is "
"a benign early-boot state, never a checkpoint failure");
char path[4096];
snprintf(path, sizeof(path), "%s/snap-%016" PRIx64 ".capnp",
f->c->persist.snap_path, f->c->persist.checkpoint_lsn);
uint8_t* section_bytes = NULL;
size_t section_len = 0;
TEST_ASSERT_EQUAL_INT(0,
snapent_read_checkpoint_config_bytes(path, &section_bytes, &section_len));
TEST_ASSERT_EQUAL_size_t_MESSAGE(0, section_len,
"no local config bytes exist yet -- the section must be explicitly "
"zero-length, not omitted or a phantom non-empty blob");
TEST_ASSERT_NULL(section_bytes);
snapent_teardown(f);
printf("test_snapent_checkpoint_config_bytes_section_empty_when_no_local_config: PASS\n");
}
/* Requirement 5.1 determinism: two independent checkpoints taken with the
* SAME active configuration unchanged on disk must produce byte-identical
* config-bytes sections -- a pure function of current on-disk content, no
* timestamp/ordering dependence.
*
* NOTE on why this does NOT also assert the two checkpoints land at
* different LSNs/paths: this fixture is a bare persist_t with no live commit
* engine (snapent_make's own doc comment -- "Deliberately OMITTED ...
* commit engine"), so appending via persist_wal_async never advances
* committed_lsn, and checkpoint_lsn (persist_checkpoint's
* last_applied_lsn snapshot) never moves past its initial value here --
* both checkpoints legitimately land at the SAME path. That is fine for
* what THIS test checks: write_snapshot's config-bytes-writing logic still
* runs twice, as two genuinely separate persist_checkpoint calls (the
* second's open/write/rename fully redoes the first's, it does not skip or
* cache anything), which is exactly the "independent materialization"
* property Req 5.1 requires -- proving it does not depend on the path
* differing. */
static void test_snapent_checkpoint_config_bytes_section_deterministic_across_checkpoints(void)
{
snapent_fixture_t* f = snapent_make(0, 0);
TEST_ASSERT_NOT_NULL(f);
snapent_wire_myconf(f);
snapent_save_test_config(f);
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(&f->c->persist));
char path1[4096];
snprintf(path1, sizeof(path1), "%s/snap-%016" PRIx64 ".capnp",
f->c->persist.snap_path, f->c->persist.checkpoint_lsn);
uint8_t* bytes1 = NULL; size_t len1 = 0;
TEST_ASSERT_EQUAL_INT(0,
snapent_read_checkpoint_config_bytes(path1, &bytes1, &len1));
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(&f->c->persist));
char path2[4096];
snprintf(path2, sizeof(path2), "%s/snap-%016" PRIx64 ".capnp",
f->c->persist.snap_path, f->c->persist.checkpoint_lsn);
uint8_t* bytes2 = NULL; size_t len2 = 0;
TEST_ASSERT_EQUAL_INT(0,
snapent_read_checkpoint_config_bytes(path2, &bytes2, &len2));
TEST_ASSERT_EQUAL_size_t_MESSAGE(len1, len2,
"independent materializations at the same unchanged configuration "
"must yield equal-length config-bytes sections");
TEST_ASSERT_EQUAL_MEMORY_MESSAGE(bytes1, bytes2, len1,
"independent materializations at the same unchanged configuration "
"must yield byte-identical config-bytes sections (Requirement 5.1)");
TEST_ASSERT_EQUAL_size_t_MESSAGE(sizeof(SNAPENT_CONFIG_TEST_PAYLOAD) - 1,
len1,
"sanity: both materializations must actually carry the real config "
"content, not two empty sections trivially matching each other");
free(bytes1);
free(bytes2);
snapent_teardown(f);
printf("test_snapent_checkpoint_config_bytes_section_deterministic_across_checkpoints: PASS\n");
}
/* ───────────────────────────────────────────────────────────────────────
* ha-entity-snapshots Task 4.2 (Req 3.1-3.3): rotation-guard RED test.
*
@ -1744,6 +2045,9 @@ void test_persist_snapshot_entities(void)
RUN_TEST(test_snapent_checkpoint_host_admin_status_roundtrip);
RUN_TEST(test_snapent_checkpoint_closed_host_yields_exactly_one_admin_only_frame);
RUN_TEST(test_snapent_checkpoint_no_hosts_yields_empty_host_section);
RUN_TEST(test_snapent_checkpoint_config_bytes_section_matches_source_file);
RUN_TEST(test_snapent_checkpoint_config_bytes_section_empty_when_no_local_config);
RUN_TEST(test_snapent_checkpoint_config_bytes_section_deterministic_across_checkpoints);
RUN_TEST(test_snapent_rotation_preserves_in_flight_checkpoint);
RUN_TEST(test_snapent_load_snapshot_restores_entities_before_advancing_position);
RUN_TEST(test_snapent_load_snapshot_corrupted_entity_leaves_position_unchanged);