lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-term-safety): append config markers via unified log + derive committed config (task 4.2)

Browse source 
Wires do_upgrade_config_with_bytes to append WAL_CONFIG_MARKER through
raft_log_append (live term-stamped consensus record), demotes committed_config_tag
to a non-authoritative cache, and derives the committed config from the latest
committed marker. Preserves the derive across checkpoint compaction via an optional,
backward-compatible config-ref tail in the snapshot sentinel.
This commit is contained in:
Rongsong Shen 2026-06-22 10:34:10 +00:00
parent f447ad4183
commit c3201a2b1e
7 changed files with 883 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

34
src/controller/testsuite/test_commit_barrier_rollout.c
View file

@ -277,6 +277,7 @@ static void test_classify_is_total_over_wal_entry_type(void)
{ WAL_JOB_PREASON, HOOK_DETAIL_CLRPREASON, "WAL_JOB_PREASON" },
{ WAL_QUEUE_CTRL, HOOK_DETAIL_QUEUE_UPDATE, "WAL_QUEUE_CTRL" },
{ WAL_WORKER_CTRL, HOOK_DETAIL_WORKER_DEAD, "WAL_WORKER_CTRL" },
{ WAL_CONFIG_MARKER, 0, "WAL_CONFIG_MARKER" },
{ WAL_CHECKPOINT, 0, "WAL_CHECKPOINT" },
};
size_t i;
@ -1167,6 +1168,8 @@ static zarray_t g_cb5_standbys;
static myconf_t g_cb5_myconf;
static char g_cb5_tmpdir[256];
static int g_cb5_have_tmpdir;
static char g_cb5_waldir[256]; /* started-persist WAL dir (task 4.2) */
static int g_cb5_have_waldir;
static void cb5_setup(void)
{
@ -1179,6 +1182,7 @@ static void cb5_setup(void)
memset(&g_cb5_req_priv, 0, sizeof(g_cb5_req_priv));
memset(&g_cb5_myconf, 0, sizeof(g_cb5_myconf));
g_cb5_have_tmpdir = 0;
g_cb5_have_waldir = 0;
g_cb5_c.myhost = &g_cb5_c.myhost_info;
g_cb5_c.myhost_info.name = (char *)"ctrl-cb5";
@ -1197,9 +1201,22 @@ static void cb5_setup(void)
}
g_cb5_c.ha.standby_endpoints = &g_cb5_standbys;
/* Replicator registry mirrors the push endpoints; wired engine carries
* n_peers=3 for the configAck majority denominator. */
TEST_ASSERT_EQUAL_INT(0, repl_init(&g_cb5_c.persist.repl));
/* Started persist so the live config-change path (config_commit_change →
* config_append_marker → ha_primary_log_append → persist_wal_async) can
* append the WAL_CONFIG_MARKER consensus record (ha-term-safety task 4.2).
* persist_init creates the replicator registry; we then mirror the push
* endpoints and wire the n_peers=3 engine (configAck majority denominator).
* A dedicated tmpdir keeps the WAL isolated from the myconf workdir. */
{
persist_config_t pcfg = persist_config_defaults();
pcfg.wal_sync_ms = 5000;
snprintf(g_cb5_waldir, sizeof(g_cb5_waldir), "/tmp/xce_cb5wal_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(g_cb5_waldir));
g_cb5_have_waldir = 1;
persist_set_workdir(&g_cb5_c.persist, g_cb5_waldir, "ctrl-cb5");
TEST_ASSERT_EQUAL_INT(0, persist_init(&g_cb5_c.persist, &pcfg, &g_cb5_c));
TEST_ASSERT_EQUAL_INT(0, persist_start(&g_cb5_c.persist));
}
TEST_ASSERT_TRUE(repl_add_standby(&g_cb5_c.persist.repl,
&g_cb5_sb_ep[0], "cb5_a", 7500) >= 0);
TEST_ASSERT_TRUE(repl_add_standby(&g_cb5_c.persist.repl,
@ -1251,7 +1268,9 @@ static void cb5_teardown(void)
cb5_drain_ep(&g_cb5_sb_ep[1]);
cb5_drain_ep(&g_cb5_req_ep);
commit_engine_fini(&g_cb5_eng);
repl_fini(&g_cb5_c.persist.repl);
/* persist_fini calls repl_fini internally (started-persist fixture). */
persist_stop(&g_cb5_c.persist);
persist_fini(&g_cb5_c.persist);
zarray_fini(&g_cb5_standbys);
cbuf_fini(&g_cb5_c.wbuf);
free(g_cb5_myconf.conffile);
@ -1263,6 +1282,13 @@ static void cb5_teardown(void)
system(cmd);
g_cb5_have_tmpdir = 0;
}
if (g_cb5_have_waldir) {
char cmd[300];
snprintf(cmd, sizeof(cmd), "rm -rf %s", g_cb5_waldir);
system(cmd);
g_cb5_have_waldir = 0;
}
}
/* ── Reply observation (raw capnp parse of ep->outq packets) ────────────── */

370
src/controller/testsuite/test_config_repl.c
View file

@ -21,11 +21,13 @@
#include <capnp_c.h>
#include "commit.h"
#include "config_repl.h"
#include "controller.capnp.h"
#include "ha.h"
#include "myconf.h"
#include "persist.h"
#include "repl.h"
#include "request.h"
#include "typemap.h"
#include "wal_types.h"
@ -590,6 +592,366 @@ static void test_config_marker_encode_short_buffer_fails(void)
printf("test_config_marker_encode_short_buffer_fails: PASS\n");
}
/* ── Config marker append + committed-config derive (task 4.2) ────────
*
* These bring a node up as the raft LEADER so config_append_marker routes the
* WAL_CONFIG_MARKER through ha_primary_log_append → raft_log_append (the same
* unified-log path job state uses), and assert the committed config is DERIVED
* from the latest committed marker within the committed prefix (R7.1/7.3/7.6/
* 7.7/7.11). Mirrors make_cluster/make_leader from test_term_safety_adapter.c. */
static cluster_t *t42_make_cluster(const char *tmpdir)
{
cluster_t *c = calloc(1, sizeof(cluster_t));
if (!c) return NULL;
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
cfg.snap_keep_n = 2;
persist_set_workdir(&c->persist, tmpdir, "ctrl-test");
if (persist_init(&c->persist, &cfg, c) != 0) { free(c); return NULL; }
if (persist_start(&c->persist) != 0) {
persist_fini(&c->persist);
free(c);
return NULL;
}
c->ha.raft_c = c;
return c;
}
static void t42_teardown(cluster_t *c)
{
if (!c) return;
persist_stop(&c->persist);
persist_fini(&c->persist);
free(c);
}
/* Bring the node up as the raft LEADER at `term` (so raft_log_append is enabled
* — it returns -EPERM off the leader; see make_leader in the adapter suite). */
static void t42_make_leader(cluster_t *c, uint64_t term, int n_peers)
{
atomic_store(&c->ha.role, CTRL_PRIMARY);
atomic_store(&c->ha.term, term);
ha_init_raft(&c->ha, c, n_peers);
c->ha.raft.term = term;
c->ha.raft.role = RAFT_LEADER;
}
/* Fill an sha256 with a deterministic pattern; tag = be64 of first 8 bytes,
* matching the config_repl version_tag convention (config_version_tag). */
static uint64_t t42_make_ref(uint8_t sha256[32], uint8_t seed)
{
for (int i = 0; i < 32; i++)
sha256[i] = (uint8_t)(seed + i);
uint64_t tag;
memcpy(&tag, sha256, 8);
return be64toh(tag);
}
/* (a) Appending a config change writes a WAL_CONFIG_MARKER through the unified
* log: the entry lands at an LSN with the live election term stamped, and its
* payload decodes to {version_tag, sha256} (R7.1/7.3). */
static void test_config_marker_append_via_unified_log(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_cr42_append_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
const uint64_t T = 7;
cluster_t *c = t42_make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
t42_make_leader(c, T, /*n_peers*/1);
uint8_t sha[32];
uint64_t tag = t42_make_ref(sha, 0x10);
uint64_t lsn = 0;
TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &lsn));
/* Library assigned LSN 1 and stamped the live term for that entry. */
TEST_ASSERT_EQUAL_UINT64(1, lsn);
TEST_ASSERT_EQUAL_UINT64(1, c->ha.raft.log_len);
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1));
t42_teardown(c);
printf("test_config_marker_append_via_unified_log: PASS\n");
}
/* (b) After the marker COMMITS (committed_lsn covers its LSN), the derived
* committed-config reference == that marker's {version_tag, sha256} (R7.7). */
static void test_committed_marker_derives_after_commit(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_cr42_derive_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t *c = t42_make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
t42_make_leader(c, /*term*/4, /*n_peers*/1);
/* A job entry then a config marker, so the marker is NOT at LSN 1. */
wal_jobstate_t js = { .jobid = 1, .state = 1 };
uint64_t jlsn = 0;
TEST_ASSERT_EQUAL_INT(0,
ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &jlsn));
uint8_t sha[32];
uint64_t tag = t42_make_ref(sha, 0x40);
uint64_t mlsn = 0;
TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &mlsn));
TEST_ASSERT_EQUAL_UINT64(2, mlsn);
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
/* Drive committed_lsn to the marker's LSN (the commit advance source). */
atomic_store_explicit(&c->persist.committed_lsn, mlsn, memory_order_release);
uint64_t got_tag = 0;
uint8_t got_sha[32];
memset(got_sha, 0, sizeof(got_sha));
TEST_ASSERT_EQUAL_INT(0, config_committed_marker(c, &got_tag, got_sha));
TEST_ASSERT_EQUAL_HEX64(tag, got_tag);
TEST_ASSERT_EQUAL_HEX8_ARRAY(sha, got_sha, 32);
t42_teardown(c);
printf("test_committed_marker_derives_after_commit: PASS\n");
}
/* (c) An UNCOMMITTED marker (LSN > committed_lsn) does NOT change the derived
* committed config: with committed_lsn at the first marker, a later second
* marker is invisible until committed (R7.7/7.11). */
static void test_uncommitted_marker_not_yet_committed_config(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_cr42_uncommit_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t *c = t42_make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
t42_make_leader(c, /*term*/5, /*n_peers*/1);
uint8_t sha1[32], sha2[32];
uint64_t tag1 = t42_make_ref(sha1, 0x10);
uint64_t tag2 = t42_make_ref(sha2, 0x80);
uint64_t l1 = 0, l2 = 0;
TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag1, sha1, &l1));
TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag2, sha2, &l2));
TEST_ASSERT_EQUAL_UINT64(1, l1);
TEST_ASSERT_EQUAL_UINT64(2, l2);
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
/* Only the FIRST marker is committed. The second is uncommitted. */
atomic_store_explicit(&c->persist.committed_lsn, l1, memory_order_release);
uint64_t got_tag = 0;
uint8_t got_sha[32];
TEST_ASSERT_EQUAL_INT(0, config_committed_marker(c, &got_tag, got_sha));
/* Derived config is the FIRST (committed) marker, NOT the uncommitted one. */
TEST_ASSERT_EQUAL_HEX64(tag1, got_tag);
TEST_ASSERT_EQUAL_HEX8_ARRAY(sha1, got_sha, 32);
/* With NOTHING committed, there is no committed config marker at all. */
atomic_store_explicit(&c->persist.committed_lsn, 0ULL, memory_order_release);
TEST_ASSERT_EQUAL_INT(-ENOENT, config_committed_marker(c, &got_tag, got_sha));
t42_teardown(c);
printf("test_uncommitted_marker_not_yet_committed_config: PASS\n");
}
/* (d) A divergent config marker at a conflicting (term,lsn) is truncated by the
* reconcile path exactly like a job entry: on_truncate_from(index) removes the
* marker (and everything above), leaving the committed prefix intact. This
* proves the marker is an ORDINARY log entry under (term,lsn) matching (R7.6).
*
* We append a committed job at LSN 1, then a divergent marker at LSN 2, drive
* committed_lsn to 1, then truncate from LSN 2 (the conflict point) and confirm
* the marker is gone while the committed job prefix survives. */
static void test_divergent_marker_truncated_like_job_entry(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_cr42_trunc_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t *c = t42_make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
t42_make_leader(c, /*term*/6, /*n_peers*/1);
/* LSN 1: committed job entry. LSN 2: a (divergent) config marker. */
wal_jobstate_t js = { .jobid = 9, .state = 2 };
uint64_t jlsn = 0;
TEST_ASSERT_EQUAL_INT(0,
ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &jlsn));
TEST_ASSERT_EQUAL_UINT64(1, jlsn);
uint8_t sha[32];
uint64_t tag = t42_make_ref(sha, 0x55);
uint64_t mlsn = 0;
TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &mlsn));
TEST_ASSERT_EQUAL_UINT64(2, mlsn);
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
/* The committed prefix is LSN 1 (the job). The marker at LSN 2 is the
* divergent suffix. The derived committed config is therefore NONE yet. */
atomic_store_explicit(&c->persist.committed_lsn, 1ULL, memory_order_release);
uint64_t got_tag = 0;
uint8_t got_sha[32];
TEST_ASSERT_EQUAL_INT(-ENOENT, config_committed_marker(c, &got_tag, got_sha));
/* Reconcile truncates from the conflict point (LSN 2) exactly as it would a
* job entry — the SAME on_truncate_from seam used by (term,lsn) matching. */
TEST_ASSERT_EQUAL_INT(0,
ha_cb_truncate_from_for_test(&c->ha, /*index*/2));
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
/* The marker is gone (term index shrank); the committed job prefix is intact. */
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(&c->persist, 2));
TEST_ASSERT_EQUAL_UINT64(6, persist_entry_term(&c->persist, 1));
/* Still no committed config marker — the divergent one never committed. */
TEST_ASSERT_EQUAL_INT(-ENOENT, config_committed_marker(c, &got_tag, got_sha));
t42_teardown(c);
printf("test_divergent_marker_truncated_like_job_entry: PASS\n");
}
/* (e) INTEGRATION (Finding 1 remediation): drive a config change through the
* LIVE commit path (config_commit_change — the helper do_upgrade_config_with_bytes
* now calls) and assert (i) a WAL_CONFIG_MARKER lands at the LIVE term, and
* (ii) once the marker COMMITS through the real WAL commit barrier, the DERIVED
* committed config (config_committed_marker) equals it. This proves the live
* config-change path appends + commits + derives the marker — not the retired
* term-blind committed_config_tag push/ack/advance.
*
* A single-member commit engine is wired so config_commit_change's WAL barrier
* (repl_wait_majority_ack/OPCode_walAck → commit_submit) self-quorums and
* advances committed_lsn for real, exactly as the production single-node path. */
static void test_config_commit_change_live_path_commits_and_derives(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_cr42_live_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
const uint64_t T = 9;
cluster_t *c = t42_make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
t42_make_leader(c, T, /*n_peers*/1);
/* Single-member commit engine: the marker's WAL LSN self-quorums, so
* driving commit_submit on it advances committed_lsn through the REAL commit
* machinery — the same path that commits the marker in production once a
* majority acks the WAL stream (no manual committed_lsn poke). */
TEST_ASSERT_EQUAL_INT(0,
commit_engine_init(&c->commit, &c->persist, /*n_peers*/1, /*n_vcpus*/8));
repl_set_commit_engine(&c->persist.repl, &c->commit);
uint8_t sha[32];
uint64_t tag = t42_make_ref(sha, 0x30);
/* The LIVE entry point do_upgrade_config_with_bytes now calls: append the
* WAL_CONFIG_MARKER consensus record + out-of-band byte push + the existing
* majority barrier. With no standbys (single member) the byte barrier
* self-quorums and the call returns 0. */
int acked = -1;
TEST_ASSERT_EQUAL_INT(0,
config_commit_change(c, (const uint8_t *)"cfg-bytes", 9, sha, tag, &acked));
/* (i) The marker landed at LSN 1 with the LIVE election term stamped. */
TEST_ASSERT_EQUAL_UINT64(1, c->ha.raft.log_len);
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1));
/* No standbys registered → zero out-of-band pushes (barrier owns the verdict). */
TEST_ASSERT_EQUAL_INT(0, acked);
/* Make the marker durable (advances durable_lsn) then commit its WAL LSN
* through the real commit engine (self-quorum), exactly as a majority WAL
* ack does in production — this advances committed_lsn so the derive sees
* the marker in the committed prefix. */
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
TEST_ASSERT_EQUAL_INT(0,
commit_submit(&c->commit, /*marker LSN*/1, COMMIT_MODE_WAIT, 1000));
TEST_ASSERT_TRUE(
atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire) >= 1);
/* (ii) The DERIVED committed config equals the change just committed. */
uint64_t got_tag = 0;
uint8_t got_sha[32];
TEST_ASSERT_EQUAL_INT(0, config_committed_marker(c, &got_tag, got_sha));
TEST_ASSERT_EQUAL_HEX64(tag, got_tag);
TEST_ASSERT_EQUAL_HEX8_ARRAY(sha, got_sha, 32);
/* committed_config_tag is now a CACHE mirroring the derived marker. */
TEST_ASSERT_EQUAL_HEX64(tag,
atomic_load_explicit(&c->ha.committed_config_tag, memory_order_acquire));
commit_engine_fini(&c->commit);
t42_teardown(c);
printf("test_config_commit_change_live_path_commits_and_derives: PASS\n");
}
/* (f) COMPACTION SAFETY (Finding 2 remediation): a checkpoint that TRIMS the WAL
* segment holding a committed marker must NOT lose the derived committed config.
* Without the sentinel-carried reference + fallback, the live-WAL scan regresses
* to -ENOENT after the trim. We commit a marker, force a NEW segment + checkpoint
* (which trims the old segment containing the marker), and assert the derive
* STILL returns the committed reference (R7.7/R7.10). */
static void test_committed_marker_survives_checkpoint_trim(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_cr42_trim_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
const uint64_t T = 11;
cluster_t *c = t42_make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
t42_make_leader(c, T, /*n_peers*/1);
TEST_ASSERT_EQUAL_INT(0,
commit_engine_init(&c->commit, &c->persist, /*n_peers*/1, /*n_vcpus*/8));
repl_set_commit_engine(&c->persist.repl, &c->commit);
uint8_t sha[32];
uint64_t tag = t42_make_ref(sha, 0x70);
/* Append the marker (segment 0, the lowest-LSN segment) and commit its WAL
* LSN through the real engine (self-quorum) so it enters the committed
* prefix — the production marker-commit path. */
int acked = -1;
TEST_ASSERT_EQUAL_INT(0,
config_commit_change(c, (const uint8_t *)"cfg", 3, sha, tag, &acked));
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
TEST_ASSERT_EQUAL_INT(0,
commit_submit(&c->commit, /*marker LSN*/1, COMMIT_MODE_WAIT, 1000));
uint64_t committed =
atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire);
TEST_ASSERT_TRUE(committed >= 1);
/* Sanity: before any trim, the live-WAL scan finds the marker. */
uint64_t got_tag = 0;
uint8_t got_sha[32];
TEST_ASSERT_EQUAL_INT(0, config_committed_marker(c, &got_tag, got_sha));
TEST_ASSERT_EQUAL_HEX64(tag, got_tag);
/* Roll to a NEW segment so the checkpoint's trim has an older segment (the
* one holding the marker) to unlink — the compaction hazard. */
TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(&c->persist));
/* Checkpoint: stamps the committed-marker reference into the sentinel, then
* trims the older segment (which physically held the marker). */
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(&c->persist));
/* The marker's segment is gone, so the live-WAL scan alone would regress to
* -ENOENT; the derive MUST fall back to the durable sentinel reference. */
got_tag = 0;
memset(got_sha, 0, sizeof(got_sha));
TEST_ASSERT_EQUAL_INT(0, config_committed_marker(c, &got_tag, got_sha));
TEST_ASSERT_EQUAL_HEX64(tag, got_tag);
TEST_ASSERT_EQUAL_HEX8_ARRAY(sha, got_sha, 32);
commit_engine_fini(&c->commit);
t42_teardown(c);
printf("test_committed_marker_survives_checkpoint_trim: PASS\n");
}
/* ── wait_all_with_timeout (XCE_TESTING shim) ───── */
#ifdef XCE_TESTING
@ -2351,6 +2713,14 @@ void test_config_repl(void)
RUN_TEST(test_config_marker_encode_decode_roundtrip);
RUN_TEST(test_config_marker_decode_truncated_fails);
RUN_TEST(test_config_marker_encode_short_buffer_fails);
/* Task 4.2: append marker via unified log + derive committed config */
RUN_TEST(test_config_marker_append_via_unified_log);
RUN_TEST(test_committed_marker_derives_after_commit);
RUN_TEST(test_uncommitted_marker_not_yet_committed_config);
RUN_TEST(test_divergent_marker_truncated_like_job_entry);
/* Task 4.2 r1: live wiring + compaction-safe derive */
RUN_TEST(test_config_commit_change_live_path_commits_and_derives);
RUN_TEST(test_committed_marker_survives_checkpoint_trim);
#ifdef XCE_TESTING
RUN_TEST(test_load_primary_config_missing_returns_error);
RUN_TEST(test_wait_all_with_timeout_slow_threads);