feat(ha-term-safety): append config markers via unified log + derive committed config (task 4.2)
Wires do_upgrade_config_with_bytes to append WAL_CONFIG_MARKER through raft_log_append (live term-stamped consensus record), demotes committed_config_tag to a non-authoritative cache, and derives the committed config from the latest committed marker. Preserves the derive across checkpoint compaction via an optional, backward-compatible config-ref tail in the snapshot sentinel.
This commit is contained in:
parent
f447ad4183
commit
c3201a2b1e
7 changed files with 883 additions and 21 deletions
|
|
@ -21,11 +21,13 @@
|
|||
|
||||
#include <capnp_c.h>
|
||||
|
||||
#include "commit.h"
|
||||
#include "config_repl.h"
|
||||
#include "controller.capnp.h"
|
||||
#include "ha.h"
|
||||
#include "myconf.h"
|
||||
#include "persist.h"
|
||||
#include "repl.h"
|
||||
#include "request.h"
|
||||
#include "typemap.h"
|
||||
#include "wal_types.h"
|
||||
|
|
@ -590,6 +592,366 @@ static void test_config_marker_encode_short_buffer_fails(void)
|
|||
printf("test_config_marker_encode_short_buffer_fails: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Config marker append + committed-config derive (task 4.2) ────────
|
||||
*
|
||||
* These bring a node up as the raft LEADER so config_append_marker routes the
|
||||
* WAL_CONFIG_MARKER through ha_primary_log_append → raft_log_append (the same
|
||||
* unified-log path job state uses), and assert the committed config is DERIVED
|
||||
* from the latest committed marker within the committed prefix (R7.1/7.3/7.6/
|
||||
* 7.7/7.11). Mirrors make_cluster/make_leader from test_term_safety_adapter.c. */
|
||||
|
||||
static cluster_t *t42_make_cluster(const char *tmpdir)
|
||||
{
|
||||
cluster_t *c = calloc(1, sizeof(cluster_t));
|
||||
if (!c) return NULL;
|
||||
|
||||
persist_config_t cfg = persist_config_defaults();
|
||||
cfg.wal_sync_ms = 5000;
|
||||
cfg.snap_keep_n = 2;
|
||||
|
||||
persist_set_workdir(&c->persist, tmpdir, "ctrl-test");
|
||||
if (persist_init(&c->persist, &cfg, c) != 0) { free(c); return NULL; }
|
||||
if (persist_start(&c->persist) != 0) {
|
||||
persist_fini(&c->persist);
|
||||
free(c);
|
||||
return NULL;
|
||||
}
|
||||
c->ha.raft_c = c;
|
||||
return c;
|
||||
}
|
||||
|
||||
static void t42_teardown(cluster_t *c)
|
||||
{
|
||||
if (!c) return;
|
||||
persist_stop(&c->persist);
|
||||
persist_fini(&c->persist);
|
||||
free(c);
|
||||
}
|
||||
|
||||
/* Bring the node up as the raft LEADER at `term` (so raft_log_append is enabled
|
||||
* — it returns -EPERM off the leader; see make_leader in the adapter suite). */
|
||||
static void t42_make_leader(cluster_t *c, uint64_t term, int n_peers)
|
||||
{
|
||||
atomic_store(&c->ha.role, CTRL_PRIMARY);
|
||||
atomic_store(&c->ha.term, term);
|
||||
ha_init_raft(&c->ha, c, n_peers);
|
||||
c->ha.raft.term = term;
|
||||
c->ha.raft.role = RAFT_LEADER;
|
||||
}
|
||||
|
||||
/* Fill an sha256 with a deterministic pattern; tag = be64 of first 8 bytes,
|
||||
* matching the config_repl version_tag convention (config_version_tag). */
|
||||
static uint64_t t42_make_ref(uint8_t sha256[32], uint8_t seed)
|
||||
{
|
||||
for (int i = 0; i < 32; i++)
|
||||
sha256[i] = (uint8_t)(seed + i);
|
||||
uint64_t tag;
|
||||
memcpy(&tag, sha256, 8);
|
||||
return be64toh(tag);
|
||||
}
|
||||
|
||||
/* (a) Appending a config change writes a WAL_CONFIG_MARKER through the unified
|
||||
* log: the entry lands at an LSN with the live election term stamped, and its
|
||||
* payload decodes to {version_tag, sha256} (R7.1/7.3). */
|
||||
static void test_config_marker_append_via_unified_log(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr42_append_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
const uint64_t T = 7;
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, T, /*n_peers*/1);
|
||||
|
||||
uint8_t sha[32];
|
||||
uint64_t tag = t42_make_ref(sha, 0x10);
|
||||
|
||||
uint64_t lsn = 0;
|
||||
TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &lsn));
|
||||
|
||||
/* Library assigned LSN 1 and stamped the live term for that entry. */
|
||||
TEST_ASSERT_EQUAL_UINT64(1, lsn);
|
||||
TEST_ASSERT_EQUAL_UINT64(1, c->ha.raft.log_len);
|
||||
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1));
|
||||
|
||||
t42_teardown(c);
|
||||
printf("test_config_marker_append_via_unified_log: PASS\n");
|
||||
}
|
||||
|
||||
/* (b) After the marker COMMITS (committed_lsn covers its LSN), the derived
|
||||
* committed-config reference == that marker's {version_tag, sha256} (R7.7). */
|
||||
static void test_committed_marker_derives_after_commit(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr42_derive_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, /*term*/4, /*n_peers*/1);
|
||||
|
||||
/* A job entry then a config marker, so the marker is NOT at LSN 1. */
|
||||
wal_jobstate_t js = { .jobid = 1, .state = 1 };
|
||||
uint64_t jlsn = 0;
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &jlsn));
|
||||
|
||||
uint8_t sha[32];
|
||||
uint64_t tag = t42_make_ref(sha, 0x40);
|
||||
uint64_t mlsn = 0;
|
||||
TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &mlsn));
|
||||
TEST_ASSERT_EQUAL_UINT64(2, mlsn);
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
|
||||
|
||||
/* Drive committed_lsn to the marker's LSN (the commit advance source). */
|
||||
atomic_store_explicit(&c->persist.committed_lsn, mlsn, memory_order_release);
|
||||
|
||||
uint64_t got_tag = 0;
|
||||
uint8_t got_sha[32];
|
||||
memset(got_sha, 0, sizeof(got_sha));
|
||||
TEST_ASSERT_EQUAL_INT(0, config_committed_marker(c, &got_tag, got_sha));
|
||||
TEST_ASSERT_EQUAL_HEX64(tag, got_tag);
|
||||
TEST_ASSERT_EQUAL_HEX8_ARRAY(sha, got_sha, 32);
|
||||
|
||||
t42_teardown(c);
|
||||
printf("test_committed_marker_derives_after_commit: PASS\n");
|
||||
}
|
||||
|
||||
/* (c) An UNCOMMITTED marker (LSN > committed_lsn) does NOT change the derived
|
||||
* committed config: with committed_lsn at the first marker, a later second
|
||||
* marker is invisible until committed (R7.7/7.11). */
|
||||
static void test_uncommitted_marker_not_yet_committed_config(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr42_uncommit_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, /*term*/5, /*n_peers*/1);
|
||||
|
||||
uint8_t sha1[32], sha2[32];
|
||||
uint64_t tag1 = t42_make_ref(sha1, 0x10);
|
||||
uint64_t tag2 = t42_make_ref(sha2, 0x80);
|
||||
|
||||
uint64_t l1 = 0, l2 = 0;
|
||||
TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag1, sha1, &l1));
|
||||
TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag2, sha2, &l2));
|
||||
TEST_ASSERT_EQUAL_UINT64(1, l1);
|
||||
TEST_ASSERT_EQUAL_UINT64(2, l2);
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
|
||||
|
||||
/* Only the FIRST marker is committed. The second is uncommitted. */
|
||||
atomic_store_explicit(&c->persist.committed_lsn, l1, memory_order_release);
|
||||
|
||||
uint64_t got_tag = 0;
|
||||
uint8_t got_sha[32];
|
||||
TEST_ASSERT_EQUAL_INT(0, config_committed_marker(c, &got_tag, got_sha));
|
||||
/* Derived config is the FIRST (committed) marker, NOT the uncommitted one. */
|
||||
TEST_ASSERT_EQUAL_HEX64(tag1, got_tag);
|
||||
TEST_ASSERT_EQUAL_HEX8_ARRAY(sha1, got_sha, 32);
|
||||
|
||||
/* With NOTHING committed, there is no committed config marker at all. */
|
||||
atomic_store_explicit(&c->persist.committed_lsn, 0ULL, memory_order_release);
|
||||
TEST_ASSERT_EQUAL_INT(-ENOENT, config_committed_marker(c, &got_tag, got_sha));
|
||||
|
||||
t42_teardown(c);
|
||||
printf("test_uncommitted_marker_not_yet_committed_config: PASS\n");
|
||||
}
|
||||
|
||||
/* (d) A divergent config marker at a conflicting (term,lsn) is truncated by the
|
||||
* reconcile path exactly like a job entry: on_truncate_from(index) removes the
|
||||
* marker (and everything above), leaving the committed prefix intact. This
|
||||
* proves the marker is an ORDINARY log entry under (term,lsn) matching (R7.6).
|
||||
*
|
||||
* We append a committed job at LSN 1, then a divergent marker at LSN 2, drive
|
||||
* committed_lsn to 1, then truncate from LSN 2 (the conflict point) and confirm
|
||||
* the marker is gone while the committed job prefix survives. */
|
||||
static void test_divergent_marker_truncated_like_job_entry(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr42_trunc_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, /*term*/6, /*n_peers*/1);
|
||||
|
||||
/* LSN 1: committed job entry. LSN 2: a (divergent) config marker. */
|
||||
wal_jobstate_t js = { .jobid = 9, .state = 2 };
|
||||
uint64_t jlsn = 0;
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &jlsn));
|
||||
TEST_ASSERT_EQUAL_UINT64(1, jlsn);
|
||||
|
||||
uint8_t sha[32];
|
||||
uint64_t tag = t42_make_ref(sha, 0x55);
|
||||
uint64_t mlsn = 0;
|
||||
TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &mlsn));
|
||||
TEST_ASSERT_EQUAL_UINT64(2, mlsn);
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
|
||||
|
||||
/* The committed prefix is LSN 1 (the job). The marker at LSN 2 is the
|
||||
* divergent suffix. The derived committed config is therefore NONE yet. */
|
||||
atomic_store_explicit(&c->persist.committed_lsn, 1ULL, memory_order_release);
|
||||
uint64_t got_tag = 0;
|
||||
uint8_t got_sha[32];
|
||||
TEST_ASSERT_EQUAL_INT(-ENOENT, config_committed_marker(c, &got_tag, got_sha));
|
||||
|
||||
/* Reconcile truncates from the conflict point (LSN 2) exactly as it would a
|
||||
* job entry — the SAME on_truncate_from seam used by (term,lsn) matching. */
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
ha_cb_truncate_from_for_test(&c->ha, /*index*/2));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
|
||||
|
||||
/* The marker is gone (term index shrank); the committed job prefix is intact. */
|
||||
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(&c->persist, 2));
|
||||
TEST_ASSERT_EQUAL_UINT64(6, persist_entry_term(&c->persist, 1));
|
||||
/* Still no committed config marker — the divergent one never committed. */
|
||||
TEST_ASSERT_EQUAL_INT(-ENOENT, config_committed_marker(c, &got_tag, got_sha));
|
||||
|
||||
t42_teardown(c);
|
||||
printf("test_divergent_marker_truncated_like_job_entry: PASS\n");
|
||||
}
|
||||
|
||||
/* (e) INTEGRATION (Finding 1 remediation): drive a config change through the
|
||||
* LIVE commit path (config_commit_change — the helper do_upgrade_config_with_bytes
|
||||
* now calls) and assert (i) a WAL_CONFIG_MARKER lands at the LIVE term, and
|
||||
* (ii) once the marker COMMITS through the real WAL commit barrier, the DERIVED
|
||||
* committed config (config_committed_marker) equals it. This proves the live
|
||||
* config-change path appends + commits + derives the marker — not the retired
|
||||
* term-blind committed_config_tag push/ack/advance.
|
||||
*
|
||||
* A single-member commit engine is wired so config_commit_change's WAL barrier
|
||||
* (repl_wait_majority_ack/OPCode_walAck → commit_submit) self-quorums and
|
||||
* advances committed_lsn for real, exactly as the production single-node path. */
|
||||
static void test_config_commit_change_live_path_commits_and_derives(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr42_live_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
const uint64_t T = 9;
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, T, /*n_peers*/1);
|
||||
|
||||
/* Single-member commit engine: the marker's WAL LSN self-quorums, so
|
||||
* driving commit_submit on it advances committed_lsn through the REAL commit
|
||||
* machinery — the same path that commits the marker in production once a
|
||||
* majority acks the WAL stream (no manual committed_lsn poke). */
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_engine_init(&c->commit, &c->persist, /*n_peers*/1, /*n_vcpus*/8));
|
||||
repl_set_commit_engine(&c->persist.repl, &c->commit);
|
||||
|
||||
uint8_t sha[32];
|
||||
uint64_t tag = t42_make_ref(sha, 0x30);
|
||||
|
||||
/* The LIVE entry point do_upgrade_config_with_bytes now calls: append the
|
||||
* WAL_CONFIG_MARKER consensus record + out-of-band byte push + the existing
|
||||
* majority barrier. With no standbys (single member) the byte barrier
|
||||
* self-quorums and the call returns 0. */
|
||||
int acked = -1;
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
config_commit_change(c, (const uint8_t *)"cfg-bytes", 9, sha, tag, &acked));
|
||||
|
||||
/* (i) The marker landed at LSN 1 with the LIVE election term stamped. */
|
||||
TEST_ASSERT_EQUAL_UINT64(1, c->ha.raft.log_len);
|
||||
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1));
|
||||
/* No standbys registered → zero out-of-band pushes (barrier owns the verdict). */
|
||||
TEST_ASSERT_EQUAL_INT(0, acked);
|
||||
|
||||
/* Make the marker durable (advances durable_lsn) then commit its WAL LSN
|
||||
* through the real commit engine (self-quorum), exactly as a majority WAL
|
||||
* ack does in production — this advances committed_lsn so the derive sees
|
||||
* the marker in the committed prefix. */
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_submit(&c->commit, /*marker LSN*/1, COMMIT_MODE_WAIT, 1000));
|
||||
TEST_ASSERT_TRUE(
|
||||
atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire) >= 1);
|
||||
|
||||
/* (ii) The DERIVED committed config equals the change just committed. */
|
||||
uint64_t got_tag = 0;
|
||||
uint8_t got_sha[32];
|
||||
TEST_ASSERT_EQUAL_INT(0, config_committed_marker(c, &got_tag, got_sha));
|
||||
TEST_ASSERT_EQUAL_HEX64(tag, got_tag);
|
||||
TEST_ASSERT_EQUAL_HEX8_ARRAY(sha, got_sha, 32);
|
||||
|
||||
/* committed_config_tag is now a CACHE mirroring the derived marker. */
|
||||
TEST_ASSERT_EQUAL_HEX64(tag,
|
||||
atomic_load_explicit(&c->ha.committed_config_tag, memory_order_acquire));
|
||||
|
||||
commit_engine_fini(&c->commit);
|
||||
t42_teardown(c);
|
||||
printf("test_config_commit_change_live_path_commits_and_derives: PASS\n");
|
||||
}
|
||||
|
||||
/* (f) COMPACTION SAFETY (Finding 2 remediation): a checkpoint that TRIMS the WAL
|
||||
* segment holding a committed marker must NOT lose the derived committed config.
|
||||
* Without the sentinel-carried reference + fallback, the live-WAL scan regresses
|
||||
* to -ENOENT after the trim. We commit a marker, force a NEW segment + checkpoint
|
||||
* (which trims the old segment containing the marker), and assert the derive
|
||||
* STILL returns the committed reference (R7.7/R7.10). */
|
||||
static void test_committed_marker_survives_checkpoint_trim(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr42_trim_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
const uint64_t T = 11;
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, T, /*n_peers*/1);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_engine_init(&c->commit, &c->persist, /*n_peers*/1, /*n_vcpus*/8));
|
||||
repl_set_commit_engine(&c->persist.repl, &c->commit);
|
||||
|
||||
uint8_t sha[32];
|
||||
uint64_t tag = t42_make_ref(sha, 0x70);
|
||||
|
||||
/* Append the marker (segment 0, the lowest-LSN segment) and commit its WAL
|
||||
* LSN through the real engine (self-quorum) so it enters the committed
|
||||
* prefix — the production marker-commit path. */
|
||||
int acked = -1;
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
config_commit_change(c, (const uint8_t *)"cfg", 3, sha, tag, &acked));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_submit(&c->commit, /*marker LSN*/1, COMMIT_MODE_WAIT, 1000));
|
||||
uint64_t committed =
|
||||
atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire);
|
||||
TEST_ASSERT_TRUE(committed >= 1);
|
||||
|
||||
/* Sanity: before any trim, the live-WAL scan finds the marker. */
|
||||
uint64_t got_tag = 0;
|
||||
uint8_t got_sha[32];
|
||||
TEST_ASSERT_EQUAL_INT(0, config_committed_marker(c, &got_tag, got_sha));
|
||||
TEST_ASSERT_EQUAL_HEX64(tag, got_tag);
|
||||
|
||||
/* Roll to a NEW segment so the checkpoint's trim has an older segment (the
|
||||
* one holding the marker) to unlink — the compaction hazard. */
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(&c->persist));
|
||||
|
||||
/* Checkpoint: stamps the committed-marker reference into the sentinel, then
|
||||
* trims the older segment (which physically held the marker). */
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(&c->persist));
|
||||
|
||||
/* The marker's segment is gone, so the live-WAL scan alone would regress to
|
||||
* -ENOENT; the derive MUST fall back to the durable sentinel reference. */
|
||||
got_tag = 0;
|
||||
memset(got_sha, 0, sizeof(got_sha));
|
||||
TEST_ASSERT_EQUAL_INT(0, config_committed_marker(c, &got_tag, got_sha));
|
||||
TEST_ASSERT_EQUAL_HEX64(tag, got_tag);
|
||||
TEST_ASSERT_EQUAL_HEX8_ARRAY(sha, got_sha, 32);
|
||||
|
||||
commit_engine_fini(&c->commit);
|
||||
t42_teardown(c);
|
||||
printf("test_committed_marker_survives_checkpoint_trim: PASS\n");
|
||||
}
|
||||
|
||||
/* ── wait_all_with_timeout (XCE_TESTING shim) ───── */
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
|
|
@ -2351,6 +2713,14 @@ void test_config_repl(void)
|
|||
RUN_TEST(test_config_marker_encode_decode_roundtrip);
|
||||
RUN_TEST(test_config_marker_decode_truncated_fails);
|
||||
RUN_TEST(test_config_marker_encode_short_buffer_fails);
|
||||
/* Task 4.2: append marker via unified log + derive committed config */
|
||||
RUN_TEST(test_config_marker_append_via_unified_log);
|
||||
RUN_TEST(test_committed_marker_derives_after_commit);
|
||||
RUN_TEST(test_uncommitted_marker_not_yet_committed_config);
|
||||
RUN_TEST(test_divergent_marker_truncated_like_job_entry);
|
||||
/* Task 4.2 r1: live wiring + compaction-safe derive */
|
||||
RUN_TEST(test_config_commit_change_live_path_commits_and_derives);
|
||||
RUN_TEST(test_committed_marker_survives_checkpoint_trim);
|
||||
#ifdef XCE_TESTING
|
||||
RUN_TEST(test_load_primary_config_missing_returns_error);
|
||||
RUN_TEST(test_wait_all_with_timeout_slow_threads);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue