/* test_config_repl.c — Unity tests for config replication Phase 1 */ #define _GNU_SOURCE #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "cluster.h" #include "commit.h" #include "config_repl.h" #include "controller.capnp.h" #include "ha.h" #include "myconf.h" #include "persist.h" #include "repl.h" #include "request.h" #include "snapdiff.h" /* snapdiff_paths -- task 8.7 Scenario 2: prove a * leadership-change-driven materialization drain * produces a byte-identical checkpoint to an ordinary, * non-promoted drain of the same marker */ #include "typemap.h" #include "wal_types.h" #include "xce/cluster_def.h" #ifdef XCE_TESTING /* Declared in ha.h — test seam for ha_apply_wal_catchup_entry. * ep and c are passed as NULL in unit tests (ACK path is skipped when * either is NULL — Task 3.1, ha-wal-replication). */ int ha_apply_wal_catchup_entry(ha_state_t *ha, persist_t *p, wal_catchup_entry_t *entry, struct endpoint_t *ep, cluster_t *c); #endif /* Per-test directory helpers — not Unity setUp/tearDown (those are in * controller_test.c). Each test that needs a temp dir calls cr_setup() * at the top and cr_teardown() via the label before returning. */ static char g_tmpdir[256]; /* g_instdir = g_tmpdir/ctrl0 — the effective_dir used by conf_save_config() */ static char g_instdir[300]; static char g_conf_dir[356]; /* tmpdir string for load_primary_config tests (no conf/current) */ static char tmpdir[256]; static void cr_setup(void) { snprintf(g_tmpdir, sizeof(g_tmpdir), "/tmp/test_cr_%d_%ld", (int)getpid(), (long)time(NULL)); mkdir(g_tmpdir, 0700); /* conf_save_config() writes to workdir/name/conf — mirror that here */ snprintf(g_instdir, sizeof(g_instdir), "%s/ctrl0", g_tmpdir); mkdir(g_instdir, 0700); snprintf(g_conf_dir, sizeof(g_conf_dir), "%s/conf", g_instdir); mkdir(g_conf_dir, 0700); } static void cr_teardown(void) { char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", g_tmpdir); system(cmd); g_tmpdir[0] = '\0'; g_conf_dir[0] = '\0'; } /* ── conf_read_current_filename ─────────────────── */ static void test_conf_read_current_filename_missing(void) { cr_setup(); char out[256] = {0}; int rc = conf_read_current_filename(g_tmpdir, out, sizeof(out)); TEST_ASSERT_EQUAL_INT(-ENOENT, rc); cr_teardown(); printf("test_conf_read_current_filename_missing: PASS\n"); } static void test_conf_read_current_filename_empty_file(void) { cr_setup(); char ptr_path[400]; snprintf(ptr_path, sizeof(ptr_path), "%s/current", g_conf_dir); int fd = open(ptr_path, O_WRONLY | O_CREAT | O_TRUNC, 0600); TEST_ASSERT_NOT_EQUAL(-1, fd); close(fd); char out[256] = {0}; /* conf/current lives under g_instdir/conf — pass g_instdir as base */ int rc = conf_read_current_filename(g_instdir, out, sizeof(out)); TEST_ASSERT_EQUAL_INT(-ENODATA, rc); cr_teardown(); printf("test_conf_read_current_filename_empty_file: PASS\n"); } static void test_conf_read_current_filename_path_separator(void) { cr_setup(); char ptr_path[400]; snprintf(ptr_path, sizeof(ptr_path), "%s/current", g_conf_dir); FILE *f = fopen(ptr_path, "w"); fprintf(f, "cluster.bin/../evil\n"); fclose(f); char out[256] = {0}; /* conf/current lives under g_instdir/conf — pass g_instdir as base */ int rc = conf_read_current_filename(g_instdir, out, sizeof(out)); TEST_ASSERT_EQUAL_INT(-EINVAL, rc); cr_teardown(); printf("test_conf_read_current_filename_path_separator: PASS\n"); } static void test_conf_read_current_filename_bad_prefix(void) { cr_setup(); char ptr_path[400]; snprintf(ptr_path, sizeof(ptr_path), "%s/current", g_conf_dir); FILE *f = fopen(ptr_path, "w"); fprintf(f, "badprefix.aabbccddeeff\n"); fclose(f); char out[256] = {0}; /* conf/current lives under g_instdir/conf — pass g_instdir as base */ int rc = conf_read_current_filename(g_instdir, out, sizeof(out)); TEST_ASSERT_EQUAL_INT(-EINVAL, rc); cr_teardown(); printf("test_conf_read_current_filename_bad_prefix: PASS\n"); } static void test_conf_read_current_filename_wrong_hex_length(void) { cr_setup(); char ptr_path[400]; snprintf(ptr_path, sizeof(ptr_path), "%s/current", g_conf_dir); FILE *f = fopen(ptr_path, "w"); fprintf(f, "cluster.bin.aabb\n"); /* only 4 hex chars, need 12 */ fclose(f); char out[256] = {0}; /* conf/current lives under g_instdir/conf — pass g_instdir as base */ int rc = conf_read_current_filename(g_instdir, out, sizeof(out)); TEST_ASSERT_EQUAL_INT(-EINVAL, rc); cr_teardown(); printf("test_conf_read_current_filename_wrong_hex_length: PASS\n"); } /* ── conf_save_config + conf_read_current_filename round-trip ─ */ static void test_conf_save_creates_pointer_file(void) { cr_setup(); const uint8_t data[] = "fake capnproto config blob"; size_t len = sizeof(data) - 1; uint8_t sha256[32]; SHA256(data, len, sha256); myconf_t myconf; memset(&myconf, 0, sizeof(myconf)); myconf.workdir = g_tmpdir; myconf.name = "ctrl0"; myconf.conf_keep_n = 2; myconf.conffile = NULL; int rc = conf_save_config(&myconf, data, len, sha256); TEST_ASSERT_EQUAL_INT(0, rc); /* conf/current must exist */ char ptr_path[400]; snprintf(ptr_path, sizeof(ptr_path), "%s/current", g_conf_dir); TEST_ASSERT_EQUAL_INT(0, access(ptr_path, F_OK)); free(myconf.conffile); cr_teardown(); printf("test_conf_save_creates_pointer_file: PASS\n"); } static void test_conf_save_filename_has_12_hex_suffix(void) { cr_setup(); const uint8_t data[] = "another fake blob"; size_t len = sizeof(data) - 1; uint8_t sha256[32]; SHA256(data, len, sha256); myconf_t myconf; memset(&myconf, 0, sizeof(myconf)); myconf.workdir = g_tmpdir; myconf.name = "ctrl0"; myconf.conf_keep_n = 2; myconf.conffile = NULL; conf_save_config(&myconf, data, len, sha256); char fname[256] = {0}; /* conf_save_config() writes to workdir/name/conf — read from g_instdir */ int rc = conf_read_current_filename(g_instdir, fname, sizeof(fname)); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(0, strncmp(fname, "cluster.bin.", 12)); const char *hex = fname + 12; TEST_ASSERT_EQUAL_INT(12, (int)strlen(hex)); TEST_ASSERT_EQUAL_INT(12, (int)strspn(hex, "0123456789abcdef")); free(myconf.conffile); cr_teardown(); printf("test_conf_save_filename_has_12_hex_suffix: PASS\n"); } /* ── conf_trim_old_versions ─────────────────────── */ static void test_conf_trim_keeps_only_n_versions(void) { cr_setup(); myconf_t myconf; memset(&myconf, 0, sizeof(myconf)); myconf.workdir = g_tmpdir; myconf.name = "ctrl0"; myconf.conf_keep_n = 2; myconf.conffile = NULL; uint8_t sha256[32]; for (int i = 0; i < 3; i++) { char data[32]; snprintf(data, sizeof(data), "config version %d", i); SHA256((uint8_t *)data, strlen(data), sha256); usleep(15000); /* ensure distinct mtime ordering */ conf_save_config(&myconf, (uint8_t *)data, strlen(data), sha256); free(myconf.conffile); myconf.conffile = NULL; } /* Count cluster.bin.* files (exclude .tmp) */ int count = 0; DIR *d = opendir(g_conf_dir); TEST_ASSERT_NOT_NULL(d); struct dirent *e; while ((e = readdir(d)) != NULL) { if (strncmp(e->d_name, "cluster.bin.", 12) == 0 && strstr(e->d_name, ".tmp") == NULL) count++; } closedir(d); TEST_ASSERT_EQUAL_INT(2, count); free(myconf.conffile); cr_teardown(); printf("test_conf_trim_keeps_only_n_versions: PASS\n"); } /* ── conf_get_saved_tag ─────────────────────────── */ static void test_conf_get_saved_tag_no_config(void) { cr_setup(); /* conf_get_saved_tag looks for conf/current under its argument; * after the path fix, conf lives under workdir/name = g_instdir */ uint64_t tag = conf_get_saved_tag(g_instdir); TEST_ASSERT_EQUAL_UINT64(0, tag); cr_teardown(); printf("test_conf_get_saved_tag_no_config: PASS\n"); } static void test_conf_get_saved_tag_after_save(void) { cr_setup(); const uint8_t data[] = "tag test blob"; size_t len = sizeof(data) - 1; uint8_t sha256[32]; SHA256(data, len, sha256); myconf_t myconf; memset(&myconf, 0, sizeof(myconf)); myconf.workdir = g_tmpdir; myconf.name = "ctrl0"; myconf.conf_keep_n = 2; myconf.conffile = NULL; conf_save_config(&myconf, data, len, sha256); free(myconf.conffile); uint64_t tag = conf_get_saved_tag(g_instdir); TEST_ASSERT_NOT_EQUAL(0, tag); /* Tag must equal first 8 bytes of sha256 in host order */ uint64_t expected; memcpy(&expected, sha256, 8); /* be64toh conversion */ #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ expected = __builtin_bswap64(expected); #endif TEST_ASSERT_EQUAL_UINT64(expected, tag); cr_teardown(); printf("test_conf_get_saved_tag_after_save: PASS\n"); } /* ── conf_check_stale ───────────────────────────── */ static void test_conf_check_stale_disabled(void) { cr_setup(); /* timeout 0 = disabled, never stale */ TEST_ASSERT_EQUAL_INT(0, conf_check_stale(g_instdir, 0)); cr_teardown(); printf("test_conf_check_stale_disabled: PASS\n"); } static void test_conf_check_stale_fresh(void) { cr_setup(); const uint8_t data[] = "stale test blob"; uint8_t sha256[32]; SHA256(data, sizeof(data) - 1, sha256); myconf_t myconf; memset(&myconf, 0, sizeof(myconf)); myconf.workdir = g_tmpdir; myconf.name = "ctrl0"; myconf.conf_keep_n = 2; myconf.conffile = NULL; conf_save_config(&myconf, data, sizeof(data) - 1, sha256); free(myconf.conffile); /* Just saved — not stale with 24h timeout */ TEST_ASSERT_EQUAL_INT(0, conf_check_stale(g_instdir, 86400)); cr_teardown(); printf("test_conf_check_stale_fresh: PASS\n"); } static void test_conf_check_stale_aged(void) { cr_setup(); const uint8_t data[] = "old config blob"; uint8_t sha256[32]; SHA256(data, sizeof(data) - 1, sha256); myconf_t myconf; memset(&myconf, 0, sizeof(myconf)); myconf.workdir = g_tmpdir; myconf.name = "ctrl0"; myconf.conf_keep_n = 2; myconf.conffile = NULL; conf_save_config(&myconf, data, sizeof(data) - 1, sha256); free(myconf.conffile); /* Age the versioned file by 20 seconds via utime */ char fname[256] = {0}; /* conf_save_config() writes to workdir/name/conf — read from g_instdir */ conf_read_current_filename(g_instdir, fname, sizeof(fname)); char ver_path[512]; snprintf(ver_path, sizeof(ver_path), "%s/%s", g_conf_dir, fname); struct utimbuf old_time; old_time.actime = time(NULL) - 20; old_time.modtime = time(NULL) - 20; utime(ver_path, &old_time); /* 5-second threshold → stale */ TEST_ASSERT_EQUAL_INT(1, conf_check_stale(g_instdir, 5)); /* 30-second threshold → not stale */ TEST_ASSERT_EQUAL_INT(0, conf_check_stale(g_instdir, 30)); cr_teardown(); printf("test_conf_check_stale_aged: PASS\n"); } /* ── config_contains_reserved_name ─────────────── */ static void test_config_contains_reserved_name_absent(void) { /* NULL config — no reserved name */ TEST_ASSERT_EQUAL_INT(0, config_contains_reserved_name(NULL)); printf("test_config_contains_reserved_name_absent: PASS\n"); } /* ── sanitise_cn ────────────────────────────────── */ static void test_sanitise_cn_printable(void) { char *out = sanitise_cn("ctrl01.example.com", 64); TEST_ASSERT_EQUAL_STRING("ctrl01.example.com", out); printf("test_sanitise_cn_printable: PASS\n"); } static void test_sanitise_cn_control_chars(void) { char *out = sanitise_cn("evil\ninjection", 64); /* newline replaced with '?' */ TEST_ASSERT_NOT_NULL(strchr(out, '?')); TEST_ASSERT_NULL(strchr(out, '\n')); printf("test_sanitise_cn_control_chars: PASS\n"); } static void test_sanitise_cn_truncated(void) { char *out = sanitise_cn("abcdefghij", 5); TEST_ASSERT_EQUAL_INT(5, (int)strlen(out)); printf("test_sanitise_cn_truncated: PASS\n"); } /* ── load_primary_config (via test-only helper) ── */ #ifdef XCE_TESTING static void test_load_primary_config_missing_returns_error(void) { /* Create a minimal cluster_t with no conf/current file */ cluster_t c; memset(&c, 0, sizeof(c)); myconf_t myconf; memset(&myconf, 0, sizeof(myconf)); /* Use a fresh tmpdir with no conf/current */ snprintf(tmpdir, sizeof(tmpdir), "/tmp/test_lpc_%d_%ld", (int)getpid(), (long)time(NULL)); mkdir(tmpdir, 0700); myconf.workdir = tmpdir; myconf.name = "ctrl0"; myconf.conf_keep_n = 2; myconf.conf_stale_timeout = 86400; myconf.conffile = NULL; c.myconf = &myconf; /* No conf/current exists — must fail (not FATAL in test context) */ int rc = load_primary_config_for_test(&c); TEST_ASSERT_TRUE(rc < 0); /* conffile must still be NULL (no heap allocation on failure) */ TEST_ASSERT_NULL(myconf.conffile); /* Cleanup */ char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_load_primary_config_missing_returns_error: PASS\n"); } #endif /* ── test_conf_save_config_path_consistency ─────── */ static void test_conf_save_config_path_consistency(void) { /* Verify conf_save_config() writes to workdir/name/conf, not workdir/conf. * This mirrors the path that load_primary_config() reads from. */ char workdir[256]; snprintf(workdir, sizeof(workdir), "/tmp/test_pathbug_%d", (int)getpid()); char instdir[300]; snprintf(instdir, sizeof(instdir), "%s/ctrl01", workdir); char conf_dir[356]; snprintf(conf_dir, sizeof(conf_dir), "%s/conf", instdir); mkdir(workdir, 0700); /* conf_save_config() builds effective_dir = workdir/name internally, * then mkdirs effective_dir/conf — but effective_dir itself must exist. */ mkdir(instdir, 0700); /* Fake 32-byte sha256 and dummy payload */ uint8_t sha256[32]; memset(sha256, 0xab, sizeof(sha256)); const uint8_t payload[] = "dummy config bytes for path test"; size_t plen = sizeof(payload) - 1; myconf_t myconf; memset(&myconf, 0, sizeof(myconf)); myconf.workdir = workdir; myconf.name = "ctrl01"; myconf.conf_keep_n = 2; myconf.conffile = NULL; int rc = conf_save_config(&myconf, payload, plen, sha256); TEST_ASSERT_EQUAL_INT(0, rc); /* Versioned file must be under workdir/name/conf/, not workdir/conf/ */ TEST_ASSERT_NOT_NULL(myconf.conffile); TEST_ASSERT_EQUAL_INT(0, strncmp(myconf.conffile, conf_dir, strlen(conf_dir))); /* conf/current pointer file must exist at workdir/name/conf/current */ char ptr_path[400]; snprintf(ptr_path, sizeof(ptr_path), "%s/current", conf_dir); TEST_ASSERT_EQUAL_INT(0, access(ptr_path, F_OK)); /* workdir/conf/ must NOT exist (wrong location) */ char wrong_conf[300]; snprintf(wrong_conf, sizeof(wrong_conf), "%s/conf", workdir); struct stat st; TEST_ASSERT_NOT_EQUAL(0, stat(wrong_conf, &st)); free(myconf.conffile); /* Cleanup */ char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", workdir); system(cmd); printf("test_conf_save_config_path_consistency: PASS\n"); } /* ── WAL_CONFIG_MARKER codec (ha-term-safety task 4.1) ─────────────── * The config-change marker payload carries ONLY {version_tag, sha256} — * a reference, never the configuration bytes (R7.2/R7.4). These tests * exercise the fixed-size encode/decode codec that rides inside the * raft_entry_blob_t envelope with type == WAL_CONFIG_MARKER. */ /* Round-trip: encode {version_tag, sha256} then decode → identical, and the * encoded form is exactly the fixed marker size (tag + 32-byte hash) with NO * configuration bytes appended. */ static void test_config_marker_encode_decode_roundtrip(void) { const uint64_t version_tag = 0x0123456789abcdefULL; uint8_t sha256[32]; for (int i = 0; i < 32; i++) sha256[i] = (uint8_t)(0xA0 + i); uint8_t buf[64]; size_t enc_len = 0; int rc = config_marker_encode(version_tag, sha256, buf, sizeof(buf), &enc_len); TEST_ASSERT_EQUAL_INT(0, rc); /* Encoded form carries NO config bytes: exactly 8 (tag) + 32 (sha256). */ TEST_ASSERT_EQUAL_UINT(WAL_CONFIG_MARKER_PAYLOAD_LEN, enc_len); TEST_ASSERT_EQUAL_UINT(8u + 32u, enc_len); uint64_t got_tag = 0; uint8_t got_sha[32]; memset(got_sha, 0, sizeof(got_sha)); rc = config_marker_decode(buf, enc_len, &got_tag, got_sha); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_HEX64(version_tag, got_tag); TEST_ASSERT_EQUAL_HEX8_ARRAY(sha256, got_sha, 32); printf("test_config_marker_encode_decode_roundtrip: PASS\n"); } /* The WAL entry type value must be exactly 50 and must not collide with the * neighbouring WAL_NOOP (40) or WAL_CHECKPOINT (100) types (design.md). */ static void test_config_marker_enum_value_is_50(void) { TEST_ASSERT_EQUAL_INT(50, WAL_CONFIG_MARKER); TEST_ASSERT_NOT_EQUAL(WAL_NOOP, WAL_CONFIG_MARKER); TEST_ASSERT_NOT_EQUAL(WAL_CHECKPOINT, WAL_CONFIG_MARKER); /* Strictly between WAL_NOOP and WAL_CHECKPOINT. */ TEST_ASSERT_TRUE(WAL_CONFIG_MARKER > WAL_NOOP); TEST_ASSERT_TRUE(WAL_CONFIG_MARKER < WAL_CHECKPOINT); printf("test_config_marker_enum_value_is_50: PASS\n"); } /* A truncated (short) buffer must fail cleanly with -EINVAL, never read past * the end of the supplied buffer. */ static void test_config_marker_decode_truncated_fails(void) { uint8_t sha256[32]; memset(sha256, 0x5c, sizeof(sha256)); uint8_t buf[WAL_CONFIG_MARKER_PAYLOAD_LEN]; size_t enc_len = 0; TEST_ASSERT_EQUAL_INT(0, config_marker_encode(0xdeadbeefcafef00dULL, sha256, buf, sizeof(buf), &enc_len)); uint64_t tag = 0; uint8_t sha_out[32]; /* One byte short of a full marker payload → reject, no OOB read. */ int rc = config_marker_decode(buf, enc_len - 1, &tag, sha_out); TEST_ASSERT_EQUAL_INT(-EINVAL, rc); /* Zero-length buffer → reject. */ rc = config_marker_decode(buf, 0, &tag, sha_out); TEST_ASSERT_EQUAL_INT(-EINVAL, rc); printf("test_config_marker_decode_truncated_fails: PASS\n"); } /* A too-small output buffer must fail with -ENOSPC and not overflow. */ static void test_config_marker_encode_short_buffer_fails(void) { uint8_t sha256[32]; memset(sha256, 0x11, sizeof(sha256)); uint8_t small[WAL_CONFIG_MARKER_PAYLOAD_LEN - 1]; size_t enc_len = 123; /* sentinel — must be left untouched on failure */ int rc = config_marker_encode(7, sha256, small, sizeof(small), &enc_len); TEST_ASSERT_EQUAL_INT(-ENOSPC, rc); printf("test_config_marker_encode_short_buffer_fails: PASS\n"); } /* ── Config marker append + committed-config derive (task 4.2) ──────── * * These bring a node up as the raft LEADER so config_append_marker routes the * WAL_CONFIG_MARKER through ha_primary_log_append → raft_log_append (the same * unified-log path job state uses), and assert the committed config is DERIVED * from the latest committed marker within the committed prefix (R7.1/7.3/7.6/ * 7.7/7.11). Mirrors make_cluster/make_leader from test_term_safety_adapter.c. */ static cluster_t *t42_make_cluster(const char *tmpdir) { cluster_t *c = calloc(1, sizeof(cluster_t)); if (!c) return NULL; persist_config_t cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; cfg.snap_keep_n = 2; persist_set_workdir(&c->persist, tmpdir, "ctrl-test"); if (persist_init(&c->persist, &cfg, c) != 0) { free(c); return NULL; } if (persist_start(&c->persist) != 0) { persist_fini(&c->persist); free(c); return NULL; } c->ha.raft_c = c; return c; } static void t42_teardown(cluster_t *c) { if (!c) return; persist_stop(&c->persist); persist_fini(&c->persist); free(c); } /* config-upgrade-snapshot task 5: committing a WAL_CONFIG_MARKER now triggers * an automatic forced checkpoint the instant the marker is APPLIED (not just * committed) via the live apply pump -- persist_apply_pump_kick's dispatch is * a CAS race between whichever thread reaches it first, which can be either * this test's own thread (inside commit_submit's synchronous * commit_resolve_pending call) or persist_start's independent fsync_worker * thread (via commit_on_durable_advance) once durable_lsn advances. When the * fsync thread wins that race, commit_submit/commit_engine's own WAIT mode * returns having waited only for COMMIT, not for this now-real APPLY+ * CHECKPOINT side effect to finish on that other thread -- a fixture's own * subsequent LSN bookkeeping or explicit persist_checkpoint() call can then * genuinely race the still-in-flight background checkpoint (observed as a * flaky -EBUSY from persist_checkpoint() or an unexpectedly-advanced * committed_lsn, depending on which check loses the race). This was always a * latent, unsynchronized race in this fixture (the same one * test_apply_pump.c's tap_wait_applied already documents and exists to * close) -- applying a marker used to be a no-op, so racing on it was * invisible; task 5 gives it a real, observable side effect for the first * time. Poll persist_last_applied_lsn() up to a bounded deadline (mirrors * tap_wait_applied exactly) before any LSN-sensitive step that follows a * marker commit, closing the race rather than leaving it flaky. */ static void t42_wait_applied(persist_t *p, uint64_t lsn, uint32_t deadline_ms) { for (uint32_t waited = 0; waited < deadline_ms; waited += 5) { if (persist_last_applied_lsn(p) >= lsn) return; usleep(5000); } } /* Bring the node up as the raft LEADER at `term` (so raft_log_append is enabled * — it returns -EPERM off the leader; see make_leader in the adapter suite). */ static void t42_make_leader(cluster_t *c, uint64_t term, int n_peers) { atomic_store(&c->ha.role, CTRL_PRIMARY); atomic_store(&c->ha.term, term); ha_init_raft(&c->ha, c, n_peers); c->ha.raft.term = term; c->ha.raft.role = RAFT_LEADER; } /* Fill an sha256 with a deterministic pattern; tag = be64 of first 8 bytes, * matching the config_repl version_tag convention (config_version_tag). */ static uint64_t t42_make_ref(uint8_t sha256[32], uint8_t seed) { for (int i = 0; i < 32; i++) sha256[i] = (uint8_t)(seed + i); uint64_t tag; memcpy(&tag, sha256, 8); return be64toh(tag); } /* (a) Appending a config change writes a WAL_CONFIG_MARKER through the unified * log: the entry lands at an LSN with the live election term stamped, and its * payload decodes to {version_tag, sha256} (R7.1/7.3). */ static void test_config_marker_append_via_unified_log(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr42_append_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 7; cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, T, /*n_peers*/1); uint8_t sha[32]; uint64_t tag = t42_make_ref(sha, 0x10); uint64_t lsn = 0; TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &lsn)); /* Library assigned LSN 1 and stamped the live term for that entry. */ TEST_ASSERT_EQUAL_UINT64(1, lsn); TEST_ASSERT_EQUAL_UINT64(1, c->ha.raft.log_len); TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1)); t42_teardown(c); printf("test_config_marker_append_via_unified_log: PASS\n"); } /* (b) After the marker COMMITS (committed_lsn covers its LSN), the derived * committed-config reference == that marker's {version_tag, sha256} (R7.7). */ static void test_committed_marker_derives_after_commit(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr42_derive_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/4, /*n_peers*/1); /* A job entry then a config marker, so the marker is NOT at LSN 1. */ wal_jobstate_t js = { .jobid = 1, .state = 1 }; uint64_t jlsn = 0; TEST_ASSERT_EQUAL_INT(0, ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &jlsn)); uint8_t sha[32]; uint64_t tag = t42_make_ref(sha, 0x40); uint64_t mlsn = 0; TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &mlsn)); TEST_ASSERT_EQUAL_UINT64(2, mlsn); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); /* Drive committed_lsn to the marker's LSN (the commit advance source). */ atomic_store_explicit(&c->persist.committed_lsn, mlsn, memory_order_release); uint64_t got_tag = 0; uint8_t got_sha[32]; memset(got_sha, 0, sizeof(got_sha)); TEST_ASSERT_EQUAL_INT(0, config_committed_marker(c, &got_tag, got_sha)); TEST_ASSERT_EQUAL_HEX64(tag, got_tag); TEST_ASSERT_EQUAL_HEX8_ARRAY(sha, got_sha, 32); t42_teardown(c); printf("test_committed_marker_derives_after_commit: PASS\n"); } /* (c) An UNCOMMITTED marker (LSN > committed_lsn) does NOT change the derived * committed config: with committed_lsn at the first marker, a later second * marker is invisible until committed (R7.7/7.11). */ static void test_uncommitted_marker_not_yet_committed_config(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr42_uncommit_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/5, /*n_peers*/1); uint8_t sha1[32], sha2[32]; uint64_t tag1 = t42_make_ref(sha1, 0x10); uint64_t tag2 = t42_make_ref(sha2, 0x80); uint64_t l1 = 0, l2 = 0; TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag1, sha1, &l1)); TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag2, sha2, &l2)); TEST_ASSERT_EQUAL_UINT64(1, l1); TEST_ASSERT_EQUAL_UINT64(2, l2); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); /* Only the FIRST marker is committed. The second is uncommitted. */ atomic_store_explicit(&c->persist.committed_lsn, l1, memory_order_release); uint64_t got_tag = 0; uint8_t got_sha[32]; TEST_ASSERT_EQUAL_INT(0, config_committed_marker(c, &got_tag, got_sha)); /* Derived config is the FIRST (committed) marker, NOT the uncommitted one. */ TEST_ASSERT_EQUAL_HEX64(tag1, got_tag); TEST_ASSERT_EQUAL_HEX8_ARRAY(sha1, got_sha, 32); /* With NOTHING committed, there is no committed config marker at all. */ atomic_store_explicit(&c->persist.committed_lsn, 0ULL, memory_order_release); TEST_ASSERT_EQUAL_INT(-ENOENT, config_committed_marker(c, &got_tag, got_sha)); t42_teardown(c); printf("test_uncommitted_marker_not_yet_committed_config: PASS\n"); } /* (d) A divergent config marker at a conflicting (term,lsn) is truncated by the * reconcile path exactly like a job entry: on_truncate_from(index) removes the * marker (and everything above), leaving the committed prefix intact. This * proves the marker is an ORDINARY log entry under (term,lsn) matching (R7.6). * * We append a committed job at LSN 1, then a divergent marker at LSN 2, drive * committed_lsn to 1, then truncate from LSN 2 (the conflict point) and confirm * the marker is gone while the committed job prefix survives. */ static void test_divergent_marker_truncated_like_job_entry(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr42_trunc_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/6, /*n_peers*/1); /* LSN 1: committed job entry. LSN 2: a (divergent) config marker. */ wal_jobstate_t js = { .jobid = 9, .state = 2 }; uint64_t jlsn = 0; TEST_ASSERT_EQUAL_INT(0, ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &jlsn)); TEST_ASSERT_EQUAL_UINT64(1, jlsn); uint8_t sha[32]; uint64_t tag = t42_make_ref(sha, 0x55); uint64_t mlsn = 0; TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &mlsn)); TEST_ASSERT_EQUAL_UINT64(2, mlsn); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); /* The committed prefix is LSN 1 (the job). The marker at LSN 2 is the * divergent suffix. The derived committed config is therefore NONE yet. */ atomic_store_explicit(&c->persist.committed_lsn, 1ULL, memory_order_release); uint64_t got_tag = 0; uint8_t got_sha[32]; TEST_ASSERT_EQUAL_INT(-ENOENT, config_committed_marker(c, &got_tag, got_sha)); /* Reconcile truncates from the conflict point (LSN 2) exactly as it would a * job entry — the SAME on_truncate_from seam used by (term,lsn) matching. */ TEST_ASSERT_EQUAL_INT(0, ha_cb_truncate_from_for_test(&c->ha, /*index*/2)); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); /* The marker is gone (term index shrank); the committed job prefix is intact. */ TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(&c->persist, 2)); TEST_ASSERT_EQUAL_UINT64(6, persist_entry_term(&c->persist, 1)); /* Still no committed config marker — the divergent one never committed. */ TEST_ASSERT_EQUAL_INT(-ENOENT, config_committed_marker(c, &got_tag, got_sha)); t42_teardown(c); printf("test_divergent_marker_truncated_like_job_entry: PASS\n"); } /* (e) INTEGRATION (Finding 1 remediation): drive a config change through the * LIVE commit path (config_commit_change — the helper do_upgrade_config_with_bytes * now calls) and assert (i) a WAL_CONFIG_MARKER lands at the LIVE term, and * (ii) once the marker COMMITS through the real WAL commit barrier, the DERIVED * committed config (config_committed_marker) equals it. This proves the live * config-change path appends + commits + derives the marker — not the retired * term-blind committed_config_tag push/ack/advance. * * A single-member commit engine is wired so config_commit_change's WAL barrier * (repl_wait_majority_ack/OPCode_walAck → commit_submit) self-quorums and * advances committed_lsn for real, exactly as the production single-node path. */ static void test_config_commit_change_live_path_commits_and_derives(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr42_live_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 9; cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, T, /*n_peers*/1); /* Single-member commit engine: the marker's WAL LSN self-quorums, so * driving commit_submit on it advances committed_lsn through the REAL commit * machinery — the same path that commits the marker in production once a * majority acks the WAL stream (no manual committed_lsn poke). */ TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, /*n_peers*/1, /*n_vcpus*/8)); repl_set_commit_engine(&c->persist.repl, &c->commit); uint8_t sha[32]; uint64_t tag = t42_make_ref(sha, 0x30); /* The LIVE entry point do_upgrade_config_with_bytes now calls: append the * WAL_CONFIG_MARKER consensus record + out-of-band byte push + the existing * majority barrier. With no standbys (single member) the byte barrier * self-quorums and the call returns 0. */ int acked = -1; TEST_ASSERT_EQUAL_INT(0, config_commit_change(c, (const uint8_t *)"cfg-bytes", 9, sha, tag, &acked)); /* (i) The marker landed at LSN 1 with the LIVE election term stamped. */ TEST_ASSERT_EQUAL_UINT64(1, c->ha.raft.log_len); TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1)); /* No standbys registered → zero out-of-band pushes (barrier owns the verdict). */ TEST_ASSERT_EQUAL_INT(0, acked); /* Make the marker durable (advances durable_lsn) then commit its WAL LSN * through the real commit engine (self-quorum), exactly as a majority WAL * ack does in production — this advances committed_lsn so the derive sees * the marker in the committed prefix. */ TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); TEST_ASSERT_EQUAL_INT(0, commit_submit(&c->commit, /*marker LSN*/1, COMMIT_MODE_WAIT, 1000)); TEST_ASSERT_TRUE( atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire) >= 1); /* (ii) The DERIVED committed config equals the change just committed. */ uint64_t got_tag = 0; uint8_t got_sha[32]; TEST_ASSERT_EQUAL_INT(0, config_committed_marker(c, &got_tag, got_sha)); TEST_ASSERT_EQUAL_HEX64(tag, got_tag); TEST_ASSERT_EQUAL_HEX8_ARRAY(sha, got_sha, 32); /* committed_config_tag is now a CACHE mirroring the derived marker. */ TEST_ASSERT_EQUAL_HEX64(tag, atomic_load_explicit(&c->ha.committed_config_tag, memory_order_acquire)); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_config_commit_change_live_path_commits_and_derives: PASS\n"); } /* (f) COMPACTION SAFETY (Finding 2 remediation): a checkpoint that TRIMS the WAL * segment holding a committed marker must NOT lose the derived committed config. * Without the sentinel-carried reference + fallback, the live-WAL scan regresses * to -ENOENT after the trim. We commit a marker, force a NEW segment + checkpoint * (which trims the old segment containing the marker), and assert the derive * STILL returns the committed reference (R7.7/R7.10). */ static void test_committed_marker_survives_checkpoint_trim(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr42_trim_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 11; cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, T, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, /*n_peers*/1, /*n_vcpus*/8)); repl_set_commit_engine(&c->persist.repl, &c->commit); uint8_t sha[32]; uint64_t tag = t42_make_ref(sha, 0x70); /* Append the marker (segment 0, the lowest-LSN segment) and commit its WAL * LSN through the real engine (self-quorum) so it enters the committed * prefix — the production marker-commit path. */ int acked = -1; TEST_ASSERT_EQUAL_INT(0, config_commit_change(c, (const uint8_t *)"cfg", 3, sha, tag, &acked)); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); TEST_ASSERT_EQUAL_INT(0, commit_submit(&c->commit, /*marker LSN*/1, COMMIT_MODE_WAIT, 1000)); uint64_t committed = atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire); TEST_ASSERT_TRUE(committed >= 1); /* config-upgrade-snapshot task 5: wait for the marker to be APPLIED (see * t42_wait_applied's doc comment) before this test's own explicit * persist_checkpoint() call below -- otherwise it can race the marker's * own now-automatic checkpoint (possibly still running on the * fsync_worker thread) and see a spurious -EBUSY from the shared * checkpoint_in_progress guard. */ t42_wait_applied(&c->persist, /*marker LSN*/1, 2000); /* Sanity: before any trim, the live-WAL scan finds the marker. */ uint64_t got_tag = 0; uint8_t got_sha[32]; TEST_ASSERT_EQUAL_INT(0, config_committed_marker(c, &got_tag, got_sha)); TEST_ASSERT_EQUAL_HEX64(tag, got_tag); /* Roll to a NEW segment so the checkpoint's trim has an older segment (the * one holding the marker) to unlink — the compaction hazard. */ TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(&c->persist)); /* Checkpoint: stamps the committed-marker reference into the sentinel, then * trims the older segment (which physically held the marker). */ TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(&c->persist)); /* The marker's segment is gone, so the live-WAL scan alone would regress to * -ENOENT; the derive MUST fall back to the durable sentinel reference. */ got_tag = 0; memset(got_sha, 0, sizeof(got_sha)); TEST_ASSERT_EQUAL_INT(0, config_committed_marker(c, &got_tag, got_sha)); TEST_ASSERT_EQUAL_HEX64(tag, got_tag); TEST_ASSERT_EQUAL_HEX8_ARRAY(sha, got_sha, 32); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_committed_marker_survives_checkpoint_trim: PASS\n"); } /* ── Config byte delivery: hash-verify-before-activate + serve-fence (task 4.3) ─ * * config_activate_committed is the CONSUMER of config_committed_marker (4.2's * derive). It keys the out-of-band config-push byte delivery to the committed * marker's {version_tag, sha256}: it VERIFIES the locally-held bytes' sha256 * against the committed marker BEFORE activating (R7.8), and on a mismatch (or * bytes absent locally) keeps the member from serving under the new configuration * and KEEPS RETRYING rather than activating mismatched config (R7.9) — the loud * non-serving fence that NEVER self-terminates. On recovery it activates the * configuration referenced by the LATEST committed marker (R7.10). * * These tests reuse the 4.2 leader fixture (t42_*) but additionally wire c->myconf * so the locally-held config bytes path (conf_save_config / conf_read_raw / * load_primary_config) is exercised. */ /* Build a VALID Cap'n Proto ClusterConfig blob (decodable by load_primary_config) * so the match path can actually ACTIVATE. Mirrors cb5_make_config_blob. */ static uint8_t *t43_make_config_blob(size_t *len_out) { cluster_config_t cfg; struct capn cap; struct capn_segment *cs; ClusterConfig_ptr p; uint8_t *buf; size_t sz; memset(&cfg, 0, sizeof(cfg)); cfg.clustername = (char *)"t43-upgrade"; cfg.config_generation = 1; capn_init_malloc(&cap); cs = capn_root(&cap).seg; encode_ClusterConfig_ptr(cs, &p, &cfg); capn_setp(capn_root(&cap), 0, p.p); sz = (size_t)capn_size(&cap); buf = (uint8_t *)malloc(sz); TEST_ASSERT_NOT_NULL(buf); capn_write_mem(&cap, buf, sz, 0); capn_free(&cap); *len_out = sz; return buf; } /* Wire a myconf_t into the cluster so conf_save_config / load_primary_config * resolve under //conf (the production layout). */ static myconf_t *t43_attach_myconf(cluster_t *c, const char *tmpdir) { myconf_t *mc = calloc(1, sizeof(myconf_t)); TEST_ASSERT_NOT_NULL(mc); mc->workdir = strdup(tmpdir); mc->name = strdup("ctrl-test"); mc->conf_keep_n = 2; mc->conffile = NULL; /* bug-1476: the real load_primary_config()/config_activate_committed() * paths this fixture drives now take mc->lock -- must be initialized. */ TEST_ASSERT_EQUAL_INT(0, mylock_init(&mc->lock)); c->myconf = mc; return mc; } static void t43_free_myconf(cluster_t *c) { myconf_t *mc = c->myconf; if (!mc) return; if (mc->cluster_config) { free_ClusterConfig(mc->cluster_config); free(mc->cluster_config); } free(mc->conffile); free(mc->workdir); free(mc->name); mylock_fini(&mc->lock); free(mc); c->myconf = NULL; } /* Commit a WAL_CONFIG_MARKER for `sha`/`tag` through the real commit engine so * config_committed_marker returns it (the committed-prefix authority 4.3 reads). * Also waits for the marker to be APPLIED (see t42_wait_applied's doc comment) * so callers' own subsequent LSN bookkeeping never races this marker's own * task-5 automatic checkpoint, wherever it happens to run. */ static void t43_commit_marker(cluster_t *c, uint64_t tag, const uint8_t sha[32]) { uint64_t mlsn = 0; TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &mlsn)); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); TEST_ASSERT_EQUAL_INT(0, commit_submit(&c->commit, mlsn, COMMIT_MODE_WAIT, 1000)); TEST_ASSERT_TRUE( atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire) >= mlsn); t42_wait_applied(&c->persist, mlsn, 2000); } /* (4.3-a) MATCH activates: a committed marker whose locally-held bytes' sha256 * MATCHES the marker activates the configuration and leaves the member UN-fenced * (R7.8). */ static void test_config_activate_match_activates_and_unfences(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr43_match_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/3, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, 1, 8)); repl_set_commit_engine(&c->persist.repl, &c->commit); myconf_t *mc = t43_attach_myconf(c, tmp); /* The locally-held bytes (saved exactly as the config-push handler would). */ size_t blen = 0; uint8_t *bytes = t43_make_config_blob(&blen); uint8_t sha[32]; SHA256(bytes, blen, sha); uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); } TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha)); free(bytes); /* Commit a marker that references EXACTLY those bytes. */ t43_commit_marker(c, tag, sha); /* HASH-VERIFIED → activate, fence stays clear. */ TEST_ASSERT_EQUAL_INT(0, config_activate_committed(c)); TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c)); /* Activation loaded the config into myconf. */ TEST_ASSERT_NOT_NULL(mc->cluster_config); TEST_ASSERT_EQUAL_HEX64(tag, mc->config_version_tag); t43_free_myconf(c); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_config_activate_match_activates_and_unfences: PASS\n"); } /* (4.3-b) MISMATCH fences + retries, NEVER activates: an injected hash mismatch * (locally-held bytes whose sha256 != the committed marker) keeps the member * FENCED (non-serving), ALIVE, and retrying, and does NOT activate the mismatched * config (R7.9). */ static void test_config_activate_mismatch_fences_and_does_not_activate(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr43_mismatch_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/4, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, 1, 8)); repl_set_commit_engine(&c->persist.repl, &c->commit); myconf_t *mc = t43_attach_myconf(c, tmp); /* Locally-held bytes are blob A (sha A). */ size_t blenA = 0; uint8_t *bytesA = t43_make_config_blob(&blenA); uint8_t shaA[32]; SHA256(bytesA, blenA, shaA); TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytesA, blenA, shaA)); free(bytesA); /* But the committed marker references a DIFFERENT hash (injected mismatch: * the bytes for this marker have NOT arrived out-of-band yet). */ uint8_t shaB[32]; uint64_t tagB = t42_make_ref(shaB, 0x99); /* deterministic != shaA */ TEST_ASSERT_FALSE(memcmp(shaA, shaB, 32) == 0); t43_commit_marker(c, tagB, shaB); /* MISMATCH → fenced, NOT activated, member stays alive (returns, no abort). */ TEST_ASSERT_EQUAL_INT(-EAGAIN, config_activate_committed(c)); TEST_ASSERT_EQUAL_INT(1, config_serve_fenced(c)); /* The mismatched marker's config was NOT activated: the active config tag is * still the locally-held blob A's tag, NEVER the marker's tagB. */ TEST_ASSERT_NOT_EQUAL(tagB, mc->config_version_tag); /* RETRYING: a second call while still mismatched stays fenced (idempotent, * never activates mismatched) — the member keeps retrying, alive. */ TEST_ASSERT_EQUAL_INT(-EAGAIN, config_activate_committed(c)); TEST_ASSERT_EQUAL_INT(1, config_serve_fenced(c)); TEST_ASSERT_NOT_EQUAL(tagB, mc->config_version_tag); t43_free_myconf(c); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_config_activate_mismatch_fences_and_does_not_activate: PASS\n"); } /* (4.3-c) OBTAINED-OUT-OF-BAND bytes are verified before activate: a member is * fenced while the marker's bytes are absent locally; once the matching bytes * arrive out-of-band (saved exactly as handle_repl_config_push would), the next * activate hash-verifies them against the marker and activates, clearing the * fence (R7.5/R7.8). */ static void test_config_activate_obtained_bytes_verified_before_activate(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr43_obtain_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/5, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, 1, 8)); repl_set_commit_engine(&c->persist.repl, &c->commit); myconf_t *mc = t43_attach_myconf(c, tmp); /* The marker's referenced bytes. Compute its hash; commit the marker BEFORE * the bytes are present locally (no conf_save_config yet). */ size_t blen = 0; uint8_t *bytes = t43_make_config_blob(&blen); uint8_t sha[32]; SHA256(bytes, blen, sha); uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); } t43_commit_marker(c, tag, sha); /* Bytes NOT present locally yet → fenced, not activated (R7.9). */ TEST_ASSERT_EQUAL_INT(-EAGAIN, config_activate_committed(c)); TEST_ASSERT_EQUAL_INT(1, config_serve_fenced(c)); /* Bytes arrive out-of-band: the config-push transport saves them locally * exactly as handle_repl_config_push does (conf_save_config). */ TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha)); free(bytes); /* Now the obtained bytes hash-verify against the committed marker → activate, * fence clears. */ TEST_ASSERT_EQUAL_INT(0, config_activate_committed(c)); TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c)); TEST_ASSERT_NOT_NULL(mc->cluster_config); TEST_ASSERT_EQUAL_HEX64(tag, mc->config_version_tag); t43_free_myconf(c); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_config_activate_obtained_bytes_verified_before_activate: PASS\n"); } /* (4.3-d) RECOVERY activates the LATEST committed marker: with two committed * markers, config_activate_committed activates the config referenced by the * LATEST committed marker (the one config_committed_marker derives), obtaining * its bytes from local out-of-band storage (R7.10). */ static void test_config_activate_recovery_uses_latest_committed_marker(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr43_recover_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/6, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, 1, 8)); repl_set_commit_engine(&c->persist.repl, &c->commit); myconf_t *mc = t43_attach_myconf(c, tmp); /* Two distinct configs, each committed as a marker in turn; the LATEST wins. */ size_t blen1 = 0, blen2 = 0; uint8_t *b1 = t43_make_config_blob(&blen1); /* Make blob 2 differ so its hash differs from blob 1. */ uint8_t *b2 = t43_make_config_blob(&blen2); TEST_ASSERT_TRUE(blen2 >= 1); b2[blen2 - 1] ^= 0xFF; /* perturb → distinct sha */ uint8_t sha1[32], sha2[32]; SHA256(b1, blen1, sha1); SHA256(b2, blen2, sha2); TEST_ASSERT_FALSE(memcmp(sha1, sha2, 32) == 0); uint64_t tag2; { uint64_t t; memcpy(&t, sha2, 8); tag2 = be64toh(t); } uint64_t tag1; { uint64_t t; memcpy(&t, sha1, 8); tag1 = be64toh(t); } t43_commit_marker(c, tag1, sha1); t43_commit_marker(c, tag2, sha2); /* On recovery, the LOCALLY-held bytes are the LATEST config (blob 2 — what the * out-of-band store holds after the latest push). */ TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, b2, blen2, sha2)); free(b1); free(b2); /* Activation targets the LATEST committed marker (tag2), hash-verifies the * local bytes, and activates it. */ TEST_ASSERT_EQUAL_INT(0, config_activate_committed(c)); TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c)); TEST_ASSERT_EQUAL_HEX64(tag2, mc->config_version_tag); t43_free_myconf(c); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_config_activate_recovery_uses_latest_committed_marker: PASS\n"); } /* ── task 4.3 REMEDIATION r1: production integration of the consumer ─────────── * * Round-1 review REJECTED 4.3 for three integration gaps, fixed below: * A — verify-before-activate was BYPASSED on the push path (handle_repl_config_ * push pre-activated via load_primary_config before the gate ran). * B — recovery-activate had NO production call site (boot loaded blindly). * C — the serve-fence had NO production reader (ha_dispatch_gate ignored it). */ /* (4.3-A) PUSH-PATH SOLE ACTIVATION: config_activate_committed is the SOLE * activation point. With a committed marker that references DIFFERENT bytes than * the ones just delivered out-of-band, the push-path sequence (conf_save_config * then config_activate_committed — NO pre-emptive load_primary_config) must NOT * make the pushed bytes live (verify-before-activate is a real GATE, not an * after-the-fact overlay). Regression for Finding A. */ static void test_push_path_mismatch_does_not_preactivate(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr43_pushgate_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/7, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, 1, 8)); repl_set_commit_engine(&c->persist.repl, &c->commit); myconf_t *mc = t43_attach_myconf(c, tmp); /* A committed marker references blob B (sha B) — the config the cluster * agreed to, whose bytes have NOT yet arrived here. */ size_t blenB = 0; uint8_t *bytesB = t43_make_config_blob(&blenB); bytesB[blenB - 1] ^= 0x5A; /* perturb → distinct sha */ uint8_t shaB[32]; SHA256(bytesB, blenB, shaB); uint64_t tagB; { uint64_t t; memcpy(&t, shaB, 8); tagB = be64toh(t); } t43_commit_marker(c, tagB, shaB); free(bytesB); /* The push delivers blob A (a DIFFERENT/older config). The push handler now * SAVES the bytes but does NOT pre-activate; activation is gated. */ size_t blenA = 0; uint8_t *bytesA = t43_make_config_blob(&blenA); uint8_t shaA[32]; SHA256(bytesA, blenA, shaA); TEST_ASSERT_FALSE(memcmp(shaA, shaB, 32) == 0); TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytesA, blenA, shaA)); free(bytesA); /* The push-path SOLE activation: gate fences (marker references blob B, not * the just-saved blob A) and does NOT load blob A. */ TEST_ASSERT_EQUAL_INT(-EAGAIN, config_activate_committed(c)); TEST_ASSERT_EQUAL_INT(1, config_serve_fenced(c)); /* Blob A was NOT made live: no decoded config, no version tag for A. */ TEST_ASSERT_NULL(mc->cluster_config); t43_free_myconf(c); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_push_path_mismatch_does_not_preactivate: PASS\n"); } /* (4.3-A/B) NO-MARKER FALLBACK: when there is NO committed marker yet (legacy / * first boot, or the marker has not committed), config_activate_committed FALLS * BACK to the existing load_primary_config behavior — it activates the * locally-held bytes (so config_version_tag is set for the freshness gate and the * push ACK) and does NOT fence. Absent-marker is NOT a mismatch. */ static void test_config_activate_no_marker_falls_back_and_loads(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr43_fallback_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/8, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, 1, 8)); repl_set_commit_engine(&c->persist.repl, &c->commit); myconf_t *mc = t43_attach_myconf(c, tmp); /* Local bytes present, but NO committed marker exists. */ size_t blen = 0; uint8_t *bytes = t43_make_config_blob(&blen); uint8_t sha[32]; SHA256(bytes, blen, sha); uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); } TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha)); free(bytes); /* No marker → fall back to load_primary_config (blind load), NOT a fence. */ TEST_ASSERT_EQUAL_INT(0, config_activate_committed(c)); TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c)); TEST_ASSERT_NOT_NULL(mc->cluster_config); TEST_ASSERT_EQUAL_HEX64(tag, mc->config_version_tag); t43_free_myconf(c); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_config_activate_no_marker_falls_back_and_loads: PASS\n"); } /* ── config-upgrade-snapshot task 6.1: Activation Fence * (upgrade_materialization_verify / upgrade_materialization_fenced, ha.c) * ────────────────────────────────────────────────────────────────────── * These tests reuse this file's own t42_/t43_ fixture helpers rather than * duplicating them into a new file — the Activation Fence's verify function * is a close sibling of config_activate_committed (tested just above), and * every fixture piece it needs (a real cluster_t, a real commit engine, a * real myconf/config file, real committed WAL_CONFIG_MARKERs) already lives * here. task 8.3 owns broader dispatch-gate/promotion-gate fence coverage in * its own new file (test_upgrade_activation_fence.c per design.md); this * task's own boundary is ha.c/ha.h only, so its tests stay narrowly scoped * to upgrade_materialization_verify/upgrade_materialization_fenced * themselves. */ extern int upgrade_materialization_verify(cluster_t *c); extern int upgrade_materialization_fenced(const cluster_t *c); /* (6.1-a) No committed marker yet: benign, not a divergence — fence stays * clear, verify succeeds (mirrors config_activate_committed's own -ENOENT * handling). */ static void test_upgrade_materialization_verify_no_marker_clears_fence(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ucaf61_nomarker_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/20, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, 1, 8)); repl_set_commit_engine(&c->persist.repl, &c->commit); myconf_t *mc = t43_attach_myconf(c, tmp); TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_verify(c)); TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c)); t43_free_myconf(c); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_upgrade_materialization_verify_no_marker_clears_fence: PASS\n"); } /* (6.1-b) A committed marker exists and its bytes are locally correct and * hash-verify, AND this member's own checkpoint has already captured a * reference to it (task 5's marker-triggered checkpoint runs synchronously * on apply, so t43_commit_marker's wait-for-applied already guarantees this) * — materialization is genuinely complete, fence clears. */ static void test_upgrade_materialization_verify_matching_materialization_clears_fence(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ucaf61_match_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/21, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, 1, 8)); repl_set_commit_engine(&c->persist.repl, &c->commit); myconf_t *mc = t43_attach_myconf(c, tmp); size_t blen = 0; uint8_t *bytes = t43_make_config_blob(&blen); uint8_t sha[32]; SHA256(bytes, blen, sha); uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); } TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha)); free(bytes); t43_commit_marker(c, tag, sha); /* Task 5's synchronous marker-triggered checkpoint already ran by the * time t43_commit_marker (which waits for apply) returns — cfgref_* * already reflects this exact marker. */ TEST_ASSERT_TRUE(c->persist.cfgref_present); TEST_ASSERT_EQUAL_HEX64(tag, c->persist.cfgref_tag); TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_verify(c)); TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c)); t43_free_myconf(c); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_upgrade_materialization_verify_matching_materialization_clears_fence: PASS\n"); } /* (6.1-c) A NEW marker commits while an earlier checkpoint is (simulated) * still in-flight, so THIS marker's own materialization never actually runs * (task 5's -EBUSY path, Req 4.4) -- config_committed_marker now returns the * NEW marker but c->persist.cfgref_* still reflects the OLDER one. * Materialization is genuinely incomplete for the CURRENT marker -> fence * SETS (distinct from config_serve_fence, which would stay clear here since * the bytes for the new marker are already present and correct). */ static void test_upgrade_materialization_verify_incomplete_materialization_fences(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ucaf61_incomplete_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/22, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, 1, 8)); repl_set_commit_engine(&c->persist.repl, &c->commit); myconf_t *mc = t43_attach_myconf(c, tmp); /* Marker A: commits, applies, materializes normally (baseline). */ size_t blenA = 0; uint8_t *bytesA = t43_make_config_blob(&blenA); uint8_t shaA[32]; SHA256(bytesA, blenA, shaA); uint64_t tagA; { uint64_t t; memcpy(&t, shaA, 8); tagA = be64toh(t); } TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytesA, blenA, shaA)); free(bytesA); t43_commit_marker(c, tagA, shaA); TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_verify(c)); TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c)); TEST_ASSERT_EQUAL_HEX64(tagA, c->persist.cfgref_tag); /* Marker B: its bytes are saved correctly (config_serve_fence would stay * clear), but its own checkpoint is forced to skip (-EBUSY) by holding * the shared checkpoint_in_progress CAS open across the commit -- the * exact real mechanism task 5's marker-apply case hits under Req 4.4. */ size_t blenB = 0; uint8_t *bytesB = t43_make_config_blob(&blenB); /* t43_make_config_blob always produces byte-identical content -- mutate * a trailing byte so blob B genuinely differs from blob A (this test * only ever hashes/saves the raw bytes via upgrade_materialization_verify, * which never parses them as capnp, so a one-byte mutation is safe). */ bytesB[blenB - 1] ^= 0xFF; uint8_t shaB[32]; SHA256(bytesB, blenB, shaB); uint64_t tagB; { uint64_t t; memcpy(&t, shaB, 8); tagB = be64toh(t); } TEST_ASSERT_FALSE(memcmp(shaA, shaB, 32) == 0); TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytesB, blenB, shaB)); free(bytesB); TEST_ASSERT_EQUAL_INT( 0, atomic_exchange_explicit(&c->persist.checkpoint_in_progress, 1, memory_order_acq_rel)); uint64_t mlsnB = 0; TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tagB, shaB, &mlsnB)); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); TEST_ASSERT_EQUAL_INT(0, commit_submit(&c->commit, mlsnB, COMMIT_MODE_WAIT, 1000)); /* apply runs (marker B's WAL_CONFIG_MARKER case fires and hits -EBUSY on * the still-held CAS -- treated as expected/self-resolving, not a * failure, so apply itself succeeds); wait for it directly since * t43_commit_marker's own wait helper is not used here. */ uint64_t deadline_iters = 200; while (deadline_iters-- > 0 && atomic_load_explicit(&c->persist.last_applied_lsn, memory_order_acquire) < mlsnB) { struct timespec ts = { .tv_sec = 0, .tv_nsec = 10 * 1000 * 1000 }; nanosleep(&ts, NULL); } TEST_ASSERT_TRUE(atomic_load_explicit(&c->persist.last_applied_lsn, memory_order_acquire) >= mlsnB); /* Release the simulated in-flight checkpoint now that marker B's apply * (and its -EBUSY'd checkpoint attempt) has already happened. */ atomic_store_explicit(&c->persist.checkpoint_in_progress, 0, memory_order_release); /* config_committed_marker now authoritatively returns marker B, but this * member's own materialization never ran for it (cfgref_tag is still A) * -- genuinely incomplete, not a bytes mismatch. */ TEST_ASSERT_EQUAL_HEX64(tagA, c->persist.cfgref_tag); TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_verify(c)); TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); /* Idempotent retry: calling again while still genuinely unresolved stays * fenced, does not error/crash/toggle unexpectedly. */ TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_verify(c)); TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); t43_free_myconf(c); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_upgrade_materialization_verify_incomplete_materialization_fences: PASS\n"); } /* (6.1-d) Materialization for the current marker DID complete (cfgref_* * correctly captured it, matching), but the LOCAL config file has since been * overwritten with different bytes (without a new marker committing) -- * simulates the exact "stale embedded copy" trap design.md warns about. * A correct implementation re-reads the LIVE bytes fresh every call rather * than trusting cfgref_sha (a checkpoint-time reference) -- so this MUST * fence, proving the fresh re-read actually happens. */ static void test_upgrade_materialization_verify_stale_local_bytes_after_materialization_fences(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ucaf61_stale_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/23, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, 1, 8)); repl_set_commit_engine(&c->persist.repl, &c->commit); myconf_t *mc = t43_attach_myconf(c, tmp); size_t blen = 0; uint8_t *bytes = t43_make_config_blob(&blen); uint8_t sha[32]; SHA256(bytes, blen, sha); uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); } TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha)); free(bytes); t43_commit_marker(c, tag, sha); TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_verify(c)); TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c)); /* The local config file is overwritten with DIFFERENT bytes, without any * new marker committing (simulates local corruption/drift, or a config * push that lands out of order) -- config_committed_marker still * returns the SAME marker/sha as before. */ size_t blen2 = 0; uint8_t *bytes2 = t43_make_config_blob(&blen2); /* t43_make_config_blob always produces byte-identical content -- mutate * a trailing byte so the "overwritten" bytes genuinely differ. */ bytes2[blen2 - 1] ^= 0xFF; uint8_t sha2[32]; SHA256(bytes2, blen2, sha2); TEST_ASSERT_FALSE(memcmp(sha, sha2, 32) == 0); TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes2, blen2, sha2)); free(bytes2); /* cfgref_sha (captured at checkpoint time) still shows the OLD matching * value -- a naive implementation trusting it would wrongly stay clear. * The correct implementation re-reads the file fresh and fences. */ TEST_ASSERT_EQUAL_HEX64(tag, c->persist.cfgref_tag); TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_verify(c)); TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); t43_free_myconf(c); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_upgrade_materialization_verify_stale_local_bytes_after_materialization_fences: PASS\n"); } /* (4.3-C) FENCE IS ENFORCED by a production reader: while the config serve-fence * is set, ha_dispatch_gate refuses config-dependent state-changing operations on * the PRIMARY (loud non-serving, member stays alive) and a fenced member is * non-promotable (ha_config_promotion_fenced). Read-only/replication ops still * pass so the bytes can arrive out-of-band. Once correct bytes arrive and * re-verify clears the fence, the member serves again. Regression for Finding C. */ static void test_serve_fence_enforced_by_dispatch_gate_and_promotion(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr43_fence_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/9, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, 1, 8)); repl_set_commit_engine(&c->persist.repl, &c->commit); myconf_t *mc = t43_attach_myconf(c, tmp); /* The committed config (blob TARGET) whose bytes have not yet arrived. */ size_t tlen = 0; uint8_t *tbytes = t43_make_config_blob(&tlen); uint8_t tsha[32]; SHA256(tbytes, tlen, tsha); uint64_t ttag; { uint64_t t; memcpy(&t, tsha, 8); ttag = be64toh(t); } t43_commit_marker(c, ttag, tsha); /* Stale local bytes (blob A) that do NOT match the committed marker. */ size_t alen = 0; uint8_t *abytes = t43_make_config_blob(&alen); abytes[alen - 1] ^= 0x33; uint8_t asha[32]; SHA256(abytes, alen, asha); TEST_ASSERT_FALSE(memcmp(asha, tsha, 32) == 0); TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, abytes, alen, asha)); free(abytes); /* Mismatch → fenced. */ TEST_ASSERT_EQUAL_INT(-EAGAIN, config_activate_committed(c)); TEST_ASSERT_EQUAL_INT(1, config_serve_fenced(c)); /* PRODUCTION READER: a fenced PRIMARY refuses config-dependent state-changing * ops (loud non-serving) but stays alive — the gate returns an error, not a * crash/exit. */ TEST_ASSERT_TRUE(ha_dispatch_gate(&c->ha, OPCode_jobsub) < 0); /* Replication / read-only ops still pass so the correct bytes can arrive. */ TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&c->ha, OPCode_replConfigPush)); TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&c->ha, OPCode_get)); /* Non-promotable while fenced. */ TEST_ASSERT_EQUAL_INT(1, ha_config_promotion_fenced(&c->ha)); /* Correct bytes arrive out-of-band (saved as the push handler would). */ TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, tbytes, tlen, tsha)); free(tbytes); /* Re-verify clears the fence → member serves and is promotable again. */ TEST_ASSERT_EQUAL_INT(0, config_activate_committed(c)); TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c)); TEST_ASSERT_EQUAL_HEX64(ttag, mc->config_version_tag); /* config-upgrade-snapshot task 6.2 remediation: t43_commit_marker above * drives the TARGET marker through the REAL apply path, which (as of * the 6.2 fix) makes apply_committed_entry's WAL_CONFIG_MARKER case * (persist.c) unconditionally re-evaluate the separate, independent * upgrade_materialization_fence too -- not just config_serve_fence, * which is all this test is otherwise about. At that moment the TARGET * bytes had not arrived locally yet (this test's own deliberate setup, * to exercise config_serve_fence), so that checkpoint's * scan_latest_committed_config_ref already stamped cfgref_{tag,sha} * from the marker itself (materialized-reference present), but the * live-bytes hash-check genuinely failed (no local bytes yet) -- * correctly setting upgrade_materialization_fence too, per Req 3.2. * That is orthogonal, correct behavior, not a bug in either fence, but * this test never previously accounted for it (the fence was * unreachable before the 6.2 fix). Re-verify it here now that the * correct bytes are locally present, mirroring config_activate_ * committed's own re-verify just above for config_serve_fence, so the * remaining assertions below exercise only the fence this test is * actually about. */ TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_verify(c)); TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c)); TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&c->ha, OPCode_jobsub)); TEST_ASSERT_EQUAL_INT(0, ha_config_promotion_fenced(&c->ha)); t43_free_myconf(c); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_serve_fence_enforced_by_dispatch_gate_and_promotion: PASS\n"); } /* Append a WAL_CONFIG_MARKER for `sha`/`tag` and flush it to the WAL, but do NOT * advance the committed prefix past it — returns its LSN. Models a STANDBY whose * local WAL tip runs ahead of its committed_lsn: on a standby committed_lsn is * driven by the primary's commit-advance (commit_set_raft_commit_index / the * library), NOT by local durability, so a flushed-but-not-yet-committed marker is * the real push-before-commit state. The caller pins committed_lsn back to the * prior value (see the test) so config_committed_marker still derives the OLDER * marker until the NEW one is explicitly committed. */ static uint64_t t43_append_marker_uncommitted(cluster_t *c, uint64_t tag, const uint8_t sha[32]) { uint64_t mlsn = 0; TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &mlsn)); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); /* config-upgrade-snapshot task 5 finding: config_append_marker's returned * lsn (via ha_primary_log_append -> raft_log_append's own s->log_len + 1) * is only reliable when EVERY prior WAL entry on this node went through * that same raft-tracked append path. A marker-triggered checkpoint (this * spec) writes its own local-only, never-replicated WAL_CHECKPOINT entry * directly (persist_write_checkpoint_marker), which advances * persist_t's current_lsn WITHOUT a corresponding raft_log_append call -- * this was already possible before this spec (the periodic WAL-threshold * checkpoint trigger does the same), but this spec's marker-triggered * checkpoint makes it the COMMON case for any second marker appended * after an earlier one's checkpoint has already run, exactly this * fixture's own scenario. The returned mlsn can then under-count the * entry's TRUE physical position. This does not corrupt commit/durability * tracking (commit_advance_locked's majority_candidate_locked is driven * entirely by persist_t's own durable_lsn/current_lsn, never by * s->log_len -- see commit.c) but it DOES make the returned value * unreliable for this test's own strict numeric-ordering comparisons. * Read the TRUE physical lsn directly instead, exactly as production * code that needs precise positioning (e.g. persist.c's apply seam, * which receives lsn from the WAL entry's own on-disk header, never from * this return value) already effectively does. */ return c->persist.current_lsn - 1; } /* (4.3-D / R7.9) COMMIT-ADVANCE RETRY clears a push-before-commit fence: the * config MARKER (WAL) and config BYTES (out-of-band push) arrive on INDEPENDENT * transports. On a 2nd upgrade the NEW bytes are saved BEFORE the NEW marker * enters the committed prefix, so config_committed_marker still returns the OLDER * marker → the freshly-saved NEW bytes hash-MISMATCH it → the member is serve- * FENCED. The NEW marker commits later via the WAL path, but on_commit_advanced * cannot re-activate (no disk I/O on the iomgr ACK thread) and the primary's * config-behind re-push net does not fire (the push already advanced the advertised * tag) — WITHOUT the commit-advance retry trigger the standby would stay * PERMANENTLY fenced. This test reproduces that ordering and asserts the retry * trigger (config_activate_retry_fn — the SAME off-thread entry the production * scheduler enqueues) CLEARS the fence and activates the NEW config once the NEW * marker is committed. Regression for Finding D. * * The single-member commit engine advances committed_lsn to the durable WAL tip, * so to hold the NEW marker out of the committed prefix this test PINS * committed_lsn at the OLD marker's LSN (modeling a standby whose committed prefix * lags its local WAL tip), then advances it past the NEW marker to model the NEW * marker committing via the WAL path. */ static void test_commit_advance_retry_clears_push_before_commit_fence(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr43_retry_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/10, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, 1, 8)); repl_set_commit_engine(&c->persist.repl, &c->commit); myconf_t *mc = t43_attach_myconf(c, tmp); /* ── Upgrade 1 (OLD): commit OLD marker + save OLD bytes + activate. The * member is UN-fenced and serving the OLD config. ───────────────────────── */ size_t blenO = 0; uint8_t *bytesO = t43_make_config_blob(&blenO); uint8_t shaO[32]; SHA256(bytesO, blenO, shaO); uint64_t tagO; { uint64_t t; memcpy(&t, shaO, 8); tagO = be64toh(t); } TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytesO, blenO, shaO)); free(bytesO); t43_commit_marker(c, tagO, shaO); TEST_ASSERT_EQUAL_INT(0, config_activate_committed(c)); TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c)); TEST_ASSERT_EQUAL_HEX64(tagO, mc->config_version_tag); /* The OLD marker is the committed prefix boundary the standby currently sees. */ uint64_t committed_at_old = atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire); /* ── Upgrade 2 (NEW): the BYTES arrive (and are saved) BEFORE the NEW marker * enters the committed prefix — the push-before-commit ordering. ────────── */ size_t blenN = 0; uint8_t *bytesN = t43_make_config_blob(&blenN); bytesN[blenN - 1] ^= 0xA5; /* perturb → distinct sha */ uint8_t shaN[32]; SHA256(bytesN, blenN, shaN); uint64_t tagN; { uint64_t t; memcpy(&t, shaN, 8); tagN = be64toh(t); } TEST_ASSERT_FALSE(memcmp(shaO, shaN, 32) == 0); /* The NEW marker is APPENDED + flushed to the WAL (local tip advances). * bug-1469 fix interaction: flushing makes durable_lsn advance, and this * single-member fixture's commit engine (majority=1) reacts to ANY * durable advance by racing committed_lsn forward on its OWN background * fsync-worker thread (commit_on_durable_advance) -- which, since * apply_committed_entry's WAL_CONFIG_MARKER case now ALSO activates * config (bug-1469), could apply-and-activate this NEW marker for real * before this test's own thread gets a chance to pin committed_lsn back * down below it. PIN IMMEDIATELY, right here, before any other work * (previously this pin lived after conf_save_config below, leaving a * multi-syscall window — several real fsyncs — for that background race * to win against a single-member fixture's over-eager auto-commit, * something a real multi-node standby never does since its * committed_lsn is solely driven by the primary's commit-advance, never * local durability). */ uint64_t new_mlsn = t43_append_marker_uncommitted(c, tagN, shaN); TEST_ASSERT_TRUE(new_mlsn > committed_at_old); atomic_store_explicit(&c->persist.committed_lsn, committed_at_old, memory_order_release); /* The OLD config is currently LIVE (activated in upgrade 1). Capture the live * decoded-config pointer as the activation witness: load_primary_config frees * and reallocates mc->cluster_config, so a CHANGED pointer proves a (re)load * happened. conf_save_config (below) only writes bytes + bumps the advertised * config_version_tag — it does NOT decode/activate — so it must NOT change this * pointer; only a successful config_activate_committed does. */ cluster_config_t *live_before = mc->cluster_config; TEST_ASSERT_NOT_NULL(live_before); /* The NEW bytes are saved out-of-band (exactly as handle_repl_config_push * does). Note conf_save_config in this single-member fixture re-evaluates the * commit barrier, which would advance committed_lsn to the durable WAL tip, and * it bumps the advertised config_version_tag to tagN (this is the very reason * the primary's config-behind re-push net does not re-fire — Finding D); we * already pinned committed_lsn back below the NEW marker immediately above (to * model a standby whose committed prefix, driven by the primary's commit- * advance, not local durability, has NOT yet advanced past the NEW marker) — * re-pin again here for defense-in-depth against the same race. */ TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytesN, blenN, shaN)); atomic_store_explicit(&c->persist.committed_lsn, committed_at_old, memory_order_release); /* Activate now: config_committed_marker still returns the OLDER marker (tagO), * so the freshly-saved NEW bytes MISMATCH it → serve-FENCED, NEW config NOT * ACTIVATED. The live decoded config is UNCHANGED (still the OLD one) — the * mismatch gate never loaded the NEW bytes. This is the permanent-fence state * the retry trigger must rescue. */ TEST_ASSERT_EQUAL_INT(-EAGAIN, config_activate_committed(c)); TEST_ASSERT_EQUAL_INT(1, config_serve_fenced(c)); TEST_ASSERT_TRUE(mc->cluster_config == live_before); /* NEW not activated */ /* ── The NEW marker COMMITS later via the WAL path (the standby's committed * prefix advances past it, as the primary's commit-advance heartbeat would * drive). config_committed_marker now derives the NEW marker. ───────────── */ atomic_store_explicit(&c->persist.committed_lsn, new_mlsn, memory_order_release); /* DRIVE THE RETRY TRIGGER: the SAME off-thread activation entry the production * commit-advance path schedules (ha_cb_commit_advanced_stub → * config_schedule_activate_retry → delaymgr → config_activate_retry_fn). In * production the delaymgr worker runs this OFF the iomgr ACK thread; here we * invoke the entry directly (the fixture's delaymgr is not started). Now * config_committed_marker returns the NEW marker and the saved NEW bytes * hash-verify → the fence CLEARS and the NEW config ACTIVATES (cluster_config * is reloaded — a fresh pointer). */ TEST_ASSERT_EQUAL_INT(0, config_activate_retry_fn(c)); TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c)); /* fence CLEARED */ TEST_ASSERT_EQUAL_HEX64(tagN, mc->config_version_tag); /* NEW now live */ TEST_ASSERT_NOT_NULL(mc->cluster_config); TEST_ASSERT_TRUE(mc->cluster_config != live_before); /* (re)loaded NEW */ free(bytesN); t43_free_myconf(c); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_commit_advance_retry_clears_push_before_commit_fence: PASS\n"); } /* ── config-upgrade-snapshot task 6.2: Activation Fence gate composition + * self-triggering retry (upgrade_materialization_fenced / ha_dispatch_gate / * raft_cb_become_leader / schedule_upgrade_materialization_retry, ha.c) ──── * * These tests reuse this file's own t42_/t43_ fixture helpers and mirror * the existing (4.3-C/D) config_serve_fence gate-composition/retry tests * above as the direct template, for the new independent * upgrade_materialization_fence instead. */ /* raft_cb_become_leader is file-static; drive it through the LIVE-cluster * XCE_TESTING seam (fix-promotion-standby-registration) so this test proves * the REAL election-time promotion decision refuses while fenced, not just a * proxy accessor read. */ extern void raft_cb_become_leader_with_cluster_for_test(cluster_t *c, uint64_t term); /* ha-term-safety task 2.4: drive the WIRED on_commit_advanced callback * (ha_cb_commit_advanced_stub) so the wiring test below exercises the real * commit-advance trigger, not a hand-rolled call. */ extern void ha_cb_commit_advanced_for_test(ha_state_t *ha, uint64_t commit_index); /* (6.2-a) UPGRADE MATERIALIZATION FENCE IS ENFORCED by two production * readers: (1) ha_dispatch_gate refuses a config-dependent state-changing * operation on the PRIMARY while fenced (loud non-serving, member stays * alive) but keeps replication/read-only ops flowing so materialization can * proceed and re-verify; (2) a REAL election-time promotion attempt * (raft_cb_become_leader_with_cluster_for_test — the actual gate, not a proxy * accessor) refuses to become CTRL_PRIMARY while this member's own * materialization for the current committed marker is incomplete. Regression * for the dispatch-gate/promotion-gate composition (Req 3.2, 3.3, 4.3, 5.5). */ static void test_upgrade_materialization_fence_enforced_by_dispatch_gate_and_promotion(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr62_fence_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/40, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, 1, 8)); repl_set_commit_engine(&c->persist.repl, &c->commit); myconf_t *mc = t43_attach_myconf(c, tmp); /* Commit a marker but never checkpoint against it -- this member's own * materialization is genuinely incomplete for it (identical setup to * test_upgrade_materialization_verify_matching_materialization_clears_fence's * sibling incomplete-materialization test, just without the -EBUSY * choreography -- a fresh marker with no checkpoint yet run is already * incomplete by construction). */ size_t blen = 0; uint8_t *bytes = t43_make_config_blob(&blen); uint8_t sha[32]; SHA256(bytes, blen, sha); uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); } TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha)); free(bytes); /* Force materialization to stay incomplete for THIS marker: hold the * shared checkpoint_in_progress CAS open across the commit (the same * mechanism test_upgrade_materialization_verify_incomplete_materialization_ * fences uses), so the marker-apply checkpoint hits -EBUSY and cfgref_* * never advances to it. */ TEST_ASSERT_EQUAL_INT( 0, atomic_exchange_explicit(&c->persist.checkpoint_in_progress, 1, memory_order_acq_rel)); uint64_t mlsn = 0; TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &mlsn)); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); TEST_ASSERT_EQUAL_INT(0, commit_submit(&c->commit, mlsn, COMMIT_MODE_WAIT, 1000)); uint64_t deadline_iters = 200; while (deadline_iters-- > 0 && atomic_load_explicit(&c->persist.last_applied_lsn, memory_order_acquire) < mlsn) { struct timespec ts = { .tv_sec = 0, .tv_nsec = 10 * 1000 * 1000 }; nanosleep(&ts, NULL); } atomic_store_explicit(&c->persist.checkpoint_in_progress, 0, memory_order_release); TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_verify(c)); TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); /* PRODUCTION READER 1: ha_dispatch_gate refuses a state-changing op on * the fenced PRIMARY but stays alive (error, not crash/exit). */ TEST_ASSERT_TRUE(ha_dispatch_gate(&c->ha, OPCode_jobsub) < 0); /* Replication / read-only ops still pass. */ TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&c->ha, OPCode_heartbeat)); TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&c->ha, OPCode_get)); /* PRODUCTION READER 2: a real election-time promotion attempt refuses to * become CTRL_PRIMARY while fenced. ha_primary_log_append above only * required c->ha.raft.role == RAFT_LEADER (set by t42_make_leader), not * ha->role == CTRL_PRIMARY, so resetting ha->role to CTRL_ELECTING here * to observe a genuine promotion decision does not disturb the marker * already committed above. */ c->ha.phase2_done = 1; /* skip cluster_init_phase2 -- unrelated first-boot path */ atomic_store_explicit(&c->ha.role, (int32_t)CTRL_ELECTING, memory_order_release); raft_cb_become_leader_with_cluster_for_test(c, /*term*/40); TEST_ASSERT_NOT_EQUAL_MESSAGE( (int32_t)CTRL_PRIMARY, atomic_load_explicit(&c->ha.role, memory_order_acquire), "config-upgrade-snapshot task 6.2 (Req 3.2/3.3/4.3/5.5): a member " "whose own upgrade-snapshot materialization is incomplete must NOT " "be promoted to CTRL_PRIMARY"); TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); /* unaffected */ t43_free_myconf(c); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_upgrade_materialization_fence_enforced_by_dispatch_gate_and_promotion: PASS\n"); } /* (6.2-b) schedule_upgrade_materialization_retry is a no-op when NOT fenced * -- steady state schedules nothing (no busy-spin), mirroring * config_schedule_activate_retry's own documented no-op behavior. */ static void test_schedule_upgrade_materialization_retry_noop_when_not_fenced(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr62_retry_noop_unfenced_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/41, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c)); /* fixture default: unfenced */ TEST_ASSERT_FALSE(c->delaymgr.running); /* fixture never starts delaymgr */ TEST_ASSERT_EQUAL_INT(0, schedule_upgrade_materialization_retry(c)); t42_teardown(c); printf("test_schedule_upgrade_materialization_retry_noop_when_not_fenced: PASS\n"); } /* (6.2-c) schedule_upgrade_materialization_retry is a no-op when the * delaymgr is not running, even while fenced -- the unit-test fixture path * (mirrors config_schedule_activate_retry's own documented delaymgr-not- * running no-op; test_commit_advance_retry_clears_push_before_commit_fence * relies on this same fixture property to drive its retry entry directly). */ static void test_schedule_upgrade_materialization_retry_noop_when_delaymgr_not_running(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr62_retry_noop_nodmgr_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/42, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, 1, 8)); repl_set_commit_engine(&c->persist.repl, &c->commit); myconf_t *mc = t43_attach_myconf(c, tmp); size_t blen = 0; uint8_t *bytes = t43_make_config_blob(&blen); uint8_t sha[32]; SHA256(bytes, blen, sha); uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); } TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha)); free(bytes); t43_commit_marker(c, tag, sha); /* Overwrite the local bytes without a new marker committing -- the same * "stale local bytes" fence trigger * test_upgrade_materialization_verify_stale_local_bytes_after_ * materialization_fences uses -- so the fence is genuinely SET without * needing the -EBUSY checkpoint choreography. */ size_t blen2 = 0; uint8_t *bytes2 = t43_make_config_blob(&blen2); bytes2[blen2 - 1] ^= 0xFF; uint8_t sha2[32]; SHA256(bytes2, blen2, sha2); TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes2, blen2, sha2)); free(bytes2); TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_verify(c)); TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); TEST_ASSERT_FALSE(c->delaymgr.running); /* delaymgr not running -- no-op, returns 0, does not crash/leak a task. */ TEST_ASSERT_EQUAL_INT(0, schedule_upgrade_materialization_retry(c)); TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); /* still fenced -- untouched */ t43_free_myconf(c); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_schedule_upgrade_materialization_retry_noop_when_delaymgr_not_running: PASS\n"); } /* (6.2-d) upgrade_materialization_retry_fn -- the same off-thread entry * point schedule_upgrade_materialization_retry's trampoline would enqueue in * production -- stays fenced (-EAGAIN) while the genuine mismatch persists, * then CLEARS the fence once the live bytes are corrected to match the * committed marker, without a new marker needing to commit. Mirrors * config_activate_retry_fn's own direct-drive regression * (test_commit_advance_retry_clears_push_before_commit_fence). */ static void test_upgrade_materialization_retry_fn_recovers_from_stale_bytes_fence(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr62_retry_fn_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/43, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, 1, 8)); repl_set_commit_engine(&c->persist.repl, &c->commit); myconf_t *mc = t43_attach_myconf(c, tmp); size_t blen = 0; uint8_t *bytes = t43_make_config_blob(&blen); uint8_t sha[32]; SHA256(bytes, blen, sha); uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); } TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha)); t43_commit_marker(c, tag, sha); TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_verify(c)); TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c)); /* Local bytes are overwritten with DIFFERENT content, without a new * marker committing -- cfgref_* still shows the old matching reference, * but a fresh live re-read must fence (same trap 6.1-d proves for * upgrade_materialization_verify itself). */ size_t blenX = 0; uint8_t *bytesX = t43_make_config_blob(&blenX); bytesX[blenX - 1] ^= 0xA5; uint8_t shaX[32]; SHA256(bytesX, blenX, shaX); TEST_ASSERT_FALSE(memcmp(sha, shaX, 32) == 0); TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytesX, blenX, shaX)); free(bytesX); TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_verify(c)); TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); /* Drive the retry entry directly while the mismatch is still genuine: * stays fenced, does not crash/toggle incorrectly. */ TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_retry_fn(c)); TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); /* Restore the CORRECT bytes (matching the still-current committed * marker) without any new marker commit -- models the out-of-band * config-push bytes finally arriving/correcting. */ TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha)); free(bytes); /* Drive the retry entry again: re-verify now succeeds -- fence CLEARS. */ TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_retry_fn(c)); TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c)); t43_free_myconf(c); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_upgrade_materialization_retry_fn_recovers_from_stale_bytes_fence: PASS\n"); } /* (6.2-e) WIRING: the commit-advance path (ha_cb_commit_advanced_stub, the * WIRED on_commit_advanced callback) consults upgrade_materialization_fenced * and calls schedule_upgrade_materialization_retry when fenced -- driven * through the REAL wired callback (ha_cb_commit_advanced_for_test), not a * hand-rolled call, so this proves the actual call-site wiring executes * without crashing/corrupting state. schedule_upgrade_materialization_retry * itself is a no-op in this fixture (delaymgr not running -- proven in * isolation by 6.2-c above), so the fence is expected to remain SET * afterward; this test's contribution is exercising the NEW branch at the * real call site itself, not re-proving the no-op contract. */ static void test_commit_advance_wiring_calls_upgrade_materialization_schedule_retry(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr62_wiring_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/44, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, 1, 8)); repl_set_commit_engine(&c->persist.repl, &c->commit); myconf_t *mc = t43_attach_myconf(c, tmp); size_t blen = 0; uint8_t *bytes = t43_make_config_blob(&blen); uint8_t sha[32]; SHA256(bytes, blen, sha); uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); } TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha)); free(bytes); t43_commit_marker(c, tag, sha); TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_verify(c)); TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c)); /* Overwrite local bytes (stale-bytes trigger, same as 6.2-c) to fence * without a new marker. */ size_t blen2 = 0; uint8_t *bytes2 = t43_make_config_blob(&blen2); bytes2[blen2 - 1] ^= 0xFF; uint8_t sha2[32]; SHA256(bytes2, blen2, sha2); TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes2, blen2, sha2)); free(bytes2); TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_verify(c)); TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); uint64_t committed_lsn = atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire); /* Drive the REAL wired commit-advance callback -- must not crash, must * not corrupt the fence state (schedule_upgrade_materialization_retry's * own no-op-when-delaymgr-not-running contract keeps it SET). */ ha_cb_commit_advanced_for_test(&c->ha, committed_lsn); TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); t43_free_myconf(c); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_commit_advance_wiring_calls_upgrade_materialization_schedule_retry: PASS\n"); } /* ── bug-1477: ha_phase2_retry_fn / ha_schedule_phase2_retry ───────────────── * * A node that takes myapp_run's skip_election passive path (bug-849 -- * "leader already known after the startup stagger") makes exactly ONE * cluster_init_phase2 attempt. Before this fix, EVERY call site (myapp.c's * skip_election block, raft_cb_become_leader, raft_cb_become_follower, * ha_elect_evaluate_and_signal's two branches) latched ha->phase2_done = 1 * UNCONDITIONALLY, regardless of cluster_init_phase2's return value. If that * one attempt raced config availability and failed, the node was left * permanently stuck: commit_engine_init never ran, so apply_committed_entry * never applied ANY committed WAL entry for the rest of that process's * lifetime -- and since a skip_election node never again contests an * election, none of the other four call sites, which only fire via THIS * node's own raft-library state transitions, ever ran again either. * * ha_phase2_retry_fn is the idempotent retry entry (mirrors * upgrade_materialization_retry_fn's shape); ha_schedule_phase2_retry is its * delaymgr-scheduled, self-limiting trigger (mirrors * schedule_upgrade_materialization_retry's no-op guards). */ /* (bug-1477-a) ha_phase2_retry_fn is a no-op once phase2_done is already * set -- does not re-invoke the hook, returns 0. */ static void test_ha_phase2_retry_fn_noop_when_already_done(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr1477_retry_noop_done_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); c->ha.phase2_done = 1; TEST_ASSERT_EQUAL_INT(0, ha_phase2_retry_fn(c)); TEST_ASSERT_EQUAL_INT(1, c->ha.phase2_done); t42_teardown(c); printf("test_ha_phase2_retry_fn_noop_when_already_done: PASS\n"); } static _Atomic int g_cr1477_hook_calls; static _Atomic int g_cr1477_hook_fail_first_n; /* Fails the first g_cr1477_hook_fail_first_n calls (rc=-EAGAIN), succeeds * (rc=0) thereafter -- models cluster_init_phase2 racing config * availability and then succeeding once it arrives. */ static int cr1477_flaky_phase2_hook(void *c) { (void)c; int n = atomic_fetch_add(&g_cr1477_hook_calls, 1); return (n < atomic_load(&g_cr1477_hook_fail_first_n)) ? -EAGAIN : 0; } /* (bug-1477-b) THE CORE REGRESSION: a failed cluster_init_phase2 attempt * must NOT latch phase2_done -- RED under the pre-fix code (which set * phase2_done = 1 unconditionally right after calling the hook, regardless * of its return value; this assertion would fail immediately). A second * call, once the underlying condition clears, then succeeds and latches. */ static void test_ha_phase2_retry_fn_does_not_latch_on_failure_and_succeeds_on_retry(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr1477_retry_fn_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); TEST_ASSERT_EQUAL_INT(0, c->ha.phase2_done); /* fixture default: not done */ atomic_store(&g_cr1477_hook_calls, 0); atomic_store(&g_cr1477_hook_fail_first_n, 1); ha_phase2_hook = cr1477_flaky_phase2_hook; /* First attempt: hook fails -- phase2_done must stay 0 (bug-1477's own * regression assertion). */ TEST_ASSERT_EQUAL_INT(-EAGAIN, ha_phase2_retry_fn(c)); TEST_ASSERT_EQUAL_INT_MESSAGE(0, c->ha.phase2_done, "a failed cluster_init_phase2 attempt must not latch phase2_done " "(bug-1477 -- the pre-fix code did this unconditionally, " "permanently stranding the member)"); /* Second attempt: hook now succeeds -- phase2_done latches. */ TEST_ASSERT_EQUAL_INT(0, ha_phase2_retry_fn(c)); TEST_ASSERT_EQUAL_INT(1, c->ha.phase2_done); /* Idempotent: a third call is a pure no-op (hook not re-invoked). */ int calls_before = atomic_load(&g_cr1477_hook_calls); TEST_ASSERT_EQUAL_INT(0, ha_phase2_retry_fn(c)); TEST_ASSERT_EQUAL_INT(calls_before, atomic_load(&g_cr1477_hook_calls)); ha_phase2_hook = NULL; /* restore -- shared process-global test seam */ t42_teardown(c); printf("test_ha_phase2_retry_fn_does_not_latch_on_failure_and_succeeds_on_retry: PASS\n"); } /* (bug-1477-c) ha_schedule_phase2_retry is a no-op when phase2_done is * already set -- steady state schedules nothing (no busy-spin), mirroring * schedule_upgrade_materialization_retry's own documented no-op behavior. */ static void test_schedule_phase2_retry_noop_when_already_done(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr1477_sched_noop_done_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); c->ha.phase2_done = 1; TEST_ASSERT_FALSE(c->delaymgr.running); /* fixture never starts delaymgr */ TEST_ASSERT_EQUAL_INT(0, ha_schedule_phase2_retry(c)); t42_teardown(c); printf("test_schedule_phase2_retry_noop_when_already_done: PASS\n"); } /* (bug-1477-d) ha_schedule_phase2_retry is a no-op when the delaymgr is not * running, even while phase2_done is still 0 -- the unit-test fixture path * (mirrors config_schedule_activate_retry / schedule_upgrade_materialization_ * retry's own documented delaymgr-not-running no-op). */ static void test_schedule_phase2_retry_noop_when_delaymgr_not_running(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr1477_sched_noop_nodmgr_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); TEST_ASSERT_EQUAL_INT(0, c->ha.phase2_done); TEST_ASSERT_FALSE(c->delaymgr.running); TEST_ASSERT_EQUAL_INT(0, ha_schedule_phase2_retry(c)); TEST_ASSERT_EQUAL_INT(0, c->ha.phase2_done); /* untouched -- no-op */ t42_teardown(c); printf("test_schedule_phase2_retry_noop_when_delaymgr_not_running: PASS\n"); } /* ── config-upgrade-snapshot task 6.2 remediation: Activation Fence set by a * REAL production trigger, not a direct test call ────────────────────────── * * Every test above (6.2-a..e, and 6.1's own 4 tests earlier in this file) * reaches the fenced state by calling upgrade_materialization_verify() * directly in the test body. That proves the fence's own logic is correct, * but NOT that a running production controller can ever reach the fenced * state in the first place: every PRODUCTION call path to * upgrade_materialization_verify() -- upgrade_materialization_retry_fn, and * both of ITS callers (schedule_upgrade_materialization_retry and the * commit-advance hook in ha_cb_commit_advanced_stub) -- is itself gated on * upgrade_materialization_fenced() already being true. Starting from the * fence's zero-initialized (unfenced) default, that is a closed loop with * no entry point: nothing in production could ever set the fence for the * first time, which made the dispatch-gate/promotion-gate checks in 6.2-a * permanently dead code against a real controller. * * The actual fix (persist.c's apply_committed_entry, WAL_CONFIG_MARKER * case) calls upgrade_materialization_verify() UNCONDITIONALLY right after * persist_checkpoint_marker_triggered() returns, regardless of that call's * own rc -- checkpoint completion (or a self-resolving skip) is the * design-intended re-evaluation trigger (design.md's Activation Fence * component). This test reaches the fenced state purely through that real * seam -- config_append_marker + commit_submit + wait-for-applied, with * persist_checkpoint_marker_triggered() forced to miss via the exact * checkpoint_in_progress technique test_upgrade_materialization_verify_ * incomplete_materialization_fences (6.1-c) and test_upgrade_materialization * _fence_enforced_by_dispatch_gate_and_promotion (6.2-a) use -- and NEVER * calls upgrade_materialization_verify() itself. */ static void test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr62_real_trigger_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/45, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, 1, 8)); repl_set_commit_engine(&c->persist.repl, &c->commit); myconf_t *mc = t43_attach_myconf(c, tmp); /* Fixture default: unfenced (fresh cluster_t, zero-initialized atomic) * -- the exact "closed loop, no entry point" starting condition the * rejection identified. This test body never calls * upgrade_materialization_verify() anywhere below -- the fence must be * set purely by the real apply-time trigger. */ TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c)); size_t blen = 0; uint8_t *bytes = t43_make_config_blob(&blen); uint8_t sha[32]; SHA256(bytes, blen, sha); uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); } TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha)); free(bytes); /* Force materialization to genuinely miss for THIS marker: hold the * shared checkpoint_in_progress CAS open across the commit, so * persist_checkpoint_marker_triggered() (called from * apply_committed_entry's WAL_CONFIG_MARKER case) hits the real -EBUSY * path (Req 4.4) -- self-resolving, not a failure, so apply itself * still succeeds and last_applied_lsn still advances. */ TEST_ASSERT_EQUAL_INT( 0, atomic_exchange_explicit(&c->persist.checkpoint_in_progress, 1, memory_order_acq_rel)); uint64_t mlsn = 0; TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &mlsn)); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); TEST_ASSERT_EQUAL_INT(0, commit_submit(&c->commit, mlsn, COMMIT_MODE_WAIT, 1000)); /* Wait for the REAL apply_committed_entry() dispatch to finish for this * marker. persist_apply_pump_run advances last_applied_lsn only AFTER * that dispatch call returns (see apply_committed_entry's own * WAL_CONFIG_MARKER case doc comment), so by the time * t42_wait_applied observes last_applied_lsn >= mlsn, the new * unconditional upgrade_materialization_verify(c) call at the end of * that case has already run synchronously on this same call stack. */ t42_wait_applied(&c->persist, mlsn, 2000); TEST_ASSERT_TRUE(persist_last_applied_lsn(&c->persist) >= mlsn); /* Release the simulated in-flight checkpoint now that the marker's own * apply (and its -EBUSY'd checkpoint attempt) has already happened. */ atomic_store_explicit(&c->persist.checkpoint_in_progress, 0, memory_order_release); /* THE ASSERTION THAT PROVES THE FIX: the fence reads SET, and this test * body never called upgrade_materialization_verify() itself -- the only * possible source is persist.c's new unconditional post-checkpoint * call. Before the fix, this read 0 forever: every production path to * upgrade_materialization_verify() was gated on the fence already being * 1, so nothing could ever set it for the first time. */ TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); /* Confirm this is a genuine incomplete-materialization fence (cfgref_* * never advanced to this marker, because the checkpoint that would have * stamped it was skipped) -- not a coincidental pass. */ TEST_ASSERT_FALSE(c->persist.cfgref_present && c->persist.cfgref_tag == tag && memcmp(c->persist.cfgref_sha, sha, 32) == 0); /* Close the loop end-to-end: the two production READERS of this fence * (ha_dispatch_gate and the promotion gate) both observe it too, using * the same real seams 6.2-a exercises -- reusing existing test * infrastructure, not re-deriving it. */ TEST_ASSERT_TRUE(ha_dispatch_gate(&c->ha, OPCode_jobsub) < 0); TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&c->ha, OPCode_heartbeat)); c->ha.phase2_done = 1; /* skip cluster_init_phase2 -- unrelated first-boot path */ atomic_store_explicit(&c->ha.role, (int32_t)CTRL_ELECTING, memory_order_release); raft_cb_become_leader_with_cluster_for_test(c, /*term*/45); TEST_ASSERT_NOT_EQUAL_MESSAGE( (int32_t)CTRL_PRIMARY, atomic_load_explicit(&c->ha.role, memory_order_acquire), "config-upgrade-snapshot task 6.2 remediation: a fence set by the " "REAL apply-time trigger must refuse promotion, same as a " "directly-set fence does"); t43_free_myconf(c); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss: PASS\n"); } /* ── bug-1469 regression: WAL_CONFIG_MARKER apply must ACTIVATE config on * the ORIGINATING member, not just receiving standbys ────────────────────── * * Live 3-node chaotic-kill soak (config-upgrade-snapshot task 8.6, * .wolf/buglog.json bug-1469): a config upgrade's WAL_CONFIG_MARKER commits * and even checkpoints successfully on the ORIGINATING member (the primary * that submitted the upgrade via do_upgrade_config_with_bytes), but * apply_committed_entry's WAL_CONFIG_MARKER case (persist.c) never called * config_activate_committed()/load_primary_config() -- so that member's * c->myconf stayed parsed from the OLD config bytes until some LATER, * independent event (a restart, or an incoming out-of-band * handle_repl_config_push) happened to refresh it. The live soak killed the * originating primary shortly after it committed the upgrade, before any * such later event, freezing a stale c->myconf into its last checkpoint -- * a genuine cross-member materialized-state divergence (ctrl-1's checkpoint * ended up a different byte size than ctrl-2/ctrl-3's, both LSN 11). * config_activate_committed WAS already being called correctly on * RECEIVING standbys, via handle_repl_config_push -- this gap is specific * to the member that originates/commits the marker on itself. */ /* Build a valid ClusterConfig blob distinguishable by config_generation + * clustername (mirrors t43_make_config_blob, parameterized so this test can * commit two genuinely different, independently-decodable configs -- an * "old" pre-upgrade config and a "new" post-upgrade config -- without * relying on a byte-mutation of an opaque capnp blob, which risks * corrupting the encoding; config-upgrade-snapshot's own standing gotcha * notes t43_make_config_blob() returns byte-IDENTICAL content on repeat * calls, so a genuinely different SECOND blob needs its own encode call). */ static uint8_t *t1469_make_config_blob(uint64_t gen, const char *name, size_t *len_out) { cluster_config_t cfg; struct capn cap; struct capn_segment *cs; ClusterConfig_ptr p; uint8_t *buf; size_t sz; memset(&cfg, 0, sizeof(cfg)); cfg.clustername = (char *)name; cfg.config_generation = gen; capn_init_malloc(&cap); cs = capn_root(&cap).seg; encode_ClusterConfig_ptr(cs, &p, &cfg); capn_setp(capn_root(&cap), 0, p.p); sz = (size_t)capn_size(&cap); buf = (uint8_t *)malloc(sz); TEST_ASSERT_NOT_NULL(buf); capn_write_mem(&cap, buf, sz, 0); capn_free(&cap); *len_out = sz; return buf; } /* This test drives the upgrade marker through the REAL production apply * path (config_append_marker + commit_submit + wait-for-applied, exactly * like test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss * above) on a single-member fixture acting as its OWN primary -- i.e. the * ORIGINATING member applying its own committed marker, never receiving an * out-of-band handle_repl_config_push. It NEVER calls * config_activate_committed() for the upgrade marker itself, so the only * way mc->cluster_config can reflect the new config is via * apply_committed_entry's own WAL_CONFIG_MARKER dispatch. * * RED (before the fix): mc->cluster_config stays pinned to the OLD config * (config_generation==1, "t1469-old") after the new marker's real, * successful commit+apply -- proving the marker-triggered dispatch never * activates config on the member that originated it. * GREEN (after the fix): mc->cluster_config reflects the NEW config * (config_generation==2, "t1469-new") immediately after the marker's apply * returns, with no restart and no out-of-band push. */ static void test_wal_config_marker_apply_activates_originating_member_config(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr1469_activate_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t42_make_leader(c, /*term*/69, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, 1, 8)); repl_set_commit_engine(&c->persist.repl, &c->commit); myconf_t *mc = t43_attach_myconf(c, tmp); /* Establish the pre-upgrade steady state: an OLD config already * committed and ACTIVATED via the legitimate, already-tested * config_activate_committed path (mirrors 4.3-a above) -- this member * is already running under config_generation=1 before any upgrade is * ever submitted, exactly like a real cluster member. */ size_t oldlen = 0; uint8_t *oldbytes = t1469_make_config_blob(1, "t1469-old", &oldlen); uint8_t oldsha[32]; SHA256(oldbytes, oldlen, oldsha); uint64_t oldtag; { uint64_t t; memcpy(&t, oldsha, 8); oldtag = be64toh(t); } TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, oldbytes, oldlen, oldsha)); free(oldbytes); t43_commit_marker(c, oldtag, oldsha); TEST_ASSERT_EQUAL_INT(0, config_activate_committed(c)); TEST_ASSERT_NOT_NULL(mc->cluster_config); TEST_ASSERT_EQUAL_UINT64(1, mc->cluster_config->config_generation); /* Now the upgrade: NEW config bytes saved locally exactly as * do_upgrade_config_with_bytes's own step 5 (conf_save_config) does on * the ORIGINATING primary, then a NEW marker committed through the * exact same real path bug-1469's live soak exercised. Deliberately * NEVER calls config_activate_committed() for this marker anywhere * below -- activation must come from apply_committed_entry's own * dispatch, or this assertion fails. */ size_t newlen = 0; uint8_t *newbytes = t1469_make_config_blob(2, "t1469-new", &newlen); uint8_t newsha[32]; SHA256(newbytes, newlen, newsha); uint64_t newtag; { uint64_t t; memcpy(&t, newsha, 8); newtag = be64toh(t); } TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, newbytes, newlen, newsha)); free(newbytes); t43_commit_marker(c, newtag, newsha); /* THE ASSERTION THAT PROVES THE FIX: mc->cluster_config must now * reflect the NEW config -- purely from apply_committed_entry's own * WAL_CONFIG_MARKER dispatch, exactly the production path the * ORIGINATING primary runs on itself. Before the fix this stays pinned * at generation 1 ("t1469-old") forever -- exactly bug-1469's observed * cross-member divergence, reproduced at the unit level. */ TEST_ASSERT_NOT_NULL(mc->cluster_config); TEST_ASSERT_EQUAL_UINT64_MESSAGE(2, mc->cluster_config->config_generation, "bug-1469: apply_committed_entry's WAL_CONFIG_MARKER case never " "activated the new config on the ORIGINATING member -- c->myconf " "stayed pinned to the OLD config"); TEST_ASSERT_EQUAL_STRING("t1469-new", mc->cluster_config->clustername); TEST_ASSERT_EQUAL_HEX64(newtag, mc->config_version_tag); TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c)); t43_free_myconf(c); commit_engine_fini(&c->commit); t42_teardown(c); printf("test_wal_config_marker_apply_activates_originating_member_config: PASS\n"); } /* ── config-upgrade-snapshot task 8.8: a second configuration-change marker * committing before the first's snapshot install completes everywhere must * cleanly SUPERSEDE the first's in-flight materialization -- never leave it * partially applied or conflated with the newer one (Req 4.4, 7.2) ──────── * * design.md ("Overlapping-marker -EBUSY handling", ~line 258, and * "Overlapping upgrades", ~line 413): when marker B's own apply runs while * marker A's forced checkpoint is still physically in flight (holding the * shared checkpoint_in_progress CAS), marker B's own checkpoint call ALSO * hits -EBUSY -- not an error, self-resolving. Content is a deterministic * function of (term, lsn, config-hash), and config_committed_marker always * resolves to the LATEST committed marker within the committed prefix * (scan_latest_committed_marker, config_repl.c:124-215, "later markers * overwrite -- latest wins", line 198). So once checkpointing is unblocked * and a fresh checkpoint attempt runs, it always reflects the latest marker * -- never a stale/partial mix of the two. * * Two independently-built single-member fixtures process the IDENTICAL * entry sequence (same term, same two deterministic config blobs via * t1469_make_config_blob -- byte-identical across calls per its own doc * comment, the same determinism-across-fixtures property task 8.7 * Scenario 2 above already relies on): * * c_control -- marker A commits+applies+checkpoints normally, THEN marker * B commits+applies+checkpoints normally. No racing. * * c_raced -- the shared checkpoint_in_progress CAS is held BEFORE marker A * commits and never released until AFTER both markers have committed and * applied: marker A's own forced checkpoint attempt -EBUSYs and skips * (still "installing"), then marker B commits+applies while A's install is * still outstanding, and marker B's own forced checkpoint attempt ALSO * -EBUSYs and skips. Only after releasing the CAS and running * persist_checkpoint() once directly (the same resolve step * test_marker_apply_busy_checkpoint_handled_gracefully in * test_upgrade_snapshot_apply.c uses) does a checkpoint actually land. * * NOTE on why this test does NOT assert whole-file byte-identity between the * two fixtures' final checkpoints: c_control's marker A checkpoint * physically WRITES a WAL_CHECKPOINT entry (persist_write_checkpoint_marker) * before marker B is ever appended, consuming one physical WAL lsn slot that * c_raced's fixture never consumes at that point (its checkpoint attempt for * A never gets past the checkpoint_in_progress CAS check, so nothing is * physically written for it). Marker B therefore lands at a DIFFERENT * absolute physical lsn in each fixture (one higher in c_control) purely as * a side effect of how many checkpoints happened to physically land before * it -- exactly the "content is a deterministic function of (term, lsn, * config-hash)" property this task's own description calls out (the * checkpoint's own lsn is legitimately part of "content"). write_snapshot's * sentinel header embeds this lsn verbatim at a fixed byte offset * (persist.c: "magic (8) + checkpoint_lsn (8) + ..."), so the two files are * expected to diverge at EXACTLY that one field. The decisive proof is * instead: byte-identical content everywhere else -- the committed-config * reference (cfgref_tag/cfgref_sha) and the entire entity+config-bytes * payload -- which is precisely what would carry any trace of marker A's * incomplete install, if one existed. snapdiff_paths is still used (for the * diagnostic corroboration that the divergence, if any, is exactly and only * that one field), alongside a direct field-aware comparison of everything * from byte 16 onward. */ static void test_second_config_marker_supersedes_first_still_installing_on_member(void) { char tmp_control[256], tmp_raced[256]; strcpy(tmp_control, "/tmp/xce_cr88_control_XXXXXX"); strcpy(tmp_raced, "/tmp/xce_cr88_raced_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp_control)); TEST_ASSERT_NOT_NULL(mkdtemp(tmp_raced)); const uint64_t term = 88; cluster_t *c_control = t42_make_cluster(tmp_control); cluster_t *c_raced = t42_make_cluster(tmp_raced); TEST_ASSERT_NOT_NULL(c_control); TEST_ASSERT_NOT_NULL(c_raced); t42_make_leader(c_control, term, /*n_peers*/1); t42_make_leader(c_raced, term, /*n_peers*/1); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c_control->commit, &c_control->persist, 1, 8)); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c_raced->commit, &c_raced->persist, 1, 8)); repl_set_commit_engine(&c_control->persist.repl, &c_control->commit); repl_set_commit_engine(&c_raced->persist.repl, &c_raced->commit); myconf_t *mc_control = t43_attach_myconf(c_control, tmp_control); myconf_t *mc_raced = t43_attach_myconf(c_raced, tmp_raced); /* Identical deterministic blobs across BOTH fixtures -- independent * encode calls per fixture (t1469_make_config_blob's own doc comment: * byte-identical for the same (gen, name) parameters). */ size_t oldlen = 0, newlen = 0; uint8_t *old_control = t1469_make_config_blob(1, "t1469-old", &oldlen); uint8_t *new_control = t1469_make_config_blob(2, "t1469-new", &newlen); uint8_t *old_raced = t1469_make_config_blob(1, "t1469-old", &oldlen); uint8_t *new_raced = t1469_make_config_blob(2, "t1469-new", &newlen); uint8_t oldsha[32], newsha[32]; SHA256(old_control, oldlen, oldsha); SHA256(new_control, newlen, newsha); uint64_t oldtag; { uint64_t t; memcpy(&t, oldsha, 8); oldtag = be64toh(t); } uint64_t newtag; { uint64_t t; memcpy(&t, newsha, 8); newtag = be64toh(t); } /* ── Fixture 1: control -- no racing; both markers checkpoint normally. */ TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc_control, old_control, oldlen, oldsha)); free(old_control); t43_commit_marker(c_control, oldtag, oldsha); TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc_control, new_control, newlen, newsha)); free(new_control); t43_commit_marker(c_control, newtag, newsha); uint64_t control_ckpt_lsn = c_control->persist.checkpoint_lsn; bool control_config_present = (mc_control->cluster_config != NULL); uint64_t gen_control = control_config_present ? mc_control->cluster_config->config_generation : 0; int fenced_control = upgrade_materialization_fenced(c_control); /* ── Fixture 2: raced -- marker A's install is still genuinely * outstanding (its checkpoint attempt never lands) when marker B * commits. Hold the shared checkpoint_in_progress CAS BEFORE marker A * commits and never release it until AFTER marker B has also committed * and applied -- this is what makes marker A genuinely "still * installing" at the moment marker B supersedes it, not a race that * might or might not manifest. */ TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc_raced, old_raced, oldlen, oldsha)); free(old_raced); TEST_ASSERT_EQUAL_INT( 0, atomic_exchange_explicit(&c_raced->persist.checkpoint_in_progress, 1, memory_order_acq_rel)); /* Marker A: inline t43_commit_marker's 4-step body -- its signature must * stay unmodified for other callers, and the CAS needs to stay held * across BOTH markers, which a single call to t43_commit_marker cannot * do. */ uint64_t lsn_a = 0; TEST_ASSERT_EQUAL_INT(0, config_append_marker(c_raced, oldtag, oldsha, &lsn_a)); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c_raced->persist)); TEST_ASSERT_EQUAL_INT(0, commit_submit(&c_raced->commit, lsn_a, COMMIT_MODE_WAIT, 1000)); t42_wait_applied(&c_raced->persist, lsn_a, 2000); TEST_ASSERT_TRUE(persist_last_applied_lsn(&c_raced->persist) >= lsn_a); /* Marker B: committed and applied while the CAS is STILL held -- marker * A's install has not happened yet. */ TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc_raced, new_raced, newlen, newsha)); free(new_raced); uint64_t lsn_b = 0; TEST_ASSERT_EQUAL_INT(0, config_append_marker(c_raced, newtag, newsha, &lsn_b)); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c_raced->persist)); TEST_ASSERT_EQUAL_INT(0, commit_submit(&c_raced->commit, lsn_b, COMMIT_MODE_WAIT, 1000)); t42_wait_applied(&c_raced->persist, lsn_b, 2000); TEST_ASSERT_TRUE(persist_last_applied_lsn(&c_raced->persist) >= lsn_b); /* ── Mid-scenario: genuinely "still installing" -- captured BEFORE * releasing the CAS. */ bool config_present_mid = (mc_raced->cluster_config != NULL); uint64_t gen_mid = config_present_mid ? mc_raced->cluster_config->config_generation : 0; char *clustername_mid = (config_present_mid && mc_raced->cluster_config->clustername) ? strdup(mc_raced->cluster_config->clustername) : NULL; bool cfgref_present_mid = c_raced->persist.cfgref_present; int fenced_mid = upgrade_materialization_fenced(c_raced); /* Release the CAS -- whatever else was (simulated to be) holding it has * finished; marker A's and marker B's own checkpoint attempts are both * long past (both -EBUSY'd and skipped). */ atomic_store_explicit(&c_raced->persist.checkpoint_in_progress, 0, memory_order_release); /* Resolve: a fresh checkpoint attempt, self-resolving per design.md -- * mirrors test_marker_apply_busy_checkpoint_handled_gracefully's own * resolve step (test_upgrade_snapshot_apply.c). A bare persist_checkpoint() * call is NOT itself a marker-apply dispatch, so it does not call * upgrade_materialization_verify() on its own -- that call only happens * from apply_committed_entry's WAL_CONFIG_MARKER case (a NEW marker * applying) or from the delaymgr-driven retry trampoline * (schedule_upgrade_materialization_retry), and this bare fixture never * initializes a delaymgr. schedule_upgrade_materialization_retry's own * doc comment is explicit about this exact situation: "No-op when... * the delaymgr is not running (the unit-test fixture path...) -- caller * drives directly". Explicitly re-verify here, mirroring the 6.1-c fence * tests above (e.g. immediately after t43_commit_marker) which do the * same for the same reason. */ int resolve_rc = persist_checkpoint(&c_raced->persist); uint64_t raced_ckpt_lsn = c_raced->persist.checkpoint_lsn; int verify_rc = upgrade_materialization_verify(c_raced); /* ── Post-resolve: the "settled" state. */ uint64_t committed_tag_after = 0; uint8_t committed_sha_after[32]; int rc_committed_after = config_committed_marker( c_raced, &committed_tag_after, committed_sha_after); bool cfgref_matches_after = c_raced->persist.cfgref_present && c_raced->persist.cfgref_tag == newtag && memcmp(c_raced->persist.cfgref_sha, newsha, 32) == 0; int fenced_after = upgrade_materialization_fenced(c_raced); bool config_present_after = (mc_raced->cluster_config != NULL); uint64_t gen_after = config_present_after ? mc_raced->cluster_config->config_generation : 0; char *clustername_after = (config_present_after && mc_raced->cluster_config->clustername) ? strdup(mc_raced->cluster_config->clustername) : NULL; /* ── Cross-fixture proof: settle both checkpoint file paths. */ char control_snap_path[4096], raced_snap_path[4096]; snprintf(control_snap_path, sizeof(control_snap_path), "%s/snap-%016" PRIx64 ".capnp", c_control->persist.snap_path, control_ckpt_lsn); snprintf(raced_snap_path, sizeof(raced_snap_path), "%s/snap-%016" PRIx64 ".capnp", c_raced->persist.snap_path, raced_ckpt_lsn); bool control_snap_exists = (access(control_snap_path, F_OK) == 0); bool raced_snap_exists = (access(raced_snap_path, F_OK) == 0); snapdiff_report_t diff_rep; memset(&diff_rep, 0, sizeof(diff_rep)); int diff_rc = snapdiff_paths(control_snap_path, raced_snap_path, NULL, &diff_rep); /* Independent, field-aware comparison: read both files fully and compare * everything AFTER the 16-byte [magic(8) + checkpoint_lsn(8)] prefix -- * i.e. cfgref_present/tag/sha, format_version, and the entire * entity+config-bytes payload (write_snapshot's sentinel layout, * persist.c). This excludes the ONE field expected to legitimately * differ (the checkpoint's own physical WAL lsn) and asserts everything * else -- which is what would carry any trace of marker A's incomplete * install -- is byte-for-byte identical. */ long control_size = -1, raced_size = -1; uint8_t *control_buf = NULL, *raced_buf = NULL; FILE *fp_control = fopen(control_snap_path, "rb"); if (fp_control) { fseek(fp_control, 0, SEEK_END); control_size = ftell(fp_control); fseek(fp_control, 0, SEEK_SET); if (control_size > 0) { control_buf = (uint8_t *)malloc((size_t)control_size); if (control_buf) TEST_ASSERT_EQUAL_INT((int)control_size, (int)fread(control_buf, 1, (size_t)control_size, fp_control)); } fclose(fp_control); } FILE *fp_raced = fopen(raced_snap_path, "rb"); if (fp_raced) { fseek(fp_raced, 0, SEEK_END); raced_size = ftell(fp_raced); fseek(fp_raced, 0, SEEK_SET); if (raced_size > 0) { raced_buf = (uint8_t *)malloc((size_t)raced_size); if (raced_buf) TEST_ASSERT_EQUAL_INT((int)raced_size, (int)fread(raced_buf, 1, (size_t)raced_size, fp_raced)); } fclose(fp_raced); } bool sizes_match = (control_size >= 0 && control_size == raced_size); bool payload_beyond_lsn_matches = false; if (sizes_match && control_buf && raced_buf && control_size > 16) { payload_beyond_lsn_matches = (memcmp(control_buf + 16, raced_buf + 16, (size_t)(control_size - 16)) == 0); } uint64_t control_lsn_in_file = 0, raced_lsn_in_file = 0; if (control_buf && control_size >= 16) memcpy(&control_lsn_in_file, control_buf + 8, 8); if (raced_buf && raced_size >= 16) memcpy(&raced_lsn_in_file, raced_buf + 8, 8); free(control_buf); free(raced_buf); t43_free_myconf(c_control); t43_free_myconf(c_raced); commit_engine_fini(&c_control->commit); commit_engine_fini(&c_raced->commit); t42_teardown(c_control); t42_teardown(c_raced); /* ── Assertions ──────────────────────────────────────────────────── */ /* Control baseline sanity. */ TEST_ASSERT_TRUE(control_config_present); TEST_ASSERT_EQUAL_UINT64(2, gen_control); TEST_ASSERT_EQUAL_INT(0, fenced_control); /* Mid-scenario: genuinely "still installing". */ TEST_ASSERT_TRUE_MESSAGE(config_present_mid, "member B's own apply must have activated config independently of " "marker A's install state"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(2, gen_mid, "activation must reflect the LATEST marker (B) purely from B's own " "apply, never re-activating to A afterward nor staying short of B"); TEST_ASSERT_NOT_NULL(clustername_mid); if (clustername_mid) TEST_ASSERT_EQUAL_STRING("t1469-new", clustername_mid); TEST_ASSERT_FALSE_MESSAGE(cfgref_present_mid, "no checkpoint has physically landed for EITHER marker yet -- both " "A's and B's checkpoint attempts were skipped via -EBUSY, proving " "this is genuinely 'still installing', not a coincidental pass"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, fenced_mid, "materialization for the latest committed marker (B) has not " "completed yet -- the fence must still be set"); /* Post-resolve: settled state reflects ONLY marker B. */ TEST_ASSERT_EQUAL_INT(0, rc_committed_after); TEST_ASSERT_EQUAL_HEX64(newtag, committed_tag_after); TEST_ASSERT_EQUAL_INT(0, memcmp(newsha, committed_sha_after, 32)); TEST_ASSERT_EQUAL_INT_MESSAGE(0, resolve_rc, "the resolving checkpoint attempt must succeed once the CAS guard " "is free -- self-resolving, per design.md"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, verify_rc, "the explicit re-verify (standing in for this bare fixture's absent " "delaymgr-driven retry trampoline) must succeed once a checkpoint " "has actually captured marker B"); TEST_ASSERT_TRUE_MESSAGE(cfgref_matches_after, "the checkpoint sentinel must now reflect marker B, never marker A"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, fenced_after, "the fence must clear once materialization for the latest marker " "(B) completes"); TEST_ASSERT_TRUE(config_present_after); TEST_ASSERT_EQUAL_UINT64(2, gen_after); TEST_ASSERT_NOT_NULL(clustername_after); if (clustername_after) TEST_ASSERT_EQUAL_STRING("t1469-new", clustername_after); /* Cross-fixture proof -- the decisive assertion for Req 4.4/7.2. */ TEST_ASSERT_TRUE_MESSAGE(control_snap_exists, "control fixture must have produced a checkpoint at marker B's lsn"); TEST_ASSERT_TRUE_MESSAGE(raced_snap_exists, "raced fixture must have produced a checkpoint once resolved"); TEST_ASSERT_TRUE_MESSAGE(sizes_match, "both fixtures' settled checkpoints must be the same total size -- " "identical entity/config content, differing at most in the " "embedded physical WAL lsn"); TEST_ASSERT_TRUE_MESSAGE(payload_beyond_lsn_matches, "Req 4.4/7.2: beyond the one field that legitimately differs " "(the checkpoint's own physical WAL lsn -- itself content per this " "task's own 'deterministic function of (term, lsn, config-hash)' " "framing), the settled checkpoint produced after racing marker A's " "incomplete install must be byte-for-byte identical to the " "non-raced control -- no trace of marker A's partially-applied " "intermediate state survives into marker B's final materialized " "output"); /* Diagnostic corroboration: snapdiff_paths independently confirms * whether the files are literally byte-identical end-to-end, and if * not, that the sole divergence is exactly that one embedded-lsn field * (offset 8, per write_snapshot's sentinel layout). */ TEST_ASSERT_TRUE_MESSAGE(diff_rc >= 0, "snapdiff_paths must not error opening either checkpoint file"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, diff_rep.size_mismatch, "file sizes must match even if the embedded physical lsn differs"); if (!diff_rep.identical) { TEST_ASSERT_EQUAL_INT_MESSAGE(8, (int)diff_rep.diff_offset, "the only expected divergence is the embedded checkpoint_lsn " "field at byte offset 8 (write_snapshot's sentinel layout)"); } TEST_ASSERT_EQUAL_UINT64(control_ckpt_lsn, control_lsn_in_file); TEST_ASSERT_EQUAL_UINT64(raced_ckpt_lsn, raced_lsn_in_file); free(clustername_mid); free(clustername_after); printf("test_second_config_marker_supersedes_first_still_installing_on_" "member: PASS\n"); } /* ── config-upgrade-snapshot task 8.7 Scenario 2: leadership change mid- * materialization does not special-case, and completes via the SAME generic * apply-drain path (Req 5.4) ──────────────────────────────────────────── * * design.md's own "Key simplification found during design" (persist.c's * WAL_CONFIG_MARKER case doc comment, apply_committed_entry): "no new * leader-specific 'resume' logic is needed either, since a newly-elected * leader satisfies this the same way, by continuing to apply its own * committed log." This test proves that claim structurally rather than * asserting it by fiat. * * The concrete mechanism (read directly out of raft_cb_become_leader, * ha.c:1516-1979): the promotion-time eligibility gate * (upgrade_materialization_fenced(c), ha.c:1763) only refuses promotion when * the fence has ALREADY been set by a prior upgrade_materialization_verify() * call. A marker that is committed but has never yet been through * apply_committed_entry() at all -- a genuine apply backlog, not a checkpoint * failure -- leaves the fence at its zero-initialized (unfenced) default, * because nothing has evaluated it yet. So promotion is NOT refused in this * case. What actually drains the backlog is ha.c:1878's * persist_apply_pump_run_sync(&c->persist) call, unconditionally issued right * after role flips to CTRL_PRIMARY -- the SAME synchronous drain function * bug-1043's job-backlog fix added for an entirely unrelated reason (a * promoted standby's un-applied committed jobs must be visible immediately). * The WAL_CONFIG_MARKER case (persist.c) rides that pre-existing, generic * drain for free: no leadership-specific config-upgrade code exists anywhere * in the promotion path. * * This test drives that exact mechanism directly * (persist_apply_pump_run_sync) rather than the full * raft_cb_become_leader_with_cluster_for_test() tail, deliberately: the rest * of that function's post-drain tail (quorum-transition logging, config push, * ctrlhello, persist-hook registration, heartbeat start, own-term gate arm) * is orthogonal to Requirement 5.4's materialization-continuation claim and * would require a substantially larger, novel fixture (full ha_init + * delaymgr_init, mirroring test_convergence_verification.c's cv_setup -- * this file's own t42_/t43_ fixtures deliberately do NOT wire that, since no * other test in this file drives promotion past the fence-refusal point) to * exercise safely -- a scope expansion this test-only task should not take * on. The GATING half of Req 5.4 (a member whose fence IS already set * refuses promotion) is already exhaustively covered by * test_upgrade_materialization_fence_enforced_by_dispatch_gate_and_promotion * (6.2-a) and * test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss (6.2 * remediation) above, both via the real raft_cb_become_leader_with_cluster_ * for_test() gate -- this test does not repeat that proof, it closes the * CONTINUATION half neither of those tests reach (both stop at the refusal; * neither goes on to clear the condition and confirm materialization * completes normally afterward). * * Rigor: builds TWO independent members (A: promoted mid-backlog: role set * CTRL_PRIMARY the same way every other test in this file simulates * promotion (t42_make_leader), THEN drained via persist_apply_pump_run_sync * -- the same function production promotion invokes; B: never promoted, * drained via the ordinary persist_apply_pump_run()) starting from the * IDENTICAL committed marker + config bytes, and asserts their resulting * checkpoints are BYTE-IDENTICAL (snapdiff_paths) -- not merely that both * "also complete," but that they complete IDENTICALLY, mirroring bug-857's * own two-member convergence-proof style (test_bug857_regression.c). */ static void test_leadership_change_mid_materialization_completes_via_same_generic_apply_drain(void) { char tmp_a[256], tmp_b[256]; strcpy(tmp_a, "/tmp/xce_cr87s2_a_XXXXXX"); strcpy(tmp_b, "/tmp/xce_cr87s2_b_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp_a)); TEST_ASSERT_NOT_NULL(mkdtemp(tmp_b)); cluster_t *c_a = t42_make_cluster(tmp_a); cluster_t *c_b = t42_make_cluster(tmp_b); TEST_ASSERT_NOT_NULL(c_a); TEST_ASSERT_NOT_NULL(c_b); myconf_t *mc_a = t43_attach_myconf(c_a, tmp_a); myconf_t *mc_b = t43_attach_myconf(c_b, tmp_b); /* Identical config bytes on both members (t43_make_config_blob is * documented byte-identical across calls -- exactly what this test * wants, since A and B must start from the SAME upgrade content for a * byte-identical-outcome comparison to be meaningful). */ size_t blen = 0; uint8_t *bytes = t43_make_config_blob(&blen); uint8_t sha[32]; SHA256(bytes, blen, sha); uint64_t tag; { uint64_t tmp; memcpy(&tmp, sha, 8); tag = be64toh(tmp); } int rc_save_a = conf_save_config(mc_a, bytes, blen, sha); int rc_save_b = conf_save_config(mc_b, bytes, blen, sha); free(bytes); uint8_t marker_payload[WAL_CONFIG_MARKER_PAYLOAD_LEN]; size_t out_len = 0; int rc_encode = config_marker_encode(tag, sha, marker_payload, WAL_CONFIG_MARKER_PAYLOAD_LEN, &out_len); /* Append the SAME marker directly to each member's physical WAL * (persist_wal_async, bypassing config_append_marker/commit_submit's * commit-engine/fsync-worker machinery entirely) and advance * committed_lsn directly -- mirrors test_bug857_regression.c's own * Scenario 2 technique exactly, chosen for the identical reason: it is * the only DETERMINISTIC way to leave a marker genuinely committed but * NOT YET applied (a real apply backlog) without racing this fixture's * background fsync_worker thread, which a commit_submit(WAIT) round * trip cannot guarantee (see t42_wait_applied's own doc comment on this * exact race). */ int rc_wal_a = persist_wal_async(&c_a->persist, WAL_CONFIG_MARKER, marker_payload, WAL_CONFIG_MARKER_PAYLOAD_LEN); uint64_t marker_lsn_a = c_a->persist.current_lsn - 1; uint64_t committed_a = persist_committed_lsn_advance_atomic( &c_a->persist, marker_lsn_a); int rc_wal_b = persist_wal_async(&c_b->persist, WAL_CONFIG_MARKER, marker_payload, WAL_CONFIG_MARKER_PAYLOAD_LEN); uint64_t marker_lsn_b = c_b->persist.current_lsn - 1; uint64_t committed_b = persist_committed_lsn_advance_atomic( &c_b->persist, marker_lsn_b); /* Precondition (validates this is genuinely the "backlog, never yet * evaluated" case, not the "already fenced" case 6.2-a covers): neither * member has applied anything yet, and neither fence has ever been * evaluated for this marker, so both read unfenced by construction. */ uint64_t last_applied_before_a = persist_last_applied_lsn(&c_a->persist); uint64_t last_applied_before_b = persist_last_applied_lsn(&c_b->persist); int fenced_before_a = upgrade_materialization_fenced(c_a); int fenced_before_b = upgrade_materialization_fenced(c_b); /* A: "leadership change" -- promoted via this fixture family's own * established promotion-simulation call (t42_make_leader; every test in * this file uses it for this purpose except the ones specifically * proving the real election-time GATE itself, which this test does not * re-prove -- see header comment). Confirm becoming primary alone has NO * bearing on the fence (still reads unfenced immediately after the role * flip -- nothing leader-specific touches it), THEN drive the exact * function production promotion uses to drain a backlog * (persist_apply_pump_run_sync, ha.c:1878). */ t42_make_leader(c_a, /*term*/60, /*n_peers*/1); int32_t role_after_promotion = atomic_load_explicit(&c_a->ha.role, memory_order_acquire); int fenced_immediately_after_promotion = upgrade_materialization_fenced(c_a); persist_apply_pump_run_sync(&c_a->persist); /* B: no leadership change at all -- drained via the ordinary, * always-available apply-pump entry point every non-promoted member * (standby or an already-serving primary catching up its own backlog) * uses identically. */ int applied_b = persist_apply_pump_run(&c_b->persist); uint64_t last_applied_after_a = persist_last_applied_lsn(&c_a->persist); uint64_t last_applied_after_b = persist_last_applied_lsn(&c_b->persist); int fenced_after_a = upgrade_materialization_fenced(c_a); int fenced_after_b = upgrade_materialization_fenced(c_b); uint64_t gen_a = mc_a->cluster_config ? mc_a->cluster_config->config_generation : 0; uint64_t gen_b = mc_b->cluster_config ? mc_b->cluster_config->config_generation : 0; bool cluster_config_a_present = (mc_a->cluster_config != NULL); bool cluster_config_b_present = (mc_b->cluster_config != NULL); char snap_a[4096], snap_b[4096]; snprintf(snap_a, sizeof(snap_a), "%s/snap-%016" PRIx64 ".capnp", c_a->persist.snap_path, marker_lsn_a); snprintf(snap_b, sizeof(snap_b), "%s/snap-%016" PRIx64 ".capnp", c_b->persist.snap_path, marker_lsn_b); bool snap_a_exists = (access(snap_a, F_OK) == 0); bool snap_b_exists = (access(snap_b, F_OK) == 0); snapdiff_report_t diff_rep; memset(&diff_rep, 0, sizeof(diff_rep)); int diff_rc = snapdiff_paths(snap_a, snap_b, NULL, &diff_rep); t43_free_myconf(c_a); t43_free_myconf(c_b); t42_teardown(c_a); t42_teardown(c_b); /* ── Assertions ──────────────────────────────────────────────────── */ TEST_ASSERT_EQUAL_INT(0, rc_save_a); TEST_ASSERT_EQUAL_INT(0, rc_save_b); TEST_ASSERT_EQUAL_INT(0, rc_encode); TEST_ASSERT_EQUAL_INT(0, rc_wal_a); TEST_ASSERT_EQUAL_INT(0, rc_wal_b); TEST_ASSERT_EQUAL_UINT64(marker_lsn_a, committed_a); TEST_ASSERT_EQUAL_UINT64(marker_lsn_b, committed_b); TEST_ASSERT_EQUAL_UINT64_MESSAGE(marker_lsn_a, marker_lsn_b, "both members must start from the SAME marker lsn for a " "byte-identical-outcome comparison to be meaningful"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, last_applied_before_a, "precondition: member A must not have applied anything yet"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, last_applied_before_b, "precondition: member B must not have applied anything yet"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, fenced_before_a, "precondition: an unevaluated backlog must NOT already be fenced -- " "this is genuinely the case existing 6.2 tests don't cover, " "distinct from a fence already set by a prior checkpoint miss"); TEST_ASSERT_EQUAL_INT(0, fenced_before_b); TEST_ASSERT_EQUAL_INT_MESSAGE((int32_t)CTRL_PRIMARY, role_after_promotion, "the promotion simulation must actually take effect"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, fenced_immediately_after_promotion, "becoming primary must have NO bearing on the fence by itself -- no " "leader-specific code sets or clears it as a side effect of the " "role transition"); TEST_ASSERT_TRUE_MESSAGE(last_applied_after_a >= marker_lsn_a, "Req 5.4: the SAME generic sync-drain function production " "promotion invokes (persist_apply_pump_run_sync) must complete " "materialization for the backlogged marker -- no special resume " "logic needed, none exists"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, applied_b, "control: the non-promoted member's ordinary apply pump must " "apply exactly the one backlogged marker entry"); TEST_ASSERT_TRUE_MESSAGE(last_applied_after_b >= marker_lsn_b, "control: the non-promoted member's ordinary apply pump must " "complete materialization identically"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, fenced_after_a, "member A's materialization must be genuinely verified complete " "(fence clears), not merely attempted"); TEST_ASSERT_EQUAL_INT(0, fenced_after_b); TEST_ASSERT_TRUE_MESSAGE(cluster_config_a_present, "member A's config must have activated as part of the same " "marker-apply dispatch, exactly as it does for any non-promoted " "member (bug-1469)"); TEST_ASSERT_TRUE(cluster_config_b_present); TEST_ASSERT_EQUAL_UINT64(1, gen_a); /* t43_make_config_blob's hardcoded generation */ TEST_ASSERT_EQUAL_UINT64(gen_b, gen_a); TEST_ASSERT_TRUE_MESSAGE(snap_a_exists, "member A must have produced a checkpoint at the marker's own lsn"); TEST_ASSERT_TRUE_MESSAGE(snap_b_exists, "member B must have produced a checkpoint at the marker's own lsn"); /* THE decisive proof: materialization triggered by a leadership change * mid-backlog produces EXACTLY the same artifact as an ordinary, * never-promoted member's continued apply of the identical marker -- * "completes identically to the non-promoted case," not merely "also * completes." */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, diff_rc, "Req 5.4: a leadership-change-driven materialization drain must " "produce a checkpoint byte-identical to the non-promoted member's"); TEST_ASSERT_TRUE_MESSAGE(diff_rep.identical, "no leader-specific divergence exists in the materialized output"); printf("test_leadership_change_mid_materialization_completes_via_" "same_generic_apply_drain: PASS\n"); } /* ── wait_all_with_timeout (XCE_TESTING shim) ───── */ #ifdef XCE_TESTING /* Thread args for timeout tests */ typedef struct { atomic_int *stop; /* poll this; exit when set to allow pthread_join to return */ push_ctx_t *ctx; } timeout_test_arg_t; /* Polls stop flag at 5ms intervals; exits when stop is set WITHOUT setting acked. * This simulates a synchronous push that respects the done signal. */ static void *slow_push_thread(void *arg) { timeout_test_arg_t *a = (timeout_test_arg_t *)arg; while (!atomic_load(a->stop)) usleep(5000); /* Thread exits without setting acked — deadline fired before completion */ a->ctx->done = 1; return NULL; } static void *fast_push_thread(void *arg) { push_ctx_t *ctx = (push_ctx_t *)arg; ctx->acked = 1; ctx->done = 1; return NULL; } static void test_wait_all_with_timeout_slow_threads(void) { /* 2 threads that poll the done flag and exit without acking. * Timeout is 100ms. Expect: acked==0, done==1, returns within ~200ms. */ const int N = 2; pthread_t threads[2]; push_ctx_t ctxs[2]; timeout_test_arg_t args[2]; atomic_int done; memset(ctxs, 0, sizeof(ctxs)); atomic_init(&done, 0); for (int i = 0; i < N; i++) { ctxs[i].started = 1; args[i].stop = &done; args[i].ctx = &ctxs[i]; pthread_create(&threads[i], NULL, slow_push_thread, &args[i]); } struct timespec t0, t1; clock_gettime(CLOCK_MONOTONIC, &t0); int acked = wait_all_with_timeout_for_test(threads, ctxs, N, 100, &done); clock_gettime(CLOCK_MONOTONIC, &t1); long elapsed_ms = (t1.tv_sec - t0.tv_sec) * 1000 + (t1.tv_nsec - t0.tv_nsec) / 1000000; /* Threads did not ack within the 100ms timeout */ TEST_ASSERT_EQUAL_INT(0, acked); /* done flag must be set by wait_all_with_timeout */ TEST_ASSERT_EQUAL_INT(1, atomic_load(&done)); /* Total elapsed should be close to the 100ms timeout + small join overhead */ TEST_ASSERT_LESS_THAN(400, (int)elapsed_ms); printf("test_wait_all_with_timeout_slow_threads: PASS (elapsed=%ldms acked=%d)\n", elapsed_ms, acked); } static void test_wait_all_with_timeout_fast_threads(void) { /* 3 threads that complete immediately; timeout is 200ms. * Expect: all 3 acked, returns quickly (well under 200ms). */ const int N = 3; pthread_t threads[3]; push_ctx_t ctxs[3]; atomic_int done; memset(ctxs, 0, sizeof(ctxs)); atomic_init(&done, 0); for (int i = 0; i < N; i++) { ctxs[i].started = 1; pthread_create(&threads[i], NULL, fast_push_thread, &ctxs[i]); } struct timespec t0, t1; clock_gettime(CLOCK_MONOTONIC, &t0); int acked = wait_all_with_timeout_for_test(threads, ctxs, N, 200, &done); clock_gettime(CLOCK_MONOTONIC, &t1); long elapsed_ms = (t1.tv_sec - t0.tv_sec) * 1000 + (t1.tv_nsec - t0.tv_nsec) / 1000000; TEST_ASSERT_EQUAL_INT(N, acked); TEST_ASSERT_EQUAL_INT(1, atomic_load(&done)); /* Should have returned long before the 200ms timeout */ TEST_ASSERT_LESS_THAN(200, (int)elapsed_ms); printf("test_wait_all_with_timeout_fast_threads: PASS (elapsed=%ldms acked=%d)\n", elapsed_ms, acked); } /* ── push_config_transport_only endpoint lifetime (bug-1381) ───── */ /* push_thread_send_hook: test-only seam declared in config_repl.c (same * pattern as xce_wal_stream_request_hook) letting this test intercept and * park push_thread's send so it can probe endpoint_ref/unref bracketing * around a concurrent teardown, without a real transport connection. */ extern int (*push_thread_send_hook)(endpoint_t *ep, xce_peerinfo_t *from, uint64_t new_tag, const uint8_t *sha256, const uint8_t *bytes, size_t blen); static atomic_int g_cpto_parked; static atomic_int g_cpto_release; static int cpto_parking_send_hook(endpoint_t *ep, xce_peerinfo_t *from, uint64_t new_tag, const uint8_t *sha256, const uint8_t *bytes, size_t blen) { (void)ep; (void)from; (void)new_tag; (void)sha256; (void)bytes; (void)blen; atomic_store_explicit(&g_cpto_parked, 1, memory_order_release); while (!atomic_load_explicit(&g_cpto_release, memory_order_acquire)) usleep(1000); return 0; } static int cpto_wait_flag(atomic_int *flag, int target, int timeout_ms) { int waited = 0; while (atomic_load_explicit(flag, memory_order_acquire) != target) { if (waited >= timeout_ms) return -1; usleep(2000); waited += 2; } return 0; } typedef struct { cluster_t *c; int acked; int n; } cpto_driver_arg_t; static void *cpto_driver_fn(void *arg) { cpto_driver_arg_t *a = (cpto_driver_arg_t *)arg; uint8_t sha[32]; uint8_t body[4] = { 1, 2, 3, 4 }; memset(sha, 0, sizeof(sha)); a->acked = push_config_transport_only(a->c, body, sizeof(body), sha, 99, &a->n); return NULL; } /* * bug-1381: push_config_transport_only spawns a push_thread per registered * standby that reads ctx->ep for the full duration of the transport send. * Before the fix, no ref was taken on ep, so a concurrent RDHUP teardown * (endpoint_killer -> endpoint_unref, the bug-1298 refcount system) could * free ep out from under the still-running push_thread -- a UAF. This test * parks a push_thread mid-send via push_thread_send_hook, simulates the * owner's teardown, and verifies ep survives until the send completes. */ static void test_configpush_transport_holds_endpoint_ref_across_send(void) { cluster_t c; memset(&c, 0, sizeof(c)); xce_peerinfo_t myhost_info; memset(&myhost_info, 0, sizeof(myhost_info)); c.myhost = &myhost_info; ha_config_t cfg; memset(&cfg, 0, sizeof(cfg)); cfg.heartbeat_timeout_ms = 10000; cfg.election_timeout_ms = 1500; ha_init(&c.ha, &cfg, 0); c.ha.standby_endpoints = calloc(1, sizeof(zarray_t)); TEST_ASSERT_NOT_NULL(c.ha.standby_endpoints); zarray_init(c.ha.standby_endpoints, 4, NULL); /* Heap-allocated endpoint standing in for a real standby connection: * refcnt starts at 1, representing the owner (the standby zarray) * holding the creator reference. */ endpoint_t *ep = endpoint_get(); TEST_ASSERT_NOT_NULL(ep); zarray_store(c.ha.standby_endpoints, ep); atomic_store(&g_cpto_parked, 0); atomic_store(&g_cpto_release, 0); push_thread_send_hook = cpto_parking_send_hook; cpto_driver_arg_t driver = { .c = &c, .acked = -1, .n = -1 }; pthread_t driver_thread; TEST_ASSERT_EQUAL_INT(0, pthread_create(&driver_thread, NULL, cpto_driver_fn, &driver)); TEST_ASSERT_EQUAL_INT_MESSAGE(0, cpto_wait_flag(&g_cpto_parked, 1, 2000), "push_thread must park in the send hook before we probe refcnt"); TEST_ASSERT_EQUAL_INT_MESSAGE(2, atomic_load_explicit(&ep->refcnt, memory_order_acquire), "push_config_transport_only must hold its own ref (owner's 1 + " "push_thread's 1) while the send is in flight"); /* Simulate the owner's teardown (RDHUP -> zarray_del -> endpoint_killer * -> endpoint_unref, bug-1298) racing the in-flight push. */ endpoint_unref(ep); TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load_explicit(&ep->refcnt, memory_order_acquire), "ep must survive the owner's unref while push_thread is still in " "flight -- bug-1381"); atomic_store_explicit(&g_cpto_release, 1, memory_order_release); pthread_join(driver_thread, NULL); TEST_ASSERT_EQUAL_INT(1, driver.n); TEST_ASSERT_EQUAL_INT(1, driver.acked); /* ep is now freed (push_thread's completion unref dropped the last * reference) -- do not touch it again. */ push_thread_send_hook = NULL; zarray_fini(c.ha.standby_endpoints); free(c.ha.standby_endpoints); c.ha.standby_endpoints = NULL; ha_fini(&c.ha); printf("test_configpush_transport_holds_endpoint_ref_across_send: PASS\n"); } /* raft-adapter-conformance task 2.1 (Req 2.2, 2.3, 2.5, 7.4): a config-push * reject must now send an explicit ReplConfigPushReply{accepted=0, term} * back to the primary instead of returning silently. This is a minimal, * representative proof that the new config_push_reply_send_hook actually * fires -- exercised here via ONE reject path (SHA-256 mismatch). Task 2.3 * expands this into full coverage of all four reject paths plus the EAGAIN * accept-path and the cluster.c term-adoption wiring; this test is kept * deliberately minimal so it doesn't collide with that broader coverage. */ static uint64_t g_cpr21_hook_term; static uint64_t g_cpr21_hook_version_tag; static int g_cpr21_hook_accepted; static int g_cpr21_hook_called; static void cpr21_reply_capture(uint64_t term, uint64_t version_tag, int accepted) { g_cpr21_hook_called = 1; g_cpr21_hook_term = term; g_cpr21_hook_version_tag = version_tag; g_cpr21_hook_accepted = accepted; } static void test_config_push_sha256_mismatch_sends_explicit_reject_reply(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr21_shamis_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); t43_attach_myconf(c, tmp); atomic_store(&c->ha.role, CTRL_STANDBY); atomic_store(&c->ha.term, 7); /* Deliberately mismatched payload: raw_sha256 does NOT hash to * raw_bytes -- exercises the real SHA-256-mismatch reject branch. */ uint8_t payload[16]; memset(payload, 0xAB, sizeof(payload)); uint8_t wrong_sha[32]; memset(wrong_sha, 0x00, sizeof(wrong_sha)); /* SHA256(payload) != all-zero */ repl_config_push_req_t rcpr; memset(&rcpr, 0, sizeof(rcpr)); rcpr.version_tag = 0x1234; rcpr.raw_bytes = payload; rcpr.raw_bytes_len = sizeof(payload); rcpr.raw_sha256 = wrong_sha; xce_request_t req; memset(&req, 0, sizeof(req)); req.body_which = Request_body_replconfigpushrequestmsg; req.body.replconfigpushrequestmsg = &rcpr; g_cpr21_hook_called = 0; config_push_reply_send_hook = cpr21_reply_capture; int rc = handle_repl_config_push(c, /*ep*/NULL, &req); config_push_reply_send_hook = NULL; /* uninstall before any assertion can fail */ TEST_ASSERT_EQUAL_INT(-EBADMSG, rc); TEST_ASSERT_TRUE_MESSAGE(g_cpr21_hook_called, "raft-adapter-conformance 2.1: a config-push SHA-256 mismatch must " "send an explicit ReplConfigPushReply, not silence it entirely"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_cpr21_hook_accepted, "the reply for a rejected config push must carry accepted=0"); TEST_ASSERT_EQUAL_UINT64(7, g_cpr21_hook_term); t43_free_myconf(c); t42_teardown(c); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp); system(cmd); printf("test_config_push_sha256_mismatch_sends_explicit_reject_reply: PASS\n"); } /* raft-adapter-conformance task 2.2 (Req 7.4, 7.5): a config-push reply * arriving on the PRIMARY's inbound standby endpoint now carries the * replying member's current term (task 1 schema addition + task 2.1 send * side). myreq_handler's ReplConfigPushReply branch (cluster.c) must route * that term through raft_on_higher_term_observed -- the SAME term-discovery * entry point the heartbeat-reply and WAL-NACK vectors already use * (bug-1421; ha.c ~6650/~6753) -- so a stale-term primary adopts the higher * term and steps down instead of retrying forever against a standby that * has already moved on. * * Drives the REAL decode+dispatch path: builds the wire bytes exactly as * the standby's send_config_push_reply puts them on the wire * (config_repl.c), then pushes them through myreq_handler_for_test -- the * production myreq_handler exposed for tests (cluster.c), same precedent as * peer_ctrl_msg_handler_for_test in ha.c. * * Fixture mirrors test_intake_higher_term_heartbeat_adopts_term * (test_term_safety_adapter.c): ha->role is left at CTRL_STANDBY (which * ha_dispatch_gate admits OPCode_reply for unconditionally, ha.c ~4637) so * the fixture doesn't need real quorum/replicator plumbing, while the RAFT * LIBRARY's own role is forced to RAFT_LEADER -- so the "adopt term and * step down from leadership" observable is scoped precisely to * raft_on_higher_term_observed's own was_leader/step-down behavior, without * dragging in the separate CTRL_PRIMARY-demotion side-effect chain (commit * engine / hooks), which is a different, already-covered code path. */ extern int myreq_handler_for_test(endpoint_t *ep, io_packet_t *pkt); /* Local fakes satisfying cluster.c's OTHER, unrelated OPCode-switch-case * dependencies now that cluster.c is linked into controller_unit_test * (meson.build -- required so myreq_handler_for_test above is reachable at * all). None of these are exercised by the OPCode_reply/ReplConfigPushReply * path this test drives; they exist purely to satisfy the linker for * cluster.c's other switch cases (job/task/scheduler/apiserver/get/ * clusterconf handling), which pull in task.c/opget.c/sched.c/apiserver.c/ * myconf.c -- files intentionally NOT part of this binary (see the sched.c/ * dataex.c comment above sched_push_unit_test_src, meson.build). Same * precedent as test_worker_join.c / test_commit_barrier_reply.c's own local * fakes for on_scheduler_join()/notify()/collect_confvars(). */ int apiserver_init(cluster_t *c) { (void)c; return 0; } int on_task_start(cluster_t *c, xce_task_pidinfo_t *info) { (void)c; (void)info; return 0; } int on_task_fail(cluster_t *c, xce_task_failinfo_t *info) { (void)c; (void)info; return 0; } int on_task_finish(cluster_t *c, xce_task_finish_t *info) { (void)c; (void)info; return 0; } int do_get_op(cluster_t *c, endpoint_t *ep, xce_request_t *req) { (void)c; (void)ep; (void)req; return 0; } int handle_scheduler_leave(cluster_t *c) { (void)c; return 0; } int add_config(void *c, char *name, char *value) { (void)c; (void)name; (void)value; return 0; } cluster_config_t *get_default_conf(void *cl) { (void)cl; return NULL; } int handle_clusterconf(cluster_t *c, endpoint_t *ep, xce_request_t *req) { (void)c; (void)ep; (void)req; return 0; } int do_runtasks(cluster_t *c, endpoint_t *ep, xce_request_t *req) { (void)c; (void)ep; (void)req; return 0; } int handle_apiserver_leave(cluster_t *c, endpoint_t *ep) { (void)c; (void)ep; return 0; } static io_packet_t *cpr22_build_config_push_reply_pkt(int accepted, uint64_t version_tag, uint64_t term) { struct capn cs; struct capn_segment *seg; struct ReplConfigPushReply ack_capn; ReplConfigPushReply_ptr ack_ptr; struct RequestReply rr_capn; RequestReply_ptr rr_ptr; struct Request rq_capn; Request_ptr rq_ptr; io_packet_t *pkt; capn_init_malloc(&cs); seg = capn_root(&cs).seg; memset(&ack_capn, 0, sizeof(ack_capn)); ack_capn.accepted = accepted ? 1 : 0; ack_capn.versionTag = version_tag; ack_capn.term = term; ack_ptr = new_ReplConfigPushReply(seg); write_ReplConfigPushReply(&ack_capn, ack_ptr); memset(&rr_capn, 0, sizeof(rr_capn)); rr_capn.seq = 1; rr_capn.opcode = OPCode_replConfigPush; rr_capn.status = 0; rr_capn.data_which = RequestReply_data_replconfigpushreplymsg; rr_capn.data.replconfigpushreplymsg = ack_ptr; rr_ptr = new_RequestReply(seg); write_RequestReply(&rr_capn, rr_ptr); memset(&rq_capn, 0, sizeof(rq_capn)); rq_capn.opcode = OPCode_reply; rq_capn.seq = 1; rq_capn.body_which = Request_body_replymsg; rq_capn.body.replymsg = rr_ptr; rq_ptr = new_Request(seg); write_Request(&rq_capn, rq_ptr); capn_setp(capn_root(&cs), 0, rq_ptr.p); pkt = calloc(1, sizeof(*pkt)); TEST_ASSERT_NOT_NULL_MESSAGE(pkt, "config-push reply pkt alloc"); pkt->sz = capn_size(&cs); TEST_ASSERT_TRUE_MESSAGE(alloc_packet(pkt) >= 0, "config-push reply pkt data alloc"); capn_write_mem(&cs, (uint8_t *)pkt->data, (size_t)pkt->sz, 0); capn_free(&cs); return pkt; } static void test_config_push_reply_higher_term_adopts_and_steps_down(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr22_term_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); atomic_store(&c->ha.role, CTRL_STANDBY); atomic_store(&c->ha.term, 5); ha_init_raft(&c->ha, c, /*n_peers*/3); c->ha.raft.term = 5; c->ha.raft.role = RAFT_LEADER; endpoint_t ep; memset(&ep, 0, sizeof(ep)); ep.owner = c; /* Reject (accepted=0) carrying a higher term (9 > 5): Req 7.4/7.5 * require the term check to fire regardless of accept/reject -- a * rejected push can still carry a useful higher term. */ io_packet_t *pkt = cpr22_build_config_push_reply_pkt(/*accepted*/0, /*version_tag*/0xABCD, /*term*/9); TEST_ASSERT_NOT_NULL(pkt); int rc = myreq_handler_for_test(&ep, pkt); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_UINT64_MESSAGE(9, c->ha.raft.term, "raft-adapter-conformance 7.4: a config-push reply's higher term " "must be adopted via raft_on_higher_term_observed"); TEST_ASSERT_NOT_EQUAL_INT_MESSAGE(RAFT_LEADER, c->ha.raft.role, "raft-adapter-conformance 7.5: leadership held at the lower term " "must be stepped down once a higher term is observed"); /* NOTE: pkt is already fully freed by this point -- myreq_handler_for_test * routes through myreq_handler -> xce_decode(), and xce_decode() * unconditionally calls free_packet(pkt)+FREEUP(pkt) before returning * (request.c). Do NOT free pkt here: it is a dangling pointer to freed * memory and a second free_packet()/free() on it is a heap * use-after-free/double-free (caught by ASAN regardless of * detect_leaks). See myreq_handler_for_test's doc comment in cluster.c * for the corrected ownership contract. */ t42_teardown(c); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp); system(cmd); printf("test_config_push_reply_higher_term_adopts_and_steps_down: PASS\n"); } #endif /* XCE_TESTING */ /* ── raw capnp shims (Task A) ───────────────────── */ /* Test 1: xce_replconfig_req_ssl encodes a request that round-trips * through the capnp decoder and preserves the version_tag. */ static void test_replconfig_req_ssl_encode(void) { int sv[2]; TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); /* Wrap the writer end of the socketpair in a bare BIO/SSL pair so we * can call SSL_write without doing a TLS handshake. Instead, we use * a memory BIO and let xce_replconfig_req_ssl write into it. */ SSL_library_init(); SSL_CTX *ctx = SSL_CTX_new(TLS_client_method()); TEST_ASSERT_NOT_NULL(ctx); SSL *ssl = SSL_new(ctx); TEST_ASSERT_NOT_NULL(ssl); /* Attach the writer fd as the underlying BIO. We don't perform a * handshake; SSL_write will just SYSCALL through to the fd. This * works because BIO_set_nbio is off and OpenSSL is willing to * write plaintext until handshake completes — except newer OpenSSL * refuses. Use a simpler approach: do the encode + raw fd write. */ /* Cleaner approach: bypass SSL wrapper and just exercise the encode * by re-using the public helper but with our own raw fd write. * We test the encoding path by invoking xce_replconfig_req_ssl * with a properly-constructed SSL only if available, otherwise * fall back to capnp encode verification via the public function * called against a duplicated socket pair. */ SSL_free(ssl); SSL_CTX_free(ctx); /* Use a pipe-based "raw" verification: capnp encode the request * ourselves the same way the function does, then decode and check * version_tag. This validates the wire layout without requiring * a full SSL handshake in unit tests. */ { struct capn c; struct capn_segment *cs; struct Request rq; struct ReplConfigRequest req_struct; ReplConfigRequest_ptr req_ptr; Request_ptr rp; capn_init_malloc(&c); cs = capn_root(&c).seg; memset(&rq, 0, sizeof(rq)); memset(&req_struct, 0, sizeof(req_struct)); req_struct.versionTag = 0xdeadbeefcafef00dULL; req_ptr = new_ReplConfigRequest(cs); write_ReplConfigRequest(&req_struct, req_ptr); rq.opcode = OPCode_replConfig; rq.body_which = Request_body_replconfigrequestmsg; rq.body.replconfigrequestmsg = req_ptr; encode_PeerInfo_ptr(cs, &rq.from, NULL); rp = new_Request(cs); write_Request(&rq, rp); capn_setp(capn_root(&c), 0, rp.p); int64_t sz = capn_size(&c); TEST_ASSERT_TRUE(sz > 0); uint8_t *buf = (uint8_t *)malloc((size_t)sz); TEST_ASSERT_NOT_NULL(buf); capn_write_mem(&c, buf, (size_t)sz, 0); capn_free(&c); /* Decode and verify */ struct capn d; TEST_ASSERT_EQUAL_INT(0, capn_init_mem(&d, buf, (size_t)sz, 0)); Request_ptr dp; dp.p = capn_getp(capn_root(&d), 0, 1); struct Request drq; read_Request(&drq, dp); TEST_ASSERT_EQUAL_INT(OPCode_replConfig, drq.opcode); TEST_ASSERT_EQUAL_INT(Request_body_replconfigrequestmsg, drq.body_which); ReplConfigRequest_ptr drp = drq.body.replconfigrequestmsg; struct ReplConfigRequest drs; read_ReplConfigRequest(&drs, drp); TEST_ASSERT_EQUAL_UINT64(0xdeadbeefcafef00dULL, drs.versionTag); capn_free(&d); free(buf); } close(sv[0]); close(sv[1]); printf("test_replconfig_req_ssl_encode: PASS\n"); } /* Test 2: xce_replconfig_reply_raw — verify NULL endpoint returns * -EINVAL and that the encode path with real bytes preserves the * sha256 and bytes payloads through a round-trip decoder. */ static void test_replconfig_reply_raw_encode(void) { /* NULL endpoint guard */ { uint8_t sha[32]; uint8_t cfg[64]; memset(sha, 0xaa, sizeof(sha)); memset(cfg, 0xbb, sizeof(cfg)); TEST_ASSERT_EQUAL_INT(-EINVAL, xce_replconfig_reply_raw(NULL, NULL, 1, 0x42, sha, sizeof(sha), cfg, sizeof(cfg))); } /* Encode-path test: directly build the same capnp structure that * xce_replconfig_reply_raw builds, then walk down Request → * RequestReply → ReplConfigReply and read sha256/bytes via raw * capn_list8 access. */ { struct capn c; struct capn_segment *cs; struct Request rq; struct RequestReply rr; struct ReplConfigReply reply_struct; ReplConfigReply_ptr reply_ptr; RequestReply_ptr rr_ptr; Request_ptr rp; const uint64_t tag = 0xfeedfacecafebabeULL; const uint64_t seq = 99; uint8_t sha256[32]; uint8_t config[128]; for (size_t i = 0; i < sizeof(sha256); i++) sha256[i] = (uint8_t)(0x10 + i); for (size_t i = 0; i < sizeof(config); i++) config[i] = (uint8_t)(i * 3); capn_init_malloc(&c); cs = capn_root(&c).seg; memset(&rq, 0, sizeof(rq)); memset(&rr, 0, sizeof(rr)); memset(&reply_struct, 0, sizeof(reply_struct)); reply_struct.upToDate = 0; reply_struct.versionTag = tag; { capn_list8 sl = capn_new_list8(cs, (int)sizeof(sha256)); capn_setv8(sl, 0, sha256, (int)sizeof(sha256)); reply_struct.sha256.p = sl.p; } { capn_list8 bl = capn_new_list8(cs, (int)sizeof(config)); capn_setv8(bl, 0, config, (int)sizeof(config)); reply_struct.bytes.p = bl.p; } reply_ptr = new_ReplConfigReply(cs); write_ReplConfigReply(&reply_struct, reply_ptr); rr.seq = seq; rr.opcode = OPCode_replConfig; rr.status = 0; rr.data_which = RequestReply_data_replconfigreplymsg; rr.data.replconfigreplymsg = reply_ptr; rr_ptr = new_RequestReply(cs); write_RequestReply(&rr, rr_ptr); rq.opcode = OPCode_reply; rq.seq = seq; rq.body_which = Request_body_replymsg; rq.body.replymsg = rr_ptr; encode_PeerInfo_ptr(cs, &rq.from, NULL); rp = new_Request(cs); write_Request(&rq, rp); capn_setp(capn_root(&c), 0, rp.p); int64_t sz = capn_size(&c); uint8_t *buf = (uint8_t *)malloc((size_t)sz); capn_write_mem(&c, buf, (size_t)sz, 0); capn_free(&c); /* Decode */ struct capn d; capn_init_mem(&d, buf, (size_t)sz, 0); Request_ptr dp; dp.p = capn_getp(capn_root(&d), 0, 1); struct Request drq; read_Request(&drq, dp); TEST_ASSERT_EQUAL_INT(Request_body_replymsg, drq.body_which); struct RequestReply drr; read_RequestReply(&drr, drq.body.replymsg); TEST_ASSERT_EQUAL_INT(RequestReply_data_replconfigreplymsg, drr.data_which); struct ReplConfigReply drs; read_ReplConfigReply(&drs, drr.data.replconfigreplymsg); TEST_ASSERT_EQUAL_INT(0, drs.upToDate); TEST_ASSERT_EQUAL_UINT64(tag, drs.versionTag); capn_list8 ds; ds.p = drs.sha256.p; capn_list8 db; db.p = drs.bytes.p; TEST_ASSERT_EQUAL_INT(32, capn_len(ds)); TEST_ASSERT_EQUAL_INT(128, capn_len(db)); uint8_t got_sha[32]; uint8_t got_cfg[128]; TEST_ASSERT_EQUAL_INT(32, capn_getv8(ds, 0, got_sha, 32)); TEST_ASSERT_EQUAL_INT(128, capn_getv8(db, 0, got_cfg, 128)); TEST_ASSERT_EQUAL_INT(0, memcmp(got_sha, sha256, 32)); TEST_ASSERT_EQUAL_INT(0, memcmp(got_cfg, config, 128)); capn_free(&d); free(buf); } printf("test_replconfig_reply_raw_encode: PASS\n"); } /* Test 3: xce_upgrade_config_req_extract_raw — build a raw * OP_upgradeConfig packet using the same list8 bypass pattern, then * parse it back and verify sha256 + bytes are recovered intact. */ static void test_upgrade_config_req_extract_raw(void) { const uint8_t sha256[32] = { 0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08, 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10, 0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18, 0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20 }; const uint8_t cfg[] = "fake-upgrade-config-blob-payload-data"; const size_t cfglen = sizeof(cfg) - 1; /* Build a raw OP_upgradeConfig request packet */ struct capn c; struct capn_segment *cs; struct Request rq; struct UpgradeConfigRequest u_struct; UpgradeConfigRequest_ptr u_ptr; Request_ptr rp; capn_init_malloc(&c); cs = capn_root(&c).seg; memset(&rq, 0, sizeof(rq)); memset(&u_struct, 0, sizeof(u_struct)); { capn_list8 sl = capn_new_list8(cs, (int)sizeof(sha256)); capn_setv8(sl, 0, sha256, (int)sizeof(sha256)); u_struct.sha256.p = sl.p; } { capn_list8 bl = capn_new_list8(cs, (int)cfglen); capn_setv8(bl, 0, cfg, (int)cfglen); u_struct.bytes.p = bl.p; } u_ptr = new_UpgradeConfigRequest(cs); write_UpgradeConfigRequest(&u_struct, u_ptr); rq.opcode = OPCode_upgradeConfig; rq.body_which = Request_body_upgradeconfigrequestmsg; rq.body.upgradeconfigrequestmsg = u_ptr; encode_PeerInfo_ptr(cs, &rq.from, NULL); rp = new_Request(cs); write_Request(&rq, rp); capn_setp(capn_root(&c), 0, rp.p); int64_t sz = capn_size(&c); uint8_t *buf = (uint8_t *)malloc((size_t)sz); capn_write_mem(&c, buf, (size_t)sz, 0); capn_free(&c); /* Extract via the function under test */ uint8_t *out_bytes = NULL; uint8_t *out_sha256 = NULL; size_t out_len = 0; int rc = xce_upgrade_config_req_extract_raw(buf, (size_t)sz, &out_bytes, &out_len, &out_sha256); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_NOT_NULL(out_sha256); TEST_ASSERT_NOT_NULL(out_bytes); TEST_ASSERT_EQUAL_INT((int)cfglen, (int)out_len); TEST_ASSERT_EQUAL_INT(0, memcmp(out_sha256, sha256, 32)); TEST_ASSERT_EQUAL_INT(0, memcmp(out_bytes, cfg, cfglen)); free(out_bytes); free(out_sha256); free(buf); /* xce_upgrade_config_req_extract: returns -EINVAL on a zeroed request * (body_which is 0, not Request_body_upgradeconfigrequestmsg). */ { xce_request_t dummy_req; memset(&dummy_req, 0, sizeof(dummy_req)); uint8_t *b = NULL, *s = NULL; size_t l = 0; TEST_ASSERT_EQUAL_INT(-EINVAL, xce_upgrade_config_req_extract(&dummy_req, &b, &l, &s)); } printf("test_upgrade_config_req_extract_raw: PASS\n"); } /* ── Task 2.1: OPCode_walCatchupEntry handler ────────────────────────────── */ #ifdef XCE_TESTING /* Helpers shared by Task 2.1 tests: create/destroy a minimal persist_t. */ static persist_t *make_test_persist_catchup(char *tmpdir_out, size_t sz) { persist_t *p = calloc(1, sizeof(persist_t)); if (!p) { fprintf(stderr, "calloc failed\n"); exit(1); } persist_config_t cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; /* no real fsync in unit tests */ snprintf(tmpdir_out, sz, "/tmp/xce_catchup_XXXXXX"); if (!mkdtemp(tmpdir_out)) { fprintf(stderr, "mkdtemp failed\n"); free(p); exit(1); } persist_set_workdir(p, tmpdir_out, "ctrl-catchup"); persist_init(p, &cfg, NULL); persist_start(p); return p; } static void destroy_test_persist_catchup(persist_t *p) { persist_stop(p); persist_fini(p); free(p); } /* bug-850/851 contiguity guard (persist.c:839): persist_wal_write_replicated * REFUSES a forward gap (entry.lsn > current_lsn → -EAGAIN) so a standby never * creates a WAL hole. A fresh persist has current_lsn == 1 (LSN 0 reserved, * persist.c:542). The catch-up tests below choose their own non-1 starting LSNs * (42, 10, 20, …); seed current_lsn to the first applied entry's LSN so the * first entry is contiguous (entry.lsn == current_lsn) and the guard accepts * the in-order stream — mirroring the real catch-up streamer, which streams * contiguously from current_lsn. current_lsn is a plain lock-protected field; * persist.c sets it directly under p->lock (e.g. persist.c:542,715), so do the * same here. */ static void seed_persist_current_lsn(persist_t *p, uint64_t next_lsn) { pthread_mutex_lock(&p->lock); p->current_lsn = next_lsn; pthread_mutex_unlock(&p->lock); } /* Test 2.1-A: successful single WAL entry advances applied_lsn */ static void test_wal_catchup_entry_advances_applied_lsn(void) { char tmpdir[256]; persist_t *p = make_test_persist_catchup(tmpdir, sizeof(tmpdir)); ha_state_t ha; memset(&ha, 0, sizeof(ha)); ha_config_t cfg; memset(&cfg, 0, sizeof(cfg)); cfg.heartbeat_timeout_ms = 10000; cfg.election_timeout_ms = 1500; ha_init(&ha, &cfg, 0); /* applied_lsn starts at 0 */ TEST_ASSERT_EQUAL_UINT64(0, (uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire)); /* Seed current_lsn so the entry at LSN 42 is contiguous (bug-850 guard). */ seed_persist_current_lsn(p, 42); /* Build a single-entry WAL catch-up with LSN=42, no payload */ wal_catchup_entry_t entry; memset(&entry, 0, sizeof(entry)); entry.lsn = 42; entry.entry_type = (uint8_t)WAL_JOB_UPDATE; entry.payload = NULL; /* payload_len=0; apply_wal_entry handles NULL */ entry.n_entries = 0; entry.entries = NULL; /* GREEN will implement ha_apply_wal_catchup_entry() */ int rc = ha_apply_wal_catchup_entry(&ha, p, &entry, NULL, NULL); TEST_ASSERT_EQUAL_INT(0, rc); /* bug-991: applied_lsn is the LAST CONTIGUOUSLY-HELD LSN, so a single applied * entry at LSN 42 advances applied_lsn to 42 (NOT 43). The +1 over-advertised * the standby's watermark and made the leader's catch-up streamer skip the * real next-needed LSN. */ TEST_ASSERT_EQUAL_UINT64(42, (uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire)); ha_fini(&ha); destroy_test_persist_catchup(p); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_wal_catchup_entry_advances_applied_lsn: PASS\n"); } /* Test 2.1-B: apply_wal_entry failure (NULL persist = -EINVAL) is logged and * skipped — applied_lsn must NOT advance, handler must not crash. * Req 3.3: corrupt/error entries are logged and skipped without crashing. */ static void test_wal_catchup_entry_error_skipped_no_crash(void) { ha_state_t ha; memset(&ha, 0, sizeof(ha)); ha_config_t cfg; memset(&cfg, 0, sizeof(cfg)); cfg.heartbeat_timeout_ms = 10000; cfg.election_timeout_ms = 1500; ha_init(&ha, &cfg, 0); atomic_store_explicit(&ha.applied_lsn, 10, memory_order_release); /* Pass NULL persist — apply_wal_entry returns -EINVAL for NULL p */ wal_catchup_entry_t entry; memset(&entry, 0, sizeof(entry)); entry.lsn = 99; entry.entry_type = (uint8_t)WAL_JOB_UPDATE; entry.payload = NULL; entry.n_entries = 0; entry.entries = NULL; /* Handler must NOT crash; must log ERROR; applied_lsn must remain at 10 */ int rc = ha_apply_wal_catchup_entry(&ha, /*persist=*/NULL, &entry, NULL, NULL); /* rc is non-zero (error propagated from apply_wal_entry(-EINVAL)) */ TEST_ASSERT_TRUE(rc != 0); /* applied_lsn must not advance on failure */ TEST_ASSERT_EQUAL_UINT64(10, (uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire)); ha_fini(&ha); printf("test_wal_catchup_entry_error_skipped_no_crash: PASS\n"); } /* bug-1479: a term-carrying entry that fails logOk (forward gap / genuine * (term,lsn) conflict) must now send a NACK {success=0} back to the primary, * not silence the reject entirely -- otherwise the primary's NACK-handling * (ha_raft_on_repl_nack, ha.c) can never engage, no matter how correct it is, * because nothing ever produces the reply it is waiting for. This is exactly * the live gap the config-upgrade soak's force-elect rotation exposed: a * standby that fell behind (a genuine forward gap against its empty log) * looped rejecting the identical re-broadcast entry forever, with no NACK * ever reaching the primary to trigger backoff + catch-up. * * ha_wal_ack_send_hook (ha.h, XCE_TESTING) captures the {term, lsn, success} * that ha_send_wal_ack_reply_ex would have sent, without a live endpoint. */ static uint64_t g_b1479_hook_term; static uint64_t g_b1479_hook_lsn; static int g_b1479_hook_success; static int g_b1479_hook_called; static void b1479_wal_ack_capture(uint64_t term, uint64_t lsn, int success) { g_b1479_hook_called = 1; g_b1479_hook_term = term; g_b1479_hook_lsn = lsn; g_b1479_hook_success = success; } static void test_bug1479_ae_reject_sends_nack_not_silence(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr_b1479_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); /* Bring the node up as a raft FOLLOWER at term 3 with an EMPTY log * (log_len=0) -- mirrors t_bug1475_make_follower further below in this * file (that fixture is defined later; duplicated inline here to avoid * an ordering dependency). */ atomic_store(&c->ha.role, CTRL_STANDBY); atomic_store(&c->ha.term, 3); ha_init_raft(&c->ha, c, /*n_peers*/3); c->ha.raft.term = 3; c->ha.raft.role = RAFT_FOLLOWER; TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.log_len); g_b1479_hook_called = 0; ha_wal_ack_send_hook = b1479_wal_ack_capture; /* A term-carrying entry at lsn=5 whose prev_log_index=4 is a forward gap * against this node's empty log (0 < 4) -- logOk MUST reject it. */ wal_catchup_entry_t inner; memset(&inner, 0, sizeof(inner)); inner.lsn = 5; inner.term = 3; inner.prev_log_index = 4; inner.prev_log_term = 3; inner.entry_type = (uint8_t)WAL_JOB_UPDATE; wal_catchup_entry_t *sub_arr[1] = { &inner }; wal_catchup_entry_t outer; memset(&outer, 0, sizeof(outer)); outer.n_entries = 1; outer.entries = sub_arr; int rc = ha_apply_wal_catchup_entry(&c->ha, &c->persist, &outer, /*ep*/NULL, c); ha_wal_ack_send_hook = NULL; /* uninstall before any assertion can fail */ TEST_ASSERT_EQUAL_INT(0, rc); /* reject, not a hard error */ TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.log_len); /* nothing was accepted */ TEST_ASSERT_TRUE_MESSAGE(g_b1479_hook_called, "bug-1479: an AppendEntries reject must send a WalAckReply " "(a NACK), not silence it entirely"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_b1479_hook_success, "the reply for a rejected entry must carry success=0"); TEST_ASSERT_EQUAL_UINT64(3, g_b1479_hook_term); TEST_ASSERT_EQUAL_UINT64(5, g_b1479_hook_lsn); t42_teardown(c); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp); system(cmd); printf("test_bug1479_ae_reject_sends_nack_not_silence: PASS\n"); } /* Test 2.1-C: NULL entry is handled gracefully (null guard) */ static void test_wal_catchup_entry_null_entry_no_crash(void) { ha_state_t ha; memset(&ha, 0, sizeof(ha)); ha_config_t cfg; memset(&cfg, 0, sizeof(cfg)); cfg.heartbeat_timeout_ms = 10000; cfg.election_timeout_ms = 1500; ha_init(&ha, &cfg, 0); /* NULL entry — must not crash, must return error */ int rc = ha_apply_wal_catchup_entry(&ha, NULL, NULL, NULL, NULL); TEST_ASSERT_TRUE(rc != 0); ha_fini(&ha); printf("test_wal_catchup_entry_null_entry_no_crash: PASS\n"); } /* ── Task 2.2: batch WAL catch-up entries on standby ─────────────────────── */ /* Test 2.2-A: batch of 3 entries — applied_lsn advances to last+1. * Requirements 5.3, 5.5: apply each in order, advance after full batch. */ static void test_wal_catchup_batch_advances_applied_lsn(void) { char tmpdir[256]; persist_t *p = make_test_persist_catchup(tmpdir, sizeof(tmpdir)); ha_state_t ha; memset(&ha, 0, sizeof(ha)); ha_config_t cfg; memset(&cfg, 0, sizeof(cfg)); cfg.heartbeat_timeout_ms = 10000; cfg.election_timeout_ms = 1500; ha_init(&ha, &cfg, 0); /* applied_lsn starts at 0 */ TEST_ASSERT_EQUAL_UINT64(0, (uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire)); /* Seed current_lsn so the batch [10,11,12] is contiguous (bug-850 guard). */ seed_persist_current_lsn(p, 10); /* Build 3 sub-entries at LSNs 10, 11, 12 */ wal_catchup_entry_t e0, e1, e2; memset(&e0, 0, sizeof(e0)); memset(&e1, 0, sizeof(e1)); memset(&e2, 0, sizeof(e2)); e0.lsn = 10; e0.entry_type = (uint8_t)WAL_JOB_UPDATE; e1.lsn = 11; e1.entry_type = (uint8_t)WAL_JOB_UPDATE; e2.lsn = 12; e2.entry_type = (uint8_t)WAL_JOB_UPDATE; wal_catchup_entry_t *batch_entries[3] = { &e0, &e1, &e2 }; /* Outer entry carries the batch; lsn/entry_type ignored when n_entries > 0 */ wal_catchup_entry_t outer; memset(&outer, 0, sizeof(outer)); outer.n_entries = 3; outer.entries = batch_entries; int rc = ha_apply_wal_catchup_entry(&ha, p, &outer, NULL, NULL); TEST_ASSERT_EQUAL_INT(0, rc); /* bug-991: applied_lsn = last contiguously-held LSN = last_entry->lsn = 12 * (NOT 13). */ TEST_ASSERT_EQUAL_UINT64(12, (uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire)); ha_fini(&ha); destroy_test_persist_catchup(p); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_wal_catchup_batch_advances_applied_lsn: PASS\n"); } /* Test 2.2-B: partial batch (2 of max 3) — applied_lsn advances correctly. * Requirement 5.4: partial batch (fewer than max) sent and applied without wait. */ static void test_wal_catchup_partial_batch(void) { char tmpdir[256]; persist_t *p = make_test_persist_catchup(tmpdir, sizeof(tmpdir)); ha_state_t ha; memset(&ha, 0, sizeof(ha)); ha_config_t cfg; memset(&cfg, 0, sizeof(cfg)); cfg.heartbeat_timeout_ms = 10000; cfg.election_timeout_ms = 1500; ha_init(&ha, &cfg, 0); /* Pre-set applied_lsn to 5 to verify monotonic advance */ atomic_store_explicit(&ha.applied_lsn, 5, memory_order_release); /* Seed current_lsn so the batch [20,21] is contiguous (bug-850 guard). */ seed_persist_current_lsn(p, 20); wal_catchup_entry_t e0, e1; memset(&e0, 0, sizeof(e0)); memset(&e1, 0, sizeof(e1)); e0.lsn = 20; e0.entry_type = (uint8_t)WAL_JOB_UPDATE; e1.lsn = 21; e1.entry_type = (uint8_t)WAL_JOB_UPDATE; wal_catchup_entry_t *batch_entries[2] = { &e0, &e1 }; wal_catchup_entry_t outer; memset(&outer, 0, sizeof(outer)); outer.n_entries = 2; outer.entries = batch_entries; int rc = ha_apply_wal_catchup_entry(&ha, p, &outer, NULL, NULL); TEST_ASSERT_EQUAL_INT(0, rc); /* bug-991: applied_lsn = last contiguously-held LSN = 21 (NOT 22). Monotonic: * advances from the pre-set 5 to 21. */ TEST_ASSERT_EQUAL_UINT64(21, (uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire)); ha_fini(&ha); destroy_test_persist_catchup(p); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_wal_catchup_partial_batch: PASS\n"); } /* Test 2.2-C: batch with one error entry — error is logged and skipped, * remaining entries are still applied, applied_lsn advances for the good ones. * Requirement 3.3/5.5: error in batch → skip that entry, continue with rest. */ static void test_wal_catchup_batch_error_entry_skipped(void) { char tmpdir[256]; persist_t *p = make_test_persist_catchup(tmpdir, sizeof(tmpdir)); ha_state_t ha; memset(&ha, 0, sizeof(ha)); ha_config_t cfg; memset(&cfg, 0, sizeof(cfg)); cfg.heartbeat_timeout_ms = 10000; cfg.election_timeout_ms = 1500; ha_init(&ha, &cfg, 0); /* Seed current_lsn so the first good entry at LSN 30 is contiguous (bug-850 * guard). The NULL sub-entry is skipped (no write, current_lsn stays at 31); * the test only requires applied_lsn >= 30 (the first good entry held). */ seed_persist_current_lsn(p, 30); /* Good entry at LSN 30, then a NULL pointer (simulates corrupt entry), * then good entry at LSN 32. */ wal_catchup_entry_t e0, e2; memset(&e0, 0, sizeof(e0)); memset(&e2, 0, sizeof(e2)); e0.lsn = 30; e0.entry_type = (uint8_t)WAL_JOB_UPDATE; e2.lsn = 32; e2.entry_type = (uint8_t)WAL_JOB_UPDATE; /* NULL in the middle represents a missing/corrupt sub-entry */ wal_catchup_entry_t *batch_entries[3] = { &e0, NULL, &e2 }; wal_catchup_entry_t outer; memset(&outer, 0, sizeof(outer)); outer.n_entries = 3; outer.entries = batch_entries; /* Handler must not crash; applied_lsn advances for successful entries */ int rc = ha_apply_wal_catchup_entry(&ha, p, &outer, NULL, NULL); /* Overall rc == 0: outer batch call succeeds even with inner skips */ TEST_ASSERT_EQUAL_INT(0, rc); /* bug-991: applied_lsn = last contiguously-held LSN. e0 (LSN 30) applies, * the NULL sub-entry is skipped, e2 (LSN 32) applies → last-held is 32 (or at * least 30 if e2 were dropped). Must reach at least 30 (e0 held). */ uint64_t final_lsn = (uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire); TEST_ASSERT_TRUE(final_lsn >= 30); ha_fini(&ha); destroy_test_persist_catchup(p); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_wal_catchup_batch_error_entry_skipped: PASS\n"); } /* Test 2.2-D: n_entries==0 falls through to single-entry path (backward compat) */ static void test_wal_catchup_batch_zero_entries_fallback(void) { char tmpdir[256]; persist_t *p = make_test_persist_catchup(tmpdir, sizeof(tmpdir)); ha_state_t ha; memset(&ha, 0, sizeof(ha)); ha_config_t cfg; memset(&cfg, 0, sizeof(cfg)); cfg.heartbeat_timeout_ms = 10000; cfg.election_timeout_ms = 1500; ha_init(&ha, &cfg, 0); /* Seed current_lsn so the fallback single entry at LSN 50 is contiguous * (bug-850 guard). */ seed_persist_current_lsn(p, 50); /* n_entries == 0 → single-entry path from task 2.1 */ wal_catchup_entry_t entry; memset(&entry, 0, sizeof(entry)); entry.lsn = 50; entry.entry_type = (uint8_t)WAL_JOB_UPDATE; entry.n_entries = 0; entry.entries = NULL; int rc = ha_apply_wal_catchup_entry(&ha, p, &entry, NULL, NULL); TEST_ASSERT_EQUAL_INT(0, rc); /* bug-991: single-entry path advances applied_lsn to the last-held LSN = 50 * (NOT 51). */ TEST_ASSERT_EQUAL_UINT64(50, (uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire)); ha_fini(&ha); destroy_test_persist_catchup(p); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_wal_catchup_batch_zero_entries_fallback: PASS\n"); } /* ── Task 6.2: additional WAL catch-up unit tests ───────────────────────── */ /* Test 6.2-A: batch of exactly 256 entries (the maximum batch size from the * sender side, Requirement 5.2) — all applied, applied_lsn advances by 256. * Tests: "apply batch of 256" from task 6.2 spec. * Requirements 3.1, 3.5, 5.3, 5.5. */ static void test_wal_catchup_batch_256_entries(void) { char tmpdir[256]; persist_t *p = make_test_persist_catchup(tmpdir, sizeof(tmpdir)); ha_state_t ha; memset(&ha, 0, sizeof(ha)); ha_config_t cfg; memset(&cfg, 0, sizeof(cfg)); cfg.heartbeat_timeout_ms = 10000; cfg.election_timeout_ms = 1500; ha_init(&ha, &cfg, 0); /* Seed current_lsn so the consecutive batch starting at LSN 1000 is * contiguous (bug-850 guard). */ seed_persist_current_lsn(p, 1000); /* Allocate 256 sub-entries at consecutive LSNs starting at 1000 */ const int N = 256; wal_catchup_entry_t entries[N]; wal_catchup_entry_t *ptrs[N]; for (int i = 0; i < N; i++) { memset(&entries[i], 0, sizeof(entries[i])); entries[i].lsn = (uint64_t)(1000 + i); entries[i].entry_type = (uint8_t)WAL_JOB_UPDATE; entries[i].n_entries = 0; entries[i].entries = NULL; ptrs[i] = &entries[i]; } wal_catchup_entry_t outer; memset(&outer, 0, sizeof(outer)); outer.n_entries = N; outer.entries = ptrs; /* applied_lsn starts at 0 */ TEST_ASSERT_EQUAL_UINT64(0, (uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire)); int rc = ha_apply_wal_catchup_entry(&ha, p, &outer, NULL, NULL); TEST_ASSERT_EQUAL_INT(0, rc); /* bug-991: applied_lsn = last contiguously-held LSN = last_entry->lsn = * 1000+255 = 1255 (NOT 1256). */ TEST_ASSERT_EQUAL_UINT64(1255, (uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire)); ha_fini(&ha); destroy_test_persist_catchup(p); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_wal_catchup_batch_256_entries: PASS\n"); } /* Test 6.2-B: applied_lsn advances only AFTER the full batch is written, not * during mid-batch entry application. This verifies Req 5.5: the standby's * next heartbeat reply reflects the post-batch state, not a partial state. * * Strategy: start with a pre-set applied_lsn, apply a batch of 4 entries, * verify that applied_lsn does NOT advance to an intermediate value and IS * exactly the last contiguously-held LSN at the end (bug-991: last_lsn, not * last_lsn+1). We confirm the "only-after-full-batch" invariant by checking the * exact final value equals last_lsn (not some intermediate held LSN from a * mid-batch advance). * Requirements 5.3, 5.5. */ static void test_wal_catchup_batch_applied_lsn_only_after_full_batch(void) { char tmpdir[256]; persist_t *p = make_test_persist_catchup(tmpdir, sizeof(tmpdir)); ha_state_t ha; memset(&ha, 0, sizeof(ha)); ha_config_t cfg; memset(&cfg, 0, sizeof(cfg)); cfg.heartbeat_timeout_ms = 10000; cfg.election_timeout_ms = 1500; ha_init(&ha, &cfg, 0); /* Pre-set applied_lsn to a known value so we can distinguish partial * from full advance. */ atomic_store_explicit(&ha.applied_lsn, 100, memory_order_release); /* Seed current_lsn so the batch [200..203] is contiguous (bug-850 guard). */ seed_persist_current_lsn(p, 200); /* Build a batch of 4 entries at LSNs 200, 201, 202, 203 */ wal_catchup_entry_t e0, e1, e2, e3; memset(&e0, 0, sizeof(e0)); memset(&e1, 0, sizeof(e1)); memset(&e2, 0, sizeof(e2)); memset(&e3, 0, sizeof(e3)); e0.lsn = 200; e0.entry_type = (uint8_t)WAL_JOB_UPDATE; e1.lsn = 201; e1.entry_type = (uint8_t)WAL_JOB_UPDATE; e2.lsn = 202; e2.entry_type = (uint8_t)WAL_JOB_UPDATE; e3.lsn = 203; e3.entry_type = (uint8_t)WAL_JOB_UPDATE; wal_catchup_entry_t *ptrs[4] = { &e0, &e1, &e2, &e3 }; wal_catchup_entry_t outer; memset(&outer, 0, sizeof(outer)); outer.n_entries = 4; outer.entries = ptrs; int rc = ha_apply_wal_catchup_entry(&ha, p, &outer, NULL, NULL); TEST_ASSERT_EQUAL_INT(0, rc); /* bug-991: applied_lsn must be exactly 203 (the last contiguously-held LSN) — * the post-batch value. The test for correctness is that the final value * equals the LAST entry's lsn, not any intermediate held LSN. * Specifically, it must NOT remain at 200 or 201 (partial advance). */ uint64_t final_lsn = (uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire); /* Must have advanced past starting value (100) */ TEST_ASSERT_GREATER_THAN_UINT64(100, final_lsn); /* Must equal the last-held LSN (203), not any intermediate value */ TEST_ASSERT_EQUAL_UINT64(203, final_lsn); ha_fini(&ha); destroy_test_persist_catchup(p); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_wal_catchup_batch_applied_lsn_only_after_full_batch: PASS\n"); } /* ── Task 2.3: OPCode_nop heartbeat reply carries stateReport ──────────── */ #ifdef XCE_TESTING /* Test seam declared in cluster.c under #ifdef XCE_TESTING — * fills *sr from ha->applied_lsn / myconf->config_version_tag, * sets reply_out->data_which = stateReportMsg and reply_out->data.state_report = sr. * Req 1.1, 1.3, 1.4. */ void ha_nop_reply_state_report(ha_state_t *ha, myconf_t *myconf, state_report_t *sr, xce_reply_t *reply_out); /* Test 2.3-A: nop reply carries stateReport with applied_lsn and config_tag. * Requirements 1.1, 1.3, 1.4: standby populates stateReport in every * OPCode_nop heartbeat reply with current applied_lsn and config_version_tag. */ static void test_nop_reply_carries_state_report(void) { ha_state_t ha; memset(&ha, 0, sizeof(ha)); ha_config_t cfg; memset(&cfg, 0, sizeof(cfg)); cfg.heartbeat_timeout_ms = 10000; cfg.election_timeout_ms = 1500; ha_init(&ha, &cfg, 0); /* Set a known applied_lsn */ atomic_store_explicit(&ha.applied_lsn, 12345ULL, memory_order_release); /* Build a minimal myconf with a known config_version_tag */ myconf_t myconf; memset(&myconf, 0, sizeof(myconf)); myconf.config_version_tag = 0xABCD1234ULL; state_report_t sr; memset(&sr, 0, sizeof(sr)); xce_reply_t reply; memset(&reply, 0, sizeof(reply)); ha_nop_reply_state_report(&ha, &myconf, &sr, &reply); /* Reply must use stateReportMsg discriminant */ TEST_ASSERT_EQUAL_INT(RequestReply_data_stateReportMsg, reply.data_which); /* state_report pointer must point at our sr */ TEST_ASSERT_EQUAL_PTR(&sr, reply.data.state_report); /* applied_lsn must match ha->applied_lsn */ TEST_ASSERT_EQUAL_UINT64(12345ULL, sr.applied_lsn); /* config_tag must match myconf->config_version_tag */ TEST_ASSERT_EQUAL_UINT64(0xABCD1234ULL, sr.config_tag); ha_fini(&ha); printf("test_nop_reply_carries_state_report: PASS\n"); } /* Test 2.3-B: nop reply when applied_lsn==0 (fresh standby, no config). * Requirement 1.4: config_tag reports zero when no config received yet. */ static void test_nop_reply_zero_state_fresh_standby(void) { ha_state_t ha; memset(&ha, 0, sizeof(ha)); ha_config_t cfg; memset(&cfg, 0, sizeof(cfg)); cfg.heartbeat_timeout_ms = 10000; cfg.election_timeout_ms = 1500; ha_init(&ha, &cfg, 0); /* applied_lsn stays at 0; config_version_tag stays at 0 → no config yet */ myconf_t myconf; memset(&myconf, 0, sizeof(myconf)); state_report_t sr; memset(&sr, 0, sizeof(sr)); xce_reply_t reply; memset(&reply, 0, sizeof(reply)); ha_nop_reply_state_report(&ha, &myconf, &sr, &reply); TEST_ASSERT_EQUAL_INT(RequestReply_data_stateReportMsg, reply.data_which); TEST_ASSERT_EQUAL_PTR(&sr, reply.data.state_report); TEST_ASSERT_EQUAL_UINT64(0ULL, sr.applied_lsn); TEST_ASSERT_EQUAL_UINT64(0ULL, sr.config_tag); ha_fini(&ha); printf("test_nop_reply_zero_state_fresh_standby: PASS\n"); } #endif /* XCE_TESTING */ #endif /* XCE_TESTING */ /* ── Task 3.4: ha_stream_wal_to_standby batch WAL streaming ──────────────── */ #ifdef XCE_TESTING /* * xce_wal_stream_request_hook: test-only seam declared in config_repl.c. * When non-NULL, ha_stream_wal_to_standby() calls this INSTEAD OF xce_request() * for each batch message, allowing tests to inspect and count batches without * a real endpoint. * * Signature: int hook(endpoint_t *ep, xce_request_t *req) * ep — outbound endpoint (may be NULL in tests) * req — the batch OPCode_walCatchupEntry request being sent * Returns 0 on success (the hook owns no resources in the req). */ extern int (*xce_wal_stream_request_hook)(endpoint_t *ep, xce_request_t *req); /* Helper: write N minimal WAL entries into a single segment file in wal_dir. * Entries have consecutive LSNs starting at start_lsn. * Returns 0 on success, -1 on error. */ static int write_test_wal_segment(const char *wal_dir, uint64_t start_lsn, int n_entries) { char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), "%s/seg-%016" PRIx64 ".wal", wal_dir, start_lsn); int fd = open(seg_path, O_WRONLY | O_CREAT | O_TRUNC, 0600); if (fd < 0) return -1; for (int i = 0; i < n_entries; i++) { wal_entry_hdr_t hdr; memset(&hdr, 0, sizeof(hdr)); hdr.lsn = start_lsn + (uint64_t)i; hdr.type = WAL_JOB_UPDATE; hdr.payload_len = 0; hdr.term = 1; ssize_t nw = write(fd, &hdr, sizeof(hdr)); if (nw != (ssize_t)sizeof(hdr)) { close(fd); return -1; } } close(fd); return 0; } /* Batch-counting state for the hook */ static int g_wal_stream_batch_count = 0; static int g_wal_stream_total_entries = 0; static int g_wal_stream_last_n_entries = 0; static int counting_wal_stream_hook(endpoint_t *ep, xce_request_t *req) { (void)ep; if (!req || !req->body.walcatchupentrymsg) return 0; wal_catchup_entry_t *batch = (wal_catchup_entry_t *)req->body.walcatchupentrymsg; g_wal_stream_batch_count++; g_wal_stream_last_n_entries = batch->n_entries; g_wal_stream_total_entries += batch->n_entries; return 0; } /* * Test 3.4-A: streaming 500 entries produces exactly 2 batches * (batch 1: 256 entries, batch 2: 244 entries). * Requirement 5.2: accumulate up to 256 entries per batch. * Requirement 5.4: partial final batch is sent immediately. */ static void test_wal_stream_500_entries_two_batches(void) { /* Set up WAL directory with 500 consecutive entries starting at LSN 1 */ char tmpdir[256]; snprintf(tmpdir, sizeof(tmpdir), "/tmp/xce_wal_stream_%d_%ld", (int)getpid(), (long)time(NULL)); TEST_ASSERT_EQUAL_INT(0, mkdir(tmpdir, 0700)); /* Write all 500 entries in a single segment starting at LSN 1 */ int rc = write_test_wal_segment(tmpdir, /*start_lsn=*/1, /*n_entries=*/500); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "Failed to write test WAL segment"); /* Install counting hook */ xce_wal_stream_request_hook = counting_wal_stream_hook; g_wal_stream_batch_count = 0; g_wal_stream_total_entries = 0; g_wal_stream_last_n_entries = 0; /* Stream entries [1..500]: standby_lsn=0, committed_lsn=500 */ rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir, /*p=*/NULL, /*leader_term=*/7, /*standby_lsn=*/0, /*committed_lsn=*/500); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "ha_stream_wal_to_standby failed"); TEST_ASSERT_EQUAL_INT_MESSAGE(2, g_wal_stream_batch_count, "500 entries must produce exactly 2 batches (256+244)"); TEST_ASSERT_EQUAL_INT_MESSAGE(500, g_wal_stream_total_entries, "All 500 entries must be sent"); TEST_ASSERT_EQUAL_INT_MESSAGE(244, g_wal_stream_last_n_entries, "Final batch must contain 244 entries (500 - 256)"); xce_wal_stream_request_hook = NULL; char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_wal_stream_500_entries_two_batches: PASS\n"); } /* * Test 3.4-B: streaming fewer than 256 entries produces exactly 1 partial batch. * Requirement 5.4: partial final batch sent immediately without waiting. */ static void test_wal_stream_partial_batch(void) { char tmpdir[256]; snprintf(tmpdir, sizeof(tmpdir), "/tmp/xce_wal_stream_partial_%d_%ld", (int)getpid(), (long)time(NULL)); TEST_ASSERT_EQUAL_INT(0, mkdir(tmpdir, 0700)); /* Write 10 entries starting at LSN 5 */ int rc = write_test_wal_segment(tmpdir, /*start_lsn=*/5, /*n_entries=*/10); TEST_ASSERT_EQUAL_INT(0, rc); xce_wal_stream_request_hook = counting_wal_stream_hook; g_wal_stream_batch_count = 0; g_wal_stream_total_entries = 0; g_wal_stream_last_n_entries = 0; /* Stream entries [6..14]: standby_lsn=5, committed_lsn=14 */ rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir, /*p=*/NULL, /*leader_term=*/7, /*standby_lsn=*/5, /*committed_lsn=*/14); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "ha_stream_wal_to_standby failed"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_wal_stream_batch_count, "< 256 entries must produce exactly 1 partial batch"); TEST_ASSERT_EQUAL_INT_MESSAGE(9, g_wal_stream_total_entries, "9 entries in range [6..14] must be sent"); xce_wal_stream_request_hook = NULL; char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_wal_stream_partial_batch: PASS\n"); } /* * Test 3.4-C: empty range (standby_lsn == committed_lsn) sends zero batches. * Requirement 5.4: no batch is sent when there are no entries to stream. */ static void test_wal_stream_empty_range_no_batches(void) { char tmpdir[256]; snprintf(tmpdir, sizeof(tmpdir), "/tmp/xce_wal_stream_empty_%d_%ld", (int)getpid(), (long)time(NULL)); TEST_ASSERT_EQUAL_INT(0, mkdir(tmpdir, 0700)); xce_wal_stream_request_hook = counting_wal_stream_hook; g_wal_stream_batch_count = 0; g_wal_stream_total_entries = 0; /* standby_lsn == committed_lsn: nothing to stream */ int rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir, /*p=*/NULL, /*leader_term=*/7, /*standby_lsn=*/100, /*committed_lsn=*/100); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "Empty range must return 0"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_wal_stream_batch_count, "Empty range must produce zero batches"); xce_wal_stream_request_hook = NULL; char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_wal_stream_empty_range_no_batches: PASS\n"); } /* Write N entries starting at start_lsn where entries[checkpoint_idx] (0-based * within the range) is a WAL_CHECKPOINT marker (local-only, never a logical * Raft entry — persist_write_checkpoint_marker consumes a real LSN slot for * it, exactly like a live checkpoint firing mid-stream would). All other * entries are WAL_JOB_UPDATE. */ static int write_test_wal_segment_with_checkpoint_at(const char *wal_dir, uint64_t start_lsn, int n_entries, int checkpoint_idx) { char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), "%s/seg-%016" PRIx64 ".wal", wal_dir, start_lsn); int fd = open(seg_path, O_WRONLY | O_CREAT | O_TRUNC, 0600); if (fd < 0) return -1; for (int i = 0; i < n_entries; i++) { wal_entry_hdr_t hdr; memset(&hdr, 0, sizeof(hdr)); hdr.lsn = start_lsn + (uint64_t)i; hdr.type = (i == checkpoint_idx) ? WAL_CHECKPOINT : WAL_JOB_UPDATE; hdr.payload_len = 0; hdr.term = 1; ssize_t nw = write(fd, &hdr, sizeof(hdr)); if (nw != (ssize_t)sizeof(hdr)) { close(fd); return -1; } } close(fd); return 0; } /* Captures every streamed entry's lsn/entry_type/prev_log_index across all * batches, in send order. */ #define CAP_CKPT_MAX 32 static uint64_t g_cap_ckpt_lsn[CAP_CKPT_MAX]; static uint8_t g_cap_ckpt_type[CAP_CKPT_MAX]; static uint64_t g_cap_ckpt_prev_idx[CAP_CKPT_MAX]; static int g_cap_ckpt_count = 0; static int capture_all_entries_wal_stream_hook(endpoint_t *ep, xce_request_t *req) { (void)ep; if (!req || !req->body.walcatchupentrymsg) return 0; wal_catchup_entry_t *batch = (wal_catchup_entry_t *)req->body.walcatchupentrymsg; for (int i = 0; i < batch->n_entries && g_cap_ckpt_count < CAP_CKPT_MAX; i++) { wal_catchup_entry_t *e = batch->entries[i]; if (!e) continue; g_cap_ckpt_lsn[g_cap_ckpt_count] = e->lsn; g_cap_ckpt_type[g_cap_ckpt_count] = e->entry_type; g_cap_ckpt_prev_idx[g_cap_ckpt_count] = e->prev_log_index; g_cap_ckpt_count++; } return 0; } /* * The catch-up streamer must SKIP WAL_CHECKPOINT markers entirely — they are * local-only bookkeeping (persist_write_checkpoint_marker's own doc comment: * "replay_wal SKIPS checkpoint markers... the recovered index never holds a * checkpoint LSN; the live index must match"), never a logical Raft log * entry, and never sent on the live broadcast path either * (persist_wal_write_replicated rejects type==WAL_CHECKPOINT with -EINVAL). * * Fixture: LSN [1..5], LSN 3 is a checkpoint marker (consumes a real LSN slot, * exactly like a live checkpoint firing mid-range). Streaming standby_lsn=0, * committed_lsn=5 must emit exactly 4 real entries (1,2,4,5), never lsn=3. * * bug-1475 (UPDATED, supersedes the original assertion): the entry after the * gap (lsn=4) must now chain prev_log_index=3 -- the checkpoint's OWN lsn -- * NOT 2 (the last real entry before it). The original assertion here (prev_ * log_index=2, "not the skipped marker's lsn=3") encoded the OLD, DEFECTIVE * design: a receiving standby that has itself already applied the same * committed WAL_CONFIG_MARKER independently writes its OWN local checkpoint * at the identical LSN (persist_write_checkpoint_marker runs deterministically * on every member that applies the marker's commit, not just the sender), so * the checkpoint's LSN is a real, HELD position on both sides -- not a gap to * skip past. Never advancing the tracker across it left every subsequent * prev_log_index permanently one below the receiver's true log length, * producing a permanent post-checkpoint catch-up stall (ae_reject_logmismatch * forever, zero progress every heartbeat) -- root-caused and fixed as part of * bug-1475; see ha_apply_term_entry_via_library's bug-1475 fix (ha.c) for the * matching receive-side co-tracking half of this fix, and * test_bug1475_catchup_survives_midstream_checkpoint below for the full * end-to-end (send+receive) regression proof. * * RED before the ORIGINAL bug (pre-existing, already fixed): the streamer * built/sent a catchup_entry for every hdr.type with no filter at all, so * lsn=3/type=WAL_CHECKPOINT itself was shipped as a real entry. */ static void test_wal_stream_skips_checkpoint_marker(void) { char tmpdir[256]; snprintf(tmpdir, sizeof(tmpdir), "/tmp/xce_wal_stream_ckpt_%d_%ld", (int)getpid(), (long)time(NULL)); TEST_ASSERT_EQUAL_INT(0, mkdir(tmpdir, 0700)); int rc = write_test_wal_segment_with_checkpoint_at( tmpdir, /*start_lsn=*/1, /*n_entries=*/5, /*checkpoint_idx=*/2); TEST_ASSERT_EQUAL_INT(0, rc); xce_wal_stream_request_hook = capture_all_entries_wal_stream_hook; g_cap_ckpt_count = 0; rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir, /*p=*/NULL, /*leader_term=*/7, /*standby_lsn=*/0, /*committed_lsn=*/5); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "ha_stream_wal_to_standby failed"); TEST_ASSERT_EQUAL_INT_MESSAGE(4, g_cap_ckpt_count, "the checkpoint marker at lsn=3 must never be streamed as an entry " "(4 real entries: 1,2,4,5)"); for (int i = 0; i < g_cap_ckpt_count; i++) { TEST_ASSERT_NOT_EQUAL_MESSAGE(3, (int)g_cap_ckpt_lsn[i], "lsn=3 (the checkpoint marker) must never appear in a streamed entry"); TEST_ASSERT_NOT_EQUAL_MESSAGE((int)WAL_CHECKPOINT, (int)g_cap_ckpt_type[i], "no streamed entry may carry entry_type==WAL_CHECKPOINT"); } /* Entries in order must be lsn 1,2,4,5. */ TEST_ASSERT_EQUAL_UINT64_MESSAGE(1, g_cap_ckpt_lsn[0], "entry 0 must be lsn=1"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(2, g_cap_ckpt_lsn[1], "entry 1 must be lsn=2"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(4, g_cap_ckpt_lsn[2], "entry 2 must be lsn=4 (lsn=3 skipped)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(5, g_cap_ckpt_lsn[3], "entry 3 must be lsn=5"); /* bug-1475: lsn=4 must chain prev_log_index=3 -- the checkpoint's OWN lsn, * which is a real HELD position on any receiver that independently * applied the same committed marker -- NOT 2 (the stale, pre-fix value * that permanently wedged catch-up one entry later; see the updated * fixture doc comment above and test_bug1475_catchup_survives_ * midstream_checkpoint for the full send+receive proof). */ TEST_ASSERT_EQUAL_UINT64_MESSAGE(3, g_cap_ckpt_prev_idx[2], "bug-1475: lsn=4 must chain prev_log_index=3 (the checkpoint's own " "lsn, a real held position on both sides), not the stale lsn=2"); xce_wal_stream_request_hook = NULL; char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_wal_stream_skips_checkpoint_marker: PASS\n"); } /* ══════════════════════════════════════════════════════════════════════════ * bug-1475 — END-TO-END regression: a standby must NOT permanently stall * catching up across a mid-stream WAL_CHECKPOINT. * * This drives all THREE parts of the fix together through the REAL production * send+receive path — not just the sender's prev_log_index computation * (already pinned above) in isolation: * - SEND side (config_repl.c, ha_stream_wal_to_standby): the tracker now * advances across a skipped WAL_CHECKPOINT instead of freezing one entry * behind. * - RECEIVE side (ha.c, ha_apply_term_entry_via_library): JIT-resyncs * ha->raft.log_len from persist_durable_floor() before consuming each AE * — mirroring ha_sync_raft_log_from_recovery's own boot-time computation, * but applied live, on the same single AE-receive call path (no new lock, * no cross-thread mutation) — so a receiver whose OWN local checkpoint * write left log_len stale (bug-1466, deliberately NOT fixed at its root * by this change) still correctly logOk-accepts the checkpoint's own lsn * as a valid predecessor. * - TERM-INDEX side (persist.c, record_entry_term/entry_term_locked): * a THIRD, previously-undiscovered gap found by live instrumentation of * THIS test after the first two fixes above still left it RED. The first * two fixes both rely on `persist_entry_term(checkpoint_lsn)` staying * "absent" (0) FOREVER on every member (documented design intent in both * comments above and in persist_write_checkpoint_marker's own comment). * That was false: record_entry_term's geometric-growth realloc zero-filled * any skipped gap slot (e.g. the checkpoint's own, deliberately-unrecorded * LSN), and entry_term_locked's `lsn <= term_index_count` presence check * then read that zero-fill back as WAL_LEGACY_TERM (1), NOT absent (0), * the INSTANT any LATER lsn was recorded and grew term_index_count past * the gap — which happens immediately in this test (and in any live * cluster) once the leader appends its next real entry after the * checkpoint. This made the checkpoint LSN's "term" a TIME-DEPENDENT * value: a fresh read right after the checkpoint returned 0 (absent, as * designed), but the SAME read after a later entry was appended returned * 1 — so the sender (which streams AFTER its own later entries exist) * computed prev_log_term=1 for the entry right after the checkpoint, * while a receiver that had not yet appended anything past its own * checkpoint still (correctly, per design) read 0 for the same LSN — * logOk's equality check then spuriously mismatched (0 != 1) and rejected * forever. Fixed with a TERM_INDEX_UNSET sentinel (UINT64_MAX) that marks * a never-explicitly-written slot as absent PERMANENTLY, independent of * later array growth — see TERM_INDEX_UNSET's comment in persist.c for * the full mechanism. Confirmed via real fprintf instrumentation at the * streamer, the JIT resync, and raft.c's logOk check (added and removed * during this investigation) that prev_log_term flipped from 0 to 1 * exactly at the point lsn=4 was appended, before the persist.c fix; the * same instrumentation confirmed prev_log_term=0 on both sides and * log_ok=1 after the fix. * * Fixture: a REAL leader persist_t WAL [lsn=1 real, lsn=2 WAL_CHECKPOINT * (via the real persist_checkpoint()), lsn=3 real, lsn=4 real]. A REAL * receiver cluster_t independently applies the SAME committed prefix through * lsn=1 and independently writes its OWN local checkpoint at lsn=2 (exactly * mirroring persist_write_checkpoint_marker's deterministic per-member * trigger — every member that applies a committed WAL_CONFIG_MARKER * checkpoints at the identical LSN, not just the leader). The receiver's * raft.log_len is left at its STALE, pre-checkpoint value (1) — the exact * live-write-time gap bug-1466 documents (persist_write_checkpoint_marker * bypasses raft_log_append, so log_len never advances for a checkpoint on * its own) — this is the precondition the permanent stall needs. * * ha_stream_wal_to_standby streams (standby_lsn=2, committed_lsn=4] and a * capture-and-replay hook feeds every batch into the REAL * ha_apply_wal_catchup_entry receive entry point against the receiver — * exactly the wire hop xce_decode → ha_apply_wal_catchup_entry performs live. * * RED (pre-fix, verified live during this investigation by reverting the * persist.c term-index sentinel fix with the other two parts still in * place): the receiver * gets permanently stuck AT OR BEFORE the checkpoint boundary — lsn=3's * prev_log_index is computed one below where the checkpoint actually left * the receiver's log, so ha_apply_term_entry_via_library's accept-check * (persist_entry_term(p, e->lsn) against the ACTUAL write position) never * matches, no ACK is sent, and every subsequent entry's prev_log_index only * grows further past the receiver's true (stale) log_len — ae_reject_ * logmismatch forever, zero progress. A second stream of the IDENTICAL range * makes IDENTICAL zero progress, proving the wedge is permanent, not a * transient one-round lag. * * GREEN (post-fix): the receiver catches all the way up to lsn=4. * ══════════════════════════════════════════════════════════════════════════ */ /* Bring a node up as a raft FOLLOWER at `term` (mirrors t42_make_leader, * make_follower in test_term_safety_adapter.c — this file has no standby-role * helper of its own yet). */ static void t_bug1475_make_follower(cluster_t *c, uint64_t term, int n_peers) { atomic_store(&c->ha.role, CTRL_STANDBY); atomic_store(&c->ha.term, term); ha_init_raft(&c->ha, c, n_peers); c->ha.raft.term = term; c->ha.raft.role = RAFT_FOLLOWER; } /* Replay-into-receiver hook: feeds every streamed batch into the REAL * ha_apply_wal_catchup_entry receive path against g_bug1475_recv, exactly as * the wire hop (xce_decode) would on a live connection. */ static cluster_t *g_bug1475_recv = NULL; static int bug1475_replay_hook(endpoint_t *ep, xce_request_t *req) { (void)ep; if (!req || !req->body.walcatchupentrymsg || !g_bug1475_recv) return 0; wal_catchup_entry_t *batch = (wal_catchup_entry_t *)req->body.walcatchupentrymsg; ha_apply_wal_catchup_entry(&g_bug1475_recv->ha, &g_bug1475_recv->persist, batch, /*ep*/NULL, g_bug1475_recv); return 0; } static void test_bug1475_catchup_survives_midstream_checkpoint(void) { char ltmp[256], rtmp[256]; strcpy(ltmp, "/tmp/xce_cr_b1475_leader_XXXXXX"); strcpy(rtmp, "/tmp/xce_cr_b1475_recv_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(ltmp)); TEST_ASSERT_NOT_NULL(mkdtemp(rtmp)); cluster_t *leader = t42_make_cluster(ltmp); cluster_t *recv = t42_make_cluster(rtmp); TEST_ASSERT_NOT_NULL(leader); TEST_ASSERT_NOT_NULL(recv); enum { TERM = 3 }; t42_make_leader(leader, TERM, /*n_peers=*/3); t_bug1475_make_follower(recv, TERM, /*n_peers=*/3); /* Identical prefix on both members through lsn=1. persist_wal_async is a * raw WAL write and does not itself update raft.log_len (that only * happens via ha_primary_log_append/raft_log_append on the real commit * path) -- both members already correctly hold index 1 (e.g. from a * prior, successful AE exchange this test does not need to re-simulate), * so both are seeded here to match. */ persist_set_term(&leader->persist, TERM); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(&leader->persist, WAL_JOB_UPDATE, NULL, 0)); /* lsn=1 */ leader->ha.raft.log_len = 1; persist_set_term(&recv->persist, TERM); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(&recv->persist, WAL_JOB_UPDATE, NULL, 0)); /* lsn=1 */ recv->ha.raft.log_len = 1; /* Each member independently checkpoints at lsn=2 -- the SAME deterministic * side effect persist_checkpoint_marker_triggered fires on every member * that applies a committed WAL_CONFIG_MARKER, not just the leader. Force a * synchronous flush right after so persist_durable_floor() (which the * ha.c bug-1475 fix reads) reflects it immediately rather than waiting on * the background fsync_worker's own timer (cfg.wal_sync_ms=5000). */ TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(&leader->persist)); /* lsn=2 */ TEST_ASSERT_EQUAL_INT(0, persist_flush(&leader->persist)); TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(&recv->persist)); /* lsn=2 */ TEST_ASSERT_EQUAL_INT(0, persist_flush(&recv->persist)); /* bug-1466 (unfixed, deliberately NOT touched by this fix): the checkpoint * write bypassed raft_log_append on BOTH members, so log_len stayed at 1 * on both even though current_lsn moved to 3 -- exactly the precondition * the live permanent stall needs. */ TEST_ASSERT_EQUAL_UINT64(1, leader->ha.raft.log_len); TEST_ASSERT_EQUAL_UINT64(1, recv->ha.raft.log_len); TEST_ASSERT_EQUAL_UINT64(3, (uint64_t)leader->persist.current_lsn); TEST_ASSERT_EQUAL_UINT64(3, (uint64_t)recv->persist.current_lsn); /* Leader appends two more REAL entries after the checkpoint. */ persist_set_term(&leader->persist, TERM); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(&leader->persist, WAL_JOB_UPDATE, NULL, 0)); /* lsn=3 */ TEST_ASSERT_EQUAL_INT(0, persist_wal_async(&leader->persist, WAL_JOB_UPDATE, NULL, 0)); /* lsn=4 */ atomic_store_explicit(&leader->persist.committed_lsn, 4, memory_order_release); /* Drive the REAL catch-up streamer -> REAL receive path. standby_lsn=2: * this receiver already durably holds through the checkpoint's own lsn. */ g_bug1475_recv = recv; xce_wal_stream_request_hook = bug1475_replay_hook; int rc = ha_stream_wal_to_standby(/*ep=*/NULL, leader->persist.wal_path, &leader->persist, /*leader_term=*/TERM, /*standby_lsn=*/2, /*committed_lsn=*/4); xce_wal_stream_request_hook = NULL; g_bug1475_recv = NULL; TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "ha_stream_wal_to_standby failed"); uint64_t applied = atomic_load_explicit(&recv->ha.applied_lsn, memory_order_acquire); TEST_ASSERT_EQUAL_UINT64_MESSAGE(4, applied, "bug-1475: the standby must catch all the way up to lsn=4 across the " "mid-stream checkpoint, not stall permanently at/before it"); TEST_ASSERT_EQUAL_UINT64(4, recv->ha.raft.log_len); TEST_ASSERT_EQUAL_UINT64(TERM, persist_entry_term(&recv->persist, 4)); /* A SECOND identical stream (the real heartbeat-driven retry) must be a * clean no-op / re-confirmation, never regress applied_lsn -- proving * this is genuine convergence, not a one-round fluke. */ g_bug1475_recv = recv; xce_wal_stream_request_hook = bug1475_replay_hook; rc = ha_stream_wal_to_standby(/*ep=*/NULL, leader->persist.wal_path, &leader->persist, /*leader_term=*/TERM, /*standby_lsn=*/4, /*committed_lsn=*/4); xce_wal_stream_request_hook = NULL; g_bug1475_recv = NULL; TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_UINT64(4, atomic_load_explicit(&recv->ha.applied_lsn, memory_order_acquire)); t42_teardown(leader); t42_teardown(recv); printf("test_bug1475_catchup_survives_midstream_checkpoint: PASS\n"); } /* * bug-1028 — the catch-up streamer must source each streamed entry's term and * the AppendEntries prev_log_term from the AUTHORITATIVE per-LSN term index * (persist_entry_term), NOT the raw on-disk hdr.term. A legacy-0 entry has * hdr.term==0 on disk but persist_entry_term maps it to WAL_LEGACY_TERM(1) — * which is exactly what the standby's logOk and the library send path compute. * If the streamer sends the raw 0, prev_log_term(0) != standby's term(1) → logOk * rejects the new-term entry forever (the "stream WAL batches" wedge). * * Fixture: LSN 1 appended at term 0 (legacy), LSN 2 at term 3. Streaming * (standby_lsn=1, committed=2] must emit LSN 2 with term=3 and prev_log_term=1 * (NOT 0). RED before the fix (streamer sent hdr.term=0 for the predecessor). */ static uint64_t g_cap_term_for_lsn2 = 0; static uint64_t g_cap_prevterm_for_lsn2 = 0; static int g_cap_saw_lsn2 = 0; static int capture_terms_wal_stream_hook(endpoint_t *ep, xce_request_t *req) { (void)ep; if (!req || !req->body.walcatchupentrymsg) return 0; wal_catchup_entry_t *batch = (wal_catchup_entry_t *)req->body.walcatchupentrymsg; for (int i = 0; i < batch->n_entries; i++) { wal_catchup_entry_t *e = batch->entries[i]; if (e && e->lsn == 2) { g_cap_saw_lsn2 = 1; g_cap_term_for_lsn2 = e->term; g_cap_prevterm_for_lsn2 = e->prev_log_term; } } return 0; } static void test_wal_stream_term_from_index_not_raw_hdr(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr_streamterm_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = t42_make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); /* LSN 1: legacy entry (term 0 on disk → persist_entry_term maps to 1). */ persist_set_term(&c->persist, 0); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(&c->persist, WAL_JOB_UPDATE, NULL, 0)); /* LSN 2: real new-term entry (term 3). */ persist_set_term(&c->persist, 3); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(&c->persist, WAL_JOB_UPDATE, NULL, 0)); /* Sanity: the authoritative index normalises legacy-0 → 1, keeps 3. */ TEST_ASSERT_EQUAL_UINT64(WAL_LEGACY_TERM, persist_entry_term(&c->persist, 1)); TEST_ASSERT_EQUAL_UINT64(3, persist_entry_term(&c->persist, 2)); xce_wal_stream_request_hook = capture_terms_wal_stream_hook; g_cap_saw_lsn2 = 0; g_cap_term_for_lsn2 = 0; g_cap_prevterm_for_lsn2 = 0; /* Standby holds LSN 1, catching up to committed 2 — streams just LSN 2, * whose prev tuple references LSN 1 (the legacy entry). */ int rc = ha_stream_wal_to_standby(/*ep=*/NULL, c->persist.wal_path, &c->persist, /*leader_term=*/atomic_load_explicit( &c->ha.term, memory_order_acquire), /*standby_lsn=*/1, /*committed_lsn=*/2); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "ha_stream_wal_to_standby failed"); xce_wal_stream_request_hook = NULL; TEST_ASSERT_TRUE_MESSAGE(g_cap_saw_lsn2, "LSN 2 must be streamed"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(3, g_cap_term_for_lsn2, "streamed entry term must come from persist_entry_term (3)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(WAL_LEGACY_TERM, g_cap_prevterm_for_lsn2, "bug-1028: prev_log_term must be the index-normalised legacy term (1), " "NOT raw on-disk hdr.term (0) — else the standby's logOk rejects forever"); t42_teardown(c); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp); system(cmd); printf("test_wal_stream_term_from_index_not_raw_hdr: PASS\n"); } /* ── Task 3.5: Snapshot chunk transfer ──────────────────────── */ static snapshot_transfer_t g_snaptest_chunks[16]; static int g_snaptest_chunk_count = 0; static void snaptest_reset(void) { memset(g_snaptest_chunks, 0, sizeof(g_snaptest_chunks)); g_snaptest_chunk_count = 0; } static int snaptest_hook(endpoint_t *ep, xce_request_t *req) { (void)ep; if (g_snaptest_chunk_count >= 16) return -1; memcpy(&g_snaptest_chunks[g_snaptest_chunk_count], req->body.snapshottransfermsg, sizeof(snapshot_transfer_t)); g_snaptest_chunk_count++; return 0; } static void test_snapshot_transfer_two_chunks(void) { char tmpdir[] = "/tmp/xcectest_snaptrans_XXXXXX"; TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); char snap_path[512]; snprintf(snap_path, sizeof(snap_path), "%s/snap-0000000000000064.capnp", tmpdir); FILE *f = fopen(snap_path, "wb"); TEST_ASSERT_NOT_NULL(f); char header[16] = "XCESNAP\0"; uint64_t lsn = 100; memcpy(header + 8, &lsn, 8); fwrite(header, 1, 16, f); char zero[4096]; memset(zero, 0, sizeof(zero)); for (int i = 16; i < 131072; i += 4096) fwrite(zero, 1, (i + 4096 > 131072) ? (size_t)(131072 - i) : 4096, f); fclose(f); extern int (*xce_snapshot_chunk_hook)(endpoint_t *, xce_request_t *); xce_snapshot_chunk_hook = snaptest_hook; snaptest_reset(); extern int ha_send_snapshot_to_standby(endpoint_t *, const char *, uint64_t, uint64_t); int rc = ha_send_snapshot_to_standby(NULL, tmpdir, lsn, 0); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(2, g_snaptest_chunk_count); TEST_ASSERT_EQUAL_UINT64(100, g_snaptest_chunks[0].snapshot_lsn); TEST_ASSERT_EQUAL_UINT32(0, g_snaptest_chunks[0].chunk_index); TEST_ASSERT_EQUAL_UINT64(0, g_snaptest_chunks[0].chunk_offset); TEST_ASSERT_EQUAL_UINT64(131072, g_snaptest_chunks[0].total_size); TEST_ASSERT_EQUAL_INT(0, g_snaptest_chunks[0].is_last); TEST_ASSERT_EQUAL_UINT64(131072, g_snaptest_chunks[1].total_size); TEST_ASSERT_EQUAL_UINT32(1, g_snaptest_chunks[1].chunk_index); TEST_ASSERT_EQUAL_UINT64(65536, g_snaptest_chunks[1].chunk_offset); TEST_ASSERT_EQUAL_INT(1, g_snaptest_chunks[1].is_last); xce_snapshot_chunk_hook = NULL; char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_snapshot_transfer_two_chunks: PASS\n"); } /* ══════════════════════════════════════════════════════════════════════════ * bug-1475 Finding 2 (coordinator review, 2026-07-27/28): the SAME permanent * catch-up wedge bug-1475 fixed for WAL-streaming catch-up, reachable * instead via the SNAPSHOT-INSTALL path (HA_REPL_SEND_SNAPSHOT — dispatched * whenever a standby has fallen too far behind for incremental streaming, a * live, common path, not dead code). * * ha_snapshot_do_send (ha.c) computes checkpoint_term = persist_entry_term( * leader, checkpoint_lsn) — which, thanks to bug-1475's TERM_INDEX_UNSET * fix, is now PERMANENTLY 0 (absent) for any real checkpoint LSN * (persist_write_checkpoint_marker never records a term for its own entry, * by design). Before THIS fix, ha_handle_snapshot_chunk's * persist_reconcile_checkpoint_term(p, checkpoint_lsn, checkpoint_term) * wrote that 0 RAW into the term index via an EXPLICIT record_entry_term * call (not a growth-fill) — so entry_term_locked's legacy-0 rule read it * back as WAL_LEGACY_TERM(1), NOT absent, permanently diverging from the * leader's own permanently-absent(0) value for the IDENTICAL LSN. A * subsequent AppendEntries whose prev_log_index==checkpoint_lsn (the entry * immediately following it) then mismatched logOk (0 != 1) forever. * * RED (pre-fix, verified by temporarily reverting persist_reconcile_ * checkpoint_term's TERM_INDEX_UNSET substitution): the receiver's * checkpoint-boundary term reads back 1 (WAL_LEGACY_TERM), not 0, and the * subsequent AppendEntries is rejected — applied_lsn never reaches 3. * GREEN (post-fix): the receiver's checkpoint-boundary term reads back 0 * (absent, matching the leader), and the subsequent AppendEntries is * accepted. * ══════════════════════════════════════════════════════════════════════════ */ static cluster_t *g_bug1475f2_recv = NULL; static int bug1475f2_snapshot_replay_hook(endpoint_t *ep, xce_request_t *req) { (void)ep; if (!req || !req->body.snapshottransfermsg || !g_bug1475f2_recv) return 0; snapshot_transfer_t *chunk = (snapshot_transfer_t *)req->body.snapshottransfermsg; return ha_handle_snapshot_chunk(&g_bug1475f2_recv->ha, &g_bug1475f2_recv->persist, g_bug1475f2_recv->persist.snap_path, chunk, chunk->raw_payload, chunk->raw_payload_len); } static void test_bug1475_finding2_snapshot_install_checkpoint_term_absent(void) { char ltmp[256], rtmp[256]; strcpy(ltmp, "/tmp/xce_cr_b1475f2_leader_XXXXXX"); strcpy(rtmp, "/tmp/xce_cr_b1475f2_recv_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(ltmp)); TEST_ASSERT_NOT_NULL(mkdtemp(rtmp)); cluster_t *leader = t42_make_cluster(ltmp); cluster_t *recv = t42_make_cluster(rtmp); TEST_ASSERT_NOT_NULL(leader); TEST_ASSERT_NOT_NULL(recv); extern int (*xce_snapshot_chunk_hook)(endpoint_t *, xce_request_t *); extern int ha_send_snapshot_to_standby(endpoint_t *, const char *, uint64_t, uint64_t); enum { TERM = 5 }; t42_make_leader(leader, TERM, /*n_peers=*/3); t_bug1475_make_follower(recv, TERM, /*n_peers=*/3); /* ha_handle_snapshot_chunk takes ha->lock directly; t42_make_cluster / * ha_init_raft never call pthread_mutex_init on it (only the AE-receive * path is exercised by the OTHER bug-1475 test, which never locks it) — * initialize it explicitly here rather than relying on a zeroed mutex * happening to behave like PTHREAD_MUTEX_INITIALIZER. */ pthread_mutex_init(&recv->ha.lock, NULL); /* Leader: a real entry at lsn=1, then a REAL checkpoint at lsn=2 via * persist_checkpoint -> persist_write_checkpoint_marker (the exact * production path — not a synthetic marker). * * p->checkpoint_lsn is NOT the WAL_CHECKPOINT entry's own physical lsn — * persist_checkpoint() (the general/periodic form ha_snapshot_do_send * actually calls) labels the snapshot with last_applied_lsn, read * immediately AFTER persist_write_checkpoint_marker has already * consumed the next lsn slot for the checkpoint itself. In a * continuously-active system checkpoint_lsn is usually one BELOW the * checkpoint's own physical lsn (the last real entry applied so far); * Finding 2 requires the specific (real, reachable) quiescent-state * timing where the apply pump has already walked all the way through * and landed exactly ON the checkpoint's own lsn (persist_apply_pump_run * advances last_applied_lsn one lsn at a time, INCLUDING a no-op pass * over a WAL_CHECKPOINT slot, before this snapshot-send cycle runs) — * e.g. right after a config upgrade with no immediate follow-up * activity. Seed that precondition directly (deterministic; the real * race depends on independent apply-pump timing this test does not * need to reproduce) so persist_checkpoint()'s OWN checkpoint_lsn * coincides with the checkpoint entry's own un-recorded lsn — exactly * the case where persist_entry_term(checkpoint_lsn) reads genuinely * absent. */ persist_set_term(&leader->persist, TERM); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(&leader->persist, WAL_JOB_UPDATE, NULL, 0)); /* lsn=1 */ TEST_ASSERT_EQUAL_UINT64(2, (uint64_t)leader->persist.current_lsn); atomic_store_explicit(&leader->persist.last_applied_lsn, 2, memory_order_release); TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(&leader->persist)); /* lsn=2 */ TEST_ASSERT_EQUAL_INT(0, persist_flush(&leader->persist)); TEST_ASSERT_EQUAL_UINT64_MESSAGE(2, leader->persist.checkpoint_lsn, "precondition: checkpoint_lsn must coincide with the checkpoint " "entry's own physical lsn for this test to exercise Finding 2 " "(otherwise it would read a real, non-absent term instead)"); /* Leader appends one more real entry AFTER the checkpoint — mirrors the * real timing that exposed Finding 2 (persist_entry_term(checkpoint_lsn) * read right after the checkpoint vs. after later growth exists). */ persist_set_term(&leader->persist, TERM); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(&leader->persist, WAL_JOB_UPDATE, NULL, 0)); /* lsn=3 */ /* Precondition: exactly what ha_snapshot_do_send's real computation * would read for the wire — genuinely absent (0), permanently, even * after later growth (TERM_INDEX_UNSET's whole point). */ uint64_t checkpoint_term = persist_entry_term(&leader->persist, leader->persist.checkpoint_lsn); TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, checkpoint_term, "precondition: a real checkpoint LSN's term reads absent (0) on the " "sender, even after later entries exist — the exact value " "ha_snapshot_do_send would put on the wire"); /* Drive the REAL snapshot send -> receive path: ha_send_snapshot_to_standby * chunks the on-disk snapshot file (written by persist_checkpoint above) * and stamps checkpoint_term on every chunk; the capture-and-replay hook * feeds each chunk into the REAL ha_handle_snapshot_chunk on the * receiver, exactly the wire hop a live transfer performs. */ g_bug1475f2_recv = recv; xce_snapshot_chunk_hook = bug1475f2_snapshot_replay_hook; int rc = ha_send_snapshot_to_standby(/*ep=*/NULL, leader->persist.snap_path, leader->persist.checkpoint_lsn, checkpoint_term); xce_snapshot_chunk_hook = NULL; g_bug1475f2_recv = NULL; TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "ha_send_snapshot_to_standby failed"); TEST_ASSERT_EQUAL_UINT64(leader->persist.checkpoint_lsn, recv->persist.checkpoint_lsn); /* The receiver must reconcile the checkpoint boundary LSN's term to * match the leader's — genuinely absent (0), NOT WAL_LEGACY_TERM(1). */ TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, persist_entry_term(&recv->persist, leader->persist.checkpoint_lsn), "bug-1475 Finding 2: the receiver's checkpoint-boundary term must " "reconcile to absent (0), matching the leader's permanently-absent " "value for the SAME lsn — NOT get reinterpreted as " "WAL_LEGACY_TERM(1) by persist_reconcile_checkpoint_term's " "record_entry_term(..., 0) call"); /* Isolate Finding 2's term-index concern from two OTHER, SEPARATE, * pre-existing (unchanged by this fix) mechanics this test is not about: * * 1. ha_handle_snapshot_chunk does not itself resync ha->raft.log_len * (only ha_apply_term_entry_via_library's JIT resync does, on the * NEXT AE) — set it directly to the checkpoint's own lsn here, the * value a real deployment reaches once durable_lsn catches up (or a * later boot-reseed runs). * 2. load_snapshot() sets p->current_lsn = stored_lsn (the checkpoint's * OWN lsn), NOT stored_lsn + 1 — confirmed via direct instrumentation * (current_lsn read back as 2, matching checkpoint_lsn, immediately * after this install). persist_wal_write_replicated's forward-gap * guard (lsn > current_lsn -> EAGAIN) would then reject ANY entry at * checkpoint_lsn+1 regardless of term, confounding this test's * specific term question. Whether that current_lsn convention * correctly bridges into the next live catch-up cycle in production * is a DIFFERENT, pre-existing question this test does not answer — * flagged here, not fixed (out of Finding 2's scope: the coordinator's * review identified a term(0-vs-1) mismatch specifically, not a * current_lsn contiguity gap). Bypass it directly so only the term * question is under test. */ recv->ha.raft.log_len = leader->persist.checkpoint_lsn; recv->persist.current_lsn = leader->persist.checkpoint_lsn + 1; /* Drive a REAL subsequent AppendEntries whose prev_log_index is the * checkpoint's own LSN — the entry immediately following it. */ wal_catchup_entry_t sub; memset(&sub, 0, sizeof(sub)); sub.lsn = 3; sub.entry_type = WAL_JOB_UPDATE; sub.term = TERM; sub.prev_log_index = leader->persist.checkpoint_lsn; /* == 2 */ sub.prev_log_term = checkpoint_term; /* == 0, absent */ sub.leader_current_term = TERM; wal_catchup_entry_t *sub_arr[1] = { &sub }; wal_catchup_entry_t batch; memset(&batch, 0, sizeof(batch)); batch.n_entries = 1; batch.entries = sub_arr; batch.term = TERM; batch.prev_log_index = sub.prev_log_index; batch.prev_log_term = sub.prev_log_term; rc = ha_apply_wal_catchup_entry(&recv->ha, &recv->persist, &batch, /*ep*/NULL, recv); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_UINT64_MESSAGE(3, atomic_load_explicit(&recv->ha.applied_lsn, memory_order_acquire), "bug-1475 Finding 2: the standby must accept the AppendEntries " "immediately following the checkpoint boundary after a snapshot " "install, not permanently reject it (ae_reject_logmismatch) due to " "a term(0) vs term(1) mismatch at the checkpoint LSN"); TEST_ASSERT_EQUAL_UINT64(TERM, persist_entry_term(&recv->persist, 3)); pthread_mutex_destroy(&recv->ha.lock); t42_teardown(leader); t42_teardown(recv); printf("test_bug1475_finding2_snapshot_install_checkpoint_term_absent: PASS\n"); } static void test_snapshot_transfer_small_less_than_one_chunk(void) { char tmpdir[] = "/tmp/xcectest_snaptrans_XXXXXX"; TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); char snap_path[512]; snprintf(snap_path, sizeof(snap_path), "%s/snap-000000000000000a.capnp", tmpdir); FILE *f = fopen(snap_path, "wb"); TEST_ASSERT_NOT_NULL(f); char buf[64]; memset(buf, 0, sizeof(buf)); memcpy(buf, "XCESNAP\0", 8); uint64_t lsn = 10; memcpy(buf + 8, &lsn, 8); fwrite(buf, 1, sizeof(buf), f); fclose(f); extern int (*xce_snapshot_chunk_hook)(endpoint_t *, xce_request_t *); xce_snapshot_chunk_hook = snaptest_hook; snaptest_reset(); extern int ha_send_snapshot_to_standby(endpoint_t *, const char *, uint64_t, uint64_t); int rc = ha_send_snapshot_to_standby(NULL, tmpdir, lsn, 0); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(1, g_snaptest_chunk_count); TEST_ASSERT_EQUAL_UINT64(10, g_snaptest_chunks[0].snapshot_lsn); TEST_ASSERT_EQUAL_UINT32(0, g_snaptest_chunks[0].chunk_index); TEST_ASSERT_EQUAL_UINT64(64, g_snaptest_chunks[0].total_size); TEST_ASSERT_EQUAL_INT(1, g_snaptest_chunks[0].is_last); xce_snapshot_chunk_hook = NULL; char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_snapshot_transfer_small_less_than_one_chunk: PASS\n"); } /* bug-1386: a chunk send that fails mid-transfer (e.g. -ECONNRESET while the * standby endpoint is mid-teardown/reconnect) must stop the loop and be * propagated, not be silently swallowed while the function still reports * "transfer complete". */ static int g_snaptest_fail_after_calls = -1; /* -1 = never fail */ static int snaptest_fail_after_hook(endpoint_t *ep, xce_request_t *req) { (void)ep; if (g_snaptest_chunk_count >= 16) return -1; memcpy(&g_snaptest_chunks[g_snaptest_chunk_count], req->body.snapshottransfermsg, sizeof(snapshot_transfer_t)); g_snaptest_chunk_count++; if (g_snaptest_fail_after_calls >= 0 && g_snaptest_chunk_count > g_snaptest_fail_after_calls) return -ECONNRESET; return 0; } static void test_snapshot_transfer_stops_and_propagates_send_failure(void) { char tmpdir[] = "/tmp/xcectest_snaptrans_XXXXXX"; TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); /* 3 chunks: 2 full 64KB chunks + a partial third, so a mid-stream * failure on chunk 1 leaves chunk 2 (the would-be is_last chunk) * unsent. */ char snap_path[512]; snprintf(snap_path, sizeof(snap_path), "%s/snap-00000000000000c8.capnp", tmpdir); FILE *f = fopen(snap_path, "wb"); TEST_ASSERT_NOT_NULL(f); char header[16] = "XCESNAP\0"; uint64_t lsn = 200; memcpy(header + 8, &lsn, 8); fwrite(header, 1, 16, f); char zero[4096]; memset(zero, 0, sizeof(zero)); size_t total = 65536 + 65536 + 4096; for (size_t i = 16; i < total; i += 4096) fwrite(zero, 1, (i + 4096 > total) ? (total - i) : 4096, f); fclose(f); extern int (*xce_snapshot_chunk_hook)(endpoint_t *, xce_request_t *); xce_snapshot_chunk_hook = snaptest_fail_after_hook; snaptest_reset(); g_snaptest_fail_after_calls = 1; /* chunk_index 0 succeeds, chunk_index 1 fails */ extern int ha_send_snapshot_to_standby(endpoint_t *, const char *, uint64_t, uint64_t); int rc = ha_send_snapshot_to_standby(NULL, tmpdir, lsn, 0); TEST_ASSERT_EQUAL_INT(-ECONNRESET, rc); /* Loop must stop right after the failing chunk — chunk_index 2 (the * is_last chunk) must never be attempted. */ TEST_ASSERT_EQUAL_INT(2, g_snaptest_chunk_count); TEST_ASSERT_EQUAL_INT(0, g_snaptest_chunks[1].is_last); xce_snapshot_chunk_hook = NULL; g_snaptest_fail_after_calls = -1; char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_snapshot_transfer_stops_and_propagates_send_failure: PASS\n"); } /* ── Task 3.5c: Standby snapshot receive handler ──────────────────────────── */ /* Helper: initialise a minimal ha_state_t for snapshot receive tests */ static void snap_recv_ha_init(ha_state_t *ha) { memset(ha, 0, sizeof(*ha)); ha_config_t cfg; memset(&cfg, 0, sizeof(cfg)); cfg.heartbeat_timeout_ms = 10000; cfg.election_timeout_ms = 1500; ha_init(ha, &cfg, 0); } /* Helper: write a minimal valid snapshot file to path (XCESNAP header + body) */ static void write_test_snapshot(const char *path, uint64_t lsn, size_t body_size) { FILE *f = fopen(path, "wb"); TEST_ASSERT_NOT_NULL_MESSAGE(f, "Failed to create test snapshot file"); char header[16]; memcpy(header, "XCESNAP\0", 8); memcpy(header + 8, &lsn, 8); fwrite(header, 1, 16, f); if (body_size > 16) { char *zeros = calloc(1, body_size - 16); fwrite(zeros, 1, body_size - 16, f); free(zeros); } fclose(f); } /* Test 3.5c-A: receive 2 chunks, verify snap.part assembled and renamed correctly, * applied_lsn set to checkpoint_lsn. * Requirements: 4.2, 4.3 */ static void test_snapshot_recv_two_chunks_assembles_and_renames(void) { char tmpdir[] = "/tmp/xcectest_snaprecv_XXXXXX"; TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); /* Create snap_dir subdirectory (matches persist->snap_path layout) */ char snap_dir[512]; snprintf(snap_dir, sizeof(snap_dir), "%s/snap", tmpdir); TEST_ASSERT_EQUAL_INT(0, mkdir(snap_dir, 0700)); /* Build a fake 128-byte snapshot body (two 64-byte chunks). * payload_bytes contains the raw file bytes for each chunk. * We skip writing an actual snapshot source file — the receiver * writes snap.part and renames it; we use the resulting file to verify. * * The body must be a WELL-FORMED entity-bearing checkpoint, not opaque * filler (ha-entity-snapshots task 5.2): load_snapshot() now actually * parses everything past the legacy header — a 0xAA/0xBB filler tail * used to be silently ignored (load_snapshot only ever read the 57-byte * header) but is now correctly rejected as an unrecognized * format_version. Layout (design.md "Checkpoint File Layout"): the * 57-byte legacy header (cfgref_present=0 at byte 16, so bytes 17-56 are * unread and left zero) + format_version=2 + job_count=0 + queue_count=0 * (69 bytes total), padded with zero filler out to 128 bytes to keep the * two-64-byte-chunk shape this test exercises — the padding is never * read (load_snapshot_entities stops right after the queue_count=0 * loop), so its content is immaterial. */ uint64_t snap_lsn = 100; size_t total_size = 128; uint8_t body[128]; memset(body, 0, sizeof(body)); memcpy(body, "XCESNAP\0", 8); memcpy(body + 8, &snap_lsn, 8); /* body[16] (cfgref_present) left 0 by the memset above. */ uint32_t fake_fmt = 2u; /* must match persist.c's SNAPSHOT_FORMAT_VERSION * (config-upgrade-snapshot task 1 bumped 1->2) */ uint32_t fake_job_count = 0u; uint32_t fake_queue_count = 0u; memcpy(body + 57, &fake_fmt, sizeof(fake_fmt)); memcpy(body + 61, &fake_job_count, sizeof(fake_job_count)); memcpy(body + 65, &fake_queue_count, sizeof(fake_queue_count)); uint8_t chunk0[64]; uint8_t chunk1[64]; memcpy(chunk0, body, 64); memcpy(chunk1, body + 64, 64); persist_t *p = calloc(1, sizeof(persist_t)); TEST_ASSERT_NOT_NULL(p); persist_config_t pcfg = persist_config_defaults(); persist_set_workdir(p, tmpdir, "ctrl-snaprecv"); persist_init(p, &pcfg, NULL); persist_start(p); ha_state_t ha; snap_recv_ha_init(&ha); /* Chunk 0 */ snapshot_transfer_t c0; memset(&c0, 0, sizeof(c0)); c0.snapshot_lsn = snap_lsn; c0.chunk_index = 0; c0.chunk_offset = 0; c0.total_size = (uint64_t)total_size; c0.is_last = 0; int rc = ha_handle_snapshot_chunk(&ha, p, snap_dir, &c0, chunk0, 64); TEST_ASSERT_EQUAL_INT(0, rc); /* snap.part must exist after first chunk */ char part_path[600]; snprintf(part_path, sizeof(part_path), "%s/snap.part", snap_dir); struct stat st; TEST_ASSERT_EQUAL_INT_MESSAGE(0, stat(part_path, &st), "snap.part should exist after chunk 0"); TEST_ASSERT_EQUAL_UINT64(64, (uint64_t)st.st_size); /* Chunk 1 (isLast=true) */ snapshot_transfer_t c1; memset(&c1, 0, sizeof(c1)); c1.snapshot_lsn = snap_lsn; c1.chunk_index = 1; c1.chunk_offset = 64; c1.total_size = (uint64_t)total_size; c1.is_last = 1; rc = ha_handle_snapshot_chunk(&ha, p, snap_dir, &c1, chunk1, 64); TEST_ASSERT_EQUAL_INT(0, rc); /* snap.part must be gone (renamed) */ TEST_ASSERT_NOT_EQUAL_MESSAGE(0, stat(part_path, &st), "snap.part should be gone after isLast"); /* Final file snap-{lsn:016x}.bin must exist */ char final_path[600]; snprintf(final_path, sizeof(final_path), "%s/snap-%016" PRIx64 ".bin", snap_dir, snap_lsn); TEST_ASSERT_EQUAL_INT_MESSAGE(0, stat(final_path, &st), "snap-{lsn}.bin should exist after isLast"); TEST_ASSERT_EQUAL_UINT64((uint64_t)total_size, (uint64_t)st.st_size); /* applied_lsn must be set to checkpoint_lsn (= snap_lsn = 100) */ uint64_t got_lsn = (uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire); TEST_ASSERT_EQUAL_UINT64(snap_lsn, got_lsn); ha_fini(&ha); persist_stop(p); persist_fini(p); free(p); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_snapshot_recv_two_chunks_assembles_and_renames: PASS\n"); } /* config-upgrade-snapshot task 1 (Req 1.9, 5.7): SNAPSHOT_FORMAT_VERSION was * bumped 1->2 to reserve the version for the HostAdminState/QueueAdminState * capnp migration of the host/queue admin-state frames. A checkpoint written * under the PRIOR format_version (1 -- the raw queue-frame layout this spec * migrates off of) must be hard-rejected fail-loud by a binary built against * the bumped SNAPSHOT_FORMAT_VERSION=2, never silently reinterpreted (Design * Invariant 4; persist.c:2541's existing `fmt != SNAPSHOT_FORMAT_VERSION` * check, unchanged logic, new constant). Same well-formed-except-for- * format-version fixture shape as * test_snapshot_recv_two_chunks_assembles_and_renames above, but the * embedded format_version is pinned to the literal PRIOR value (1) * regardless of whatever persist.c's SNAPSHOT_FORMAT_VERSION currently is — * this is exactly the "old on-disk checkpoint, new binary" skew the version * bump must reject. */ static void test_snapshot_recv_old_format_version_rejected_fail_loud(void) { char tmpdir[] = "/tmp/xcectest_snaprecv_oldfmt_XXXXXX"; TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); char snap_dir[512]; snprintf(snap_dir, sizeof(snap_dir), "%s/snap", tmpdir); TEST_ASSERT_EQUAL_INT(0, mkdir(snap_dir, 0700)); uint64_t snap_lsn = 200; size_t total_size = 128; uint8_t body[128]; memset(body, 0, sizeof(body)); memcpy(body, "XCESNAP\0", 8); memcpy(body + 8, &snap_lsn, 8); /* body[16] (cfgref_present) left 0 by the memset above. */ uint32_t old_fmt = 1u; /* the pre-migration raw queue-frame layout -- * deliberately NOT persist.c's current * SNAPSHOT_FORMAT_VERSION */ uint32_t fake_job_count = 0u; uint32_t fake_queue_count = 0u; memcpy(body + 57, &old_fmt, sizeof(old_fmt)); memcpy(body + 61, &fake_job_count, sizeof(fake_job_count)); memcpy(body + 65, &fake_queue_count, sizeof(fake_queue_count)); uint8_t chunk0[64]; uint8_t chunk1[64]; memcpy(chunk0, body, 64); memcpy(chunk1, body + 64, 64); persist_t *p = calloc(1, sizeof(persist_t)); TEST_ASSERT_NOT_NULL(p); persist_config_t pcfg = persist_config_defaults(); persist_set_workdir(p, tmpdir, "ctrl-snaprecv-oldfmt"); persist_init(p, &pcfg, NULL); persist_start(p); ha_state_t ha; snap_recv_ha_init(&ha); /* Chunk 0 — assembling snap.part never itself parses the body, so this * must still succeed regardless of what format_version the bytes carry. */ snapshot_transfer_t c0; memset(&c0, 0, sizeof(c0)); c0.snapshot_lsn = snap_lsn; c0.chunk_index = 0; c0.chunk_offset = 0; c0.total_size = (uint64_t)total_size; c0.is_last = 0; int rc = ha_handle_snapshot_chunk(&ha, p, snap_dir, &c0, chunk0, 64); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "assembling a chunk never parses the body"); /* Chunk 1 (isLast=true) triggers rename + persist_load_snapshot — this is * where the format-version check fires and must fail loud. */ snapshot_transfer_t c1; memset(&c1, 0, sizeof(c1)); c1.snapshot_lsn = snap_lsn; c1.chunk_index = 1; c1.chunk_offset = 64; c1.total_size = (uint64_t)total_size; c1.is_last = 1; rc = ha_handle_snapshot_chunk(&ha, p, snap_dir, &c1, chunk1, 64); TEST_ASSERT_TRUE_MESSAGE(rc != 0, "a checkpoint encoded under the prior SNAPSHOT_FORMAT_VERSION must be " "rejected fail-loud by a binary built against the bumped version, " "never silently reinterpreted (Requirement 5.7)"); /* A rejected load must never advance applied_lsn (no partial/silent * acceptance of mismatched-format bytes). */ uint64_t got_lsn = (uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire); TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, got_lsn, "a rejected format-mismatched checkpoint must never advance applied_lsn"); ha_fini(&ha); persist_stop(p); persist_fini(p); free(p); char cmd2[512]; snprintf(cmd2, sizeof(cmd2), "rm -rf %s", tmpdir); system(cmd2); printf("test_snapshot_recv_old_format_version_rejected_fail_loud: PASS\n"); } /* Test 6.1: a legacy (pre-ha-entity-snapshots) position-only snapshot — * exactly the 16-byte XCESNAP magic+lsn header, no entity payload — driven * through the real receive handler end-to-end. persist_load_snapshot's * entities_restored out-param comes back false (design.md "Checkpoint File * Layout": "does more data exist" is the legacy discriminator), and * ha_handle_snapshot_chunk must still: (a) return rc==0 (a legacy file is * not a failure, Requirement 2.4/5.1/5.2), and (b) advance ha->applied_lsn * exactly as it does for an entity-bearing file (Requirement 4.1/4.2 — * the applied_lsn gating is unchanged by task 6.1; only a new informational * log line is added on this path). * Requirements: 2.4, 4.1, 4.2, 5.1, 5.2 */ static void test_snapshot_recv_legacy_file_advances_without_entities(void) { char tmpdir[] = "/tmp/xcectest_snaprecv_legacy_XXXXXX"; TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); char snap_dir[512]; snprintf(snap_dir, sizeof(snap_dir), "%s/snap", tmpdir); TEST_ASSERT_EQUAL_INT(0, mkdir(snap_dir, 0700)); /* Legacy body: magic(8) + checkpoint_lsn(8), nothing else — the file * ends exactly at the legacy header, so load_snapshot's "does more data * exist" discriminator reports this as legacy (entities_restored=false) * rather than corrupt or entity-bearing. */ uint64_t snap_lsn = 300; uint8_t body[16]; memcpy(body, "XCESNAP\0", 8); memcpy(body + 8, &snap_lsn, 8); persist_t *p = calloc(1, sizeof(persist_t)); TEST_ASSERT_NOT_NULL(p); persist_config_t pcfg = persist_config_defaults(); persist_set_workdir(p, tmpdir, "ctrl-snaprecv-legacy"); persist_init(p, &pcfg, NULL); persist_start(p); ha_state_t ha; snap_recv_ha_init(&ha); snapshot_transfer_t c0; memset(&c0, 0, sizeof(c0)); c0.snapshot_lsn = snap_lsn; c0.chunk_index = 0; c0.chunk_offset = 0; c0.total_size = sizeof(body); c0.is_last = 1; int rc = ha_handle_snapshot_chunk(&ha, p, snap_dir, &c0, body, sizeof(body)); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "a legacy position-only checkpoint must not be treated as a failure"); /* Final file must exist (rename still happens on the legacy path) */ char final_path[600]; snprintf(final_path, sizeof(final_path), "%s/snap-%016" PRIx64 ".bin", snap_dir, snap_lsn); struct stat st; TEST_ASSERT_EQUAL_INT_MESSAGE(0, stat(final_path, &st), "snap-{lsn}.bin should exist after isLast"); /* applied_lsn advances to the legacy file's recorded position — same * rc-gates-applied_lsn ordering as the entity-bearing path, unchanged by * task 6.1 (Requirement 4.1, 4.2). */ uint64_t got_lsn = (uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire); TEST_ASSERT_EQUAL_UINT64(snap_lsn, got_lsn); ha_fini(&ha); persist_stop(p); persist_fini(p); free(p); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_snapshot_recv_legacy_file_advances_without_entities: PASS\n"); } /* Test 3.5c-B: size mismatch on isLast returns error (verify size == total_size). * Requirements: 4.3 */ static void test_snapshot_recv_size_mismatch_fails(void) { char tmpdir[] = "/tmp/xcectest_snaprecv2_XXXXXX"; TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); char snap_dir[512]; snprintf(snap_dir, sizeof(snap_dir), "%s/snap", tmpdir); TEST_ASSERT_EQUAL_INT(0, mkdir(snap_dir, 0700)); persist_t *p = calloc(1, sizeof(persist_t)); TEST_ASSERT_NOT_NULL(p); persist_config_t pcfg = persist_config_defaults(); persist_set_workdir(p, tmpdir, "ctrl-snaprecv2"); persist_init(p, &pcfg, NULL); persist_start(p); ha_state_t ha; snap_recv_ha_init(&ha); uint64_t snap_lsn = 200; uint8_t chunk0[64]; memset(chunk0, 0xCC, 64); /* Claim total_size=200 but only send 64 bytes with is_last=true */ snapshot_transfer_t c0; memset(&c0, 0, sizeof(c0)); c0.snapshot_lsn = snap_lsn; c0.chunk_index = 0; c0.chunk_offset = 0; c0.total_size = 200; /* mismatch — only 64 bytes will be written */ c0.is_last = 1; int rc = ha_handle_snapshot_chunk(&ha, p, snap_dir, &c0, chunk0, 64); /* Must fail — size mismatch */ TEST_ASSERT_NOT_EQUAL_MESSAGE(0, rc, "size mismatch must return error"); /* applied_lsn must remain 0 (not advanced on failure) */ uint64_t got_lsn = (uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire); TEST_ASSERT_EQUAL_UINT64(0, got_lsn); ha_fini(&ha); persist_stop(p); persist_fini(p); free(p); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_snapshot_recv_size_mismatch_fails: PASS\n"); } /* Test 3.5c-C: NULL snap_dir or NULL chunk returns error gracefully */ static void test_snapshot_recv_null_args_fail(void) { ha_state_t ha; snap_recv_ha_init(&ha); int rc = ha_handle_snapshot_chunk(&ha, NULL, NULL, NULL, NULL, 0); TEST_ASSERT_NOT_EQUAL(0, rc); ha_fini(&ha); printf("test_snapshot_recv_null_args_fail: PASS\n"); } /* Test 6.4-A: 128KB snapshot in 2 equal chunks (65536 bytes each) — standby * receives both chunks, verifies total_size matches, snap.part is atomically * renamed to snap-{lsn}.bin, persist_load_snapshot is called successfully, * and applied_lsn equals the checkpoint_lsn embedded in the snapshot. * * This is the primary task-6.4 coverage test for Req 4.2, 4.3. * Uses the same 65536-byte chunk size as the sender test * (test_snapshot_transfer_two_chunks) to exercise the full receive path. */ static void test_snapshot_recv_128kb_two_chunks(void) { char tmpdir[] = "/tmp/xcectest_snap128kb_XXXXXX"; TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); char snap_dir[512]; snprintf(snap_dir, sizeof(snap_dir), "%s/snap", tmpdir); TEST_ASSERT_EQUAL_INT(0, mkdir(snap_dir, 0700)); persist_t *p = calloc(1, sizeof(persist_t)); TEST_ASSERT_NOT_NULL(p); persist_config_t pcfg = persist_config_defaults(); persist_set_workdir(p, tmpdir, "ctrl-snap128kb"); persist_init(p, &pcfg, NULL); persist_start(p); ha_state_t ha; snap_recv_ha_init(&ha); /* 128KB snapshot split into two 65536-byte chunks */ const uint64_t snap_lsn = 0x64; /* LSN 100 — matches sender test */ const size_t chunk_size = 65536; const size_t total_size = 131072; /* 128 * 1024 */ /* Allocate and zero-fill chunks */ uint8_t *chunk0 = calloc(1, chunk_size); uint8_t *chunk1 = calloc(1, chunk_size); TEST_ASSERT_NOT_NULL(chunk0); TEST_ASSERT_NOT_NULL(chunk1); /* Embed valid XCESNAP header in chunk0 so persist_load_snapshot succeeds. * chunk0/chunk1 are otherwise zero-filled, but a well-formed entity- * bearing checkpoint (ha-entity-snapshots task 5.2) needs format_version * = SNAPSHOT_FORMAT_VERSION right after the 57-byte legacy header, or * load_snapshot() correctly rejects an all-zero format_version as * unrecognized rather than silently ignoring the body as before. The * job_count=0/queue_count=0 that follow are already satisfied by the * zero-fill; the rest of the 128KB body is never read (load_snapshot * stops right after the queue_count=0 loop) so it stays harmless * filler. */ memcpy(chunk0, "XCESNAP\0", 8); memcpy(chunk0 + 8, &snap_lsn, 8); /* chunk0[16] (cfgref_present) already 0 from calloc. */ const uint32_t fake_fmt = 2u; /* must match persist.c's SNAPSHOT_FORMAT_VERSION * (config-upgrade-snapshot task 1 bumped 1->2) */ memcpy(chunk0 + 57, &fake_fmt, sizeof(fake_fmt)); /* ── Chunk 0: not last ── */ snapshot_transfer_t c0; memset(&c0, 0, sizeof(c0)); c0.snapshot_lsn = snap_lsn; c0.chunk_index = 0; c0.chunk_offset = 0; c0.total_size = (uint64_t)total_size; c0.is_last = 0; int rc = ha_handle_snapshot_chunk(&ha, p, snap_dir, &c0, chunk0, chunk_size); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "chunk 0 must succeed"); /* snap.part must exist and be exactly chunk_size bytes */ char part_path[600]; snprintf(part_path, sizeof(part_path), "%s/snap.part", snap_dir); struct stat st; TEST_ASSERT_EQUAL_INT_MESSAGE(0, stat(part_path, &st), "snap.part must exist after chunk 0"); TEST_ASSERT_EQUAL_UINT64_MESSAGE((uint64_t)chunk_size, (uint64_t)st.st_size, "snap.part must be chunk_size bytes after chunk 0"); /* ── Chunk 1: isLast=true ── */ snapshot_transfer_t c1; memset(&c1, 0, sizeof(c1)); c1.snapshot_lsn = snap_lsn; c1.chunk_index = 1; c1.chunk_offset = (uint64_t)chunk_size; c1.total_size = (uint64_t)total_size; c1.is_last = 1; rc = ha_handle_snapshot_chunk(&ha, p, snap_dir, &c1, chunk1, chunk_size); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "chunk 1 (isLast) must succeed"); /* snap.part must be gone (atomically renamed) */ TEST_ASSERT_NOT_EQUAL_MESSAGE(0, stat(part_path, &st), "snap.part must be gone after isLast rename"); /* Final file snap-{lsn:016x}.bin must exist with total_size bytes */ char final_path[600]; snprintf(final_path, sizeof(final_path), "%s/snap-%016" PRIx64 ".bin", snap_dir, snap_lsn); TEST_ASSERT_EQUAL_INT_MESSAGE(0, stat(final_path, &st), "snap-{lsn}.bin must exist after isLast"); TEST_ASSERT_EQUAL_UINT64_MESSAGE((uint64_t)total_size, (uint64_t)st.st_size, "snap-{lsn}.bin must be exactly total_size bytes"); /* applied_lsn must equal checkpoint_lsn (snap_lsn) after successful load */ uint64_t got_lsn = (uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire); TEST_ASSERT_EQUAL_UINT64_MESSAGE(snap_lsn, got_lsn, "applied_lsn must equal checkpoint_lsn after load"); free(chunk0); free(chunk1); ha_fini(&ha); persist_stop(p); persist_fini(p); free(p); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_snapshot_recv_128kb_two_chunks: PASS\n"); } /* bug-1389: a live --force-snapshot soak observed the vtask pool dispatching * a 12-chunk transfer's chunks out of chunk_index order — chunk 2 was * processed AFTER chunks 3 and 4 (arrival order 0,1,3,4,2,5,6,7,8,9,10,11). * Live crc32 instrumentation proved every chunk's CONTENT survived the wire * intact; the bug was that ha_handle_snapshot_chunk appended bytes in * ARRIVAL order (fopen "ab") instead of writing each chunk at its own * chunk_offset, so chunks 2/3/4 landed at the wrong file positions — * same total size, wrong content, exactly matching the corruption the soak * saw. This test replays that exact arrival order and asserts the * assembled file is byte-identical to the correctly-ordered source, * regardless of the order ha_handle_snapshot_chunk was called in. */ static void test_snapshot_recv_out_of_order_chunks_assemble_correctly(void) { char tmpdir[] = "/tmp/xcectest_snaprecv3_XXXXXX"; TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); char snap_dir[512]; snprintf(snap_dir, sizeof(snap_dir), "%s/snap", tmpdir); TEST_ASSERT_EQUAL_INT(0, mkdir(snap_dir, 0700)); /* 4 chunks of 64 bytes each = 256-byte body. Must be a WELL-FORMED * entity-bearing checkpoint (same reason as * test_snapshot_recv_two_chunks_assembles_and_renames above): legacy * 57-byte header (cfgref_present=0) + format_version=2 + job_count=0 + * queue_count=0 + host_count=0 (config-upgrade-snapshot task 4 -- * load_snapshot_entities now reads a host section after the queue * section, so it must likewise be pinned to a well-formed 0 here rather * than left as position-dependent filler), then filler out to 256 * bytes. The filler is position-dependent (distinct per source byte, * not per chunk) so ANY chunk landing at the wrong file offset is * immediately detectable via a whole-buffer memcmp, regardless of * which physical chunk a given structural field happens to fall in -- * except for the 4 structural-field byte ranges explicitly overlaid * below (fmt/job_count/queue_count/host_count), which must stay * well-formed for the load to succeed at all. */ uint64_t snap_lsn = 400; size_t total_size = 256; uint8_t body[256]; memset(body, 0, sizeof(body)); memcpy(body, "XCESNAP\0", 8); memcpy(body + 8, &snap_lsn, 8); for (int i = 16; i < 256; i++) body[i] = (uint8_t)(i * 3 + 7); /* position-dependent, non-repeating */ uint32_t fake_fmt = 2u; /* must match persist.c's SNAPSHOT_FORMAT_VERSION * (config-upgrade-snapshot task 1 bumped 1->2) */ uint32_t fake_job_count = 0u; uint32_t fake_queue_count = 0u; uint32_t fake_host_count = 0u; memcpy(body + 57, &fake_fmt, sizeof(fake_fmt)); memcpy(body + 61, &fake_job_count, sizeof(fake_job_count)); memcpy(body + 65, &fake_queue_count, sizeof(fake_queue_count)); memcpy(body + 69, &fake_host_count, sizeof(fake_host_count)); persist_t *p = calloc(1, sizeof(persist_t)); TEST_ASSERT_NOT_NULL(p); persist_config_t pcfg = persist_config_defaults(); persist_set_workdir(p, tmpdir, "ctrl-snaprecv3"); persist_init(p, &pcfg, NULL); persist_start(p); ha_state_t ha; snap_recv_ha_init(&ha); /* Deliver out of chunk_index order: 0, 2, 1, 3 (isLast). */ int order[4] = { 0, 2, 1, 3 }; for (int oi = 0; oi < 4; oi++) { int idx = order[oi]; snapshot_transfer_t c; memset(&c, 0, sizeof(c)); c.snapshot_lsn = snap_lsn; c.chunk_index = (uint32_t)idx; c.chunk_offset = (uint64_t)(idx * 64); c.total_size = (uint64_t)total_size; c.is_last = (idx == 3); int rc = ha_handle_snapshot_chunk(&ha, p, snap_dir, &c, body + idx * 64, 64); TEST_ASSERT_EQUAL_INT(0, rc); } /* Final file must exist (finalize triggers once all 4 chunks are in, * regardless of the delivery order above) and be byte-identical to the * correctly-ordered source body — proves each chunk landed at its own * chunk_offset, not at the position it happened to arrive in. */ char final_path[600]; snprintf(final_path, sizeof(final_path), "%s/snap-%016" PRIx64 ".bin", snap_dir, snap_lsn); struct stat st; TEST_ASSERT_EQUAL_INT_MESSAGE(0, stat(final_path, &st), "snap-{lsn}.bin should exist once all " "chunks are in, regardless of arrival order"); TEST_ASSERT_EQUAL_UINT64((uint64_t)total_size, (uint64_t)st.st_size); FILE *f = fopen(final_path, "rb"); TEST_ASSERT_NOT_NULL(f); uint8_t got[256]; TEST_ASSERT_EQUAL_size_t(256, fread(got, 1, 256, f)); fclose(f); TEST_ASSERT_EQUAL_MEMORY_MESSAGE(body, got, 256, "assembled file must be byte-identical to the source despite " "out-of-chunk_index-order delivery (bug-1389)"); uint64_t got_lsn = (uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire); TEST_ASSERT_EQUAL_UINT64(snap_lsn, got_lsn); ha_fini(&ha); persist_stop(p); persist_fini(p); free(p); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_snapshot_recv_out_of_order_chunks_assemble_correctly: PASS\n"); } /* Test 3.5d: reconnected standby discards snap.part and restarts from chunk 0. * * Design §Snapshot Transfer: "Mid-transfer reconnect always restarts snapshot * from chunk 0 — the standby discards its partial snap.part file on reconnect." * The discard (unlink) is done by the ctrlhello handler before the first chunk * of a new transfer arrives. This test verifies that: * 1. After receiving chunk 0 (creating snap.part), if the caller unlinks * snap.part (simulating reconnect/reset) and then calls ha_handle_snapshot_chunk * again with chunk_index=0, the function creates a fresh snap.part * (not appending to a previous stale file). * 2. The resulting snap.part contains only the new chunk's bytes (not a * concatenation of old + new data). * * Requirement: 4.4 (ha-repl-hardening) — retry by reconnecting initiates new transfer. */ static void test_snapshot_recv_reconnect_discards_snappart(void) { char tmpdir[] = "/tmp/xcectest_snapreconn_XXXXXX"; TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); char snap_dir[512]; snprintf(snap_dir, sizeof(snap_dir), "%s/snap", tmpdir); TEST_ASSERT_EQUAL_INT(0, mkdir(snap_dir, 0700)); persist_t *p = calloc(1, sizeof(persist_t)); TEST_ASSERT_NOT_NULL(p); persist_config_t pcfg = persist_config_defaults(); persist_set_workdir(p, tmpdir, "ctrl-snapreconn"); persist_init(p, &pcfg, NULL); persist_start(p); ha_state_t ha; snap_recv_ha_init(&ha); uint64_t snap_lsn = 300; uint8_t first[64]; uint8_t second[64]; memset(first, 0xAA, 64); memset(second, 0xBB, 64); /* ── First transfer attempt: send chunk 0 ── */ snapshot_transfer_t c0; memset(&c0, 0, sizeof(c0)); c0.snapshot_lsn = snap_lsn; c0.chunk_index = 0; c0.chunk_offset = 0; c0.total_size = 64; /* single-chunk transfer */ c0.is_last = 0; /* not last — simulates partial in-flight transfer */ int rc = ha_handle_snapshot_chunk(&ha, p, snap_dir, &c0, first, 64); TEST_ASSERT_EQUAL_INT(0, rc); /* snap.part must exist and contain exactly 64 bytes (first chunk) */ char part_path[600]; snprintf(part_path, sizeof(part_path), "%s/snap.part", snap_dir); struct stat st; TEST_ASSERT_EQUAL_INT_MESSAGE(0, stat(part_path, &st), "snap.part must exist after first chunk"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(64, (uint64_t)st.st_size, "snap.part must be 64 bytes after first chunk"); /* ── Simulate reconnect: caller discards snap.part ── */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, unlink(part_path), "unlink snap.part must succeed (simulating reconnect)"); /* ── Second transfer attempt: restart from chunk 0 ── */ snapshot_transfer_t c0b; memset(&c0b, 0, sizeof(c0b)); c0b.snapshot_lsn = snap_lsn; c0b.chunk_index = 0; c0b.chunk_offset = 0; c0b.total_size = 64; c0b.is_last = 0; /* still not last — just verifying discard */ rc = ha_handle_snapshot_chunk(&ha, p, snap_dir, &c0b, second, 64); TEST_ASSERT_EQUAL_INT(0, rc); /* snap.part must exist and contain exactly 64 bytes (second chunk only, * not 128 bytes which would indicate old data was appended) */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, stat(part_path, &st), "snap.part must exist after reconnect chunk 0"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(64, (uint64_t)st.st_size, "snap.part must be 64 bytes (not 128) — stale data discarded"); /* Verify the content is the second chunk (0xBB bytes), not the first (0xAA) */ FILE *f = fopen(part_path, "rb"); TEST_ASSERT_NOT_NULL_MESSAGE(f, "snap.part must be readable"); uint8_t buf[64]; size_t n = fread(buf, 1, 64, f); fclose(f); TEST_ASSERT_EQUAL_UINT64_MESSAGE(64, (uint64_t)n, "must read 64 bytes from snap.part"); TEST_ASSERT_EQUAL_UINT8_MESSAGE(0xBB, buf[0], "snap.part content must be second chunk (0xBB), not first (0xAA)"); ha_fini(&ha); persist_stop(p); persist_fini(p); free(p); char cmd[512]; snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); system(cmd); printf("test_snapshot_recv_reconnect_discards_snappart: PASS\n"); } /* ── Task 5.1: handle_ctrlhello — connection setup only ───────────────────── * * Verify that after the ha-repl-hardening simplification, handle_ctrlhello * does connection bookkeeping only and enqueues NO replication messages. * * Requirements: 1.1, 1.2 */ static int g_ctrlhello_repl_msg_count = 0; /* Count any xce_request calls (via the wal stream hook) that arrive during * handle_ctrlhello. After simplification this must remain 0. * Note: the WAL streaming hook is for ha_stream_wal_to_standby; if * handle_ctrlhello were still calling xce_request directly the test would * need a separate seam — but since the simplified code has none, we simply * verify no crash and correct state. */ static void test_ctrlhello_no_repl_messages_enqueued(void) { /* Build a minimal cluster_t in CTRL_STANDBY role. * Intentionally leave persist.repl, persist.wal_path, and * standby_endpoints uninitialised (zero/NULL) — the simplified * handle_ctrlhello must not touch any of these fields. * The old code would crash or corrupt memory if it reached * repl_add_standby / WAL streaming / push_config_to_standbys * with NULL repl state. */ cluster_t c; memset(&c, 0, sizeof(c)); ha_config_t hcfg; memset(&hcfg, 0, sizeof(hcfg)); hcfg.heartbeat_timeout_ms = 10000; hcfg.election_timeout_ms = 1500; ha_init(&c.ha, &hcfg, 0); c.ha.primary_ctrl_fd = -1; atomic_store_explicit(&c.ha.role, (int32_t)CTRL_STANDBY, memory_order_release); /* Give myconf a non-zero config_version_tag so the old config-push branch * would have fired (standby.config_tag=0 != primary_tag=0xCAFE). */ myconf_t conf; memset(&conf, 0, sizeof(conf)); conf.config_version_tag = 0xCAFEULL; c.myconf = &conf; /* Give persist a non-zero committed_lsn so the old WAL-streaming branch * would have fired (standby.applied_lsn=0 < committed_lsn=100). */ atomic_store_explicit(&c.persist.committed_lsn, 100ULL, memory_order_release); /* Leave persist.wal_path as NULL / empty — old code would have opened NULL. */ /* Build a minimal CtrlHelloRequest: standby reports lsn=0, tag=0 */ ctrl_hello_req_t hello; memset(&hello, 0, sizeof(hello)); hello.name = "standby-test"; hello.applied_lsn = 0; hello.config_tag = 0; xce_request_t req; memset(&req, 0, sizeof(req)); req.body_which = Request_body_ctrlhellomsg; req.body.ctrlhellomsg = &hello; endpoint_t ep; memset(&ep, 0, sizeof(ep)); ep.fd = 77; /* Reset repl message counter */ g_ctrlhello_repl_msg_count = 0; /* Call the simplified handle_ctrlhello — must return 0 without crash */ int rc = handle_ctrlhello(&c, &ep, &req); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "handle_ctrlhello must return 0 (Req 1.1)"); /* Connection bookkeeping must have happened */ TEST_ASSERT_EQUAL_INT_MESSAGE(77, c.ha.primary_ctrl_fd, "handle_ctrlhello must set primary_ctrl_fd = ep->fd (Req 1.1)"); /* ha_on_heartbeat_received must have been called (last_activity updated) */ TEST_ASSERT_TRUE_MESSAGE(c.ha.last_activity > 0, "handle_ctrlhello must call ha_on_heartbeat_received (Req 1.2)"); /* Idempotency: second call must not overwrite primary_ctrl_fd */ ep.fd = 99; handle_ctrlhello(&c, &ep, &req); TEST_ASSERT_EQUAL_INT_MESSAGE(77, c.ha.primary_ctrl_fd, "handle_ctrlhello: second call must not overwrite primary_ctrl_fd (Req 1.1)"); /* Verify no replication messages were counted. * With the simplified implementation, handle_ctrlhello never calls * xce_request() — so g_ctrlhello_repl_msg_count stays 0. If xce_request * were called for OPCode_replConfigPush or OPCode_walCatchupEntry the * test binary would crash (NULL ep infrastructure) before reaching here. */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_ctrlhello_repl_msg_count, "handle_ctrlhello must enqueue no OPCode_replConfigPush or " "OPCode_walCatchupEntry messages (Req 1.1, 1.2)"); ha_fini(&c.ha); printf("test_ctrlhello_no_repl_messages_enqueued: PASS\n"); } /* Declare handle_ctrlhello for this translation unit (defined in config_repl.c) */ extern int handle_ctrlhello(cluster_t *c, endpoint_t *ep, xce_request_t *req); #endif /* XCE_TESTING */ /* ═══════════════════════════════════════════════════════════════════════════ * xce-audit-invariants controller pass, thread-role violation 2 fix * (2026-07-18, the more urgent violation — catch-up can stream an unbounded * amount of WAL data): ha_catchup_continuation_fn must not block the * delaymgr thread — the SAME single thread that drives heartbeats and * election timeouts cluster-wide (ha-async-io.md thread-role contract). * * These tests drive a REAL, STARTED delaymgr (single worker thread, exactly * as production — delaymgr_t.worker is a single pthread, not a pool) so the * scheduled one-shot continuation task is picked up and run BY that thread, * proving whether it blocks. ha_repl_stream_wal_hook gates the blocking * streaming call so the test controls exactly when it completes. A cheap * PROBE task queued on the SAME delaymgr right after the continuation is * scheduled stands in for "the next heartbeat/election timer" — whether it * fires promptly is the load-bearing assertion. * ═══════════════════════════════════════════════════════════════════════════ */ #ifdef XCE_TESTING typedef struct { pthread_mutex_t mu; pthread_cond_t release_cv; pthread_cond_t entered_cv; int gate_open; _Atomic int entered; _Atomic int calls; } cc_gate_t; static cc_gate_t g_cc_gate; static void cc_gate_init(cc_gate_t *g) { pthread_mutex_init(&g->mu, NULL); pthread_cond_init(&g->release_cv, NULL); pthread_cond_init(&g->entered_cv, NULL); g->gate_open = 0; atomic_store(&g->entered, 0); atomic_store(&g->calls, 0); } static void cc_gate_destroy(cc_gate_t *g) { pthread_mutex_destroy(&g->mu); pthread_cond_destroy(&g->release_cv); pthread_cond_destroy(&g->entered_cv); } static void cc_gate_open(cc_gate_t *g) { pthread_mutex_lock(&g->mu); g->gate_open = 1; pthread_cond_broadcast(&g->release_cv); pthread_mutex_unlock(&g->mu); } static int cc_gate_wait_entered(cc_gate_t *g, int n, int timeout_ms) { struct timespec deadline; clock_gettime(CLOCK_REALTIME, &deadline); deadline.tv_sec += timeout_ms / 1000; deadline.tv_nsec += (long)(timeout_ms % 1000) * 1000000L; if (deadline.tv_nsec >= 1000000000L) { deadline.tv_sec++; deadline.tv_nsec -= 1000000000L; } pthread_mutex_lock(&g->mu); int rc = 0; while (atomic_load(&g->entered) < n) { rc = pthread_cond_timedwait(&g->entered_cv, &g->mu, &deadline); if (rc != 0) break; } int entered = atomic_load(&g->entered); pthread_mutex_unlock(&g->mu); return entered >= n ? 0 : -1; } /* Stand-in for the blocking WAL stream (ha_stream_wal_to_standby): count, * signal entry, park until the test opens the gate, then return success. * Runs on a slow-lane worker (flag ON) or inline on the delaymgr worker * thread (flag OFF). */ static int cc_gated_stream_hook(void *ep, const char *wal_path, uint64_t standby_lsn, uint64_t committed_lsn) { (void)ep; (void)wal_path; (void)standby_lsn; (void)committed_lsn; atomic_fetch_add(&g_cc_gate.calls, 1); pthread_mutex_lock(&g_cc_gate.mu); atomic_fetch_add(&g_cc_gate.entered, 1); pthread_cond_broadcast(&g_cc_gate.entered_cv); while (!g_cc_gate.gate_open) pthread_cond_wait(&g_cc_gate.release_cv, &g_cc_gate.mu); pthread_mutex_unlock(&g_cc_gate.mu); return 0; } /* Real, started delaymgr pass-through runner — production's run_delay_tasks * (cluster.c) is not linked into controller_unit_test; mirrors * test_commit_durability.c's dur_run_delay_task. */ static int cc_run_delay_task(delay_task_entry_fn_t entry, void *arg) { return entry(arg); } static int g_cc_ep_placeholder; /* fake non-NULL repl_ep_t identity */ typedef struct { _Atomic int fired; } cc_probe_t; static int cc_probe_fn(void *arg) { cc_probe_t *p = (cc_probe_t *)arg; atomic_store(&p->fired, 1); return 0; } static int cc_poll_flag(_Atomic int *flag, int timeout_ms) { for (int i = 0; i < timeout_ms; i++) { if (atomic_load(flag)) break; struct timespec ts = { .tv_sec = 0, .tv_nsec = 1000000L }; /* 1 ms */ nanosleep(&ts, NULL); } return atomic_load(flag); } static void cc_run_offload_probe(int offload_on, const char *label) { char tmpdir[256]; snprintf(tmpdir, sizeof(tmpdir), "/tmp/xce_cc_%s_XXXXXX", offload_on ? "on" : "off"); TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); cluster_t c; memset(&c, 0, sizeof(c)); persist_config_t pcfg = persist_config_defaults(); pcfg.wal_sync_ms = 5000; persist_set_workdir(&c.persist, tmpdir, "ctrl-test"); TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &pcfg, &c)); TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist)); pthread_mutex_lock(&c.persist.lock); c.persist.current_lsn = 11; /* tip = 10 */ pthread_mutex_unlock(&c.persist.lock); atomic_store(&c.persist.committed_lsn, 10); atomic_store(&c.persist.durable_lsn, 10); TEST_ASSERT_TRUE(repl_add_standby(&c.persist.repl, &g_cc_ep_placeholder, "ccsb0", 7600) >= 0); ha_config_t hcfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &hcfg, 0)); TEST_ASSERT_EQUAL_INT(0, slowlane_init(&c.slowlane, 2, 4)); TEST_ASSERT_EQUAL_INT(0, delaymgr_init(&c.delaymgr, 5, cc_run_delay_task)); TEST_ASSERT_EQUAL_INT(0, delaymgr_start(&c.delaymgr)); cc_gate_init(&g_cc_gate); ha_repl_stream_wal_hook = cc_gated_stream_hook; ha_set_catchup_offload_for_test(offload_on); /* acked_lsn=0 < target(=10) -> schedules exactly one continuation. */ TEST_ASSERT_EQUAL_INT(0, ha_walack_continuation_for_test(&c, &g_cc_ep_placeholder, 0, 0)); TEST_ASSERT_EQUAL_INT_MESSAGE(0, cc_gate_wait_entered(&g_cc_gate, 1, 1000), "the streaming call must reach the gate (on the slow lane, or — " "RED — inline on the delaymgr thread)"); /* Queue a cheap probe task RIGHT AFTER, on the SAME single-worker * delaymgr — a stand-in for the next heartbeat/election timer. */ cc_probe_t probe; atomic_store(&probe.fired, 0); TEST_ASSERT_EQUAL_INT(0, delay_task_ms(&c.delaymgr, 1, 1, cc_probe_fn, &probe)); if (offload_on) { TEST_ASSERT_TRUE_MESSAGE(cc_poll_flag(&probe.fired, 1000), "GREEN: with catch-up offloaded to the slow lane, the delaymgr " "worker thread must be free to run the NEXT queued task " "promptly, even while the streaming body is still parked"); } else { TEST_ASSERT_FALSE_MESSAGE(cc_poll_flag(&probe.fired, 300), "RED: with offload OFF the streaming body runs INLINE on the " "delaymgr worker thread, so the next queued task (standing in " "for a heartbeat/election timer) cannot fire until it returns " "-- this is the blocking-on-delaymgr shape the fix removes"); } cc_gate_open(&g_cc_gate); /* Drain: wait for catchup_inflight[0] to clear (lane/inline body done), * then confirm the probe (queued behind it in the OFF/RED case) fires. */ for (int i = 0; i < 1000; i++) { if (atomic_load_explicit(&c.ha.catchup_inflight[0], memory_order_acquire) == 0) break; struct timespec ts = { .tv_sec = 0, .tv_nsec = 1000000L }; nanosleep(&ts, NULL); } TEST_ASSERT_TRUE_MESSAGE(cc_poll_flag(&probe.fired, 1000), "the probe task must eventually fire once the gate is released"); ha_repl_stream_wal_hook = NULL; ha_set_catchup_offload_for_test(0); /* restore production default */ cc_gate_destroy(&g_cc_gate); delaymgr_stop(&c.delaymgr); delaymgr_fini(&c.delaymgr); slowlane_fini(&c.slowlane); ha_fini(&c.ha); persist_stop(&c.persist); persist_fini(&c.persist); repl_fini(&c.persist.repl); printf("%s: PASS\n", label); } static void test_catchup_offload_probe_runs_promptly(void) { cc_run_offload_probe(1, "test_catchup_offload_probe_runs_promptly"); } static void test_catchup_inline_probe_blocks_RED(void) { cc_run_offload_probe(0, "test_catchup_inline_probe_blocks_RED"); } /* ── bug-1476: concurrent load_primary_config() race on myconf->cluster_config ── * * load_primary_config()'s free-decode-publish sequence for * myconf->cluster_config was completely unguarded by any lock. A live * 3-node ASan soak (config-upgrade-snapshot task 8.9 re-verification, * ctrl-3) caught a genuine heap-use-after-free: two concurrent * handle_repl_config_push deliveries for the SAME standby raced two * concurrent load_primary_config() calls on separate VTASK_LIVE_BLOCK * workers -- one call's decode had already published a non-NULL, * still-populating cluster_config_t* into the shared myconf->cluster_config * field (decode_ClusterConfig_ptr calloc's *and publishes* the pointer via * its out-parameter BEFORE decode_ClusterConfig populates the nested fields * such as resources -- confirmed by reading * lib/controller.capnp.c:17067-17077) while a second, concurrent invocation * observed that same non-NULL pointer, assumed (per the single free-then- * decode sequence) it was a prior FULLY-decoded config, and freed it out * from under the first call's still-in-flight decode. * * load_primary_config_race_hook (XCE_TESTING-only, config_repl.c) lets this * test deterministically force two REAL pthreads through that exact race * window instead of depending on scheduler timing: thread A captures the * config pointer it is about to retire and parks at the hook; thread B (a * full, real, concurrent load_primary_config() call for the SAME myconf_t) * is released to run while A is parked. * * RED (pre-fix, no lock): B runs to completion unblocked -- freeing the * SAME pointer A already captured and publishing its own config. Opening * the gate lets A retire (free) that already-freed pointer a SECOND time: a * genuine, deterministic, ASan-catchable double-free from the identical * missing-lock root cause the live crash exposed. (The exact * decode_Resource_list-mid-write UAF the live trace shows additionally * requires pausing inside the generated capnp decoder, which the fix * deliberately avoids touching -- see the fix's own comment in * load_primary_config; this reproduction targets the same unguarded * read-free-decode-publish sequence and is caught by ASan the same way.) * * GREEN (post-fix): myconf->lock serializes the two calls end-to-end -- B * cannot even acquire the lock (let alone reach its own retire step or * corrupt anything) until A's whole call, hook included, returns and * releases it. No pointer is ever freed twice, and load_primary_config (fix * part 2) no longer decodes directly into the shared field at all, closing * the exposure for any OTHER reader that never takes the lock. */ typedef struct { pthread_mutex_t mu; pthread_cond_t release_cv; pthread_cond_t entered_cv; int gate_open; _Atomic int entered; } lpc_gate_t; static lpc_gate_t g_lpc_gate; static _Atomic int g_lpc_hook_block_next; /* 1 = the NEXT hook call should park */ static void lpc_gate_init(lpc_gate_t *g) { pthread_mutex_init(&g->mu, NULL); pthread_cond_init(&g->release_cv, NULL); pthread_cond_init(&g->entered_cv, NULL); g->gate_open = 0; atomic_store(&g->entered, 0); } static void lpc_gate_destroy(lpc_gate_t *g) { pthread_mutex_destroy(&g->mu); pthread_cond_destroy(&g->release_cv); pthread_cond_destroy(&g->entered_cv); } static void lpc_gate_open(lpc_gate_t *g) { pthread_mutex_lock(&g->mu); g->gate_open = 1; pthread_cond_broadcast(&g->release_cv); pthread_mutex_unlock(&g->mu); } static int lpc_gate_wait_entered(lpc_gate_t *g, int timeout_ms) { struct timespec deadline; clock_gettime(CLOCK_REALTIME, &deadline); deadline.tv_sec += timeout_ms / 1000; deadline.tv_nsec += (long)(timeout_ms % 1000) * 1000000L; if (deadline.tv_nsec >= 1000000000L) { deadline.tv_sec++; deadline.tv_nsec -= 1000000000L; } pthread_mutex_lock(&g->mu); int rc = 0; while (atomic_load(&g->entered) < 1) { rc = pthread_cond_timedwait(&g->entered_cv, &g->mu, &deadline); if (rc != 0) break; } int entered = atomic_load(&g->entered); pthread_mutex_unlock(&g->mu); return entered >= 1 ? 0 : -1; } /* Fires from INSIDE load_primary_config -- holding no lock of its own under * the OLD (pre-fix) code, or holding myconf->lock under the fixed code. * Only the FIRST call (armed via g_lpc_hook_block_next) actually parks; * later calls pass straight through, so thread B's own call runs unimpeded * when it reaches this same hook (pre-fix), or never even reaches it * (post-fix -- it blocks acquiring myconf->lock before getting this far). */ static void lpc_race_hook(void) { if (!atomic_exchange(&g_lpc_hook_block_next, 0)) return; pthread_mutex_lock(&g_lpc_gate.mu); atomic_fetch_add(&g_lpc_gate.entered, 1); pthread_cond_broadcast(&g_lpc_gate.entered_cv); while (!g_lpc_gate.gate_open) pthread_cond_wait(&g_lpc_gate.release_cv, &g_lpc_gate.mu); pthread_mutex_unlock(&g_lpc_gate.mu); } typedef struct { cluster_t *c; _Atomic int done; _Atomic int rc; } lpc_thread_arg_t; static void *lpc_call_load_primary_config(void *arg) { lpc_thread_arg_t *a = (lpc_thread_arg_t *)arg; int rc = load_primary_config(a->c); atomic_store(&a->rc, rc); atomic_store(&a->done, 1); return NULL; } static void test_bug1476_concurrent_load_primary_config_no_uaf_or_double_free(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cr1476_race_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t c; memset(&c, 0, sizeof(c)); myconf_t *mc = t43_attach_myconf(&c, tmp); /* inits mc->lock */ /* conf_save_config only mkdir()s the LAST path component (workdir/name/ * conf); unlike other t43_* fixtures this test does not call * t42_make_cluster (which creates workdir/name via persist_init) since * load_primary_config only touches c->myconf -- create workdir/name * directly instead. */ char ctrldir[300]; snprintf(ctrldir, sizeof(ctrldir), "%s/ctrl-test", tmp); TEST_ASSERT_EQUAL_INT(0, mkdir(ctrldir, 0700)); /* Establish an initial live config so BOTH racing calls exercise the * "free the OLD config" retire step (not the first-ever-load path) -- * matches the live crash, which happened on a standby reloading an * ALREADY-activated config, not on first boot. */ size_t blen0 = 0; uint8_t *bytes0 = t43_make_config_blob(&blen0); uint8_t sha0[32]; SHA256(bytes0, blen0, sha0); TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes0, blen0, sha0)); free(bytes0); TEST_ASSERT_EQUAL_INT(0, load_primary_config(&c)); TEST_ASSERT_NOT_NULL(mc->cluster_config); /* Both racing calls independently resolve + decode the SAME conf/current * file -- real production behaviour: two concurrent * handle_repl_config_push deliveries for the same standby race over the * SAME on-disk config, not different ones. */ lpc_gate_init(&g_lpc_gate); atomic_store(&g_lpc_hook_block_next, 1); load_primary_config_race_hook = lpc_race_hook; lpc_thread_arg_t arg_a = { .c = &c, .done = 0, .rc = 0 }; lpc_thread_arg_t arg_b = { .c = &c, .done = 0, .rc = 0 }; pthread_t thread_a; TEST_ASSERT_EQUAL_INT(0, pthread_create(&thread_a, NULL, lpc_call_load_primary_config, &arg_a)); TEST_ASSERT_EQUAL_INT_MESSAGE(0, lpc_gate_wait_entered(&g_lpc_gate, 2000), "thread A must reach the race-hook checkpoint inside " "load_primary_config before the test proceeds"); pthread_t thread_b; TEST_ASSERT_EQUAL_INT(0, pthread_create(&thread_b, NULL, lpc_call_load_primary_config, &arg_b)); /* Snapshot whether B completed WHILE A was still parked -- the * serialization proof. Captured into a plain variable (not asserted * yet) so a pre-fix failure here does not abort the test before it * reaches the release below: pre-fix (RED) the crash triggered by that * release is the primary, load-bearing signal; this flag is * corroborating evidence, asserted only after both threads have safely * joined post-release. */ int b_raced_through_while_a_parked = cc_poll_flag(&arg_b.done, 300); lpc_gate_open(&g_lpc_gate); TEST_ASSERT_TRUE_MESSAGE(cc_poll_flag(&arg_a.done, 2000), "thread A must complete once released"); pthread_join(thread_a, NULL); TEST_ASSERT_TRUE_MESSAGE(cc_poll_flag(&arg_b.done, 2000), "thread B must complete once unblocked by A's release"); pthread_join(thread_b, NULL); load_primary_config_race_hook = NULL; lpc_gate_destroy(&g_lpc_gate); TEST_ASSERT_EQUAL_INT(0, atomic_load(&arg_a.rc)); TEST_ASSERT_EQUAL_INT(0, atomic_load(&arg_b.rc)); /* No corruption: the final published config is fully valid (whichever * of the two calls happened to finish last -- both raced the SAME * conf/current file, so either outcome is a legitimate, fully-decoded * config, never a torn/freed one). */ TEST_ASSERT_NOT_NULL(mc->cluster_config); TEST_ASSERT_NOT_NULL(mc->cluster_config->clustername); TEST_ASSERT_EQUAL_STRING("t43-upgrade", mc->cluster_config->clustername); /* GREEN: myconf->lock must have kept B blocked the whole time A was * parked mid-call. RED (no lock) reliably fails this -- but reliably * crashes (ASan double-free abort) even earlier, at lpc_gate_open() * above, before ever reaching this line. */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, b_raced_through_while_a_parked, "thread B's load_primary_config must stay blocked (serialized by " "myconf->lock) while thread A is parked mid-call"); t43_free_myconf(&c); printf("test_bug1476_concurrent_load_primary_config_no_uaf_or_double_free: PASS\n"); } #endif /* XCE_TESTING */ /* ── dispatch ───────────────────────────────────── */ void test_config_repl(void) { UnitySetTestFile(__FILE__); RUN_TEST(test_conf_read_current_filename_missing); RUN_TEST(test_conf_read_current_filename_empty_file); RUN_TEST(test_conf_read_current_filename_path_separator); RUN_TEST(test_conf_read_current_filename_bad_prefix); RUN_TEST(test_conf_read_current_filename_wrong_hex_length); RUN_TEST(test_conf_save_creates_pointer_file); RUN_TEST(test_conf_save_filename_has_12_hex_suffix); RUN_TEST(test_conf_trim_keeps_only_n_versions); RUN_TEST(test_conf_get_saved_tag_no_config); RUN_TEST(test_conf_get_saved_tag_after_save); RUN_TEST(test_conf_check_stale_disabled); RUN_TEST(test_conf_check_stale_fresh); RUN_TEST(test_conf_check_stale_aged); RUN_TEST(test_config_contains_reserved_name_absent); RUN_TEST(test_sanitise_cn_printable); RUN_TEST(test_sanitise_cn_control_chars); RUN_TEST(test_sanitise_cn_truncated); RUN_TEST(test_conf_save_config_path_consistency); /* Task 4.1: WAL_CONFIG_MARKER type + reference-only codec */ RUN_TEST(test_config_marker_enum_value_is_50); RUN_TEST(test_config_marker_encode_decode_roundtrip); RUN_TEST(test_config_marker_decode_truncated_fails); RUN_TEST(test_config_marker_encode_short_buffer_fails); /* Task 4.2: append marker via unified log + derive committed config */ RUN_TEST(test_config_marker_append_via_unified_log); RUN_TEST(test_committed_marker_derives_after_commit); RUN_TEST(test_uncommitted_marker_not_yet_committed_config); RUN_TEST(test_divergent_marker_truncated_like_job_entry); /* Task 4.2 r1: live wiring + compaction-safe derive */ RUN_TEST(test_config_commit_change_live_path_commits_and_derives); RUN_TEST(test_committed_marker_survives_checkpoint_trim); RUN_TEST(test_config_activate_match_activates_and_unfences); RUN_TEST(test_config_activate_mismatch_fences_and_does_not_activate); RUN_TEST(test_config_activate_obtained_bytes_verified_before_activate); RUN_TEST(test_config_activate_recovery_uses_latest_committed_marker); /* Task 4.3 r1 remediation: production integration (push-path sole activation, * no-marker fallback, serve-fence enforced by dispatch gate + promotion). */ RUN_TEST(test_push_path_mismatch_does_not_preactivate); RUN_TEST(test_config_activate_no_marker_falls_back_and_loads); /* config-upgrade-snapshot task 6.1: Activation Fence. */ RUN_TEST(test_upgrade_materialization_verify_no_marker_clears_fence); RUN_TEST(test_upgrade_materialization_verify_matching_materialization_clears_fence); RUN_TEST(test_upgrade_materialization_verify_incomplete_materialization_fences); RUN_TEST(test_upgrade_materialization_verify_stale_local_bytes_after_materialization_fences); RUN_TEST(test_serve_fence_enforced_by_dispatch_gate_and_promotion); /* Task 4.3 r2 remediation: commit-advance retry clears a push-before-commit * serve-fence (Finding D / R7.9). */ RUN_TEST(test_commit_advance_retry_clears_push_before_commit_fence); /* config-upgrade-snapshot task 6.2: Activation Fence gate composition + * self-triggering retry. */ RUN_TEST(test_upgrade_materialization_fence_enforced_by_dispatch_gate_and_promotion); RUN_TEST(test_schedule_upgrade_materialization_retry_noop_when_not_fenced); RUN_TEST(test_schedule_upgrade_materialization_retry_noop_when_delaymgr_not_running); RUN_TEST(test_upgrade_materialization_retry_fn_recovers_from_stale_bytes_fence); RUN_TEST(test_commit_advance_wiring_calls_upgrade_materialization_schedule_retry); RUN_TEST(test_ha_phase2_retry_fn_noop_when_already_done); RUN_TEST(test_ha_phase2_retry_fn_does_not_latch_on_failure_and_succeeds_on_retry); RUN_TEST(test_schedule_phase2_retry_noop_when_already_done); RUN_TEST(test_schedule_phase2_retry_noop_when_delaymgr_not_running); /* config-upgrade-snapshot task 6.2 remediation: fence set by a REAL * production trigger (apply_committed_entry's WAL_CONFIG_MARKER case), * not a direct test call -- proves the fence is reachable at all. */ RUN_TEST(test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss); /* bug-1469: WAL_CONFIG_MARKER apply must activate config on the * ORIGINATING member too, not just receiving standbys. */ RUN_TEST(test_wal_config_marker_apply_activates_originating_member_config); /* config-upgrade-snapshot task 8.8: a second configuration-change * marker committing before the first's snapshot install completes * everywhere must cleanly supersede it, never leave it partially * applied or conflated with the newer one. */ RUN_TEST(test_second_config_marker_supersedes_first_still_installing_on_member); /* config-upgrade-snapshot task 8.7 Scenario 2: leadership change * mid-materialization completes via the same generic apply-drain path. */ RUN_TEST(test_leadership_change_mid_materialization_completes_via_same_generic_apply_drain); #ifdef XCE_TESTING RUN_TEST(test_load_primary_config_missing_returns_error); RUN_TEST(test_wait_all_with_timeout_slow_threads); RUN_TEST(test_wait_all_with_timeout_fast_threads); /* bug-1381: push_config_transport_only endpoint-lifetime regression */ RUN_TEST(test_configpush_transport_holds_endpoint_ref_across_send); /* raft-adapter-conformance task 2.1: config-push reject reply integrity */ RUN_TEST(test_config_push_sha256_mismatch_sends_explicit_reject_reply); /* raft-adapter-conformance task 2.2: config-push reply term adoption */ RUN_TEST(test_config_push_reply_higher_term_adopts_and_steps_down); /* Task 2.1: OPCode_walCatchupEntry handler */ RUN_TEST(test_wal_catchup_entry_advances_applied_lsn); RUN_TEST(test_wal_catchup_entry_error_skipped_no_crash); RUN_TEST(test_bug1479_ae_reject_sends_nack_not_silence); RUN_TEST(test_wal_catchup_entry_null_entry_no_crash); /* Task 2.2: batch WAL catch-up entries */ RUN_TEST(test_wal_catchup_batch_advances_applied_lsn); RUN_TEST(test_wal_catchup_partial_batch); RUN_TEST(test_wal_catchup_batch_error_entry_skipped); RUN_TEST(test_wal_catchup_batch_zero_entries_fallback); /* Task 6.2: additional WAL catch-up and batch handling tests */ RUN_TEST(test_wal_catchup_batch_256_entries); RUN_TEST(test_wal_catchup_batch_applied_lsn_only_after_full_batch); /* Task 2.3: OPCode_nop heartbeat reply carries stateReport */ RUN_TEST(test_nop_reply_carries_state_report); RUN_TEST(test_nop_reply_zero_state_fresh_standby); /* Task 3.4: ha_stream_wal_to_standby batch WAL streaming */ RUN_TEST(test_wal_stream_500_entries_two_batches); RUN_TEST(test_wal_stream_partial_batch); RUN_TEST(test_wal_stream_empty_range_no_batches); RUN_TEST(test_wal_stream_skips_checkpoint_marker); RUN_TEST(test_bug1475_catchup_survives_midstream_checkpoint); /* bug-1475 */ RUN_TEST(test_wal_stream_term_from_index_not_raw_hdr); /* bug-1028 */ /* Task 3.5: snapshot chunk transfer (sender side) */ RUN_TEST(test_snapshot_transfer_two_chunks); RUN_TEST(test_bug1475_finding2_snapshot_install_checkpoint_term_absent); /* bug-1475 Finding 2 */ RUN_TEST(test_snapshot_transfer_small_less_than_one_chunk); RUN_TEST(test_snapshot_transfer_stops_and_propagates_send_failure); /* Task 3.5c: standby snapshot receive handler */ RUN_TEST(test_snapshot_recv_two_chunks_assembles_and_renames); /* config-upgrade-snapshot task 1: prior SNAPSHOT_FORMAT_VERSION rejected * fail-loud by a binary built against the bumped version */ RUN_TEST(test_snapshot_recv_old_format_version_rejected_fail_loud); /* Task 6.1: legacy position-only file consumed end-to-end via the * receive handler — entities_restored=false path */ RUN_TEST(test_snapshot_recv_legacy_file_advances_without_entities); RUN_TEST(test_snapshot_recv_size_mismatch_fails); RUN_TEST(test_snapshot_recv_null_args_fail); /* Task 6.4-A: 128KB snapshot in 2 chunks — full-size receive path */ RUN_TEST(test_snapshot_recv_128kb_two_chunks); RUN_TEST(test_snapshot_recv_out_of_order_chunks_assemble_correctly); /* Task 3.5d: reconnect discards snap.part */ RUN_TEST(test_snapshot_recv_reconnect_discards_snappart); /* Task 5.1: handle_ctrlhello connection setup only — no repl messages */ RUN_TEST(test_ctrlhello_no_repl_messages_enqueued); /* xce-audit-invariants controller pass, thread-role violation 2 fix * (2026-07-18): catch-up continuation slow-lane offload */ RUN_TEST(test_catchup_offload_probe_runs_promptly); RUN_TEST(test_catchup_inline_probe_blocks_RED); /* bug-1476: concurrent load_primary_config() UAF/double-free race */ RUN_TEST(test_bug1476_concurrent_load_primary_config_no_uaf_or_double_free); #endif /* Task A: raw capnp shims */ RUN_TEST(test_replconfig_req_ssl_encode); RUN_TEST(test_replconfig_reply_raw_encode); RUN_TEST(test_upgrade_config_req_extract_raw); }