#define _GNU_SOURCE #include #include #include #include #include #include #include #include #include "config.h" #include "config_repl.h" #include "persist.h" #include "wal_types.h" #include "repl.h" #include "commit.h" #include "xce/cluster_def.h" #include "ha.h" /* ha-term-safety task 1.3 — WAL term seam + recovery read-back + legacy-0 rule. * * The seam under test: * - persist_set_term(p, T) sets the controller's live election term so a * subsequent append stamps hdr.term = T (R1.1, R1.6). * - replay_wal retains each entry's hdr.term per LSN in an in-memory index * (R1.5), read back via persist_entry_term(p, lsn). * - persist_entry_term returns 0 for an absent LSN; a legacy entry carrying * hdr.term == 0 maps to the single lowest-known real Raft term (1) so a * legacy-0 entry is never spuriously divergent from another legacy-0 entry * and recovery never fails on legacy-0 (R1.8). * - the on-disk wal_entry_hdr_t stays 40 bytes (R1.7). */ /* The defined legacy-0 mapping (lowest real Raft term). Kept local to the test * so the assertion is independent of the implementation constant. */ #define LEGACY_LOWEST_TERM 1u /* ------------------------------------------------------------------ */ /* Helpers */ /* ------------------------------------------------------------------ */ static persist_t* make_fresh_persist(const char* tmpdir) { persist_t* p = calloc(1, sizeof(persist_t)); if (!p) return NULL; persist_config_t cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; /* slow fsync — no races in tests */ cfg.snap_keep_n = 2; persist_set_workdir(p, tmpdir, "ctrl-test"); if (persist_init(p, &cfg, NULL) != 0) { free(p); return NULL; } if (persist_start(p) != 0) { persist_fini(p); free(p); return NULL; } return p; } static void teardown_persist(persist_t* p) { persist_stop(p); persist_fini(p); free(p); } /* ------------------------------------------------------------------ */ /* Test: header layout preserved (R1.7) — value fix, not a format */ /* change. The on-disk WAL entry header must stay exactly 40 bytes. */ /* ------------------------------------------------------------------ */ static void test_wal_entry_hdr_size_is_40(void) { TEST_ASSERT_EQUAL_UINT64(40u, (uint64_t)sizeof(wal_entry_hdr_t)); printf("test_wal_entry_hdr_size_is_40: PASS\n"); } /* ------------------------------------------------------------------ */ /* Test: persist_set_term makes a subsequent append stamp hdr.term */ /* (R1.1, R1.6), recovered back via persist_entry_term. */ /* ------------------------------------------------------------------ */ static void test_set_term_stamps_append(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_term_set_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 7; /* Phase 1: set the live term, then append — the append stamps hdr.term=T. */ { persist_t* p = make_fresh_persist(tmp); TEST_ASSERT_NOT_NULL(p); persist_set_term(p, T); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1)); TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); /* Live read-back of the just-appended entry's term (LSN 1). */ TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(p, 1)); teardown_persist(p); } /* Phase 2: fresh persist_t recovers the WAL and reads the term back. */ { persist_t* p2 = make_fresh_persist(tmp); TEST_ASSERT_NOT_NULL(p2); TEST_ASSERT_EQUAL_INT(0, persist_recover(p2)); TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(p2, 1)); teardown_persist(p2); } printf("test_set_term_stamps_append: PASS\n"); } /* ------------------------------------------------------------------ */ /* Test: replay_wal of a WAL with mixed real-term and legacy-0 */ /* entries — persist_entry_term returns the real term for real */ /* entries and the defined lowest-known term for legacy-0 (R1.5, */ /* R1.8). Recovery never fails on legacy-0 entries. */ /* ------------------------------------------------------------------ */ static void test_recover_mixed_real_and_legacy_terms(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_term_mixed_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); /* Phase 1: write LSN 1,2 with term 0 (legacy — the pre-feature default), * then LSN 3,4 with real term 5. The append path stamps hdr.term=p->term, * so leaving the term unset (0) produces genuine legacy-0 entries. */ { persist_t* p = make_fresh_persist(tmp); TEST_ASSERT_NOT_NULL(p); /* Legacy-0 entries: no persist_set_term call (p->term stays 0). */ TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "l", 1)); /* lsn 1 */ TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "m", 1)); /* lsn 2 */ /* Real-term entries. */ persist_set_term(p, 5); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "r", 1)); /* lsn 3 */ TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "s", 1)); /* lsn 4 */ TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); teardown_persist(p); } /* Phase 2: fresh recover — read back each LSN's term. */ { persist_t* p2 = make_fresh_persist(tmp); TEST_ASSERT_NOT_NULL(p2); /* Recovery must NOT fail on the legacy-0 entries. */ TEST_ASSERT_EQUAL_INT(0, persist_recover(p2)); TEST_ASSERT_EQUAL_INT(RECOVER_CLEAN, persist_recovery_verdict(p2)); /* Legacy-0 entries map to the single lowest-known real term (1) — both * the same, so two legacy-0 entries are never seen as divergent. */ TEST_ASSERT_EQUAL_UINT64(LEGACY_LOWEST_TERM, persist_entry_term(p2, 1)); TEST_ASSERT_EQUAL_UINT64(LEGACY_LOWEST_TERM, persist_entry_term(p2, 2)); /* Real-term entries return their recorded term verbatim. */ TEST_ASSERT_EQUAL_UINT64(5, persist_entry_term(p2, 3)); TEST_ASSERT_EQUAL_UINT64(5, persist_entry_term(p2, 4)); teardown_persist(p2); } printf("test_recover_mixed_real_and_legacy_terms: PASS\n"); } /* ------------------------------------------------------------------ */ /* Test: persist_entry_term returns 0 for an absent LSN (no entry). */ /* Distinguishes "absent" (0) from a present legacy-0 entry (mapped */ /* to lowest-known), which is the precision REVIEW ISSUE 2 requires. */ /* ------------------------------------------------------------------ */ static void test_entry_term_absent_lsn_is_zero(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_term_absent_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = make_fresh_persist(tmp); TEST_ASSERT_NOT_NULL(p); persist_set_term(p, 3); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1)); /* lsn 1 */ /* LSN 0 is reserved (never an entry) and LSN 2 was never appended. */ TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(p, 0)); TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(p, 2)); TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(p, 999999)); /* The present entry still answers its real term. */ TEST_ASSERT_EQUAL_UINT64(3, persist_entry_term(p, 1)); teardown_persist(p); printf("test_entry_term_absent_lsn_is_zero: PASS\n"); } /* ------------------------------------------------------------------ */ /* Test: persist_entry_term is NULL-safe. */ /* ------------------------------------------------------------------ */ static void test_entry_term_null_safe(void) { TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(NULL, 1)); /* persist_set_term must not crash on NULL. */ persist_set_term(NULL, 9); printf("test_entry_term_null_safe: PASS\n"); } /* ================================================================== */ /* Task 2.1 — on_persist_entry callback body */ /* ================================================================== */ /* Build a cluster_t whose embedded persist module is initialised + started in * `tmpdir`, with ha.raft_c self-referencing the cluster (the convention every * raft callback uses to recover persist/role from ctx == &c->ha). Returns a * calloc'd cluster_t the caller must teardown via teardown_cluster. */ static cluster_t* make_cluster(const char* tmpdir) { cluster_t* c = calloc(1, sizeof(cluster_t)); if (!c) return NULL; persist_config_t cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; cfg.snap_keep_n = 2; persist_set_workdir(&c->persist, tmpdir, "ctrl-test"); if (persist_init(&c->persist, &cfg, c) != 0) { free(c); return NULL; } if (persist_start(&c->persist) != 0) { persist_fini(&c->persist); free(c); return NULL; } /* The callback recovers persist/role from ctx == &c->ha → c->ha.raft_c. */ c->ha.raft_c = c; return c; } static void teardown_cluster(cluster_t* c) { if (!c) return; persist_stop(&c->persist); persist_fini(&c->persist); free(c); } /* Pack one adapter entry blob: [raft_entry_blob_t{type,len}][payload]. */ static size_t pack_entry(uint8_t* buf, uint32_t type, const void* payload, uint32_t plen) { raft_entry_blob_t* h = (raft_entry_blob_t*)buf; h->type = type; h->payload_len = plen; if (payload && plen > 0) memcpy(buf + sizeof(*h), payload, plen); return sizeof(*h) + plen; } /* Test: the WIRED on_persist_entry stamps hdr.term from the library term at the * supplied 1-based index == LSN, persisting through the existing WAL path; the * recovered hdr.term equals the supplied term (R1.1–R1.4). Drives the standby * (replicated) path so persist-before-ACK is exercised. */ static void test_on_persist_entry_stamps_term_standby(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_op_sb_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 9; /* Phase 1: standby applies a replicated entry at LSN 1 with leader term T. */ { cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); atomic_store(&c->ha.role, CTRL_STANDBY); uint8_t blob[64]; size_t n = pack_entry(blob, WAL_JOB_UPDATE, "x", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, /*index*/1, T, blob, n)); /* Persisted at LSN 1 with hdr.term == T (live read-back). */ TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1)); TEST_ASSERT_EQUAL_UINT64(2, c->persist.current_lsn); teardown_cluster(c); } /* Phase 2: fresh recover reads the stamped term back from disk (R1.5). */ { cluster_t* c2 = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c2); TEST_ASSERT_EQUAL_INT(0, persist_recover(&c2->persist)); TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c2->persist, 1)); teardown_cluster(c2); } printf("test_on_persist_entry_stamps_term_standby: PASS\n"); } /* Test: the primary's own-append path through on_persist_entry stamps the term * and persists at the contiguous next LSN. */ static void test_on_persist_entry_stamps_term_primary(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_op_pr_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 4; cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); atomic_store(&c->ha.role, CTRL_PRIMARY); uint8_t blob[64]; size_t n = pack_entry(blob, WAL_NOOP, NULL, 0); /* own-term promotion no-op */ TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, /*index*/1, T, blob, n)); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1)); TEST_ASSERT_EQUAL_UINT64(2, c->persist.current_lsn); teardown_cluster(c); printf("test_on_persist_entry_stamps_term_primary: PASS\n"); } /* Test: a forced persist failure (fdatasync fail-loud seam on the replicated * write) makes on_persist_entry return non-zero and DOES NOT advance the log / * ACK (persist-before-ACK preserved). */ static void test_on_persist_entry_failure_propagates_no_ack(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_op_fail_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); atomic_store(&c->ha.role, CTRL_STANDBY); uint64_t durable_before = atomic_load(&c->persist.durable_lsn); /* Force the persist-before-ACK fdatasync to be treated as failed. */ atomic_store(&c->persist.test_force_fsync_fail, 1); uint8_t blob[64]; size_t n = pack_entry(blob, WAL_JOB_UPDATE, "y", 1); int rc = ha_cb_persist_entry_for_test(&c->ha, /*index*/1, 7, blob, n); TEST_ASSERT_NOT_EQUAL_INT(0, rc); /* failure propagates → caller does NOT ACK */ /* Persist-before-ACK durability contract: the entry is NOT durable, so the * durable watermark must NOT advance (the standby would ACK only on a * successful, durable write — the non-zero rc above is exactly the signal * that suppresses the ACK). durable_lsn (not the write cursor current_lsn) * is the durability authority; it stays put on fdatasync failure. */ TEST_ASSERT_EQUAL_UINT64(durable_before, atomic_load(&c->persist.durable_lsn)); atomic_store(&c->persist.test_force_fsync_fail, 0); teardown_cluster(c); printf("test_on_persist_entry_failure_propagates_no_ack: PASS\n"); } /* ================================================================== */ /* Task 2.2 — on_get_entry_term + on_truncate_from callback bodies */ /* ================================================================== */ /* Persist a couple of replicated entries (each at a 1-based index == LSN) * through the WIRED on_persist_entry callback, stamping a leader term per * entry. Returns the (started) cluster the caller tears down. */ static cluster_t* persist_two_entries(const char* tmp, uint64_t t1, uint64_t t2) { cluster_t* c = make_cluster(tmp); if (!c) return NULL; atomic_store(&c->ha.role, CTRL_STANDBY); uint8_t blob[64]; size_t n1 = pack_entry(blob, WAL_JOB_UPDATE, "a", 1); if (ha_cb_persist_entry_for_test(&c->ha, /*index*/1, t1, blob, n1) != 0) { teardown_cluster(c); return NULL; } size_t n2 = pack_entry(blob, WAL_JOB_UPDATE, "b", 1); if (ha_cb_persist_entry_for_test(&c->ha, /*index*/2, t2, blob, n2) != 0) { teardown_cluster(c); return NULL; } return c; } /* Test: on_get_entry_term returns the stamped per-LSN term for present entries, * the lowest-known term for a legacy-0 entry, and 0 for an absent LSN * (R1.5, R3.1). */ static void test_on_get_entry_term_present_legacy_absent(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_get_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); atomic_store(&c->ha.role, CTRL_STANDBY); uint8_t blob[64]; /* LSN 1: real term 6. */ size_t n = pack_entry(blob, WAL_JOB_UPDATE, "r", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, /*index*/1, /*term*/6, blob, n)); /* LSN 2: legacy-0 entry (leader term 0) → reads back as WAL_LEGACY_TERM. */ n = pack_entry(blob, WAL_JOB_UPDATE, "l", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, /*index*/2, /*term*/0, blob, n)); /* Present real-term entry returns its term verbatim. */ TEST_ASSERT_EQUAL_UINT64(6, ha_cb_get_entry_term_for_test(&c->ha, 1)); /* Present legacy-0 entry maps to the lowest-known real term. */ TEST_ASSERT_EQUAL_UINT64(LEGACY_LOWEST_TERM, ha_cb_get_entry_term_for_test(&c->ha, 2)); /* Absent LSNs (0 reserved, 3 never appended) return 0. */ TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 0)); TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 3)); teardown_cluster(c); printf("test_on_get_entry_term_present_legacy_absent: PASS\n"); } /* Test: on_truncate_from(k) deletes local entries at LSN >= k; the prefix < k * stays intact and on_get_entry_term returns 0 for the truncated LSNs while the * term index shrinks consistently (R1.8, R3.1, R3.2). */ static void test_on_truncate_from_drops_suffix(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_trunc_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); /* Persist LSN 1 (term 3) and LSN 2 (term 5). */ cluster_t* c = persist_two_entries(tmp, /*t1*/3, /*t2*/5); TEST_ASSERT_NOT_NULL(c); /* Both present before truncation. */ TEST_ASSERT_EQUAL_UINT64(3, ha_cb_get_entry_term_for_test(&c->ha, 1)); TEST_ASSERT_EQUAL_UINT64(5, ha_cb_get_entry_term_for_test(&c->ha, 2)); TEST_ASSERT_EQUAL_UINT64(3, c->persist.current_lsn); /* Truncate from LSN 2 → entries at LSN >= 2 are deleted. */ TEST_ASSERT_EQUAL_INT(0, ha_cb_truncate_from_for_test(&c->ha, 2)); /* Prefix < 2 intact, suffix >= 2 gone (term index returns 0). */ TEST_ASSERT_EQUAL_UINT64(3, ha_cb_get_entry_term_for_test(&c->ha, 1)); TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 2)); TEST_ASSERT_EQUAL_UINT64(2, c->persist.current_lsn); teardown_cluster(c); printf("test_on_truncate_from_drops_suffix: PASS\n"); } /* Test: on_truncate_from(1) deletes the whole log (LSN >= 1); every entry's * term reads back absent and the tip collapses to LSN 1. */ static void test_on_truncate_from_removes_all(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_trunc_all_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = persist_two_entries(tmp, /*t1*/2, /*t2*/2); TEST_ASSERT_NOT_NULL(c); TEST_ASSERT_EQUAL_INT(0, ha_cb_truncate_from_for_test(&c->ha, 1)); TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 1)); TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 2)); TEST_ASSERT_EQUAL_UINT64(1, c->persist.current_lsn); teardown_cluster(c); printf("test_on_truncate_from_removes_all: PASS\n"); } /* ================================================================== */ /* Task 2.3 — on_send_append_entries callback body */ /* ================================================================== */ /* A dummy non-NULL endpoint pointer for a registered standby slot. The send * is diverted to the capture hook, so the pointer is never dereferenced. */ static int g_ae_dummy_ep; /* Register one standby in the cluster's embedded replicator at slot 0, wiring * ha.repl so the callback maps peer_idx 0 → this endpoint. */ static void register_one_standby(cluster_t *c) { c->ha.repl = &c->persist.repl; TEST_ASSERT_EQUAL_INT(0, repl_add_standby(&c->persist.repl, &g_ae_dummy_ep, "sb0", 7000)); } /* Test: on_send_append_entries for n_entries==1 builds an envelope carrying the * correct (prev_log_index, prev_log_term, leader_commit, n_entries) and, for the * single entry sourced from the leader's own WAL at prev_log_index+1, its term * (== persist_entry_term(prev_log_index+1)) and payload (R1.3, R3.1). */ static void test_send_append_entries_one_entry_from_wal(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_ae1_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); atomic_store(&c->ha.role, CTRL_PRIMARY); /* Populate the leader's own WAL BEFORE registering a standby — otherwise the * persist append fans out via repl_broadcast to the dummy int ep (ASAN * global-buffer-overflow reading ep->peer). With no standby, the append is a * no-op broadcast. The entry to be sent is at prev_log_index+1 == LSN 2. */ uint8_t blob[64]; size_t n1 = pack_entry(blob, WAL_JOB_UPDATE, "p", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, /*index*/1, /*term*/4, blob, n1)); size_t n2 = pack_entry(blob, WAL_JOB_UPDATE, "QQ", 2); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, /*index*/2, /*term*/6, blob, n2)); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); /* Now register the standby so peer_idx 0 maps to a slot; the AE send below * is diverted to the capture hook, so the dummy ep is never dereferenced. */ register_one_standby(c); /* The entry-term oracle the send must carry for LSN 2. */ uint64_t expect_term = persist_entry_term(&c->persist, 2); TEST_ASSERT_EQUAL_UINT64(6, expect_term); /* Drive the callback: prev_log_index=1, prev_log_term=4 (term of LSN 1), * leader_commit=1, n_entries=1 → sends the entry at LSN 2. */ repl_ae_capture_t cap; memset(&cap, 0, sizeof(cap)); repl_send_ae_capture_hook = ∩ int rc = ha_cb_send_append_entries_for_test(&c->ha, /*peer*/0, /*prev_idx*/1, /*prev_term*/4, /*leader_commit*/1, /*n*/1); repl_send_ae_capture_hook = NULL; TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(1, cap.called); TEST_ASSERT_EQUAL_INT(0, cap.peer_idx); TEST_ASSERT_EQUAL_UINT64(1, cap.prev_log_index); TEST_ASSERT_EQUAL_UINT64(4, cap.prev_log_term); TEST_ASSERT_EQUAL_UINT64(1, cap.leader_commit); TEST_ASSERT_EQUAL_INT(1, cap.n_entries); /* The single entry is sourced from the local WAL at prev_log_index+1 == 2. */ TEST_ASSERT_EQUAL_UINT64(2, cap.entry_lsn); TEST_ASSERT_EQUAL_UINT64(expect_term, cap.entry_term); TEST_ASSERT_EQUAL_UINT8(WAL_JOB_UPDATE, cap.entry_type); TEST_ASSERT_EQUAL_UINT32(2, cap.payload_len); TEST_ASSERT_EQUAL_MEMORY("QQ", cap.payload, 2); teardown_cluster(c); printf("test_send_append_entries_one_entry_from_wal: PASS\n"); } /* Test: on_send_append_entries for n_entries==0 emits a heartbeat carrying the * prev tuple + leader_commit and ZERO entries (R1.3). */ static void test_send_append_entries_heartbeat_zero_entries(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_ae0_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); atomic_store(&c->ha.role, CTRL_PRIMARY); /* One entry on disk so prev_log_index=1 is a real tip; heartbeat sends none. * Persist BEFORE registering the standby (see one-entry test note). */ uint8_t blob[64]; size_t n1 = pack_entry(blob, WAL_JOB_UPDATE, "h", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, /*index*/1, /*term*/3, blob, n1)); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); register_one_standby(c); repl_ae_capture_t cap; memset(&cap, 0, sizeof(cap)); repl_send_ae_capture_hook = ∩ int rc = ha_cb_send_append_entries_for_test(&c->ha, /*peer*/0, /*prev_idx*/1, /*prev_term*/3, /*leader_commit*/1, /*n*/0); repl_send_ae_capture_hook = NULL; TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(1, cap.called); TEST_ASSERT_EQUAL_UINT64(1, cap.prev_log_index); TEST_ASSERT_EQUAL_UINT64(3, cap.prev_log_term); TEST_ASSERT_EQUAL_UINT64(1, cap.leader_commit); TEST_ASSERT_EQUAL_INT(0, cap.n_entries); /* No entry sourced for a heartbeat. */ TEST_ASSERT_EQUAL_UINT32(0, cap.payload_len); teardown_cluster(c); printf("test_send_append_entries_heartbeat_zero_entries: PASS\n"); } /* ================================================================== */ /* Task 2.4 — on_commit_advanced trigger + commit-source switch */ /* ================================================================== */ /* * Task 2.4 feeds the library-supplied current-term-gated `commit_index` into * the existing synchronous commit barrier as the commit SOURCE. The barrier * mechanism is unchanged: committed_lsn still advances only when self+majority * agree AND the bytes are durable locally (durable_lsn cap). The library commit * index becomes an ADDITIONAL ceiling so the barrier never advances committed * past the library's current-term-safe point. on_commit_advanced only TRIGGERS * (records the ceiling + re-evaluates); it never performs durability itself. * * These cases build the commit engine standalone over a stub persist_t + * replicator (the test_commit_engine.c pattern) so the barrier interaction is * tested in isolation, plus one case driving the WIRED on_commit_advanced * callback end-to-end through ha_cb_commit_advanced_for_test. * * RED: against the no-op ha_cb_commit_advanced_stub (and the missing * commit_set_raft_commit_index ceiling), the library commit index is NOT a * commit source — committed_lsn would advance on durable-majority alone with * no current-term gate. GREEN: the wired callback feeds the ceiling so the * barrier holds until BOTH durable majority and the library commit index agree. */ /* Stub-persist fixture for the commit-source tests (mirrors test_commit_engine.c). */ static persist_t g_t24_p; static commit_engine_t g_t24_ce; static int g_t24_eps[MAX_STANDBYS]; static void t24_engine_setup(int n_peers, int n_standbys, uint64_t appended_lsn) { char name[16]; memset(&g_t24_p, 0, sizeof(g_t24_p)); TEST_ASSERT_EQUAL_INT(0, repl_init(&g_t24_p.repl)); for (int i = 0; i < n_standbys; i++) { snprintf(name, sizeof(name), "sb%d", i); TEST_ASSERT_EQUAL_INT( i, repl_add_standby(&g_t24_p.repl, &g_t24_eps[i], name, 7000 + (uint32_t)i)); } g_t24_p.current_lsn = appended_lsn + 1; atomic_store(&g_t24_p.committed_lsn, 0); atomic_store(&g_t24_p.durable_lsn, appended_lsn); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&g_t24_ce, &g_t24_p, n_peers, 8)); } static void t24_engine_teardown(void) { commit_engine_fini(&g_t24_ce); repl_fini(&g_t24_p.repl); } static void t24_standby_ack(int idx, uint64_t acked_lsn) { atomic_store(&g_t24_p.repl.standbys[idx].acked_lsn, acked_lsn); commit_on_ack(&g_t24_ce, idx, acked_lsn); } static uint64_t t24_committed(void) { return atomic_load(&g_t24_p.committed_lsn); } /* commit-ceiling-optA — REFRAMED from test_commit_advanced_is_commit_source. * * The durable-majority barrier is AUTHORITATIVE for commit-advance; the recorded * library commit_index is inert-advisory and does NOT cap committed_lsn. A * durable majority covering LSN 3 commits to 3 EVEN WHILE the recorded library * commit_index lags at 1 — this is the starvation the old ceiling caused (commit * pinned at 1 below the safe durable majority) and the Option A fix removes. * * Meaningful both ways: it FAILS if commit were wrongly gated by the recorded * index (committed would stick at 1), AND it FAILS if commit over-advanced past * the durable majority (it asserts == 3, the exact durable watermark). */ static void test_commit_advances_to_durable_majority_index_inert(void) { /* 3 peers = self + 2 standbys; majority needs self + 1 standby. */ t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/3); /* Record an advisory library commit_index that LAGS at 1 (e.g. match_index[] * not yet advanced via catch-up). Under the old design this pinned committed * at 1; under Option A it is inert. */ commit_set_raft_commit_index(&g_t24_ce, 1); /* A standby ACKs LSN 3; self is durable to LSN 3 → durable majority = 3. * committed advances to the durable majority (3), NOT pinned at the lagging * advisory index (1). */ t24_standby_ack(0, 3); TEST_ASSERT_EQUAL_UINT64(3, t24_committed()); /* Advancing the advisory record changes nothing — already at the durable * majority; record is inert. */ commit_set_raft_commit_index(&g_t24_ce, 3); TEST_ASSERT_EQUAL_UINT64(3, t24_committed()); t24_engine_teardown(); printf("test_commit_advances_to_durable_majority_index_inert: PASS\n"); } /* commit-ceiling-optA — REFRAMED from test_commit_advanced_ceiling_caps_durable_majority. * * The recorded library commit_index is NOT a ceiling: committed advances to the * full durable majority even when the advisory index sits below it. Pins the * removal of the redundant ceiling. * * Meaningful both ways: FAILS if the advisory index still capped committed at 3 * (the old, now-rejected behavior), AND FAILS if committed over-advanced past * the durable majority of 5 (asserts == 5 exactly). */ static void test_commit_index_record_is_not_a_ceiling(void) { t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/5); /* Advisory record lags at 3 while the durable majority will reach 5. */ commit_set_raft_commit_index(&g_t24_ce, 3); /* Durable majority covers up to LSN 5 (self durable 5 + standby acked 5). * committed advances to 5 — the lagging advisory index (3) does NOT cap it. */ t24_standby_ack(0, 5); TEST_ASSERT_EQUAL_UINT64(5, t24_committed()); /* Catching the advisory record up to 5 is a no-op (already there). */ commit_set_raft_commit_index(&g_t24_ce, 5); TEST_ASSERT_EQUAL_UINT64(5, t24_committed()); t24_engine_teardown(); printf("test_commit_index_record_is_not_a_ceiling: PASS\n"); } /* commit-ceiling-optA — RETAINED (unchanged contract): committed never advances * past what is durable locally. The durable self-cap (persist_durable_lsn) is * still authoritative regardless of any advisory library index. With the * advisory index well ahead (4) but self durable only to LSN 2 and the standby * acked LSN 2, committed stays at the durable-majority floor (2). This invariant * is unchanged by Option A — the durable cap was always orthogonal to the * removed ceiling — but is re-pinned here to guard Commit Durability (R5.4). */ static void test_commit_advanced_does_not_bypass_durable_cap(void) { t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/4); atomic_store(&g_t24_p.durable_lsn, 2); /* self durable only to 2 */ t24_standby_ack(0, 2); /* standby agrees to 2 */ commit_set_raft_commit_index(&g_t24_ce, 4); /* advisory index well ahead */ commit_on_ack(&g_t24_ce, 0, 2); /* Durable-majority floor is 2; nothing (advisory index 4 included) raises it. */ TEST_ASSERT_EQUAL_UINT64(2, t24_committed()); t24_engine_teardown(); printf("test_commit_advanced_does_not_bypass_durable_cap: PASS\n"); } /* commit-ceiling-optA — NEW load-bearing safety test: the own-term writable gate * (Figure-8). With Option A removing the library-commit-index ceiling, the * own-term NOOP gate (commit_set_writable_lsn / wait_done_locked) is now the * load-bearing realization of "no acked-committed prior-term entry until the * current-term NOOP is durable on a majority". * * Scenario: self holds a PRIOR-term entry at LSN 1 and the current-term NOOP at * LSN 2 (the promotion NOOP). The own-term gate is armed at LSN 2. A durable * majority reaches LSN 1 FIRST (the NOOP not yet majority-durable): * - committed_lsn MAY advance to 1 (the barrier is contiguous-prefix), but a * WAIT for LSN 1 must NOT complete — the gate blocks acknowledgement until * committed_lsn >= writable_lsn (2). This is the safety property: the * prior-term entry at LSN 1 is NOT acknowledged-committed yet. * Then the NOOP becomes majority-durable (committed reaches 2): * - the gate opens and the WAIT completes. * * RED if the writable gate were bypassed: a WAIT(lsn=1) would complete as soon * as committed_lsn >= 1, acknowledging a prior-term entry before the own-term * NOOP is majority-durable — the Figure-8 violation. GREEN: it blocks (times * out) until the NOOP commits, then completes. */ static void test_own_term_gate_blocks_priorterm_ack_until_noop_durable(void) { /* 3 peers; majority needs self + 1 standby. Appended tip = LSN 2 (prior-term * entry at 1 + current-term NOOP at 2). */ t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/2); /* Arm the own-term gate at the NOOP's LSN (2): no acknowledgement of any * entry until committed_lsn >= 2 (the current-term NOOP is majority-durable). */ commit_set_writable_lsn(&g_t24_ce, 2); /* Durable majority reaches LSN 1 only: self durable to 2, but the standby has * acked just LSN 1, so the median (self + 1 of 2 peers) covers LSN 1. */ atomic_store(&g_t24_p.durable_lsn, 2); t24_standby_ack(0, 1); /* The contiguous-prefix barrier MAY advance committed_lsn to 1 ... */ TEST_ASSERT_EQUAL_UINT64(1, t24_committed()); /* ... but a WAIT for LSN 1 must NOT complete: the own-term gate (writable_lsn * == 2) holds it. Pre-NOOP-durable, this is the Figure-8 safety block — the * prior-term entry at LSN 1 is committed-but-not-acknowledged. A short * timeout proves the gate blocks (returns -ETIMEDOUT, not 0). */ TEST_ASSERT_EQUAL_INT(-ETIMEDOUT, commit_submit(&g_t24_ce, /*lsn*/1, COMMIT_MODE_WAIT, /*timeout_ms*/150)); /* The current-term NOOP at LSN 2 now becomes majority-durable (standby acks * LSN 2). committed_lsn advances to 2 and the gate opens. */ t24_standby_ack(0, 2); TEST_ASSERT_EQUAL_UINT64(2, t24_committed()); /* The WAIT for LSN 1 now completes: committed_lsn (2) >= lsn (1) AND * >= writable_lsn (2). The prior-term entry is acknowledged only AFTER the * own-term NOOP is majority-durable. */ TEST_ASSERT_EQUAL_INT(0, commit_submit(&g_t24_ce, /*lsn*/1, COMMIT_MODE_WAIT, /*timeout_ms*/500)); t24_engine_teardown(); printf("test_own_term_gate_blocks_priorterm_ack_until_noop_durable: PASS\n"); } /* Test (pins the own-term-gate DECISION — RETAINED, complementary, no deadlock): * the own-term writable_lsn gate is a WAIT-completion predicate, NOT a cap on * committed_lsn. With the gate armed at LSN 2 and the durable majority at 2, * committed advances to 2 AND a WAIT(lsn=2) completes — the gate never holds * committed_lsn back, and once committed reaches the gate the WAIT completes (no * deadlock). The advisory library index is irrelevant here (inert under optA). */ static void test_own_term_gate_retained_no_deadlock(void) { t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/2); /* Arm the own-term gate at LSN 2 (the load-bearing complementary gate). */ commit_set_writable_lsn(&g_t24_ce, 2); /* Durable majority reaches LSN 2. */ t24_standby_ack(0, 2); commit_set_raft_commit_index(&g_t24_ce, 2); /* advisory; inert */ commit_on_ack(&g_t24_ce, 0, 2); /* committed_lsn advanced to 2 (own-term gate never caps committed_lsn). */ TEST_ASSERT_EQUAL_UINT64(2, t24_committed()); /* A WAIT for LSN 2 completes immediately: committed_lsn (2) >= lsn (2) AND * >= writable_lsn (2). No deadlock between the gate and the barrier. */ TEST_ASSERT_EQUAL_INT(0, commit_submit(&g_t24_ce, /*lsn*/2, COMMIT_MODE_WAIT, /*timeout_ms*/500)); t24_engine_teardown(); printf("test_own_term_gate_retained_no_deadlock: PASS\n"); } /* commit-ceiling-optA — REFRAMED from test_on_commit_advanced_wired_feeds_barrier. * * The WIRED on_commit_advanced callback (ha_cb_commit_advanced_for_test) routes * the library commit_index into the cluster's commit engine as an INERT-ADVISORY * record — it records ce->raft_commit_index and wakes the barrier, but does NOT * gate committed_lsn. The durable-majority barrier is authoritative: with a * durable majority covering LSN 2, committed advances to 2 even though the wired * callback last fed a LAGGING commit_index of 1. * * Meaningful both ways: FAILS if the wired callback still capped committed at 1 * (the old, now-rejected ceiling — the starvation), AND FAILS if committed * over-advanced past the durable majority of 2 (asserts == 2). Also pins that * the wired callback still RECORDS the advisory index (sink intact). */ static void test_on_commit_advanced_wired_records_advisory_index(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_commitadv_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); /* Init the cluster's embedded commit engine over the same persist as a * 3-peer cluster (self + 2 standbys), so a majority is self + 1 standby. */ TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, /*n_peers*/3, /*n_vcpus*/8)); /* Persist two entries BEFORE registering a standby — otherwise the primary * own-append fans out via repl_broadcast to the dummy int ep (ASAN * global-buffer-overflow reading ep->peer); with no standby the broadcast * is a no-op. Append LSN 1,2 with term 4 → durable floor reaches LSN 2. */ atomic_store(&c->ha.role, CTRL_PRIMARY); uint8_t blob[64]; size_t n1 = pack_entry(blob, WAL_JOB_UPDATE, "a", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 4, blob, n1)); size_t n2 = pack_entry(blob, WAL_JOB_UPDATE, "b", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 2, 4, blob, n2)); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); /* Now register the standby so a majority (self + 1) is reachable. The ACK * below is driven directly into the registry + commit_on_ack (no network). */ TEST_ASSERT_EQUAL_INT(0, repl_add_standby(&c->persist.repl, &g_ae_dummy_ep, "sb0", 7000)); /* Fire the WIRED callback with a LAGGING commit_index of 1 (e.g. match_index[] * not yet caught up). Under optA this only RECORDS the advisory index. */ ha_cb_commit_advanced_for_test(&c->ha, 1); TEST_ASSERT_EQUAL_UINT64(1, c->commit.raft_commit_index); /* sink intact */ /* Standby ACKs LSN 2 → durable majority covers LSN 2. committed advances to * the durable majority (2), NOT pinned at the lagging advisory index (1): * the advisory record does not gate commit. */ atomic_store(&c->persist.repl.standbys[0].acked_lsn, 2); commit_on_ack(&c->commit, 0, 2); TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn)); /* A later callback advancing the advisory record to 2 is a no-op for commit. */ ha_cb_commit_advanced_for_test(&c->ha, 2); TEST_ASSERT_EQUAL_UINT64(2, c->commit.raft_commit_index); TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn)); commit_engine_fini(&c->commit); teardown_cluster(c); printf("test_on_commit_advanced_wired_records_advisory_index: PASS\n"); } /* ================================================================== */ /* Task 2.5 — library init restore from the recovered WAL floor */ /* ================================================================== */ /* Test: ha_init_raft restores the library log/commit floor from the * recovered WAL instead of booting at LSN 0 (R1.5, R2.1). * * After a member recovers a WAL whose durable tip is LSN 3 (term 7) with a * committed prefix at LSN 2, raft_init must boot with: * raft.log_len == 3 (initial_log_len = persist_durable_floor) * raft.commit_index == 2 (initial_commit_index = recovered committed_lsn) * and the last-log term must be recoverable via the wired on_get_entry_term * (== 7). Booting from an empty log (log_len/commit_index == 0) is the pre-fix * behaviour this test fails on. */ static void test_init_restores_recovered_floor(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_initrestore_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 7; /* Phase 1: lay down a 3-entry WAL (term 7) and flush so the recovered tip * reaches LSN 3. */ { cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); atomic_store(&c->ha.role, CTRL_PRIMARY); uint8_t blob[64]; for (uint64_t i = 1; i <= 3; i++) { size_t n = pack_entry(blob, WAL_JOB_UPDATE, "x", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, i, T, blob, n)); } TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); teardown_cluster(c); } /* Phase 2: recover into a fresh cluster; durable floor == 3. Simulate a * recovered committed prefix at LSN 2 (recovery re-derives the durable tip; * the committed position is set by the surrounding boot path). */ { cluster_t* c2 = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c2); TEST_ASSERT_EQUAL_INT(0, persist_recover(&c2->persist)); TEST_ASSERT_EQUAL_UINT64(3, persist_durable_floor(&c2->persist)); atomic_store(&c2->persist.committed_lsn, 2ULL); /* Initialize the embedded raft as a single-node cluster. The restore * must source the floor from the recovered persist, not LSN 0. */ ha_init_raft(&c2->ha, c2, /*n_peers*/1); TEST_ASSERT_EQUAL_UINT64(3, c2->ha.raft.log_len); TEST_ASSERT_EQUAL_UINT64(2, c2->ha.raft.commit_index); /* The recovered last-log term is surfaced via the wired callback. */ TEST_ASSERT_EQUAL_UINT64(T, ha_cb_get_entry_term_for_test(&c2->ha, c2->ha.raft.log_len)); /* All five term-safety callbacks remain wired (non-NULL). */ TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs); TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_persist_entry); TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_get_entry_term); TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_send_append_entries); TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_truncate_from); TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_commit_advanced); /* Existing election callbacks remain wired. */ TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_become_leader); TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_persist); teardown_cluster(c2); } printf("test_init_restores_recovered_floor: PASS\n"); } /* Test: an empty-WAL first boot restores a zero floor (no spurious non-zero * indices) — the recovery-read-failure / empty-log edge of R1.5. */ static void test_init_empty_wal_zero_floor(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_initempty_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); TEST_ASSERT_EQUAL_INT(0, persist_recover(&c->persist)); TEST_ASSERT_EQUAL_UINT64(0, persist_durable_floor(&c->persist)); ha_init_raft(&c->ha, c, /*n_peers*/1); TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.log_len); TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.commit_index); teardown_cluster(c); printf("test_init_empty_wal_zero_floor: PASS\n"); } /* ================================================================== */ /* Task 3.1 — drive raft_log_append on the primary append path */ /* ================================================================== */ /* Bring a make_cluster()'d node up as the raft LEADER so raft_log_append is * enabled (it returns -EPERM off the leader). We init the library from the * recovered floor (task 2.5) and set the live election term + raft role * directly — modelling a node that has already won an election at `term` * (the cerebrum raft-sim convention: set state.term after a clean election; * raft_log_append stamps entries from s->term). */ static void make_leader(cluster_t* c, uint64_t term, int n_peers) { atomic_store(&c->ha.role, CTRL_PRIMARY); atomic_store(&c->ha.term, term); ha_init_raft(&c->ha, c, n_peers); c->ha.raft.term = term; c->ha.raft.role = RAFT_LEADER; } /* Test: a primary job-state append driven through ha_primary_log_append goes * through the library — the library assigns the LSN (== s->log_len post-append), * stamps the live election term into the WAL header, and keeps the persist * write cursor co-tracked with the library index (current_lsn == log_len + 1). * Against the pre-3.1 hand-rolled path this entry point does not exist * (compile RED) — and once it exists the library MUST be the LSN/term source * (R1.1, R1.2, R2.1). */ static void test_primary_log_append_stamps_term_and_assigns_lsn(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_plap_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 7; cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); make_leader(c, T, /*n_peers*/1); /* Fresh leader, empty log: first append takes library index 1. */ TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.log_len); wal_jobstate_t js = { .jobid = 42, .state = 3 }; uint64_t lsn = 0; TEST_ASSERT_EQUAL_INT(0, ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &lsn)); /* (a) The LIBRARY assigned the LSN and grew its log to it. */ TEST_ASSERT_EQUAL_UINT64(1, lsn); TEST_ASSERT_EQUAL_UINT64(1, c->ha.raft.log_len); /* (b) The WAL header recorded the live election term for that LSN. */ TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1)); /* (d) Co-tracking invariant: the persist write cursor advanced exactly in * lockstep with the library index — current_lsn == log_len + 1 (next LSN). */ TEST_ASSERT_EQUAL_UINT64(c->ha.raft.log_len + 1, c->persist.current_lsn); /* A second append keeps the two counters co-tracked. */ js.jobid = 43; TEST_ASSERT_EQUAL_INT(0, ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &lsn)); TEST_ASSERT_EQUAL_UINT64(2, lsn); TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.log_len); TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 2)); TEST_ASSERT_EQUAL_UINT64(c->ha.raft.log_len + 1, c->persist.current_lsn); teardown_cluster(c); printf("test_primary_log_append_stamps_term_and_assigns_lsn: PASS\n"); } /* Test: the own-term promotion no-op (ha_arm_own_term_gate) is routed through * the library, so the WAL_NOOP entry records the newly established term (not 0) * and the library index tracks it (R1.2). Pre-3.1 the noop went through a * hand-rolled persist_wal_async with a best-effort p->term and did NOT advance * s->log_len. */ static void test_own_term_noop_records_live_term_via_library(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_noop_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 5; cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); make_leader(c, T, /*n_peers*/1); /* Register the persist hooks then arm the own-term gate exactly as the * promotion tail does (ha_promote_to_primary / raft_cb_become_leader). */ ha_arm_own_term_gate_for_test(c); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); /* The no-op landed at library index 1 with the live term recorded. */ TEST_ASSERT_EQUAL_UINT64(1, c->ha.raft.log_len); TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1)); TEST_ASSERT_EQUAL_UINT64(c->ha.raft.log_len + 1, c->persist.current_lsn); teardown_cluster(c); printf("test_own_term_noop_records_live_term_via_library: PASS\n"); } /* Test: when raft leadership is NOT active (the library is not RAFT_LEADER — * the init-time non-raft promotion road and engine-less test contexts), * ha_primary_log_append still appends locally through the legacy path so the * non-raft promotion tail keeps working, WITHOUT growing the library log (the * library is not tracking the log on that road, so no co-tracking hazard). * The entry still records the live election term via persist_set_term. */ static void test_primary_log_append_non_leader_legacy_fallback(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_plapfb_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 6; cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); /* Non-raft promotion road: controller role PRIMARY + live term, but the * raft library was never driven to LEADER (raft.role stays FOLLOWER). */ atomic_store(&c->ha.role, CTRL_PRIMARY); atomic_store(&c->ha.term, T); ha_init_raft(&c->ha, c, /*n_peers*/1); TEST_ASSERT_NOT_EQUAL_INT(RAFT_LEADER, c->ha.raft.role); wal_jobstate_t js = { .jobid = 7, .state = 1 }; uint64_t lsn = 0; TEST_ASSERT_EQUAL_INT(0, ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &lsn)); /* Appended at the contiguous next LSN (current_lsn started at 1). */ TEST_ASSERT_EQUAL_UINT64(1, lsn); TEST_ASSERT_EQUAL_UINT64(2, c->persist.current_lsn); /* Live term still recorded for the entry. */ TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1)); /* The library log was NOT grown by the legacy fallback (no raft leadership). */ TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.log_len); teardown_cluster(c); printf("test_primary_log_append_non_leader_legacy_fallback: PASS\n"); } /* ================================================================== */ /* Task 3.2 — route heartbeat/replication intake through the library */ /* raft_on_append_entries_request: UpdateTerm (G5) + logOk (G2) + */ /* (term,lsn) conflict-truncate (G3) via the wired callbacks. */ /* */ /* These drive the production intake helper ha_consume_append_entries */ /* (the in-process route that replaces the term-blind handling in */ /* ha_consume_heartbeat_committed + the committed-boundary truncate in */ /* ha_rejoin_reconcile). RED: that helper does not exist pre-3.2 */ /* (compile RED); once it exists the three observable behaviors hold. */ /* ================================================================== */ /* Make a node a STANDBY follower at a given term + raft log floor, with the * raft library initialized so the intake can drive it. Seeds the library log * to match the persisted WAL tip (the ha_init_raft floor restore). */ static void make_follower(cluster_t* c, uint64_t term, int n_peers) { atomic_store(&c->ha.role, CTRL_STANDBY); atomic_store(&c->ha.term, term); ha_init_raft(&c->ha, c, n_peers); c->ha.raft.term = term; c->ha.raft.role = RAFT_FOLLOWER; } /* Build a decoded single-entry WalCatchupEntry (n_entries==0 path == one entry * carried in the top-level container, matching the on-wire single-entry mode). * For a heartbeat use n_entries set so the helper sees zero entries. */ static void fill_ae_entry(wal_catchup_entry_t* e, int is_heartbeat, uint64_t prev_log_index, uint64_t prev_log_term, uint64_t leader_commit, uint64_t entry_lsn, uint8_t entry_type, uint64_t entry_term, uint8_t* payload, uint32_t plen) { memset(e, 0, sizeof(*e)); e->prev_log_index = prev_log_index; e->prev_log_term = prev_log_term; e->leader_commit = leader_commit; if (is_heartbeat) { e->n_entries = 0; e->lsn = 0; } else { /* single-entry mode: top-level container carries the entry, n_entries==1 * with no nested sub-entries (one-entry AppendEntries). */ e->n_entries = 1; e->lsn = entry_lsn; e->entry_type = entry_type; e->term = entry_term; e->raw_payload = payload; e->raw_payload_len = plen; } } /* (a) A higher-term heartbeat (0-entry AppendEntries) routed through the library * makes the follower ADOPT the higher term and step down — term recorded * (R2.6 / G5). Pre-3.2 ha_consume_heartbeat_committed only DROPPED stale * heartbeats and never adopted a higher term. */ static void test_intake_higher_term_heartbeat_adopts_term(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_intake_ht_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); make_follower(c, /*term*/3, /*n_peers*/3); /* The follower briefly believes itself a candidate/leader at term 3. */ c->ha.raft.role = RAFT_LEADER; /* Heartbeat from a leader at term 7, empty log (prev=0), commit 0. The * heartbeat's top-level container carries the leader's live term. */ wal_catchup_entry_t hb; fill_ae_entry(&hb, /*heartbeat*/1, /*prev_idx*/0, /*prev_term*/0, /*leader_commit*/0, 0, 0, 0, NULL, 0); hb.term = 7; int rc = ha_consume_append_entries(c, &hb, /*leader_idx*/1); TEST_ASSERT_EQUAL_INT(0, rc); /* The library adopted the higher term and stepped down to follower. */ TEST_ASSERT_EQUAL_UINT64(7, c->ha.raft.term); TEST_ASSERT_NOT_EQUAL_INT(RAFT_LEADER, c->ha.raft.role); teardown_cluster(c); printf("test_intake_higher_term_heartbeat_adopts_term: PASS\n"); } /* (b) A divergent same-LSN tail is truncated and the leader's entry accepted * (R3.1/R3.2 / G3), and the committed prefix below the divergence is untouched * (R3.3). The follower locally holds [LSN1@term2, LSN2@term2]; the leader sends * an AppendEntries appending LSN2@term5 on top of LSN1@term2. The library * detects the (term,lsn) conflict at LSN2, truncates LSN>=2, and re-persists * the leader's LSN2@term5. LSN1 (the agreed prefix) stays intact. */ static void test_intake_divergent_tail_truncated_and_reconverged(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_intake_div_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); make_follower(c, /*term*/2, /*n_peers*/3); /* Seed the local log: LSN1@term2, LSN2@term2 (the divergent tail). */ uint8_t blob[64]; size_t n = pack_entry(blob, WAL_JOB_UPDATE, "a", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 2, blob, n)); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 2, 2, blob, n)); c->ha.raft.log_len = 2; TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 1)); TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 2)); /* Leader (term 5) sends AppendEntries: prev=(LSN1,term2), entry LSN2@term5. * logOk holds at LSN1 (term2==term2); conflict at LSN2 (local term2 != * incoming term5) → truncate LSN>=2 then persist LSN2@term5. */ uint8_t lpay[1] = { 'b' }; wal_catchup_entry_t ae; fill_ae_entry(&ae, /*heartbeat*/0, /*prev_idx*/1, /*prev_term*/2, /*leader_commit*/1, /*entry_lsn*/2, WAL_JOB_UPDATE, /*entry_term*/5, lpay, 1); int rc = ha_consume_append_entries(c, &ae, /*leader_idx*/1); TEST_ASSERT_EQUAL_INT(0, rc); /* (term,lsn) reconverged: LSN2 now carries the leader's term 5. */ TEST_ASSERT_EQUAL_UINT64(5, persist_entry_term(&c->persist, 2)); /* The agreed committed prefix (LSN1@term2) is untouched (R3.3). */ TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 1)); /* The higher leader term was adopted on the RPC (G5). */ TEST_ASSERT_EQUAL_UINT64(5, c->ha.raft.term); teardown_cluster(c); printf("test_intake_divergent_tail_truncated_and_reconverged: PASS\n"); } /* (c) An unreconcilable member (logOk fails: the leader's prev tuple does not * match the follower's log) is REJECTED — the follower neither persists the * leader's entry nor reports false convergence, and stays available to retry * (no crash, no committed advance). R3.5/R3.6: the member keeps its divergent * state pending a later catch-up rather than spuriously accepting. */ static void test_intake_logmismatch_rejected_no_false_convergence(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_intake_rej_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); make_follower(c, /*term*/4, /*n_peers*/3); /* Local log: LSN1@term2. The leader claims prev=(LSN1,term3) — a term * mismatch at LSN1 → logOk fails. */ uint8_t blob[64]; size_t n = pack_entry(blob, WAL_JOB_UPDATE, "a", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 2, blob, n)); c->ha.raft.log_len = 1; uint8_t lpay[1] = { 'z' }; wal_catchup_entry_t ae; fill_ae_entry(&ae, /*heartbeat*/0, /*prev_idx*/1, /*prev_term*/3, /*leader_commit*/0, /*entry_lsn*/2, WAL_JOB_UPDATE, /*entry_term*/4, lpay, 1); int rc = ha_consume_append_entries(c, &ae, /*leader_idx*/1); /* The helper returns 0 (no crash) but the AE was NOT accepted: the leader's * entry was not persisted at LSN2 and the local LSN1 entry is unchanged. */ TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(&c->persist, 2)); /* absent */ TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 1)); /* intact */ teardown_cluster(c); printf("test_intake_logmismatch_rejected_no_false_convergence: PASS\n"); } /* ================================================================== */ /* Task 3.4 — live broadcast/catch-up receive cutover (closes bug-990) */ /* ================================================================== */ /* The live cutover's observable: a standby applying a LIVE-broadcast-shaped * AppendEntries (a single entry in the `entries` LIST carrying the leader's term * + prev tuple — exactly what repl_broadcast → xce_wal_batch_req_raw produces and * xce_decode yields) through the PRODUCTION receive entry point * ha_apply_wal_catchup_entry persists the LEADER's term (hdr.term == leader_term), * NOT 0 (the retired term-0 legacy direct-apply path) and NOT its local term. * * Before the cutover repl_broadcast sent entry->term == 0, so this entry took the * term-0 legacy direct-apply branch and persisted hdr.term = local p->term — the * bug-990 term-identity gap. With the cutover the term-carrying entry routes * through raft_on_append_entries_request → on_persist_entry → leader term stamped. */ static void test_live_broadcast_receive_persists_leader_term(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_live_bcast_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); /* The standby is a follower at a LOW local term (3) so we can prove the * persisted term is the LEADER's (7), not the local term. */ make_follower(c, /*term*/3, /*n_peers*/3); /* Seed LSN 1 at the leader's term 7 so the broadcast of LSN 2 satisfies the * library's logOk (prev=(1,7)). This mirrors the contiguous live stream. */ uint8_t blob[64]; size_t n = pack_entry(blob, WAL_JOB_SUBMIT, "a", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 7, blob, n)); c->ha.raft.log_len = 1; TEST_ASSERT_EQUAL_UINT64(7, persist_entry_term(&c->persist, 1)); /* Build the LIVE receive shape: a single entry in the `entries` list, each * sub-entry carrying its term + prev tuple (what wal_entry_to_wire now encodes * per element and xce_decode reproduces). repl_broadcast at term 7 of LSN 2 * with prev=(1,7). */ uint8_t lpay[2] = { 'b', 'c' }; wal_catchup_entry_t sub; memset(&sub, 0, sizeof(sub)); sub.lsn = 2; sub.entry_type = WAL_JOB_SUBMIT; sub.term = 7; /* the LEADER's term on the wire */ sub.prev_log_index = 1; sub.prev_log_term = 7; sub.raw_payload = lpay; sub.raw_payload_len = sizeof(lpay); wal_catchup_entry_t* sub_arr[1] = { &sub }; wal_catchup_entry_t batch; memset(&batch, 0, sizeof(batch)); batch.n_entries = 1; batch.entries = sub_arr; /* Top-level container also carries the AE metadata (the encoder mirrors it). */ batch.term = 7; batch.prev_log_index = 1; batch.prev_log_term = 7; int rc = ha_apply_wal_catchup_entry(&c->ha, &c->persist, &batch, /*ep*/NULL, c); TEST_ASSERT_EQUAL_INT(0, rc); /* The load-bearing assertion (bug-990): LSN 2 persists the LEADER's term 7, * NOT 0 and NOT the local term 3. Before the cutover this read back as the * standby's local term via the term-0 direct-apply path. */ TEST_ASSERT_EQUAL_UINT64(7, persist_entry_term(&c->persist, 2)); /* The agreed prefix (LSN 1@term7) is untouched. */ TEST_ASSERT_EQUAL_UINT64(7, persist_entry_term(&c->persist, 1)); /* The library adopted the leader's term on the RPC (G5). */ TEST_ASSERT_EQUAL_UINT64(7, c->ha.raft.term); teardown_cluster(c); printf("test_live_broadcast_receive_persists_leader_term: PASS\n"); } /* ================================================================== */ /* bug-991 — catch-up applied_lsn must be "last contiguously-held LSN" */ /* (NOT next-to-apply / lsn+1) so the leader's catch-up */ /* streamer never SKIPS the standby's real next-needed LSN. */ /* ================================================================== */ /* The standby's applied_lsn is the watermark it advertises to the leader * (ha_nop_reply_state_report → sr.applied_lsn → standby_reported_lsn). The * leader's catch-up streamer (ha_stream_wal_to_standby) and the snapshot/stream * decision (ha_evaluate_repl_action) BOTH treat that value as the * "last contiguously-held LSN" and stream the half-open range * (standby_lsn, committed_lsn]. The recovery/checkpoint/init setters already use * last-held semantics (persist_durable_floor / checkpoint_lsn / 0). The three * live/catch-up apply paths historically advanced to lsn+1 (next-to-apply) — * an off-by-one that made the standby advertise one LSN too high, so after it * applied catch-up entry N the leader streamed (N+1, committed] and SKIPPED the * real next-needed entry N+1 forever (logOk wedge → no WalAck → commit stall). * * RED (pre-fix): after applying a catch-up entry at LSN N the standby's * applied_lsn reads N+1, and ha_evaluate_repl_action(standby_lsn=applied_lsn, * committed=N) reports CONVERGED one entry early (skip). * GREEN (post-fix): applied_lsn reads N (last-held); a standby that has applied * up to N with committed > N still STREAMS (no skip), and the streamer's * (standby_lsn, committed] range begins at exactly N+1. */ static void test_catchup_applied_lsn_is_last_held_no_skip(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_bug991_lasthelp_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); make_follower(c, /*term*/7, /*n_peers*/3); /* applied_lsn starts at 0 (init = empty/last-held-nothing). */ TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&c->ha.applied_lsn)); /* Catch-up stream of LSN 1 then LSN 2 at the leader's term 7, contiguous * (prev tuples chain 0/0 → 1/7), routed through the production receive entry * point exactly as ha_stream_wal_to_standby → xce_decode would deliver them. */ uint8_t p1[1] = { 'a' }; wal_catchup_entry_t sub1; memset(&sub1, 0, sizeof(sub1)); sub1.lsn = 1; sub1.entry_type = WAL_JOB_SUBMIT; sub1.term = 7; sub1.prev_log_index = 0; sub1.prev_log_term = 0; sub1.raw_payload = p1; sub1.raw_payload_len = sizeof(p1); wal_catchup_entry_t* arr1[1] = { &sub1 }; wal_catchup_entry_t batch1; memset(&batch1, 0, sizeof(batch1)); batch1.n_entries = 1; batch1.entries = arr1; batch1.term = 7; batch1.prev_log_index = 0; batch1.prev_log_term = 0; TEST_ASSERT_EQUAL_INT(0, ha_apply_wal_catchup_entry(&c->ha, &c->persist, &batch1, /*ep*/NULL, c)); TEST_ASSERT_EQUAL_UINT64(7, persist_entry_term(&c->persist, 1)); /* THE bug-991 load-bearing assertion: after applying LSN 1 the standby * advertises applied_lsn == 1 (last-held), NOT 2 (next-to-apply). Pre-fix * this read 2 and the leader's next stream skipped LSN 2. */ TEST_ASSERT_EQUAL_UINT64(1, atomic_load(&c->ha.applied_lsn)); /* No-skip property at the leader's decision seam: a standby holding up to * LSN 1 with committed_lsn == 2 must STREAM (its next-needed LSN 2 is missing), * NOT report CONVERGED. With the off-by-one advertised value (2) the leader * would have seen standby_lsn(2) >= committed(2) → CONVERGED (the skip). */ { uint64_t standby_lsn = atomic_load(&c->ha.applied_lsn); ha_repl_action_t act = ha_evaluate_repl_action( /*standby_tag*/1, standby_lsn, /*committed_tag*/1, /*committed_lsn*/2, /*oldest_wal_lsn*/1); TEST_ASSERT_EQUAL_INT(HA_REPL_STREAM_WAL, act); /* The streamer's start LSN is standby_lsn + 1 == 2 — exactly the missing * entry, no skip. */ TEST_ASSERT_EQUAL_UINT64(2, standby_lsn + 1); } /* Apply the next contiguous entry (LSN 2) — applied_lsn becomes 2 (last-held), * not 3 — and the standby is now CONVERGED at committed_lsn == 2. */ uint8_t p2[1] = { 'b' }; wal_catchup_entry_t sub2; memset(&sub2, 0, sizeof(sub2)); sub2.lsn = 2; sub2.entry_type = WAL_JOB_SUBMIT; sub2.term = 7; sub2.prev_log_index = 1; sub2.prev_log_term = 7; sub2.raw_payload = p2; sub2.raw_payload_len = sizeof(p2); wal_catchup_entry_t* arr2[1] = { &sub2 }; wal_catchup_entry_t batch2; memset(&batch2, 0, sizeof(batch2)); batch2.n_entries = 1; batch2.entries = arr2; batch2.term = 7; batch2.prev_log_index = 1; batch2.prev_log_term = 7; TEST_ASSERT_EQUAL_INT(0, ha_apply_wal_catchup_entry(&c->ha, &c->persist, &batch2, /*ep*/NULL, c)); TEST_ASSERT_EQUAL_UINT64(7, persist_entry_term(&c->persist, 2)); TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->ha.applied_lsn)); { ha_repl_action_t act = ha_evaluate_repl_action( /*standby_tag*/1, atomic_load(&c->ha.applied_lsn), /*committed_tag*/1, /*committed_lsn*/2, /*oldest_wal_lsn*/1); TEST_ASSERT_EQUAL_INT(HA_REPL_CONVERGED, act); } teardown_cluster(c); printf("test_catchup_applied_lsn_is_last_held_no_skip: PASS\n"); } /* ================================================================== */ /* bug-1067 — cross-term catch-up: the AppendEntries header term is the */ /* leader's CURRENT term, NOT the streamed entry's stored term */ /* ================================================================== */ /* Root cause: ha_consume_append_entries used the entry's stored term (entry_term) * as the AppendEntries header term. For LIVE appends that is the leader's current * term (leader appends at its term), but for CATCH-UP streaming of HISTORICAL * entries written by a PRIOR leader, entry_term < the current leader's term. The * library's stale-term check (term < s->term → ae_reject_stale) then permanently * rejected every historical entry, so the lagging standby could never catch up * (standby_reported_lsn never advanced; the streamer retransmitted forever). * * The fix carries the leader's current term out-of-band per entry * (wal_catchup_entry_t.leader_current_term, WalCatchupEntry.leaderCurrentTerm @8), * set by the streamer to ha->term, and uses THAT as the AE header term while the * entry's stored term remains the per-entry term for logOk / on_persist_entry. * * RED before the fix: the leader_current_term field does not exist (compile * error); once added but before the receiver reads it, ae_term == entry_term (4) < * s->term (5) → stale-reject → LSN 2 never persists. */ static void test_intake_crossterm_catchup_uses_leader_current_term(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_xterm_catchup_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); /* Standby has already adopted the current leader's term (5) from heartbeats. */ make_follower(c, /*term*/5, /*n_peers*/3); /* Local log holds a HISTORICAL entry LSN1@term4 (written by the prior leader * at term 4, replicated to this standby). */ uint8_t blob[64]; size_t n = pack_entry(blob, WAL_JOB_SUBMIT, "a", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 4, blob, n)); c->ha.raft.log_len = 1; TEST_ASSERT_EQUAL_UINT64(4, persist_entry_term(&c->persist, 1)); /* The CURRENT leader (term 5) streams the historical entry LSN2@term4 as a * catch-up sub-entry: the entry's STORED term is 4, but the AppendEntries * header term is the leader's CURRENT term 5 (leader_current_term). prev tuple * (1,4) matches the local log so logOk holds. */ uint8_t lpay[1] = { 'b' }; wal_catchup_entry_t sub; memset(&sub, 0, sizeof(sub)); sub.lsn = 2; sub.entry_type = WAL_JOB_SUBMIT; sub.term = 4; /* the entry's STORED term (prior leader) */ sub.leader_current_term = 5; /* the streaming leader's CURRENT term */ sub.prev_log_index = 1; sub.prev_log_term = 4; sub.raw_payload = lpay; sub.raw_payload_len = sizeof(lpay); wal_catchup_entry_t* sub_arr[1] = { &sub }; wal_catchup_entry_t batch; memset(&batch, 0, sizeof(batch)); batch.n_entries = 1; batch.entries = sub_arr; batch.term = 4; batch.leader_current_term = 5; batch.prev_log_index = 1; batch.prev_log_term = 4; int rc = ha_apply_wal_catchup_entry(&c->ha, &c->persist, &batch, /*ep*/NULL, c); TEST_ASSERT_EQUAL_INT(0, rc); /* THE bug-1067 assertion: the historical entry is ACCEPTED (not stale-rejected) * and persisted at its stored term 4. Pre-fix it was rejected (absent). */ TEST_ASSERT_EQUAL_UINT64(4, persist_entry_term(&c->persist, 2)); /* The agreed prefix (LSN1@term4) is untouched. */ TEST_ASSERT_EQUAL_UINT64(4, persist_entry_term(&c->persist, 1)); /* The leader's current term (5) equals our term — no spurious term change. */ TEST_ASSERT_EQUAL_UINT64(5, c->ha.raft.term); teardown_cluster(c); printf("test_intake_crossterm_catchup_uses_leader_current_term: PASS\n"); } /* Safety guard: a GENUINELY stale leader (leader_current_term < our term) must * STILL be rejected — the fix must not open a hole letting a deposed leader's * divergent entries through. */ static void test_intake_stale_leader_term_still_rejected(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_stale_leader_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); make_follower(c, /*term*/5, /*n_peers*/3); uint8_t blob[64]; size_t n = pack_entry(blob, WAL_JOB_SUBMIT, "a", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 4, blob, n)); c->ha.raft.log_len = 1; /* A deposed leader at term 3 (< our term 5) tries to stream. Even though the * entry's stored term and prev tuple look locally plausible, the AE header * term (leader_current_term=3) is stale → reject. */ uint8_t lpay[1] = { 'z' }; wal_catchup_entry_t sub; memset(&sub, 0, sizeof(sub)); sub.lsn = 2; sub.entry_type = WAL_JOB_SUBMIT; sub.term = 3; sub.leader_current_term = 3; /* STALE leader term < our term 5 */ sub.prev_log_index = 1; sub.prev_log_term = 4; sub.raw_payload = lpay; sub.raw_payload_len = sizeof(lpay); wal_catchup_entry_t* sub_arr[1] = { &sub }; wal_catchup_entry_t batch; memset(&batch, 0, sizeof(batch)); batch.n_entries = 1; batch.entries = sub_arr; batch.term = 3; batch.leader_current_term = 3; batch.prev_log_index = 1; batch.prev_log_term = 4; int rc = ha_apply_wal_catchup_entry(&c->ha, &c->persist, &batch, /*ep*/NULL, c); TEST_ASSERT_EQUAL_INT(0, rc); /* Stale leader's entry was REJECTED — LSN 2 absent, our term unchanged. */ TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(&c->persist, 2)); TEST_ASSERT_EQUAL_UINT64(5, c->ha.raft.term); teardown_cluster(c); printf("test_intake_stale_leader_term_still_rejected: PASS\n"); } /* ================================================================== */ /* Task 3.3 — drive raft_on_append_entries_reply + raft_advance_commit */ /* on the replication ACK path */ /* ================================================================== */ /* These tests drive the ACK seam ha_raft_on_repl_ack(c, abs_peer_idx, acked_lsn, * reply_term): on a replication ACK the controller routes the reply through * raft_on_append_entries_reply (updating match_index[]/next_index[]) then * raft_advance_commit, which recomputes the current-term-gated commit and MAY * fire on_commit_advanced (the task-2.4 callback that feeds the barrier). * * Peer-index contract (REMEDIATION round 1): ha_raft_on_repl_ack now takes the * ABSOLUTE library peer index directly (the match_index[]/next_index[] index * space, j != self_idx). The CALLER resolves it per endpoint type via * ha_repl_ack_abs_peer_idx BEFORE this seam — promoted-vote eps carry a relative * "votes" index (remapped), heartbeat standby eps carry an already-absolute * index (passed through). The earlier blind remap inside the seam corrupted the * heartbeat path (wrong-peer false quorum / OOB match_index[] write); the * endpoint-type round-trip is pinned by the two remediation tests below. * * Fixture: make_leader(c, term, n_peers) brings the node up as the raft LEADER * with self_idx == my_priority == 0 (make_cluster calloc's my_priority=0), so * the only standby's absolute peer index is 1. * * Against the pre-3.3 ACK path (repl_on_ack durability only, no library reply), * ha_raft_on_repl_ack does not exist (compile RED); once wired, a current-term * quorum ACK MUST advance the library commit_index and fire on_commit_advanced * (R4.1/R4.5), a prior-term agreed entry MUST hold, and a stale-term reply MUST * be dropped. */ /* Seed `count` entries at the given term into the leader's local WAL via the * wired on_persist_entry, growing the library log_len in lockstep. */ static void leader_seed_entries(cluster_t* c, uint64_t term, int count) { uint8_t blob[64]; for (int i = 1; i <= count; i++) { size_t n = pack_entry(blob, WAL_JOB_UPDATE, "x", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, (uint64_t)i, term, blob, n)); c->ha.raft.log_len = (uint64_t)i; } TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); } /* (a) A current-term quorum ACK drives raft_on_append_entries_reply → * raft_advance_commit → on_commit_advanced fires with the agreed commit_index, * which is recorded as the advisory library index AND wakes the barrier. 3-peer * cluster (self + 2 standbys, quorum=2): self holds LSN 1,2 at the current term; * one standby ACKs LSN 2 → {self, peer1} is a quorum for LSN 2, the agreed entry * is current-term, so the library commit_index advances to 2; the durable * majority (also at 2) is what advances committed_lsn (optA: the durable-majority * barrier is authoritative; the recorded index is inert-advisory). This leg * still verifies the full ACK→library→callback→barrier wiring is intact. */ static void test_repl_ack_currentterm_quorum_advances_commit(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_ack_adv_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 6; cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); make_leader(c, T, /*n_peers*/3); /* Init the cluster's commit engine over the same 3-peer cluster so a * majority is self + 1 standby, and register one standby whose durable ACK * supplies the durable-majority floor. */ TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, /*n_peers*/3, /*n_vcpus*/8)); /* Seed LSN 1,2 at the current term T → durable floor reaches LSN 2. */ leader_seed_entries(c, T, /*count*/2); TEST_ASSERT_EQUAL_INT(0, repl_add_standby(&c->persist.repl, &g_ae_dummy_ep, "sb0", 7000)); /* Durable majority: the standby's durable position covers LSN 2. Under optA * the durable-majority barrier is authoritative, so this ACK already advances * committed_lsn to 2 (asserted at the end); the library wiring below is still * exercised to confirm the ACK→library→callback path is intact. */ atomic_store(&c->persist.repl.standbys[0].acked_lsn, 2); commit_on_ack(&c->commit, /*peer_idx*/0, /*acked_lsn*/2); /* The library has not advanced commit_index yet (no ACK routed through the * library) so the advisory record is still 0 — but committed_lsn is NOT * gated by it (optA). */ TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.commit_index); /* The standby ACKs LSN 2 at the leader's current term. The caller has * already resolved the absolute peer index (self_idx 0 → the only standby is * absolute peer 1); the seam takes that absolute index directly. */ int rc = ha_raft_on_repl_ack(c, /*abs_peer_idx*/1, /*acked_lsn*/2, /*reply_term*/T); TEST_ASSERT_EQUAL_INT(0, rc); /* (1) The library recorded the peer's match_index at the absolute index. */ TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.match_index[1]); TEST_ASSERT_EQUAL_UINT64(3, c->ha.raft.next_index[1]); /* (2) raft_advance_commit advanced the current-term-gated commit_index to 2 * ({self, peer1} quorum, entry at LSN 2 is term T). */ TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.commit_index); /* (3) committed_lsn advanced to the durable majority (2). on_commit_advanced * fired and recorded the advisory index (2), but the durable-majority barrier * is what governs committed_lsn under optA (R4.1 realized via the authoritative * durable barrier + own-term gate). */ TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn)); TEST_ASSERT_EQUAL_UINT64(2, c->commit.raft_commit_index); /* advisory recorded */ commit_engine_fini(&c->commit); teardown_cluster(c); printf("test_repl_ack_currentterm_quorum_advances_commit: PASS\n"); } /* (b) A quorum that agrees only on a PRIOR-term entry does NOT advance commit * (the Figure-8 current-term rule, R4.1/R4.5). self holds LSN 1@term3 (a * prior-term entry) and is now leader at term 6 with no current-term entry yet. * A standby ACKs LSN 1 → {self, peer1} is a quorum for LSN 1, but the entry at * LSN 1 is term 3 != current term 6, so raft_advance_commit holds commit_index * at 0 and on_commit_advanced never fires. */ static void test_repl_ack_priorterm_quorum_holds_commit(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_ack_prior_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t PRIOR = 3; const uint64_t CUR = 6; cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); make_leader(c, CUR, /*n_peers*/3); /* Seed a single PRIOR-term entry at LSN 1 (replicated from an old leader). */ leader_seed_entries(c, PRIOR, /*count*/1); /* A standby ACKs LSN 1 at the leader's current term (the wire reply term is * the leader's term; the ENTRY's term is what gates the commit). */ int rc = ha_raft_on_repl_ack(c, /*abs_peer_idx*/1, /*acked_lsn*/1, /*reply_term*/CUR); TEST_ASSERT_EQUAL_INT(0, rc); /* match_index updated (the reply itself is current-term, not stale). */ TEST_ASSERT_EQUAL_UINT64(1, c->ha.raft.match_index[1]); /* But commit is HELD: the agreed entry (LSN1@term3) is not current-term → * Figure-8 rule blocks the advance, commit_index stays 0. */ TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.commit_index); teardown_cluster(c); printf("test_repl_ack_priorterm_quorum_holds_commit: PASS\n"); } /* (c) A stale-term reply is dropped: raft_on_append_entries_reply discards a * reply whose term != s->term, so match_index[] is NOT updated and no commit * advance occurs (R4.5 / raft.tla 415). self holds LSN 1,2 at the current term; * a stale reply (reply_term < current term) for LSN 2 must leave match_index and * commit_index untouched. */ static void test_repl_ack_staleterm_reply_dropped(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_ack_stale_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 8; const uint64_t STALE = 5; cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); make_leader(c, T, /*n_peers*/3); leader_seed_entries(c, T, /*count*/2); /* Stale-term ACK for LSN 2 → dropped by the library before any state change. */ int rc = ha_raft_on_repl_ack(c, /*abs_peer_idx*/1, /*acked_lsn*/2, /*reply_term*/STALE); TEST_ASSERT_EQUAL_INT(0, rc); /* No match_index update (the peer's slot stays at the post-init default 0). */ TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.match_index[1]); /* No commit advance. */ TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.commit_index); teardown_cluster(c); printf("test_repl_ack_staleterm_reply_dropped: PASS\n"); } /* ================================================================== */ /* Task 3.3 REMEDIATION round 1 — endpoint-type-correct peer mapping */ /* ================================================================== */ /* (d) The bug: the ACK walAck branch fires for BOTH promoted-vote eps (relative * votes index) AND heartbeat standby eps (already-absolute index). The earlier * seam unconditionally remapped, so a heartbeat ep's absolute index was mapped a * SECOND time — crediting the WRONG peer (false quorum) or writing OUT OF BOUNDS. * ha_repl_ack_abs_peer_idx must remap ONLY the vote ep and pass the standby ep * through unchanged. We pin both endpoint types AND both my_priority values so * the remap-vs-passthrough distinction actually round-trips (the pre-fix tests * only used my_priority==0, which masked the bug). * * Round-trip oracle: for a vote ep, peer absolute index i is sent as votes_idx * (i < self ? i : i-1); ha_repl_ack_abs_peer_idx must recover exactly i. For a * standby ep, the stored peer_idx IS i and must pass through unchanged. */ static void test_repl_ack_abs_peer_idx_endpoint_type_roundtrip(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_ack_map_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); /* --- self_idx (my_priority) == 0: every peer is after self --- */ c->ha.my_priority = 0; /* Heartbeat standby ep (ctx==NULL → ep_is_vote=0): absolute index passes * through UNCHANGED. Absolute peer 1 must stay 1, absolute peer 2 stay 2. * (The pre-fix blind remap would have turned 1→2 and 2→3 — the bug.) */ TEST_ASSERT_EQUAL_INT(1, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/0, /*peer*/1)); TEST_ASSERT_EQUAL_INT(2, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/0, /*peer*/2)); /* Promoted vote ep (ep_is_vote=1): votes_idx 0 → abs 1, votes_idx 1 → abs 2. */ TEST_ASSERT_EQUAL_INT(1, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/1, /*votes*/0)); TEST_ASSERT_EQUAL_INT(2, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/1, /*votes*/1)); /* --- self_idx (my_priority) == 1: a peer below self is NOT shifted --- */ c->ha.my_priority = 1; /* Standby ep: absolute indices 0 and 2 (peer 1 is self) pass through. */ TEST_ASSERT_EQUAL_INT(0, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/0, /*peer*/0)); TEST_ASSERT_EQUAL_INT(2, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/0, /*peer*/2)); /* Vote ep with self_idx==1: votes_idx 0 (peer absolute 0, < self) → 0; * votes_idx 1 (peer absolute 2, >= self) → 2. This is where remap and * passthrough DIVERGE — the case the my_priority==0 tests could not reach. */ TEST_ASSERT_EQUAL_INT(0, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/1, /*votes*/0)); TEST_ASSERT_EQUAL_INT(2, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/1, /*votes*/1)); teardown_cluster(c); printf("test_repl_ack_abs_peer_idx_endpoint_type_roundtrip: PASS\n"); } /* (e) End-to-end heartbeat-standby ACK: a standby ep carries an ALREADY-ABSOLUTE * peer index. Resolving it via ha_repl_ack_abs_peer_idx (ep_is_vote=0) and * driving ha_raft_on_repl_ack must land match_index on the CORRECT absolute slot * (not the +1 wrong slot the blind remap produced). 3-peer cluster, self=0; the * standby's absolute index is 1. A current-term quorum ACK for LSN 2 advances * the library commit_index to 2 — but ONLY because match_index[1] (not [2]) is * credited. Under the broken blind remap the credit landed on match_index[2] * (a phantom peer): no quorum at LSN 2, commit_index would stay 0. */ static void test_repl_ack_heartbeat_standby_absolute_slot(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_ack_hb_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 6; cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); make_leader(c, T, /*n_peers*/3); TEST_ASSERT_EQUAL_INT(0, c->ha.my_priority); /* self_idx 0 */ TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, /*n_peers*/3, /*n_vcpus*/8)); leader_seed_entries(c, T, /*count*/2); TEST_ASSERT_EQUAL_INT(0, repl_add_standby(&c->persist.repl, &g_ae_dummy_ep, "sb0", 7000)); atomic_store(&c->persist.repl.standbys[0].acked_lsn, 2); commit_on_ack(&c->commit, /*peer_idx*/0, /*acked_lsn*/2); /* The standby ep stores the ABSOLUTE index 1 (standby_ep_ctx_t.peer_idx). * Resolve it as a heartbeat ep (ep_is_vote=0) — it must pass through as 1. */ int abs_idx = ha_repl_ack_abs_peer_idx(&c->ha, /*ep_is_vote*/0, /*peer_idx*/1); TEST_ASSERT_EQUAL_INT(1, abs_idx); /* NOT 2 — the blind-remap bug slot */ int rc = ha_raft_on_repl_ack(c, abs_idx, /*acked_lsn*/2, /*reply_term*/T); TEST_ASSERT_EQUAL_INT(0, rc); /* The CORRECT absolute slot was credited; the phantom slot 2 stayed empty. */ TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.match_index[1]); TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.match_index[2]); /* {self, peer1} is a real quorum for LSN 2 → current-term commit advances. */ TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.commit_index); TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn)); commit_engine_fini(&c->commit); teardown_cluster(c); printf("test_repl_ack_heartbeat_standby_absolute_slot: PASS\n"); } /* (f) Defense-in-depth: an out-of-range / self absolute index is rejected loudly * (returns -EINVAL) and performs NO library accounting — fencing the unchecked * match_index[] write at the controller boundary rather than corrupting memory. */ static void test_repl_ack_out_of_range_peer_rejected(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_ack_oob_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 6; cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); make_leader(c, T, /*n_peers*/3); /* valid peers are absolute 1,2 */ leader_seed_entries(c, T, /*count*/2); /* abs index == n_peers (3) is out of range → rejected, no state change. */ TEST_ASSERT_EQUAL_INT(-EINVAL, ha_raft_on_repl_ack(c, /*abs_peer_idx*/3, /*acked_lsn*/2, /*reply_term*/T)); /* self (0) is never a valid peer → rejected. */ TEST_ASSERT_EQUAL_INT(-EINVAL, ha_raft_on_repl_ack(c, /*abs_peer_idx*/0, /*acked_lsn*/2, /*reply_term*/T)); /* Nothing was credited and commit did not advance. */ TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.match_index[1]); TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.commit_index); teardown_cluster(c); printf("test_repl_ack_out_of_range_peer_rejected: PASS\n"); } /* ================================================================== */ /* Task 5.1 — every-callback + marker-path end-to-end integration */ /* ================================================================== */ /* * The per-task tests above each exercise ONE callback (or one leg of the * marker path) in isolation. This integration test drives a single realistic * leader-lifecycle flow in which ALL FIVE adapter callbacks AND the config * marker path interoperate end-to-end, so a regression that breaks the * COMPOSITION of the callbacks (not just one in isolation) is caught here: * * on_persist_entry — primary job append + config-marker append both * land in the WAL with the live term stamped * on_get_entry_term — the leader's last-log term is read back * on_send_append_entries— the leader emits the marker entry to a peer with the * correct (prev tuple, leader_commit) and the entry's * recorded term/type/payload * on_commit_advanced — a current-term quorum ACK advances commit; the * marker enters the committed prefix and the DERIVED * committed config (config_committed_marker) equals it * on_truncate_from — a divergent same-LSN tail is truncated then the * leader's entry re-persisted, the committed prefix * (including the committed marker) left intact * * Against any single broken callback the corresponding assertion below fails * (e.g. neutering on_commit_advanced leaves committed_lsn behind the durable * majority so config_committed_marker returns -ENOENT; neutering on_truncate_from * leaves the stale divergent term at the conflicting LSN). */ static void test_every_callback_and_marker_path_end_to_end(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_e2e_all_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 9; cluster_t *c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); /* Bring the node up as the raft LEADER at term T (init-restore floor path, * task 2.5). A 3-peer cluster so a majority is self + 1 standby. */ make_leader(c, T, /*n_peers*/3); TEST_ASSERT_EQUAL_INT(0, c->ha.my_priority); /* self_idx 0 */ TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.log_len); /* The cluster's own commit engine over the same persist (3 peers). */ TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, /*n_peers*/3, /*n_vcpus*/8)); /* ── on_persist_entry (job append) ────────────────────────────────────── * A primary job-state append routes through the library → on_persist_entry * stamps the live term at LSN 1. */ wal_jobstate_t js = { .jobid = 77, .state = 2 }; uint64_t lsn_job = 0; TEST_ASSERT_EQUAL_INT(0, ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &lsn_job)); TEST_ASSERT_EQUAL_UINT64(1, lsn_job); TEST_ASSERT_EQUAL_UINT64(1, c->ha.raft.log_len); TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1)); /* ── on_persist_entry (config marker — the marker path) ───────────────── * A config marker rides the SAME unified append path (R7.1/R7.3): it lands * at LSN 2 with the live term stamped, exactly like a job entry. */ uint8_t sha[32]; for (int i = 0; i < 32; i++) sha[i] = (uint8_t)(0xA0 + i); uint64_t tag = 0xA0A1A2A3A4A5A6A7ULL; /* be64 of the first 8 sha bytes */ uint64_t lsn_marker = 0; TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &lsn_marker)); TEST_ASSERT_EQUAL_UINT64(2, lsn_marker); TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.log_len); TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 2)); /* A third, NON-committed job entry at LSN 3 (current term). It is appended * BEFORE the standby is registered so the primary's repl_broadcast is a * no-op (registering the dummy int ep first would fan the append out to a * non-endpoint → ASAN SEGV — the same ordering constraint the send-AE tests * above document). LSN 3 is the divergent suffix the on_truncate_from leg * removes once the committed prefix (LSN 1,2) is established below. */ wal_jobstate_t js2 = { .jobid = 78, .state = 1 }; uint64_t lsn3 = 0; TEST_ASSERT_EQUAL_INT(0, ha_primary_log_append(c, WAL_JOB_UPDATE, &js2, sizeof(js2), &lsn3)); TEST_ASSERT_EQUAL_UINT64(3, lsn3); TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 3)); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); /* ── on_get_entry_term ────────────────────────────────────────────────── * The leader's last-log term (LSN 3) reads back as T via the wired callback. */ TEST_ASSERT_EQUAL_UINT64(T, ha_cb_get_entry_term_for_test(&c->ha, c->ha.raft.log_len)); /* ── on_send_append_entries ───────────────────────────────────────────── * The leader emits the marker entry (LSN 2) to peer 0, carrying the prev * tuple (LSN 1, term T), leader_commit, and the marker's own recorded term + * WAL_CONFIG_MARKER type + 40-byte reference payload (read from its own WAL). */ register_one_standby(c); repl_ae_capture_t cap; memset(&cap, 0, sizeof(cap)); repl_send_ae_capture_hook = ∩ int rc = ha_cb_send_append_entries_for_test(&c->ha, /*peer*/0, /*prev_idx*/1, /*prev_term*/T, /*leader_commit*/0, /*n*/1); repl_send_ae_capture_hook = NULL; TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(1, cap.called); TEST_ASSERT_EQUAL_UINT64(1, cap.prev_log_index); TEST_ASSERT_EQUAL_UINT64(T, cap.prev_log_term); TEST_ASSERT_EQUAL_UINT64(2, cap.entry_lsn); TEST_ASSERT_EQUAL_UINT64(T, cap.entry_term); TEST_ASSERT_EQUAL_UINT8(WAL_CONFIG_MARKER, cap.entry_type); TEST_ASSERT_EQUAL_UINT32(WAL_CONFIG_MARKER_PAYLOAD_LEN, cap.payload_len); /* ── on_commit_advanced (+ committed marker derive) ───────────────────── * Before commit advances, the marker is appended but NOT in the committed * prefix, so the derived committed config is absent. */ uint64_t got_tag = 0; uint8_t got_sha[32]; TEST_ASSERT_EQUAL_INT(-ENOENT, config_committed_marker(c, &got_tag, got_sha)); /* on_commit_advanced records the advisory library index and wakes the * barrier (optA: it does NOT gate committed_lsn). Production order: the ACK * path fires on_commit_advanced with a (possibly lagging) commit_index, then * the durable-majority barrier — which is authoritative — advances * committed_lsn. Fire the WIRED callback with a LAGGING commit_index=1, then * supply a durable majority covering LSN 2: committed_lsn advances to the * durable majority (2), NOT pinned at the lagging advisory index (1). Under * the OLD (rejected) ceiling design this would have stuck committed at 1 — * the starvation. The advisory index is still RECORDED (sink intact). */ ha_cb_commit_advanced_for_test(&c->ha, 1); TEST_ASSERT_EQUAL_UINT64(1, c->commit.raft_commit_index); /* advisory recorded */ atomic_store(&c->persist.repl.standbys[0].acked_lsn, 2); commit_on_ack(&c->commit, /*peer_idx*/0, /*acked_lsn*/2); TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn)); /* Route the real current-term quorum ACK through the library * (raft_on_append_entries_reply → raft_advance_commit → on_commit_advanced). * The agreed entries are current-term (T) so the library commit advances to 2 * and the callback records the advisory index 2. committed_lsn is already 2 * (durable barrier). This still exercises the full ACK→library→callback wiring. */ TEST_ASSERT_EQUAL_INT(0, ha_raft_on_repl_ack(c, /*abs_peer_idx*/1, /*acked_lsn*/2, /*reply_term*/T)); TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.commit_index); TEST_ASSERT_EQUAL_UINT64(2, c->commit.raft_commit_index); /* advisory recorded */ TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn)); /* The marker (LSN 2) is now in the committed prefix → the DERIVED committed * config equals the appended marker's {tag, sha} (R7.7). */ TEST_ASSERT_EQUAL_INT(0, config_committed_marker(c, &got_tag, got_sha)); TEST_ASSERT_EQUAL_UINT64(tag, got_tag); TEST_ASSERT_EQUAL_MEMORY(sha, got_sha, 32); /* ── on_truncate_from ─────────────────────────────────────────────────── * The uncommitted suffix (the LSN 3 job entry appended above) is removed by * truncate-from LSN 3, while the committed prefix (job LSN 1 + committed * marker LSN 2) stays byte-stable (R3.2/R3.3). on_truncate_from is the * callback under test; the committed marker derive must still hold after. */ TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 3)); /* present */ TEST_ASSERT_EQUAL_INT(0, ha_cb_truncate_from_for_test(&c->ha, 3)); TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(&c->persist, 3)); /* gone */ /* The committed prefix is intact: LSN 1 (job) + LSN 2 (marker) keep their * terms, and the committed marker still derives to the same {tag, sha}. */ TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1)); TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 2)); TEST_ASSERT_EQUAL_INT(0, config_committed_marker(c, &got_tag, got_sha)); TEST_ASSERT_EQUAL_UINT64(tag, got_tag); TEST_ASSERT_EQUAL_MEMORY(sha, got_sha, 32); /* All five term-safety callbacks remain wired (non-NULL) through the flow. */ TEST_ASSERT_NOT_NULL(c->ha.raft.cbs->on_persist_entry); TEST_ASSERT_NOT_NULL(c->ha.raft.cbs->on_get_entry_term); TEST_ASSERT_NOT_NULL(c->ha.raft.cbs->on_send_append_entries); TEST_ASSERT_NOT_NULL(c->ha.raft.cbs->on_truncate_from); TEST_ASSERT_NOT_NULL(c->ha.raft.cbs->on_commit_advanced); commit_engine_fini(&c->commit); teardown_cluster(c); printf("test_every_callback_and_marker_path_end_to_end: PASS\n"); } /* ------------------------------------------------------------------ */ /* Suite entry point */ /* ------------------------------------------------------------------ */ void test_term_safety_adapter(void) { RUN_TEST(test_wal_entry_hdr_size_is_40); RUN_TEST(test_set_term_stamps_append); RUN_TEST(test_recover_mixed_real_and_legacy_terms); RUN_TEST(test_entry_term_absent_lsn_is_zero); RUN_TEST(test_entry_term_null_safe); RUN_TEST(test_on_persist_entry_stamps_term_standby); RUN_TEST(test_on_persist_entry_stamps_term_primary); RUN_TEST(test_on_persist_entry_failure_propagates_no_ack); RUN_TEST(test_on_get_entry_term_present_legacy_absent); RUN_TEST(test_on_truncate_from_drops_suffix); RUN_TEST(test_on_truncate_from_removes_all); RUN_TEST(test_send_append_entries_one_entry_from_wal); RUN_TEST(test_send_append_entries_heartbeat_zero_entries); RUN_TEST(test_commit_advances_to_durable_majority_index_inert); RUN_TEST(test_commit_index_record_is_not_a_ceiling); RUN_TEST(test_commit_advanced_does_not_bypass_durable_cap); RUN_TEST(test_own_term_gate_blocks_priorterm_ack_until_noop_durable); RUN_TEST(test_own_term_gate_retained_no_deadlock); RUN_TEST(test_on_commit_advanced_wired_records_advisory_index); RUN_TEST(test_init_restores_recovered_floor); RUN_TEST(test_init_empty_wal_zero_floor); RUN_TEST(test_primary_log_append_stamps_term_and_assigns_lsn); RUN_TEST(test_own_term_noop_records_live_term_via_library); RUN_TEST(test_primary_log_append_non_leader_legacy_fallback); RUN_TEST(test_intake_higher_term_heartbeat_adopts_term); RUN_TEST(test_intake_divergent_tail_truncated_and_reconverged); RUN_TEST(test_intake_logmismatch_rejected_no_false_convergence); RUN_TEST(test_live_broadcast_receive_persists_leader_term); RUN_TEST(test_catchup_applied_lsn_is_last_held_no_skip); RUN_TEST(test_intake_crossterm_catchup_uses_leader_current_term); RUN_TEST(test_intake_stale_leader_term_still_rejected); RUN_TEST(test_repl_ack_currentterm_quorum_advances_commit); RUN_TEST(test_repl_ack_priorterm_quorum_holds_commit); RUN_TEST(test_repl_ack_staleterm_reply_dropped); RUN_TEST(test_repl_ack_abs_peer_idx_endpoint_type_roundtrip); RUN_TEST(test_repl_ack_heartbeat_standby_absolute_slot); RUN_TEST(test_repl_ack_out_of_range_peer_rejected); /* Task 5.1: every-callback + marker-path end-to-end integration. */ RUN_TEST(test_every_callback_and_marker_path_end_to_end); }