#define _GNU_SOURCE #include #include #include #include #include #include #include #include "config.h" #include "persist.h" #include "wal_types.h" #include "repl.h" #include "commit.h" #include "xce/cluster_def.h" #include "ha.h" /* ha-term-safety task 1.3 — WAL term seam + recovery read-back + legacy-0 rule. * * The seam under test: * - persist_set_term(p, T) sets the controller's live election term so a * subsequent append stamps hdr.term = T (R1.1, R1.6). * - replay_wal retains each entry's hdr.term per LSN in an in-memory index * (R1.5), read back via persist_entry_term(p, lsn). * - persist_entry_term returns 0 for an absent LSN; a legacy entry carrying * hdr.term == 0 maps to the single lowest-known real Raft term (1) so a * legacy-0 entry is never spuriously divergent from another legacy-0 entry * and recovery never fails on legacy-0 (R1.8). * - the on-disk wal_entry_hdr_t stays 40 bytes (R1.7). */ /* The defined legacy-0 mapping (lowest real Raft term). Kept local to the test * so the assertion is independent of the implementation constant. */ #define LEGACY_LOWEST_TERM 1u /* ------------------------------------------------------------------ */ /* Helpers */ /* ------------------------------------------------------------------ */ static persist_t* make_fresh_persist(const char* tmpdir) { persist_t* p = calloc(1, sizeof(persist_t)); if (!p) return NULL; persist_config_t cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; /* slow fsync — no races in tests */ cfg.snap_keep_n = 2; persist_set_workdir(p, tmpdir, "ctrl-test"); if (persist_init(p, &cfg, NULL) != 0) { free(p); return NULL; } if (persist_start(p) != 0) { persist_fini(p); free(p); return NULL; } return p; } static void teardown_persist(persist_t* p) { persist_stop(p); persist_fini(p); free(p); } /* ------------------------------------------------------------------ */ /* Test: header layout preserved (R1.7) — value fix, not a format */ /* change. The on-disk WAL entry header must stay exactly 40 bytes. */ /* ------------------------------------------------------------------ */ static void test_wal_entry_hdr_size_is_40(void) { TEST_ASSERT_EQUAL_UINT64(40u, (uint64_t)sizeof(wal_entry_hdr_t)); printf("test_wal_entry_hdr_size_is_40: PASS\n"); } /* ------------------------------------------------------------------ */ /* Test: persist_set_term makes a subsequent append stamp hdr.term */ /* (R1.1, R1.6), recovered back via persist_entry_term. */ /* ------------------------------------------------------------------ */ static void test_set_term_stamps_append(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_term_set_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 7; /* Phase 1: set the live term, then append — the append stamps hdr.term=T. */ { persist_t* p = make_fresh_persist(tmp); TEST_ASSERT_NOT_NULL(p); persist_set_term(p, T); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1)); TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); /* Live read-back of the just-appended entry's term (LSN 1). */ TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(p, 1)); teardown_persist(p); } /* Phase 2: fresh persist_t recovers the WAL and reads the term back. */ { persist_t* p2 = make_fresh_persist(tmp); TEST_ASSERT_NOT_NULL(p2); TEST_ASSERT_EQUAL_INT(0, persist_recover(p2)); TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(p2, 1)); teardown_persist(p2); } printf("test_set_term_stamps_append: PASS\n"); } /* ------------------------------------------------------------------ */ /* Test: replay_wal of a WAL with mixed real-term and legacy-0 */ /* entries — persist_entry_term returns the real term for real */ /* entries and the defined lowest-known term for legacy-0 (R1.5, */ /* R1.8). Recovery never fails on legacy-0 entries. */ /* ------------------------------------------------------------------ */ static void test_recover_mixed_real_and_legacy_terms(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_term_mixed_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); /* Phase 1: write LSN 1,2 with term 0 (legacy — the pre-feature default), * then LSN 3,4 with real term 5. The append path stamps hdr.term=p->term, * so leaving the term unset (0) produces genuine legacy-0 entries. */ { persist_t* p = make_fresh_persist(tmp); TEST_ASSERT_NOT_NULL(p); /* Legacy-0 entries: no persist_set_term call (p->term stays 0). */ TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "l", 1)); /* lsn 1 */ TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "m", 1)); /* lsn 2 */ /* Real-term entries. */ persist_set_term(p, 5); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "r", 1)); /* lsn 3 */ TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "s", 1)); /* lsn 4 */ TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); teardown_persist(p); } /* Phase 2: fresh recover — read back each LSN's term. */ { persist_t* p2 = make_fresh_persist(tmp); TEST_ASSERT_NOT_NULL(p2); /* Recovery must NOT fail on the legacy-0 entries. */ TEST_ASSERT_EQUAL_INT(0, persist_recover(p2)); TEST_ASSERT_EQUAL_INT(RECOVER_CLEAN, persist_recovery_verdict(p2)); /* Legacy-0 entries map to the single lowest-known real term (1) — both * the same, so two legacy-0 entries are never seen as divergent. */ TEST_ASSERT_EQUAL_UINT64(LEGACY_LOWEST_TERM, persist_entry_term(p2, 1)); TEST_ASSERT_EQUAL_UINT64(LEGACY_LOWEST_TERM, persist_entry_term(p2, 2)); /* Real-term entries return their recorded term verbatim. */ TEST_ASSERT_EQUAL_UINT64(5, persist_entry_term(p2, 3)); TEST_ASSERT_EQUAL_UINT64(5, persist_entry_term(p2, 4)); teardown_persist(p2); } printf("test_recover_mixed_real_and_legacy_terms: PASS\n"); } /* ------------------------------------------------------------------ */ /* Test: persist_entry_term returns 0 for an absent LSN (no entry). */ /* Distinguishes "absent" (0) from a present legacy-0 entry (mapped */ /* to lowest-known), which is the precision REVIEW ISSUE 2 requires. */ /* ------------------------------------------------------------------ */ static void test_entry_term_absent_lsn_is_zero(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_term_absent_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = make_fresh_persist(tmp); TEST_ASSERT_NOT_NULL(p); persist_set_term(p, 3); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1)); /* lsn 1 */ /* LSN 0 is reserved (never an entry) and LSN 2 was never appended. */ TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(p, 0)); TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(p, 2)); TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(p, 999999)); /* The present entry still answers its real term. */ TEST_ASSERT_EQUAL_UINT64(3, persist_entry_term(p, 1)); teardown_persist(p); printf("test_entry_term_absent_lsn_is_zero: PASS\n"); } /* ------------------------------------------------------------------ */ /* Test: persist_entry_term is NULL-safe. */ /* ------------------------------------------------------------------ */ static void test_entry_term_null_safe(void) { TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(NULL, 1)); /* persist_set_term must not crash on NULL. */ persist_set_term(NULL, 9); printf("test_entry_term_null_safe: PASS\n"); } /* ================================================================== */ /* Task 2.1 — on_persist_entry callback body */ /* ================================================================== */ /* Build a cluster_t whose embedded persist module is initialised + started in * `tmpdir`, with ha.raft_c self-referencing the cluster (the convention every * raft callback uses to recover persist/role from ctx == &c->ha). Returns a * calloc'd cluster_t the caller must teardown via teardown_cluster. */ static cluster_t* make_cluster(const char* tmpdir) { cluster_t* c = calloc(1, sizeof(cluster_t)); if (!c) return NULL; persist_config_t cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; cfg.snap_keep_n = 2; persist_set_workdir(&c->persist, tmpdir, "ctrl-test"); if (persist_init(&c->persist, &cfg, c) != 0) { free(c); return NULL; } if (persist_start(&c->persist) != 0) { persist_fini(&c->persist); free(c); return NULL; } /* The callback recovers persist/role from ctx == &c->ha → c->ha.raft_c. */ c->ha.raft_c = c; return c; } static void teardown_cluster(cluster_t* c) { if (!c) return; persist_stop(&c->persist); persist_fini(&c->persist); free(c); } /* Pack one adapter entry blob: [raft_entry_blob_t{type,len}][payload]. */ static size_t pack_entry(uint8_t* buf, uint32_t type, const void* payload, uint32_t plen) { raft_entry_blob_t* h = (raft_entry_blob_t*)buf; h->type = type; h->payload_len = plen; if (payload && plen > 0) memcpy(buf + sizeof(*h), payload, plen); return sizeof(*h) + plen; } /* Test: the WIRED on_persist_entry stamps hdr.term from the library term at the * supplied 1-based index == LSN, persisting through the existing WAL path; the * recovered hdr.term equals the supplied term (R1.1–R1.4). Drives the standby * (replicated) path so persist-before-ACK is exercised. */ static void test_on_persist_entry_stamps_term_standby(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_op_sb_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 9; /* Phase 1: standby applies a replicated entry at LSN 1 with leader term T. */ { cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); atomic_store(&c->ha.role, CTRL_STANDBY); uint8_t blob[64]; size_t n = pack_entry(blob, WAL_JOB_UPDATE, "x", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, /*index*/1, T, blob, n)); /* Persisted at LSN 1 with hdr.term == T (live read-back). */ TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1)); TEST_ASSERT_EQUAL_UINT64(2, c->persist.current_lsn); teardown_cluster(c); } /* Phase 2: fresh recover reads the stamped term back from disk (R1.5). */ { cluster_t* c2 = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c2); TEST_ASSERT_EQUAL_INT(0, persist_recover(&c2->persist)); TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c2->persist, 1)); teardown_cluster(c2); } printf("test_on_persist_entry_stamps_term_standby: PASS\n"); } /* Test: the primary's own-append path through on_persist_entry stamps the term * and persists at the contiguous next LSN. */ static void test_on_persist_entry_stamps_term_primary(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_op_pr_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 4; cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); atomic_store(&c->ha.role, CTRL_PRIMARY); uint8_t blob[64]; size_t n = pack_entry(blob, WAL_NOOP, NULL, 0); /* own-term promotion no-op */ TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, /*index*/1, T, blob, n)); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1)); TEST_ASSERT_EQUAL_UINT64(2, c->persist.current_lsn); teardown_cluster(c); printf("test_on_persist_entry_stamps_term_primary: PASS\n"); } /* Test: a forced persist failure (fdatasync fail-loud seam on the replicated * write) makes on_persist_entry return non-zero and DOES NOT advance the log / * ACK (persist-before-ACK preserved). */ static void test_on_persist_entry_failure_propagates_no_ack(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_op_fail_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); atomic_store(&c->ha.role, CTRL_STANDBY); uint64_t durable_before = atomic_load(&c->persist.durable_lsn); /* Force the persist-before-ACK fdatasync to be treated as failed. */ atomic_store(&c->persist.test_force_fsync_fail, 1); uint8_t blob[64]; size_t n = pack_entry(blob, WAL_JOB_UPDATE, "y", 1); int rc = ha_cb_persist_entry_for_test(&c->ha, /*index*/1, 7, blob, n); TEST_ASSERT_NOT_EQUAL_INT(0, rc); /* failure propagates → caller does NOT ACK */ /* Persist-before-ACK durability contract: the entry is NOT durable, so the * durable watermark must NOT advance (the standby would ACK only on a * successful, durable write — the non-zero rc above is exactly the signal * that suppresses the ACK). durable_lsn (not the write cursor current_lsn) * is the durability authority; it stays put on fdatasync failure. */ TEST_ASSERT_EQUAL_UINT64(durable_before, atomic_load(&c->persist.durable_lsn)); atomic_store(&c->persist.test_force_fsync_fail, 0); teardown_cluster(c); printf("test_on_persist_entry_failure_propagates_no_ack: PASS\n"); } /* ================================================================== */ /* Task 2.2 — on_get_entry_term + on_truncate_from callback bodies */ /* ================================================================== */ /* Persist a couple of replicated entries (each at a 1-based index == LSN) * through the WIRED on_persist_entry callback, stamping a leader term per * entry. Returns the (started) cluster the caller tears down. */ static cluster_t* persist_two_entries(const char* tmp, uint64_t t1, uint64_t t2) { cluster_t* c = make_cluster(tmp); if (!c) return NULL; atomic_store(&c->ha.role, CTRL_STANDBY); uint8_t blob[64]; size_t n1 = pack_entry(blob, WAL_JOB_UPDATE, "a", 1); if (ha_cb_persist_entry_for_test(&c->ha, /*index*/1, t1, blob, n1) != 0) { teardown_cluster(c); return NULL; } size_t n2 = pack_entry(blob, WAL_JOB_UPDATE, "b", 1); if (ha_cb_persist_entry_for_test(&c->ha, /*index*/2, t2, blob, n2) != 0) { teardown_cluster(c); return NULL; } return c; } /* Test: on_get_entry_term returns the stamped per-LSN term for present entries, * the lowest-known term for a legacy-0 entry, and 0 for an absent LSN * (R1.5, R3.1). */ static void test_on_get_entry_term_present_legacy_absent(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_get_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); atomic_store(&c->ha.role, CTRL_STANDBY); uint8_t blob[64]; /* LSN 1: real term 6. */ size_t n = pack_entry(blob, WAL_JOB_UPDATE, "r", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, /*index*/1, /*term*/6, blob, n)); /* LSN 2: legacy-0 entry (leader term 0) → reads back as WAL_LEGACY_TERM. */ n = pack_entry(blob, WAL_JOB_UPDATE, "l", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, /*index*/2, /*term*/0, blob, n)); /* Present real-term entry returns its term verbatim. */ TEST_ASSERT_EQUAL_UINT64(6, ha_cb_get_entry_term_for_test(&c->ha, 1)); /* Present legacy-0 entry maps to the lowest-known real term. */ TEST_ASSERT_EQUAL_UINT64(LEGACY_LOWEST_TERM, ha_cb_get_entry_term_for_test(&c->ha, 2)); /* Absent LSNs (0 reserved, 3 never appended) return 0. */ TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 0)); TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 3)); teardown_cluster(c); printf("test_on_get_entry_term_present_legacy_absent: PASS\n"); } /* Test: on_truncate_from(k) deletes local entries at LSN >= k; the prefix < k * stays intact and on_get_entry_term returns 0 for the truncated LSNs while the * term index shrinks consistently (R1.8, R3.1, R3.2). */ static void test_on_truncate_from_drops_suffix(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_trunc_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); /* Persist LSN 1 (term 3) and LSN 2 (term 5). */ cluster_t* c = persist_two_entries(tmp, /*t1*/3, /*t2*/5); TEST_ASSERT_NOT_NULL(c); /* Both present before truncation. */ TEST_ASSERT_EQUAL_UINT64(3, ha_cb_get_entry_term_for_test(&c->ha, 1)); TEST_ASSERT_EQUAL_UINT64(5, ha_cb_get_entry_term_for_test(&c->ha, 2)); TEST_ASSERT_EQUAL_UINT64(3, c->persist.current_lsn); /* Truncate from LSN 2 → entries at LSN >= 2 are deleted. */ TEST_ASSERT_EQUAL_INT(0, ha_cb_truncate_from_for_test(&c->ha, 2)); /* Prefix < 2 intact, suffix >= 2 gone (term index returns 0). */ TEST_ASSERT_EQUAL_UINT64(3, ha_cb_get_entry_term_for_test(&c->ha, 1)); TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 2)); TEST_ASSERT_EQUAL_UINT64(2, c->persist.current_lsn); teardown_cluster(c); printf("test_on_truncate_from_drops_suffix: PASS\n"); } /* Test: on_truncate_from(1) deletes the whole log (LSN >= 1); every entry's * term reads back absent and the tip collapses to LSN 1. */ static void test_on_truncate_from_removes_all(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_trunc_all_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = persist_two_entries(tmp, /*t1*/2, /*t2*/2); TEST_ASSERT_NOT_NULL(c); TEST_ASSERT_EQUAL_INT(0, ha_cb_truncate_from_for_test(&c->ha, 1)); TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 1)); TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 2)); TEST_ASSERT_EQUAL_UINT64(1, c->persist.current_lsn); teardown_cluster(c); printf("test_on_truncate_from_removes_all: PASS\n"); } /* ================================================================== */ /* Task 2.3 — on_send_append_entries callback body */ /* ================================================================== */ /* A dummy non-NULL endpoint pointer for a registered standby slot. The send * is diverted to the capture hook, so the pointer is never dereferenced. */ static int g_ae_dummy_ep; /* Register one standby in the cluster's embedded replicator at slot 0, wiring * ha.repl so the callback maps peer_idx 0 → this endpoint. */ static void register_one_standby(cluster_t *c) { c->ha.repl = &c->persist.repl; TEST_ASSERT_EQUAL_INT(0, repl_add_standby(&c->persist.repl, &g_ae_dummy_ep, "sb0", 7000)); } /* Test: on_send_append_entries for n_entries==1 builds an envelope carrying the * correct (prev_log_index, prev_log_term, leader_commit, n_entries) and, for the * single entry sourced from the leader's own WAL at prev_log_index+1, its term * (== persist_entry_term(prev_log_index+1)) and payload (R1.3, R3.1). */ static void test_send_append_entries_one_entry_from_wal(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_ae1_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); atomic_store(&c->ha.role, CTRL_PRIMARY); /* Populate the leader's own WAL BEFORE registering a standby — otherwise the * persist append fans out via repl_broadcast to the dummy int ep (ASAN * global-buffer-overflow reading ep->peer). With no standby, the append is a * no-op broadcast. The entry to be sent is at prev_log_index+1 == LSN 2. */ uint8_t blob[64]; size_t n1 = pack_entry(blob, WAL_JOB_UPDATE, "p", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, /*index*/1, /*term*/4, blob, n1)); size_t n2 = pack_entry(blob, WAL_JOB_UPDATE, "QQ", 2); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, /*index*/2, /*term*/6, blob, n2)); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); /* Now register the standby so peer_idx 0 maps to a slot; the AE send below * is diverted to the capture hook, so the dummy ep is never dereferenced. */ register_one_standby(c); /* The entry-term oracle the send must carry for LSN 2. */ uint64_t expect_term = persist_entry_term(&c->persist, 2); TEST_ASSERT_EQUAL_UINT64(6, expect_term); /* Drive the callback: prev_log_index=1, prev_log_term=4 (term of LSN 1), * leader_commit=1, n_entries=1 → sends the entry at LSN 2. */ repl_ae_capture_t cap; memset(&cap, 0, sizeof(cap)); repl_send_ae_capture_hook = ∩ int rc = ha_cb_send_append_entries_for_test(&c->ha, /*peer*/0, /*prev_idx*/1, /*prev_term*/4, /*leader_commit*/1, /*n*/1); repl_send_ae_capture_hook = NULL; TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(1, cap.called); TEST_ASSERT_EQUAL_INT(0, cap.peer_idx); TEST_ASSERT_EQUAL_UINT64(1, cap.prev_log_index); TEST_ASSERT_EQUAL_UINT64(4, cap.prev_log_term); TEST_ASSERT_EQUAL_UINT64(1, cap.leader_commit); TEST_ASSERT_EQUAL_INT(1, cap.n_entries); /* The single entry is sourced from the local WAL at prev_log_index+1 == 2. */ TEST_ASSERT_EQUAL_UINT64(2, cap.entry_lsn); TEST_ASSERT_EQUAL_UINT64(expect_term, cap.entry_term); TEST_ASSERT_EQUAL_UINT8(WAL_JOB_UPDATE, cap.entry_type); TEST_ASSERT_EQUAL_UINT32(2, cap.payload_len); TEST_ASSERT_EQUAL_MEMORY("QQ", cap.payload, 2); teardown_cluster(c); printf("test_send_append_entries_one_entry_from_wal: PASS\n"); } /* Test: on_send_append_entries for n_entries==0 emits a heartbeat carrying the * prev tuple + leader_commit and ZERO entries (R1.3). */ static void test_send_append_entries_heartbeat_zero_entries(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_ae0_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); atomic_store(&c->ha.role, CTRL_PRIMARY); /* One entry on disk so prev_log_index=1 is a real tip; heartbeat sends none. * Persist BEFORE registering the standby (see one-entry test note). */ uint8_t blob[64]; size_t n1 = pack_entry(blob, WAL_JOB_UPDATE, "h", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, /*index*/1, /*term*/3, blob, n1)); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); register_one_standby(c); repl_ae_capture_t cap; memset(&cap, 0, sizeof(cap)); repl_send_ae_capture_hook = ∩ int rc = ha_cb_send_append_entries_for_test(&c->ha, /*peer*/0, /*prev_idx*/1, /*prev_term*/3, /*leader_commit*/1, /*n*/0); repl_send_ae_capture_hook = NULL; TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(1, cap.called); TEST_ASSERT_EQUAL_UINT64(1, cap.prev_log_index); TEST_ASSERT_EQUAL_UINT64(3, cap.prev_log_term); TEST_ASSERT_EQUAL_UINT64(1, cap.leader_commit); TEST_ASSERT_EQUAL_INT(0, cap.n_entries); /* No entry sourced for a heartbeat. */ TEST_ASSERT_EQUAL_UINT32(0, cap.payload_len); teardown_cluster(c); printf("test_send_append_entries_heartbeat_zero_entries: PASS\n"); } /* ================================================================== */ /* Task 2.4 — on_commit_advanced trigger + commit-source switch */ /* ================================================================== */ /* * Task 2.4 feeds the library-supplied current-term-gated `commit_index` into * the existing synchronous commit barrier as the commit SOURCE. The barrier * mechanism is unchanged: committed_lsn still advances only when self+majority * agree AND the bytes are durable locally (durable_lsn cap). The library commit * index becomes an ADDITIONAL ceiling so the barrier never advances committed * past the library's current-term-safe point. on_commit_advanced only TRIGGERS * (records the ceiling + re-evaluates); it never performs durability itself. * * These cases build the commit engine standalone over a stub persist_t + * replicator (the test_commit_engine.c pattern) so the barrier interaction is * tested in isolation, plus one case driving the WIRED on_commit_advanced * callback end-to-end through ha_cb_commit_advanced_for_test. * * RED: against the no-op ha_cb_commit_advanced_stub (and the missing * commit_set_raft_commit_index ceiling), the library commit index is NOT a * commit source — committed_lsn would advance on durable-majority alone with * no current-term gate. GREEN: the wired callback feeds the ceiling so the * barrier holds until BOTH durable majority and the library commit index agree. */ /* Stub-persist fixture for the commit-source tests (mirrors test_commit_engine.c). */ static persist_t g_t24_p; static commit_engine_t g_t24_ce; static int g_t24_eps[MAX_STANDBYS]; static void t24_engine_setup(int n_peers, int n_standbys, uint64_t appended_lsn) { char name[16]; memset(&g_t24_p, 0, sizeof(g_t24_p)); TEST_ASSERT_EQUAL_INT(0, repl_init(&g_t24_p.repl)); for (int i = 0; i < n_standbys; i++) { snprintf(name, sizeof(name), "sb%d", i); TEST_ASSERT_EQUAL_INT( i, repl_add_standby(&g_t24_p.repl, &g_t24_eps[i], name, 7000 + (uint32_t)i)); } g_t24_p.current_lsn = appended_lsn + 1; atomic_store(&g_t24_p.committed_lsn, 0); atomic_store(&g_t24_p.durable_lsn, appended_lsn); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&g_t24_ce, &g_t24_p, n_peers, 8)); } static void t24_engine_teardown(void) { commit_engine_fini(&g_t24_ce); repl_fini(&g_t24_p.repl); } static void t24_standby_ack(int idx, uint64_t acked_lsn) { atomic_store(&g_t24_p.repl.standbys[idx].acked_lsn, acked_lsn); commit_on_ack(&g_t24_ce, idx, acked_lsn); } static uint64_t t24_committed(void) { return atomic_load(&g_t24_p.committed_lsn); } /* Test: the library commit index is the commit SOURCE once fed. A durable * majority covers LSN 3, but the library has only admitted (current-term commit) * up to LSN 1, so the barrier caps committed at 1 — NOT at the durable-majority * floor of 3. Raising the library ceiling to 3 then releases the barrier to 3. * This proves the library current-term commit index gates the commit, replacing * the term-blind safe-commit (R4.1, R4.5, R5.4). * * (raft_commit_index == 0 is the INERT pre-wire state — the library source is * not yet active and the barrier keeps autonomous durable-majority behavior; * that flag-OFF fallback is covered by the existing test_commit_engine suite, * whose autonomous semantics this task must not regress.) */ static void test_commit_advanced_is_commit_source(void) { /* 3 peers = self + 2 standbys; majority needs self + 1 standby. */ t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/3); /* The library is the commit source from the first commit: establish the * ceiling at LSN 1 BEFORE any ACK drives evaluation (production order — the * ACK path drives raft_advance_commit, which fires on_commit_advanced, then * the barrier re-evaluates). */ commit_set_raft_commit_index(&g_t24_ce, 1); /* A standby ACKs LSN 3; self is durable to LSN 3 → durable majority = 3. * The barrier must cap committed at the library ceiling (1), NOT at the * durable majority (3). */ t24_standby_ack(0, 3); TEST_ASSERT_EQUAL_UINT64(1, t24_committed()); /* Library admits up to LSN 3 → barrier releases to the durable majority (3). */ commit_set_raft_commit_index(&g_t24_ce, 3); TEST_ASSERT_EQUAL_UINT64(3, t24_committed()); t24_engine_teardown(); printf("test_commit_advanced_is_commit_source: PASS\n"); } /* Test: the library commit index is a CEILING — committed never advances past * it even when a durable majority covers a higher LSN (current-term gate). The * barrier's durable cap is the OTHER ceiling: committed = min(durable-majority, * raft_commit_index). No gap: raising either ceiling alone cannot overshoot the * other (R4.1, R4.5). */ static void test_commit_advanced_ceiling_caps_durable_majority(void) { t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/5); /* Establish the library ceiling at 3 BEFORE any ACK (production order). The * library has only committed up to 3 (higher entries prev-term or not yet * current-term-agreed). */ commit_set_raft_commit_index(&g_t24_ce, 3); /* Durable majority covers up to LSN 5 (self durable 5 + standby acked 5). * committed must cap at the library ceiling (3), not the durable floor (5). */ t24_standby_ack(0, 5); TEST_ASSERT_EQUAL_UINT64(3, t24_committed()); /* Library advances the ceiling to 5 → committed catches up to the durable * majority floor (5). Monotonic, no regression. */ commit_set_raft_commit_index(&g_t24_ce, 5); TEST_ASSERT_EQUAL_UINT64(5, t24_committed()); t24_engine_teardown(); printf("test_commit_advanced_ceiling_caps_durable_majority: PASS\n"); } /* Test: the library commit index NEVER lets committed advance past what is * durable locally. With raft_commit_index=4 but self only durable to LSN 2 and * the standby acked LSN 2, committed must stay at the durable-majority floor (2) * — the library ceiling does not bypass persist-before-ACK / durable cap * (R5.4 — barrier durability mechanism unchanged). */ static void test_commit_advanced_does_not_bypass_durable_cap(void) { t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/4); atomic_store(&g_t24_p.durable_lsn, 2); /* self durable only to 2 */ t24_standby_ack(0, 2); /* standby agrees to 2 */ commit_set_raft_commit_index(&g_t24_ce, 4); /* library ceiling well ahead */ commit_on_ack(&g_t24_ce, 0, 2); /* Durable-majority floor is 2; the library ceiling (4) cannot raise it. */ TEST_ASSERT_EQUAL_UINT64(2, t24_committed()); t24_engine_teardown(); printf("test_commit_advanced_does_not_bypass_durable_cap: PASS\n"); } /* Test (pins the own-term-gate DECISION — RETAINED, complementary, no deadlock): * the own-term writable_lsn gate is a WAIT-completion predicate, NOT a cap on * committed_lsn. It is retained alongside the library commit source. With the * gate armed at LSN 2 and the library commit index + durable majority both at 2, * committed advances to 2 AND a WAIT(lsn=2) completes — the two gates cannot * deadlock (the own-term gate never holds committed_lsn back; the library * ceiling never holds the WAIT predicate back beyond committed_lsn). */ static void test_own_term_gate_retained_no_deadlock(void) { t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/2); /* Arm the own-term gate at LSN 2 (retained complementary durability gate). */ commit_set_writable_lsn(&g_t24_ce, 2); /* Library commit source + durable majority both reach LSN 2. */ t24_standby_ack(0, 2); commit_set_raft_commit_index(&g_t24_ce, 2); commit_on_ack(&g_t24_ce, 0, 2); /* committed_lsn advanced to 2 (own-term gate never caps committed_lsn). */ TEST_ASSERT_EQUAL_UINT64(2, t24_committed()); /* A WAIT for LSN 2 completes immediately: committed_lsn (2) >= lsn (2) AND * >= writable_lsn (2). No deadlock between the gate and the library source. */ TEST_ASSERT_EQUAL_INT(0, commit_submit(&g_t24_ce, /*lsn*/2, COMMIT_MODE_WAIT, /*timeout_ms*/500)); t24_engine_teardown(); printf("test_own_term_gate_retained_no_deadlock: PASS\n"); } /* Test: the WIRED on_commit_advanced callback routes commit_index into the * cluster's commit engine end-to-end (ha_cb_commit_advanced_for_test) as the * commit SOURCE. A durable majority covers LSN 2, but the wired callback feeds * commit_index=1 first → the barrier caps committed at 1 (library current-term * gate), NOT at the durable majority of 2. A second callback at commit_index=2 * releases the barrier to 2. Proves the stub→real transition: the no-op stub * never fed the source, so committed would have followed the durable majority. */ static void test_on_commit_advanced_wired_feeds_barrier(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_commitadv_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t *c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); /* Init the cluster's embedded commit engine over the same persist as a * 3-peer cluster (self + 2 standbys), so a majority is self + 1 standby. */ TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&c->commit, &c->persist, /*n_peers*/3, /*n_vcpus*/8)); /* Persist two entries BEFORE registering a standby — otherwise the primary * own-append fans out via repl_broadcast to the dummy int ep (ASAN * global-buffer-overflow reading ep->peer); with no standby the broadcast * is a no-op. Append LSN 1,2 with term 4 → durable floor reaches LSN 2. */ atomic_store(&c->ha.role, CTRL_PRIMARY); uint8_t blob[64]; size_t n1 = pack_entry(blob, WAL_JOB_UPDATE, "a", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 4, blob, n1)); size_t n2 = pack_entry(blob, WAL_JOB_UPDATE, "b", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 2, 4, blob, n2)); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); /* Now register the standby so a majority (self + 1) is reachable. The ACK * below is driven directly into the registry + commit_on_ack (no network). */ TEST_ASSERT_EQUAL_INT(0, repl_add_standby(&c->persist.repl, &g_ae_dummy_ep, "sb0", 7000)); /* Fire the WIRED callback FIRST to establish the library ceiling at LSN 1 * (production order: the ACK path drives raft_advance_commit, which fires * on_commit_advanced before/with the barrier re-evaluation). */ ha_cb_commit_advanced_for_test(&c->ha, 1); /* Standby ACKs LSN 2 → durable majority covers LSN 2. The wired source caps * committed at 1, BELOW the durable majority (2): the library current-term * gate governs (the term-blind safe-commit is retired as the source). */ atomic_store(&c->persist.repl.standbys[0].acked_lsn, 2); commit_on_ack(&c->commit, 0, 2); TEST_ASSERT_EQUAL_UINT64(1, atomic_load(&c->persist.committed_lsn)); /* Library admits LSN 2 → barrier releases to the durable majority (2). */ ha_cb_commit_advanced_for_test(&c->ha, 2); TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn)); commit_engine_fini(&c->commit); teardown_cluster(c); printf("test_on_commit_advanced_wired_feeds_barrier: PASS\n"); } /* ================================================================== */ /* Task 2.5 — library init restore from the recovered WAL floor */ /* ================================================================== */ /* Test: ha_init_raft restores the library log/commit floor from the * recovered WAL instead of booting at LSN 0 (R1.5, R2.1). * * After a member recovers a WAL whose durable tip is LSN 3 (term 7) with a * committed prefix at LSN 2, raft_init must boot with: * raft.log_len == 3 (initial_log_len = persist_durable_floor) * raft.commit_index == 2 (initial_commit_index = recovered committed_lsn) * and the last-log term must be recoverable via the wired on_get_entry_term * (== 7). Booting from an empty log (log_len/commit_index == 0) is the pre-fix * behaviour this test fails on. */ static void test_init_restores_recovered_floor(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_initrestore_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 7; /* Phase 1: lay down a 3-entry WAL (term 7) and flush so the recovered tip * reaches LSN 3. */ { cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); atomic_store(&c->ha.role, CTRL_PRIMARY); uint8_t blob[64]; for (uint64_t i = 1; i <= 3; i++) { size_t n = pack_entry(blob, WAL_JOB_UPDATE, "x", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, i, T, blob, n)); } TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); teardown_cluster(c); } /* Phase 2: recover into a fresh cluster; durable floor == 3. Simulate a * recovered committed prefix at LSN 2 (recovery re-derives the durable tip; * the committed position is set by the surrounding boot path). */ { cluster_t* c2 = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c2); TEST_ASSERT_EQUAL_INT(0, persist_recover(&c2->persist)); TEST_ASSERT_EQUAL_UINT64(3, persist_durable_floor(&c2->persist)); atomic_store(&c2->persist.committed_lsn, 2ULL); /* Initialize the embedded raft as a single-node cluster. The restore * must source the floor from the recovered persist, not LSN 0. */ ha_init_raft(&c2->ha, c2, /*n_peers*/1); TEST_ASSERT_EQUAL_UINT64(3, c2->ha.raft.log_len); TEST_ASSERT_EQUAL_UINT64(2, c2->ha.raft.commit_index); /* The recovered last-log term is surfaced via the wired callback. */ TEST_ASSERT_EQUAL_UINT64(T, ha_cb_get_entry_term_for_test(&c2->ha, c2->ha.raft.log_len)); /* All five term-safety callbacks remain wired (non-NULL). */ TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs); TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_persist_entry); TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_get_entry_term); TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_send_append_entries); TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_truncate_from); TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_commit_advanced); /* Existing election callbacks remain wired. */ TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_become_leader); TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_persist); teardown_cluster(c2); } printf("test_init_restores_recovered_floor: PASS\n"); } /* Test: an empty-WAL first boot restores a zero floor (no spurious non-zero * indices) — the recovery-read-failure / empty-log edge of R1.5. */ static void test_init_empty_wal_zero_floor(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_initempty_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); TEST_ASSERT_EQUAL_INT(0, persist_recover(&c->persist)); TEST_ASSERT_EQUAL_UINT64(0, persist_durable_floor(&c->persist)); ha_init_raft(&c->ha, c, /*n_peers*/1); TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.log_len); TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.commit_index); teardown_cluster(c); printf("test_init_empty_wal_zero_floor: PASS\n"); } /* ================================================================== */ /* Task 3.1 — drive raft_log_append on the primary append path */ /* ================================================================== */ /* Bring a make_cluster()'d node up as the raft LEADER so raft_log_append is * enabled (it returns -EPERM off the leader). We init the library from the * recovered floor (task 2.5) and set the live election term + raft role * directly — modelling a node that has already won an election at `term` * (the cerebrum raft-sim convention: set state.term after a clean election; * raft_log_append stamps entries from s->term). */ static void make_leader(cluster_t* c, uint64_t term, int n_peers) { atomic_store(&c->ha.role, CTRL_PRIMARY); atomic_store(&c->ha.term, term); ha_init_raft(&c->ha, c, n_peers); c->ha.raft.term = term; c->ha.raft.role = RAFT_LEADER; } /* Test: a primary job-state append driven through ha_primary_log_append goes * through the library — the library assigns the LSN (== s->log_len post-append), * stamps the live election term into the WAL header, and keeps the persist * write cursor co-tracked with the library index (current_lsn == log_len + 1). * Against the pre-3.1 hand-rolled path this entry point does not exist * (compile RED) — and once it exists the library MUST be the LSN/term source * (R1.1, R1.2, R2.1). */ static void test_primary_log_append_stamps_term_and_assigns_lsn(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_plap_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 7; cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); make_leader(c, T, /*n_peers*/1); /* Fresh leader, empty log: first append takes library index 1. */ TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.log_len); wal_jobstate_t js = { .jobid = 42, .state = 3 }; uint64_t lsn = 0; TEST_ASSERT_EQUAL_INT(0, ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &lsn)); /* (a) The LIBRARY assigned the LSN and grew its log to it. */ TEST_ASSERT_EQUAL_UINT64(1, lsn); TEST_ASSERT_EQUAL_UINT64(1, c->ha.raft.log_len); /* (b) The WAL header recorded the live election term for that LSN. */ TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1)); /* (d) Co-tracking invariant: the persist write cursor advanced exactly in * lockstep with the library index — current_lsn == log_len + 1 (next LSN). */ TEST_ASSERT_EQUAL_UINT64(c->ha.raft.log_len + 1, c->persist.current_lsn); /* A second append keeps the two counters co-tracked. */ js.jobid = 43; TEST_ASSERT_EQUAL_INT(0, ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &lsn)); TEST_ASSERT_EQUAL_UINT64(2, lsn); TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.log_len); TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 2)); TEST_ASSERT_EQUAL_UINT64(c->ha.raft.log_len + 1, c->persist.current_lsn); teardown_cluster(c); printf("test_primary_log_append_stamps_term_and_assigns_lsn: PASS\n"); } /* Test: the own-term promotion no-op (ha_arm_own_term_gate) is routed through * the library, so the WAL_NOOP entry records the newly established term (not 0) * and the library index tracks it (R1.2). Pre-3.1 the noop went through a * hand-rolled persist_wal_async with a best-effort p->term and did NOT advance * s->log_len. */ static void test_own_term_noop_records_live_term_via_library(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_noop_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 5; cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); make_leader(c, T, /*n_peers*/1); /* Register the persist hooks then arm the own-term gate exactly as the * promotion tail does (ha_promote_to_primary / raft_cb_become_leader). */ ha_arm_own_term_gate_for_test(c); TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist)); /* The no-op landed at library index 1 with the live term recorded. */ TEST_ASSERT_EQUAL_UINT64(1, c->ha.raft.log_len); TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1)); TEST_ASSERT_EQUAL_UINT64(c->ha.raft.log_len + 1, c->persist.current_lsn); teardown_cluster(c); printf("test_own_term_noop_records_live_term_via_library: PASS\n"); } /* Test: when raft leadership is NOT active (the library is not RAFT_LEADER — * the init-time non-raft promotion road and engine-less test contexts), * ha_primary_log_append still appends locally through the legacy path so the * non-raft promotion tail keeps working, WITHOUT growing the library log (the * library is not tracking the log on that road, so no co-tracking hazard). * The entry still records the live election term via persist_set_term. */ static void test_primary_log_append_non_leader_legacy_fallback(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_plapfb_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const uint64_t T = 6; cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); /* Non-raft promotion road: controller role PRIMARY + live term, but the * raft library was never driven to LEADER (raft.role stays FOLLOWER). */ atomic_store(&c->ha.role, CTRL_PRIMARY); atomic_store(&c->ha.term, T); ha_init_raft(&c->ha, c, /*n_peers*/1); TEST_ASSERT_NOT_EQUAL_INT(RAFT_LEADER, c->ha.raft.role); wal_jobstate_t js = { .jobid = 7, .state = 1 }; uint64_t lsn = 0; TEST_ASSERT_EQUAL_INT(0, ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &lsn)); /* Appended at the contiguous next LSN (current_lsn started at 1). */ TEST_ASSERT_EQUAL_UINT64(1, lsn); TEST_ASSERT_EQUAL_UINT64(2, c->persist.current_lsn); /* Live term still recorded for the entry. */ TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1)); /* The library log was NOT grown by the legacy fallback (no raft leadership). */ TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.log_len); teardown_cluster(c); printf("test_primary_log_append_non_leader_legacy_fallback: PASS\n"); } /* ================================================================== */ /* Task 3.2 — route heartbeat/replication intake through the library */ /* raft_on_append_entries_request: UpdateTerm (G5) + logOk (G2) + */ /* (term,lsn) conflict-truncate (G3) via the wired callbacks. */ /* */ /* These drive the production intake helper ha_consume_append_entries */ /* (the in-process route that replaces the term-blind handling in */ /* ha_consume_heartbeat_committed + the committed-boundary truncate in */ /* ha_rejoin_reconcile). RED: that helper does not exist pre-3.2 */ /* (compile RED); once it exists the three observable behaviors hold. */ /* ================================================================== */ /* Make a node a STANDBY follower at a given term + raft log floor, with the * raft library initialized so the intake can drive it. Seeds the library log * to match the persisted WAL tip (the ha_init_raft floor restore). */ static void make_follower(cluster_t* c, uint64_t term, int n_peers) { atomic_store(&c->ha.role, CTRL_STANDBY); atomic_store(&c->ha.term, term); ha_init_raft(&c->ha, c, n_peers); c->ha.raft.term = term; c->ha.raft.role = RAFT_FOLLOWER; } /* Build a decoded single-entry WalCatchupEntry (n_entries==0 path == one entry * carried in the top-level container, matching the on-wire single-entry mode). * For a heartbeat use n_entries set so the helper sees zero entries. */ static void fill_ae_entry(wal_catchup_entry_t* e, int is_heartbeat, uint64_t prev_log_index, uint64_t prev_log_term, uint64_t leader_commit, uint64_t entry_lsn, uint8_t entry_type, uint64_t entry_term, uint8_t* payload, uint32_t plen) { memset(e, 0, sizeof(*e)); e->prev_log_index = prev_log_index; e->prev_log_term = prev_log_term; e->leader_commit = leader_commit; if (is_heartbeat) { e->n_entries = 0; e->lsn = 0; } else { /* single-entry mode: top-level container carries the entry, n_entries==1 * with no nested sub-entries (one-entry AppendEntries). */ e->n_entries = 1; e->lsn = entry_lsn; e->entry_type = entry_type; e->term = entry_term; e->raw_payload = payload; e->raw_payload_len = plen; } } /* (a) A higher-term heartbeat (0-entry AppendEntries) routed through the library * makes the follower ADOPT the higher term and step down — term recorded * (R2.6 / G5). Pre-3.2 ha_consume_heartbeat_committed only DROPPED stale * heartbeats and never adopted a higher term. */ static void test_intake_higher_term_heartbeat_adopts_term(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_intake_ht_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); make_follower(c, /*term*/3, /*n_peers*/3); /* The follower briefly believes itself a candidate/leader at term 3. */ c->ha.raft.role = RAFT_LEADER; /* Heartbeat from a leader at term 7, empty log (prev=0), commit 0. The * heartbeat's top-level container carries the leader's live term. */ wal_catchup_entry_t hb; fill_ae_entry(&hb, /*heartbeat*/1, /*prev_idx*/0, /*prev_term*/0, /*leader_commit*/0, 0, 0, 0, NULL, 0); hb.term = 7; int rc = ha_consume_append_entries(c, &hb, /*leader_idx*/1); TEST_ASSERT_EQUAL_INT(0, rc); /* The library adopted the higher term and stepped down to follower. */ TEST_ASSERT_EQUAL_UINT64(7, c->ha.raft.term); TEST_ASSERT_NOT_EQUAL_INT(RAFT_LEADER, c->ha.raft.role); teardown_cluster(c); printf("test_intake_higher_term_heartbeat_adopts_term: PASS\n"); } /* (b) A divergent same-LSN tail is truncated and the leader's entry accepted * (R3.1/R3.2 / G3), and the committed prefix below the divergence is untouched * (R3.3). The follower locally holds [LSN1@term2, LSN2@term2]; the leader sends * an AppendEntries appending LSN2@term5 on top of LSN1@term2. The library * detects the (term,lsn) conflict at LSN2, truncates LSN>=2, and re-persists * the leader's LSN2@term5. LSN1 (the agreed prefix) stays intact. */ static void test_intake_divergent_tail_truncated_and_reconverged(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_intake_div_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); make_follower(c, /*term*/2, /*n_peers*/3); /* Seed the local log: LSN1@term2, LSN2@term2 (the divergent tail). */ uint8_t blob[64]; size_t n = pack_entry(blob, WAL_JOB_UPDATE, "a", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 2, blob, n)); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 2, 2, blob, n)); c->ha.raft.log_len = 2; TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 1)); TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 2)); /* Leader (term 5) sends AppendEntries: prev=(LSN1,term2), entry LSN2@term5. * logOk holds at LSN1 (term2==term2); conflict at LSN2 (local term2 != * incoming term5) → truncate LSN>=2 then persist LSN2@term5. */ uint8_t lpay[1] = { 'b' }; wal_catchup_entry_t ae; fill_ae_entry(&ae, /*heartbeat*/0, /*prev_idx*/1, /*prev_term*/2, /*leader_commit*/1, /*entry_lsn*/2, WAL_JOB_UPDATE, /*entry_term*/5, lpay, 1); int rc = ha_consume_append_entries(c, &ae, /*leader_idx*/1); TEST_ASSERT_EQUAL_INT(0, rc); /* (term,lsn) reconverged: LSN2 now carries the leader's term 5. */ TEST_ASSERT_EQUAL_UINT64(5, persist_entry_term(&c->persist, 2)); /* The agreed committed prefix (LSN1@term2) is untouched (R3.3). */ TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 1)); /* The higher leader term was adopted on the RPC (G5). */ TEST_ASSERT_EQUAL_UINT64(5, c->ha.raft.term); teardown_cluster(c); printf("test_intake_divergent_tail_truncated_and_reconverged: PASS\n"); } /* (c) An unreconcilable member (logOk fails: the leader's prev tuple does not * match the follower's log) is REJECTED — the follower neither persists the * leader's entry nor reports false convergence, and stays available to retry * (no crash, no committed advance). R3.5/R3.6: the member keeps its divergent * state pending a later catch-up rather than spuriously accepting. */ static void test_intake_logmismatch_rejected_no_false_convergence(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_ts_intake_rej_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = make_cluster(tmp); TEST_ASSERT_NOT_NULL(c); make_follower(c, /*term*/4, /*n_peers*/3); /* Local log: LSN1@term2. The leader claims prev=(LSN1,term3) — a term * mismatch at LSN1 → logOk fails. */ uint8_t blob[64]; size_t n = pack_entry(blob, WAL_JOB_UPDATE, "a", 1); TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 2, blob, n)); c->ha.raft.log_len = 1; uint8_t lpay[1] = { 'z' }; wal_catchup_entry_t ae; fill_ae_entry(&ae, /*heartbeat*/0, /*prev_idx*/1, /*prev_term*/3, /*leader_commit*/0, /*entry_lsn*/2, WAL_JOB_UPDATE, /*entry_term*/4, lpay, 1); int rc = ha_consume_append_entries(c, &ae, /*leader_idx*/1); /* The helper returns 0 (no crash) but the AE was NOT accepted: the leader's * entry was not persisted at LSN2 and the local LSN1 entry is unchanged. */ TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(&c->persist, 2)); /* absent */ TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 1)); /* intact */ teardown_cluster(c); printf("test_intake_logmismatch_rejected_no_false_convergence: PASS\n"); } /* ------------------------------------------------------------------ */ /* Suite entry point */ /* ------------------------------------------------------------------ */ void test_term_safety_adapter(void) { RUN_TEST(test_wal_entry_hdr_size_is_40); RUN_TEST(test_set_term_stamps_append); RUN_TEST(test_recover_mixed_real_and_legacy_terms); RUN_TEST(test_entry_term_absent_lsn_is_zero); RUN_TEST(test_entry_term_null_safe); RUN_TEST(test_on_persist_entry_stamps_term_standby); RUN_TEST(test_on_persist_entry_stamps_term_primary); RUN_TEST(test_on_persist_entry_failure_propagates_no_ack); RUN_TEST(test_on_get_entry_term_present_legacy_absent); RUN_TEST(test_on_truncate_from_drops_suffix); RUN_TEST(test_on_truncate_from_removes_all); RUN_TEST(test_send_append_entries_one_entry_from_wal); RUN_TEST(test_send_append_entries_heartbeat_zero_entries); RUN_TEST(test_commit_advanced_is_commit_source); RUN_TEST(test_commit_advanced_ceiling_caps_durable_majority); RUN_TEST(test_commit_advanced_does_not_bypass_durable_cap); RUN_TEST(test_own_term_gate_retained_no_deadlock); RUN_TEST(test_on_commit_advanced_wired_feeds_barrier); RUN_TEST(test_init_restores_recovered_floor); RUN_TEST(test_init_empty_wal_zero_floor); RUN_TEST(test_primary_log_append_stamps_term_and_assigns_lsn); RUN_TEST(test_own_term_noop_records_live_term_via_library); RUN_TEST(test_primary_log_append_non_leader_legacy_fallback); RUN_TEST(test_intake_higher_term_heartbeat_adopts_term); RUN_TEST(test_intake_divergent_tail_truncated_and_reconverged); RUN_TEST(test_intake_logmismatch_rejected_no_false_convergence); }