lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
myxce/src/controller/testsuite/test_term_safety_adapter.c

1311 lines
54 KiB
C
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#define _GNU_SOURCE
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <unistd.h>
#include <stdatomic.h>
#include <unity/unity.h>
#include "config.h"
#include "persist.h"
#include "wal_types.h"
#include "repl.h"
#include "commit.h"
#include "xce/cluster_def.h"
#include "ha.h"
/* ha-term-safety task 1.3 — WAL term seam + recovery read-back + legacy-0 rule.
*
* The seam under test:
* - persist_set_term(p, T) sets the controller's live election term so a
* subsequent append stamps hdr.term = T (R1.1, R1.6).
* - replay_wal retains each entry's hdr.term per LSN in an in-memory index
* (R1.5), read back via persist_entry_term(p, lsn).
* - persist_entry_term returns 0 for an absent LSN; a legacy entry carrying
* hdr.term == 0 maps to the single lowest-known real Raft term (1) so a
* legacy-0 entry is never spuriously divergent from another legacy-0 entry
* and recovery never fails on legacy-0 (R1.8).
* - the on-disk wal_entry_hdr_t stays 40 bytes (R1.7).
*/
/* The defined legacy-0 mapping (lowest real Raft term). Kept local to the test
* so the assertion is independent of the implementation constant. */
#define LEGACY_LOWEST_TERM 1u
/* ------------------------------------------------------------------ */
/* Helpers */
/* ------------------------------------------------------------------ */
static persist_t* make_fresh_persist(const char* tmpdir)
{
persist_t* p = calloc(1, sizeof(persist_t));
if (!p) return NULL;
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000; /* slow fsync — no races in tests */
cfg.snap_keep_n = 2;
persist_set_workdir(p, tmpdir, "ctrl-test");
if (persist_init(p, &cfg, NULL) != 0) {
free(p);
return NULL;
}
if (persist_start(p) != 0) {
persist_fini(p);
free(p);
return NULL;
}
return p;
}
static void teardown_persist(persist_t* p)
{
persist_stop(p);
persist_fini(p);
free(p);
}
/* ------------------------------------------------------------------ */
/* Test: header layout preserved (R1.7) — value fix, not a format */
/* change. The on-disk WAL entry header must stay exactly 40 bytes. */
/* ------------------------------------------------------------------ */
static void test_wal_entry_hdr_size_is_40(void)
{
TEST_ASSERT_EQUAL_UINT64(40u, (uint64_t)sizeof(wal_entry_hdr_t));
printf("test_wal_entry_hdr_size_is_40: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Test: persist_set_term makes a subsequent append stamp hdr.term */
/* (R1.1, R1.6), recovered back via persist_entry_term. */
/* ------------------------------------------------------------------ */
static void test_set_term_stamps_append(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_term_set_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
const uint64_t T = 7;
/* Phase 1: set the live term, then append — the append stamps hdr.term=T. */
{
persist_t* p = make_fresh_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
persist_set_term(p, T);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
/* Live read-back of the just-appended entry's term (LSN 1). */
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(p, 1));
teardown_persist(p);
}
/* Phase 2: fresh persist_t recovers the WAL and reads the term back. */
{
persist_t* p2 = make_fresh_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
TEST_ASSERT_EQUAL_INT(0, persist_recover(p2));
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(p2, 1));
teardown_persist(p2);
}
printf("test_set_term_stamps_append: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Test: replay_wal of a WAL with mixed real-term and legacy-0 */
/* entries — persist_entry_term returns the real term for real */
/* entries and the defined lowest-known term for legacy-0 (R1.5, */
/* R1.8). Recovery never fails on legacy-0 entries. */
/* ------------------------------------------------------------------ */
static void test_recover_mixed_real_and_legacy_terms(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_term_mixed_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Phase 1: write LSN 1,2 with term 0 (legacy — the pre-feature default),
* then LSN 3,4 with real term 5. The append path stamps hdr.term=p->term,
* so leaving the term unset (0) produces genuine legacy-0 entries. */
{
persist_t* p = make_fresh_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Legacy-0 entries: no persist_set_term call (p->term stays 0). */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "l", 1)); /* lsn 1 */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "m", 1)); /* lsn 2 */
/* Real-term entries. */
persist_set_term(p, 5);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "r", 1)); /* lsn 3 */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "s", 1)); /* lsn 4 */
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
teardown_persist(p);
}
/* Phase 2: fresh recover — read back each LSN's term. */
{
persist_t* p2 = make_fresh_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
/* Recovery must NOT fail on the legacy-0 entries. */
TEST_ASSERT_EQUAL_INT(0, persist_recover(p2));
TEST_ASSERT_EQUAL_INT(RECOVER_CLEAN, persist_recovery_verdict(p2));
/* Legacy-0 entries map to the single lowest-known real term (1) — both
* the same, so two legacy-0 entries are never seen as divergent. */
TEST_ASSERT_EQUAL_UINT64(LEGACY_LOWEST_TERM, persist_entry_term(p2, 1));
TEST_ASSERT_EQUAL_UINT64(LEGACY_LOWEST_TERM, persist_entry_term(p2, 2));
/* Real-term entries return their recorded term verbatim. */
TEST_ASSERT_EQUAL_UINT64(5, persist_entry_term(p2, 3));
TEST_ASSERT_EQUAL_UINT64(5, persist_entry_term(p2, 4));
teardown_persist(p2);
}
printf("test_recover_mixed_real_and_legacy_terms: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Test: persist_entry_term returns 0 for an absent LSN (no entry). */
/* Distinguishes "absent" (0) from a present legacy-0 entry (mapped */
/* to lowest-known), which is the precision REVIEW ISSUE 2 requires. */
/* ------------------------------------------------------------------ */
static void test_entry_term_absent_lsn_is_zero(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_term_absent_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_fresh_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
persist_set_term(p, 3);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1)); /* lsn 1 */
/* LSN 0 is reserved (never an entry) and LSN 2 was never appended. */
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(p, 0));
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(p, 2));
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(p, 999999));
/* The present entry still answers its real term. */
TEST_ASSERT_EQUAL_UINT64(3, persist_entry_term(p, 1));
teardown_persist(p);
printf("test_entry_term_absent_lsn_is_zero: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Test: persist_entry_term is NULL-safe. */
/* ------------------------------------------------------------------ */
static void test_entry_term_null_safe(void)
{
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(NULL, 1));
/* persist_set_term must not crash on NULL. */
persist_set_term(NULL, 9);
printf("test_entry_term_null_safe: PASS\n");
}
/* ================================================================== */
/* Task 2.1 — on_persist_entry callback body */
/* ================================================================== */
/* Build a cluster_t whose embedded persist module is initialised + started in
* `tmpdir`, with ha.raft_c self-referencing the cluster (the convention every
* raft callback uses to recover persist/role from ctx == &c->ha). Returns a
* calloc'd cluster_t the caller must teardown via teardown_cluster. */
static cluster_t* make_cluster(const char* tmpdir)
{
cluster_t* c = calloc(1, sizeof(cluster_t));
if (!c) return NULL;
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
cfg.snap_keep_n = 2;
persist_set_workdir(&c->persist, tmpdir, "ctrl-test");
if (persist_init(&c->persist, &cfg, c) != 0) { free(c); return NULL; }
if (persist_start(&c->persist) != 0) {
persist_fini(&c->persist);
free(c);
return NULL;
}
/* The callback recovers persist/role from ctx == &c->ha → c->ha.raft_c. */
c->ha.raft_c = c;
return c;
}
static void teardown_cluster(cluster_t* c)
{
if (!c) return;
persist_stop(&c->persist);
persist_fini(&c->persist);
free(c);
}
/* Pack one adapter entry blob: [raft_entry_blob_t{type,len}][payload]. */
static size_t pack_entry(uint8_t* buf, uint32_t type,
const void* payload, uint32_t plen)
{
raft_entry_blob_t* h = (raft_entry_blob_t*)buf;
h->type = type;
h->payload_len = plen;
if (payload && plen > 0)
memcpy(buf + sizeof(*h), payload, plen);
return sizeof(*h) + plen;
}
/* Test: the WIRED on_persist_entry stamps hdr.term from the library term at the
* supplied 1-based index == LSN, persisting through the existing WAL path; the
* recovered hdr.term equals the supplied term (R1.1–R1.4). Drives the standby
* (replicated) path so persist-before-ACK is exercised. */
static void test_on_persist_entry_stamps_term_standby(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_op_sb_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
const uint64_t T = 9;
/* Phase 1: standby applies a replicated entry at LSN 1 with leader term T. */
{
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
atomic_store(&c->ha.role, CTRL_STANDBY);
uint8_t blob[64];
size_t n = pack_entry(blob, WAL_JOB_UPDATE, "x", 1);
TEST_ASSERT_EQUAL_INT(0,
ha_cb_persist_entry_for_test(&c->ha, /*index*/1, T, blob, n));
/* Persisted at LSN 1 with hdr.term == T (live read-back). */
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1));
TEST_ASSERT_EQUAL_UINT64(2, c->persist.current_lsn);
teardown_cluster(c);
}
/* Phase 2: fresh recover reads the stamped term back from disk (R1.5). */
{
cluster_t* c2 = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c2);
TEST_ASSERT_EQUAL_INT(0, persist_recover(&c2->persist));
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c2->persist, 1));
teardown_cluster(c2);
}
printf("test_on_persist_entry_stamps_term_standby: PASS\n");
}
/* Test: the primary's own-append path through on_persist_entry stamps the term
* and persists at the contiguous next LSN. */
static void test_on_persist_entry_stamps_term_primary(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_op_pr_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
const uint64_t T = 4;
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
atomic_store(&c->ha.role, CTRL_PRIMARY);
uint8_t blob[64];
size_t n = pack_entry(blob, WAL_NOOP, NULL, 0); /* own-term promotion no-op */
TEST_ASSERT_EQUAL_INT(0,
ha_cb_persist_entry_for_test(&c->ha, /*index*/1, T, blob, n));
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1));
TEST_ASSERT_EQUAL_UINT64(2, c->persist.current_lsn);
teardown_cluster(c);
printf("test_on_persist_entry_stamps_term_primary: PASS\n");
}
/* Test: a forced persist failure (fdatasync fail-loud seam on the replicated
* write) makes on_persist_entry return non-zero and DOES NOT advance the log /
* ACK (persist-before-ACK preserved). */
static void test_on_persist_entry_failure_propagates_no_ack(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_op_fail_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
atomic_store(&c->ha.role, CTRL_STANDBY);
uint64_t durable_before =
atomic_load(&c->persist.durable_lsn);
/* Force the persist-before-ACK fdatasync to be treated as failed. */
atomic_store(&c->persist.test_force_fsync_fail, 1);
uint8_t blob[64];
size_t n = pack_entry(blob, WAL_JOB_UPDATE, "y", 1);
int rc = ha_cb_persist_entry_for_test(&c->ha, /*index*/1, 7, blob, n);
TEST_ASSERT_NOT_EQUAL_INT(0, rc); /* failure propagates → caller does NOT ACK */
/* Persist-before-ACK durability contract: the entry is NOT durable, so the
* durable watermark must NOT advance (the standby would ACK only on a
* successful, durable write — the non-zero rc above is exactly the signal
* that suppresses the ACK). durable_lsn (not the write cursor current_lsn)
* is the durability authority; it stays put on fdatasync failure. */
TEST_ASSERT_EQUAL_UINT64(durable_before,
atomic_load(&c->persist.durable_lsn));
atomic_store(&c->persist.test_force_fsync_fail, 0);
teardown_cluster(c);
printf("test_on_persist_entry_failure_propagates_no_ack: PASS\n");
}
/* ================================================================== */
/* Task 2.2 — on_get_entry_term + on_truncate_from callback bodies */
/* ================================================================== */
/* Persist a couple of replicated entries (each at a 1-based index == LSN)
* through the WIRED on_persist_entry callback, stamping a leader term per
* entry. Returns the (started) cluster the caller tears down. */
static cluster_t* persist_two_entries(const char* tmp, uint64_t t1, uint64_t t2)
{
cluster_t* c = make_cluster(tmp);
if (!c) return NULL;
atomic_store(&c->ha.role, CTRL_STANDBY);
uint8_t blob[64];
size_t n1 = pack_entry(blob, WAL_JOB_UPDATE, "a", 1);
if (ha_cb_persist_entry_for_test(&c->ha, /*index*/1, t1, blob, n1) != 0) {
teardown_cluster(c);
return NULL;
}
size_t n2 = pack_entry(blob, WAL_JOB_UPDATE, "b", 1);
if (ha_cb_persist_entry_for_test(&c->ha, /*index*/2, t2, blob, n2) != 0) {
teardown_cluster(c);
return NULL;
}
return c;
}
/* Test: on_get_entry_term returns the stamped per-LSN term for present entries,
* the lowest-known term for a legacy-0 entry, and 0 for an absent LSN
* (R1.5, R3.1). */
static void test_on_get_entry_term_present_legacy_absent(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_get_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
atomic_store(&c->ha.role, CTRL_STANDBY);
uint8_t blob[64];
/* LSN 1: real term 6. */
size_t n = pack_entry(blob, WAL_JOB_UPDATE, "r", 1);
TEST_ASSERT_EQUAL_INT(0,
ha_cb_persist_entry_for_test(&c->ha, /*index*/1, /*term*/6, blob, n));
/* LSN 2: legacy-0 entry (leader term 0) → reads back as WAL_LEGACY_TERM. */
n = pack_entry(blob, WAL_JOB_UPDATE, "l", 1);
TEST_ASSERT_EQUAL_INT(0,
ha_cb_persist_entry_for_test(&c->ha, /*index*/2, /*term*/0, blob, n));
/* Present real-term entry returns its term verbatim. */
TEST_ASSERT_EQUAL_UINT64(6, ha_cb_get_entry_term_for_test(&c->ha, 1));
/* Present legacy-0 entry maps to the lowest-known real term. */
TEST_ASSERT_EQUAL_UINT64(LEGACY_LOWEST_TERM,
ha_cb_get_entry_term_for_test(&c->ha, 2));
/* Absent LSNs (0 reserved, 3 never appended) return 0. */
TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 0));
TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 3));
teardown_cluster(c);
printf("test_on_get_entry_term_present_legacy_absent: PASS\n");
}
/* Test: on_truncate_from(k) deletes local entries at LSN >= k; the prefix < k
* stays intact and on_get_entry_term returns 0 for the truncated LSNs while the
* term index shrinks consistently (R1.8, R3.1, R3.2). */
static void test_on_truncate_from_drops_suffix(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_trunc_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Persist LSN 1 (term 3) and LSN 2 (term 5). */
cluster_t* c = persist_two_entries(tmp, /*t1*/3, /*t2*/5);
TEST_ASSERT_NOT_NULL(c);
/* Both present before truncation. */
TEST_ASSERT_EQUAL_UINT64(3, ha_cb_get_entry_term_for_test(&c->ha, 1));
TEST_ASSERT_EQUAL_UINT64(5, ha_cb_get_entry_term_for_test(&c->ha, 2));
TEST_ASSERT_EQUAL_UINT64(3, c->persist.current_lsn);
/* Truncate from LSN 2 → entries at LSN >= 2 are deleted. */
TEST_ASSERT_EQUAL_INT(0, ha_cb_truncate_from_for_test(&c->ha, 2));
/* Prefix < 2 intact, suffix >= 2 gone (term index returns 0). */
TEST_ASSERT_EQUAL_UINT64(3, ha_cb_get_entry_term_for_test(&c->ha, 1));
TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 2));
TEST_ASSERT_EQUAL_UINT64(2, c->persist.current_lsn);
teardown_cluster(c);
printf("test_on_truncate_from_drops_suffix: PASS\n");
}
/* Test: on_truncate_from(1) deletes the whole log (LSN >= 1); every entry's
* term reads back absent and the tip collapses to LSN 1. */
static void test_on_truncate_from_removes_all(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_trunc_all_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = persist_two_entries(tmp, /*t1*/2, /*t2*/2);
TEST_ASSERT_NOT_NULL(c);
TEST_ASSERT_EQUAL_INT(0, ha_cb_truncate_from_for_test(&c->ha, 1));
TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 1));
TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 2));
TEST_ASSERT_EQUAL_UINT64(1, c->persist.current_lsn);
teardown_cluster(c);
printf("test_on_truncate_from_removes_all: PASS\n");
}
/* ================================================================== */
/* Task 2.3 — on_send_append_entries callback body */
/* ================================================================== */
/* A dummy non-NULL endpoint pointer for a registered standby slot. The send
* is diverted to the capture hook, so the pointer is never dereferenced. */
static int g_ae_dummy_ep;
/* Register one standby in the cluster's embedded replicator at slot 0, wiring
* ha.repl so the callback maps peer_idx 0 → this endpoint. */
static void register_one_standby(cluster_t *c)
{
c->ha.repl = &c->persist.repl;
TEST_ASSERT_EQUAL_INT(0,
repl_add_standby(&c->persist.repl, &g_ae_dummy_ep, "sb0", 7000));
}
/* Test: on_send_append_entries for n_entries==1 builds an envelope carrying the
* correct (prev_log_index, prev_log_term, leader_commit, n_entries) and, for the
* single entry sourced from the leader's own WAL at prev_log_index+1, its term
* (== persist_entry_term(prev_log_index+1)) and payload (R1.3, R3.1). */
static void test_send_append_entries_one_entry_from_wal(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_ae1_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t *c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
atomic_store(&c->ha.role, CTRL_PRIMARY);
/* Populate the leader's own WAL BEFORE registering a standby — otherwise the
* persist append fans out via repl_broadcast to the dummy int ep (ASAN
* global-buffer-overflow reading ep->peer). With no standby, the append is a
* no-op broadcast. The entry to be sent is at prev_log_index+1 == LSN 2. */
uint8_t blob[64];
size_t n1 = pack_entry(blob, WAL_JOB_UPDATE, "p", 1);
TEST_ASSERT_EQUAL_INT(0,
ha_cb_persist_entry_for_test(&c->ha, /*index*/1, /*term*/4, blob, n1));
size_t n2 = pack_entry(blob, WAL_JOB_UPDATE, "QQ", 2);
TEST_ASSERT_EQUAL_INT(0,
ha_cb_persist_entry_for_test(&c->ha, /*index*/2, /*term*/6, blob, n2));
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
/* Now register the standby so peer_idx 0 maps to a slot; the AE send below
* is diverted to the capture hook, so the dummy ep is never dereferenced. */
register_one_standby(c);
/* The entry-term oracle the send must carry for LSN 2. */
uint64_t expect_term = persist_entry_term(&c->persist, 2);
TEST_ASSERT_EQUAL_UINT64(6, expect_term);
/* Drive the callback: prev_log_index=1, prev_log_term=4 (term of LSN 1),
* leader_commit=1, n_entries=1 → sends the entry at LSN 2. */
repl_ae_capture_t cap;
memset(&cap, 0, sizeof(cap));
repl_send_ae_capture_hook = &cap;
int rc = ha_cb_send_append_entries_for_test(&c->ha, /*peer*/0,
/*prev_idx*/1, /*prev_term*/4,
/*leader_commit*/1, /*n*/1);
repl_send_ae_capture_hook = NULL;
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(1, cap.called);
TEST_ASSERT_EQUAL_INT(0, cap.peer_idx);
TEST_ASSERT_EQUAL_UINT64(1, cap.prev_log_index);
TEST_ASSERT_EQUAL_UINT64(4, cap.prev_log_term);
TEST_ASSERT_EQUAL_UINT64(1, cap.leader_commit);
TEST_ASSERT_EQUAL_INT(1, cap.n_entries);
/* The single entry is sourced from the local WAL at prev_log_index+1 == 2. */
TEST_ASSERT_EQUAL_UINT64(2, cap.entry_lsn);
TEST_ASSERT_EQUAL_UINT64(expect_term, cap.entry_term);
TEST_ASSERT_EQUAL_UINT8(WAL_JOB_UPDATE, cap.entry_type);
TEST_ASSERT_EQUAL_UINT32(2, cap.payload_len);
TEST_ASSERT_EQUAL_MEMORY("QQ", cap.payload, 2);
teardown_cluster(c);
printf("test_send_append_entries_one_entry_from_wal: PASS\n");
}
/* Test: on_send_append_entries for n_entries==0 emits a heartbeat carrying the
* prev tuple + leader_commit and ZERO entries (R1.3). */
static void test_send_append_entries_heartbeat_zero_entries(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_ae0_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t *c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
atomic_store(&c->ha.role, CTRL_PRIMARY);
/* One entry on disk so prev_log_index=1 is a real tip; heartbeat sends none.
* Persist BEFORE registering the standby (see one-entry test note). */
uint8_t blob[64];
size_t n1 = pack_entry(blob, WAL_JOB_UPDATE, "h", 1);
TEST_ASSERT_EQUAL_INT(0,
ha_cb_persist_entry_for_test(&c->ha, /*index*/1, /*term*/3, blob, n1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
register_one_standby(c);
repl_ae_capture_t cap;
memset(&cap, 0, sizeof(cap));
repl_send_ae_capture_hook = &cap;
int rc = ha_cb_send_append_entries_for_test(&c->ha, /*peer*/0,
/*prev_idx*/1, /*prev_term*/3,
/*leader_commit*/1, /*n*/0);
repl_send_ae_capture_hook = NULL;
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(1, cap.called);
TEST_ASSERT_EQUAL_UINT64(1, cap.prev_log_index);
TEST_ASSERT_EQUAL_UINT64(3, cap.prev_log_term);
TEST_ASSERT_EQUAL_UINT64(1, cap.leader_commit);
TEST_ASSERT_EQUAL_INT(0, cap.n_entries);
/* No entry sourced for a heartbeat. */
TEST_ASSERT_EQUAL_UINT32(0, cap.payload_len);
teardown_cluster(c);
printf("test_send_append_entries_heartbeat_zero_entries: PASS\n");
}
/* ================================================================== */
/* Task 2.4 — on_commit_advanced trigger + commit-source switch */
/* ================================================================== */
/*
* Task 2.4 feeds the library-supplied current-term-gated `commit_index` into
* the existing synchronous commit barrier as the commit SOURCE. The barrier
* mechanism is unchanged: committed_lsn still advances only when self+majority
* agree AND the bytes are durable locally (durable_lsn cap). The library commit
* index becomes an ADDITIONAL ceiling so the barrier never advances committed
* past the library's current-term-safe point. on_commit_advanced only TRIGGERS
* (records the ceiling + re-evaluates); it never performs durability itself.
*
* These cases build the commit engine standalone over a stub persist_t +
* replicator (the test_commit_engine.c pattern) so the barrier interaction is
* tested in isolation, plus one case driving the WIRED on_commit_advanced
* callback end-to-end through ha_cb_commit_advanced_for_test.
*
* RED: against the no-op ha_cb_commit_advanced_stub (and the missing
* commit_set_raft_commit_index ceiling), the library commit index is NOT a
* commit source — committed_lsn would advance on durable-majority alone with
* no current-term gate. GREEN: the wired callback feeds the ceiling so the
* barrier holds until BOTH durable majority and the library commit index agree.
*/
/* Stub-persist fixture for the commit-source tests (mirrors test_commit_engine.c). */
static persist_t g_t24_p;
static commit_engine_t g_t24_ce;
static int g_t24_eps[MAX_STANDBYS];
static void t24_engine_setup(int n_peers, int n_standbys, uint64_t appended_lsn)
{
char name[16];
memset(&g_t24_p, 0, sizeof(g_t24_p));
TEST_ASSERT_EQUAL_INT(0, repl_init(&g_t24_p.repl));
for (int i = 0; i < n_standbys; i++) {
snprintf(name, sizeof(name), "sb%d", i);
TEST_ASSERT_EQUAL_INT(
i, repl_add_standby(&g_t24_p.repl, &g_t24_eps[i], name,
7000 + (uint32_t)i));
}
g_t24_p.current_lsn = appended_lsn + 1;
atomic_store(&g_t24_p.committed_lsn, 0);
atomic_store(&g_t24_p.durable_lsn, appended_lsn);
TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&g_t24_ce, &g_t24_p, n_peers, 8));
}
static void t24_engine_teardown(void)
{
commit_engine_fini(&g_t24_ce);
repl_fini(&g_t24_p.repl);
}
static void t24_standby_ack(int idx, uint64_t acked_lsn)
{
atomic_store(&g_t24_p.repl.standbys[idx].acked_lsn, acked_lsn);
commit_on_ack(&g_t24_ce, idx, acked_lsn);
}
static uint64_t t24_committed(void)
{
return atomic_load(&g_t24_p.committed_lsn);
}
/* Test: the library commit index is the commit SOURCE once fed. A durable
* majority covers LSN 3, but the library has only admitted (current-term commit)
* up to LSN 1, so the barrier caps committed at 1 — NOT at the durable-majority
* floor of 3. Raising the library ceiling to 3 then releases the barrier to 3.
* This proves the library current-term commit index gates the commit, replacing
* the term-blind safe-commit (R4.1, R4.5, R5.4).
*
* (raft_commit_index == 0 is the INERT pre-wire state — the library source is
* not yet active and the barrier keeps autonomous durable-majority behavior;
* that flag-OFF fallback is covered by the existing test_commit_engine suite,
* whose autonomous semantics this task must not regress.) */
static void test_commit_advanced_is_commit_source(void)
{
/* 3 peers = self + 2 standbys; majority needs self + 1 standby. */
t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/3);
/* The library is the commit source from the first commit: establish the
* ceiling at LSN 1 BEFORE any ACK drives evaluation (production order — the
* ACK path drives raft_advance_commit, which fires on_commit_advanced, then
* the barrier re-evaluates). */
commit_set_raft_commit_index(&g_t24_ce, 1);
/* A standby ACKs LSN 3; self is durable to LSN 3 → durable majority = 3.
* The barrier must cap committed at the library ceiling (1), NOT at the
* durable majority (3). */
t24_standby_ack(0, 3);
TEST_ASSERT_EQUAL_UINT64(1, t24_committed());
/* Library admits up to LSN 3 → barrier releases to the durable majority (3). */
commit_set_raft_commit_index(&g_t24_ce, 3);
TEST_ASSERT_EQUAL_UINT64(3, t24_committed());
t24_engine_teardown();
printf("test_commit_advanced_is_commit_source: PASS\n");
}
/* Test: the library commit index is a CEILING — committed never advances past
* it even when a durable majority covers a higher LSN (current-term gate). The
* barrier's durable cap is the OTHER ceiling: committed = min(durable-majority,
* raft_commit_index). No gap: raising either ceiling alone cannot overshoot the
* other (R4.1, R4.5). */
static void test_commit_advanced_ceiling_caps_durable_majority(void)
{
t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/5);
/* Establish the library ceiling at 3 BEFORE any ACK (production order). The
* library has only committed up to 3 (higher entries prev-term or not yet
* current-term-agreed). */
commit_set_raft_commit_index(&g_t24_ce, 3);
/* Durable majority covers up to LSN 5 (self durable 5 + standby acked 5).
* committed must cap at the library ceiling (3), not the durable floor (5). */
t24_standby_ack(0, 5);
TEST_ASSERT_EQUAL_UINT64(3, t24_committed());
/* Library advances the ceiling to 5 → committed catches up to the durable
* majority floor (5). Monotonic, no regression. */
commit_set_raft_commit_index(&g_t24_ce, 5);
TEST_ASSERT_EQUAL_UINT64(5, t24_committed());
t24_engine_teardown();
printf("test_commit_advanced_ceiling_caps_durable_majority: PASS\n");
}
/* Test: the library commit index NEVER lets committed advance past what is
* durable locally. With raft_commit_index=4 but self only durable to LSN 2 and
* the standby acked LSN 2, committed must stay at the durable-majority floor (2)
* — the library ceiling does not bypass persist-before-ACK / durable cap
* (R5.4 — barrier durability mechanism unchanged). */
static void test_commit_advanced_does_not_bypass_durable_cap(void)
{
t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/4);
atomic_store(&g_t24_p.durable_lsn, 2); /* self durable only to 2 */
t24_standby_ack(0, 2); /* standby agrees to 2 */
commit_set_raft_commit_index(&g_t24_ce, 4); /* library ceiling well ahead */
commit_on_ack(&g_t24_ce, 0, 2);
/* Durable-majority floor is 2; the library ceiling (4) cannot raise it. */
TEST_ASSERT_EQUAL_UINT64(2, t24_committed());
t24_engine_teardown();
printf("test_commit_advanced_does_not_bypass_durable_cap: PASS\n");
}
/* Test (pins the own-term-gate DECISION — RETAINED, complementary, no deadlock):
* the own-term writable_lsn gate is a WAIT-completion predicate, NOT a cap on
* committed_lsn. It is retained alongside the library commit source. With the
* gate armed at LSN 2 and the library commit index + durable majority both at 2,
* committed advances to 2 AND a WAIT(lsn=2) completes — the two gates cannot
* deadlock (the own-term gate never holds committed_lsn back; the library
* ceiling never holds the WAIT predicate back beyond committed_lsn). */
static void test_own_term_gate_retained_no_deadlock(void)
{
t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/2);
/* Arm the own-term gate at LSN 2 (retained complementary durability gate). */
commit_set_writable_lsn(&g_t24_ce, 2);
/* Library commit source + durable majority both reach LSN 2. */
t24_standby_ack(0, 2);
commit_set_raft_commit_index(&g_t24_ce, 2);
commit_on_ack(&g_t24_ce, 0, 2);
/* committed_lsn advanced to 2 (own-term gate never caps committed_lsn). */
TEST_ASSERT_EQUAL_UINT64(2, t24_committed());
/* A WAIT for LSN 2 completes immediately: committed_lsn (2) >= lsn (2) AND
* >= writable_lsn (2). No deadlock between the gate and the library source. */
TEST_ASSERT_EQUAL_INT(0,
commit_submit(&g_t24_ce, /*lsn*/2, COMMIT_MODE_WAIT, /*timeout_ms*/500));
t24_engine_teardown();
printf("test_own_term_gate_retained_no_deadlock: PASS\n");
}
/* Test: the WIRED on_commit_advanced callback routes commit_index into the
* cluster's commit engine end-to-end (ha_cb_commit_advanced_for_test) as the
* commit SOURCE. A durable majority covers LSN 2, but the wired callback feeds
* commit_index=1 first → the barrier caps committed at 1 (library current-term
* gate), NOT at the durable majority of 2. A second callback at commit_index=2
* releases the barrier to 2. Proves the stub→real transition: the no-op stub
* never fed the source, so committed would have followed the durable majority. */
static void test_on_commit_advanced_wired_feeds_barrier(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_commitadv_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t *c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
/* Init the cluster's embedded commit engine over the same persist as a
* 3-peer cluster (self + 2 standbys), so a majority is self + 1 standby. */
TEST_ASSERT_EQUAL_INT(0,
commit_engine_init(&c->commit, &c->persist, /*n_peers*/3, /*n_vcpus*/8));
/* Persist two entries BEFORE registering a standby — otherwise the primary
* own-append fans out via repl_broadcast to the dummy int ep (ASAN
* global-buffer-overflow reading ep->peer); with no standby the broadcast
* is a no-op. Append LSN 1,2 with term 4 → durable floor reaches LSN 2. */
atomic_store(&c->ha.role, CTRL_PRIMARY);
uint8_t blob[64];
size_t n1 = pack_entry(blob, WAL_JOB_UPDATE, "a", 1);
TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 4, blob, n1));
size_t n2 = pack_entry(blob, WAL_JOB_UPDATE, "b", 1);
TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 2, 4, blob, n2));
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
/* Now register the standby so a majority (self + 1) is reachable. The ACK
* below is driven directly into the registry + commit_on_ack (no network). */
TEST_ASSERT_EQUAL_INT(0,
repl_add_standby(&c->persist.repl, &g_ae_dummy_ep, "sb0", 7000));
/* Fire the WIRED callback FIRST to establish the library ceiling at LSN 1
* (production order: the ACK path drives raft_advance_commit, which fires
* on_commit_advanced before/with the barrier re-evaluation). */
ha_cb_commit_advanced_for_test(&c->ha, 1);
/* Standby ACKs LSN 2 → durable majority covers LSN 2. The wired source caps
* committed at 1, BELOW the durable majority (2): the library current-term
* gate governs (the term-blind safe-commit is retired as the source). */
atomic_store(&c->persist.repl.standbys[0].acked_lsn, 2);
commit_on_ack(&c->commit, 0, 2);
TEST_ASSERT_EQUAL_UINT64(1, atomic_load(&c->persist.committed_lsn));
/* Library admits LSN 2 → barrier releases to the durable majority (2). */
ha_cb_commit_advanced_for_test(&c->ha, 2);
TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn));
commit_engine_fini(&c->commit);
teardown_cluster(c);
printf("test_on_commit_advanced_wired_feeds_barrier: PASS\n");
}
/* ================================================================== */
/* Task 2.5 — library init restore from the recovered WAL floor */
/* ================================================================== */
/* Test: ha_init_raft restores the library log/commit floor from the
* recovered WAL instead of booting at LSN 0 (R1.5, R2.1).
*
* After a member recovers a WAL whose durable tip is LSN 3 (term 7) with a
* committed prefix at LSN 2, raft_init must boot with:
* raft.log_len == 3 (initial_log_len = persist_durable_floor)
* raft.commit_index == 2 (initial_commit_index = recovered committed_lsn)
* and the last-log term must be recoverable via the wired on_get_entry_term
* (== 7). Booting from an empty log (log_len/commit_index == 0) is the pre-fix
* behaviour this test fails on. */
static void test_init_restores_recovered_floor(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_initrestore_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
const uint64_t T = 7;
/* Phase 1: lay down a 3-entry WAL (term 7) and flush so the recovered tip
* reaches LSN 3. */
{
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
atomic_store(&c->ha.role, CTRL_PRIMARY);
uint8_t blob[64];
for (uint64_t i = 1; i <= 3; i++) {
size_t n = pack_entry(blob, WAL_JOB_UPDATE, "x", 1);
TEST_ASSERT_EQUAL_INT(0,
ha_cb_persist_entry_for_test(&c->ha, i, T, blob, n));
}
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
teardown_cluster(c);
}
/* Phase 2: recover into a fresh cluster; durable floor == 3. Simulate a
* recovered committed prefix at LSN 2 (recovery re-derives the durable tip;
* the committed position is set by the surrounding boot path). */
{
cluster_t* c2 = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c2);
TEST_ASSERT_EQUAL_INT(0, persist_recover(&c2->persist));
TEST_ASSERT_EQUAL_UINT64(3, persist_durable_floor(&c2->persist));
atomic_store(&c2->persist.committed_lsn, 2ULL);
/* Initialize the embedded raft as a single-node cluster. The restore
* must source the floor from the recovered persist, not LSN 0. */
ha_init_raft(&c2->ha, c2, /*n_peers*/1);
TEST_ASSERT_EQUAL_UINT64(3, c2->ha.raft.log_len);
TEST_ASSERT_EQUAL_UINT64(2, c2->ha.raft.commit_index);
/* The recovered last-log term is surfaced via the wired callback. */
TEST_ASSERT_EQUAL_UINT64(T,
ha_cb_get_entry_term_for_test(&c2->ha, c2->ha.raft.log_len));
/* All five term-safety callbacks remain wired (non-NULL). */
TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs);
TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_persist_entry);
TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_get_entry_term);
TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_send_append_entries);
TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_truncate_from);
TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_commit_advanced);
/* Existing election callbacks remain wired. */
TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_become_leader);
TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_persist);
teardown_cluster(c2);
}
printf("test_init_restores_recovered_floor: PASS\n");
}
/* Test: an empty-WAL first boot restores a zero floor (no spurious non-zero
* indices) — the recovery-read-failure / empty-log edge of R1.5. */
static void test_init_empty_wal_zero_floor(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_initempty_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
TEST_ASSERT_EQUAL_INT(0, persist_recover(&c->persist));
TEST_ASSERT_EQUAL_UINT64(0, persist_durable_floor(&c->persist));
ha_init_raft(&c->ha, c, /*n_peers*/1);
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.log_len);
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.commit_index);
teardown_cluster(c);
printf("test_init_empty_wal_zero_floor: PASS\n");
}
/* ================================================================== */
/* Task 3.1 — drive raft_log_append on the primary append path */
/* ================================================================== */
/* Bring a make_cluster()'d node up as the raft LEADER so raft_log_append is
* enabled (it returns -EPERM off the leader). We init the library from the
* recovered floor (task 2.5) and set the live election term + raft role
* directly — modelling a node that has already won an election at `term`
* (the cerebrum raft-sim convention: set state.term after a clean election;
* raft_log_append stamps entries from s->term). */
static void make_leader(cluster_t* c, uint64_t term, int n_peers)
{
atomic_store(&c->ha.role, CTRL_PRIMARY);
atomic_store(&c->ha.term, term);
ha_init_raft(&c->ha, c, n_peers);
c->ha.raft.term = term;
c->ha.raft.role = RAFT_LEADER;
}
/* Test: a primary job-state append driven through ha_primary_log_append goes
* through the library — the library assigns the LSN (== s->log_len post-append),
* stamps the live election term into the WAL header, and keeps the persist
* write cursor co-tracked with the library index (current_lsn == log_len + 1).
* Against the pre-3.1 hand-rolled path this entry point does not exist
* (compile RED) — and once it exists the library MUST be the LSN/term source
* (R1.1, R1.2, R2.1). */
static void test_primary_log_append_stamps_term_and_assigns_lsn(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_plap_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
const uint64_t T = 7;
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
make_leader(c, T, /*n_peers*/1);
/* Fresh leader, empty log: first append takes library index 1. */
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.log_len);
wal_jobstate_t js = { .jobid = 42, .state = 3 };
uint64_t lsn = 0;
TEST_ASSERT_EQUAL_INT(0,
ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &lsn));
/* (a) The LIBRARY assigned the LSN and grew its log to it. */
TEST_ASSERT_EQUAL_UINT64(1, lsn);
TEST_ASSERT_EQUAL_UINT64(1, c->ha.raft.log_len);
/* (b) The WAL header recorded the live election term for that LSN. */
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1));
/* (d) Co-tracking invariant: the persist write cursor advanced exactly in
* lockstep with the library index — current_lsn == log_len + 1 (next LSN). */
TEST_ASSERT_EQUAL_UINT64(c->ha.raft.log_len + 1, c->persist.current_lsn);
/* A second append keeps the two counters co-tracked. */
js.jobid = 43;
TEST_ASSERT_EQUAL_INT(0,
ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &lsn));
TEST_ASSERT_EQUAL_UINT64(2, lsn);
TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.log_len);
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 2));
TEST_ASSERT_EQUAL_UINT64(c->ha.raft.log_len + 1, c->persist.current_lsn);
teardown_cluster(c);
printf("test_primary_log_append_stamps_term_and_assigns_lsn: PASS\n");
}
/* Test: the own-term promotion no-op (ha_arm_own_term_gate) is routed through
* the library, so the WAL_NOOP entry records the newly established term (not 0)
* and the library index tracks it (R1.2). Pre-3.1 the noop went through a
* hand-rolled persist_wal_async with a best-effort p->term and did NOT advance
* s->log_len. */
static void test_own_term_noop_records_live_term_via_library(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_noop_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
const uint64_t T = 5;
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
make_leader(c, T, /*n_peers*/1);
/* Register the persist hooks then arm the own-term gate exactly as the
* promotion tail does (ha_promote_to_primary / raft_cb_become_leader). */
ha_arm_own_term_gate_for_test(c);
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
/* The no-op landed at library index 1 with the live term recorded. */
TEST_ASSERT_EQUAL_UINT64(1, c->ha.raft.log_len);
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1));
TEST_ASSERT_EQUAL_UINT64(c->ha.raft.log_len + 1, c->persist.current_lsn);
teardown_cluster(c);
printf("test_own_term_noop_records_live_term_via_library: PASS\n");
}
/* Test: when raft leadership is NOT active (the library is not RAFT_LEADER —
* the init-time non-raft promotion road and engine-less test contexts),
* ha_primary_log_append still appends locally through the legacy path so the
* non-raft promotion tail keeps working, WITHOUT growing the library log (the
* library is not tracking the log on that road, so no co-tracking hazard).
* The entry still records the live election term via persist_set_term. */
static void test_primary_log_append_non_leader_legacy_fallback(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_plapfb_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
const uint64_t T = 6;
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
/* Non-raft promotion road: controller role PRIMARY + live term, but the
* raft library was never driven to LEADER (raft.role stays FOLLOWER). */
atomic_store(&c->ha.role, CTRL_PRIMARY);
atomic_store(&c->ha.term, T);
ha_init_raft(&c->ha, c, /*n_peers*/1);
TEST_ASSERT_NOT_EQUAL_INT(RAFT_LEADER, c->ha.raft.role);
wal_jobstate_t js = { .jobid = 7, .state = 1 };
uint64_t lsn = 0;
TEST_ASSERT_EQUAL_INT(0,
ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &lsn));
/* Appended at the contiguous next LSN (current_lsn started at 1). */
TEST_ASSERT_EQUAL_UINT64(1, lsn);
TEST_ASSERT_EQUAL_UINT64(2, c->persist.current_lsn);
/* Live term still recorded for the entry. */
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1));
/* The library log was NOT grown by the legacy fallback (no raft leadership). */
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.log_len);
teardown_cluster(c);
printf("test_primary_log_append_non_leader_legacy_fallback: PASS\n");
}
/* ================================================================== */
/* Task 3.2 — route heartbeat/replication intake through the library */
/* raft_on_append_entries_request: UpdateTerm (G5) + logOk (G2) + */
/* (term,lsn) conflict-truncate (G3) via the wired callbacks. */
/* */
/* These drive the production intake helper ha_consume_append_entries */
/* (the in-process route that replaces the term-blind handling in */
/* ha_consume_heartbeat_committed + the committed-boundary truncate in */
/* ha_rejoin_reconcile). RED: that helper does not exist pre-3.2 */
/* (compile RED); once it exists the three observable behaviors hold. */
/* ================================================================== */
/* Make a node a STANDBY follower at a given term + raft log floor, with the
* raft library initialized so the intake can drive it. Seeds the library log
* to match the persisted WAL tip (the ha_init_raft floor restore). */
static void make_follower(cluster_t* c, uint64_t term, int n_peers)
{
atomic_store(&c->ha.role, CTRL_STANDBY);
atomic_store(&c->ha.term, term);
ha_init_raft(&c->ha, c, n_peers);
c->ha.raft.term = term;
c->ha.raft.role = RAFT_FOLLOWER;
}
/* Build a decoded single-entry WalCatchupEntry (n_entries==0 path == one entry
* carried in the top-level container, matching the on-wire single-entry mode).
* For a heartbeat use n_entries set so the helper sees zero entries. */
static void fill_ae_entry(wal_catchup_entry_t* e, int is_heartbeat,
uint64_t prev_log_index, uint64_t prev_log_term,
uint64_t leader_commit, uint64_t entry_lsn,
uint8_t entry_type, uint64_t entry_term,
uint8_t* payload, uint32_t plen)
{
memset(e, 0, sizeof(*e));
e->prev_log_index = prev_log_index;
e->prev_log_term = prev_log_term;
e->leader_commit = leader_commit;
if (is_heartbeat) {
e->n_entries = 0;
e->lsn = 0;
} else {
/* single-entry mode: top-level container carries the entry, n_entries==1
* with no nested sub-entries (one-entry AppendEntries). */
e->n_entries = 1;
e->lsn = entry_lsn;
e->entry_type = entry_type;
e->term = entry_term;
e->raw_payload = payload;
e->raw_payload_len = plen;
}
}
/* (a) A higher-term heartbeat (0-entry AppendEntries) routed through the library
* makes the follower ADOPT the higher term and step down — term recorded
* (R2.6 / G5). Pre-3.2 ha_consume_heartbeat_committed only DROPPED stale
* heartbeats and never adopted a higher term. */
static void test_intake_higher_term_heartbeat_adopts_term(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_intake_ht_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
make_follower(c, /*term*/3, /*n_peers*/3);
/* The follower briefly believes itself a candidate/leader at term 3. */
c->ha.raft.role = RAFT_LEADER;
/* Heartbeat from a leader at term 7, empty log (prev=0), commit 0. The
* heartbeat's top-level container carries the leader's live term. */
wal_catchup_entry_t hb;
fill_ae_entry(&hb, /*heartbeat*/1, /*prev_idx*/0, /*prev_term*/0,
/*leader_commit*/0, 0, 0, 0, NULL, 0);
hb.term = 7;
int rc = ha_consume_append_entries(c, &hb, /*leader_idx*/1);
TEST_ASSERT_EQUAL_INT(0, rc);
/* The library adopted the higher term and stepped down to follower. */
TEST_ASSERT_EQUAL_UINT64(7, c->ha.raft.term);
TEST_ASSERT_NOT_EQUAL_INT(RAFT_LEADER, c->ha.raft.role);
teardown_cluster(c);
printf("test_intake_higher_term_heartbeat_adopts_term: PASS\n");
}
/* (b) A divergent same-LSN tail is truncated and the leader's entry accepted
* (R3.1/R3.2 / G3), and the committed prefix below the divergence is untouched
* (R3.3). The follower locally holds [LSN1@term2, LSN2@term2]; the leader sends
* an AppendEntries appending LSN2@term5 on top of LSN1@term2. The library
* detects the (term,lsn) conflict at LSN2, truncates LSN>=2, and re-persists
* the leader's LSN2@term5. LSN1 (the agreed prefix) stays intact. */
static void test_intake_divergent_tail_truncated_and_reconverged(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_intake_div_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
make_follower(c, /*term*/2, /*n_peers*/3);
/* Seed the local log: LSN1@term2, LSN2@term2 (the divergent tail). */
uint8_t blob[64];
size_t n = pack_entry(blob, WAL_JOB_UPDATE, "a", 1);
TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 2, blob, n));
TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 2, 2, blob, n));
c->ha.raft.log_len = 2;
TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 1));
TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 2));
/* Leader (term 5) sends AppendEntries: prev=(LSN1,term2), entry LSN2@term5.
* logOk holds at LSN1 (term2==term2); conflict at LSN2 (local term2 !=
* incoming term5) → truncate LSN>=2 then persist LSN2@term5. */
uint8_t lpay[1] = { 'b' };
wal_catchup_entry_t ae;
fill_ae_entry(&ae, /*heartbeat*/0, /*prev_idx*/1, /*prev_term*/2,
/*leader_commit*/1, /*entry_lsn*/2, WAL_JOB_UPDATE,
/*entry_term*/5, lpay, 1);
int rc = ha_consume_append_entries(c, &ae, /*leader_idx*/1);
TEST_ASSERT_EQUAL_INT(0, rc);
/* (term,lsn) reconverged: LSN2 now carries the leader's term 5. */
TEST_ASSERT_EQUAL_UINT64(5, persist_entry_term(&c->persist, 2));
/* The agreed committed prefix (LSN1@term2) is untouched (R3.3). */
TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 1));
/* The higher leader term was adopted on the RPC (G5). */
TEST_ASSERT_EQUAL_UINT64(5, c->ha.raft.term);
teardown_cluster(c);
printf("test_intake_divergent_tail_truncated_and_reconverged: PASS\n");
}
/* (c) An unreconcilable member (logOk fails: the leader's prev tuple does not
* match the follower's log) is REJECTED — the follower neither persists the
* leader's entry nor reports false convergence, and stays available to retry
* (no crash, no committed advance). R3.5/R3.6: the member keeps its divergent
* state pending a later catch-up rather than spuriously accepting. */
static void test_intake_logmismatch_rejected_no_false_convergence(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_intake_rej_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
make_follower(c, /*term*/4, /*n_peers*/3);
/* Local log: LSN1@term2. The leader claims prev=(LSN1,term3) — a term
* mismatch at LSN1 → logOk fails. */
uint8_t blob[64];
size_t n = pack_entry(blob, WAL_JOB_UPDATE, "a", 1);
TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 2, blob, n));
c->ha.raft.log_len = 1;
uint8_t lpay[1] = { 'z' };
wal_catchup_entry_t ae;
fill_ae_entry(&ae, /*heartbeat*/0, /*prev_idx*/1, /*prev_term*/3,
/*leader_commit*/0, /*entry_lsn*/2, WAL_JOB_UPDATE,
/*entry_term*/4, lpay, 1);
int rc = ha_consume_append_entries(c, &ae, /*leader_idx*/1);
/* The helper returns 0 (no crash) but the AE was NOT accepted: the leader's
* entry was not persisted at LSN2 and the local LSN1 entry is unchanged. */
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(&c->persist, 2)); /* absent */
TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 1)); /* intact */
teardown_cluster(c);
printf("test_intake_logmismatch_rejected_no_false_convergence: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Suite entry point */
/* ------------------------------------------------------------------ */
void test_term_safety_adapter(void)
{
RUN_TEST(test_wal_entry_hdr_size_is_40);
RUN_TEST(test_set_term_stamps_append);
RUN_TEST(test_recover_mixed_real_and_legacy_terms);
RUN_TEST(test_entry_term_absent_lsn_is_zero);
RUN_TEST(test_entry_term_null_safe);
RUN_TEST(test_on_persist_entry_stamps_term_standby);
RUN_TEST(test_on_persist_entry_stamps_term_primary);
RUN_TEST(test_on_persist_entry_failure_propagates_no_ack);
RUN_TEST(test_on_get_entry_term_present_legacy_absent);
RUN_TEST(test_on_truncate_from_drops_suffix);
RUN_TEST(test_on_truncate_from_removes_all);
RUN_TEST(test_send_append_entries_one_entry_from_wal);
RUN_TEST(test_send_append_entries_heartbeat_zero_entries);
RUN_TEST(test_commit_advanced_is_commit_source);
RUN_TEST(test_commit_advanced_ceiling_caps_durable_majority);
RUN_TEST(test_commit_advanced_does_not_bypass_durable_cap);
RUN_TEST(test_own_term_gate_retained_no_deadlock);
RUN_TEST(test_on_commit_advanced_wired_feeds_barrier);
RUN_TEST(test_init_restores_recovered_floor);
RUN_TEST(test_init_empty_wal_zero_floor);
RUN_TEST(test_primary_log_append_stamps_term_and_assigns_lsn);
RUN_TEST(test_own_term_noop_records_live_term_via_library);
RUN_TEST(test_primary_log_append_non_leader_legacy_fallback);
RUN_TEST(test_intake_higher_term_heartbeat_adopts_term);
RUN_TEST(test_intake_divergent_tail_truncated_and_reconverged);
RUN_TEST(test_intake_logmismatch_rejected_no_false_convergence);
}
Reference in a new issue View git blame Copy permalink