forked from lei.chen/myxce
2213 lines
99 KiB
C
2213 lines
99 KiB
C
#define _GNU_SOURCE
|
||
#include <errno.h>
|
||
#include <stdint.h>
|
||
#include <stdlib.h>
|
||
#include <string.h>
|
||
#include <stdio.h>
|
||
#include <unistd.h>
|
||
|
||
#include <stdatomic.h>
|
||
|
||
#include <unity/unity.h>
|
||
|
||
#include "config.h"
|
||
#include "config_repl.h"
|
||
#include "persist.h"
|
||
#include "wal_types.h"
|
||
#include "repl.h"
|
||
#include "commit.h"
|
||
#include "xce/cluster_def.h"
|
||
#include "ha.h"
|
||
|
||
/* ha-term-safety task 1.3 — WAL term seam + recovery read-back + legacy-0 rule.
|
||
*
|
||
* The seam under test:
|
||
* - persist_set_term(p, T) sets the controller's live election term so a
|
||
* subsequent append stamps hdr.term = T (R1.1, R1.6).
|
||
* - replay_wal retains each entry's hdr.term per LSN in an in-memory index
|
||
* (R1.5), read back via persist_entry_term(p, lsn).
|
||
* - persist_entry_term returns 0 for an absent LSN; a legacy entry carrying
|
||
* hdr.term == 0 maps to the single lowest-known real Raft term (1) so a
|
||
* legacy-0 entry is never spuriously divergent from another legacy-0 entry
|
||
* and recovery never fails on legacy-0 (R1.8).
|
||
* - the on-disk wal_entry_hdr_t stays 40 bytes (R1.7).
|
||
*/
|
||
|
||
/* The defined legacy-0 mapping (lowest real Raft term). Kept local to the test
|
||
* so the assertion is independent of the implementation constant. */
|
||
#define LEGACY_LOWEST_TERM 1u
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Helpers */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
static persist_t* make_fresh_persist(const char* tmpdir)
|
||
{
|
||
persist_t* p = calloc(1, sizeof(persist_t));
|
||
if (!p) return NULL;
|
||
|
||
persist_config_t cfg = persist_config_defaults();
|
||
cfg.wal_sync_ms = 5000; /* slow fsync — no races in tests */
|
||
cfg.snap_keep_n = 2;
|
||
|
||
persist_set_workdir(p, tmpdir, "ctrl-test");
|
||
|
||
if (persist_init(p, &cfg, NULL) != 0) {
|
||
free(p);
|
||
return NULL;
|
||
}
|
||
if (persist_start(p) != 0) {
|
||
persist_fini(p);
|
||
free(p);
|
||
return NULL;
|
||
}
|
||
return p;
|
||
}
|
||
|
||
static void teardown_persist(persist_t* p)
|
||
{
|
||
persist_stop(p);
|
||
persist_fini(p);
|
||
free(p);
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Test: header layout preserved (R1.7) — value fix, not a format */
|
||
/* change. The on-disk WAL entry header must stay exactly 40 bytes. */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
static void test_wal_entry_hdr_size_is_40(void)
|
||
{
|
||
TEST_ASSERT_EQUAL_UINT64(40u, (uint64_t)sizeof(wal_entry_hdr_t));
|
||
printf("test_wal_entry_hdr_size_is_40: PASS\n");
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Test: persist_set_term makes a subsequent append stamp hdr.term */
|
||
/* (R1.1, R1.6), recovered back via persist_entry_term. */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
static void test_set_term_stamps_append(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_term_set_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
const uint64_t T = 7;
|
||
|
||
/* Phase 1: set the live term, then append — the append stamps hdr.term=T. */
|
||
{
|
||
persist_t* p = make_fresh_persist(tmp);
|
||
TEST_ASSERT_NOT_NULL(p);
|
||
|
||
persist_set_term(p, T);
|
||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
|
||
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
|
||
|
||
/* Live read-back of the just-appended entry's term (LSN 1). */
|
||
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(p, 1));
|
||
|
||
teardown_persist(p);
|
||
}
|
||
|
||
/* Phase 2: fresh persist_t recovers the WAL and reads the term back. */
|
||
{
|
||
persist_t* p2 = make_fresh_persist(tmp);
|
||
TEST_ASSERT_NOT_NULL(p2);
|
||
|
||
TEST_ASSERT_EQUAL_INT(0, persist_recover(p2));
|
||
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(p2, 1));
|
||
|
||
teardown_persist(p2);
|
||
}
|
||
|
||
printf("test_set_term_stamps_append: PASS\n");
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Test: replay_wal of a WAL with mixed real-term and legacy-0 */
|
||
/* entries — persist_entry_term returns the real term for real */
|
||
/* entries and the defined lowest-known term for legacy-0 (R1.5, */
|
||
/* R1.8). Recovery never fails on legacy-0 entries. */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
static void test_recover_mixed_real_and_legacy_terms(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_term_mixed_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
/* Phase 1: write LSN 1,2 with term 0 (legacy — the pre-feature default),
|
||
* then LSN 3,4 with real term 5. The append path stamps hdr.term=p->term,
|
||
* so leaving the term unset (0) produces genuine legacy-0 entries. */
|
||
{
|
||
persist_t* p = make_fresh_persist(tmp);
|
||
TEST_ASSERT_NOT_NULL(p);
|
||
|
||
/* Legacy-0 entries: no persist_set_term call (p->term stays 0). */
|
||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "l", 1)); /* lsn 1 */
|
||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "m", 1)); /* lsn 2 */
|
||
|
||
/* Real-term entries. */
|
||
persist_set_term(p, 5);
|
||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "r", 1)); /* lsn 3 */
|
||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "s", 1)); /* lsn 4 */
|
||
|
||
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
|
||
teardown_persist(p);
|
||
}
|
||
|
||
/* Phase 2: fresh recover — read back each LSN's term. */
|
||
{
|
||
persist_t* p2 = make_fresh_persist(tmp);
|
||
TEST_ASSERT_NOT_NULL(p2);
|
||
|
||
/* Recovery must NOT fail on the legacy-0 entries. */
|
||
TEST_ASSERT_EQUAL_INT(0, persist_recover(p2));
|
||
TEST_ASSERT_EQUAL_INT(RECOVER_CLEAN, persist_recovery_verdict(p2));
|
||
|
||
/* Legacy-0 entries map to the single lowest-known real term (1) — both
|
||
* the same, so two legacy-0 entries are never seen as divergent. */
|
||
TEST_ASSERT_EQUAL_UINT64(LEGACY_LOWEST_TERM, persist_entry_term(p2, 1));
|
||
TEST_ASSERT_EQUAL_UINT64(LEGACY_LOWEST_TERM, persist_entry_term(p2, 2));
|
||
|
||
/* Real-term entries return their recorded term verbatim. */
|
||
TEST_ASSERT_EQUAL_UINT64(5, persist_entry_term(p2, 3));
|
||
TEST_ASSERT_EQUAL_UINT64(5, persist_entry_term(p2, 4));
|
||
|
||
teardown_persist(p2);
|
||
}
|
||
|
||
printf("test_recover_mixed_real_and_legacy_terms: PASS\n");
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Test: persist_entry_term returns 0 for an absent LSN (no entry). */
|
||
/* Distinguishes "absent" (0) from a present legacy-0 entry (mapped */
|
||
/* to lowest-known), which is the precision REVIEW ISSUE 2 requires. */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
static void test_entry_term_absent_lsn_is_zero(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_term_absent_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
persist_t* p = make_fresh_persist(tmp);
|
||
TEST_ASSERT_NOT_NULL(p);
|
||
|
||
persist_set_term(p, 3);
|
||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1)); /* lsn 1 */
|
||
|
||
/* LSN 0 is reserved (never an entry) and LSN 2 was never appended. */
|
||
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(p, 0));
|
||
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(p, 2));
|
||
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(p, 999999));
|
||
|
||
/* The present entry still answers its real term. */
|
||
TEST_ASSERT_EQUAL_UINT64(3, persist_entry_term(p, 1));
|
||
|
||
teardown_persist(p);
|
||
printf("test_entry_term_absent_lsn_is_zero: PASS\n");
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Test: persist_entry_term is NULL-safe. */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
static void test_entry_term_null_safe(void)
|
||
{
|
||
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(NULL, 1));
|
||
/* persist_set_term must not crash on NULL. */
|
||
persist_set_term(NULL, 9);
|
||
printf("test_entry_term_null_safe: PASS\n");
|
||
}
|
||
|
||
/* ================================================================== */
|
||
/* Task 2.1 — on_persist_entry callback body */
|
||
/* ================================================================== */
|
||
|
||
/* Build a cluster_t whose embedded persist module is initialised + started in
|
||
* `tmpdir`, with ha.raft_c self-referencing the cluster (the convention every
|
||
* raft callback uses to recover persist/role from ctx == &c->ha). Returns a
|
||
* calloc'd cluster_t the caller must teardown via teardown_cluster. */
|
||
static cluster_t* make_cluster(const char* tmpdir)
|
||
{
|
||
cluster_t* c = calloc(1, sizeof(cluster_t));
|
||
if (!c) return NULL;
|
||
|
||
persist_config_t cfg = persist_config_defaults();
|
||
cfg.wal_sync_ms = 5000;
|
||
cfg.snap_keep_n = 2;
|
||
|
||
persist_set_workdir(&c->persist, tmpdir, "ctrl-test");
|
||
if (persist_init(&c->persist, &cfg, c) != 0) { free(c); return NULL; }
|
||
if (persist_start(&c->persist) != 0) {
|
||
persist_fini(&c->persist);
|
||
free(c);
|
||
return NULL;
|
||
}
|
||
|
||
/* The callback recovers persist/role from ctx == &c->ha → c->ha.raft_c. */
|
||
c->ha.raft_c = c;
|
||
return c;
|
||
}
|
||
|
||
static void teardown_cluster(cluster_t* c)
|
||
{
|
||
if (!c) return;
|
||
persist_stop(&c->persist);
|
||
persist_fini(&c->persist);
|
||
free(c);
|
||
}
|
||
|
||
/* Pack one adapter entry blob: [raft_entry_blob_t{type,len}][payload]. */
|
||
static size_t pack_entry(uint8_t* buf, uint32_t type,
|
||
const void* payload, uint32_t plen)
|
||
{
|
||
raft_entry_blob_t* h = (raft_entry_blob_t*)buf;
|
||
h->type = type;
|
||
h->payload_len = plen;
|
||
if (payload && plen > 0)
|
||
memcpy(buf + sizeof(*h), payload, plen);
|
||
return sizeof(*h) + plen;
|
||
}
|
||
|
||
/* Test: the WIRED on_persist_entry stamps hdr.term from the library term at the
|
||
* supplied 1-based index == LSN, persisting through the existing WAL path; the
|
||
* recovered hdr.term equals the supplied term (R1.1–R1.4). Drives the standby
|
||
* (replicated) path so persist-before-ACK is exercised. */
|
||
static void test_on_persist_entry_stamps_term_standby(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_op_sb_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
const uint64_t T = 9;
|
||
|
||
/* Phase 1: standby applies a replicated entry at LSN 1 with leader term T. */
|
||
{
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
atomic_store(&c->ha.role, CTRL_STANDBY);
|
||
|
||
uint8_t blob[64];
|
||
size_t n = pack_entry(blob, WAL_JOB_UPDATE, "x", 1);
|
||
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
ha_cb_persist_entry_for_test(&c->ha, /*index*/1, T, blob, n));
|
||
|
||
/* Persisted at LSN 1 with hdr.term == T (live read-back). */
|
||
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1));
|
||
TEST_ASSERT_EQUAL_UINT64(2, c->persist.current_lsn);
|
||
|
||
teardown_cluster(c);
|
||
}
|
||
|
||
/* Phase 2: fresh recover reads the stamped term back from disk (R1.5). */
|
||
{
|
||
cluster_t* c2 = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c2);
|
||
TEST_ASSERT_EQUAL_INT(0, persist_recover(&c2->persist));
|
||
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c2->persist, 1));
|
||
teardown_cluster(c2);
|
||
}
|
||
|
||
printf("test_on_persist_entry_stamps_term_standby: PASS\n");
|
||
}
|
||
|
||
/* Test: the primary's own-append path through on_persist_entry stamps the term
|
||
* and persists at the contiguous next LSN. */
|
||
static void test_on_persist_entry_stamps_term_primary(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_op_pr_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
const uint64_t T = 4;
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
atomic_store(&c->ha.role, CTRL_PRIMARY);
|
||
|
||
uint8_t blob[64];
|
||
size_t n = pack_entry(blob, WAL_NOOP, NULL, 0); /* own-term promotion no-op */
|
||
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
ha_cb_persist_entry_for_test(&c->ha, /*index*/1, T, blob, n));
|
||
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
|
||
|
||
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1));
|
||
TEST_ASSERT_EQUAL_UINT64(2, c->persist.current_lsn);
|
||
|
||
teardown_cluster(c);
|
||
printf("test_on_persist_entry_stamps_term_primary: PASS\n");
|
||
}
|
||
|
||
/* Test: a forced persist failure (fdatasync fail-loud seam on the replicated
|
||
* write) makes on_persist_entry return non-zero and DOES NOT advance the log /
|
||
* ACK (persist-before-ACK preserved). */
|
||
static void test_on_persist_entry_failure_propagates_no_ack(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_op_fail_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
atomic_store(&c->ha.role, CTRL_STANDBY);
|
||
|
||
uint64_t durable_before =
|
||
atomic_load(&c->persist.durable_lsn);
|
||
|
||
/* Force the persist-before-ACK fdatasync to be treated as failed. */
|
||
atomic_store(&c->persist.test_force_fsync_fail, 1);
|
||
|
||
uint8_t blob[64];
|
||
size_t n = pack_entry(blob, WAL_JOB_UPDATE, "y", 1);
|
||
|
||
int rc = ha_cb_persist_entry_for_test(&c->ha, /*index*/1, 7, blob, n);
|
||
TEST_ASSERT_NOT_EQUAL_INT(0, rc); /* failure propagates → caller does NOT ACK */
|
||
|
||
/* Persist-before-ACK durability contract: the entry is NOT durable, so the
|
||
* durable watermark must NOT advance (the standby would ACK only on a
|
||
* successful, durable write — the non-zero rc above is exactly the signal
|
||
* that suppresses the ACK). durable_lsn (not the write cursor current_lsn)
|
||
* is the durability authority; it stays put on fdatasync failure. */
|
||
TEST_ASSERT_EQUAL_UINT64(durable_before,
|
||
atomic_load(&c->persist.durable_lsn));
|
||
|
||
atomic_store(&c->persist.test_force_fsync_fail, 0);
|
||
teardown_cluster(c);
|
||
printf("test_on_persist_entry_failure_propagates_no_ack: PASS\n");
|
||
}
|
||
|
||
/* ================================================================== */
|
||
/* Task 2.2 — on_get_entry_term + on_truncate_from callback bodies */
|
||
/* ================================================================== */
|
||
|
||
/* Persist a couple of replicated entries (each at a 1-based index == LSN)
|
||
* through the WIRED on_persist_entry callback, stamping a leader term per
|
||
* entry. Returns the (started) cluster the caller tears down. */
|
||
static cluster_t* persist_two_entries(const char* tmp, uint64_t t1, uint64_t t2)
|
||
{
|
||
cluster_t* c = make_cluster(tmp);
|
||
if (!c) return NULL;
|
||
atomic_store(&c->ha.role, CTRL_STANDBY);
|
||
|
||
uint8_t blob[64];
|
||
size_t n1 = pack_entry(blob, WAL_JOB_UPDATE, "a", 1);
|
||
if (ha_cb_persist_entry_for_test(&c->ha, /*index*/1, t1, blob, n1) != 0) {
|
||
teardown_cluster(c);
|
||
return NULL;
|
||
}
|
||
size_t n2 = pack_entry(blob, WAL_JOB_UPDATE, "b", 1);
|
||
if (ha_cb_persist_entry_for_test(&c->ha, /*index*/2, t2, blob, n2) != 0) {
|
||
teardown_cluster(c);
|
||
return NULL;
|
||
}
|
||
return c;
|
||
}
|
||
|
||
/* Test: on_get_entry_term returns the stamped per-LSN term for present entries,
|
||
* the lowest-known term for a legacy-0 entry, and 0 for an absent LSN
|
||
* (R1.5, R3.1). */
|
||
static void test_on_get_entry_term_present_legacy_absent(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_get_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
atomic_store(&c->ha.role, CTRL_STANDBY);
|
||
|
||
uint8_t blob[64];
|
||
|
||
/* LSN 1: real term 6. */
|
||
size_t n = pack_entry(blob, WAL_JOB_UPDATE, "r", 1);
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
ha_cb_persist_entry_for_test(&c->ha, /*index*/1, /*term*/6, blob, n));
|
||
|
||
/* LSN 2: legacy-0 entry (leader term 0) → reads back as WAL_LEGACY_TERM. */
|
||
n = pack_entry(blob, WAL_JOB_UPDATE, "l", 1);
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
ha_cb_persist_entry_for_test(&c->ha, /*index*/2, /*term*/0, blob, n));
|
||
|
||
/* Present real-term entry returns its term verbatim. */
|
||
TEST_ASSERT_EQUAL_UINT64(6, ha_cb_get_entry_term_for_test(&c->ha, 1));
|
||
/* Present legacy-0 entry maps to the lowest-known real term. */
|
||
TEST_ASSERT_EQUAL_UINT64(LEGACY_LOWEST_TERM,
|
||
ha_cb_get_entry_term_for_test(&c->ha, 2));
|
||
/* Absent LSNs (0 reserved, 3 never appended) return 0. */
|
||
TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 0));
|
||
TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 3));
|
||
|
||
teardown_cluster(c);
|
||
printf("test_on_get_entry_term_present_legacy_absent: PASS\n");
|
||
}
|
||
|
||
/* Test: on_truncate_from(k) deletes local entries at LSN >= k; the prefix < k
|
||
* stays intact and on_get_entry_term returns 0 for the truncated LSNs while the
|
||
* term index shrinks consistently (R1.8, R3.1, R3.2). */
|
||
static void test_on_truncate_from_drops_suffix(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_trunc_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
/* Persist LSN 1 (term 3) and LSN 2 (term 5). */
|
||
cluster_t* c = persist_two_entries(tmp, /*t1*/3, /*t2*/5);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
|
||
/* Both present before truncation. */
|
||
TEST_ASSERT_EQUAL_UINT64(3, ha_cb_get_entry_term_for_test(&c->ha, 1));
|
||
TEST_ASSERT_EQUAL_UINT64(5, ha_cb_get_entry_term_for_test(&c->ha, 2));
|
||
TEST_ASSERT_EQUAL_UINT64(3, c->persist.current_lsn);
|
||
|
||
/* Truncate from LSN 2 → entries at LSN >= 2 are deleted. */
|
||
TEST_ASSERT_EQUAL_INT(0, ha_cb_truncate_from_for_test(&c->ha, 2));
|
||
|
||
/* Prefix < 2 intact, suffix >= 2 gone (term index returns 0). */
|
||
TEST_ASSERT_EQUAL_UINT64(3, ha_cb_get_entry_term_for_test(&c->ha, 1));
|
||
TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 2));
|
||
TEST_ASSERT_EQUAL_UINT64(2, c->persist.current_lsn);
|
||
|
||
teardown_cluster(c);
|
||
printf("test_on_truncate_from_drops_suffix: PASS\n");
|
||
}
|
||
|
||
/* Test: on_truncate_from(1) deletes the whole log (LSN >= 1); every entry's
|
||
* term reads back absent and the tip collapses to LSN 1. */
|
||
static void test_on_truncate_from_removes_all(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_trunc_all_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
cluster_t* c = persist_two_entries(tmp, /*t1*/2, /*t2*/2);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
|
||
TEST_ASSERT_EQUAL_INT(0, ha_cb_truncate_from_for_test(&c->ha, 1));
|
||
|
||
TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 1));
|
||
TEST_ASSERT_EQUAL_UINT64(0, ha_cb_get_entry_term_for_test(&c->ha, 2));
|
||
TEST_ASSERT_EQUAL_UINT64(1, c->persist.current_lsn);
|
||
|
||
teardown_cluster(c);
|
||
printf("test_on_truncate_from_removes_all: PASS\n");
|
||
}
|
||
|
||
/* ================================================================== */
|
||
/* Task 2.3 — on_send_append_entries callback body */
|
||
/* ================================================================== */
|
||
|
||
/* A dummy non-NULL endpoint pointer for a registered standby slot. The send
|
||
* is diverted to the capture hook, so the pointer is never dereferenced. */
|
||
static int g_ae_dummy_ep;
|
||
|
||
/* Register one standby in the cluster's embedded replicator at slot 0, wiring
|
||
* ha.repl so the callback maps peer_idx 0 → this endpoint. */
|
||
static void register_one_standby(cluster_t *c)
|
||
{
|
||
c->ha.repl = &c->persist.repl;
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
repl_add_standby(&c->persist.repl, &g_ae_dummy_ep, "sb0", 7000));
|
||
}
|
||
|
||
/* Test: on_send_append_entries for n_entries==1 builds an envelope carrying the
|
||
* correct (prev_log_index, prev_log_term, leader_commit, n_entries) and, for the
|
||
* single entry sourced from the leader's own WAL at prev_log_index+1, its term
|
||
* (== persist_entry_term(prev_log_index+1)) and payload (R1.3, R3.1). */
|
||
static void test_send_append_entries_one_entry_from_wal(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_ae1_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
cluster_t *c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
atomic_store(&c->ha.role, CTRL_PRIMARY);
|
||
|
||
/* Populate the leader's own WAL BEFORE registering a standby — otherwise the
|
||
* persist append fans out via repl_broadcast to the dummy int ep (ASAN
|
||
* global-buffer-overflow reading ep->peer). With no standby, the append is a
|
||
* no-op broadcast. The entry to be sent is at prev_log_index+1 == LSN 2. */
|
||
uint8_t blob[64];
|
||
size_t n1 = pack_entry(blob, WAL_JOB_UPDATE, "p", 1);
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
ha_cb_persist_entry_for_test(&c->ha, /*index*/1, /*term*/4, blob, n1));
|
||
size_t n2 = pack_entry(blob, WAL_JOB_UPDATE, "QQ", 2);
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
ha_cb_persist_entry_for_test(&c->ha, /*index*/2, /*term*/6, blob, n2));
|
||
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
|
||
|
||
/* Now register the standby so peer_idx 0 maps to a slot; the AE send below
|
||
* is diverted to the capture hook, so the dummy ep is never dereferenced. */
|
||
register_one_standby(c);
|
||
|
||
/* The entry-term oracle the send must carry for LSN 2. */
|
||
uint64_t expect_term = persist_entry_term(&c->persist, 2);
|
||
TEST_ASSERT_EQUAL_UINT64(6, expect_term);
|
||
|
||
/* Drive the callback: prev_log_index=1, prev_log_term=4 (term of LSN 1),
|
||
* leader_commit=1, n_entries=1 → sends the entry at LSN 2. */
|
||
repl_ae_capture_t cap;
|
||
memset(&cap, 0, sizeof(cap));
|
||
repl_send_ae_capture_hook = ∩
|
||
|
||
int rc = ha_cb_send_append_entries_for_test(&c->ha, /*peer*/0,
|
||
/*prev_idx*/1, /*prev_term*/4,
|
||
/*leader_commit*/1, /*n*/1);
|
||
repl_send_ae_capture_hook = NULL;
|
||
|
||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||
TEST_ASSERT_EQUAL_INT(1, cap.called);
|
||
TEST_ASSERT_EQUAL_INT(0, cap.peer_idx);
|
||
TEST_ASSERT_EQUAL_UINT64(1, cap.prev_log_index);
|
||
TEST_ASSERT_EQUAL_UINT64(4, cap.prev_log_term);
|
||
TEST_ASSERT_EQUAL_UINT64(1, cap.leader_commit);
|
||
TEST_ASSERT_EQUAL_INT(1, cap.n_entries);
|
||
|
||
/* The single entry is sourced from the local WAL at prev_log_index+1 == 2. */
|
||
TEST_ASSERT_EQUAL_UINT64(2, cap.entry_lsn);
|
||
TEST_ASSERT_EQUAL_UINT64(expect_term, cap.entry_term);
|
||
TEST_ASSERT_EQUAL_UINT8(WAL_JOB_UPDATE, cap.entry_type);
|
||
TEST_ASSERT_EQUAL_UINT32(2, cap.payload_len);
|
||
TEST_ASSERT_EQUAL_MEMORY("QQ", cap.payload, 2);
|
||
|
||
teardown_cluster(c);
|
||
printf("test_send_append_entries_one_entry_from_wal: PASS\n");
|
||
}
|
||
|
||
/* Test: on_send_append_entries for n_entries==0 emits a heartbeat carrying the
|
||
* prev tuple + leader_commit and ZERO entries (R1.3). */
|
||
static void test_send_append_entries_heartbeat_zero_entries(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_ae0_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
cluster_t *c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
atomic_store(&c->ha.role, CTRL_PRIMARY);
|
||
|
||
/* One entry on disk so prev_log_index=1 is a real tip; heartbeat sends none.
|
||
* Persist BEFORE registering the standby (see one-entry test note). */
|
||
uint8_t blob[64];
|
||
size_t n1 = pack_entry(blob, WAL_JOB_UPDATE, "h", 1);
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
ha_cb_persist_entry_for_test(&c->ha, /*index*/1, /*term*/3, blob, n1));
|
||
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
|
||
|
||
register_one_standby(c);
|
||
|
||
repl_ae_capture_t cap;
|
||
memset(&cap, 0, sizeof(cap));
|
||
repl_send_ae_capture_hook = ∩
|
||
|
||
int rc = ha_cb_send_append_entries_for_test(&c->ha, /*peer*/0,
|
||
/*prev_idx*/1, /*prev_term*/3,
|
||
/*leader_commit*/1, /*n*/0);
|
||
repl_send_ae_capture_hook = NULL;
|
||
|
||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||
TEST_ASSERT_EQUAL_INT(1, cap.called);
|
||
TEST_ASSERT_EQUAL_UINT64(1, cap.prev_log_index);
|
||
TEST_ASSERT_EQUAL_UINT64(3, cap.prev_log_term);
|
||
TEST_ASSERT_EQUAL_UINT64(1, cap.leader_commit);
|
||
TEST_ASSERT_EQUAL_INT(0, cap.n_entries);
|
||
/* No entry sourced for a heartbeat. */
|
||
TEST_ASSERT_EQUAL_UINT32(0, cap.payload_len);
|
||
|
||
teardown_cluster(c);
|
||
printf("test_send_append_entries_heartbeat_zero_entries: PASS\n");
|
||
}
|
||
|
||
/* ================================================================== */
|
||
/* Task 2.4 — on_commit_advanced trigger + commit-source switch */
|
||
/* ================================================================== */
|
||
|
||
/*
|
||
* Task 2.4 feeds the library-supplied current-term-gated `commit_index` into
|
||
* the existing synchronous commit barrier as the commit SOURCE. The barrier
|
||
* mechanism is unchanged: committed_lsn still advances only when self+majority
|
||
* agree AND the bytes are durable locally (durable_lsn cap). The library commit
|
||
* index becomes an ADDITIONAL ceiling so the barrier never advances committed
|
||
* past the library's current-term-safe point. on_commit_advanced only TRIGGERS
|
||
* (records the ceiling + re-evaluates); it never performs durability itself.
|
||
*
|
||
* These cases build the commit engine standalone over a stub persist_t +
|
||
* replicator (the test_commit_engine.c pattern) so the barrier interaction is
|
||
* tested in isolation, plus one case driving the WIRED on_commit_advanced
|
||
* callback end-to-end through ha_cb_commit_advanced_for_test.
|
||
*
|
||
* RED: against the no-op ha_cb_commit_advanced_stub (and the missing
|
||
* commit_set_raft_commit_index ceiling), the library commit index is NOT a
|
||
* commit source — committed_lsn would advance on durable-majority alone with
|
||
* no current-term gate. GREEN: the wired callback feeds the ceiling so the
|
||
* barrier holds until BOTH durable majority and the library commit index agree.
|
||
*/
|
||
|
||
/* Stub-persist fixture for the commit-source tests (mirrors test_commit_engine.c). */
|
||
static persist_t g_t24_p;
|
||
static commit_engine_t g_t24_ce;
|
||
static int g_t24_eps[MAX_STANDBYS];
|
||
|
||
static void t24_engine_setup(int n_peers, int n_standbys, uint64_t appended_lsn)
|
||
{
|
||
char name[16];
|
||
memset(&g_t24_p, 0, sizeof(g_t24_p));
|
||
TEST_ASSERT_EQUAL_INT(0, repl_init(&g_t24_p.repl));
|
||
for (int i = 0; i < n_standbys; i++) {
|
||
snprintf(name, sizeof(name), "sb%d", i);
|
||
TEST_ASSERT_EQUAL_INT(
|
||
i, repl_add_standby(&g_t24_p.repl, &g_t24_eps[i], name,
|
||
7000 + (uint32_t)i));
|
||
}
|
||
g_t24_p.current_lsn = appended_lsn + 1;
|
||
atomic_store(&g_t24_p.committed_lsn, 0);
|
||
atomic_store(&g_t24_p.durable_lsn, appended_lsn);
|
||
TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&g_t24_ce, &g_t24_p, n_peers, 8));
|
||
}
|
||
|
||
static void t24_engine_teardown(void)
|
||
{
|
||
commit_engine_fini(&g_t24_ce);
|
||
repl_fini(&g_t24_p.repl);
|
||
}
|
||
|
||
static void t24_standby_ack(int idx, uint64_t acked_lsn)
|
||
{
|
||
atomic_store(&g_t24_p.repl.standbys[idx].acked_lsn, acked_lsn);
|
||
commit_on_ack(&g_t24_ce, idx, acked_lsn);
|
||
}
|
||
|
||
static uint64_t t24_committed(void)
|
||
{
|
||
return atomic_load(&g_t24_p.committed_lsn);
|
||
}
|
||
|
||
/* commit-ceiling-optA — REFRAMED from test_commit_advanced_is_commit_source.
|
||
*
|
||
* The durable-majority barrier is AUTHORITATIVE for commit-advance; the recorded
|
||
* library commit_index is inert-advisory and does NOT cap committed_lsn. A
|
||
* durable majority covering LSN 3 commits to 3 EVEN WHILE the recorded library
|
||
* commit_index lags at 1 — this is the starvation the old ceiling caused (commit
|
||
* pinned at 1 below the safe durable majority) and the Option A fix removes.
|
||
*
|
||
* Meaningful both ways: it FAILS if commit were wrongly gated by the recorded
|
||
* index (committed would stick at 1), AND it FAILS if commit over-advanced past
|
||
* the durable majority (it asserts == 3, the exact durable watermark). */
|
||
static void test_commit_advances_to_durable_majority_index_inert(void)
|
||
{
|
||
/* 3 peers = self + 2 standbys; majority needs self + 1 standby. */
|
||
t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/3);
|
||
|
||
/* Record an advisory library commit_index that LAGS at 1 (e.g. match_index[]
|
||
* not yet advanced via catch-up). Under the old design this pinned committed
|
||
* at 1; under Option A it is inert. */
|
||
commit_set_raft_commit_index(&g_t24_ce, 1);
|
||
|
||
/* A standby ACKs LSN 3; self is durable to LSN 3 → durable majority = 3.
|
||
* committed advances to the durable majority (3), NOT pinned at the lagging
|
||
* advisory index (1). */
|
||
t24_standby_ack(0, 3);
|
||
TEST_ASSERT_EQUAL_UINT64(3, t24_committed());
|
||
|
||
/* Advancing the advisory record changes nothing — already at the durable
|
||
* majority; record is inert. */
|
||
commit_set_raft_commit_index(&g_t24_ce, 3);
|
||
TEST_ASSERT_EQUAL_UINT64(3, t24_committed());
|
||
|
||
t24_engine_teardown();
|
||
printf("test_commit_advances_to_durable_majority_index_inert: PASS\n");
|
||
}
|
||
|
||
/* commit-ceiling-optA — REFRAMED from test_commit_advanced_ceiling_caps_durable_majority.
|
||
*
|
||
* The recorded library commit_index is NOT a ceiling: committed advances to the
|
||
* full durable majority even when the advisory index sits below it. Pins the
|
||
* removal of the redundant ceiling.
|
||
*
|
||
* Meaningful both ways: FAILS if the advisory index still capped committed at 3
|
||
* (the old, now-rejected behavior), AND FAILS if committed over-advanced past
|
||
* the durable majority of 5 (asserts == 5 exactly). */
|
||
static void test_commit_index_record_is_not_a_ceiling(void)
|
||
{
|
||
t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/5);
|
||
|
||
/* Advisory record lags at 3 while the durable majority will reach 5. */
|
||
commit_set_raft_commit_index(&g_t24_ce, 3);
|
||
|
||
/* Durable majority covers up to LSN 5 (self durable 5 + standby acked 5).
|
||
* committed advances to 5 — the lagging advisory index (3) does NOT cap it. */
|
||
t24_standby_ack(0, 5);
|
||
TEST_ASSERT_EQUAL_UINT64(5, t24_committed());
|
||
|
||
/* Catching the advisory record up to 5 is a no-op (already there). */
|
||
commit_set_raft_commit_index(&g_t24_ce, 5);
|
||
TEST_ASSERT_EQUAL_UINT64(5, t24_committed());
|
||
|
||
t24_engine_teardown();
|
||
printf("test_commit_index_record_is_not_a_ceiling: PASS\n");
|
||
}
|
||
|
||
/* commit-ceiling-optA — RETAINED (unchanged contract): committed never advances
|
||
* past what is durable locally. The durable self-cap (persist_durable_lsn) is
|
||
* still authoritative regardless of any advisory library index. With the
|
||
* advisory index well ahead (4) but self durable only to LSN 2 and the standby
|
||
* acked LSN 2, committed stays at the durable-majority floor (2). This invariant
|
||
* is unchanged by Option A — the durable cap was always orthogonal to the
|
||
* removed ceiling — but is re-pinned here to guard Commit Durability (R5.4). */
|
||
static void test_commit_advanced_does_not_bypass_durable_cap(void)
|
||
{
|
||
t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/4);
|
||
atomic_store(&g_t24_p.durable_lsn, 2); /* self durable only to 2 */
|
||
|
||
t24_standby_ack(0, 2); /* standby agrees to 2 */
|
||
commit_set_raft_commit_index(&g_t24_ce, 4); /* advisory index well ahead */
|
||
commit_on_ack(&g_t24_ce, 0, 2);
|
||
|
||
/* Durable-majority floor is 2; nothing (advisory index 4 included) raises it. */
|
||
TEST_ASSERT_EQUAL_UINT64(2, t24_committed());
|
||
|
||
t24_engine_teardown();
|
||
printf("test_commit_advanced_does_not_bypass_durable_cap: PASS\n");
|
||
}
|
||
|
||
/* commit-ceiling-optA — NEW load-bearing safety test: the own-term writable gate
|
||
* (Figure-8). With Option A removing the library-commit-index ceiling, the
|
||
* own-term NOOP gate (commit_set_writable_lsn / wait_done_locked) is now the
|
||
* load-bearing realization of "no acked-committed prior-term entry until the
|
||
* current-term NOOP is durable on a majority".
|
||
*
|
||
* Scenario: self holds a PRIOR-term entry at LSN 1 and the current-term NOOP at
|
||
* LSN 2 (the promotion NOOP). The own-term gate is armed at LSN 2. A durable
|
||
* majority reaches LSN 1 FIRST (the NOOP not yet majority-durable):
|
||
* - committed_lsn MAY advance to 1 (the barrier is contiguous-prefix), but a
|
||
* WAIT for LSN 1 must NOT complete — the gate blocks acknowledgement until
|
||
* committed_lsn >= writable_lsn (2). This is the safety property: the
|
||
* prior-term entry at LSN 1 is NOT acknowledged-committed yet.
|
||
* Then the NOOP becomes majority-durable (committed reaches 2):
|
||
* - the gate opens and the WAIT completes.
|
||
*
|
||
* RED if the writable gate were bypassed: a WAIT(lsn=1) would complete as soon
|
||
* as committed_lsn >= 1, acknowledging a prior-term entry before the own-term
|
||
* NOOP is majority-durable — the Figure-8 violation. GREEN: it blocks (times
|
||
* out) until the NOOP commits, then completes. */
|
||
static void test_own_term_gate_blocks_priorterm_ack_until_noop_durable(void)
|
||
{
|
||
/* 3 peers; majority needs self + 1 standby. Appended tip = LSN 2 (prior-term
|
||
* entry at 1 + current-term NOOP at 2). */
|
||
t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/2);
|
||
|
||
/* Arm the own-term gate at the NOOP's LSN (2): no acknowledgement of any
|
||
* entry until committed_lsn >= 2 (the current-term NOOP is majority-durable). */
|
||
commit_set_writable_lsn(&g_t24_ce, 2);
|
||
|
||
/* Durable majority reaches LSN 1 only: self durable to 2, but the standby has
|
||
* acked just LSN 1, so the median (self + 1 of 2 peers) covers LSN 1. */
|
||
atomic_store(&g_t24_p.durable_lsn, 2);
|
||
t24_standby_ack(0, 1);
|
||
|
||
/* The contiguous-prefix barrier MAY advance committed_lsn to 1 ... */
|
||
TEST_ASSERT_EQUAL_UINT64(1, t24_committed());
|
||
|
||
/* ... but a WAIT for LSN 1 must NOT complete: the own-term gate (writable_lsn
|
||
* == 2) holds it. Pre-NOOP-durable, this is the Figure-8 safety block — the
|
||
* prior-term entry at LSN 1 is committed-but-not-acknowledged. A short
|
||
* timeout proves the gate blocks (returns -ETIMEDOUT, not 0). */
|
||
TEST_ASSERT_EQUAL_INT(-ETIMEDOUT,
|
||
commit_submit(&g_t24_ce, /*lsn*/1, COMMIT_MODE_WAIT, /*timeout_ms*/150));
|
||
|
||
/* The current-term NOOP at LSN 2 now becomes majority-durable (standby acks
|
||
* LSN 2). committed_lsn advances to 2 and the gate opens. */
|
||
t24_standby_ack(0, 2);
|
||
TEST_ASSERT_EQUAL_UINT64(2, t24_committed());
|
||
|
||
/* The WAIT for LSN 1 now completes: committed_lsn (2) >= lsn (1) AND
|
||
* >= writable_lsn (2). The prior-term entry is acknowledged only AFTER the
|
||
* own-term NOOP is majority-durable. */
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
commit_submit(&g_t24_ce, /*lsn*/1, COMMIT_MODE_WAIT, /*timeout_ms*/500));
|
||
|
||
t24_engine_teardown();
|
||
printf("test_own_term_gate_blocks_priorterm_ack_until_noop_durable: PASS\n");
|
||
}
|
||
|
||
/* Test (pins the own-term-gate DECISION — RETAINED, complementary, no deadlock):
|
||
* the own-term writable_lsn gate is a WAIT-completion predicate, NOT a cap on
|
||
* committed_lsn. With the gate armed at LSN 2 and the durable majority at 2,
|
||
* committed advances to 2 AND a WAIT(lsn=2) completes — the gate never holds
|
||
* committed_lsn back, and once committed reaches the gate the WAIT completes (no
|
||
* deadlock). The advisory library index is irrelevant here (inert under optA). */
|
||
static void test_own_term_gate_retained_no_deadlock(void)
|
||
{
|
||
t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/2);
|
||
|
||
/* Arm the own-term gate at LSN 2 (the load-bearing complementary gate). */
|
||
commit_set_writable_lsn(&g_t24_ce, 2);
|
||
|
||
/* Durable majority reaches LSN 2. */
|
||
t24_standby_ack(0, 2);
|
||
commit_set_raft_commit_index(&g_t24_ce, 2); /* advisory; inert */
|
||
commit_on_ack(&g_t24_ce, 0, 2);
|
||
|
||
/* committed_lsn advanced to 2 (own-term gate never caps committed_lsn). */
|
||
TEST_ASSERT_EQUAL_UINT64(2, t24_committed());
|
||
|
||
/* A WAIT for LSN 2 completes immediately: committed_lsn (2) >= lsn (2) AND
|
||
* >= writable_lsn (2). No deadlock between the gate and the barrier. */
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
commit_submit(&g_t24_ce, /*lsn*/2, COMMIT_MODE_WAIT, /*timeout_ms*/500));
|
||
|
||
t24_engine_teardown();
|
||
printf("test_own_term_gate_retained_no_deadlock: PASS\n");
|
||
}
|
||
|
||
/* commit-ceiling-optA — REFRAMED from test_on_commit_advanced_wired_feeds_barrier.
|
||
*
|
||
* The WIRED on_commit_advanced callback (ha_cb_commit_advanced_for_test) routes
|
||
* the library commit_index into the cluster's commit engine as an INERT-ADVISORY
|
||
* record — it records ce->raft_commit_index and wakes the barrier, but does NOT
|
||
* gate committed_lsn. The durable-majority barrier is authoritative: with a
|
||
* durable majority covering LSN 2, committed advances to 2 even though the wired
|
||
* callback last fed a LAGGING commit_index of 1.
|
||
*
|
||
* Meaningful both ways: FAILS if the wired callback still capped committed at 1
|
||
* (the old, now-rejected ceiling — the starvation), AND FAILS if committed
|
||
* over-advanced past the durable majority of 2 (asserts == 2). Also pins that
|
||
* the wired callback still RECORDS the advisory index (sink intact). */
|
||
static void test_on_commit_advanced_wired_records_advisory_index(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_commitadv_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
cluster_t *c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
|
||
/* Init the cluster's embedded commit engine over the same persist as a
|
||
* 3-peer cluster (self + 2 standbys), so a majority is self + 1 standby. */
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
commit_engine_init(&c->commit, &c->persist, /*n_peers*/3, /*n_vcpus*/8));
|
||
|
||
/* Persist two entries BEFORE registering a standby — otherwise the primary
|
||
* own-append fans out via repl_broadcast to the dummy int ep (ASAN
|
||
* global-buffer-overflow reading ep->peer); with no standby the broadcast
|
||
* is a no-op. Append LSN 1,2 with term 4 → durable floor reaches LSN 2. */
|
||
atomic_store(&c->ha.role, CTRL_PRIMARY);
|
||
uint8_t blob[64];
|
||
size_t n1 = pack_entry(blob, WAL_JOB_UPDATE, "a", 1);
|
||
TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 4, blob, n1));
|
||
size_t n2 = pack_entry(blob, WAL_JOB_UPDATE, "b", 1);
|
||
TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 2, 4, blob, n2));
|
||
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
|
||
|
||
/* Now register the standby so a majority (self + 1) is reachable. The ACK
|
||
* below is driven directly into the registry + commit_on_ack (no network). */
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
repl_add_standby(&c->persist.repl, &g_ae_dummy_ep, "sb0", 7000));
|
||
|
||
/* Fire the WIRED callback with a LAGGING commit_index of 1 (e.g. match_index[]
|
||
* not yet caught up). Under optA this only RECORDS the advisory index. */
|
||
ha_cb_commit_advanced_for_test(&c->ha, 1);
|
||
TEST_ASSERT_EQUAL_UINT64(1, c->commit.raft_commit_index); /* sink intact */
|
||
|
||
/* Standby ACKs LSN 2 → durable majority covers LSN 2. committed advances to
|
||
* the durable majority (2), NOT pinned at the lagging advisory index (1):
|
||
* the advisory record does not gate commit. */
|
||
atomic_store(&c->persist.repl.standbys[0].acked_lsn, 2);
|
||
commit_on_ack(&c->commit, 0, 2);
|
||
TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn));
|
||
|
||
/* A later callback advancing the advisory record to 2 is a no-op for commit. */
|
||
ha_cb_commit_advanced_for_test(&c->ha, 2);
|
||
TEST_ASSERT_EQUAL_UINT64(2, c->commit.raft_commit_index);
|
||
TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn));
|
||
|
||
commit_engine_fini(&c->commit);
|
||
teardown_cluster(c);
|
||
printf("test_on_commit_advanced_wired_records_advisory_index: PASS\n");
|
||
}
|
||
|
||
/* ================================================================== */
|
||
/* Task 2.5 — library init restore from the recovered WAL floor */
|
||
/* ================================================================== */
|
||
|
||
/* Test: ha_init_raft restores the library log/commit floor from the
|
||
* recovered WAL instead of booting at LSN 0 (R1.5, R2.1).
|
||
*
|
||
* After a member recovers a WAL whose durable tip is LSN 3 (term 7) with a
|
||
* committed prefix at LSN 2, raft_init must boot with:
|
||
* raft.log_len == 3 (initial_log_len = persist_durable_floor)
|
||
* raft.commit_index == 2 (initial_commit_index = recovered committed_lsn)
|
||
* and the last-log term must be recoverable via the wired on_get_entry_term
|
||
* (== 7). Booting from an empty log (log_len/commit_index == 0) is the pre-fix
|
||
* behaviour this test fails on. */
|
||
static void test_init_restores_recovered_floor(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_initrestore_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
const uint64_t T = 7;
|
||
|
||
/* Phase 1: lay down a 3-entry WAL (term 7) and flush so the recovered tip
|
||
* reaches LSN 3. */
|
||
{
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
atomic_store(&c->ha.role, CTRL_PRIMARY);
|
||
|
||
uint8_t blob[64];
|
||
for (uint64_t i = 1; i <= 3; i++) {
|
||
size_t n = pack_entry(blob, WAL_JOB_UPDATE, "x", 1);
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
ha_cb_persist_entry_for_test(&c->ha, i, T, blob, n));
|
||
}
|
||
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
|
||
teardown_cluster(c);
|
||
}
|
||
|
||
/* Phase 2: recover into a fresh cluster; durable floor == 3. Simulate a
|
||
* recovered committed prefix at LSN 2 (recovery re-derives the durable tip;
|
||
* the committed position is set by the surrounding boot path). */
|
||
{
|
||
cluster_t* c2 = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c2);
|
||
TEST_ASSERT_EQUAL_INT(0, persist_recover(&c2->persist));
|
||
|
||
TEST_ASSERT_EQUAL_UINT64(3, persist_durable_floor(&c2->persist));
|
||
atomic_store(&c2->persist.committed_lsn, 2ULL);
|
||
|
||
/* Initialize the embedded raft as a single-node cluster. The restore
|
||
* must source the floor from the recovered persist, not LSN 0. */
|
||
ha_init_raft(&c2->ha, c2, /*n_peers*/1);
|
||
|
||
TEST_ASSERT_EQUAL_UINT64(3, c2->ha.raft.log_len);
|
||
TEST_ASSERT_EQUAL_UINT64(2, c2->ha.raft.commit_index);
|
||
|
||
/* The recovered last-log term is surfaced via the wired callback. */
|
||
TEST_ASSERT_EQUAL_UINT64(T,
|
||
ha_cb_get_entry_term_for_test(&c2->ha, c2->ha.raft.log_len));
|
||
|
||
/* All five term-safety callbacks remain wired (non-NULL). */
|
||
TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs);
|
||
TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_persist_entry);
|
||
TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_get_entry_term);
|
||
TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_send_append_entries);
|
||
TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_truncate_from);
|
||
TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_commit_advanced);
|
||
/* Existing election callbacks remain wired. */
|
||
TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_become_leader);
|
||
TEST_ASSERT_NOT_NULL(c2->ha.raft.cbs->on_persist);
|
||
|
||
teardown_cluster(c2);
|
||
}
|
||
|
||
printf("test_init_restores_recovered_floor: PASS\n");
|
||
}
|
||
|
||
/* Test: an empty-WAL first boot restores a zero floor (no spurious non-zero
|
||
* indices) — the recovery-read-failure / empty-log edge of R1.5. */
|
||
static void test_init_empty_wal_zero_floor(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_initempty_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
TEST_ASSERT_EQUAL_INT(0, persist_recover(&c->persist));
|
||
TEST_ASSERT_EQUAL_UINT64(0, persist_durable_floor(&c->persist));
|
||
|
||
ha_init_raft(&c->ha, c, /*n_peers*/1);
|
||
|
||
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.log_len);
|
||
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.commit_index);
|
||
|
||
teardown_cluster(c);
|
||
printf("test_init_empty_wal_zero_floor: PASS\n");
|
||
}
|
||
|
||
/* ================================================================== */
|
||
/* Task 3.1 — drive raft_log_append on the primary append path */
|
||
/* ================================================================== */
|
||
|
||
/* Bring a make_cluster()'d node up as the raft LEADER so raft_log_append is
|
||
* enabled (it returns -EPERM off the leader). We init the library from the
|
||
* recovered floor (task 2.5) and set the live election term + raft role
|
||
* directly — modelling a node that has already won an election at `term`
|
||
* (the cerebrum raft-sim convention: set state.term after a clean election;
|
||
* raft_log_append stamps entries from s->term). */
|
||
static void make_leader(cluster_t* c, uint64_t term, int n_peers)
|
||
{
|
||
atomic_store(&c->ha.role, CTRL_PRIMARY);
|
||
atomic_store(&c->ha.term, term);
|
||
ha_init_raft(&c->ha, c, n_peers);
|
||
c->ha.raft.term = term;
|
||
c->ha.raft.role = RAFT_LEADER;
|
||
}
|
||
|
||
/* Test: a primary job-state append driven through ha_primary_log_append goes
|
||
* through the library — the library assigns the LSN (== s->log_len post-append),
|
||
* stamps the live election term into the WAL header, and keeps the persist
|
||
* write cursor co-tracked with the library index (current_lsn == log_len + 1).
|
||
* Against the pre-3.1 hand-rolled path this entry point does not exist
|
||
* (compile RED) — and once it exists the library MUST be the LSN/term source
|
||
* (R1.1, R1.2, R2.1). */
|
||
static void test_primary_log_append_stamps_term_and_assigns_lsn(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_plap_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
const uint64_t T = 7;
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
make_leader(c, T, /*n_peers*/1);
|
||
|
||
/* Fresh leader, empty log: first append takes library index 1. */
|
||
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.log_len);
|
||
|
||
wal_jobstate_t js = { .jobid = 42, .state = 3 };
|
||
uint64_t lsn = 0;
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &lsn));
|
||
|
||
/* (a) The LIBRARY assigned the LSN and grew its log to it. */
|
||
TEST_ASSERT_EQUAL_UINT64(1, lsn);
|
||
TEST_ASSERT_EQUAL_UINT64(1, c->ha.raft.log_len);
|
||
|
||
/* (b) The WAL header recorded the live election term for that LSN. */
|
||
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1));
|
||
|
||
/* (d) Co-tracking invariant: the persist write cursor advanced exactly in
|
||
* lockstep with the library index — current_lsn == log_len + 1 (next LSN). */
|
||
TEST_ASSERT_EQUAL_UINT64(c->ha.raft.log_len + 1, c->persist.current_lsn);
|
||
|
||
/* A second append keeps the two counters co-tracked. */
|
||
js.jobid = 43;
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &lsn));
|
||
TEST_ASSERT_EQUAL_UINT64(2, lsn);
|
||
TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.log_len);
|
||
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 2));
|
||
TEST_ASSERT_EQUAL_UINT64(c->ha.raft.log_len + 1, c->persist.current_lsn);
|
||
|
||
teardown_cluster(c);
|
||
printf("test_primary_log_append_stamps_term_and_assigns_lsn: PASS\n");
|
||
}
|
||
|
||
/* Test: the own-term promotion no-op (ha_arm_own_term_gate) is routed through
|
||
* the library, so the WAL_NOOP entry records the newly established term (not 0)
|
||
* and the library index tracks it (R1.2). Pre-3.1 the noop went through a
|
||
* hand-rolled persist_wal_async with a best-effort p->term and did NOT advance
|
||
* s->log_len. */
|
||
static void test_own_term_noop_records_live_term_via_library(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_noop_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
const uint64_t T = 5;
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
make_leader(c, T, /*n_peers*/1);
|
||
|
||
/* Register the persist hooks then arm the own-term gate exactly as the
|
||
* promotion tail does (ha_promote_to_primary / raft_cb_become_leader). */
|
||
ha_arm_own_term_gate_for_test(c);
|
||
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
|
||
|
||
/* The no-op landed at library index 1 with the live term recorded. */
|
||
TEST_ASSERT_EQUAL_UINT64(1, c->ha.raft.log_len);
|
||
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1));
|
||
TEST_ASSERT_EQUAL_UINT64(c->ha.raft.log_len + 1, c->persist.current_lsn);
|
||
|
||
teardown_cluster(c);
|
||
printf("test_own_term_noop_records_live_term_via_library: PASS\n");
|
||
}
|
||
|
||
/* Test: when raft leadership is NOT active (the library is not RAFT_LEADER —
|
||
* the init-time non-raft promotion road and engine-less test contexts),
|
||
* ha_primary_log_append still appends locally through the legacy path so the
|
||
* non-raft promotion tail keeps working, WITHOUT growing the library log (the
|
||
* library is not tracking the log on that road, so no co-tracking hazard).
|
||
* The entry still records the live election term via persist_set_term. */
|
||
static void test_primary_log_append_non_leader_legacy_fallback(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_plapfb_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
const uint64_t T = 6;
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
/* Non-raft promotion road: controller role PRIMARY + live term, but the
|
||
* raft library was never driven to LEADER (raft.role stays FOLLOWER). */
|
||
atomic_store(&c->ha.role, CTRL_PRIMARY);
|
||
atomic_store(&c->ha.term, T);
|
||
ha_init_raft(&c->ha, c, /*n_peers*/1);
|
||
TEST_ASSERT_NOT_EQUAL_INT(RAFT_LEADER, c->ha.raft.role);
|
||
|
||
wal_jobstate_t js = { .jobid = 7, .state = 1 };
|
||
uint64_t lsn = 0;
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &lsn));
|
||
|
||
/* Appended at the contiguous next LSN (current_lsn started at 1). */
|
||
TEST_ASSERT_EQUAL_UINT64(1, lsn);
|
||
TEST_ASSERT_EQUAL_UINT64(2, c->persist.current_lsn);
|
||
/* Live term still recorded for the entry. */
|
||
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1));
|
||
/* The library log was NOT grown by the legacy fallback (no raft leadership). */
|
||
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.log_len);
|
||
|
||
teardown_cluster(c);
|
||
printf("test_primary_log_append_non_leader_legacy_fallback: PASS\n");
|
||
}
|
||
|
||
/* ================================================================== */
|
||
/* Task 3.2 — route heartbeat/replication intake through the library */
|
||
/* raft_on_append_entries_request: UpdateTerm (G5) + logOk (G2) + */
|
||
/* (term,lsn) conflict-truncate (G3) via the wired callbacks. */
|
||
/* */
|
||
/* These drive the production intake helper ha_consume_append_entries */
|
||
/* (the in-process route that replaces the term-blind handling in */
|
||
/* ha_consume_heartbeat_committed + the committed-boundary truncate in */
|
||
/* ha_rejoin_reconcile). RED: that helper does not exist pre-3.2 */
|
||
/* (compile RED); once it exists the three observable behaviors hold. */
|
||
/* ================================================================== */
|
||
|
||
/* Make a node a STANDBY follower at a given term + raft log floor, with the
|
||
* raft library initialized so the intake can drive it. Seeds the library log
|
||
* to match the persisted WAL tip (the ha_init_raft floor restore). */
|
||
static void make_follower(cluster_t* c, uint64_t term, int n_peers)
|
||
{
|
||
atomic_store(&c->ha.role, CTRL_STANDBY);
|
||
atomic_store(&c->ha.term, term);
|
||
ha_init_raft(&c->ha, c, n_peers);
|
||
c->ha.raft.term = term;
|
||
c->ha.raft.role = RAFT_FOLLOWER;
|
||
}
|
||
|
||
/* Build a decoded single-entry WalCatchupEntry (n_entries==0 path == one entry
|
||
* carried in the top-level container, matching the on-wire single-entry mode).
|
||
* For a heartbeat use n_entries set so the helper sees zero entries. */
|
||
static void fill_ae_entry(wal_catchup_entry_t* e, int is_heartbeat,
|
||
uint64_t prev_log_index, uint64_t prev_log_term,
|
||
uint64_t leader_commit, uint64_t entry_lsn,
|
||
uint8_t entry_type, uint64_t entry_term,
|
||
uint8_t* payload, uint32_t plen)
|
||
{
|
||
memset(e, 0, sizeof(*e));
|
||
e->prev_log_index = prev_log_index;
|
||
e->prev_log_term = prev_log_term;
|
||
e->leader_commit = leader_commit;
|
||
if (is_heartbeat) {
|
||
e->n_entries = 0;
|
||
e->lsn = 0;
|
||
} else {
|
||
/* single-entry mode: top-level container carries the entry, n_entries==1
|
||
* with no nested sub-entries (one-entry AppendEntries). */
|
||
e->n_entries = 1;
|
||
e->lsn = entry_lsn;
|
||
e->entry_type = entry_type;
|
||
e->term = entry_term;
|
||
e->raw_payload = payload;
|
||
e->raw_payload_len = plen;
|
||
}
|
||
}
|
||
|
||
/* (a) A higher-term heartbeat (0-entry AppendEntries) routed through the library
|
||
* makes the follower ADOPT the higher term and step down — term recorded
|
||
* (R2.6 / G5). Pre-3.2 ha_consume_heartbeat_committed only DROPPED stale
|
||
* heartbeats and never adopted a higher term. */
|
||
static void test_intake_higher_term_heartbeat_adopts_term(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_intake_ht_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
make_follower(c, /*term*/3, /*n_peers*/3);
|
||
|
||
/* The follower briefly believes itself a candidate/leader at term 3. */
|
||
c->ha.raft.role = RAFT_LEADER;
|
||
|
||
/* Heartbeat from a leader at term 7, empty log (prev=0), commit 0. The
|
||
* heartbeat's top-level container carries the leader's live term. */
|
||
wal_catchup_entry_t hb;
|
||
fill_ae_entry(&hb, /*heartbeat*/1, /*prev_idx*/0, /*prev_term*/0,
|
||
/*leader_commit*/0, 0, 0, 0, NULL, 0);
|
||
hb.term = 7;
|
||
|
||
int rc = ha_consume_append_entries(c, &hb, /*leader_idx*/1);
|
||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||
|
||
/* The library adopted the higher term and stepped down to follower. */
|
||
TEST_ASSERT_EQUAL_UINT64(7, c->ha.raft.term);
|
||
TEST_ASSERT_NOT_EQUAL_INT(RAFT_LEADER, c->ha.raft.role);
|
||
|
||
teardown_cluster(c);
|
||
printf("test_intake_higher_term_heartbeat_adopts_term: PASS\n");
|
||
}
|
||
|
||
/* (b) A divergent same-LSN tail is truncated and the leader's entry accepted
|
||
* (R3.1/R3.2 / G3), and the committed prefix below the divergence is untouched
|
||
* (R3.3). The follower locally holds [LSN1@term2, LSN2@term2]; the leader sends
|
||
* an AppendEntries appending LSN2@term5 on top of LSN1@term2. The library
|
||
* detects the (term,lsn) conflict at LSN2, truncates LSN>=2, and re-persists
|
||
* the leader's LSN2@term5. LSN1 (the agreed prefix) stays intact. */
|
||
static void test_intake_divergent_tail_truncated_and_reconverged(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_intake_div_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
make_follower(c, /*term*/2, /*n_peers*/3);
|
||
|
||
/* Seed the local log: LSN1@term2, LSN2@term2 (the divergent tail). */
|
||
uint8_t blob[64];
|
||
size_t n = pack_entry(blob, WAL_JOB_UPDATE, "a", 1);
|
||
TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 2, blob, n));
|
||
TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 2, 2, blob, n));
|
||
c->ha.raft.log_len = 2;
|
||
TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 1));
|
||
TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 2));
|
||
|
||
/* Leader (term 5) sends AppendEntries: prev=(LSN1,term2), entry LSN2@term5.
|
||
* logOk holds at LSN1 (term2==term2); conflict at LSN2 (local term2 !=
|
||
* incoming term5) → truncate LSN>=2 then persist LSN2@term5. */
|
||
uint8_t lpay[1] = { 'b' };
|
||
wal_catchup_entry_t ae;
|
||
fill_ae_entry(&ae, /*heartbeat*/0, /*prev_idx*/1, /*prev_term*/2,
|
||
/*leader_commit*/1, /*entry_lsn*/2, WAL_JOB_UPDATE,
|
||
/*entry_term*/5, lpay, 1);
|
||
|
||
int rc = ha_consume_append_entries(c, &ae, /*leader_idx*/1);
|
||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||
|
||
/* (term,lsn) reconverged: LSN2 now carries the leader's term 5. */
|
||
TEST_ASSERT_EQUAL_UINT64(5, persist_entry_term(&c->persist, 2));
|
||
/* The agreed committed prefix (LSN1@term2) is untouched (R3.3). */
|
||
TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 1));
|
||
/* The higher leader term was adopted on the RPC (G5). */
|
||
TEST_ASSERT_EQUAL_UINT64(5, c->ha.raft.term);
|
||
|
||
teardown_cluster(c);
|
||
printf("test_intake_divergent_tail_truncated_and_reconverged: PASS\n");
|
||
}
|
||
|
||
/* (c) An unreconcilable member (logOk fails: the leader's prev tuple does not
|
||
* match the follower's log) is REJECTED — the follower neither persists the
|
||
* leader's entry nor reports false convergence, and stays available to retry
|
||
* (no crash, no committed advance). R3.5/R3.6: the member keeps its divergent
|
||
* state pending a later catch-up rather than spuriously accepting. */
|
||
static void test_intake_logmismatch_rejected_no_false_convergence(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_intake_rej_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
make_follower(c, /*term*/4, /*n_peers*/3);
|
||
|
||
/* Local log: LSN1@term2. The leader claims prev=(LSN1,term3) — a term
|
||
* mismatch at LSN1 → logOk fails. */
|
||
uint8_t blob[64];
|
||
size_t n = pack_entry(blob, WAL_JOB_UPDATE, "a", 1);
|
||
TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 2, blob, n));
|
||
c->ha.raft.log_len = 1;
|
||
|
||
uint8_t lpay[1] = { 'z' };
|
||
wal_catchup_entry_t ae;
|
||
fill_ae_entry(&ae, /*heartbeat*/0, /*prev_idx*/1, /*prev_term*/3,
|
||
/*leader_commit*/0, /*entry_lsn*/2, WAL_JOB_UPDATE,
|
||
/*entry_term*/4, lpay, 1);
|
||
|
||
int rc = ha_consume_append_entries(c, &ae, /*leader_idx*/1);
|
||
/* The helper returns 0 (no crash) but the AE was NOT accepted: the leader's
|
||
* entry was not persisted at LSN2 and the local LSN1 entry is unchanged. */
|
||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(&c->persist, 2)); /* absent */
|
||
TEST_ASSERT_EQUAL_UINT64(2, persist_entry_term(&c->persist, 1)); /* intact */
|
||
|
||
teardown_cluster(c);
|
||
printf("test_intake_logmismatch_rejected_no_false_convergence: PASS\n");
|
||
}
|
||
|
||
/* ================================================================== */
|
||
/* Task 3.4 — live broadcast/catch-up receive cutover (closes bug-990) */
|
||
/* ================================================================== */
|
||
|
||
/* The live cutover's observable: a standby applying a LIVE-broadcast-shaped
|
||
* AppendEntries (a single entry in the `entries` LIST carrying the leader's term
|
||
* + prev tuple — exactly what repl_broadcast → xce_wal_batch_req_raw produces and
|
||
* xce_decode yields) through the PRODUCTION receive entry point
|
||
* ha_apply_wal_catchup_entry persists the LEADER's term (hdr.term == leader_term),
|
||
* NOT 0 (the retired term-0 legacy direct-apply path) and NOT its local term.
|
||
*
|
||
* Before the cutover repl_broadcast sent entry->term == 0, so this entry took the
|
||
* term-0 legacy direct-apply branch and persisted hdr.term = local p->term — the
|
||
* bug-990 term-identity gap. With the cutover the term-carrying entry routes
|
||
* through raft_on_append_entries_request → on_persist_entry → leader term stamped. */
|
||
static void test_live_broadcast_receive_persists_leader_term(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_live_bcast_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
/* The standby is a follower at a LOW local term (3) so we can prove the
|
||
* persisted term is the LEADER's (7), not the local term. */
|
||
make_follower(c, /*term*/3, /*n_peers*/3);
|
||
|
||
/* Seed LSN 1 at the leader's term 7 so the broadcast of LSN 2 satisfies the
|
||
* library's logOk (prev=(1,7)). This mirrors the contiguous live stream. */
|
||
uint8_t blob[64];
|
||
size_t n = pack_entry(blob, WAL_JOB_SUBMIT, "a", 1);
|
||
TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 7, blob, n));
|
||
c->ha.raft.log_len = 1;
|
||
TEST_ASSERT_EQUAL_UINT64(7, persist_entry_term(&c->persist, 1));
|
||
|
||
/* Build the LIVE receive shape: a single entry in the `entries` list, each
|
||
* sub-entry carrying its term + prev tuple (what wal_entry_to_wire now encodes
|
||
* per element and xce_decode reproduces). repl_broadcast at term 7 of LSN 2
|
||
* with prev=(1,7). */
|
||
uint8_t lpay[2] = { 'b', 'c' };
|
||
wal_catchup_entry_t sub;
|
||
memset(&sub, 0, sizeof(sub));
|
||
sub.lsn = 2;
|
||
sub.entry_type = WAL_JOB_SUBMIT;
|
||
sub.term = 7; /* the LEADER's term on the wire */
|
||
sub.prev_log_index = 1;
|
||
sub.prev_log_term = 7;
|
||
sub.raw_payload = lpay;
|
||
sub.raw_payload_len = sizeof(lpay);
|
||
|
||
wal_catchup_entry_t* sub_arr[1] = { &sub };
|
||
wal_catchup_entry_t batch;
|
||
memset(&batch, 0, sizeof(batch));
|
||
batch.n_entries = 1;
|
||
batch.entries = sub_arr;
|
||
/* Top-level container also carries the AE metadata (the encoder mirrors it). */
|
||
batch.term = 7;
|
||
batch.prev_log_index = 1;
|
||
batch.prev_log_term = 7;
|
||
|
||
int rc = ha_apply_wal_catchup_entry(&c->ha, &c->persist, &batch,
|
||
/*ep*/NULL, c);
|
||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||
|
||
/* The load-bearing assertion (bug-990): LSN 2 persists the LEADER's term 7,
|
||
* NOT 0 and NOT the local term 3. Before the cutover this read back as the
|
||
* standby's local term via the term-0 direct-apply path. */
|
||
TEST_ASSERT_EQUAL_UINT64(7, persist_entry_term(&c->persist, 2));
|
||
/* The agreed prefix (LSN 1@term7) is untouched. */
|
||
TEST_ASSERT_EQUAL_UINT64(7, persist_entry_term(&c->persist, 1));
|
||
/* The library adopted the leader's term on the RPC (G5). */
|
||
TEST_ASSERT_EQUAL_UINT64(7, c->ha.raft.term);
|
||
|
||
teardown_cluster(c);
|
||
printf("test_live_broadcast_receive_persists_leader_term: PASS\n");
|
||
}
|
||
|
||
/* ================================================================== */
|
||
/* bug-991 — catch-up applied_lsn must be "last contiguously-held LSN" */
|
||
/* (NOT next-to-apply / lsn+1) so the leader's catch-up */
|
||
/* streamer never SKIPS the standby's real next-needed LSN. */
|
||
/* ================================================================== */
|
||
|
||
/* The standby's applied_lsn is the watermark it advertises to the leader
|
||
* (ha_nop_reply_state_report → sr.applied_lsn → standby_reported_lsn). The
|
||
* leader's catch-up streamer (ha_stream_wal_to_standby) and the snapshot/stream
|
||
* decision (ha_evaluate_repl_action) BOTH treat that value as the
|
||
* "last contiguously-held LSN" and stream the half-open range
|
||
* (standby_lsn, committed_lsn]. The recovery/checkpoint/init setters already use
|
||
* last-held semantics (persist_durable_floor / checkpoint_lsn / 0). The three
|
||
* live/catch-up apply paths historically advanced to lsn+1 (next-to-apply) —
|
||
* an off-by-one that made the standby advertise one LSN too high, so after it
|
||
* applied catch-up entry N the leader streamed (N+1, committed] and SKIPPED the
|
||
* real next-needed entry N+1 forever (logOk wedge → no WalAck → commit stall).
|
||
*
|
||
* RED (pre-fix): after applying a catch-up entry at LSN N the standby's
|
||
* applied_lsn reads N+1, and ha_evaluate_repl_action(standby_lsn=applied_lsn,
|
||
* committed=N) reports CONVERGED one entry early (skip).
|
||
* GREEN (post-fix): applied_lsn reads N (last-held); a standby that has applied
|
||
* up to N with committed > N still STREAMS (no skip), and the streamer's
|
||
* (standby_lsn, committed] range begins at exactly N+1. */
|
||
static void test_catchup_applied_lsn_is_last_held_no_skip(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_bug991_lasthelp_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
make_follower(c, /*term*/7, /*n_peers*/3);
|
||
|
||
/* applied_lsn starts at 0 (init = empty/last-held-nothing). */
|
||
TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&c->ha.applied_lsn));
|
||
|
||
/* Catch-up stream of LSN 1 then LSN 2 at the leader's term 7, contiguous
|
||
* (prev tuples chain 0/0 → 1/7), routed through the production receive entry
|
||
* point exactly as ha_stream_wal_to_standby → xce_decode would deliver them. */
|
||
uint8_t p1[1] = { 'a' };
|
||
wal_catchup_entry_t sub1;
|
||
memset(&sub1, 0, sizeof(sub1));
|
||
sub1.lsn = 1; sub1.entry_type = WAL_JOB_SUBMIT; sub1.term = 7;
|
||
sub1.prev_log_index = 0; sub1.prev_log_term = 0;
|
||
sub1.raw_payload = p1; sub1.raw_payload_len = sizeof(p1);
|
||
wal_catchup_entry_t* arr1[1] = { &sub1 };
|
||
wal_catchup_entry_t batch1;
|
||
memset(&batch1, 0, sizeof(batch1));
|
||
batch1.n_entries = 1; batch1.entries = arr1;
|
||
batch1.term = 7; batch1.prev_log_index = 0; batch1.prev_log_term = 0;
|
||
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
ha_apply_wal_catchup_entry(&c->ha, &c->persist, &batch1, /*ep*/NULL, c));
|
||
TEST_ASSERT_EQUAL_UINT64(7, persist_entry_term(&c->persist, 1));
|
||
|
||
/* THE bug-991 load-bearing assertion: after applying LSN 1 the standby
|
||
* advertises applied_lsn == 1 (last-held), NOT 2 (next-to-apply). Pre-fix
|
||
* this read 2 and the leader's next stream skipped LSN 2. */
|
||
TEST_ASSERT_EQUAL_UINT64(1, atomic_load(&c->ha.applied_lsn));
|
||
|
||
/* No-skip property at the leader's decision seam: a standby holding up to
|
||
* LSN 1 with committed_lsn == 2 must STREAM (its next-needed LSN 2 is missing),
|
||
* NOT report CONVERGED. With the off-by-one advertised value (2) the leader
|
||
* would have seen standby_lsn(2) >= committed(2) → CONVERGED (the skip). */
|
||
{
|
||
uint64_t standby_lsn = atomic_load(&c->ha.applied_lsn);
|
||
ha_repl_action_t act = ha_evaluate_repl_action(
|
||
/*standby_tag*/1, standby_lsn,
|
||
/*committed_tag*/1, /*committed_lsn*/2, /*oldest_wal_lsn*/1);
|
||
TEST_ASSERT_EQUAL_INT(HA_REPL_STREAM_WAL, act);
|
||
/* The streamer's start LSN is standby_lsn + 1 == 2 — exactly the missing
|
||
* entry, no skip. */
|
||
TEST_ASSERT_EQUAL_UINT64(2, standby_lsn + 1);
|
||
}
|
||
|
||
/* Apply the next contiguous entry (LSN 2) — applied_lsn becomes 2 (last-held),
|
||
* not 3 — and the standby is now CONVERGED at committed_lsn == 2. */
|
||
uint8_t p2[1] = { 'b' };
|
||
wal_catchup_entry_t sub2;
|
||
memset(&sub2, 0, sizeof(sub2));
|
||
sub2.lsn = 2; sub2.entry_type = WAL_JOB_SUBMIT; sub2.term = 7;
|
||
sub2.prev_log_index = 1; sub2.prev_log_term = 7;
|
||
sub2.raw_payload = p2; sub2.raw_payload_len = sizeof(p2);
|
||
wal_catchup_entry_t* arr2[1] = { &sub2 };
|
||
wal_catchup_entry_t batch2;
|
||
memset(&batch2, 0, sizeof(batch2));
|
||
batch2.n_entries = 1; batch2.entries = arr2;
|
||
batch2.term = 7; batch2.prev_log_index = 1; batch2.prev_log_term = 7;
|
||
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
ha_apply_wal_catchup_entry(&c->ha, &c->persist, &batch2, /*ep*/NULL, c));
|
||
TEST_ASSERT_EQUAL_UINT64(7, persist_entry_term(&c->persist, 2));
|
||
TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->ha.applied_lsn));
|
||
{
|
||
ha_repl_action_t act = ha_evaluate_repl_action(
|
||
/*standby_tag*/1, atomic_load(&c->ha.applied_lsn),
|
||
/*committed_tag*/1, /*committed_lsn*/2, /*oldest_wal_lsn*/1);
|
||
TEST_ASSERT_EQUAL_INT(HA_REPL_CONVERGED, act);
|
||
}
|
||
|
||
teardown_cluster(c);
|
||
printf("test_catchup_applied_lsn_is_last_held_no_skip: PASS\n");
|
||
}
|
||
|
||
/* ================================================================== */
|
||
/* bug-1067 — cross-term catch-up: the AppendEntries header term is the */
|
||
/* leader's CURRENT term, NOT the streamed entry's stored term */
|
||
/* ================================================================== */
|
||
|
||
/* Root cause: ha_consume_append_entries used the entry's stored term (entry_term)
|
||
* as the AppendEntries header term. For LIVE appends that is the leader's current
|
||
* term (leader appends at its term), but for CATCH-UP streaming of HISTORICAL
|
||
* entries written by a PRIOR leader, entry_term < the current leader's term. The
|
||
* library's stale-term check (term < s->term → ae_reject_stale) then permanently
|
||
* rejected every historical entry, so the lagging standby could never catch up
|
||
* (standby_reported_lsn never advanced; the streamer retransmitted forever).
|
||
*
|
||
* The fix carries the leader's current term out-of-band per entry
|
||
* (wal_catchup_entry_t.leader_current_term, WalCatchupEntry.leaderCurrentTerm @8),
|
||
* set by the streamer to ha->term, and uses THAT as the AE header term while the
|
||
* entry's stored term remains the per-entry term for logOk / on_persist_entry.
|
||
*
|
||
* RED before the fix: the leader_current_term field does not exist (compile
|
||
* error); once added but before the receiver reads it, ae_term == entry_term (4) <
|
||
* s->term (5) → stale-reject → LSN 2 never persists. */
|
||
static void test_intake_crossterm_catchup_uses_leader_current_term(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_xterm_catchup_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
/* Standby has already adopted the current leader's term (5) from heartbeats. */
|
||
make_follower(c, /*term*/5, /*n_peers*/3);
|
||
|
||
/* Local log holds a HISTORICAL entry LSN1@term4 (written by the prior leader
|
||
* at term 4, replicated to this standby). */
|
||
uint8_t blob[64];
|
||
size_t n = pack_entry(blob, WAL_JOB_SUBMIT, "a", 1);
|
||
TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 4, blob, n));
|
||
c->ha.raft.log_len = 1;
|
||
TEST_ASSERT_EQUAL_UINT64(4, persist_entry_term(&c->persist, 1));
|
||
|
||
/* The CURRENT leader (term 5) streams the historical entry LSN2@term4 as a
|
||
* catch-up sub-entry: the entry's STORED term is 4, but the AppendEntries
|
||
* header term is the leader's CURRENT term 5 (leader_current_term). prev tuple
|
||
* (1,4) matches the local log so logOk holds. */
|
||
uint8_t lpay[1] = { 'b' };
|
||
wal_catchup_entry_t sub;
|
||
memset(&sub, 0, sizeof(sub));
|
||
sub.lsn = 2;
|
||
sub.entry_type = WAL_JOB_SUBMIT;
|
||
sub.term = 4; /* the entry's STORED term (prior leader) */
|
||
sub.leader_current_term = 5; /* the streaming leader's CURRENT term */
|
||
sub.prev_log_index = 1;
|
||
sub.prev_log_term = 4;
|
||
sub.raw_payload = lpay;
|
||
sub.raw_payload_len = sizeof(lpay);
|
||
|
||
wal_catchup_entry_t* sub_arr[1] = { &sub };
|
||
wal_catchup_entry_t batch;
|
||
memset(&batch, 0, sizeof(batch));
|
||
batch.n_entries = 1;
|
||
batch.entries = sub_arr;
|
||
batch.term = 4;
|
||
batch.leader_current_term = 5;
|
||
batch.prev_log_index = 1;
|
||
batch.prev_log_term = 4;
|
||
|
||
int rc = ha_apply_wal_catchup_entry(&c->ha, &c->persist, &batch, /*ep*/NULL, c);
|
||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||
|
||
/* THE bug-1067 assertion: the historical entry is ACCEPTED (not stale-rejected)
|
||
* and persisted at its stored term 4. Pre-fix it was rejected (absent). */
|
||
TEST_ASSERT_EQUAL_UINT64(4, persist_entry_term(&c->persist, 2));
|
||
/* The agreed prefix (LSN1@term4) is untouched. */
|
||
TEST_ASSERT_EQUAL_UINT64(4, persist_entry_term(&c->persist, 1));
|
||
/* The leader's current term (5) equals our term — no spurious term change. */
|
||
TEST_ASSERT_EQUAL_UINT64(5, c->ha.raft.term);
|
||
|
||
teardown_cluster(c);
|
||
printf("test_intake_crossterm_catchup_uses_leader_current_term: PASS\n");
|
||
}
|
||
|
||
/* Safety guard: a GENUINELY stale leader (leader_current_term < our term) must
|
||
* STILL be rejected — the fix must not open a hole letting a deposed leader's
|
||
* divergent entries through. */
|
||
static void test_intake_stale_leader_term_still_rejected(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_stale_leader_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
make_follower(c, /*term*/5, /*n_peers*/3);
|
||
|
||
uint8_t blob[64];
|
||
size_t n = pack_entry(blob, WAL_JOB_SUBMIT, "a", 1);
|
||
TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 4, blob, n));
|
||
c->ha.raft.log_len = 1;
|
||
|
||
/* A deposed leader at term 3 (< our term 5) tries to stream. Even though the
|
||
* entry's stored term and prev tuple look locally plausible, the AE header
|
||
* term (leader_current_term=3) is stale → reject. */
|
||
uint8_t lpay[1] = { 'z' };
|
||
wal_catchup_entry_t sub;
|
||
memset(&sub, 0, sizeof(sub));
|
||
sub.lsn = 2;
|
||
sub.entry_type = WAL_JOB_SUBMIT;
|
||
sub.term = 3;
|
||
sub.leader_current_term = 3; /* STALE leader term < our term 5 */
|
||
sub.prev_log_index = 1;
|
||
sub.prev_log_term = 4;
|
||
sub.raw_payload = lpay;
|
||
sub.raw_payload_len = sizeof(lpay);
|
||
|
||
wal_catchup_entry_t* sub_arr[1] = { &sub };
|
||
wal_catchup_entry_t batch;
|
||
memset(&batch, 0, sizeof(batch));
|
||
batch.n_entries = 1;
|
||
batch.entries = sub_arr;
|
||
batch.term = 3;
|
||
batch.leader_current_term = 3;
|
||
batch.prev_log_index = 1;
|
||
batch.prev_log_term = 4;
|
||
|
||
int rc = ha_apply_wal_catchup_entry(&c->ha, &c->persist, &batch, /*ep*/NULL, c);
|
||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||
|
||
/* Stale leader's entry was REJECTED — LSN 2 absent, our term unchanged. */
|
||
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(&c->persist, 2));
|
||
TEST_ASSERT_EQUAL_UINT64(5, c->ha.raft.term);
|
||
|
||
teardown_cluster(c);
|
||
printf("test_intake_stale_leader_term_still_rejected: PASS\n");
|
||
}
|
||
|
||
/* ================================================================== */
|
||
/* Task 3.3 — drive raft_on_append_entries_reply + raft_advance_commit */
|
||
/* on the replication ACK path */
|
||
/* ================================================================== */
|
||
|
||
/* These tests drive the ACK seam ha_raft_on_repl_ack(c, abs_peer_idx, acked_lsn,
|
||
* reply_term): on a replication ACK the controller routes the reply through
|
||
* raft_on_append_entries_reply (updating match_index[]/next_index[]) then
|
||
* raft_advance_commit, which recomputes the current-term-gated commit and MAY
|
||
* fire on_commit_advanced (the task-2.4 callback that feeds the barrier).
|
||
*
|
||
* Peer-index contract (REMEDIATION round 1): ha_raft_on_repl_ack now takes the
|
||
* ABSOLUTE library peer index directly (the match_index[]/next_index[] index
|
||
* space, j != self_idx). The CALLER resolves it per endpoint type via
|
||
* ha_repl_ack_abs_peer_idx BEFORE this seam — promoted-vote eps carry a relative
|
||
* "votes" index (remapped), heartbeat standby eps carry an already-absolute
|
||
* index (passed through). The earlier blind remap inside the seam corrupted the
|
||
* heartbeat path (wrong-peer false quorum / OOB match_index[] write); the
|
||
* endpoint-type round-trip is pinned by the two remediation tests below.
|
||
*
|
||
* Fixture: make_leader(c, term, n_peers) brings the node up as the raft LEADER
|
||
* with self_idx == my_priority == 0 (make_cluster calloc's my_priority=0), so
|
||
* the only standby's absolute peer index is 1.
|
||
*
|
||
* Against the pre-3.3 ACK path (repl_on_ack durability only, no library reply),
|
||
* ha_raft_on_repl_ack does not exist (compile RED); once wired, a current-term
|
||
* quorum ACK MUST advance the library commit_index and fire on_commit_advanced
|
||
* (R4.1/R4.5), a prior-term agreed entry MUST hold, and a stale-term reply MUST
|
||
* be dropped. */
|
||
|
||
/* Seed `count` entries at the given term into the leader's local WAL via the
|
||
* wired on_persist_entry, growing the library log_len in lockstep. */
|
||
static void leader_seed_entries(cluster_t* c, uint64_t term, int count)
|
||
{
|
||
uint8_t blob[64];
|
||
for (int i = 1; i <= count; i++) {
|
||
size_t n = pack_entry(blob, WAL_JOB_UPDATE, "x", 1);
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
ha_cb_persist_entry_for_test(&c->ha, (uint64_t)i, term, blob, n));
|
||
c->ha.raft.log_len = (uint64_t)i;
|
||
}
|
||
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
|
||
}
|
||
|
||
/* (a) A current-term quorum ACK drives raft_on_append_entries_reply →
|
||
* raft_advance_commit → on_commit_advanced fires with the agreed commit_index,
|
||
* which is recorded as the advisory library index AND wakes the barrier. 3-peer
|
||
* cluster (self + 2 standbys, quorum=2): self holds LSN 1,2 at the current term;
|
||
* one standby ACKs LSN 2 → {self, peer1} is a quorum for LSN 2, the agreed entry
|
||
* is current-term, so the library commit_index advances to 2; the durable
|
||
* majority (also at 2) is what advances committed_lsn (optA: the durable-majority
|
||
* barrier is authoritative; the recorded index is inert-advisory). This leg
|
||
* still verifies the full ACK→library→callback→barrier wiring is intact. */
|
||
static void test_repl_ack_currentterm_quorum_advances_commit(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_ack_adv_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
const uint64_t T = 6;
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
make_leader(c, T, /*n_peers*/3);
|
||
|
||
/* Init the cluster's commit engine over the same 3-peer cluster so a
|
||
* majority is self + 1 standby, and register one standby whose durable ACK
|
||
* supplies the durable-majority floor. */
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
commit_engine_init(&c->commit, &c->persist, /*n_peers*/3, /*n_vcpus*/8));
|
||
|
||
/* Seed LSN 1,2 at the current term T → durable floor reaches LSN 2. */
|
||
leader_seed_entries(c, T, /*count*/2);
|
||
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
repl_add_standby(&c->persist.repl, &g_ae_dummy_ep, "sb0", 7000));
|
||
/* Durable majority: the standby's durable position covers LSN 2. Under optA
|
||
* the durable-majority barrier is authoritative, so this ACK already advances
|
||
* committed_lsn to 2 (asserted at the end); the library wiring below is still
|
||
* exercised to confirm the ACK→library→callback path is intact. */
|
||
atomic_store(&c->persist.repl.standbys[0].acked_lsn, 2);
|
||
commit_on_ack(&c->commit, /*peer_idx*/0, /*acked_lsn*/2);
|
||
|
||
/* The library has not advanced commit_index yet (no ACK routed through the
|
||
* library) so the advisory record is still 0 — but committed_lsn is NOT
|
||
* gated by it (optA). */
|
||
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.commit_index);
|
||
|
||
/* The standby ACKs LSN 2 at the leader's current term. The caller has
|
||
* already resolved the absolute peer index (self_idx 0 → the only standby is
|
||
* absolute peer 1); the seam takes that absolute index directly. */
|
||
int rc = ha_raft_on_repl_ack(c, /*abs_peer_idx*/1, /*acked_lsn*/2, /*reply_term*/T);
|
||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||
|
||
/* (1) The library recorded the peer's match_index at the absolute index. */
|
||
TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.match_index[1]);
|
||
TEST_ASSERT_EQUAL_UINT64(3, c->ha.raft.next_index[1]);
|
||
|
||
/* (2) raft_advance_commit advanced the current-term-gated commit_index to 2
|
||
* ({self, peer1} quorum, entry at LSN 2 is term T). */
|
||
TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.commit_index);
|
||
|
||
/* (3) committed_lsn advanced to the durable majority (2). on_commit_advanced
|
||
* fired and recorded the advisory index (2), but the durable-majority barrier
|
||
* is what governs committed_lsn under optA (R4.1 realized via the authoritative
|
||
* durable barrier + own-term gate). */
|
||
TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn));
|
||
TEST_ASSERT_EQUAL_UINT64(2, c->commit.raft_commit_index); /* advisory recorded */
|
||
|
||
commit_engine_fini(&c->commit);
|
||
teardown_cluster(c);
|
||
printf("test_repl_ack_currentterm_quorum_advances_commit: PASS\n");
|
||
}
|
||
|
||
/* (b) A quorum that agrees only on a PRIOR-term entry does NOT advance commit
|
||
* (the Figure-8 current-term rule, R4.1/R4.5). self holds LSN 1@term3 (a
|
||
* prior-term entry) and is now leader at term 6 with no current-term entry yet.
|
||
* A standby ACKs LSN 1 → {self, peer1} is a quorum for LSN 1, but the entry at
|
||
* LSN 1 is term 3 != current term 6, so raft_advance_commit holds commit_index
|
||
* at 0 and on_commit_advanced never fires. */
|
||
static void test_repl_ack_priorterm_quorum_holds_commit(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_ack_prior_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
const uint64_t PRIOR = 3;
|
||
const uint64_t CUR = 6;
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
make_leader(c, CUR, /*n_peers*/3);
|
||
|
||
/* Seed a single PRIOR-term entry at LSN 1 (replicated from an old leader). */
|
||
leader_seed_entries(c, PRIOR, /*count*/1);
|
||
|
||
/* A standby ACKs LSN 1 at the leader's current term (the wire reply term is
|
||
* the leader's term; the ENTRY's term is what gates the commit). */
|
||
int rc = ha_raft_on_repl_ack(c, /*abs_peer_idx*/1, /*acked_lsn*/1, /*reply_term*/CUR);
|
||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||
|
||
/* match_index updated (the reply itself is current-term, not stale). */
|
||
TEST_ASSERT_EQUAL_UINT64(1, c->ha.raft.match_index[1]);
|
||
|
||
/* But commit is HELD: the agreed entry (LSN1@term3) is not current-term →
|
||
* Figure-8 rule blocks the advance, commit_index stays 0. */
|
||
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.commit_index);
|
||
|
||
teardown_cluster(c);
|
||
printf("test_repl_ack_priorterm_quorum_holds_commit: PASS\n");
|
||
}
|
||
|
||
/* (c) A stale-term reply is dropped: raft_on_append_entries_reply discards a
|
||
* reply whose term != s->term, so match_index[] is NOT updated and no commit
|
||
* advance occurs (R4.5 / raft.tla 415). self holds LSN 1,2 at the current term;
|
||
* a stale reply (reply_term < current term) for LSN 2 must leave match_index and
|
||
* commit_index untouched. */
|
||
static void test_repl_ack_staleterm_reply_dropped(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_ack_stale_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
const uint64_t T = 8;
|
||
const uint64_t STALE = 5;
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
make_leader(c, T, /*n_peers*/3);
|
||
|
||
leader_seed_entries(c, T, /*count*/2);
|
||
|
||
/* Stale-term ACK for LSN 2 → dropped by the library before any state change. */
|
||
int rc = ha_raft_on_repl_ack(c, /*abs_peer_idx*/1, /*acked_lsn*/2, /*reply_term*/STALE);
|
||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||
|
||
/* No match_index update (the peer's slot stays at the post-init default 0). */
|
||
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.match_index[1]);
|
||
/* No commit advance. */
|
||
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.commit_index);
|
||
|
||
teardown_cluster(c);
|
||
printf("test_repl_ack_staleterm_reply_dropped: PASS\n");
|
||
}
|
||
|
||
/* ================================================================== */
|
||
/* Task 3.3 REMEDIATION round 1 — endpoint-type-correct peer mapping */
|
||
/* ================================================================== */
|
||
|
||
/* (d) The bug: the ACK walAck branch fires for BOTH promoted-vote eps (relative
|
||
* votes index) AND heartbeat standby eps (already-absolute index). The earlier
|
||
* seam unconditionally remapped, so a heartbeat ep's absolute index was mapped a
|
||
* SECOND time — crediting the WRONG peer (false quorum) or writing OUT OF BOUNDS.
|
||
* ha_repl_ack_abs_peer_idx must remap ONLY the vote ep and pass the standby ep
|
||
* through unchanged. We pin both endpoint types AND both my_priority values so
|
||
* the remap-vs-passthrough distinction actually round-trips (the pre-fix tests
|
||
* only used my_priority==0, which masked the bug).
|
||
*
|
||
* Round-trip oracle: for a vote ep, peer absolute index i is sent as votes_idx
|
||
* (i < self ? i : i-1); ha_repl_ack_abs_peer_idx must recover exactly i. For a
|
||
* standby ep, the stored peer_idx IS i and must pass through unchanged. */
|
||
static void test_repl_ack_abs_peer_idx_endpoint_type_roundtrip(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_ack_map_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
|
||
/* --- self_idx (my_priority) == 0: every peer is after self --- */
|
||
c->ha.my_priority = 0;
|
||
/* Heartbeat standby ep (ctx==NULL → ep_is_vote=0): absolute index passes
|
||
* through UNCHANGED. Absolute peer 1 must stay 1, absolute peer 2 stay 2.
|
||
* (The pre-fix blind remap would have turned 1→2 and 2→3 — the bug.) */
|
||
TEST_ASSERT_EQUAL_INT(1, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/0, /*peer*/1));
|
||
TEST_ASSERT_EQUAL_INT(2, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/0, /*peer*/2));
|
||
/* Promoted vote ep (ep_is_vote=1): votes_idx 0 → abs 1, votes_idx 1 → abs 2. */
|
||
TEST_ASSERT_EQUAL_INT(1, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/1, /*votes*/0));
|
||
TEST_ASSERT_EQUAL_INT(2, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/1, /*votes*/1));
|
||
|
||
/* --- self_idx (my_priority) == 1: a peer below self is NOT shifted --- */
|
||
c->ha.my_priority = 1;
|
||
/* Standby ep: absolute indices 0 and 2 (peer 1 is self) pass through. */
|
||
TEST_ASSERT_EQUAL_INT(0, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/0, /*peer*/0));
|
||
TEST_ASSERT_EQUAL_INT(2, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/0, /*peer*/2));
|
||
/* Vote ep with self_idx==1: votes_idx 0 (peer absolute 0, < self) → 0;
|
||
* votes_idx 1 (peer absolute 2, >= self) → 2. This is where remap and
|
||
* passthrough DIVERGE — the case the my_priority==0 tests could not reach. */
|
||
TEST_ASSERT_EQUAL_INT(0, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/1, /*votes*/0));
|
||
TEST_ASSERT_EQUAL_INT(2, ha_repl_ack_abs_peer_idx(&c->ha, /*vote*/1, /*votes*/1));
|
||
|
||
teardown_cluster(c);
|
||
printf("test_repl_ack_abs_peer_idx_endpoint_type_roundtrip: PASS\n");
|
||
}
|
||
|
||
/* (e) End-to-end heartbeat-standby ACK: a standby ep carries an ALREADY-ABSOLUTE
|
||
* peer index. Resolving it via ha_repl_ack_abs_peer_idx (ep_is_vote=0) and
|
||
* driving ha_raft_on_repl_ack must land match_index on the CORRECT absolute slot
|
||
* (not the +1 wrong slot the blind remap produced). 3-peer cluster, self=0; the
|
||
* standby's absolute index is 1. A current-term quorum ACK for LSN 2 advances
|
||
* the library commit_index to 2 — but ONLY because match_index[1] (not [2]) is
|
||
* credited. Under the broken blind remap the credit landed on match_index[2]
|
||
* (a phantom peer): no quorum at LSN 2, commit_index would stay 0. */
|
||
static void test_repl_ack_heartbeat_standby_absolute_slot(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_ack_hb_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
const uint64_t T = 6;
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
make_leader(c, T, /*n_peers*/3);
|
||
TEST_ASSERT_EQUAL_INT(0, c->ha.my_priority); /* self_idx 0 */
|
||
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
commit_engine_init(&c->commit, &c->persist, /*n_peers*/3, /*n_vcpus*/8));
|
||
|
||
leader_seed_entries(c, T, /*count*/2);
|
||
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
repl_add_standby(&c->persist.repl, &g_ae_dummy_ep, "sb0", 7000));
|
||
atomic_store(&c->persist.repl.standbys[0].acked_lsn, 2);
|
||
commit_on_ack(&c->commit, /*peer_idx*/0, /*acked_lsn*/2);
|
||
|
||
/* The standby ep stores the ABSOLUTE index 1 (standby_ep_ctx_t.peer_idx).
|
||
* Resolve it as a heartbeat ep (ep_is_vote=0) — it must pass through as 1. */
|
||
int abs_idx = ha_repl_ack_abs_peer_idx(&c->ha, /*ep_is_vote*/0, /*peer_idx*/1);
|
||
TEST_ASSERT_EQUAL_INT(1, abs_idx); /* NOT 2 — the blind-remap bug slot */
|
||
|
||
int rc = ha_raft_on_repl_ack(c, abs_idx, /*acked_lsn*/2, /*reply_term*/T);
|
||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||
|
||
/* The CORRECT absolute slot was credited; the phantom slot 2 stayed empty. */
|
||
TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.match_index[1]);
|
||
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.match_index[2]);
|
||
|
||
/* {self, peer1} is a real quorum for LSN 2 → current-term commit advances. */
|
||
TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.commit_index);
|
||
TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn));
|
||
|
||
commit_engine_fini(&c->commit);
|
||
teardown_cluster(c);
|
||
printf("test_repl_ack_heartbeat_standby_absolute_slot: PASS\n");
|
||
}
|
||
|
||
/* (f) Defense-in-depth: an out-of-range / self absolute index is rejected loudly
|
||
* (returns -EINVAL) and performs NO library accounting — fencing the unchecked
|
||
* match_index[] write at the controller boundary rather than corrupting memory. */
|
||
static void test_repl_ack_out_of_range_peer_rejected(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_ack_oob_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
const uint64_t T = 6;
|
||
|
||
cluster_t* c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
make_leader(c, T, /*n_peers*/3); /* valid peers are absolute 1,2 */
|
||
leader_seed_entries(c, T, /*count*/2);
|
||
|
||
/* abs index == n_peers (3) is out of range → rejected, no state change. */
|
||
TEST_ASSERT_EQUAL_INT(-EINVAL,
|
||
ha_raft_on_repl_ack(c, /*abs_peer_idx*/3, /*acked_lsn*/2, /*reply_term*/T));
|
||
/* self (0) is never a valid peer → rejected. */
|
||
TEST_ASSERT_EQUAL_INT(-EINVAL,
|
||
ha_raft_on_repl_ack(c, /*abs_peer_idx*/0, /*acked_lsn*/2, /*reply_term*/T));
|
||
|
||
/* Nothing was credited and commit did not advance. */
|
||
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.match_index[1]);
|
||
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.commit_index);
|
||
|
||
teardown_cluster(c);
|
||
printf("test_repl_ack_out_of_range_peer_rejected: PASS\n");
|
||
}
|
||
|
||
/* ================================================================== */
|
||
/* Task 5.1 — every-callback + marker-path end-to-end integration */
|
||
/* ================================================================== */
|
||
|
||
/*
|
||
* The per-task tests above each exercise ONE callback (or one leg of the
|
||
* marker path) in isolation. This integration test drives a single realistic
|
||
* leader-lifecycle flow in which ALL FIVE adapter callbacks AND the config
|
||
* marker path interoperate end-to-end, so a regression that breaks the
|
||
* COMPOSITION of the callbacks (not just one in isolation) is caught here:
|
||
*
|
||
* on_persist_entry — primary job append + config-marker append both
|
||
* land in the WAL with the live term stamped
|
||
* on_get_entry_term — the leader's last-log term is read back
|
||
* on_send_append_entries— the leader emits the marker entry to a peer with the
|
||
* correct (prev tuple, leader_commit) and the entry's
|
||
* recorded term/type/payload
|
||
* on_commit_advanced — a current-term quorum ACK advances commit; the
|
||
* marker enters the committed prefix and the DERIVED
|
||
* committed config (config_committed_marker) equals it
|
||
* on_truncate_from — a divergent same-LSN tail is truncated then the
|
||
* leader's entry re-persisted, the committed prefix
|
||
* (including the committed marker) left intact
|
||
*
|
||
* Against any single broken callback the corresponding assertion below fails
|
||
* (e.g. neutering on_commit_advanced leaves committed_lsn behind the durable
|
||
* majority so config_committed_marker returns -ENOENT; neutering on_truncate_from
|
||
* leaves the stale divergent term at the conflicting LSN).
|
||
*/
|
||
static void test_every_callback_and_marker_path_end_to_end(void)
|
||
{
|
||
char tmp[256];
|
||
strcpy(tmp, "/tmp/xce_ts_e2e_all_XXXXXX");
|
||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||
|
||
const uint64_t T = 9;
|
||
|
||
cluster_t *c = make_cluster(tmp);
|
||
TEST_ASSERT_NOT_NULL(c);
|
||
|
||
/* Bring the node up as the raft LEADER at term T (init-restore floor path,
|
||
* task 2.5). A 3-peer cluster so a majority is self + 1 standby. */
|
||
make_leader(c, T, /*n_peers*/3);
|
||
TEST_ASSERT_EQUAL_INT(0, c->ha.my_priority); /* self_idx 0 */
|
||
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.log_len);
|
||
|
||
/* The cluster's own commit engine over the same persist (3 peers). */
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
commit_engine_init(&c->commit, &c->persist, /*n_peers*/3, /*n_vcpus*/8));
|
||
|
||
/* ── on_persist_entry (job append) ──────────────────────────────────────
|
||
* A primary job-state append routes through the library → on_persist_entry
|
||
* stamps the live term at LSN 1. */
|
||
wal_jobstate_t js = { .jobid = 77, .state = 2 };
|
||
uint64_t lsn_job = 0;
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
ha_primary_log_append(c, WAL_JOB_UPDATE, &js, sizeof(js), &lsn_job));
|
||
TEST_ASSERT_EQUAL_UINT64(1, lsn_job);
|
||
TEST_ASSERT_EQUAL_UINT64(1, c->ha.raft.log_len);
|
||
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1));
|
||
|
||
/* ── on_persist_entry (config marker — the marker path) ─────────────────
|
||
* A config marker rides the SAME unified append path (R7.1/R7.3): it lands
|
||
* at LSN 2 with the live term stamped, exactly like a job entry. */
|
||
uint8_t sha[32];
|
||
for (int i = 0; i < 32; i++) sha[i] = (uint8_t)(0xA0 + i);
|
||
uint64_t tag = 0xA0A1A2A3A4A5A6A7ULL; /* be64 of the first 8 sha bytes */
|
||
uint64_t lsn_marker = 0;
|
||
TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &lsn_marker));
|
||
TEST_ASSERT_EQUAL_UINT64(2, lsn_marker);
|
||
TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.log_len);
|
||
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 2));
|
||
|
||
/* A third, NON-committed job entry at LSN 3 (current term). It is appended
|
||
* BEFORE the standby is registered so the primary's repl_broadcast is a
|
||
* no-op (registering the dummy int ep first would fan the append out to a
|
||
* non-endpoint → ASAN SEGV — the same ordering constraint the send-AE tests
|
||
* above document). LSN 3 is the divergent suffix the on_truncate_from leg
|
||
* removes once the committed prefix (LSN 1,2) is established below. */
|
||
wal_jobstate_t js2 = { .jobid = 78, .state = 1 };
|
||
uint64_t lsn3 = 0;
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
ha_primary_log_append(c, WAL_JOB_UPDATE, &js2, sizeof(js2), &lsn3));
|
||
TEST_ASSERT_EQUAL_UINT64(3, lsn3);
|
||
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 3));
|
||
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
|
||
|
||
/* ── on_get_entry_term ──────────────────────────────────────────────────
|
||
* The leader's last-log term (LSN 3) reads back as T via the wired callback. */
|
||
TEST_ASSERT_EQUAL_UINT64(T,
|
||
ha_cb_get_entry_term_for_test(&c->ha, c->ha.raft.log_len));
|
||
|
||
/* ── on_send_append_entries ─────────────────────────────────────────────
|
||
* The leader emits the marker entry (LSN 2) to peer 0, carrying the prev
|
||
* tuple (LSN 1, term T), leader_commit, and the marker's own recorded term +
|
||
* WAL_CONFIG_MARKER type + 40-byte reference payload (read from its own WAL). */
|
||
register_one_standby(c);
|
||
repl_ae_capture_t cap;
|
||
memset(&cap, 0, sizeof(cap));
|
||
repl_send_ae_capture_hook = ∩
|
||
int rc = ha_cb_send_append_entries_for_test(&c->ha, /*peer*/0,
|
||
/*prev_idx*/1, /*prev_term*/T,
|
||
/*leader_commit*/0, /*n*/1);
|
||
repl_send_ae_capture_hook = NULL;
|
||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||
TEST_ASSERT_EQUAL_INT(1, cap.called);
|
||
TEST_ASSERT_EQUAL_UINT64(1, cap.prev_log_index);
|
||
TEST_ASSERT_EQUAL_UINT64(T, cap.prev_log_term);
|
||
TEST_ASSERT_EQUAL_UINT64(2, cap.entry_lsn);
|
||
TEST_ASSERT_EQUAL_UINT64(T, cap.entry_term);
|
||
TEST_ASSERT_EQUAL_UINT8(WAL_CONFIG_MARKER, cap.entry_type);
|
||
TEST_ASSERT_EQUAL_UINT32(WAL_CONFIG_MARKER_PAYLOAD_LEN, cap.payload_len);
|
||
|
||
/* ── on_commit_advanced (+ committed marker derive) ─────────────────────
|
||
* Before commit advances, the marker is appended but NOT in the committed
|
||
* prefix, so the derived committed config is absent. */
|
||
uint64_t got_tag = 0;
|
||
uint8_t got_sha[32];
|
||
TEST_ASSERT_EQUAL_INT(-ENOENT,
|
||
config_committed_marker(c, &got_tag, got_sha));
|
||
|
||
/* on_commit_advanced records the advisory library index and wakes the
|
||
* barrier (optA: it does NOT gate committed_lsn). Production order: the ACK
|
||
* path fires on_commit_advanced with a (possibly lagging) commit_index, then
|
||
* the durable-majority barrier — which is authoritative — advances
|
||
* committed_lsn. Fire the WIRED callback with a LAGGING commit_index=1, then
|
||
* supply a durable majority covering LSN 2: committed_lsn advances to the
|
||
* durable majority (2), NOT pinned at the lagging advisory index (1). Under
|
||
* the OLD (rejected) ceiling design this would have stuck committed at 1 —
|
||
* the starvation. The advisory index is still RECORDED (sink intact). */
|
||
ha_cb_commit_advanced_for_test(&c->ha, 1);
|
||
TEST_ASSERT_EQUAL_UINT64(1, c->commit.raft_commit_index); /* advisory recorded */
|
||
atomic_store(&c->persist.repl.standbys[0].acked_lsn, 2);
|
||
commit_on_ack(&c->commit, /*peer_idx*/0, /*acked_lsn*/2);
|
||
TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn));
|
||
|
||
/* Route the real current-term quorum ACK through the library
|
||
* (raft_on_append_entries_reply → raft_advance_commit → on_commit_advanced).
|
||
* The agreed entries are current-term (T) so the library commit advances to 2
|
||
* and the callback records the advisory index 2. committed_lsn is already 2
|
||
* (durable barrier). This still exercises the full ACK→library→callback wiring. */
|
||
TEST_ASSERT_EQUAL_INT(0,
|
||
ha_raft_on_repl_ack(c, /*abs_peer_idx*/1, /*acked_lsn*/2, /*reply_term*/T));
|
||
TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.commit_index);
|
||
TEST_ASSERT_EQUAL_UINT64(2, c->commit.raft_commit_index); /* advisory recorded */
|
||
TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn));
|
||
|
||
/* The marker (LSN 2) is now in the committed prefix → the DERIVED committed
|
||
* config equals the appended marker's {tag, sha} (R7.7). */
|
||
TEST_ASSERT_EQUAL_INT(0, config_committed_marker(c, &got_tag, got_sha));
|
||
TEST_ASSERT_EQUAL_UINT64(tag, got_tag);
|
||
TEST_ASSERT_EQUAL_MEMORY(sha, got_sha, 32);
|
||
|
||
/* ── on_truncate_from ───────────────────────────────────────────────────
|
||
* The uncommitted suffix (the LSN 3 job entry appended above) is removed by
|
||
* truncate-from LSN 3, while the committed prefix (job LSN 1 + committed
|
||
* marker LSN 2) stays byte-stable (R3.2/R3.3). on_truncate_from is the
|
||
* callback under test; the committed marker derive must still hold after. */
|
||
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 3)); /* present */
|
||
TEST_ASSERT_EQUAL_INT(0, ha_cb_truncate_from_for_test(&c->ha, 3));
|
||
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(&c->persist, 3)); /* gone */
|
||
|
||
/* The committed prefix is intact: LSN 1 (job) + LSN 2 (marker) keep their
|
||
* terms, and the committed marker still derives to the same {tag, sha}. */
|
||
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 1));
|
||
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(&c->persist, 2));
|
||
TEST_ASSERT_EQUAL_INT(0, config_committed_marker(c, &got_tag, got_sha));
|
||
TEST_ASSERT_EQUAL_UINT64(tag, got_tag);
|
||
TEST_ASSERT_EQUAL_MEMORY(sha, got_sha, 32);
|
||
|
||
/* All five term-safety callbacks remain wired (non-NULL) through the flow. */
|
||
TEST_ASSERT_NOT_NULL(c->ha.raft.cbs->on_persist_entry);
|
||
TEST_ASSERT_NOT_NULL(c->ha.raft.cbs->on_get_entry_term);
|
||
TEST_ASSERT_NOT_NULL(c->ha.raft.cbs->on_send_append_entries);
|
||
TEST_ASSERT_NOT_NULL(c->ha.raft.cbs->on_truncate_from);
|
||
TEST_ASSERT_NOT_NULL(c->ha.raft.cbs->on_commit_advanced);
|
||
|
||
commit_engine_fini(&c->commit);
|
||
teardown_cluster(c);
|
||
printf("test_every_callback_and_marker_path_end_to_end: PASS\n");
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Suite entry point */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
void test_term_safety_adapter(void)
|
||
{
|
||
RUN_TEST(test_wal_entry_hdr_size_is_40);
|
||
RUN_TEST(test_set_term_stamps_append);
|
||
RUN_TEST(test_recover_mixed_real_and_legacy_terms);
|
||
RUN_TEST(test_entry_term_absent_lsn_is_zero);
|
||
RUN_TEST(test_entry_term_null_safe);
|
||
RUN_TEST(test_on_persist_entry_stamps_term_standby);
|
||
RUN_TEST(test_on_persist_entry_stamps_term_primary);
|
||
RUN_TEST(test_on_persist_entry_failure_propagates_no_ack);
|
||
RUN_TEST(test_on_get_entry_term_present_legacy_absent);
|
||
RUN_TEST(test_on_truncate_from_drops_suffix);
|
||
RUN_TEST(test_on_truncate_from_removes_all);
|
||
RUN_TEST(test_send_append_entries_one_entry_from_wal);
|
||
RUN_TEST(test_send_append_entries_heartbeat_zero_entries);
|
||
RUN_TEST(test_commit_advances_to_durable_majority_index_inert);
|
||
RUN_TEST(test_commit_index_record_is_not_a_ceiling);
|
||
RUN_TEST(test_commit_advanced_does_not_bypass_durable_cap);
|
||
RUN_TEST(test_own_term_gate_blocks_priorterm_ack_until_noop_durable);
|
||
RUN_TEST(test_own_term_gate_retained_no_deadlock);
|
||
RUN_TEST(test_on_commit_advanced_wired_records_advisory_index);
|
||
RUN_TEST(test_init_restores_recovered_floor);
|
||
RUN_TEST(test_init_empty_wal_zero_floor);
|
||
RUN_TEST(test_primary_log_append_stamps_term_and_assigns_lsn);
|
||
RUN_TEST(test_own_term_noop_records_live_term_via_library);
|
||
RUN_TEST(test_primary_log_append_non_leader_legacy_fallback);
|
||
RUN_TEST(test_intake_higher_term_heartbeat_adopts_term);
|
||
RUN_TEST(test_intake_divergent_tail_truncated_and_reconverged);
|
||
RUN_TEST(test_intake_logmismatch_rejected_no_false_convergence);
|
||
RUN_TEST(test_live_broadcast_receive_persists_leader_term);
|
||
RUN_TEST(test_catchup_applied_lsn_is_last_held_no_skip);
|
||
RUN_TEST(test_intake_crossterm_catchup_uses_leader_current_term);
|
||
RUN_TEST(test_intake_stale_leader_term_still_rejected);
|
||
RUN_TEST(test_repl_ack_currentterm_quorum_advances_commit);
|
||
RUN_TEST(test_repl_ack_priorterm_quorum_holds_commit);
|
||
RUN_TEST(test_repl_ack_staleterm_reply_dropped);
|
||
RUN_TEST(test_repl_ack_abs_peer_idx_endpoint_type_roundtrip);
|
||
RUN_TEST(test_repl_ack_heartbeat_standby_absolute_slot);
|
||
RUN_TEST(test_repl_ack_out_of_range_peer_rejected);
|
||
/* Task 5.1: every-callback + marker-path end-to-end integration. */
|
||
RUN_TEST(test_every_callback_and_marker_path_end_to_end);
|
||
}
|