feat(ha-commit-durability): WAL recovery integrity verdict (CLEAN/TRUNCATED) + recovery_truncated flag (task 3.1)
This commit is contained in:
parent
526bca9679
commit
dd4d70863b
4 changed files with 338 additions and 7 deletions
|
|
@ -1,4 +1,6 @@
|
|||
#define _GNU_SOURCE
|
||||
#include <fcntl.h>
|
||||
#include <inttypes.h>
|
||||
#include <pthread.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdio.h>
|
||||
|
|
@ -11,6 +13,7 @@
|
|||
#include "commit.h"
|
||||
#include "config.h"
|
||||
#include "persist.h"
|
||||
#include "wal_types.h"
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* ha-commit-durability — task 1.2: durability-verification test seam *
|
||||
|
|
@ -775,6 +778,218 @@ static void test_commit_never_exceeds_durable_invariant(void)
|
|||
printf("test_commit_never_exceeds_durable_invariant: PASS\n");
|
||||
}
|
||||
|
||||
/* ================================================================== *
|
||||
* task 3.1 — recovery integrity verdict (Req 2.1) *
|
||||
* *
|
||||
* persist_recover/replay_wal must distinguish a CLEAN log end (EOF, *
|
||||
* pre-allocated-zero tail, snapshot-sentinel + clean replay, empty *
|
||||
* first-boot WAL) from a corruption-TRUNCATED end (CRC chain break, *
|
||||
* CRC mismatch, short/partial entry read). A corruption-truncated *
|
||||
* recovery is NEVER reported as a clean recovery and sets a *
|
||||
* member-level recovery_truncated flag the promotion gate (4.3) *
|
||||
* consumes. 3.1 only SETS the verdict/flag — it does NOT FATAL and *
|
||||
* does NOT fence (3.3 / 4.3 own those). *
|
||||
* *
|
||||
* NB the verdict is about STRUCTURAL integrity of the local log, not *
|
||||
* about being behind the cluster: a clean log that ends below the *
|
||||
* cluster high-water is still CLEAN (the freshness/promotion fence *
|
||||
* catches "behind", not this verdict). *
|
||||
* ------------------------------------------------------------------ */
|
||||
|
||||
/* The persist paths derive as <workdir>/<name>/data/wal; make_durability_persist
|
||||
* uses name "ctrl-test" (see persist_set_workdir call above). */
|
||||
#define DUR_WAL_SEG_FMT "%s/ctrl-test/data/wal/seg-%016" PRIx64 ".wal"
|
||||
|
||||
/* Open the first WAL segment (starts at LSN 1; LSN 0 reserved) for the
|
||||
* given tmpdir and return its fd, or -1. */
|
||||
static int open_first_seg_rw(const char* tmpdir)
|
||||
{
|
||||
char seg_path[512];
|
||||
/* The first-ever entry is LSN 1, so the first segment is seg-...0001. */
|
||||
snprintf(seg_path, sizeof(seg_path), DUR_WAL_SEG_FMT, tmpdir,
|
||||
(uint64_t)1);
|
||||
return open(seg_path, O_RDWR);
|
||||
}
|
||||
|
||||
/* Open a SECOND persist instance over the same on-disk WAL with NO
|
||||
* in-memory writer state, run recovery, and return the recovery verdict
|
||||
* (via persist_recovery_truncated). The writer must already be torn down
|
||||
* so the segments are closed and on disk. */
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* Test 3.1a: an intact log yields RECOVER_CLEAN (flag NOT set). *
|
||||
* ------------------------------------------------------------------ */
|
||||
static void test_recover_verdict_clean_intact_log(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_recclean_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
/* Writer: append + flush three entries, then "die". */
|
||||
{
|
||||
persist_t* p = make_durability_persist(tmp);
|
||||
TEST_ASSERT_NOT_NULL(p);
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
|
||||
teardown_durability_persist(p);
|
||||
}
|
||||
|
||||
/* Recover into a fresh instance: an intact CRC chain must be CLEAN. */
|
||||
{
|
||||
persist_t* p2 = make_durability_persist(tmp);
|
||||
TEST_ASSERT_NOT_NULL(p2);
|
||||
int rc = persist_recover(p2);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
RECOVER_CLEAN, persist_recovery_verdict(p2),
|
||||
"an intact CRC chain must recover CLEAN");
|
||||
TEST_ASSERT_FALSE_MESSAGE(
|
||||
persist_recovery_truncated(p2),
|
||||
"a clean recovery must NOT set the recovery_truncated flag");
|
||||
/* current_lsn advanced past the three replayed entries. */
|
||||
TEST_ASSERT_EQUAL_UINT64(4, p2->current_lsn);
|
||||
teardown_durability_persist(p2);
|
||||
}
|
||||
printf("test_recover_verdict_clean_intact_log: PASS\n");
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* Test 3.1b: an empty first-boot WAL is CLEAN, not corruption. *
|
||||
* ------------------------------------------------------------------ */
|
||||
static void test_recover_verdict_empty_first_boot_clean(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_recempty_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
persist_t* p = make_durability_persist(tmp);
|
||||
TEST_ASSERT_NOT_NULL(p);
|
||||
|
||||
/* No appends: a first-boot WAL has nothing to replay — that is a clean
|
||||
* (empty) recovery, never a corruption-truncated one. */
|
||||
int rc = persist_recover(p);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
RECOVER_CLEAN, persist_recovery_verdict(p),
|
||||
"an empty first-boot WAL must be CLEAN (no entries != corruption)");
|
||||
TEST_ASSERT_FALSE(persist_recovery_truncated(p));
|
||||
|
||||
teardown_durability_persist(p);
|
||||
printf("test_recover_verdict_empty_first_boot_clean: PASS\n");
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* Test 3.1c (core RED): a CRC break injected at the FIRST entry yields *
|
||||
* RECOVER_TRUNCATED and SETS the recovery_truncated flag. *
|
||||
* *
|
||||
* We append+flush two entries, then corrupt the on-disk `crc` field of *
|
||||
* the FIRST entry (LSN 1). On recovery the CRC verification at LSN 1 *
|
||||
* fails, replay stops before any clean boundary, and the verdict must *
|
||||
* be TRUNCATED — never silently accepted as a complete clean recovery. *
|
||||
* ------------------------------------------------------------------ */
|
||||
static void test_recover_verdict_crc_break_first_entry_truncated(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_recbreak_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
/* Writer: two durable entries, then die. */
|
||||
{
|
||||
persist_t* p = make_durability_persist(tmp);
|
||||
TEST_ASSERT_NOT_NULL(p);
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
|
||||
teardown_durability_persist(p);
|
||||
}
|
||||
|
||||
/* Corrupt the FIRST entry's CRC field on disk: read its header at
|
||||
* offset 0, flip the crc, write it back. This breaks the chain at
|
||||
* LSN 1 — replay cannot reach any clean boundary. */
|
||||
{
|
||||
int fd = open_first_seg_rw(tmp);
|
||||
TEST_ASSERT_TRUE_MESSAGE(fd >= 0, "first WAL segment must exist");
|
||||
wal_entry_hdr_t hdr;
|
||||
TEST_ASSERT_EQUAL_INT((int)sizeof(hdr),
|
||||
(int)pread(fd, &hdr, sizeof(hdr), 0));
|
||||
hdr.crc ^= 0xDEADBEEFu; /* corrupt the entry CRC */
|
||||
TEST_ASSERT_EQUAL_INT((int)sizeof(hdr),
|
||||
(int)pwrite(fd, &hdr, sizeof(hdr), 0));
|
||||
fsync(fd);
|
||||
close(fd);
|
||||
}
|
||||
|
||||
/* Recover: the corruption at LSN 1 must yield TRUNCATED + set the flag. */
|
||||
{
|
||||
persist_t* p2 = make_durability_persist(tmp);
|
||||
TEST_ASSERT_NOT_NULL(p2);
|
||||
int rc = persist_recover(p2);
|
||||
/* persist_recover still returns 0 (no I/O error); the VERDICT is the
|
||||
* signal — 3.1 classifies, it does not FATAL (that is task 3.3). */
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
RECOVER_TRUNCATED, persist_recovery_verdict(p2),
|
||||
"a CRC break at the first entry must yield RECOVER_TRUNCATED");
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
persist_recovery_truncated(p2),
|
||||
"a corruption-truncated recovery must SET the recovery_truncated "
|
||||
"flag (consumed by the promotion gate, task 4.3)");
|
||||
teardown_durability_persist(p2);
|
||||
}
|
||||
printf("test_recover_verdict_crc_break_first_entry_truncated: PASS\n");
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* Test 3.1d (snapshot-sentinel edge): a member that recovered from a *
|
||||
* 16-byte LSN-sentinel snapshot has a committed position at the *
|
||||
* sentinel LSN but NO log entries below it. A clean replay over that *
|
||||
* sentinel snapshot must classify CLEAN, never TRUNCATED — the absence *
|
||||
* of sub-sentinel entries is normal, not corruption (design Open *
|
||||
* Questions / snapshot-recovery edge). *
|
||||
* *
|
||||
* We checkpoint (writes the sentinel snapshot + trims the WAL), append *
|
||||
* + flush one fresh entry above the checkpoint, then recover in a fresh *
|
||||
* instance: snapshot-sentinel + clean replay ⇒ CLEAN. *
|
||||
* ------------------------------------------------------------------ */
|
||||
static void test_recover_verdict_snapshot_sentinel_clean(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_recsnap_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
/* Writer: a couple of entries, a checkpoint (sentinel snapshot), then a
|
||||
* further entry above the checkpoint, all flushed; then die. */
|
||||
{
|
||||
persist_t* p = make_durability_persist(tmp);
|
||||
TEST_ASSERT_NOT_NULL(p);
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
|
||||
/* Sentinel snapshot at the current committed/checkpoint position. */
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
|
||||
teardown_durability_persist(p);
|
||||
}
|
||||
|
||||
/* Recover: load_snapshot sets checkpoint_lsn from the sentinel, replay
|
||||
* picks up entries above it cleanly — verdict CLEAN, flag NOT set. */
|
||||
{
|
||||
persist_t* p2 = make_durability_persist(tmp);
|
||||
TEST_ASSERT_NOT_NULL(p2);
|
||||
int rc = persist_recover(p2);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
RECOVER_CLEAN, persist_recovery_verdict(p2),
|
||||
"snapshot-sentinel + clean replay must be CLEAN, not TRUNCATED");
|
||||
TEST_ASSERT_FALSE(persist_recovery_truncated(p2));
|
||||
teardown_durability_persist(p2);
|
||||
}
|
||||
printf("test_recover_verdict_snapshot_sentinel_clean: PASS\n");
|
||||
}
|
||||
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
|
@ -797,6 +1012,10 @@ void test_commit_durability(void)
|
|||
RUN_TEST(test_legacy_sync_append_holds_committed_at_durable);
|
||||
RUN_TEST(test_legacy_sync_append_lsn0_edge_valid);
|
||||
RUN_TEST(test_commit_never_exceeds_durable_invariant);
|
||||
RUN_TEST(test_recover_verdict_clean_intact_log);
|
||||
RUN_TEST(test_recover_verdict_empty_first_boot_clean);
|
||||
RUN_TEST(test_recover_verdict_crc_break_first_entry_truncated);
|
||||
RUN_TEST(test_recover_verdict_snapshot_sentinel_clean);
|
||||
#else
|
||||
printf("test_commit_durability: skipped (no XCE_TESTING)\n");
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue