lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-term-safety): WAL term seam + recovery read-back + legacy-0 rule (task 1.3)

Browse source 
persist_set_term sets the live election term so appends stamp hdr.term (fixes the
G1 term-0 stamping); replay_wal builds an in-memory per-LSN term index (freed in
persist_fini, no on-disk change); persist_entry_term reads it back (0=absent).
Legacy-0 -> WAL_LEGACY_TERM=1 at read-back only (raw 0 retained so present-legacy-0
is distinguishable from absent). 40-byte wal_entry_hdr_t preserved (value fix, R1.7).
5 new tests (test_term_safety_adapter.c); reviewer APPROVED. Feeds task 2.2.

All controller test groups report 0 failures; the meson full-suite timeout is the
bug-924 overlayfs /tmp fdatasync artifact, not this task.
This commit is contained in:
Rongsong Shen 2026-06-22 04:32:29 +00:00
parent a95d57f733
commit d540ebd183
6 changed files with 420 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/controller/meson.build
View file

@ -138,6 +138,7 @@ controller_unit_test_src = [
'testsuite/test_ctrlstatus.c',
'testsuite/test_ha_heartbeat_committed.c',
'testsuite/test_ha_vote_floor_heartbeat.c',
'testsuite/test_term_safety_adapter.c',
'testsuite/test_waldump.c',
'testsuite/waldump.c',
'ctrlstatus.c',

127
src/controller/persist.c
View file

@ -392,6 +392,90 @@ static void* fsync_worker(void* arg)
return NULL;
}
/* ------------------------------------------------------------------ *
* WAL term adapter (ha-term-safety task 1.3, R1.1/R1.5/R1.6/R1.7/R1.8) *
* ------------------------------------------------------------------ */
/* Record an entry's RAW hdr.term at its 1-based LSN in the in-memory per-LSN
* term index (R1.5). Grows the index on demand; LSNs replay/append contiguously
* so the array is dense (slot i == LSN i+1). Any gap left by a non-contiguous
* LSN (should not happen on the contiguous WAL) is zero-filled by realloc-then-
* memset, which read back as "absent" until/unless written. The raw term
* (including a legacy 0) is stored verbatim; the legacy-0 -> WAL_LEGACY_TERM
* mapping is applied only at READ-BACK (persist_entry_term), keeping "present
* legacy-0" distinguishable from "absent".
*
* Callers: the live append path (holds p->lock) and the single-threaded
* recovery path (replay_wal — no other thread touches the index before the node
* participates, so no lock is required there). Best-effort: an allocation
* failure is logged loudly and leaves the index unextended (read-back then
* reports the LSN absent) — it never aborts and never corrupts existing
* entries. */
static void record_entry_term(persist_t* p, uint64_t lsn, uint64_t term)
{
if (!p || lsn == 0)
return; /* LSN 0 is reserved — never an entry */
if (lsn > p->term_index_cap) {
/* Grow geometrically to amortize; never below the requested LSN. */
uint64_t new_cap = p->term_index_cap ? p->term_index_cap * 2 : 64;
if (new_cap < lsn)
new_cap = lsn;
uint64_t* grown = realloc(p->term_index, new_cap * sizeof(uint64_t));
if (!grown) {
ERROR("persist: term-index realloc failed (lsn=%" PRIu64
", cap=%" PRIu64 "->%" PRIu64 "); term for this LSN will read "
"back as absent", lsn, p->term_index_cap, new_cap);
return;
}
/* Zero-fill the newly grown region so any unwritten slot reads absent. */
memset(grown + p->term_index_cap, 0,
(new_cap - p->term_index_cap) * sizeof(uint64_t));
p->term_index = grown;
p->term_index_cap = new_cap;
}
p->term_index[lsn - 1] = term;
if (lsn > p->term_index_count)
p->term_index_count = lsn;
}
void persist_set_term(persist_t* p, uint64_t term)
{
if (!p)
return;
/* p->term is read on the append path under p->lock; take the lock so the
* stamped value is consistent with the append that reads it. */
pthread_mutex_lock(&p->lock);
p->term = term;
pthread_mutex_unlock(&p->lock);
}
uint64_t persist_entry_term(const persist_t* p, uint64_t lsn)
{
if (!p || lsn == 0)
return 0;
/* Take p->lock to read consistently against concurrent record_entry_term
* on the append path. Cast away const — the lock is logically a mutable
* sub-object of an otherwise-const read. */
persist_t* mp = (persist_t*)p;
pthread_mutex_lock(&mp->lock);
uint64_t result = 0;
if (lsn <= mp->term_index_count && mp->term_index != NULL) {
uint64_t raw = mp->term_index[lsn - 1];
/* Legacy-zero rule (R1.8): a present entry carrying term 0 maps to the
* single lowest-known real Raft term, applied uniformly so two legacy-0
* entries at the same LSN match. Absent LSNs are caught by the bound
* above and return 0. */
result = (raw == 0) ? WAL_LEGACY_TERM : raw;
}
pthread_mutex_unlock(&mp->lock);
return result;
}
/* ------------------------------------------------------------------ */
/* Public API */
/* ------------------------------------------------------------------ */
@ -460,6 +544,13 @@ int persist_init(persist_t* p, persist_config_t* cfg, cluster_t* c)
p->on_durable_advance = NULL;
p->durable_cb_ctx = NULL;
/* Per-LSN term index starts empty (ha-term-safety 1.3, R1.5): no entries
* recorded yet. Grown on demand by record_entry_term during replay/append;
* freed in persist_fini. */
p->term_index = NULL;
p->term_index_count = 0;
p->term_index_cap = 0;
/* Create WAL and snapshot directories */
if (mkdir_p(p->wal_path, 0700) != 0) return -errno;
if (mkdir_p(p->snap_path, 0700) != 0) return -errno;
@ -560,6 +651,12 @@ void persist_fini(persist_t* p)
pthread_cond_destroy(&p->checkpoint_done);
pthread_mutex_destroy(&p->fsync_mutex);
pthread_cond_destroy(&p->fsync_cond);
/* Free the in-memory per-LSN term index (ha-term-safety 1.3). */
free(p->term_index);
p->term_index = NULL;
p->term_index_count = 0;
p->term_index_cap = 0;
}
/* Forward declaration — defined after checkpoint internals */
@ -621,6 +718,10 @@ int persist_wal_async(persist_t* p, uint32_t type,
p->total_wal_bytes += entry_size;
p->last_crc = hdr.crc;
/* Record this entry's term at its LSN for recovery read-back (ha-term-safety
* 1.3, R1.5). Under p->lock; stamps the same hdr.term written to disk. */
record_entry_term(p, hdr.lsn, hdr.term);
/* Replicate to standbys — never replicate WAL_CHECKPOINT entries.
* last_replicated_crc is distinct from last_crc:
* last_crc includes checkpoint markers; last_replicated_crc skips them.
@ -757,6 +858,14 @@ int persist_wal_write_replicated(persist_t* p, uint64_t lsn, uint32_t type,
p->total_wal_bytes += entry_size;
p->last_crc = hdr.crc;
/* Record this entry's term at its LSN for recovery read-back (ha-term-safety
* 1.3, R1.5). Stamps the same hdr.term written to disk. NOTE: the streamed
* envelope does not yet carry the leader's term, so this records the local
* best-effort p->term (the design's known hazard closed by task 2.x, which
* carries the leader's term on the envelope; this task only provides the
* read-back seam over whatever term was persisted). */
record_entry_term(p, hdr.lsn, hdr.term);
/* Advance current_lsn to one past the given lsn (max-idempotent — mirrors
* apply_wal_entry's monotonic advance). */
if (lsn + 1 > p->current_lsn)
@ -1058,6 +1167,11 @@ static int persist_write_checkpoint_marker(persist_t* p)
p->total_wal_bytes += sizeof(hdr);
p->last_crc = hdr.crc;
/* NOTE: last_replicated_crc is NOT updated — WAL_CHECKPOINT is never replicated */
/* NOTE: the term index is NOT updated for WAL_CHECKPOINT — replay_wal SKIPS
* checkpoint markers (local bookkeeping, not a logical Raft entry), so the
* recovered index never holds a checkpoint LSN; the live index must match
* (ha-term-safety 1.3). persist_entry_term is for (term,lsn) reconcile, which
* never compares against a checkpoint marker. */
pthread_mutex_unlock(&p->lock);
return 0;
@ -1583,6 +1697,19 @@ static int replay_wal(persist_t* p, uint64_t from_lsn,
/* Advance segment-local CRC */
last_crc = hdr.crc;
/* Record this entry's term at its LSN for recovery read-back
* (ha-term-safety 1.3, R1.5). Record EVERY CRC-validated logical
* entry — including snapshot-covered ones below from_lsn — so the
* index spans the full LSN range the reconcile/promotion logic may
* query. EXCLUDE WAL_CHECKPOINT markers (type == WAL_CHECKPOINT) and
* unknown future types (type > WAL_CHECKPOINT): they are skipped
* below and are never logical Raft entries, so the live append path
* does not record them either (kept symmetric). The raw hdr.term is
* stored verbatim (legacy 0 included); the legacy-0 -> WAL_LEGACY_TERM
* mapping is applied only at read-back. */
if (hdr.type < WAL_CHECKPOINT)
record_entry_term(p, hdr.lsn, hdr.term);
/* Skip entries already covered by the snapshot */
if (hdr.lsn <= from_lsn) {
free(payload);

2
src/controller/testsuite/controller_test.c
View file

@ -63,6 +63,7 @@ extern void test_commit_durability(void);
extern void test_ctrlstatus(void);
extern void test_ha_heartbeat_committed(void);
extern void test_ha_vote_floor_heartbeat(void);
extern void test_term_safety_adapter(void);
extern void test_waldump(void);
void setUp(void) { return; }
@ -196,6 +197,7 @@ int main(int argc, char* argv[])
TEST(test_ctrlstatus);
TEST(test_ha_heartbeat_committed);
TEST(test_ha_vote_floor_heartbeat);
TEST(test_term_safety_adapter);
TEST(test_waldump);
return UNITY_END();

229
src/controller/testsuite/test_term_safety_adapter.c Normal file
View file

@ -0,0 +1,229 @@
#define _GNU_SOURCE
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <unistd.h>
#include <unity/unity.h>
#include "config.h"
#include "persist.h"
#include "wal_types.h"
/* ha-term-safety task 1.3 — WAL term seam + recovery read-back + legacy-0 rule.
*
* The seam under test:
* - persist_set_term(p, T) sets the controller's live election term so a
* subsequent append stamps hdr.term = T (R1.1, R1.6).
* - replay_wal retains each entry's hdr.term per LSN in an in-memory index
* (R1.5), read back via persist_entry_term(p, lsn).
* - persist_entry_term returns 0 for an absent LSN; a legacy entry carrying
* hdr.term == 0 maps to the single lowest-known real Raft term (1) so a
* legacy-0 entry is never spuriously divergent from another legacy-0 entry
* and recovery never fails on legacy-0 (R1.8).
* - the on-disk wal_entry_hdr_t stays 40 bytes (R1.7).
*/
/* The defined legacy-0 mapping (lowest real Raft term). Kept local to the test
* so the assertion is independent of the implementation constant. */
#define LEGACY_LOWEST_TERM 1u
/* ------------------------------------------------------------------ */
/* Helpers */
/* ------------------------------------------------------------------ */
static persist_t* make_fresh_persist(const char* tmpdir)
{
persist_t* p = calloc(1, sizeof(persist_t));
if (!p) return NULL;
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000; /* slow fsync — no races in tests */
cfg.snap_keep_n = 2;
persist_set_workdir(p, tmpdir, "ctrl-test");
if (persist_init(p, &cfg, NULL) != 0) {
free(p);
return NULL;
}
if (persist_start(p) != 0) {
persist_fini(p);
free(p);
return NULL;
}
return p;
}
static void teardown_persist(persist_t* p)
{
persist_stop(p);
persist_fini(p);
free(p);
}
/* ------------------------------------------------------------------ */
/* Test: header layout preserved (R1.7) — value fix, not a format */
/* change. The on-disk WAL entry header must stay exactly 40 bytes. */
/* ------------------------------------------------------------------ */
static void test_wal_entry_hdr_size_is_40(void)
{
TEST_ASSERT_EQUAL_UINT64(40u, (uint64_t)sizeof(wal_entry_hdr_t));
printf("test_wal_entry_hdr_size_is_40: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Test: persist_set_term makes a subsequent append stamp hdr.term */
/* (R1.1, R1.6), recovered back via persist_entry_term. */
/* ------------------------------------------------------------------ */
static void test_set_term_stamps_append(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_term_set_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
const uint64_t T = 7;
/* Phase 1: set the live term, then append — the append stamps hdr.term=T. */
{
persist_t* p = make_fresh_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
persist_set_term(p, T);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
/* Live read-back of the just-appended entry's term (LSN 1). */
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(p, 1));
teardown_persist(p);
}
/* Phase 2: fresh persist_t recovers the WAL and reads the term back. */
{
persist_t* p2 = make_fresh_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
TEST_ASSERT_EQUAL_INT(0, persist_recover(p2));
TEST_ASSERT_EQUAL_UINT64(T, persist_entry_term(p2, 1));
teardown_persist(p2);
}
printf("test_set_term_stamps_append: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Test: replay_wal of a WAL with mixed real-term and legacy-0 */
/* entries — persist_entry_term returns the real term for real */
/* entries and the defined lowest-known term for legacy-0 (R1.5, */
/* R1.8). Recovery never fails on legacy-0 entries. */
/* ------------------------------------------------------------------ */
static void test_recover_mixed_real_and_legacy_terms(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_term_mixed_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Phase 1: write LSN 1,2 with term 0 (legacy — the pre-feature default),
* then LSN 3,4 with real term 5. The append path stamps hdr.term=p->term,
* so leaving the term unset (0) produces genuine legacy-0 entries. */
{
persist_t* p = make_fresh_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Legacy-0 entries: no persist_set_term call (p->term stays 0). */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "l", 1)); /* lsn 1 */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "m", 1)); /* lsn 2 */
/* Real-term entries. */
persist_set_term(p, 5);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "r", 1)); /* lsn 3 */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "s", 1)); /* lsn 4 */
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
teardown_persist(p);
}
/* Phase 2: fresh recover — read back each LSN's term. */
{
persist_t* p2 = make_fresh_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
/* Recovery must NOT fail on the legacy-0 entries. */
TEST_ASSERT_EQUAL_INT(0, persist_recover(p2));
TEST_ASSERT_EQUAL_INT(RECOVER_CLEAN, persist_recovery_verdict(p2));
/* Legacy-0 entries map to the single lowest-known real term (1) — both
* the same, so two legacy-0 entries are never seen as divergent. */
TEST_ASSERT_EQUAL_UINT64(LEGACY_LOWEST_TERM, persist_entry_term(p2, 1));
TEST_ASSERT_EQUAL_UINT64(LEGACY_LOWEST_TERM, persist_entry_term(p2, 2));
/* Real-term entries return their recorded term verbatim. */
TEST_ASSERT_EQUAL_UINT64(5, persist_entry_term(p2, 3));
TEST_ASSERT_EQUAL_UINT64(5, persist_entry_term(p2, 4));
teardown_persist(p2);
}
printf("test_recover_mixed_real_and_legacy_terms: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Test: persist_entry_term returns 0 for an absent LSN (no entry). */
/* Distinguishes "absent" (0) from a present legacy-0 entry (mapped */
/* to lowest-known), which is the precision REVIEW ISSUE 2 requires. */
/* ------------------------------------------------------------------ */
static void test_entry_term_absent_lsn_is_zero(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_term_absent_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_fresh_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
persist_set_term(p, 3);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1)); /* lsn 1 */
/* LSN 0 is reserved (never an entry) and LSN 2 was never appended. */
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(p, 0));
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(p, 2));
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(p, 999999));
/* The present entry still answers its real term. */
TEST_ASSERT_EQUAL_UINT64(3, persist_entry_term(p, 1));
teardown_persist(p);
printf("test_entry_term_absent_lsn_is_zero: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Test: persist_entry_term is NULL-safe. */
/* ------------------------------------------------------------------ */
static void test_entry_term_null_safe(void)
{
TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(NULL, 1));
/* persist_set_term must not crash on NULL. */
persist_set_term(NULL, 9);
printf("test_entry_term_null_safe: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Suite entry point */
/* ------------------------------------------------------------------ */
void test_term_safety_adapter(void)
{
RUN_TEST(test_wal_entry_hdr_size_is_40);
RUN_TEST(test_set_term_stamps_append);
RUN_TEST(test_recover_mixed_real_and_legacy_terms);
RUN_TEST(test_entry_term_absent_lsn_is_zero);
RUN_TEST(test_entry_term_null_safe);
}