lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-entity-snapshots): track in-flight snapshot transfer LSN per peer (task 4.1)

Browse source 
Adds snapshot_inflight_lsn[MAX_HA_CANDIDATES] to ha_state_t, alongside the
existing snapshot_inflight[] flag it mirrors exactly: set to the
checkpoint's LSN at ha_snapshot_send_dispatch before a transfer's first
byte goes out, cleared to 0 at every completion/failure path
(snapshot_lane_fn's normal completion, and all 3 early-return failure
sites: malloc failure, slowlane dedup-refuse, slowlane submit failure).

This is the foundation task 4.3 will consult from rotate_snapshots to
avoid deleting a checkpoint file a peer is still mid-transfer reading
from -- the one genuinely new retention guard this feature needs (the
existing write_snapshot-before-trim_wal_segments sequencing already
satisfies the WAL-trim side of Requirement 3).

Proven via a new gated unit test (test_ha_liveness.c) that observes the
tracked LSN mid-flight while the send body is parked at a test gate, not
just before/after -- confirms the lifecycle ordering, not just the
endpoints.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JDYFbrgyDHbvNYcxApKhrX
This commit is contained in:
Rongsong Shen 2026-07-01 07:26:44 +00:00
parent eac7bb8ad8
commit 8d1ee8b761
4 changed files with 108 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

25
src/controller/ha.c
View file

@ -2778,7 +2778,12 @@ static void snapshot_lane_fn(void *arg)
ha_snapshot_do_send(c, sx->ep, peer_idx);
/* Clear in-flight LAST so a concurrent heartbeat tick cannot enqueue a
* second snapshot for this peer until this one has fully finished. */
* second snapshot for this peer until this one has fully finished.
* Clear the tracked LSN (Req 3.1-3.3) alongside the flag — same
* lifecycle, one more field — so rotate_snapshots never sees a stale
* in-flight LSN for a transfer that has already completed. */
atomic_store_explicit(&c->ha.snapshot_inflight_lsn[peer_idx], 0,
memory_order_release);
atomic_store_explicit(&c->ha.snapshot_inflight[peer_idx], 0,
memory_order_release);
free(sx);
@ -2810,8 +2815,19 @@ static void ha_snapshot_send_dispatch(ha_state_t *ha, cluster_t *c,
return;
}
/* Record the checkpoint position this transfer references (Req 3.1-3.3)
* at the SAME point the flag above is set — before the first byte of the
* transfer goes out — so a reader that observes snapshot_inflight[peer_idx]
* == 1 can find a meaningful LSN alongside it. rotate_snapshots (task 4.3)
* consults this to avoid deleting the checkpoint file this transfer is
* still reading from. */
atomic_store_explicit(&ha->snapshot_inflight_lsn[peer_idx],
c->persist.checkpoint_lsn, memory_order_release);
snapshot_send_ctx_t *sx = malloc(sizeof(*sx));
if (!sx) {
atomic_store_explicit(&ha->snapshot_inflight_lsn[peer_idx], 0,
memory_order_release);
atomic_store_explicit(&ha->snapshot_inflight[peer_idx], 0,
memory_order_release);
return;
@ -2827,6 +2843,8 @@ static void ha_snapshot_send_dispatch(ha_state_t *ha, cluster_t *c,
* refused — release the flag we set so no clear is lost, and let the
* next tick retry. */
free(sx);
atomic_store_explicit(&ha->snapshot_inflight_lsn[peer_idx], 0,
memory_order_release);
atomic_store_explicit(&ha->snapshot_inflight[peer_idx], 0,
memory_order_release);
return;
@ -2838,11 +2856,14 @@ static void ha_snapshot_send_dispatch(ha_state_t *ha, cluster_t *c,
WARN("HA repl: slowlane_submit snapshot peer[%d] failed: %d — retry "
"next cycle", peer_idx, srv);
free(sx);
atomic_store_explicit(&ha->snapshot_inflight_lsn[peer_idx], 0,
memory_order_release);
atomic_store_explicit(&ha->snapshot_inflight[peer_idx], 0,
memory_order_release);
return;
}
/* 0 or -EAGAIN: enqueued exactly once; snapshot_lane_fn clears the flag. */
/* 0 or -EAGAIN: enqueued exactly once; snapshot_lane_fn clears the flag
* and the tracked LSN. */
}
/* ───────────────────────────────────────────────────────────────────────

19
src/controller/ha.h
View file

@ -300,6 +300,25 @@ typedef struct {
* set, the heartbeat decision loop /
* promotion enqueues no duplicate config
* push. 0 at init (ha_init memset). */
/* ── New field added by ha-entity-snapshots spec (Task 4.1) ──────── */
_Atomic uint64_t snapshot_inflight_lsn[MAX_HA_CANDIDATES]; /* Per-peer
* checkpoint LSN referenced by an
* in-flight snapshot transfer (Req
* 3.1-3.3): set in
* ha_snapshot_send_dispatch at the
* SAME point snapshot_inflight[peer_idx]
* is set to 1 (before dispatch),
* cleared at the same points the flag
* is cleared to 0 (on completion or
* CAS-release failure) — identical
* lifecycle, one more field. Consulted
* by rotate_snapshots (task 4.3) to
* skip deleting a checkpoint file a
* peer transfer is still reading from.
* Explicitly zeroed on every clear so
* a stale value can never be mistaken
* for a live in-flight LSN. 0 at
* init (ha_init memset). */
} ha_state_t;
/* ── Public API ────────────────────────────────────────────── */

64
src/controller/testsuite/test_ha_liveness.c
View file

@ -1969,6 +1969,68 @@ static void test_snapshot_offload_dedups_per_peer(void)
printf("test_snapshot_offload_dedups_per_peer: PASS\n");
}
/* ───────────────────────────────────────────────────────────────────────
* ha-entity-snapshots Task 4.1 (Req 3.1-3.3): the in-flight-LSN tracking
* array must be set to the checkpoint's position BEFORE a transfer's first
* byte goes out, and cleared once the transfer completes -- proven with the
* same gate mechanism the dedup test above uses to observe the tracker
* mid-flight (parked), not just before/after.
* ─────────────────────────────────────────────────────────────────────── */
static void test_snapshot_inflight_lsn_set_before_send_cleared_after(void)
{
cluster_t c;
memset(&c, 0, sizeof(c));
ha_init(&c.ha, &g_cfg, 0);
TEST_ASSERT_EQUAL_INT(0, slowlane_init(&c.slowlane, 2, 4));
c.persist.checkpoint_lsn = 42;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
snap_reset_probe();
atomic_store(&g_snap_gate_on, 1); /* lane body parks at the gate */
ha_repl_send_snapshot_hook = snap_probe;
ha_set_snapshot_offload_for_test(1);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(0,
atomic_load_explicit(&c.ha.snapshot_inflight_lsn[0], memory_order_acquire),
"Req 3.1-3.3: no transfer in flight yet -- tracked LSN must be unset");
TEST_ASSERT_EQUAL_INT(0, snap_dispatch_once(&c, &ep, 0));
TEST_ASSERT_EQUAL_INT_MESSAGE(0, snap_wait_parked(2000),
"the snapshot body must reach the gate (running on the lane)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(42,
atomic_load_explicit(&c.ha.snapshot_inflight_lsn[0], memory_order_acquire),
"Req 3.1-3.3: while the transfer is in flight (parked before its "
"first byte is released), the tracked LSN must equal the checkpoint "
"position this transfer references");
snap_release_gate();
TEST_ASSERT_EQUAL_INT_MESSAGE(1, snap_wait_runs(1, 2000),
"the parked snapshot body must complete after release");
/* Bounded poll: snapshot_lane_fn clears the LSN then the flag after the
* send body returns -- give it a moment to finish its own cleanup. */
uint64_t cleared = 1;
for (int i = 0; i < 2000; i++) {
cleared = atomic_load_explicit(&c.ha.snapshot_inflight_lsn[0],
memory_order_acquire);
if (cleared == 0) break;
struct timespec ts = { .tv_sec = 0, .tv_nsec = 1000000L };
nanosleep(&ts, NULL);
}
TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, cleared,
"Req 3.1-3.3: once the transfer completes, the tracked LSN must be "
"cleared so rotate_snapshots never sees a stale in-flight LSN");
ha_set_snapshot_offload_for_test(0); /* restore production default (OFF) */
ha_repl_send_snapshot_hook = NULL;
slowlane_fini(&c.slowlane);
ha_fini(&c.ha);
printf("test_snapshot_inflight_lsn_set_before_send_cleared_after: PASS\n");
}
/* ───────────────────────────────────────────────────────────────────────
* Task 3.3 — requester-less ConfigPush slow-lane offload (Req 1.1, 1.5; bug-1198).
*
@ -2214,6 +2276,8 @@ void test_ha_liveness(void)
RUN_TEST(test_snapshot_offload_runs_on_lane);
RUN_TEST(test_snapshot_inline_when_flag_off);
RUN_TEST(test_snapshot_offload_dedups_per_peer);
/* ha-entity-snapshots Task 4.1 — in-flight snapshot LSN tracking (Req 3.1-3.3) */
RUN_TEST(test_snapshot_inflight_lsn_set_before_send_cleared_after);
/* raft-drive-async Task 3.3 — requester-less ConfigPush slow-lane offload (Req 1.1, 1.5, 6.x) */
RUN_TEST(test_configpush_offload_runs_on_lane);
RUN_TEST(test_configpush_inline_when_flag_off);