feat(ha-convergence-recovery): ACK-driven catch-up continuation + parity re-inclusion (task 4.2)
This commit is contained in:
parent
c2221077ff
commit
ffd08c6472
5 changed files with 243 additions and 22 deletions
|
|
@ -1651,6 +1651,201 @@ int ha_stream_wal_to_standby(endpoint_t *ep, const char *wal_path,
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* ── ACK-driven catch-up continuation (ha-convergence-recovery task 4.2) ──────
|
||||
*
|
||||
* Design D6 (Req 5.1, 5.2, 7.1). The walAck branch of
|
||||
* repl_ack_decode_and_dispatch (ha.c, iomgr) calls ha_walack_on_ack after
|
||||
* repl_on_ack; the unit test drives the same logic through
|
||||
* ha_walack_continuation_for_test. Streaming reads WAL files, so it must never
|
||||
* run inline on iomgr — the decision schedules a one-shot delaymgr task that
|
||||
* does the streaming and clears the per-standby single-flight flag.
|
||||
*
|
||||
* This lives in config_repl.c (per design) alongside ha_stream_wal_to_standby:
|
||||
* the continuation wraps the existing streaming primitive with no change to its
|
||||
* internals.
|
||||
* ──────────────────────────────────────────────────────────────────────────── */
|
||||
|
||||
/* repl_ep_still_registered: is `ep` still an active standby slot? The ep may
|
||||
* have RDHUPed between schedule and run; on its disappearance the continuation
|
||||
* is a no-op (design Error Handling: clear the flag and return, no error). */
|
||||
static int repl_ep_still_registered(replicator_t *r, repl_ep_t ep)
|
||||
{
|
||||
int found = 0;
|
||||
mylock_lock(&r->lock);
|
||||
for (int i = 0; i < MAX_STANDBYS; i++) {
|
||||
if (r->standbys[i].ep == ep) {
|
||||
found = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
mylock_unlock(&r->lock);
|
||||
return found;
|
||||
}
|
||||
|
||||
/* re-derive the replication target exactly as ha.c:2771-2776 / D6:
|
||||
* target = max(committed_lsn, last_written), last_written = current_lsn - 1. */
|
||||
static uint64_t catchup_derive_target(cluster_t *c)
|
||||
{
|
||||
uint64_t committed = atomic_load_explicit(&c->persist.committed_lsn,
|
||||
memory_order_acquire);
|
||||
pthread_mutex_lock(&c->persist.lock);
|
||||
uint64_t last_written = c->persist.current_lsn > 0
|
||||
? c->persist.current_lsn - 1
|
||||
: 0;
|
||||
pthread_mutex_unlock(&c->persist.lock);
|
||||
return last_written > committed ? last_written : committed;
|
||||
}
|
||||
|
||||
/* Invoke the catch-up streaming path for [standby_lsn+1, target]. Mirrors the
|
||||
* HA_REPL_STREAM_WAL dispatch in ha.c: under XCE_TESTING the
|
||||
* ha_repl_stream_wal_hook seam intercepts so unit tests can count the stream
|
||||
* without a real endpoint or WAL directory; otherwise ha_stream_wal_to_standby
|
||||
* runs unchanged. */
|
||||
static void catchup_stream_range(cluster_t *c, repl_ep_t ep,
|
||||
uint64_t standby_lsn, uint64_t target)
|
||||
{
|
||||
#ifdef XCE_TESTING
|
||||
if (ha_repl_stream_wal_hook != NULL) {
|
||||
const char *wal_path = (c && c->persist.wal_path[0])
|
||||
? c->persist.wal_path : "";
|
||||
ha_repl_stream_wal_hook(ep, wal_path, standby_lsn, target);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
if (c && c->persist.wal_path[0])
|
||||
ha_stream_wal_to_standby((endpoint_t *)ep, c->persist.wal_path,
|
||||
standby_lsn, target);
|
||||
}
|
||||
|
||||
/* ha_catchup_continuation_ctx_t — heap context for the one-shot delaymgr task.
|
||||
* Carries the standby slot identity (peer_idx — stable schedule→run, indexes the
|
||||
* single-flight flag) and the ACKed LSN; target is re-derived at run time so the
|
||||
* stream extends to entries appended meanwhile (Req 5.1). */
|
||||
typedef struct {
|
||||
cluster_t *c;
|
||||
repl_ep_t ep;
|
||||
int peer_idx;
|
||||
uint64_t acked_lsn;
|
||||
} ha_catchup_continuation_ctx_t;
|
||||
|
||||
/*
|
||||
* ha_catchup_continuation_fn — delaymgr task body (runs OFF the iomgr walAck
|
||||
* branch, D6/D1). Re-validate the endpoint against the registry, re-derive
|
||||
* target, stream [acked_lsn+1, target], and clear catchup_inflight on EVERY exit
|
||||
* path (success, ep-gone, nothing-to-stream) so a stuck flag never blocks future
|
||||
* continuations (Req 5.2). Frees its own ctx (body-frees-own-ctx pattern,
|
||||
* mirroring ha_demotion_convergence_fn).
|
||||
*/
|
||||
int ha_catchup_continuation_fn(void *arg)
|
||||
{
|
||||
ha_catchup_continuation_ctx_t *ctx = (ha_catchup_continuation_ctx_t *)arg;
|
||||
if (ctx == NULL || ctx->c == NULL) {
|
||||
free(ctx);
|
||||
return -EINVAL;
|
||||
}
|
||||
cluster_t *c = ctx->c;
|
||||
repl_ep_t ep = ctx->ep;
|
||||
int peer_idx = ctx->peer_idx;
|
||||
uint64_t acked = ctx->acked_lsn;
|
||||
|
||||
/* Endpoint may have RDHUPed between schedule and run — no-op if gone. */
|
||||
if (!repl_ep_still_registered(&c->persist.repl, ep)) {
|
||||
INFO("HA catch-up continuation: standby[%d] endpoint gone before run "
|
||||
"— skipping (heartbeat fallback re-evaluates)", peer_idx);
|
||||
goto clear_and_done;
|
||||
}
|
||||
|
||||
uint64_t target = catchup_derive_target(c);
|
||||
if (acked < target) {
|
||||
INFO("HA catch-up continuation: standby[%d] acked=%" PRIu64
|
||||
" < target=%" PRIu64 " → streaming [%" PRIu64 "+1..%" PRIu64 "]",
|
||||
peer_idx, acked, target, acked, target);
|
||||
catchup_stream_range(c, ep, acked, target);
|
||||
}
|
||||
/* If acked >= target the standby reached parity meanwhile — nothing to
|
||||
* stream; parity re-inclusion is owned by the decision path, not here. */
|
||||
|
||||
clear_and_done:
|
||||
/* Req 5.2: release the single-flight slot so the next lagging ACK schedules
|
||||
* again. release order so the decision path's acquire load observes it. */
|
||||
if (peer_idx >= 0 && peer_idx < MAX_STANDBYS)
|
||||
atomic_store_explicit(&c->ha.catchup_inflight[peer_idx], 0,
|
||||
memory_order_release);
|
||||
free(ctx);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* ha_schedule_catchup_continuation — enqueue one one-shot delaymgr task running
|
||||
* ha_catchup_continuation_fn. The single-flight flag is set by the caller
|
||||
* (ha_walack_on_ack) before this call; on a scheduling failure this clears it so
|
||||
* the slot does not stay stuck. Returns 0 on success, negative on failure.
|
||||
*/
|
||||
static int ha_schedule_catchup_continuation(cluster_t *c, repl_ep_t ep,
|
||||
int peer_idx, uint64_t acked_lsn)
|
||||
{
|
||||
ha_catchup_continuation_ctx_t *ctx = calloc(1, sizeof(*ctx));
|
||||
if (ctx == NULL) {
|
||||
ERROR("HA catch-up continuation: ctx alloc failed for standby[%d]",
|
||||
peer_idx);
|
||||
atomic_store_explicit(&c->ha.catchup_inflight[peer_idx], 0,
|
||||
memory_order_release);
|
||||
return -ENOMEM;
|
||||
}
|
||||
ctx->c = c;
|
||||
ctx->ep = ep;
|
||||
ctx->peer_idx = peer_idx;
|
||||
ctx->acked_lsn = acked_lsn;
|
||||
|
||||
/* One-shot (runcount=1), minimal delay: streaming reads WAL files and must
|
||||
* run off the iomgr walAck thread. delay_task_ms only enqueues (ordq_put);
|
||||
* the delaymgr worker runs it. In XCE_TESTING the fixture drives the queued
|
||||
* task synchronously via its stored entry/args. */
|
||||
int rc = delay_task_ms(&c->delaymgr, 1, 1,
|
||||
(delay_task_entry_fn_t)ha_catchup_continuation_fn,
|
||||
ctx);
|
||||
if (rc < 0) {
|
||||
ERROR("HA catch-up continuation: delay_task_ms failed for standby[%d]:"
|
||||
" %d", peer_idx, rc);
|
||||
free(ctx);
|
||||
atomic_store_explicit(&c->ha.catchup_inflight[peer_idx], 0,
|
||||
memory_order_release);
|
||||
return rc;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int ha_walack_on_ack(cluster_t *c, void *ep, int peer_idx, uint64_t acked_lsn)
|
||||
{
|
||||
if (c == NULL || ep == NULL)
|
||||
return -EINVAL;
|
||||
if (peer_idx < 0 || peer_idx >= MAX_STANDBYS)
|
||||
return -EINVAL;
|
||||
|
||||
uint64_t target = catchup_derive_target(c);
|
||||
|
||||
/* Parity (Req 7.1): the standby reached the boundary — re-include it in live
|
||||
* replication via the existing-but-untriggered re-inclusion call. No
|
||||
* continuation; ha-commit-engine wiring propagates commit_on_membership_change. */
|
||||
if (acked_lsn >= target) {
|
||||
repl_complete_catchup(&c->persist.repl, (repl_ep_t)ep);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Behind target (Req 5.1/5.2): single-flight — claim the per-standby slot
|
||||
* 0→1; if already in flight, schedule nothing. */
|
||||
int expected = 0;
|
||||
if (!atomic_compare_exchange_strong_explicit(
|
||||
&c->ha.catchup_inflight[peer_idx], &expected, 1,
|
||||
memory_order_acq_rel, memory_order_acquire)) {
|
||||
/* A continuation is already in flight for this standby (Req 5.2). */
|
||||
return 0;
|
||||
}
|
||||
|
||||
return ha_schedule_catchup_continuation(c, (repl_ep_t)ep, peer_idx,
|
||||
acked_lsn);
|
||||
}
|
||||
|
||||
/* ── Snapshot chunk transfer ───────────────────────────────────────────── */
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue