wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity

feat(ha-term-safety): route AppendEntries intake through library + leaderCommit wire (task 3.2)

Browse source
This commit is contained in:
Rongsong Shen 2026-06-22 08:42:35 +00:00
parent 6a519a928b
commit 754e586f8f
10 changed files with 508 additions and 168 deletions
Download patch file Download diff file Expand all files Collapse all files

125
src/controller/ha_convergence.c
View file

@ -219,54 +219,13 @@ int ha_schedule_demotion_convergence(cluster_t *c)
}
/*
* ha_rejoin_reconcile_ctx_t — heap context for the one-shot rejoin delaymgr task.
* Carries the clamped cutoff computed at trigger time (under the same lock that
* read the tip) plus the prior tip for the WARN range, so the deferred body does
* not re-read state that may have advanced between schedule and run.
* ha-term-safety task 3.2: the rejoin delaymgr task body (ha_rejoin_reconcile_fn)
* and its heap context (ha_rejoin_reconcile_ctx_t) are REMOVED. They drove the
* watermark-clamped truncate + self-restart that is now retired — divergent-suffix
* reconciliation is owned by the raft library's (term,lsn) AppendEntries path
* (raft_on_append_entries_request → on_truncate_from). See ha_rejoin_reconcile
* below (now a no-op CtrlHelloReply hook).
*/
typedef struct {
cluster_t *c;
uint64_t cutoff; /* truncate above this LSN (clamped, Req 2.2) */
uint64_t prior_tip; /* local tip at trigger time (WARN range, Req 2.1) */
} ha_rejoin_reconcile_ctx_t;
/*
* ha_rejoin_reconcile_fn — delaymgr task body (runs OFF the CtrlHelloReply intake
* thread, which may be iomgr; D5/D1). Truncate above the clamped cutoff (Req 2.1,
* 2.2), WARN with the discarded range, then ha_self_restart (Req 2.1). On
* truncation failure ERROR then restart anyway — boot recovery + a subsequent
* rejoin converge the untruncated case (crash-safe by design, mirrors demotion).
*/
int ha_rejoin_reconcile_fn(void *arg)
{
ha_rejoin_reconcile_ctx_t *ctx = (ha_rejoin_reconcile_ctx_t *)arg;
if (ctx == NULL || ctx->c == NULL) {
free(ctx);
return -EINVAL;
}
cluster_t *c = ctx->c;
uint64_t cutoff = ctx->cutoff;
uint64_t prior_tip = ctx->prior_tip;
int trc = persist_truncate_wal_above(&c->persist, cutoff);
if (trc < 0) {
ERROR("HA rejoin reconcile: truncate above cutoff=%" PRIu64
" failed: %d — restarting anyway (boot recovery + next rejoin will "
"converge)", cutoff, trc);
} else {
/* Req 2.1: log the discarded range — (cutoff, prior tip]. */
WARN("HA rejoin reconcile: local tip=%" PRIu64 " exceeded the cluster "
"committed boundary; truncated WAL above cutoff=%" PRIu64
" (clamped to the locally committed prefix), restarting once to "
"rejoin from converged state", prior_tip, cutoff);
}
/* Free the task ctx before restart (production exec never returns). */
free(ctx);
/* Req 2.1: rebuild in-memory state through boot recovery. */
return ha_self_restart(c);
}
/*
* ha_rejoin_reconcile — converge a rejoining node whose local WAL log runs ahead
@ -297,58 +256,26 @@ int ha_rejoin_reconcile(cluster_t *c, uint64_t primary_committed_lsn)
if (c == NULL)
return -EINVAL;
/* Read the local tip + committed prefix together under the persist lock so
* the clamp decision is consistent. */
pthread_mutex_lock(&c->persist.lock);
uint64_t current_lsn = c->persist.current_lsn;
pthread_mutex_unlock(&c->persist.lock);
uint64_t local_tip = current_lsn > 0 ? current_lsn - 1 : 0;
/* Req 2.3: tip does not exceed the boundary → join with no truncation/restart. */
if (local_tip <= primary_committed_lsn)
return 0;
/* Req 2.1 (defensive once-guard): reconcile at most once per process lifetime.
* CAS 0→1 so a concurrent/duplicate CtrlHelloReply cannot schedule twice. */
int expected = 0;
if (!atomic_compare_exchange_strong_explicit(
&c->ha.rejoin_reconciled, &expected, 1,
memory_order_acq_rel, memory_order_acquire))
return 0; /* already reconciled — no second restart */
/* Req 2.2 (clamp): never truncate at/below the locally known committed prefix.
* cutoff = max(primary_committed_lsn, local committed_lsn). */
uint64_t local_committed = atomic_load_explicit(&c->persist.committed_lsn,
memory_order_acquire);
uint64_t cutoff = primary_committed_lsn > local_committed
? primary_committed_lsn
: local_committed;
ha_rejoin_reconcile_ctx_t *ctx = calloc(1, sizeof(*ctx));
if (ctx == NULL) {
ERROR("HA rejoin reconcile: ctx alloc failed — divergent suffix will be "
"reconciled at the next rejoin");
atomic_store_explicit(&c->ha.rejoin_reconciled, 0,
memory_order_release); /* roll back: allow retry */
return -ENOMEM;
}
ctx->c = c;
ctx->cutoff = cutoff;
ctx->prior_tip = local_tip;
/* D5/D1: defer truncation + restart off the (possibly iomgr) intake thread.
* One-shot, minimal delay — mirrors ha_schedule_demotion_convergence. Under
* XCE_TESTING the fixture drives the queued task synchronously. */
int rc = delay_task_ms(&c->delaymgr, 1, 1,
(delay_task_entry_fn_t)ha_rejoin_reconcile_fn, ctx);
if (rc < 0) {
ERROR("HA rejoin reconcile: delay_task_ms failed: %d", rc);
free(ctx);
atomic_store_explicit(&c->ha.rejoin_reconciled, 0,
memory_order_release); /* roll back: allow retry */
return rc;
}
/* ha-term-safety task 3.2: the committed-boundary (watermark) truncate is
* RETIRED. Divergent-suffix reconciliation is now owned by the raft library:
* an incoming AppendEntries from the leader is routed through
* raft_on_append_entries_request (ha_consume_append_entries), and the (term,
* lsn) log-matching there drives on_truncate_from to delete exactly the
* divergent suffix (G3) — never a committed-prefix entry (R3.3), and never by
* an LSN watermark that is term-blind. A node whose tip runs ahead of the
* cluster keeps its entries until the leader's AE proves the conflict by term,
* then the library truncates precisely; until reconciled the existing
* promotion-time fence keeps the member non-promotable (R3.5) without
* self-restarting.
*
* This function is retained as the CtrlHelloReply hook so the learned
* committed boundary is still recorded by the caller (cluster_apply_ctrlhello_
* reply), but it no longer performs a watermark truncate + self-restart. The
* local_tip vs boundary comparison and the clamped-cutoff heuristic that drove
* the old delaymgr truncate/restart task are removed: they are replaced by the
* library's term-aware truncation, not duplicated here.
*/
(void)primary_committed_lsn;
return 0;
}