Rongsong Shen
f88e4ef431
/xce-audit-invariants controller found two live violations of the async I/O thread-role contract: ha_demotion_convergence_fn (ha_convergence.c) ran WAL truncation + tlog_exit()+execv() directly on the delaymgr thread, and ha_catchup_continuation_fn (config_repl.c) ran unbounded WAL-streaming to a standby directly on delaymgr -- the same single thread that drives heartbeats and election timeouts cluster-wide, so either blocks liveness for the whole cluster and risks a spurious election. A prior vtask-pool offload attempt (2026-07-10, bug-1429) deadlocked controller_unit_test because c->ha.pool is never initialized by the lightweight unit-test fixtures these functions run under, and was fully reverted. This fix instead uses the codebase's established slow-lane pattern (the same mechanism already used for snapshot-send, config-push, and peer connect) -- a 4th and 5th instance, not a new mechanism. New g_drive_async_demotion_offload / g_drive_async_catchup_offload flags ship default OFF, matching the original rollout discipline of the three reference offloads (shipped OFF, flipped ON later after a dedicated compatibility audit). Flipping ON was tried during validation and reproduced two distinct hazards against pre-existing fixtures that never call slowlane_init: the already-known hang, and a new ASan SEGV in commit_advance_locked (logged as bug-1452, not root-caused, deferred to the future flip-to-ON audit). Both flags verified OFF and the full 976-test suite clean, then the committed state re-verified clean against a live 10-round chaotic and 10-round sequential HA soak (zero job loss, zero divergence, both PASS).
614 lines
29 KiB
C
614 lines
29 KiB
C
/*
|
||
* ha_convergence.c — demotion convergence + self-restart primitive
|
||
* (ha-convergence-recovery Requirement 1).
|
||
*
|
||
* ───────────────────────────────────────────────────────────────────────────
|
||
* This TU owns the self-restart primitive and the deferred demotion-convergence
|
||
* task body. The synchronous demotion BRANCH (hook disable + task scheduling)
|
||
* lives in raft_cb_become_follower (ha.c) because that callback is file-static;
|
||
* it calls ha_schedule_demotion_convergence() declared here.
|
||
*
|
||
* Split of responsibilities (design "Demotion Convergence + Self-Restart"):
|
||
* - raft_cb_become_follower (ha.c): read prior role → if was-PRIMARY, disable
|
||
* the 3 persistence hooks INLINE (Req 1.1),
|
||
* then schedule the one-shot convergence
|
||
* task off the callback thread (D1).
|
||
* - ha_demotion_convergence_fn (here, delaymgr task body): truncate above
|
||
* committed_lsn (Req 1.2) → WARN range
|
||
* (Req 1.5) → ha_self_restart (Req 1.3).
|
||
* - ha_self_restart (here): flush logs → execv(saved argv); exec
|
||
* return → ERROR + abort (FATAL, Req 1.4);
|
||
* under XCE_TESTING route to the restart /
|
||
* fatal seams (Req 1.6, 1.4).
|
||
*
|
||
* controller_saved_argv lives in main.c (the real argv capture); this TU only
|
||
* consumes it through the ha.h accessor.
|
||
*
|
||
* fd hygiene (Req 1.3 / design fd audit): every controller-to-controller socket
|
||
* — listener, accepted peer/replication connections, and outbound vote/peer
|
||
* connect sockets — is created FD_CLOEXEC in src/lib/netio.c (socket_create
|
||
* sets it at netio.c:219; socket_accept at netio.c:308). The re-exec'd image
|
||
* therefore inherits no live sockets and re-creates all endpoints through the
|
||
* boot path; no extra close-before-exec sweep is required here.
|
||
* ───────────────────────────────────────────────────────────────────────────
|
||
*/
|
||
|
||
#define _GNU_SOURCE
|
||
#include <errno.h>
|
||
#include <inttypes.h>
|
||
#include <pthread.h>
|
||
#include <stdatomic.h>
|
||
#include <stddef.h>
|
||
#include <stdint.h>
|
||
#include <stdlib.h>
|
||
#include <unistd.h>
|
||
|
||
#include <tlog.h>
|
||
|
||
#include "cluster.h"
|
||
#include "delay.h"
|
||
#include "ha.h"
|
||
#include "persist.h"
|
||
#include "xce/cluster_def.h"
|
||
|
||
#ifdef XCE_TESTING
|
||
/* XCE_TESTING seams (Req 1.6, 1.4) — defined here, declared in ha.h.
|
||
* Tests set these before exercising the restart/fatal contract. */
|
||
int (*ha_restart_hook)(cluster_t *c) = NULL;
|
||
void (*ha_fatal_hook)(const char *msg) = NULL;
|
||
|
||
/* Quorum-transition emission-count seam (task 5.x, Req 8.1-8.3) — defined here,
|
||
* declared in ha.h. ha_log_quorum_transition invokes this once per WARN it
|
||
* emits so the counter-hook tests can assert once-per-transition + counts.
|
||
* NULL until a test installs it. */
|
||
void (*ha_quorum_transition_log_hook)(int lost, int connected,
|
||
int required) = NULL;
|
||
#endif
|
||
|
||
#ifndef XCE_TESTING
|
||
#define HA_SELF_RESTART_RETRY_MS 5000
|
||
|
||
static int ha_self_restart_retry_fn(void *arg);
|
||
|
||
/* ha_self_restart_enter_fence_and_retry: the non-abort response to a failed
|
||
* re-exec (2026-07-10, supersedes ha-convergence-recovery Req 1.4's original
|
||
* "terminate fatally" text — see requirements.md's superseding note). Sets
|
||
* the sticky, never-live-cleared ha->self_restart_pending flag (checked
|
||
* unconditionally by raft_cb_become_leader / promotion_fence_timeout_fn
|
||
* alongside apply_divergent, ha.c) so this node can never become primary
|
||
* while it still carries its stale pre-demotion in-memory state, then
|
||
* schedules a bounded periodic retry of the re-exec. This satisfies the same
|
||
* safety property Req 1.4 protects — a divergent ex-primary never serves
|
||
* authoritatively — via the project-wide fail-loud invariant (a controller
|
||
* MUST NEVER self-terminate as a designed response to an operational
|
||
* condition; CLAUDE.md "Process Lifecycle") instead of abort(): staying
|
||
* alive and retrying self-heals automatically once the transient condition
|
||
* (ENOMEM, EMFILE, a permissions change mid-upgrade) clears, whereas a dead
|
||
* process needs manual admin recovery and degrades quorum meanwhile. */
|
||
static void ha_self_restart_enter_fence_and_retry(cluster_t *c)
|
||
{
|
||
atomic_store_explicit(&c->ha.self_restart_pending, 1, memory_order_release);
|
||
|
||
if (c->delaymgr.running) {
|
||
int rc = delay_task_ms(&c->delaymgr, HA_SELF_RESTART_RETRY_MS, 1,
|
||
ha_self_restart_retry_fn, c);
|
||
if (rc < 0)
|
||
WARN("HA self-restart: retry scheduling failed: %d — node stays "
|
||
"fenced (non-promotable) with no further automatic retry "
|
||
"attempt", rc);
|
||
} else {
|
||
WARN("HA self-restart: delaymgr not running — node stays fenced "
|
||
"(non-promotable) with no automatic retry");
|
||
}
|
||
}
|
||
|
||
/* ha_self_restart_retry_fn: delaymgr task body — bounded periodic retry of
|
||
* the re-exec. A successful retry replaces the process image entirely (this
|
||
* function never returns in that case); a repeated failure re-arms itself
|
||
* via ha_self_restart's own failure path. */
|
||
static int ha_self_restart_retry_fn(void *arg)
|
||
{
|
||
cluster_t *c = (cluster_t *)arg;
|
||
if (!c)
|
||
return -EINVAL;
|
||
return ha_self_restart(c);
|
||
}
|
||
#endif
|
||
|
||
/*
|
||
* ha_self_restart — the truncate-and-restart primitive's restart half.
|
||
*
|
||
* Production (Req 1.3): flush logs, then execv() the saved argv so the
|
||
* controller rebuilds in-memory state purely from the surviving (committed)
|
||
* WAL prefix via boot recovery. NEVER returns on success (execv replaces the
|
||
* process image). If exec cannot be performed, do NOT abort (see
|
||
* ha_self_restart_enter_fence_and_retry above) — enter a permanent
|
||
* non-promotable fence and keep retrying periodically, per the project's
|
||
* never-self-terminate invariant.
|
||
*
|
||
* Test (Req 1.6): under XCE_TESTING invoke ha_restart_hook(c) instead of
|
||
* execv and return its result on the success path; a non-zero hook return
|
||
* models an exec failure and routes to ha_fatal_hook (retained as the test
|
||
* seam name — it still marks "restart could not be performed", now backed by
|
||
* the fence-and-retry response in production rather than abort), so a test
|
||
* binary can assert that failure path was taken without dying.
|
||
*/
|
||
int ha_self_restart(cluster_t *c)
|
||
{
|
||
#ifdef XCE_TESTING
|
||
/* Test contract: the restart seam stands in for execv. */
|
||
int rc = 0;
|
||
if (ha_restart_hook != NULL)
|
||
rc = ha_restart_hook(c);
|
||
|
||
if (rc != 0 || ha_restart_hook == NULL) {
|
||
/* Restart could not be performed — same non-abort response as
|
||
* production (Req 1.6 mirrors ha_self_restart_enter_fence_and_retry's
|
||
* flag-setting without the real delaymgr retry, since a unit fixture
|
||
* has no live delaymgr to retry on). */
|
||
if (c)
|
||
atomic_store_explicit(&c->ha.self_restart_pending, 1,
|
||
memory_order_release);
|
||
if (ha_fatal_hook != NULL)
|
||
ha_fatal_hook("ha_self_restart: restart seam failed — fenced "
|
||
"non-promotable, staying alive");
|
||
return rc != 0 ? rc : -1;
|
||
}
|
||
return 0;
|
||
#else
|
||
char **argv = controller_saved_argv();
|
||
if (argv == NULL || argv[0] == NULL) {
|
||
ERROR("HA self-restart: no saved argv — cannot re-exec; entering "
|
||
"non-promotable fence and retrying periodically (stay alive)");
|
||
ha_self_restart_enter_fence_and_retry(c);
|
||
return -1;
|
||
}
|
||
|
||
INFO("HA self-restart: re-executing %s to rebuild from committed state",
|
||
argv[0]);
|
||
tlog_exit(); /* flush + drain the async log buffer before exec replaces us */
|
||
|
||
execv(argv[0], argv);
|
||
|
||
/* execv only returns on failure. A demoted ex-primary must never continue
|
||
* serving with potentially divergent state (ha-convergence-recovery Req
|
||
* 1.4) — but per the project-wide fail-loud invariant (never self-
|
||
* terminate as a designed response to an operational condition), the
|
||
* correct response is a permanent non-promotable fence that stays alive
|
||
* and keeps retrying, not abort(). */
|
||
ERROR("HA self-restart: execv(%s) failed: %d — entering non-promotable "
|
||
"fence and retrying periodically (stay alive)", argv[0], errno);
|
||
ha_self_restart_enter_fence_and_retry(c);
|
||
return -1;
|
||
#endif
|
||
}
|
||
|
||
/*
|
||
* ha_demotion_convergence_ctx_t — heap context for the one-shot delaymgr task
|
||
* (and, with the offload flag ON, the slow-lane task it hands off to). Holds
|
||
* only the cluster pointer; committed_lsn is read at task time (not schedule
|
||
* time) so a late-advancing commit still truncates to the right boundary.
|
||
*/
|
||
typedef struct {
|
||
cluster_t *c;
|
||
} ha_demotion_convergence_ctx_t;
|
||
|
||
/*
|
||
* g_drive_async_demotion_offload / ha_set_demotion_offload_for_test /
|
||
* ha_demotion_offload_enabled (xce-audit-invariants controller pass,
|
||
* thread-role violation 1 fix, 2026-07-18):
|
||
*
|
||
* Gates whether ha_demotion_convergence_fn hands the truncate + self-restart
|
||
* body off to the slow lane (ON — closes the thread-role violation: the
|
||
* delaymgr thread that also drives heartbeats/election timeouts cluster-wide
|
||
* never blocks on the truncation I/O or the tlog_exit()+execv() inside
|
||
* ha_self_restart) or runs it INLINE on the calling (delaymgr) thread (OFF,
|
||
* the PRODUCTION DEFAULT for now — see below). This is the SAME established
|
||
* slow-lane pattern already used for snapshot send
|
||
* (ha_snapshot_send_dispatch/snapshot_lane_fn), config-push
|
||
* (ha_config_push_offloadable/configpush_lane_fn), and peer connect
|
||
* (ha_connect_to_peer/reconnect_lane_fn) in ha.c — not a new mechanism. A
|
||
* prior attempt to fix this via a VTASK_LIVE_BLOCK vtask-pool dispatch
|
||
* (2026-07-10) deadlocked controller_unit_test and was reverted (bug-1429):
|
||
* the vtask pool used there (c->ha.pool) is never initialized/started by
|
||
* this file's lightweight unit-test fixtures (dc_setup), so a vtask
|
||
* enqueued VTASK_LIVE_BLOCK blocked forever waiting for a worker that never
|
||
* existed. The slow lane's own submit contract cannot deadlock a caller the
|
||
* way the vtask pool did — but a second, distinct hazard surfaces the SAME
|
||
* class of problem: raft_cb_become_follower (the demotion callback this task
|
||
* is scheduled from) is exercised by a very broad set of PRE-EXISTING
|
||
* controller_unit_test fixtures (HA lifecycle, connection-reuse,
|
||
* vep-teardown, startup, …) that were never written with a live slow lane in
|
||
* mind. Flipping this flag ON by default was tried and reproduced a REAL
|
||
* hang: such a fixture demotes a PRIMARY, the truncate+restart work is
|
||
* silently submitted to an uninitialized c->slowlane (queued, but never run
|
||
* — no live workers), and downstream code that waits for the now-never-
|
||
* happening completion hangs. OFF (0, PRODUCTION DEFAULT) matches the
|
||
* ORIGINAL rollout discipline of all three reference offloads — every one of
|
||
* snapshot/config-push/reconnect shipped OFF-by-default and was flipped ON
|
||
* only in a LATER, dedicated pass (raft-drive-async task 3.4) once the full
|
||
* test suite was audited for compatibility. This change lands the
|
||
* mechanism, correct and covered by its own targeted tests below; flipping
|
||
* the cluster-wide default to ON is the same kind of follow-up audit, out of
|
||
* scope for this fix (bug-1450, buglog.json). Compiled in ALL builds; the
|
||
* setter is the test seam to flip RED<->GREEN.
|
||
*/
|
||
static _Atomic int g_drive_async_demotion_offload = 0;
|
||
|
||
void ha_set_demotion_offload_for_test(int on)
|
||
{
|
||
atomic_store_explicit(&g_drive_async_demotion_offload, on ? 1 : 0,
|
||
memory_order_release);
|
||
}
|
||
|
||
int ha_demotion_offload_enabled(void)
|
||
{
|
||
return atomic_load_explicit(&g_drive_async_demotion_offload,
|
||
memory_order_acquire);
|
||
}
|
||
|
||
/*
|
||
* ha_demotion_convergence_do_run — the actual (blocking) convergence work:
|
||
* truncate above committed_lsn (Req 1.2), WARN with the truncated range
|
||
* (Req 1.5), then self-restart (Req 1.3). On truncation failure (Req error
|
||
* strategy) ERROR then restart anyway — the raft library's own (term,lsn) AE
|
||
* reconcile converges the untruncated case once the leader's AppendEntries
|
||
* arrives. Shared verbatim by the inline path (offload flag OFF) and the
|
||
* slow-lane worker (demotion_lane_fn, offload flag ON) so both behave
|
||
* identically — mirrors ha_snapshot_do_send / ha_config_push_do_send in
|
||
* ha.c.
|
||
*/
|
||
static int ha_demotion_convergence_do_run(cluster_t *c)
|
||
{
|
||
uint64_t committed = atomic_load_explicit(&c->persist.committed_lsn,
|
||
memory_order_acquire);
|
||
|
||
/* Prior log tip for the WARN range (read under the persist lock). */
|
||
pthread_mutex_lock(&c->persist.lock);
|
||
uint64_t prior_tip = c->persist.current_lsn > 0
|
||
? c->persist.current_lsn - 1
|
||
: 0;
|
||
pthread_mutex_unlock(&c->persist.lock);
|
||
|
||
int trc = persist_truncate_wal_above(&c->persist, committed);
|
||
if (trc < 0) {
|
||
ERROR("HA demotion convergence: truncate above committed_lsn=%" PRIu64
|
||
" failed: %d — restarting anyway (rejoin reconciliation will "
|
||
"converge)", committed, trc);
|
||
} else {
|
||
/* Req 1.5: log the discarded range — (committed_lsn, prior tip]. */
|
||
WARN("HA demotion convergence: truncated WAL above committed_lsn=%"
|
||
PRIu64 " (prior tip=%" PRIu64 "); discarding provisional suffix, "
|
||
"restarting to rebuild from committed state",
|
||
committed, prior_tip);
|
||
}
|
||
|
||
/* Req 1.3: rebuild in-memory state through boot recovery. Production:
|
||
* execv() replaces the process image and never returns. XCE_TESTING:
|
||
* the restart seam returns so the caller can observe the outcome. */
|
||
return ha_self_restart(c);
|
||
}
|
||
|
||
/*
|
||
* demotion_lane_fn — slow-lane worker entry. Runs ha_demotion_convergence_do_run
|
||
* OFF the delaymgr thread; frees ctx and clears the cluster-wide single-flight
|
||
* guard on the return path. In production ha_self_restart's execv() never
|
||
* returns, so this point is reached only under XCE_TESTING (or the rare
|
||
* production restart-seam-failure fence-and-retry path, which also returns).
|
||
*/
|
||
static void demotion_lane_fn(void *arg)
|
||
{
|
||
ha_demotion_convergence_ctx_t *ctx = (ha_demotion_convergence_ctx_t *)arg;
|
||
if (ctx == NULL)
|
||
return;
|
||
cluster_t *c = ctx->c;
|
||
free(ctx);
|
||
|
||
(void)ha_demotion_convergence_do_run(c);
|
||
|
||
atomic_store_explicit(&c->ha.demotion_convergence_inflight, 0,
|
||
memory_order_release);
|
||
}
|
||
|
||
/*
|
||
* ha_demotion_convergence_fn — delaymgr task body (runs OFF the become-follower
|
||
* callback thread, D1 — the FIRST hop). With the offload flag ON (production
|
||
* default), this function does only fast, non-blocking admin work (ctx
|
||
* unpack, single-flight CAS, slowlane_submit) and returns immediately — the
|
||
* SECOND hop, off the delaymgr thread itself, onto a slow-lane worker
|
||
* (demotion_lane_fn), which is where the truncate + self-restart actually
|
||
* run. This closes the thread-role violation: delaymgr also drives
|
||
* heartbeats and election timeouts cluster-wide, so blocking it here would
|
||
* stall liveness for the whole cluster (ha-async-io.md thread-role
|
||
* contract). With the flag OFF, runs the legacy inline blocking path
|
||
* (retained solely so a TDD RED test can show the pre-fix stall).
|
||
*
|
||
* In XCE_TESTING the test fixture drives this entry directly (un-started
|
||
* delaymgr) via its stored entry/args, exactly as before this offload.
|
||
*/
|
||
int ha_demotion_convergence_fn(void *arg)
|
||
{
|
||
ha_demotion_convergence_ctx_t *ctx = (ha_demotion_convergence_ctx_t *)arg;
|
||
if (ctx == NULL || ctx->c == NULL) {
|
||
free(ctx);
|
||
return -EINVAL;
|
||
}
|
||
cluster_t *c = ctx->c;
|
||
|
||
if (!ha_demotion_offload_enabled()) {
|
||
/* Legacy inline path (flag OFF) — blocks the calling (delaymgr)
|
||
* thread. Retained solely so the RED test can show the pre-change
|
||
* stall; production runs with the flag ON. */
|
||
int rc = ha_demotion_convergence_do_run(c);
|
||
free(ctx);
|
||
return rc;
|
||
}
|
||
|
||
/* Cluster-wide single-flight (demotion convergence is a single one-shot
|
||
* event, like configpush — not per-peer): CAS 0->1. Already 1 should be
|
||
* structurally unreachable — raft_cb_become_follower only schedules on
|
||
* the was-PRIMARY edge, and the role flips to STANDBY synchronously
|
||
* BEFORE scheduling — but stay defensive rather than assume. */
|
||
int expected = 0;
|
||
if (!atomic_compare_exchange_strong_explicit(
|
||
&c->ha.demotion_convergence_inflight, &expected, 1,
|
||
memory_order_acq_rel, memory_order_acquire)) {
|
||
WARN("HA demotion convergence: a convergence task is already in "
|
||
"flight — skipping duplicate schedule");
|
||
free(ctx);
|
||
return 0;
|
||
}
|
||
|
||
int srv = slowlane_submit(&c->slowlane, /*key=*/1, demotion_lane_fn, ctx);
|
||
if (srv == -EEXIST) {
|
||
/* Lane still holds key=1 from a prior (already-cleared) convergence
|
||
* task whose key removal hasn't caught up yet. Release the guard we
|
||
* just set (mirrors the reconnect/configpush -EEXIST handling in
|
||
* ha.c) and fall back to running inline: unlike reconnect/configpush
|
||
* — which retry on the next heartbeat cycle — a demotion-convergence
|
||
* event is one-shot with no later trigger, and this is a
|
||
* safety-critical action (an ex-primary discarding provisional state
|
||
* before it can be trusted again), so it must never be silently
|
||
* dropped. */
|
||
atomic_store_explicit(&c->ha.demotion_convergence_inflight, 0,
|
||
memory_order_release);
|
||
int rc = ha_demotion_convergence_do_run(c);
|
||
free(ctx);
|
||
return rc;
|
||
}
|
||
if (srv < 0 && srv != -EAGAIN) {
|
||
/* Hard submit failure (alloc / lane stopping). -EAGAIN is NOT a
|
||
* failure (still queued, will run). Same "never drop" reasoning as
|
||
* -EEXIST above. */
|
||
WARN("HA demotion convergence: slowlane_submit failed: %d — running "
|
||
"inline (a one-shot safety action, never dropped)", srv);
|
||
atomic_store_explicit(&c->ha.demotion_convergence_inflight, 0,
|
||
memory_order_release);
|
||
int rc = ha_demotion_convergence_do_run(c);
|
||
free(ctx);
|
||
return rc;
|
||
}
|
||
/* 0 or -EAGAIN: enqueued exactly once; demotion_lane_fn frees ctx and
|
||
* clears the flag after the (production: non-returning) restart. */
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
* ha_schedule_demotion_convergence — called from raft_cb_become_follower's
|
||
* was-PRIMARY branch AFTER the persistence hooks have been disabled inline.
|
||
* Schedules exactly one one-shot delaymgr task running ha_demotion_convergence_fn
|
||
* so the file I/O (truncation) and exec never run on the callback thread (D1).
|
||
*
|
||
* Returns 0 on success, negative on allocation/scheduling failure. On failure
|
||
* the node stays demoted with hooks disabled; the untruncated suffix is then
|
||
* reconciled by rejoin reconciliation at the next join (crash-safe by design).
|
||
*/
|
||
int ha_schedule_demotion_convergence(cluster_t *c)
|
||
{
|
||
if (c == NULL)
|
||
return -EINVAL;
|
||
|
||
ha_demotion_convergence_ctx_t *ctx = calloc(1, sizeof(*ctx));
|
||
if (ctx == NULL) {
|
||
ERROR("HA demotion convergence: ctx alloc failed — suffix will be "
|
||
"reconciled at next rejoin");
|
||
return -ENOMEM;
|
||
}
|
||
ctx->c = c;
|
||
|
||
/* One-shot (runcount=1), minimal delay: the work must run promptly but off
|
||
* the iomgr callback thread. delay_task_ms only enqueues (ordq_put); the
|
||
* delaymgr worker runs it. In XCE_TESTING the fixture drives the queued
|
||
* task synchronously via its stored entry/args. */
|
||
int rc = delay_task_ms(&c->delaymgr, 1, 1,
|
||
(delay_task_entry_fn_t)ha_demotion_convergence_fn,
|
||
ctx);
|
||
if (rc < 0) {
|
||
ERROR("HA demotion convergence: delay_task_ms failed: %d", rc);
|
||
free(ctx);
|
||
return rc;
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
* ha-term-safety task 3.2: the rejoin delaymgr task body (ha_rejoin_reconcile_fn)
|
||
* and its heap context (ha_rejoin_reconcile_ctx_t) are REMOVED. They drove the
|
||
* watermark-clamped truncate + self-restart that is now retired — divergent-suffix
|
||
* reconciliation is owned by the raft library's (term,lsn) AppendEntries path
|
||
* (raft_on_append_entries_request → on_truncate_from). See ha_rejoin_reconcile
|
||
* below (now a no-op CtrlHelloReply hook).
|
||
*/
|
||
|
||
/*
|
||
* ha_rejoin_reconcile — converge a rejoining node whose local WAL log runs ahead
|
||
* of the cluster's committed boundary (ha-convergence-recovery Req 2.1–2.3, D5).
|
||
*
|
||
* Called from the CtrlHelloReply intake (cluster_apply_ctrlhello_reply) after the
|
||
* primary-reported committed boundary is stored — the moment the rejoining node
|
||
* learns where the cluster has committed.
|
||
*
|
||
* Decision (design "Rejoin Reconciliation", research D5):
|
||
* - local_tip = persist.current_lsn - 1.
|
||
* - If local_tip <= primary_committed_lsn → return 0, NO side effects (Req 2.3).
|
||
* - If the rejoin_reconciled once-guard is already set → return 0; never restart
|
||
* a second time in one process lifetime (Req 2.1 defensive guard; "once" is
|
||
* also structural — after restart the tip equals the cutoff, so the condition
|
||
* cannot re-fire for the same divergence).
|
||
* - Else: cutoff = max(primary_committed_lsn, local committed_lsn) — the CLAMP
|
||
* (Req 2.2): never truncate at or below the locally known committed prefix.
|
||
* Set the once-guard, then defer truncation + restart to a delaymgr one-shot
|
||
* (the intake may run on iomgr, so file I/O / exec must leave it — D5/D1).
|
||
*
|
||
* Returns 0 on the no-action paths and on a successfully scheduled reconcile;
|
||
* negative on a scheduling/allocation failure (the once-guard is rolled back so a
|
||
* later rejoin can retry).
|
||
*/
|
||
int ha_rejoin_reconcile(cluster_t *c, uint64_t primary_committed_lsn)
|
||
{
|
||
if (c == NULL)
|
||
return -EINVAL;
|
||
|
||
/* ha-term-safety task 3.2: the committed-boundary (watermark) truncate is
|
||
* RETIRED. Divergent-suffix reconciliation is now owned by the raft library:
|
||
* an incoming AppendEntries from the leader is routed through
|
||
* raft_on_append_entries_request (ha_consume_append_entries), and the (term,
|
||
* lsn) log-matching there drives on_truncate_from to delete exactly the
|
||
* divergent suffix (G3) — never a committed-prefix entry (R3.3), and never by
|
||
* an LSN watermark that is term-blind. A node whose tip runs ahead of the
|
||
* cluster keeps its entries until the leader's AE proves the conflict by term,
|
||
* then the library truncates precisely; until reconciled the existing
|
||
* promotion-time fence keeps the member non-promotable (R3.5) without
|
||
* self-restarting.
|
||
*
|
||
* This function is retained as the CtrlHelloReply hook so the learned
|
||
* committed boundary is still recorded by the caller (cluster_apply_ctrlhello_
|
||
* reply), but it no longer performs a watermark truncate + self-restart. The
|
||
* local_tip vs boundary comparison and the clamped-cutoff heuristic that drove
|
||
* the old delaymgr truncate/restart task are removed: they are replaced by the
|
||
* library's term-aware truncation, not duplicated here.
|
||
*/
|
||
(void)primary_committed_lsn;
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
* cluster_apply_ctrlhello_reply — standby-side CtrlHelloReply intake body.
|
||
*
|
||
* Lives in this (unit-linkable) TU rather than cluster.c so the rejoin
|
||
* intake-integration test can drive the real path: cluster.c (the daemon-only
|
||
* vtask handler graph) is NOT part of controller_unit_test_src, but
|
||
* ha_convergence.c is. cluster.c's myreq_handler calls this on every
|
||
* CtrlHelloReply; the function stores the primary-reported committed boundary +
|
||
* config tag (existing behavior), then is the rejoin trigger point.
|
||
*
|
||
* Rejoin trigger (Req 2.1–2.3, design "Rejoin Reconciliation", research D5):
|
||
* AFTER the store, ha_rejoin_reconcile(c, committed_lsn) evaluates the local tip
|
||
* against the primary-reported boundary and (when the tip runs ahead) defers a
|
||
* clamped truncate + one-time restart. ha_rejoin_reconcile is internally a
|
||
* no-op when the tip is at/below the boundary or already reconciled, so it is
|
||
* safe to call on every CtrlHelloReply.
|
||
*/
|
||
void cluster_apply_ctrlhello_reply(cluster_t *c, uint64_t committed_lsn,
|
||
uint64_t config_tag)
|
||
{
|
||
atomic_store_explicit(&c->ha.last_known_committed_lsn, committed_lsn,
|
||
memory_order_release);
|
||
atomic_store_explicit(&c->ha.last_known_committed_config_tag, config_tag,
|
||
memory_order_release);
|
||
INFO("HA: CtrlHelloReply from primary: committed_lsn=%" PRIu64
|
||
" config_tag=%" PRIu64, committed_lsn, config_tag);
|
||
|
||
/* Req 2.1–2.3: reconcile a log that runs ahead of the cluster boundary. */
|
||
(void)ha_rejoin_reconcile(c, committed_lsn);
|
||
}
|
||
|
||
/*
|
||
* ha_log_quorum_transition — write-quorum loss/restore edge logger
|
||
* (ha-convergence-recovery task 5.2, Req 8.1, 8.2, 8.3).
|
||
*
|
||
* Evaluate the current write-quorum state and compare it against the
|
||
* ha_state_t.quorum_lost edge-detector flag. On a TRANSITION — and only on a
|
||
* transition — atomically flip the flag via CAS and emit exactly one WARN
|
||
* carrying the connected/required member counts:
|
||
* - loss: "HA quorum lost: connected=%d required=%d" (Req 8.1)
|
||
* - restore: "HA quorum restored: connected=%d required=%d" (Req 8.2)
|
||
*
|
||
* The flag makes the logging an edge detector, NOT a latch: repeated calls while
|
||
* the state is unchanged emit nothing (Req 8.3 — no log storms across the
|
||
* dispatch-gate backstop's per-rejection calls), but a later loss/restore cycle
|
||
* logs again. The CAS guarantees once-per-transition under concurrency: only the
|
||
* caller that actually flips the flag emits the WARN.
|
||
*
|
||
* Observability only — this never returns a status and never gates an operation
|
||
* outcome; the dispatch gate's existing -XCE_ERR_NOT_READY behavior is unchanged.
|
||
*
|
||
* connected/required mirror the ha_has_write_quorum BODY (ha.c): with the repl
|
||
* back-pointer wired, connected = repl_active_standby_count(repl) and the
|
||
* threshold required = n_peers/2 (integer division — for n_peers=3 → 1). When
|
||
* the back-pointer is NULL (legacy contexts), connected falls back to the
|
||
* standby_endpoints count, matching the gate's own fallback exactly. The quorum
|
||
* decision math is NOT changed here — this only reports what that body decides.
|
||
*/
|
||
void ha_log_quorum_transition(ha_state_t *ha)
|
||
{
|
||
if (ha == NULL)
|
||
return;
|
||
|
||
/* Single-node clusters always hold quorum (ha_has_write_quorum returns true
|
||
* for n_peers <= 1); there is no transition to observe. */
|
||
if (ha->n_peers <= 1)
|
||
return;
|
||
|
||
/* Mirror ha_has_write_quorum's connected-count derivation (ha.c body). */
|
||
int connected;
|
||
if (ha->repl != NULL)
|
||
connected = repl_active_standby_count(ha->repl);
|
||
else
|
||
connected = ha->standby_endpoints ? ha->standby_endpoints->count : 0;
|
||
|
||
int required = (int)(ha->n_peers / 2);
|
||
int lost_now = (connected < required) ? 1 : 0;
|
||
|
||
/* Edge detection: only the caller that flips quorum_lost from its prior value
|
||
* to lost_now emits. CAS expected=!lost_now → desired=lost_now; if the flag
|
||
* already equals lost_now (no transition) the CAS fails and we return
|
||
* silently (Req 8.3). */
|
||
int expected = lost_now ? 0 : 1;
|
||
if (!atomic_compare_exchange_strong_explicit(
|
||
&ha->quorum_lost, &expected, lost_now,
|
||
memory_order_acq_rel, memory_order_acquire))
|
||
return; /* no edge (or another thread already logged it) — no WARN */
|
||
|
||
if (lost_now)
|
||
WARN("HA quorum lost: connected=%d required=%d", connected, required);
|
||
else
|
||
WARN("HA quorum restored: connected=%d required=%d", connected, required);
|
||
|
||
#ifdef XCE_TESTING
|
||
/* Emission-count seam (Req 8.1-8.3 observable): invoked once per WARN. */
|
||
if (ha_quorum_transition_log_hook != NULL)
|
||
ha_quorum_transition_log_hook(lost_now, connected, required);
|
||
#endif
|
||
}
|
||
|
||
#ifdef XCE_TESTING
|
||
/* ───────────────────────────────────────────────────────────────────────────
|
||
* ha_walack_continuation_for_test — task 4.2 GREEN.
|
||
*
|
||
* Test-only shim onto the REAL post-ACK catch-up decision ha_walack_on_ack
|
||
* (config_repl.c, D6). The production walAck branch of
|
||
* repl_ack_decode_and_dispatch (ha.c) calls the SAME ha_walack_on_ack after
|
||
* repl_on_ack, so production and the unit tests exercise identical logic
|
||
* (derive target = max(committed_lsn, last_written); behind target → schedule
|
||
* one single-flight continuation; at parity → repl_complete_catchup, Req
|
||
* 5.1/5.2/7.1). This shim exists only because the production caller decodes a
|
||
* real Cap'n Proto WalAckReply off the wire, which is too heavy to drive from a
|
||
* unit fixture.
|
||
* ─────────────────────────────────────────────────────────────────────────── */
|
||
int ha_walack_continuation_for_test(cluster_t *c, void *ep, int peer_idx,
|
||
uint64_t acked_lsn)
|
||
{
|
||
return ha_walack_on_ack(c, ep, peer_idx, acked_lsn);
|
||
}
|
||
#endif /* XCE_TESTING */
|