lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-term-safety): out-of-band config bytes, hash-verify-before-activate, serve-fence (task 4.3)

Browse source 
Wires config_committed_marker to a production consumer (config_activate_committed):
verifies pushed bytes' sha256 against the committed marker before activation, fences
(loud, non-serving, stay-alive) on mismatch, and activates the latest committed marker
on recovery. Enforced via ha_dispatch_gate + promotion gate.

Fixes bug-981: a 2nd+ config upgrade could permanently fence a standby on a
push-before-commit ordering. Adds an off-iomgr delaymgr retry scheduled from the
commit-advance hook (atomic read + non-blocking enqueue only on the ACK thread) that
re-activates and clears the stale fence once the new marker commits and bytes verify.
This commit is contained in:
Rongsong Shen 2026-06-22 11:47:15 +00:00
parent c3201a2b1e
commit be4c1f0721
7 changed files with 1074 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

233
src/controller/config_repl.c
View file

@ -244,6 +244,211 @@ int config_committed_marker(const cluster_t *c, uint64_t *version_tag,
return rc; /* genuinely no committed config marker anywhere */
}
/* ── Config byte delivery: hash-verify-before-activate + serve-fence (task 4.3) ─
*
* config_committed_marker (4.2) is the AUTHORITY for which configuration is in
* effect; the configuration BYTES travel out-of-band on the existing config-push
* transport (OPCode_replConfigPush / handle_repl_config_push), never inside a WAL
* payload (R7.4/R7.5). config_activate_committed keys that byte delivery to the
* committed marker's {version_tag, sha256}: it VERIFIES the locally-held bytes'
* sha256 against the committed marker BEFORE activating (R7.8), and on a mismatch
* (or bytes absent locally) keeps the member from serving under the new config and
* KEEPS RETRYING rather than activating mismatched config (R7.9) — the loud
* non-serving fence that NEVER self-terminates. On recovery it activates the config
* referenced by the LATEST committed marker (R7.10).
*/
int config_serve_fenced(const cluster_t *c)
{
if (!c)
return 0;
return atomic_load_explicit(&c->ha.config_serve_fence,
memory_order_acquire) ? 1 : 0;
}
/* Read the locally-held config bytes (resolved via myconf->conffile or
* conf/current) and compute their sha256 into out_sha. Returns 0 with the hash on
* success; -ENOENT when no local config bytes exist yet (the out-of-band push has
* not delivered them). The bytes are the out-of-band-delivered config, NEVER a WAL
* payload (R7.4/R7.5). */
static int local_config_sha256(cluster_t *c, uint8_t out_sha[32])
{
myconf_t *mc = c->myconf;
if (!mc || !mc->workdir || !mc->name)
return -EINVAL;
/* Resolve the current locally-held versioned config file. */
char conf_dir[4096];
snprintf(conf_dir, sizeof(conf_dir), "%s/%s", mc->workdir, mc->name);
char fname[256] = {0};
int rc = conf_read_current_filename(conf_dir, fname, sizeof(fname));
if (rc != 0)
return -ENOENT; /* no local bytes yet — caller fences + retries */
char path[4352];
snprintf(path, sizeof(path), "%s/conf/%s", conf_dir, fname);
uint8_t *bytes = NULL;
size_t blen = 0;
rc = conf_read_raw(path, &bytes, &blen);
if (rc != 0)
return -ENOENT;
SHA256(bytes, blen, out_sha);
free(bytes);
return 0;
}
int config_activate_committed(cluster_t *c)
{
if (!c)
return -EINVAL;
/* 1. The committed marker is the authority for the target config (R7.7/R7.10:
* on recovery this is the LATEST committed marker). */
uint64_t marker_tag = 0;
uint8_t marker_sha[32];
int rc = config_committed_marker(c, &marker_tag, marker_sha);
if (rc == -ENOENT) {
/* No committed config marker exists (legacy / first boot, or the marker
* has not committed yet). Absent-marker is NOT a hash MISMATCH, so it must
* NOT fence the node (a benign "awaiting bytes/marker" state, never a hard
* divergence fence — R7.9/R7.10). FALL BACK to the existing
* load_primary_config behavior so any locally-held bytes are activated
* (config_version_tag is set for the freshness gate and the push ACK).
* A missing local config file (-ENOENT from load_primary_config) is also
* benign first-boot — the node fetches config post-election. Clear any
* stale fence either way: an absent marker is never a divergence. */
atomic_store_explicit(&c->ha.config_serve_fence, 0, memory_order_release);
int lrc = load_primary_config(c);
if (lrc != 0 && lrc != -ENOENT)
WARN("config_activate_committed: no committed marker; "
"load_primary_config fallback rc=%d", lrc);
return 0;
}
if (rc != 0)
return rc; /* -EINVAL / -EIO surfaced to the caller */
/* 2. Hash-verify the LOCALLY-HELD (out-of-band-delivered) bytes against the
* committed marker BEFORE activating (R7.8). */
uint8_t local_sha[32];
rc = local_config_sha256(c, local_sha);
bool verified = (rc == 0 && memcmp(local_sha, marker_sha, 32) == 0);
if (!verified) {
/* 3. MISMATCH or bytes not yet present locally: the bytes for this
* committed marker have not arrived/verified out-of-band. Keep the
* member NON-SERVING under the new config and KEEP RETRYING — the loud
* non-self-terminating fence (R7.9). NEVER activate mismatched bytes.
* Stays ALIVE: returns -EAGAIN so the caller retries when the matching
* bytes arrive via the existing config-push transport. */
int was = atomic_exchange_explicit(&c->ha.config_serve_fence, 1,
memory_order_acq_rel);
if (!was)
ERROR("config serve-fence: committed config marker tag=%016" PRIx64
" references bytes whose sha256 is not present/matching locally "
"(rc=%d) -- member NON-SERVING under the new config, awaiting "
"out-of-band config-push bytes; staying alive and retrying",
marker_tag, rc);
return -EAGAIN;
}
/* 4. VERIFIED: the locally-held bytes hash-match the committed marker —
* activate the configuration (load it into myconf), then clear the fence. */
rc = load_primary_config(c);
if (rc != 0) {
/* The verified bytes failed to load (e.g. decode error). Stay fenced and
* retrying rather than serving an unloadable config — never self-terminate. */
int was = atomic_exchange_explicit(&c->ha.config_serve_fence, 1,
memory_order_acq_rel);
if (!was)
ERROR("config serve-fence: hash-verified config bytes for committed "
"marker tag=%016" PRIx64 " failed to load (rc=%d) -- member "
"NON-SERVING, staying alive and retrying", marker_tag, rc);
return rc;
}
atomic_store_explicit(&c->ha.config_serve_fence, 0, memory_order_release);
INFO("config activated from committed marker tag=%016" PRIx64
" (hash-verified out-of-band bytes)", marker_tag);
return 0;
}
/* ── Commit-advance retry trigger (task 4.3 remediation r2, Finding D / R7.9) ──
*
* The config MARKER (WAL stream) and the config BYTES (out-of-band config-push)
* arrive on INDEPENDENT transports. On a 2nd-or-later upgrade the bytes can be
* saved locally BEFORE the standby's committed prefix advances past the NEW
* marker: at that moment config_committed_marker still returns the OLDER marker,
* so the freshly-saved NEW bytes hash-MISMATCH it and config_activate_committed
* sets the serve-fence (-EAGAIN). The NEW marker commits later via the WAL path,
* but on_commit_advanced runs on the iomgr ACK thread and MUST NOT do disk I/O,
* so it cannot re-activate inline; and the primary's "config-behind" re-push net
* does NOT fire because conf_save_config already advanced this member's advertised
* config_version_tag. Without a retry the standby stays PERMANENTLY fenced (and
* non-promotable) for that upgrade until reboot — a §7.9 liveness violation.
*
* The fix: when commit advances WHILE this member is serve-fenced, schedule a
* one-shot OFF-IOMGR delaymgr task that re-runs config_activate_committed. The
* delaymgr worker runs on a thread allowed to block on the disk reads
* (scan_latest_committed_marker pread + load_primary_config). Once the NEW marker
* is in the committed prefix AND the locally-held bytes hash-match it, the retry
* activates and CLEARS the fence; a genuine still-MISMATCH (or no-bytes-yet) keeps
* the fence and the next commit-advance reschedules. -ENOENT still falls back
* without fencing (the no-marker path is benign). Idempotent + self-limiting: the
* scheduler is a no-op once unfenced, so there is no busy-spin. */
/* config_activate_retry_fn: the delaymgr task body (runs OFF the iomgr ACK
* thread). Re-runs config_activate_committed so a freshly-committed config marker
* is activated — and a stale serve-fence cleared — once its out-of-band bytes
* verify. `arg` is the cluster_t* (passed directly; no heap ctx to free, so the
* delaymgr node-killer that frees only the delay_task_t never leaks). */
int config_activate_retry_fn(void *arg)
{
cluster_t *c = (cluster_t *)arg;
if (!c)
return -EINVAL;
if (!config_serve_fenced(c))
return 0; /* already cleared by another path — nothing to retry */
int rc = config_activate_committed(c);
if (rc == 0)
INFO("config commit-advance retry: serve-fence CLEARED — newly committed "
"config marker activated (hash-verified out-of-band bytes)");
else if (rc == -EAGAIN)
TRACE("config commit-advance retry: still fenced (marker/bytes not yet "
"matching) — will retry on the next commit advance");
else
WARN("config commit-advance retry: config_activate_committed rc=%d", rc);
return rc;
}
/* config_schedule_activate_retry: enqueue config_activate_retry_fn on the
* delaymgr (NON-BLOCKING — delay_task_ms only does ordq_put). Called from the
* commit-advance path (iomgr ACK thread) so activation's disk I/O runs OFF that
* thread (ha-async-io invariant). No-op when the member is NOT serve-fenced (so
* the steady state schedules nothing) or when the delaymgr is not running (the
* unit-test fixture path, which never inits delaymgr — avoids leaking a task into
* an un-drained queue). Returns 0 on enqueue/no-op, negative on enqueue failure. */
int config_schedule_activate_retry(cluster_t *c)
{
if (!c)
return -EINVAL;
if (!config_serve_fenced(c))
return 0; /* steady state: nothing to retry, schedule nothing */
if (!c->delaymgr.running)
return 0; /* delaymgr not running (test fixture) — caller drives directly */
/* One-shot, minimal delay: run promptly but off the iomgr ACK thread. */
int rc = delay_task_ms(&c->delaymgr, 1, 1,
(delay_task_entry_fn_t)config_activate_retry_fn, c);
if (rc < 0)
WARN("config commit-advance retry: delay_task_ms failed: %d — fence will "
"clear on a later commit advance or config-push", rc);
return rc;
}
/* config_commit_change (ha-term-safety task 4.2 — LIVE wiring): order a config
* change through the unified (term,lsn) log as the CONSENSUS RECORD and make the
* derived committed marker authoritative, retiring committed_config_tag to a
@ -991,7 +1196,14 @@ int handle_repl_config_push(cluster_t *c, endpoint_t *ep, xce_request_t *req)
return -EBADMSG;
}
/* Save to disk and reload */
/* ha-term-safety task 4.3 (remediation r1, Finding A): SAVE the pushed bytes
* to disk out-of-band, but do NOT activate them here. config_activate_committed
* is the SOLE activation point — it loads the bytes into the live config ONLY
* after hash-verifying them against the latest committed WAL_CONFIG_MARKER
* (R7.5/R7.8). A pre-emptive load_primary_config here would make the just-pushed
* bytes LIVE before the gate runs, turning verify-before-activate into an
* after-the-fact overlay. The bytes travel out-of-band via this transport and
* NEVER inside a WAL payload (R7.4/R7.5). */
int rc = conf_save_config(c->myconf, rcpr->raw_bytes, rcpr->raw_bytes_len,
rcpr->raw_sha256);
if (rc != 0) {
@ -999,14 +1211,19 @@ int handle_repl_config_push(cluster_t *c, endpoint_t *ep, xce_request_t *req)
return rc;
}
rc = load_primary_config(c);
if (rc != 0) {
WARN("handle_repl_config_push: load_primary_config failed: %d", rc);
return rc;
}
INFO("handle_repl_config_push: config bytes saved out-of-band "
"(pushed tag=%016" PRIx64 ") — activation gated on committed marker",
rcpr->version_tag);
INFO("handle_repl_config_push: config saved and loaded (tag=%016" PRIx64 ")",
c->myconf->config_version_tag);
/* Key activation to the committed marker. On mismatch (the marker references
* different/later bytes than the ones just pushed) config_activate_committed
* sets the loud non-serving serve-fence and returns -EAGAIN; the member STAYS
* ALIVE and a later config-push / commit-advance retries (R7.9). -ENOENT
* (no committed marker yet — first boot/bootstrap) falls back to loading the
* locally-held bytes; a successful verify activates. */
int act_rc = config_activate_committed(c);
if (act_rc != 0 && act_rc != -EAGAIN)
WARN("handle_repl_config_push: config_activate_committed rc=%d", act_rc);
/* ACK the primary */
repl_config_push_reply_t reply_body = {