feat(config-upgrade-snapshot): add materialization-and-hash-verification activation fence
Adds a new, independent fence (ha_state_t.upgrade_materialization_fence) distinct from the existing config-byte serve-fence: that fence tracks whether the locally-held config bytes hash-match the committed marker; this one tracks whether THIS MEMBER'S OWN upgrade-snapshot materialization (task 5's marker-triggered checkpoint) has actually completed for the current committed marker, with the live on-disk config bytes still hash-verifying against it right now. upgrade_materialization_verify(c) mirrors config_activate_committed's shape: reads the current committed marker, checks whether persist.cfgref_present/cfgref_tag/cfgref_sha already reflect it (stamped by every checkpoint via scan_latest_committed_config_ref -- proof this member's own materialization ran for this exact marker), and separately re-reads the LIVE on-disk config bytes fresh on every call (never a cached/embedded copy) to hash-verify them too. Both must hold for the fence to clear. Logs loud only on the 0->1 transition, never exits/aborts, always fail-loud + fence + stay alive + retriable. upgrade_materialization_fenced(c) is the trivial atomic-read accessor. Not yet wired into the dispatch gate, promotion gate, or an automatic retry trigger -- that's task 6.2. Task 6.1 of 19. Requirements: 1.8, 3.2, 3.4
This commit is contained in:
parent
1da26b043e
commit
7b507156a0
4 changed files with 509 additions and 1 deletions
|
|
@ -22,6 +22,7 @@
|
|||
|
||||
#include <openssl/ssl.h>
|
||||
#include <openssl/err.h>
|
||||
#include <openssl/sha.h>
|
||||
|
||||
/* Circular-include safety: cluster_def.h includes ha.h (include-guarded),
|
||||
* and ha.h only forward-declares cluster_t — no cycle. */
|
||||
|
|
@ -105,6 +106,188 @@ extern int config_serve_fenced(const cluster_t *c);
|
|||
* stale serve-fence (non-blocking ordq_put only). */
|
||||
extern int config_schedule_activate_retry(cluster_t *c);
|
||||
|
||||
/* ── config-upgrade-snapshot task 6.1: Activation Fence ───────────────────
|
||||
* upgrade_materialization_verify (below) is a SEPARATE fence from the
|
||||
* config-byte serve-fence above — see ha_state_t's upgrade_materialization_
|
||||
* fence field doc comment (ha.h) for the full "why a second fence" rationale.
|
||||
* It reads the SAME committed-marker authority config_activate_committed
|
||||
* itself reads. */
|
||||
extern int config_committed_marker(const cluster_t *c, uint64_t *version_tag,
|
||||
uint8_t sha256_out[32]);
|
||||
|
||||
/* upgrade_materialization_read_live_config_sha256 (config-upgrade-snapshot
|
||||
* task 6.1, Req 3.2/3.4): resolve and hash THIS MEMBER'S CURRENTLY-ACTIVE
|
||||
* config bytes fresh, right now — never a cached/frozen copy (in particular,
|
||||
* never c->persist.cfgref_sha, which is only a snapshot-time REFERENCE
|
||||
* captured at whatever moment the last checkpoint ran, not proof the bytes
|
||||
* are still current this instant).
|
||||
*
|
||||
* DELIBERATE DESIGN CHOICE: this calls the same two already-PUBLIC
|
||||
* config_repl.c entry points (conf_read_current_filename, conf_read_raw)
|
||||
* that config_activate_committed's own local_config_sha256 helper uses
|
||||
* internally, rather than re-deriving the <workdir>/<name>/conf/current ->
|
||||
* cluster.bin.<hex> resolution a third time (persist.c's
|
||||
* read_active_config_bytes already duplicates that resolution inline once,
|
||||
* with its own documented rationale for why persist.c specifically avoids a
|
||||
* persist.c -> config_repl.c call edge). ha.c carries no such constraint —
|
||||
* it already depends on config_repl.c directly for config_activate_
|
||||
* committed/config_serve_fenced/config_schedule_activate_retry/conf_read_raw
|
||||
* above, so calling these two existing, already-public accessors here is
|
||||
* in-boundary, ordinary integration along an established edge, not a new
|
||||
* layering violation or a third duplicate implementation of path validation.
|
||||
*
|
||||
* Returns 0 with out_sha populated on success; -ENOENT when no local config
|
||||
* bytes exist yet (benign — e.g. c->myconf unset, or no conf/current pointer
|
||||
* written yet); another negative errno surfaces a genuine read/validation
|
||||
* failure from the underlying calls. */
|
||||
static int upgrade_materialization_read_live_config_sha256(cluster_t *c,
|
||||
uint8_t out_sha[32])
|
||||
{
|
||||
extern int conf_read_current_filename(const char *workdir, char *out,
|
||||
size_t outsz);
|
||||
extern int conf_read_raw(const char *conffile, uint8_t **bytes,
|
||||
size_t *len);
|
||||
|
||||
myconf_t *mc = c->myconf;
|
||||
if (!mc || !mc->workdir || !mc->name)
|
||||
return -ENOENT;
|
||||
|
||||
/* effective_dir = workdir/name — must match conf_save_config's own path
|
||||
* construction (config_repl.c) exactly, mirroring local_config_sha256. */
|
||||
char conf_dir[4096];
|
||||
snprintf(conf_dir, sizeof(conf_dir), "%s/%s", mc->workdir, mc->name);
|
||||
|
||||
char fname[256] = {0};
|
||||
int rc = conf_read_current_filename(conf_dir, fname, sizeof(fname));
|
||||
if (rc != 0)
|
||||
return -ENOENT; /* no local bytes yet — caller fences + retries */
|
||||
|
||||
char path[4352];
|
||||
snprintf(path, sizeof(path), "%s/conf/%s", conf_dir, fname);
|
||||
|
||||
uint8_t *bytes = NULL;
|
||||
size_t blen = 0;
|
||||
rc = conf_read_raw(path, &bytes, &blen);
|
||||
if (rc != 0)
|
||||
return -ENOENT;
|
||||
|
||||
SHA256(bytes, blen, out_sha);
|
||||
free(bytes);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* upgrade_materialization_verify (config-upgrade-snapshot task 6.1, Req 1.8,
|
||||
* 3.2, 3.4): the Activation Fence's own verify function — mirrors
|
||||
* config_activate_committed's exact shape (config_repl.c:304-377) for a
|
||||
* DIFFERENT condition. config_serve_fence (config_repl.c) asks "do the
|
||||
* locally-held config BYTES hash-match the committed marker?"; THIS fence
|
||||
* asks "has THIS MEMBER'S OWN materialization (the upgrade-snapshot
|
||||
* checkpoint, config-upgrade-snapshot task 5) actually completed for the
|
||||
* CURRENT committed marker, with matching config bytes?" — a distinct,
|
||||
* independently-trackable condition: bytes can already hash-match
|
||||
* (config_serve_fence clear) while this member's own checkpoint for that
|
||||
* exact marker has not run yet, e.g. immediately after a marker commits but
|
||||
* before apply_committed_entry's WAL_CONFIG_MARKER case has actually fired
|
||||
* and written a checkpoint.
|
||||
*
|
||||
* "Materialization complete" is determined via c->persist.cfgref_present /
|
||||
* cfgref_tag / cfgref_sha — persist.c's scan_latest_committed_config_ref,
|
||||
* invoked from EVERY checkpoint (persist_checkpoint_write_locked), stamps
|
||||
* these fields with the latest committed WAL_CONFIG_MARKER as of that
|
||||
* checkpoint. cfgref_present && cfgref_tag == marker_tag && cfgref_sha ==
|
||||
* marker_sha therefore means "some checkpoint on THIS member has already
|
||||
* captured a reference to this exact committed marker" — materialization for
|
||||
* it has happened. This is the SAME field config_committed_marker's own
|
||||
* compaction fallback already reads (config_repl.c), so this is a read of an
|
||||
* existing, already-correct signal, not a new one.
|
||||
*
|
||||
* That alone is not sufficient: the checkpoint's cfgref_sha is a REFERENCE
|
||||
* captured AT CHECKPOINT TIME, not proof the bytes are STILL current right
|
||||
* now (the WAL marker's raft-commit and the R7 config-push byte delivery are
|
||||
* two independent channels with no strict ordering guarantee). So this
|
||||
* function ALSO re-reads the live on-disk config bytes fresh on every call
|
||||
* (never c->persist.cfgref_sha, never any other cached/embedded copy) via
|
||||
* upgrade_materialization_read_live_config_sha256 (above), and hash-verifies
|
||||
* that fresh read against the committed marker too. Both must hold for the
|
||||
* fence to clear.
|
||||
*
|
||||
* Never exits/aborts on this condition (controller/CLAUDE.md's never-self-
|
||||
* terminate invariant) — always fail loud + fence + stay alive + retriable,
|
||||
* mirroring config_activate_committed exactly: logs loud only on the 0->1
|
||||
* fence transition (never spams on repeated failed retries), returns
|
||||
* -EAGAIN on failure so the caller retries, clears + returns 0 on success. */
|
||||
int upgrade_materialization_verify(cluster_t *c)
|
||||
{
|
||||
if (!c)
|
||||
return -EINVAL;
|
||||
|
||||
/* 1. The committed marker is the authority for the target upgrade. An
|
||||
* absent marker (no configuration change has ever committed on this
|
||||
* cluster, or the marker has not committed yet) is NOT a divergence —
|
||||
* clear any stale fence and succeed, mirroring config_activate_
|
||||
* committed's own -ENOENT handling exactly. */
|
||||
uint64_t marker_tag = 0;
|
||||
uint8_t marker_sha[32];
|
||||
int rc = config_committed_marker(c, &marker_tag, marker_sha);
|
||||
if (rc == -ENOENT) {
|
||||
atomic_store_explicit(&c->ha.upgrade_materialization_fence, 0,
|
||||
memory_order_release);
|
||||
return 0;
|
||||
}
|
||||
if (rc != 0)
|
||||
return rc; /* -EINVAL / -EIO surfaced to the caller */
|
||||
|
||||
/* 2. Materialization-complete signal: has ANY checkpoint on THIS member
|
||||
* already captured a reference to this EXACT committed marker? */
|
||||
bool materialized = c->persist.cfgref_present &&
|
||||
c->persist.cfgref_tag == marker_tag &&
|
||||
memcmp(c->persist.cfgref_sha, marker_sha, 32) == 0;
|
||||
|
||||
/* 3. Hash-verify: re-read the LIVE, currently-active config bytes fresh
|
||||
* on every attempt and compare against the committed marker. */
|
||||
uint8_t live_sha[32];
|
||||
int live_rc = upgrade_materialization_read_live_config_sha256(c, live_sha);
|
||||
bool bytes_verified = (live_rc == 0) &&
|
||||
(memcmp(live_sha, marker_sha, 32) == 0);
|
||||
|
||||
if (!materialized || !bytes_verified) {
|
||||
int was = atomic_exchange_explicit(&c->ha.upgrade_materialization_fence,
|
||||
1, memory_order_acq_rel);
|
||||
if (!was)
|
||||
ERROR("upgrade materialization fence: committed config marker "
|
||||
"tag=%016" PRIx64 " -- materialization %s "
|
||||
"(cfgref_present=%d cfgref_tag=%016" PRIx64 "), live config "
|
||||
"bytes %s (rc=%d) -- member's OWN upgrade-snapshot "
|
||||
"materialization for this marker is INCOMPLETE or "
|
||||
"MISMATCHED; member NON-SERVING and NON-PROMOTABLE under the "
|
||||
"new configuration, staying alive and retrying "
|
||||
"(config-upgrade-snapshot R1.8/3.2/3.4)",
|
||||
marker_tag, materialized ? "captured" : "NOT YET captured",
|
||||
c->persist.cfgref_present, c->persist.cfgref_tag,
|
||||
bytes_verified ? "hash-match" : "do NOT hash-match", live_rc);
|
||||
return -EAGAIN;
|
||||
}
|
||||
|
||||
atomic_store_explicit(&c->ha.upgrade_materialization_fence, 0,
|
||||
memory_order_release);
|
||||
INFO("upgrade materialization VERIFIED for committed config marker "
|
||||
"tag=%016" PRIx64 " -- materialization complete, live config bytes "
|
||||
"hash-verified", marker_tag);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* upgrade_materialization_fenced: trivial atomic-read accessor — mirrors
|
||||
* config_serve_fenced (config_repl.c:262-268) exactly, for the new
|
||||
* independent fence. Pure atomic read (no I/O), safe on the election/
|
||||
* dispatch path. */
|
||||
int upgrade_materialization_fenced(const cluster_t *c)
|
||||
{
|
||||
if (!c)
|
||||
return 0;
|
||||
return atomic_load_explicit(&c->ha.upgrade_materialization_fence,
|
||||
memory_order_acquire) ? 1 : 0;
|
||||
}
|
||||
|
||||
static void on_vote_connected(endpoint_t *ep);
|
||||
static void on_pre_vote_connected(endpoint_t *ep);
|
||||
static int vote_ep_err_handler(void *ep_raw);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue