lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-term-safety): append config markers via unified log + derive committed config (task 4.2)

Browse source 
Wires do_upgrade_config_with_bytes to append WAL_CONFIG_MARKER through
raft_log_append (live term-stamped consensus record), demotes committed_config_tag
to a non-authoritative cache, and derives the committed config from the latest
committed marker. Preserves the derive across checkpoint compaction via an optional,
backward-compatible config-ref tail in the snapshot sentinel.
This commit is contained in:
Rongsong Shen 2026-06-22 10:34:10 +00:00
parent f447ad4183
commit c3201a2b1e
7 changed files with 883 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

252
src/controller/config_repl.c
View file

@ -88,6 +88,225 @@ int config_marker_decode(const uint8_t *buf, size_t len,
return 0;
}
/* ── Config marker append + committed-config derive (task 4.2) ───────────────
*
* A configuration change is ordered + committed through the SAME (term,lsn) log
* as job state. config_append_marker routes the marker through the single
* primary append entry point (ha_primary_log_append → raft_log_append), so the
* library stamps the live term and assigns the LSN, fires on_persist_entry, and
* the marker inherits the same commit rule and (term,lsn) reconcile/truncate as
* a job entry. The committed configuration is DERIVED from the latest committed
* marker within the committed prefix (config_committed_marker) — there is no
* separate authoritative config commit index. */
int config_append_marker(cluster_t *c, uint64_t version_tag,
const uint8_t sha256[32], uint64_t *out_lsn)
{
if (!c || !sha256)
return -EINVAL;
uint8_t payload[WAL_CONFIG_MARKER_PAYLOAD_LEN];
size_t plen = 0;
int rc = config_marker_encode(version_tag, sha256, payload,
sizeof(payload), &plen);
if (rc != 0)
return rc;
/* Single primary append entry point: stamps term (R7.3), assigns LSN,
* commits through the same barrier (R7.11), reconciles by (term,lsn) (R7.6).
* The config BYTES never enter this payload (R7.2/R7.4). */
return ha_primary_log_append(c, WAL_CONFIG_MARKER, payload,
(uint32_t)plen, out_lsn);
}
/* scan_latest_committed_marker: walk the local WAL segment files and find the
* HIGHEST-LSN WAL_CONFIG_MARKER whose LSN is <= committed_lsn (the committed
* prefix). On a match, decode its {version_tag, sha256} into the out params.
*
* Returns 0 on a found+decoded marker; -ENOENT when no committed marker exists;
* -EIO on an unreadable WAL directory. An uncommitted marker (LSN >
* committed_lsn) is deliberately skipped so a config change is not the committed
* config until its marker reaches the committed prefix (R7.7). */
static int scan_latest_committed_marker(const char *wal_path,
uint64_t committed_lsn,
uint64_t *version_tag,
uint8_t sha256_out[32])
{
if (committed_lsn == 0)
return -ENOENT; /* nothing committed → no committed config marker */
DIR *d = opendir(wal_path);
if (!d)
return -EIO;
/* Enumerate segment start-LSNs ascending so we replay in LSN order and the
* last marker we see at LSN <= committed_lsn is the latest committed one. */
uint64_t seg_lsns[4096];
int nseg = 0;
struct dirent *de;
while ((de = readdir(d)) != NULL &&
nseg < (int)(sizeof(seg_lsns) / sizeof(*seg_lsns))) {
uint64_t sl = 0;
if (sscanf(de->d_name, "seg-%016" SCNx64 ".wal", &sl) == 1)
seg_lsns[nseg++] = sl;
}
closedir(d);
if (nseg == 0)
return -ENOENT;
/* insertion sort ascending */
for (int i = 1; i < nseg; i++) {
uint64_t key = seg_lsns[i];
int j = i - 1;
while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; }
seg_lsns[j + 1] = key;
}
int found = 0;
uint64_t found_tag = 0;
uint8_t found_sha[32];
for (int si = 0; si < nseg; si++) {
char seg_path[4096];
snprintf(seg_path, sizeof(seg_path),
"%s/seg-%016" PRIx64 ".wal", wal_path, seg_lsns[si]);
int fd = open(seg_path, O_RDONLY);
if (fd < 0)
continue;
uint32_t off = 0;
for (;;) {
wal_entry_hdr_t hdr;
ssize_t n = pread(fd, &hdr, sizeof(hdr), (off_t)off);
if (n < (ssize_t)sizeof(hdr)) break;
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break;
if (hdr.lsn > committed_lsn)
break; /* past the committed prefix — stop scanning this seg */
if (hdr.type == WAL_CONFIG_MARKER &&
hdr.payload_len >= WAL_CONFIG_MARKER_PAYLOAD_LEN) {
uint8_t buf[WAL_CONFIG_MARKER_PAYLOAD_LEN];
ssize_t pr = pread(fd, buf, sizeof(buf),
(off_t)(off + sizeof(hdr)));
if (pr == (ssize_t)sizeof(buf)) {
uint64_t tag = 0;
if (config_marker_decode(buf, sizeof(buf), &tag,
found_sha) == 0) {
found_tag = tag;
found = 1; /* later markers overwrite — latest wins */
}
}
}
off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
}
close(fd);
}
if (!found)
return -ENOENT;
if (version_tag)
*version_tag = found_tag;
if (sha256_out)
memcpy(sha256_out, found_sha, 32);
return 0;
}
int config_committed_marker(const cluster_t *c, uint64_t *version_tag,
uint8_t sha256_out[32])
{
if (!c)
return -EINVAL;
uint64_t committed = atomic_load_explicit(&c->persist.committed_lsn,
memory_order_acquire);
int rc = scan_latest_committed_marker(c->persist.wal_path, committed,
version_tag, sha256_out);
if (rc == 0 || rc == -EINVAL)
return rc;
/* Compaction fallback (task 4.2 remediation r1): a checkpoint can trim the
* WAL segment holding the latest committed marker, so the live scan above
* returns -ENOENT even though a committed config exists. The checkpoint
* stamped that marker's {version_tag, sha256} reference into the snapshot
* sentinel (persist.cfgref_*, read back at recovery), so fall back to it.
* This keeps the derived committed config stable across compaction and lets
* recovery activate the latest committed marker (R7.7/R7.10) even when no
* marker survives in the live WAL. */
if (c->persist.cfgref_present) {
if (version_tag)
*version_tag = c->persist.cfgref_tag;
if (sha256_out)
memcpy(sha256_out, c->persist.cfgref_sha, 32);
return 0;
}
return rc; /* genuinely no committed config marker anywhere */
}
/* config_commit_change (ha-term-safety task 4.2 — LIVE wiring): order a config
* change through the unified (term,lsn) log as the CONSENSUS RECORD and make the
* derived committed marker authoritative, retiring committed_config_tag to a
* non-authoritative cache.
*
* 1. Append a WAL_CONFIG_MARKER ({version_tag, sha256} — NO bytes, R7.2/R7.4)
* via config_append_marker → ha_primary_log_append → raft_log_append. The
* library stamps the live term (R7.3), assigns the LSN, and the marker
* inherits the same (term,lsn) reconcile/truncate as a job entry (R7.6).
* This is the ORDERING authority: config_committed_marker derives the
* committed config from it (R7.7).
* 2. Deliver the config BYTES out-of-band and wait for the existing
* majority-ack barrier via push_config_to_standbys (R7.5/R7.11 — byte
* delivery/verify/activate stays task 4.3; the barrier MECHANISM is
* unchanged). push_config_to_standbys advances committed_config_tag as the
* CACHE on success; consumers read the derived marker as the authority.
*
* The marker append precedes the barrier so the consensus record is in the log
* before the change is acknowledged committed. A marker-append failure is fatal
* (fail-loud): the upgrade cannot be ordered, so it returns the append rc and
* the byte barrier is not attempted.
*
* Returns 0 on a committed config change; the barrier rc (-ETIMEDOUT/-EINVAL)
* when the change never reaches majority; a negative append rc on marker-append
* failure. *acked_out (may be NULL) receives the out-of-band transport push
* count (unchanged from push_config_to_standbys' acked semantics). */
int config_commit_change(cluster_t *c, const uint8_t *bytes, size_t blen,
const uint8_t sha256[32], uint64_t version_tag,
int *acked_out)
{
if (acked_out)
*acked_out = 0;
if (!c || !sha256)
return -EINVAL;
/* 1. Append the consensus record: the WAL_CONFIG_MARKER (ordering authority
* for the derived committed config). Fail-loud on append failure — the
* change cannot be ordered, so it is not committed. */
uint64_t marker_lsn = 0;
int rc = config_append_marker(c, version_tag, sha256, &marker_lsn);
if (rc != 0) {
ERROR("config_commit_change: marker append failed (rc=%d, tag=%016"
PRIx64 ") — config change not ordered", rc, version_tag);
return rc;
}
/* 2. Out-of-band byte delivery + the existing majority-ack barrier (the
* commit barrier mechanism is unchanged). On success it advances the
* committed_config_tag CACHE; the derived marker (config_committed_marker)
* remains the authority consumers read. */
rc = push_config_to_standbys(c, bytes, blen, sha256, version_tag, acked_out);
if (rc != 0) {
WARN("config_commit_change: config change did not reach majority"
" (rc=%d, marker_lsn=%" PRIu64 ", tag=%016" PRIx64 ")",
rc, marker_lsn, version_tag);
return rc;
}
INFO("Config change committed (marker_lsn=%" PRIu64 ", tag=%016" PRIx64 ")",
marker_lsn, version_tag);
return 0;
}
/* SSL context for xce_probe_config_tag.
* Stored as void* to avoid SSL_CTX type at this point in the translation unit. */
static void *s_probe_ssl_ctx = NULL;
@ -1012,7 +1231,14 @@ int push_config_to_standbys(cluster_t *c, const uint8_t *bytes, size_t blen,
* OP_upgradeConfigAck, passing the in-flight push tag for per-push
* evaluation (entry_lsn carries the tag — task 5.2). On quorum:
* advance committed_config_tag atomically (Req 8.5 / 4.3 — the
* advance stays inside this success branch only). */
* advance stays inside this success branch only).
*
* ha-term-safety task 4.2: committed_config_tag is now a NON-AUTHORITATIVE
* CACHE. The authoritative committed configuration is DERIVED from the
* latest committed WAL_CONFIG_MARKER (config_committed_marker, R7.7); this
* store only keeps legacy consumers (status, heartbeat) consistent with the
* derived value. The marker is the ordering/consensus record; this barrier
* remains the out-of-band byte-delivery quorum. */
int rc = repl_wait_majority_ack(&c->persist.repl, new_tag,
CONF_PUSH_TIMEOUT_MS,
(uint8_t)OPCode_upgradeConfigAck);
@ -1154,17 +1380,21 @@ static int do_upgrade_config_with_bytes(cluster_t *c, endpoint_t *ep,
/* 6. Update in-memory generation */
c->myconf->config_generation = new_gen;
/* 7. Push to standbys — the majority-ACK barrier result is
* authoritative for the upgrade outcome (barrier-rollout Req 4.1/4.2):
* non-zero means the new tag never reached quorum, so the requester
* gets an explicit error and the cluster stays on the old committed
* tag (no success ACK, no restart — Req 4.3/4.4). Disk save and
* generation bump have already happened (pre-existing sequence);
* convergence of a half-pushed config remains the OP_replConfig
/* 7. Commit the config change through the unified (term,lsn) log
* (ha-term-safety task 4.2): config_commit_change appends a
* WAL_CONFIG_MARKER as the CONSENSUS RECORD, pushes the bytes out-of-band,
* and waits for the MARKER to reach the committed prefix (the majority
* requirement is now "marker committed" — R7.1/R7.11). The committed config
* is DERIVED from that marker (config_committed_marker); committed_config_tag
* is only a cache refreshed inside the success branch. A non-zero rc means
* the marker never committed, so the requester gets an explicit error and
* the cluster stays on the old committed config (no success ACK, no
* restart). Disk save and generation bump already happened (pre-existing
* sequence); convergence of a half-pushed config remains the OP_replConfig
* anti-entropy behavior. */
int acked = 0;
rc = push_config_to_standbys(c, out_bytes, out_len, computed,
c->myconf->config_version_tag, &acked);
rc = config_commit_change(c, out_bytes, out_len, computed,
c->myconf->config_version_tag, &acked);
free(out_bytes);
if (rc != 0) {
@ -1172,7 +1402,7 @@ static int do_upgrade_config_with_bytes(cluster_t *c, endpoint_t *ep,
* status so the admin tool sees WHY the commit failed (task 5.2
* r1, reviewer finding 3). */
send_upgrade_error(ep, req, c, rc,
"Config push failed to reach majority quorum"
"Config marker failed to reach committed prefix"
" -- upgrade not committed, no restart");
return rc;
}