fix(ha-wal-replication): apply_wal_entry must not own the WAL CRC chain (bug-778)
On the standby replicated-apply path, ha_apply_single_entry wrote the entry via persist_wal_write_replicated (which correctly set the self-computed CRC chain link p->last_crc), then called apply_wal_entry with a SYNTHETIC header whose crc=0. apply_wal_entry's `p->last_crc = hdr->crc` clobbered the chain link with 0, so every subsequent replicated write stamped prev_crc=0 on disk. The on-disk CRC chain broke at the second replicated entry — invisible while live (standbys serve in-memory state) but fatal on COLD RESTART: replay hit the break, returned RECOVER_TRUNCATED, recovered only the config prefix, and lost all committed jobs (joblist=0). The sticky truncated verdict then drove a promotion-fence availability wedge (bug-777) when every survivor was truncated. Fix — single ownership of the WAL header + CRC chain: - apply_wal_entry is now LOGICAL entity-restore only: signature (p, type, lsn, payload, payload_len); no header. It advances the current_lsn high-water (monotonic) but NEVER touches p->last_crc. - The physical layer is the sole owner of the CRC chain: persist_wal_write_- replicated on the live path, replay_wal/persist_recover on recovery. - ha_apply_single_entry drops the fabricated header and passes entry fields directly. The persist_recover replay caller is updated to the new signature. Verified: new RED->GREEN test test_replicated_apply_chain_survives_cold_restart (drives the real ha_apply_wal_catchup_entry path, cold-replays -> RECOVER_CLEAN, all N recovered); controller suite 0 failures; 3-node rolling-SIGKILL soak on a real ext2/3 fs now passes 5/5 (was wedge+loss at round 3); full cold restart recovers all 6 committed jobs (was 0) with CLEAN verdicts and a normal election. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
276e26c37b
commit
64ab210cbf
7 changed files with 136 additions and 45 deletions
|
|
@ -1302,49 +1302,60 @@ int persist_load_snapshot(persist_t *p, const char *path, uint64_t lsn)
|
|||
*
|
||||
* Returns 0 on success (including skip for checkpoint / unknown types),
|
||||
* negative errno on error (NULL args, oversized payload). */
|
||||
int apply_wal_entry(persist_t* p, const wal_entry_hdr_t* hdr, const void* payload)
|
||||
int apply_wal_entry(persist_t* p, uint32_t type, uint64_t lsn,
|
||||
const void* payload, uint32_t payload_len)
|
||||
{
|
||||
if (!p || !hdr)
|
||||
if (!p)
|
||||
return -EINVAL;
|
||||
|
||||
/* Reject oversized payloads BEFORE any allocation */
|
||||
if (hdr->payload_len > MAX_WAL_PAYLOAD_BYTES) {
|
||||
if (payload_len > MAX_WAL_PAYLOAD_BYTES) {
|
||||
ERROR("apply_wal_entry: payload_len %u exceeds MAX_WAL_PAYLOAD_BYTES %u",
|
||||
hdr->payload_len, MAX_WAL_PAYLOAD_BYTES);
|
||||
payload_len, MAX_WAL_PAYLOAD_BYTES);
|
||||
return -EMSGSIZE;
|
||||
}
|
||||
|
||||
/* LOGICAL entity restore ONLY. The on-disk WAL header, the CRC chain
|
||||
* (p->last_crc), and current_lsn are owned exclusively by the physical
|
||||
* layer — persist_wal_write_replicated on the live replicated path, and
|
||||
* replay_wal/persist_recover on recovery. This function MUST NOT touch
|
||||
* them. (bug-778: it previously set p->last_crc from a synthetic crc=0
|
||||
* header, breaking the on-disk CRC chain and cold-restart recovery.) */
|
||||
|
||||
/* WAL_CHECKPOINT (100) is a local-only marker — standbys never receive it
|
||||
* via replication. Skip silently; do NOT advance LSN. */
|
||||
if (hdr->type == WAL_CHECKPOINT)
|
||||
* via replication. Skip silently. */
|
||||
if (type == WAL_CHECKPOINT)
|
||||
return 0;
|
||||
|
||||
/* Unknown types above WAL_ENTRY_TYPE_MAX: log WARN and skip for forward
|
||||
* compatibility — an older controller can boot on WAL written by a newer
|
||||
* one without hard-failing on unrecognised entry types. */
|
||||
if (hdr->type > WAL_ENTRY_TYPE_MAX) {
|
||||
if (type > WAL_ENTRY_TYPE_MAX) {
|
||||
WARN("apply_wal_entry: unknown type %u > WAL_ENTRY_TYPE_MAX %u"
|
||||
" (lsn=%" PRIu64 ") — skipping for forward compatibility",
|
||||
hdr->type, WAL_ENTRY_TYPE_MAX, hdr->lsn);
|
||||
type, WAL_ENTRY_TYPE_MAX, lsn);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Advance LSN chain: track the highest replicated LSN seen. */
|
||||
/* Advance the current_lsn high-water (monotonic, idempotent). This is a
|
||||
* write-cursor high-water that is safe to bump from multiple paths. The
|
||||
* CRC chain link (p->last_crc) is deliberately NOT touched here — it has a
|
||||
* single owner: the physical writer (persist_wal_write_replicated) and
|
||||
* replay (persist_recover). (bug-778) */
|
||||
pthread_mutex_lock(&p->lock);
|
||||
if (hdr->lsn + 1 > p->current_lsn)
|
||||
p->current_lsn = hdr->lsn + 1;
|
||||
p->last_crc = hdr->crc;
|
||||
if (lsn + 1 > p->current_lsn)
|
||||
p->current_lsn = lsn + 1;
|
||||
pthread_mutex_unlock(&p->lock);
|
||||
|
||||
if (!p->cluster)
|
||||
return 0; /* unit tests without a full cluster — skip entity restore */
|
||||
|
||||
switch (hdr->type) {
|
||||
switch (type) {
|
||||
case WAL_JOB_SUBMIT:
|
||||
/* Payload: Cap'n Proto serialized xce_job_t. Thread the entry LSN
|
||||
* through so the restored job is tagged for the egress watermark
|
||||
* (ha-convergence-recovery 3.3). */
|
||||
restore_job(p->cluster, 0, payload, hdr->payload_len, hdr->lsn);
|
||||
restore_job(p->cluster, 0, payload, payload_len, lsn);
|
||||
break;
|
||||
|
||||
case WAL_JOB_START:
|
||||
|
|
@ -1357,7 +1368,7 @@ int apply_wal_entry(persist_t* p, const wal_entry_hdr_t* hdr, const void* payloa
|
|||
case WAL_JOB_PREASON:
|
||||
case WAL_JOB_RUSAGE:
|
||||
/* Payload: wal_jobstate_t {jobid, state} */
|
||||
if (hdr->payload_len >= sizeof(wal_jobstate_t)) {
|
||||
if (payload_len >= sizeof(wal_jobstate_t)) {
|
||||
const wal_jobstate_t* js = (const wal_jobstate_t*)payload;
|
||||
restore_job_state(p->cluster, js->jobid, js->state);
|
||||
}
|
||||
|
|
@ -1365,13 +1376,13 @@ int apply_wal_entry(persist_t* p, const wal_entry_hdr_t* hdr, const void* payloa
|
|||
|
||||
case WAL_WORKER_CTRL:
|
||||
/* Payload: [uint32_t detail][capnp worker] */
|
||||
if (hdr->payload_len > WORKER_WAL_HDR_SIZE)
|
||||
worker_from_capnp(p->cluster, payload, hdr->payload_len);
|
||||
if (payload_len > WORKER_WAL_HDR_SIZE)
|
||||
worker_from_capnp(p->cluster, payload, payload_len);
|
||||
break;
|
||||
|
||||
case WAL_QUEUE_CTRL:
|
||||
/* Payload: [wal_queuehdr_t][name NUL-terminated] */
|
||||
if (hdr->payload_len > sizeof(wal_queuehdr_t)) {
|
||||
if (payload_len > sizeof(wal_queuehdr_t)) {
|
||||
const wal_queuehdr_t* qh = (const wal_queuehdr_t*)payload;
|
||||
const char* name = (const char*)(payload + sizeof(wal_queuehdr_t));
|
||||
restore_queue_state(p->cluster, name, qh->status);
|
||||
|
|
@ -1380,8 +1391,8 @@ int apply_wal_entry(persist_t* p, const wal_entry_hdr_t* hdr, const void* payloa
|
|||
|
||||
case WAL_NOOP:
|
||||
/* Internal promotion no-op: empty payload, no entity restore, no
|
||||
* hooks. The common-path current_lsn advance (above) is its only
|
||||
* effect — recognised explicitly here rather than via default. */
|
||||
* hooks. current_lsn is advanced by the physical layer (writer /
|
||||
* replay), not here — recognised explicitly rather than via default. */
|
||||
break;
|
||||
|
||||
default:
|
||||
|
|
@ -1581,8 +1592,10 @@ static int replay_wal(persist_t* p, uint64_t from_lsn,
|
|||
continue;
|
||||
}
|
||||
|
||||
/* Apply the entry (P0 stub: no-op) */
|
||||
apply_wal_entry(p, &hdr, payload);
|
||||
/* Logical entity restore. The CRC chain + current_lsn are owned by
|
||||
* this replay loop (current_lsn advanced below; last_crc at line
|
||||
* ~1601), not by apply_wal_entry. */
|
||||
apply_wal_entry(p, hdr.type, hdr.lsn, payload, hdr.payload_len);
|
||||
free(payload);
|
||||
|
||||
/* Advance current_lsn to one past this entry */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue