fix(ha-wal-replication): standby durably persists replicated WAL before ACK
A standby applied replicated WAL entries via apply_wal_entry (the replay/ in-memory path) which restores entities + advances applied_lsn but NEVER writes the entry to the standby's WAL segment. The standby then ACKed (feeding the primary's commit majority) without persisting — a Raft durability violation: entries counted toward a commit could be lost on standby restart, and standby WAL files stayed empty (contradicting steering: log[i] = WAL segments for every node). This is why ctrl-2/ctrl-3 received + acked the streamed own-term NOOP yet kept empty seg-*.wal. Fix: new persist_wal_write_replicated(p, lsn, type, payload, len) writes the entry to the local WAL at the PRIMARY's lsn (no lsn-reassignment), with a self-consistent intra-segment CRC chain (the streamed entry carries no term/crc; each node chains independently and recovery validates intra-segment), fdatasyncs for durability, advances current_lsn = max(current_lsn, lsn+1), rotates segments, and does NOT re-broadcast. Wired into ha_apply_single_entry as persist(durable) -> apply(in-memory) -> ACK; on persist failure: no apply, no ACK (a standby never ACKs an entry it didn't persist). Covers single + batch catch-up; OPCode_replEntry has no separate apply path (allow-list only), so all live WAL apply now persists. Dedup guard (lsn+1<=current_lsn) makes re-streams idempotent while still ACKing. TDD: standby write -> drop persist_t -> reopen + persist_recover replays it (single, 3-entry crc chain, post-dedup); checkpoint/oversized rejected. Full controller suite green under ASAN; persist_wal_async / recovery / truncation / streamer / wire untouched.
This commit is contained in:
parent
007bf0c403
commit
708f2279e3
4 changed files with 359 additions and 0 deletions
|
|
@ -397,6 +397,110 @@ int persist_wal_async(persist_t* p, uint32_t type,
|
|||
return 0;
|
||||
}
|
||||
|
||||
int persist_wal_write_replicated(persist_t* p, uint64_t lsn, uint32_t type,
|
||||
const void* payload, uint32_t len)
|
||||
{
|
||||
if (!p)
|
||||
return -EINVAL;
|
||||
|
||||
/* WAL_CHECKPOINT is a local-only marker, never streamed to standbys. */
|
||||
if (type == WAL_CHECKPOINT)
|
||||
return -EINVAL;
|
||||
|
||||
/* Reject oversized payloads BEFORE touching the segment. */
|
||||
if (len > MAX_WAL_PAYLOAD_BYTES)
|
||||
return -EMSGSIZE;
|
||||
|
||||
pthread_mutex_lock(&p->lock);
|
||||
|
||||
/* Idempotency / dedup guard: if this lsn is already persisted (current_lsn
|
||||
* already advanced past it) treat the re-stream as a no-op. This keeps the
|
||||
* write idempotent and avoids a duplicate entry / CRC-chain corruption when
|
||||
* the primary re-sends an lsn the standby already wrote. */
|
||||
if (lsn + 1 <= p->current_lsn) {
|
||||
pthread_mutex_unlock(&p->lock);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Open first segment lazily. open_new_segment names the file from
|
||||
* current_lsn — for a fresh standby receiving lsn 1 first, current_lsn is
|
||||
* 1 (reserve-lsn-0), so the segment is seg-...001.wal. */
|
||||
if (p->wal_fd < 0) {
|
||||
if (open_new_segment(p) != 0) {
|
||||
pthread_mutex_unlock(&p->lock);
|
||||
return -EIO;
|
||||
}
|
||||
}
|
||||
|
||||
/* Build entry header. Unlike persist_wal_async, lsn is GIVEN (not
|
||||
* current_lsn++). term isn't streamed, so use our best-effort p->term.
|
||||
* The CRC chain is SELF-computed (the streamed entry carries no crc):
|
||||
* recovery validates an intra-segment chain (prev_crc == last_crc), so the
|
||||
* standby's own writes must chain to each other — cross-node CRC equality
|
||||
* is neither required nor possible. */
|
||||
wal_entry_hdr_t hdr;
|
||||
memset(&hdr, 0, sizeof(hdr));
|
||||
hdr.term = p->term;
|
||||
hdr.lsn = lsn;
|
||||
hdr.timestamp = now_us();
|
||||
hdr.type = type;
|
||||
hdr.payload_len = len;
|
||||
hdr.prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc;
|
||||
|
||||
uint32_t c = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
|
||||
if (payload && len > 0)
|
||||
c = crc32(c, (const Bytef*)payload, len);
|
||||
hdr.crc = c;
|
||||
|
||||
/* Write header */
|
||||
ssize_t hw = pwrite(p->wal_fd, &hdr, sizeof(hdr), (off_t)p->seg_offset);
|
||||
if (hw != (ssize_t)sizeof(hdr)) {
|
||||
pthread_mutex_unlock(&p->lock);
|
||||
return -EIO;
|
||||
}
|
||||
|
||||
/* Write payload (if any) */
|
||||
if (payload && len > 0) {
|
||||
ssize_t pw = pwrite(p->wal_fd, payload, len,
|
||||
(off_t)(p->seg_offset + sizeof(hdr)));
|
||||
if (pw != (ssize_t)len) {
|
||||
pthread_mutex_unlock(&p->lock);
|
||||
return -EIO;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t entry_size = (uint32_t)(sizeof(hdr) + len);
|
||||
p->seg_offset += entry_size;
|
||||
p->total_wal_bytes += entry_size;
|
||||
p->last_crc = hdr.crc;
|
||||
|
||||
/* Advance current_lsn to one past the given lsn (max-idempotent — mirrors
|
||||
* apply_wal_entry's monotonic advance). */
|
||||
if (lsn + 1 > p->current_lsn)
|
||||
p->current_lsn = lsn + 1;
|
||||
|
||||
/* Durability: fsync the segment BEFORE returning so the caller's ACK
|
||||
* implies the entry is on stable storage (persist-before-ACK).
|
||||
* persist_wal_async defers fsync to the background thread / persist_flush;
|
||||
* here we sync inline because the entry feeds the primary's commit
|
||||
* majority and must survive a standby restart. fdatasync is sufficient
|
||||
* (entry bytes + segment metadata are pre-allocated by open_new_segment). */
|
||||
fdatasync(p->wal_fd);
|
||||
|
||||
/* Rotate segment when full (same threshold as the primary append). */
|
||||
if (p->seg_offset >= WAL_SEGMENT_SIZE_BYTES) {
|
||||
if (rotate_segment(p) != 0) {
|
||||
pthread_mutex_unlock(&p->lock);
|
||||
return -EIO;
|
||||
}
|
||||
}
|
||||
|
||||
/* NOTE: no repl_broadcast — standbys never re-replicate. */
|
||||
|
||||
pthread_mutex_unlock(&p->lock);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int persist_flush(persist_t* p)
|
||||
{
|
||||
pthread_mutex_lock(&p->lock);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue