lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix bug-1060 bug-1067

Browse source
This commit is contained in:
Rongsong Shen 2026-06-24 17:05:43 +08:00
parent 6067ba6c94
commit 61e1391cd0
28 changed files with 7021 additions and 2926 deletions
Download patch file Download diff file Expand all files Collapse all files

114
src/controller/ha.c
View file

@ -568,6 +568,18 @@ void ha_compute_safe_committed_lsn(ha_elect_ctx_t *ctx,
" (from %d granted voters)", *safe_lsn_out, *safe_tag_out, n_granted);
}
/* ── bug-1060: never truncate the new leader's own durable log ───────────────
*
* The post-election WAL truncation cutoff must never fall below the leader's own
* durable tip, or it destroys entries committed by a prior leader (which this
* node had acked as a standby, and so are committed) — a Raft §5.4 Leader
* Completeness violation that diverges WALs and deadlocks the next election.
* See ha.h for the full rationale. */
uint64_t ha_promotion_truncate_floor(uint64_t safe_lsn, uint64_t durable_tip)
{
return safe_lsn > durable_tip ? safe_lsn : durable_tip;
}
/* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ───────────
*
* ha_promotion_gate_decision: pure decision (Req 2.2, 2.5, 2.6). Build the Boot
@ -939,9 +951,22 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
promotion_safe_lsn = safe_lsn;
/* Store results for use in WAL truncation and post-election commit.
* safe_lsn is the highest LSN on a majority; entries above it
* may not have been committed on all nodes and must be truncated. */
* may not have been committed on all nodes and must be truncated.
*
* bug-1060 (Leader Completeness, Raft §5.4): the truncation cutoff is
* clamped to the leader's OWN durable tip so it never destroys entries
* this node holds durably — those were acked by this node as a standby
* and so were committed by the prior leader. Truncating to the bare
* majority element (which can sit BELOW the leader's durable tip when
* a slower peer is in the granting majority) diverged WALs and dead-
* locked the next election. With the clamp, persist_truncate_wal_above
* is a no-op on the leader's durable log; safe_lsn only governs removal
* of a non-durable divergent tail strictly above the durable tip. */
if (c) {
persist_truncate_wal_above(&c->persist, safe_lsn);
uint64_t durable_tip = persist_durable_floor(&c->persist);
uint64_t trunc_floor =
ha_promotion_truncate_floor(safe_lsn, durable_tip);
persist_truncate_wal_above(&c->persist, trunc_floor);
}
/* Task 4.2 (ha-repl-hardening): set committed_lsn = safe_lsn.
* persist_truncate_wal_above already sets c->persist.committed_lsn
@ -2128,6 +2153,21 @@ static int ha_apply_term_entry_via_library(ha_state_t *ha, persist_t *p,
* e->lsn + 2 and SKIPS the real next-needed entry e->lsn + 1, wedging
* logOk and stalling commit. */
(void)before_term;
/* bug-1043: the library route (on_persist_entry → persist_wal_write_
* replicated) writes the entry to the WAL but does NOT apply it to the
* in-memory entity state (job/queue/worker tables). The legacy direct-
* apply path (ha_apply_single_entry) called apply_wal_entry after the
* persist; the term-safe library route dropped it. So a standby that
* caught up entries 8..N via replication held them on disk but NOT in the
* queryable job table — on promotion `joblist`/`expect-job` could not
* find a committed job (it IS on disk; walcompare passes), looking like a
* committed-job loss. Re-apply here, mirroring the legacy path. Idempotent
* on a re-stream (restore_job upserts by jobid). WAL_NOOP / WAL_CONFIG_
* MARKER carry no entity payload — apply_wal_entry no-ops on them. */
apply_wal_entry(p, (uint32_t)e->entry_type, e->lsn,
e->raw_payload, (uint32_t)e->raw_payload_len);
ha_advance_applied_lsn(ha, e->lsn);
ha_send_wal_ack_reply(ha, ep, c, e->lsn);
return 1;
@ -2323,9 +2363,16 @@ int ha_consume_append_entries(cluster_t *c, wal_catchup_entry_t *entry,
const void *payload = e->raw_payload;
uint32_t plen = (uint32_t)e->raw_payload_len;
entry_term = e->term;
/* ae_term for UpdateTerm is the entry's term (the leader appends at its
* current term, so entry_term == leader term for a live append). */
ae_term = entry_term;
/* bug-1067: the AppendEntries HEADER term (UpdateTerm/stale-check) is the
* leader's CURRENT term, NOT the entry's stored term. For a LIVE append
* the two are equal (the leader appends at its current term), but for a
* CATCH-UP entry written by a PRIOR leader the stored term is LOWER than
* the current leader's term — using it as the AE term made the library
* stale-reject (term < s->term) every historical entry, wedging cross-term
* catch-up forever. The streamer stamps leader_current_term = ha->term;
* fall back to the stored term only when it is unset (legacy sender). */
ae_term = (e->leader_current_term != 0) ? e->leader_current_term
: entry_term;
blob_len = sizeof(raft_entry_blob_t) + plen;
uint8_t *buf = (blob_len <= sizeof(stackbuf)) ? stackbuf
@ -2680,10 +2727,19 @@ int ha_dispatch_repl_action_for_standby(ha_state_t *ha,
if (c && c->persist.wal_path[0]) {
extern int ha_stream_wal_to_standby(endpoint_t *ep,
const char *wal_path,
const persist_t *p,
uint64_t leader_term,
uint64_t standby_lsn,
uint64_t committed_lsn);
ha_stream_wal_to_standby(ep, c->persist.wal_path,
standby_lsn, committed_lsn);
/* bug-1028: pass &c->persist so streamed entry terms come from the
* authoritative per-LSN index (persist_entry_term), matching the
* standby's logOk + the library send path.
* bug-1067: pass the leader's CURRENT term so historical entries
* carry the AE header term (not their lower stored term). */
uint64_t leader_term =
atomic_load_explicit(&c->ha.term, memory_order_acquire);
ha_stream_wal_to_standby(ep, c->persist.wal_path, &c->persist,
leader_term, standby_lsn, committed_lsn);
}
break;
@ -3521,6 +3577,50 @@ void ha_sync_applied_lsn_from_recovery(ha_state_t *ha, persist_t *p)
memory_order_release);
}
/* bug-1042: re-sync the raft library's log_len / commit_index / term from the
* recovered WAL AFTER persist_recover.
*
* Init-ordering hazard: ha_init_raft (myapp.c, before the election) sources
* raft.initial_log_len from persist_durable_floor — but on a STANDBY-REJOIN boot
* persist_recover has not run yet at that point, so the library comes up with
* log_len=0. After cluster_init_phase2 runs persist_recover (recovering e.g.
* durable_floor=7), the library still thinks its log is EMPTY. Every incoming
* AppendEntries then fails logOk (prevLogIndex>=1 against log_len=0 → reject),
* so the rejoined member is PERMANENTLY stuck as a non-converging standby and
* the primary re-streams [tip+1..committed] every heartbeat forever.
*
* Fix: after recovery, raise the library's log_len to the recovered WAL tip and
* its commit_index to the recovered durable floor (clamped to log_len, honoring
* the commit_index<=log_len invariant), and align raft.term with the persisted
* term so the per-LSN (term,index) reconcile reads consistent state. Monotonic:
* never lowers an already-higher library position (e.g. if catch-up advanced it
* before this runs). The library stores no entry bytes — on_get_entry_term reads
* the recovered per-LSN term index — so raising log_len is sufficient for logOk. */
void ha_sync_raft_log_from_recovery(ha_state_t *ha, persist_t *p)
{
if (!ha || ha->raft_c == NULL || !p)
return;
uint64_t recovered_tip = persist_durable_floor(p);
if (recovered_tip > ha->raft.log_len)
ha->raft.log_len = recovered_tip;
/* commit_index advances to the recovered durable floor but never past the
* log tip (library invariant commit_index <= log_len), and never decreases. */
uint64_t recovered_commit = recovered_tip;
if (recovered_commit > ha->raft.log_len)
recovered_commit = ha->raft.log_len;
if (recovered_commit > ha->raft.commit_index)
ha->raft.commit_index = recovered_commit;
/* Align the library term with the persisted/restored HA term so the
* (term,lsn) reconcile and own-term gate read a consistent value after a
* rejoin (ha_restore_term already loaded ha->term before ha_init_raft). */
uint64_t ha_term = atomic_load_explicit(&ha->term, memory_order_acquire);
if (ha_term > ha->raft.term)
ha->raft.term = ha_term;
}
/* ── Vote Handler (Req 2.2–2.6, 6.1, 6.2, 6.4, 9.1) ────────────────────────
*
* handle_request_vote: process an incoming OP_requestVote RPC.