lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-commit-barrier-rollout): config barrier with production ACK ingestion (task 5.2 + r1)

Browse source 
configAck branch: per-push majority (acked_config_lsn >= push_tag,
1+count > n_peers/2 over r->ce->n_peers, n_peers<=1 immediate,
ce==NULL -EINVAL); legacy >0 counting and n_active==0 shortcut dead.
push_config_to_standbys returns the barrier result; upgrade failure ->
send_upgrade_error (rc forwarded), no success ACK, no restart,
committed_config_tag success-only.

Remediation r1 (two CRITICAL review findings):
- F1: wire production config-ACK ingestion — replconfigpushreplymsg
  decode branch in repl_ack_decode_and_dispatch -> repl_on_config_ack
  on the promoted-vote ep; + defensive cluster.c OPCode_reply case.
  Closes the unlanded half of ha-config-bootstrap 8.1; without this the
  barrier would brick every multi-member config upgrade. No schema change.
- F2: keep the barrier off the iomgr thread — export
  push_config_transport_only (non-blocking) for the 5 ha.c + join.c
  streaming sites; barrier path now vtask-only (do_upgrade_config).

Wire-path test drives real capnp ACK bytes through the production decode
branch. 30/30 rollout, 6/6 repl_majority, 46/46 config_repl; census 0
FAIL; pre-existing fixture leak only (less than baseline).
This commit is contained in:
Rongsong Shen 2026-06-13 03:46:29 +00:00
parent 9cdeba5816
commit 5a2fae1d37
12 changed files with 1034 additions and 399 deletions
Download patch file Download diff file Expand all files Collapse all files

71
src/controller/ha.c
View file

@ -68,15 +68,23 @@ typedef struct {
cluster_t *c;
} committing_timeout_ctx_t;
/* cluster_init_phase2 / push_config_to_standbys / ha_send_ctrlhello_to_all_standbys:
* defined in cluster.c / config_repl.c; only referenced in production builds.
* (XCE_TESTING uses hook shims and does not link cluster.c.) */
/* cluster_init_phase2 / push_config_transport_only /
* ha_send_ctrlhello_to_all_standbys: defined in cluster.c / config_repl.c;
* only referenced in production builds.
* (XCE_TESTING uses hook shims and does not link cluster.c.)
* ha.c uses push_config_transport_only ONLY (task 5.2 remediation r1):
* every ha.c push site is reachable from peer_ctrl_msg_handler / io_worker
* (vote replies, heartbeat replies), where the push_config_to_standbys
* barrier wait would block the single iomgr epoll thread — and the config
* ACK it waits for would arrive on that same blocked thread (structural
* deadlock). Convergence is owned by the OP_replConfig anti-entropy pull
* and the heartbeat decision loop, not by these best-effort pushes. */
#ifndef XCE_TESTING
extern int cluster_init_phase2(cluster_t *c);
extern void ha_send_ctrlhello_to_all_standbys(cluster_t *c);
extern int push_config_to_standbys(cluster_t *c, const uint8_t *bytes, size_t blen,
const uint8_t sha256[32], uint64_t new_tag,
int *acked_out);
extern int push_config_transport_only(cluster_t *c, const uint8_t *bytes,
size_t blen, const uint8_t sha256[32],
uint64_t new_tag, int *n_out);
#endif
static void on_vote_connected(endpoint_t *ep);
@ -567,12 +575,12 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
extern int conf_read_raw(const char *, uint8_t **, size_t *);
if (c->myconf && c->myconf->conffile &&
conf_read_raw(c->myconf->conffile, &cfg_bytes, &cfg_len) == 0) {
push_config_to_standbys(c, cfg_bytes, cfg_len,
push_config_transport_only(c, cfg_bytes, cfg_len,
c->myconf->config_sha256,
c->myconf->config_version_tag, NULL);
free(cfg_bytes);
} else {
push_config_to_standbys(c, NULL, 0, NULL,
push_config_transport_only(c, NULL, 0, NULL,
c->myconf ? c->myconf->config_version_tag : 0,
NULL);
}
@ -1362,7 +1370,7 @@ void ha_check_majority_convergence(ha_state_t *ha, cluster_t *c)
* (stored by ha_process_standby_state on the same heartbeat reply), calls
* ha_evaluate_repl_action, and acts on the result:
*
* HA_REPL_PUSH_CONFIG → push_config_to_standbys(c, …) [or test hook] (Task 3.3)
* HA_REPL_PUSH_CONFIG → push_config_transport_only(c, …) [or test hook] (Task 3.3)
* HA_REPL_SEND_SNAPSHOT→ stub (Task 3.5)
* HA_REPL_STREAM_WAL → ha_stream_wal_to_standby(c, ha, idx) (Task 3.4)
* HA_REPL_CONVERGED → no-op
@ -1407,12 +1415,12 @@ int ha_dispatch_repl_action_for_standby(ha_state_t *ha,
size_t cfg_len = 0;
if (c->myconf && c->myconf->conffile &&
conf_read_raw(c->myconf->conffile, &cfg_bytes, &cfg_len) == 0) {
push_config_to_standbys(c, cfg_bytes, cfg_len,
push_config_transport_only(c, cfg_bytes, cfg_len,
c->myconf->config_sha256,
c->myconf->config_version_tag, NULL);
free(cfg_bytes);
} else {
push_config_to_standbys(c, NULL, 0, NULL,
push_config_transport_only(c, NULL, 0, NULL,
c->myconf ? c->myconf->config_version_tag : 0,
NULL);
}
@ -2078,7 +2086,7 @@ int handle_request_vote(ha_state_t *ha,
/* ── Test-only hook shims (XCE_TESTING only) ──────────────────────────────
* Type definitions (election_timeout_ctx_t, committing_timeout_ctx_t),
* extern declarations (cluster_init_phase2, push_config_to_standbys,
* extern declarations (cluster_init_phase2, push_config_transport_only,
* ha_send_ctrlhello_to_all_standbys), and static forward declarations
* (committing_timeout_fn, ha_heartbeat_is_fresh) have been moved to the
* forward-declarations block near the top of this file so that the raft
@ -2101,9 +2109,9 @@ int (*ha_phase2_hook)(void *c) = NULL;
int (*ha_send_ctrlhello_hook)(void *c) = NULL;
/*
* ha_push_config_hook: test-only seam for push_config_to_standbys.
* ha_push_config_hook: test-only seam for push_config_transport_only.
* When non-NULL, the COMMITTING path calls this INSTEAD OF the real function
* (config_repl.c's push_config_to_standbys needs a real cluster). Req 7.1.
* (config_repl.c's push_config_transport_only needs a real cluster). Req 7.1.
*/
int (*ha_push_config_hook)(void *c) = NULL;
@ -2118,7 +2126,7 @@ int (*ha_nop_req_hook)(endpoint_t *ep) = NULL;
* ha_repl_push_config_for_standby_hook: test-only seam for the per-standby
* config push triggered by the heartbeat-reply decision loop (Task 3.3).
* When non-NULL, ha_dispatch_repl_action_for_standby() calls this INSTEAD OF
* push_config_to_standbys() when the decision is HA_REPL_PUSH_CONFIG.
* push_config_transport_only() when the decision is HA_REPL_PUSH_CONFIG.
* Requirements: 1.4 (ha-repl-hardening)
*/
int (*ha_repl_push_config_for_standby_hook)(void *c, int peer_idx) = NULL;
@ -2283,7 +2291,7 @@ apply_role:
ha_push_config_hook(c);
#else
if (c != NULL)
push_config_to_standbys(c, NULL, 0, NULL, 0, NULL);
push_config_transport_only(c, NULL, 0, NULL, 0, NULL);
#endif
/* Send CtrlHelloRequest to all standbys to trigger WAL catch-up (Req 7.2) */
@ -2649,7 +2657,10 @@ done:
*
* - OPCode_replCatchUpDone → ha_repl_ack_received
* - OPCode_reply with stateReportMsg → ha_process_standby_state (Task 3.1)
* - OPCode_nop or OPCode_reply without stateReport → silently ignored
* - OPCode_reply with walackreplymsg → repl_on_ack (Task 4.1)
* - OPCode_reply with replconfigpushreplymsg → repl_on_config_ack
* (barrier-rollout task 5.2 r1 / ha-config-bootstrap task 8.1)
* - OPCode_nop or OPCode_reply with any other body → silently ignored
* - anything else → WARN + drop
*/
static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx)
@ -2775,6 +2786,32 @@ static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx)
WARN("repl_ack_decode_and_dispatch: WAL NACK from standby"
" lsn=%lu success=0 — ignoring", (unsigned long)acked_lsn);
}
} else if (rep_s.data_which == RequestReply_data_replconfigpushreplymsg) {
/* Config-push ACK from standby (ha-commit-barrier-rollout task
* 5.2 remediation r1; completes the unlanded half of
* ha-config-bootstrap task 8.1): the standby replies to
* OP_replConfigPush with replconfigpushreplymsg carrying
* {accepted, versionTag} (config_repl.c:744-752). Decode the
* acked tag and apply it to the replicator so the configAck
* barrier (repl_wait_majority_ack) can observe the covering
* ACK — without this branch every multi-member config upgrade
* would time out. read_ReplConfigPushReply reads scalars only
* — safe to call before capn_free. */
struct ReplConfigPushReply rcp_raw;
memset(&rcp_raw, 0, sizeof(rcp_raw));
read_ReplConfigPushReply(&rcp_raw, rep_s.data.replconfigpushreplymsg);
uint64_t acked_tag = (uint64_t)rcp_raw.versionTag;
int accepted = rcp_raw.accepted ? 1 : 0;
capn_free(&capn_ctx);
if (accepted && c) {
repl_on_config_ack(&c->persist.repl, ep, acked_tag);
TRACE("repl_ack_decode_and_dispatch: config ACK from standby"
" tag=%016" PRIx64 " accepted=1", acked_tag);
} else if (!accepted) {
WARN("repl_ack_decode_and_dispatch: config push rejected by"
" standby (tag=%016" PRIx64 ") — ignoring", acked_tag);
}
} else {
capn_free(&capn_ctx);
}