lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore: checkpoint all prior work for clean state

Browse source 
Captures all uncommitted work from ha-config-bootstrap,
ha-heartbeat-liveness, and associated spec/tooling updates:

- src: ha.c, ha.h, cluster.c, config_repl.c, myapp.c, raft.c —
  heartbeat liveness, RDHUP integration, committing phase fixes
- tests: test_ha_role.c, test_config_repl.c, test_ha_rdhup_integration.c,
  test_p1_integration.c, test_p3_hardening.c, scheduler test
- specs: election-iomgr-refactor, ha-heartbeat-liveness,
  ha-peer-connection-reuse, raft-library (new); ha-config-bootstrap
  and election-iomgr-refactor task/research updates
- tooling: .wolf/ state, ha/data cluster fixtures, raft/raft.tla,
  src/build symlink, xce-build skill update
This commit is contained in:
Rongsong Shen 2026-06-04 13:04:16 +00:00
parent 15bd291bdd
commit e31673dd88
66 changed files with 122422 additions and 860 deletions
Download patch file Download diff file Expand all files Collapse all files

397
src/controller/ha.c
View file

@ -441,17 +441,27 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
ha_start_primary_heartbeat(c);
/* Arm committing timeout watchdog */
{
/* Cancel any prior committing timeout (e.g. from a previous term) */
committing_timeout_ctx_t *prev = (committing_timeout_ctx_t *)
atomic_exchange(&ha->committing_timeout_ctx, NULL);
if (prev) {
atomic_store(&prev->cancelled, 1);
}
}
committing_timeout_ctx_t *ct = calloc(1, sizeof(*ct));
if (ct) {
atomic_init(&ct->cancelled, 0);
ct->ha = ha;
ct->c = c;
atomic_store(&ha->committing_timeout_ctx, ct);
int rc = delay_task_ms(&c->delaymgr,
(int)ha->config.committing_timeout_ms,
1, committing_timeout_fn, ct);
if (rc < 0) {
WARN("raft_cb_become_leader: delay_task_ms for committing timeout failed: %d",
rc);
atomic_store(&ha->committing_timeout_ctx, NULL);
free(ct);
}
}
@ -474,6 +484,18 @@ static void raft_cb_become_follower(void *ctx, uint64_t term, int leader_idx)
/* Cancel any pending election timeout */
raft_cb_cancel_timeout(ctx);
/* Cancel any pending committing timeout — we are no longer leader.
* Without this, a node that stepped down due to a higher-term vote
* request would still promote to PRIMARY when the old committing
* timeout fires → split-brain. */
{
committing_timeout_ctx_t *ct = (committing_timeout_ctx_t *)
atomic_exchange(&ha->committing_timeout_ctx, NULL);
if (ct) {
atomic_store(&ct->cancelled, 1);
}
}
/* Sync ha->term */
atomic_store_explicit(&ha->term, term, memory_order_release);
@ -647,8 +669,7 @@ int ha_init(ha_state_t *ha, const ha_config_t *cfg, int my_priority)
atomic_store_explicit(&ha->promotion_failed, 0, memory_order_relaxed);
atomic_store_explicit(&ha->promotion_retry_count, 0, memory_order_relaxed);
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_relaxed);
atomic_store_explicit(&ha->failover_requested, 0, memory_order_relaxed);
atomic_store_explicit(&ha->watchdog_stop, 0, memory_order_relaxed);
/* failover_requested / watchdog_stop deleted — watchdog replaced by ha_arm_heartbeat_timeout_vtask */
pthread_mutex_init(&ha->lock, NULL);
@ -706,30 +727,8 @@ int ha_init(ha_state_t *ha, const ha_config_t *cfg, int my_priority)
return 0;
}
/* ── Promotion / demotion / SIGUSR1 ────────────────────────── */
/* Async-signal-safe: atomic store on int-width type (POSIX.1-2017 §2.4.3).
* Pointer set once before signals are enabled — no locking required. */
static ha_state_t *g_sigusr1_ha = NULL;
static void sigusr1_handler(int sig)
{
(void)sig;
if (g_sigusr1_ha)
atomic_store_explicit(&g_sigusr1_ha->failover_requested, 1,
memory_order_release);
}
void ha_install_sigusr1(ha_state_t *ha)
{
g_sigusr1_ha = ha;
struct sigaction sa;
memset(&sa, 0, sizeof(sa));
sa.sa_handler = sigusr1_handler;
sigemptyset(&sa.sa_mask);
sa.sa_flags = SA_RESTART;
sigaction(SIGUSR1, &sa, NULL);
}
/* SIGUSR1 handler / ha_install_sigusr1() deleted — failover_requested replaced by
* ha_arm_heartbeat_timeout_vtask (delaymgr) in election-iomgr-refactor */
void ha_do_promote(ha_state_t *ha)
{
@ -754,182 +753,9 @@ void ha_do_demote(ha_state_t *ha)
atomic_load_explicit(&ha->term, memory_order_acquire));
}
/* ── P1-6: Election probing ─────────────────────────────────── */
/* probe_thread() / wait_any_probe() deleted — only called by am_i_primary() (election-iomgr-refactor) */
/*
* probe_thread: attempt a TCP (optionally SSL) connection to one candidate.
* Sets ctx->got_ack = 1 if the connection succeeds before election_done is set.
*/
static void *probe_thread(void *arg)
{
probe_ctx_t *ctx = (probe_ctx_t *)arg;
struct addrinfo hints, *res = NULL;
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
char port_str[16];
snprintf(port_str, sizeof(port_str), "%u", (unsigned)ctx->candidate->port);
if (getaddrinfo(ctx->candidate->hostname, port_str, &hints, &res) != 0)
goto done;
int sock = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
if (sock < 0) { freeaddrinfo(res); goto done; }
/* Non-blocking connect */
int flags = fcntl(sock, F_GETFL, 0);
fcntl(sock, F_SETFL, flags | O_NONBLOCK);
connect(sock, res->ai_addr, res->ai_addrlen);
freeaddrinfo(res);
res = NULL;
/* Poll loop: check election_done between iterations */
while (!atomic_load(ctx->election_done)) {
struct pollfd pfd = { .fd = sock, .events = POLLOUT | POLLERR | POLLHUP };
int pr = poll(&pfd, 1, (int)ctx->timeout_ms);
if (pr < 0 && errno == EINTR) continue;
if (pr == 0) goto cleanup; /* poll timeout */
if (pfd.revents & (POLLERR | POLLHUP)) goto cleanup;
if (pfd.revents & POLLOUT) break;
}
if (atomic_load(ctx->election_done)) goto cleanup;
/* Verify connect succeeded */
{
int err = 0;
socklen_t el = sizeof(err);
getsockopt(sock, SOL_SOCKET, SO_ERROR, &err, &el);
if (err != 0) goto cleanup;
}
/* SSL handshake — only if ssl_ctx is set */
if (ctx->ssl_ctx) {
SSL *ssl = SSL_new((SSL_CTX *)ctx->ssl_ctx);
if (!ssl) goto cleanup;
/* Restore blocking for SSL */
fcntl(sock, F_SETFL, flags & ~O_NONBLOCK);
SSL_set_fd(ssl, sock);
int sr;
do {
sr = SSL_connect(ssl);
if (sr <= 0) {
int ssl_err = SSL_get_error(ssl, sr);
if (ssl_err != SSL_ERROR_WANT_READ &&
ssl_err != SSL_ERROR_WANT_WRITE)
break;
}
} while (sr <= 0 && !atomic_load(ctx->election_done));
SSL_shutdown(ssl);
SSL_free(ssl);
if (sr <= 0 || atomic_load(ctx->election_done))
goto cleanup;
}
ctx->got_ack = 1;
cleanup:
close(sock);
done:
return NULL;
}
/*
* wait_any_probe: spin-poll until any probe sets got_ack, or deadline passes.
* Returns 1 if any probe acknowledged, 0 if deadline expired.
*/
static int wait_any_probe(probe_ctx_t *probes, int n,
const struct timespec *deadline)
{
while (1) {
for (int i = 0; i < n; i++) {
if (probes[i].got_ack)
return 1;
}
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
if (now.tv_sec > deadline->tv_sec ||
(now.tv_sec == deadline->tv_sec &&
now.tv_nsec >= deadline->tv_nsec))
return 0;
/* Sleep 10ms between checks */
struct timespec sleep_ts = { .tv_sec = 0, .tv_nsec = 10 * 1000000L };
nanosleep(&sleep_ts, NULL);
}
}
/*
* am_i_primary: election probe logic.
*
* Priority-0 is unconditionally primary — never probes.
* Priority-N probes the N candidates with lower index (higher priority).
* If any responds before election_timeout + priority_step*N + jitter, this
* node becomes standby. If deadline expires with no response, self-promote.
*/
int am_i_primary(cluster_t *c)
{
/* Priority-0 never probes — it is unconditionally primary */
if (c->ha.my_priority == 0)
return 1;
const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg;
int n_probes = (c->ha.my_priority < pcfg->candidates_count)
? c->ha.my_priority
: pcfg->candidates_count;
if (n_probes == 0)
return 1;
/* Seed jitter PRNG if needed */
if (c->ha.jitter_seed == 0)
ha_seed_jitter(&c->ha.jitter_seed, c->ha.my_priority);
atomic_int election_done;
atomic_init(&election_done, 0);
probe_ctx_t probes[MAX_HA_CANDIDATES];
for (int i = 0; i < n_probes; i++) {
probes[i].candidate = (xce_candidate_t *)&pcfg->candidates[i];
probes[i].ssl_ctx = c->ha.ssl_ctx;
probes[i].timeout_ms = c->ha.config.probe_connect_timeout_ms;
probes[i].election_done = &election_done;
probes[i].got_ack = 0;
pthread_create(&probes[i].thread, NULL, probe_thread, &probes[i]);
}
/* Compute deadline = election_timeout + priority_step * my_priority + jitter */
long to_ms = (long)c->ha.config.election_timeout_ms
+ (long)c->ha.my_priority
* (long)c->ha.config.election_priority_step_ms;
if (c->ha.config.election_jitter_max_ms > 0) {
to_ms += (long)(xorshift64_next(&c->ha.jitter_seed)
% ((uint64_t)c->ha.config.election_jitter_max_ms + 1));
}
struct timespec deadline;
clock_gettime(CLOCK_MONOTONIC, &deadline);
deadline.tv_sec += to_ms / 1000;
deadline.tv_nsec += (to_ms % 1000) * 1000000L;
if (deadline.tv_nsec >= 1000000000L) {
deadline.tv_sec++;
deadline.tv_nsec -= 1000000000L;
}
int got_ack = wait_any_probe(probes, n_probes, &deadline);
/* Signal all threads to stop */
atomic_store(&election_done, 1);
for (int i = 0; i < n_probes; i++)
pthread_join(probes[i].thread, NULL);
if (got_ack) {
WARN("HA election: higher-priority candidate responded — "
"starting as standby (my_priority=%d)", c->ha.my_priority);
return 0;
}
INFO("HA election: no higher-priority candidate responded within %ldms — "
"starting as primary (my_priority=%d)", to_ms, c->ha.my_priority);
return 1;
}
/* am_i_primary() deleted — replaced by ha_start_election_async (election-iomgr-refactor) */
/*
* ha_is_primary: lock-free role check.
@ -941,23 +767,7 @@ int ha_is_primary(const ha_state_t *ha)
}
/* ha_check_priority_stepdown() deleted — replaced by quorum election (Req 2.1) */
/*
* start_heartbeat_listener / stop_heartbeat_listener:
* Stub implementations — full listener is implemented in Task 7 (cluster.c).
* The stubs allow am_i_primary() callers (main.c / cluster.c) to compile and
* link without requiring the full Phase-2 network stack.
*/
void start_heartbeat_listener(cluster_t *c)
{
(void)c;
/* Heartbeat listener stub — full implementation in cluster.c Task 7 */
}
void stop_heartbeat_listener(cluster_t *c)
{
(void)c;
}
/* start_heartbeat_listener() / stop_heartbeat_listener() deleted — never called (election-iomgr-refactor) */
/* ── P3-6: Test-only failover shim ────────────────────────────── */
@ -970,96 +780,14 @@ uint64_t ha_trigger_manual_failover_for_test(ha_state_t *ha)
}
#endif
/* ha_check_heartbeat_nocluster() removed from the public API — logic inlined in
* ha_watchdog_thread(). XCE_TESTING shim preserved so existing unit tests compile. */
#ifdef XCE_TESTING
void ha_check_heartbeat_nocluster(ha_state_t *ha)
{
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire);
if (role != CTRL_STANDBY) return;
if (ha->primary_host[0] == '\0') return;
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
pthread_mutex_lock(&ha->lock);
struct timespec last = ha->last_heartbeat;
pthread_mutex_unlock(&ha->lock);
uint64_t elapsed_ms =
(uint64_t)(now.tv_sec - last.tv_sec) * 1000ULL +
(uint64_t)(now.tv_nsec - last.tv_nsec) / 1000000ULL;
if (elapsed_ms >= ha->config.heartbeat_timeout_ms)
atomic_store_explicit(&ha->failover_requested, 1, memory_order_release);
}
#endif /* XCE_TESTING */
static void *ha_watchdog_thread(void *arg)
{
ha_state_t *ha = arg;
while (!atomic_load_explicit(&ha->watchdog_stop, memory_order_acquire)) {
long ms = (long)ha->config.heartbeat_interval_ms;
struct timespec ts = {
.tv_sec = ms / 1000,
.tv_nsec = (ms % 1000) * 1000000L,
};
nanosleep(&ts, NULL);
/* Re-check stop flag after sleep */
if (atomic_load_explicit(&ha->watchdog_stop, memory_order_acquire)) break;
/* Heartbeat timeout check (inlined from deleted ha_check_heartbeat_nocluster) */
{
ctrl_role_t role =
(ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire);
if (role == CTRL_STANDBY && ha->primary_host[0] != '\0') {
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
pthread_mutex_lock(&ha->lock);
struct timespec last = ha->last_heartbeat;
pthread_mutex_unlock(&ha->lock);
uint64_t elapsed_ms =
(uint64_t)(now.tv_sec - last.tv_sec) * 1000ULL +
(uint64_t)(now.tv_nsec - last.tv_nsec) / 1000000ULL;
if (elapsed_ms >= ha->config.heartbeat_timeout_ms)
atomic_store_explicit(&ha->failover_requested, 1,
memory_order_release);
}
}
if (atomic_load_explicit(&ha->failover_requested, memory_order_acquire)) {
atomic_store_explicit(&ha->failover_requested, 0, memory_order_release);
/* Promotion requires cluster_t; the watchdog only has ha_state_t.
* Full ha_promote_to_primary(c) is called from the cluster's own
* failover path which holds cluster_t context. */
}
}
return NULL;
}
int ha_start_watchdog(ha_state_t *ha)
{
atomic_store_explicit(&ha->watchdog_stop, 0, memory_order_release);
int rc = pthread_create(&ha->watchdog_thread, NULL, ha_watchdog_thread, ha);
if (rc != 0) return -rc;
return 0;
}
void ha_stop_watchdog(ha_state_t *ha)
{
atomic_store_explicit(&ha->watchdog_stop, 1, memory_order_release);
if (ha->watchdog_thread != 0) {
pthread_join(ha->watchdog_thread, NULL);
ha->watchdog_thread = 0;
}
}
/* ha_check_heartbeat_nocluster() / ha_watchdog_thread() / ha_start_watchdog() /
* ha_stop_watchdog() deleted — watchdog pthread replaced by
* ha_arm_heartbeat_timeout_vtask (delaymgr) in election-iomgr-refactor */
void ha_fini(ha_state_t *ha)
{
if (!ha)
return;
ha_stop_watchdog(ha); /* join watchdog thread before destroying mutex */
pthread_mutex_destroy(&ha->lock);
sem_destroy(&ha->heartbeat_seen);
@ -1473,10 +1201,42 @@ int handle_request_vote(ha_state_t *ha,
uint64_t my_last_lsn,
int in_catch_up_flag)
{
/* Req 6.1, 6.2: role-based deny gate — checked before any Raft state mutation.
* PRIMARY and PRIMARY_COMMITTING never grant votes (protect the live primary).
* STANDBY with a fresh heartbeat also denies (primary is alive, no new election
* needed). CTRL_INIT and CTRL_ELECTING fall through to Raft evaluation. */
/* Req 2.3: higher term → step down BEFORE any role gate.
*
* Raft §5.1: "If RPC request or response contains term T > currentTerm:
* set currentTerm = T, convert to follower."
*
* Without this ordering, a node in CTRL_PRIMARY_COMMITTING at term=N
* would deny a vote request at term=N+1 without ever seeing the higher
* term. Both nodes then independently promote to PRIMARY via their
* committing timeout → split-brain. */
uint64_t my_term = atomic_load_explicit(&ha->term, memory_order_acquire);
if (term > my_term) {
atomic_store_explicit(&ha->term, term, memory_order_release);
ha->voted_for_term = 0;
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
/* Step down from leader/committing: a higher-term election is in
* progress and we must not block it with the role gate below. */
ctrl_role_t cur_role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (cur_role == CTRL_PRIMARY || cur_role == CTRL_PRIMARY_COMMITTING) {
ha_do_demote(ha);
INFO("HA: stepping down (term %" PRIu64 " → %" PRIu64
") — higher-term election in progress", my_term, term);
}
my_term = term;
}
/* Req 2.2: stale term → deny, do not update any state */
if (term < my_term) {
return 0;
}
/* Req 6.1, 6.2: role-based deny gate.
* Evaluated AFTER the term catch-up above: if we just stepped down due to
* a higher term, role is now CTRL_STANDBY and we fall through to voting.
* PRIMARY and PRIMARY_COMMITTING at the current term still deny votes
* (protecting the live primary from same-term challengers). */
ctrl_role_t my_role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (my_role == CTRL_PRIMARY || my_role == CTRL_PRIMARY_COMMITTING)
@ -1491,21 +1251,6 @@ int handle_request_vote(ha_state_t *ha,
/* Protect voted_for check+set from concurrent vote requests on different vtasks */
pthread_mutex_lock(&ha->lock);
uint64_t my_term = atomic_load_explicit(&ha->term, memory_order_acquire);
/* Req 2.2: stale term → deny, do not update any state */
if (term < my_term) {
pthread_mutex_unlock(&ha->lock);
return 0;
}
/* Req 2.3: higher term → advance term, clear prior vote record */
if (term > my_term) {
atomic_store_explicit(&ha->term, term, memory_order_release);
ha->voted_for_term = 0;
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
}
/* Req 2.6: already voted for a different candidate in this term → deny */
if (ha->voted_for_term == term &&
ha->voted_for_name[0] != '\0' &&
@ -1737,17 +1482,25 @@ apply_role:
/* Arm committing timeout watchdog (degraded-mode fallback, Req 8.4) */
if (c) {
/* Cancel any prior committing timeout */
committing_timeout_ctx_t *prev = (committing_timeout_ctx_t *)
atomic_exchange(&ha->committing_timeout_ctx, NULL);
if (prev) {
atomic_store(&prev->cancelled, 1);
}
committing_timeout_ctx_t *ct = calloc(1, sizeof(*ct));
if (ct) {
atomic_init(&ct->cancelled, 0);
ct->ha = ha;
ct->c = c;
atomic_store(&ha->committing_timeout_ctx, ct);
int rc = delay_task_ms(&c->delaymgr,
(int)ha->config.committing_timeout_ms,
1, committing_timeout_fn, ct);
if (rc < 0) {
WARN("ha_elect_evaluate_and_signal: delay_task_ms "
"for committing timeout failed: %d", rc);
atomic_store(&ha->committing_timeout_ctx, NULL);
free(ct);
}
}