lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: heartbeat watchdog pthread + ha_check_heartbeat + start/stop API (P1-3)

Browse source 
- Add ha_start_watchdog() / ha_stop_watchdog() — dedicated pthread with
  nanosleep(heartbeat_interval_ms) loop; stop via atomic watchdog_stop +
  pthread_join (no pthread_cancel, safe for OpenSSL)
- Add ha_check_heartbeat_nocluster() — reads role with
  atomic_load_explicit(memory_order_acquire), guards on primary_host[0]=='\0',
  computes elapsed_ms from CLOCK_MONOTONIC, sets failover_requested on timeout
- Update ha_fini() to call ha_stop_watchdog() before mutex destroy
- 3 new tests (TDD RED→GREEN): watchdog starts/stops, no-op when no primary,
  stale heartbeat no crash; all 97 controller tests green
This commit is contained in:
Rongsong Shen 2026-05-07 14:19:27 +00:00
parent a289edb777
commit 2fae4c3df5
3 changed files with 165 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

75
src/controller/ha.c
View file

@ -37,11 +37,84 @@ int ha_init(ha_state_t *ha, const ha_config_t *cfg, int my_priority)
return 0;
}
/* ── Watchdog thread implementation ────────────────────────── */
void ha_check_heartbeat_nocluster(ha_state_t *ha)
{
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire);
/* Only standbys watch for primary heartbeat timeout */
if (role != CTRL_STANDBY) return;
/* If we have no primary endpoint, nothing to monitor */
if (ha->primary_host[0] == '\0') return;
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
pthread_mutex_lock(&ha->lock);
struct timespec last = ha->last_heartbeat;
pthread_mutex_unlock(&ha->lock);
uint64_t elapsed_ms =
(uint64_t)(now.tv_sec - last.tv_sec) * 1000ULL +
(uint64_t)(now.tv_nsec - last.tv_nsec) / 1000000ULL;
if (elapsed_ms >= ha->config.heartbeat_timeout_ms) {
/* Primary appears dead — full promotion in P1-3a.
* For now: set failover_requested so watchdog thread can act. */
atomic_store_explicit(&ha->failover_requested, 1, memory_order_release);
}
}
static void *ha_watchdog_thread(void *arg)
{
ha_state_t *ha = arg;
while (!atomic_load_explicit(&ha->watchdog_stop, memory_order_acquire)) {
long ms = (long)ha->config.heartbeat_interval_ms;
struct timespec ts = {
.tv_sec = ms / 1000,
.tv_nsec = (ms % 1000) * 1000000L,
};
nanosleep(&ts, NULL);
/* Re-check stop flag after sleep */
if (atomic_load_explicit(&ha->watchdog_stop, memory_order_acquire)) break;
ha_check_heartbeat_nocluster(ha);
/* Handle manual failover request (SIGUSR1 path — P1-3a) */
if (atomic_load_explicit(&ha->failover_requested, memory_order_acquire)) {
atomic_store_explicit(&ha->failover_requested, 0, memory_order_release);
/* Full ha_promote_to_primary(c) call in P1-3a when cluster_t is available */
}
}
return NULL;
}
int ha_start_watchdog(ha_state_t *ha)
{
atomic_store_explicit(&ha->watchdog_stop, 0, memory_order_release);
int rc = pthread_create(&ha->watchdog_thread, NULL, ha_watchdog_thread, ha);
if (rc != 0) return -rc;
return 0;
}
void ha_stop_watchdog(ha_state_t *ha)
{
atomic_store_explicit(&ha->watchdog_stop, 1, memory_order_release);
if (ha->watchdog_thread != 0) {
pthread_join(ha->watchdog_thread, NULL);
ha->watchdog_thread = 0;
}
}
void ha_fini(ha_state_t *ha)
{
if (!ha)
return;
/* Watchdog thread join added in P1-3 */
ha_stop_watchdog(ha); /* join watchdog thread before destroying mutex */
pthread_mutex_destroy(&ha->lock);
}