myxce/src/lib/delay.c
Rongsong Shen b9ad01878c fix(lib): normalize delaymgr to one CLOCK_MONOTONIC ms timescale, fixing seconds-task starvation
Root-caused via raft-ha-core-safety 5.10a investigation: schedule_checkpoint()
(persist.c) fires correctly once wal_record_count crosses its threshold, but
its delay_task()-scheduled checkpoint_task_fn never executed in a live
multi-node deployment -- confirmed live via temporary diagnostic logging
(wal_record_count=100 max=100 logged, but the task itself never ran).

Root cause: delay_task_t carried TWO incomparable time representations --
`nextrun` (wall-clock time_t seconds, used by delay_task()) and `nextrun_ts`
(CLOCK_MONOTONIC timespec, used by delay_task_ms()). task_cmp()'s mixed-type
fallback compared them directly: an ms-task's nextrun is always 0, a
seconds-task's nextrun is a real epoch value (e.g. 1785600123) -- so 0
always sorted "less than" any seconds-task, regardless of actual due time.
ordq (src/lib/ordq.c) keeps its queue as an ascending sorted linked list by
this comparator, and delay_worker() only ever inspects the list HEAD
(ordq_peek with no matchfn returns q->head immediately, no scanning). Since
cluster_t.delaymgr is shared with ha.c's ~10 recurring delay_task_ms() call
sites (heartbeats, election timers, RDHUP polling), the head was
structurally almost-always occupied by an ms-task -- any delay_task()
(seconds) caller sharing that delaymgr, including schedule_checkpoint's
checkpoint_task_fn, could be starved indefinitely.

Fix: eliminate the dual representation entirely. Every task -- whether
scheduled via delay_task() (seconds) or delay_task_ms() (milliseconds) --
now stores one flat uint64_t CLOCK_MONOTONIC millisecond deadline
(nextrun_ms). delay_task() becomes a thin seconds->ms wrapper around
delay_task_ms(). task_cmp() and task_due() collapse to a single integer
comparison; delay_worker()'s reschedule is a flat integer add (no more
tv_sec/tv_nsec carry arithmetic, no more is_ms_task branching). Net -42
lines. Live-verified: after this fix, a forced checkpoint round produces a
real installed snapshot file (previously: schedule_checkpoint fired but the
task silently never ran).

controller_unit_test: full 58-group sweep, 0 Failures, 0 ASAN reports
(before and after this change) -- no regression, including every
delaymgr-touching group (test_demotion_convergence, test_ha_start_election_async,
test_ha_core_safety, test_ha_startup, etc).
2026-07-02 03:40:08 +00:00

238 lines
5.6 KiB
C

#include <errno.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "delay.h"
#include "utils.h"
/* Flat CLOCK_MONOTONIC "now", in milliseconds. The single time source every
* task deadline (delay_task_t.nextrun_ms) is compared against. */
static uint64_t monotonic_ms(void)
{
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
return (uint64_t)now.tv_sec * 1000ULL + (uint64_t)now.tv_nsec / 1000000ULL;
}
/* Return 1 if a task's deadline has passed, 0 otherwise. Every task in the
* queue uses this single millisecond time domain (see delay.h) -- there is
* no separate "seconds task" due-check anymore. */
static int task_due(const delay_task_t* t)
{
return monotonic_ms() >= t->nextrun_ms;
}
static void* delay_worker(void* args)
{
delaymgr_t* mgr = (delaymgr_t*)args;
while (1) {
ordq_item_t* it;
delay_task_t* t;
pthread_testcancel();
{
long ms = (long)mgr->precision_ms;
struct timespec ts = {
.tv_sec = ms / 1000,
.tv_nsec = (ms % 1000) * 1000000L,
};
nanosleep(&ts, NULL);
}
it = ordq_peek(&(mgr->tasks), NULL);
if (it == NULL) {
continue;
}
t = (delay_task_t*)it->data;
if (!task_due(t)) {
continue;
}
it = ordq_get(&(mgr->tasks), NULL);
t = (delay_task_t*)it->data;
mgr->runner_fn(t->entry, t->args);
if (t->runcount > 0) {
t->runcount--;
}
if (t->runcount == 0) {
FREEUP(t);
} else {
/* advance deadline by run_period_ms -- flat integer add, no
* sec/nsec carry arithmetic needed. */
t->nextrun_ms += (uint64_t)t->run_period_ms;
ordq_put(&(mgr->tasks), t);
}
FREEUP(it);
}
return NULL;
}
/* Single normalized comparison: every task carries one flat millisecond
* deadline (nextrun_ms), so ordering is always apples-to-apples regardless
* of whether the task was scheduled via delay_task() or delay_task_ms(). */
static int task_cmp(void* ctx, void* arg1, void* arg2)
{
(void)ctx;
delay_task_t* t1 = (delay_task_t*)arg1;
delay_task_t* t2 = (delay_task_t*)arg2;
if (t1->nextrun_ms > t2->nextrun_ms) return 1;
if (t1->nextrun_ms < t2->nextrun_ms) return -1;
return 0;
}
static void delay_task_killer(delay_task_t* t)
{
if (t == NULL) {
return;
}
FREEUP(t);
}
static int default_runner_fn(delay_task_entry_fn_t entry, void* arg)
{
if (entry == NULL) {
return 0;
}
return entry(arg);
}
int delaymgr_init(delaymgr_t* mgr, int precision, task_run_fn_t runner_fn)
{
if ((mgr == NULL) || (precision <= 0)) {
return -EINVAL;
}
if (runner_fn == NULL) {
runner_fn = default_runner_fn;
}
mgr->precision_ms = precision;
mgr->running = 0;
mgr->runner_fn = runner_fn;
mgr->worker = 0;
ordq_init(&(mgr->tasks), (ordq_data_killer_t)delay_task_killer,
(ordq_cmp_fn_t)task_cmp);
return 0;
}
void delaymgr_stop(delaymgr_t* mgr)
{
int rc;
if (mgr == NULL) {
return;
}
if (!(mgr->running)) {
return;
}
if (mgr->worker != 0) {
rc = pthread_cancel(mgr->worker);
if (rc == 0) {
void* res = NULL;
pthread_join(mgr->worker, &res);
if (res == PTHREAD_CANCELED) {
mgr->worker = 0;
}
}
}
return;
}
void delaymgr_fini(delaymgr_t* mgr)
{
if (mgr == NULL) {
return;
}
if (mgr->running) {
delaymgr_stop(mgr);
}
ordq_fini(&(mgr->tasks));
}
int delaymgr_start(delaymgr_t* mgr)
{
int rc;
if (mgr == NULL) {
return -EINVAL;
}
rc = pthread_create(&(mgr->worker), NULL, &delay_worker, mgr);
if (rc == 0) {
mgr->running = 1;
} else {
SET_ERROR(errno);
return -errno;
}
return 0;
}
/* delay_task: seconds-granularity scheduling API. A thin wrapper over
* delay_task_ms() -- every task, regardless of which API scheduled it, is
* stored and compared on the single CLOCK_MONOTONIC timescale (delay.h), so
* a seconds-task and an ms-task in the same delaymgr are always correctly
* ordered against each other (previously a mixed wall-clock/monotonic
* comparison meant a seconds-task could be starved indefinitely behind a
* steady stream of ms-tasks -- see the bug this simplification fixes).
* INT_MAX/1000 seconds (~24 days) is far beyond any real caller's delay; a
* value that would overflow the ms conversion is clamped rather than
* wrapped. */
int delay_task(delaymgr_t* mgr, int delay, int runcount,
delay_task_entry_fn_t entry, void* args)
{
if (delay <= 0) {
return -EINVAL;
}
long delay_ms = (long)delay * 1000L;
if (delay_ms > INT_MAX) {
delay_ms = INT_MAX;
}
return delay_task_ms(mgr, (int)delay_ms, runcount, entry, args);
}
int delay_task_ms(delaymgr_t* mgr, int delay_ms, int runcount,
delay_task_entry_fn_t entry, void* args)
{
delay_task_t* t;
if ((mgr == NULL) || (delay_ms <= 0) || (entry == NULL)) {
return -EINVAL;
}
t = (delay_task_t*)calloc(1, sizeof(*t));
if (t == NULL) {
return -ENOMEM;
}
t->nextrun_ms = monotonic_ms() + (uint64_t)delay_ms;
t->runcount = runcount;
t->run_period_ms = delay_ms;
t->entry = entry;
t->args = args;
return ordq_put(&(mgr->tasks), t);
}