forked from lei.chen/myxce
Rongsong Shen
102e263cce
Two independent call sites duplicated target = max(committed_lsn, last_written): ha.c's heartbeat/stateReport dispatch (repl_ack_decode_and_ dispatch) and config_repl.c's catchup_derive_target (used by the ACK-driven catch-up continuation path). Once config-upgrade-snapshot's marker-triggered checkpoint advances last_written one LSN past an already-committed committed_lsn (the checkpoint's own consumed, never-replicated LSN slot), the unconditional max() permanently wedges every standby's catch-up target at an unreachable value -- the primary re-streams an empty range every heartbeat forever. Extracted the shared, pure ha_compute_repl_target_lsn(committed_lsn, last_written) (ha.h/ha.c), which only falls back to last_written on genuine committed_lsn==0 bootstrap, and pointed both call sites at it so they can never disagree again. Fixing the second call site exposed three pre-existing tests in test_convergence_verification.c that hardcoded the old buggy formula's output as their expected target; updated their fixture values (committed_lsn 4->9, matching appended_lsn) to decouple them from the now-fixed edge case, which already has dedicated pure-function coverage in test_ha_role.c. GREEN: full controller_unit_test suite = 1040 tests, 0 failures, clean across multiple runs. Live-verified via run-ha-soak.sh --config-upgrade-wal-trim (18 jobs, zero job loss, ctrl-1 reconverged 3-of-3 using the preserved floor segment) -- unblocks config-upgrade-snapshot task 8.9.
1490 lines
70 KiB
C
1490 lines
70 KiB
C
#define _GNU_SOURCE
|
|
/*
|
|
* test_convergence_verification.c — RED-phase tests for ACK-driven catch-up
|
|
* continuation, single-flight, and parity re-inclusion
|
|
* (ha-convergence-recovery task 4.1).
|
|
*
|
|
* Requirements (this spec's IDs — see requirements.md):
|
|
* 5.1 — when the primary receives a WAL ACK from a standby that remains BEHIND
|
|
* the replication target, it continues streaming the missing entries to
|
|
* that standby WITHOUT waiting for the next heartbeat exchange (the
|
|
* continuation streams [acked_lsn+1, target] and dispatches no
|
|
* heartbeat/stateReport replication action of its own).
|
|
* 5.2 — while a catch-up continuation for a standby is already in flight, the
|
|
* primary does NOT start a duplicate continuation for that standby
|
|
* (per-standby single-flight guard); once the in-flight continuation
|
|
* completes, the next lagging ACK schedules again.
|
|
* 5.3 — the heartbeat-driven replication evaluation remains in place as the
|
|
* fallback initiator of catch-up AND the initiator of snapshot-transfer
|
|
* decisions: ha_evaluate_repl_action still returns HA_REPL_STREAM_WAL /
|
|
* HA_REPL_SEND_SNAPSHOT for the same inputs (decision boundary pin).
|
|
* 7.1 — when a lagging standby catches up to the replication boundary (a WAL
|
|
* ACK at/above target), the primary re-includes it in live replication
|
|
* via repl_complete_catchup, which clears the standby's in_catch_up.
|
|
*
|
|
* RED status against the current tree: the production walAck branch
|
|
* (repl_ack_decode_and_dispatch, ha.c — the OPCode_reply /
|
|
* RequestReply_data_walackreplymsg case) only calls repl_on_ack; it derives no
|
|
* target, schedules no continuation, and never calls repl_complete_catchup
|
|
* (which has NO production caller, repl.c). The task-4.1 test seam
|
|
* ha_walack_continuation_for_test is a RED scaffold (ha_convergence.c) that
|
|
* records the ACK only. The continuation / single-flight / parity cases below
|
|
* therefore FAIL until task 4.2 lands the real D6 decision; the 5.3 decision-pin
|
|
* case exercises existing behavior and PASSES now (pinning the boundary so 4.2
|
|
* cannot disturb the heartbeat path).
|
|
*
|
|
* Harness: a partial-but-real cluster_t (real persist over a private tmpdir,
|
|
* a wired commit engine, stub standby registry, un-started delaymgr) mirroring
|
|
* test_demotion_convergence.c's dc_setup, plus the catch-up streaming count seam
|
|
* ha_repl_stream_wal_hook and a dispatch-observer seam to prove the continuation
|
|
* takes no heartbeat path.
|
|
*
|
|
* AAA structure throughout.
|
|
*/
|
|
#include <assert.h>
|
|
#include <errno.h>
|
|
#include <inttypes.h>
|
|
#include <pthread.h>
|
|
#include <stdatomic.h>
|
|
#include <stdint.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <unistd.h>
|
|
|
|
#include <unity/unity.h>
|
|
|
|
#include "commit.h" /* commit_engine_t, commit_engine_init */
|
|
#include "controller.capnp.h" /* OPCode_* (task 6.1 (c) dispatch-gate pin) */
|
|
#include "delay.h" /* delaymgr_t, delay_task_t */
|
|
#include "ha.h" /* ha_walack_continuation_for_test, ha_evaluate_repl_action,
|
|
* ha_repl_stream_wal_hook, HA_REPL_* */
|
|
#include "ordq.h" /* ordq_item_t (drive the scheduled task) */
|
|
#include "persist.h" /* persist_t */
|
|
#include "repl.h" /* repl_add_standby / repl_remove_standby /
|
|
* repl_active_standby_count / standby slots */
|
|
#include "xce/cluster_def.h" /* full cluster_t layout */
|
|
#include "xce/errnox.h" /* XCE_ERR_NOT_READY (task 6.1 (c) gate pin) */
|
|
|
|
#ifdef XCE_TESTING
|
|
|
|
/* ── Broadcast drop-hook (precedent: test_demotion_convergence.c) — the stub int
|
|
* endpoints registered via repl_add_standby are not real endpoint_t, so any
|
|
* repl_broadcast triggered by priming appends must not dereference ep->peer. */
|
|
extern int (*repl_broadcast_send_hook)(void *ep,
|
|
wal_catchup_entry_t **entries,
|
|
int n_entries);
|
|
static int cv_drop_send_hook(void *ep, wal_catchup_entry_t **entries,
|
|
int n_entries)
|
|
{
|
|
(void)ep; (void)entries; (void)n_entries;
|
|
return 0;
|
|
}
|
|
|
|
/* ── Catch-up streaming count seam (Req 5.1) ─────────────────────────────────
|
|
* The ACK-driven continuation must stream the missing range through the existing
|
|
* catch-up streaming path. ha_repl_stream_wal_hook is the established XCE_TESTING
|
|
* seam for that path (config_repl.c ha_stream_wal_to_standby / ha.c
|
|
* HA_REPL_STREAM_WAL). We count invocations and capture the [start, target]
|
|
* range so the continuation's bounds can be asserted. */
|
|
static atomic_int g_stream_calls;
|
|
static atomic_uint_least64_t g_stream_last_standby_lsn;
|
|
static atomic_uint_least64_t g_stream_last_target_lsn;
|
|
|
|
static int cv_stream_hook(void *ep, const char *wal_path,
|
|
uint64_t standby_lsn, uint64_t committed_lsn)
|
|
{
|
|
(void)ep; (void)wal_path;
|
|
atomic_fetch_add(&g_stream_calls, 1);
|
|
atomic_store(&g_stream_last_standby_lsn, standby_lsn);
|
|
atomic_store(&g_stream_last_target_lsn, committed_lsn);
|
|
return 0;
|
|
}
|
|
|
|
/* ── Cluster fixture ─────────────────────────────────────────────────────── */
|
|
static cluster_t g_cv; /* large — must not live on the test stack */
|
|
static int g_cv_eps[MAX_STANDBYS];
|
|
#define CV_PERSIST_NAME "ctrl-test"
|
|
|
|
/* ── Fan-out counting send-hook (task 6.1 (a)/(d), umbrella 6.1/6.4 pins) ─────
|
|
* repl_broadcast routes each enqueue through repl_broadcast_send_hook under
|
|
* XCE_TESTING; this counting variant records, per standby slot, how many entries
|
|
* repl_broadcast tried to send to that slot's ep. An active, non-catch-up
|
|
* standby must receive exactly one send; an empty slot or an in_catch_up slot
|
|
* must receive ZERO. Indexed by matching ep against the g_cv_eps[] addresses.
|
|
* (Declared after g_cv_eps so the slot match resolves.) */
|
|
static atomic_int g_bcast_total_calls;
|
|
static atomic_int g_bcast_slot_calls[MAX_STANDBYS];
|
|
|
|
static int cv_count_send_hook(void *ep, wal_catchup_entry_t **entries,
|
|
int n_entries)
|
|
{
|
|
(void)entries; (void)n_entries;
|
|
atomic_fetch_add(&g_bcast_total_calls, 1);
|
|
for (int i = 0; i < MAX_STANDBYS; i++) {
|
|
if (ep == (void *)&g_cv_eps[i]) {
|
|
atomic_fetch_add(&g_bcast_slot_calls[i], 1);
|
|
break;
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static void cv_bcast_reset_counters(void)
|
|
{
|
|
atomic_store(&g_bcast_total_calls, 0);
|
|
for (int i = 0; i < MAX_STANDBYS; i++)
|
|
atomic_store(&g_bcast_slot_calls[i], 0);
|
|
}
|
|
|
|
static ha_config_t cv_ha_cfg(void)
|
|
{
|
|
ha_config_t cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
};
|
|
return cfg;
|
|
}
|
|
|
|
/* Build g_cv: real persist over a fresh tmpdir, n_standbys stub standbys, a
|
|
* wired commit engine, an initialised-but-UNSTARTED delaymgr (so any task in
|
|
* tasks.head was armed by the code under test, not by a background runner).
|
|
* Role left at INIT (the walAck continuation is a primary-side concern but the
|
|
* decision under test needs only the persist watermark + repl registry, not the
|
|
* full promotion sequence). Leaves committed_lsn at 0; callers set it. */
|
|
static void cv_setup(const char *suffix, int n_peers, int n_standbys,
|
|
uint64_t appended_lsn, int n_vcpus)
|
|
{
|
|
char name[32];
|
|
char tmpdir[256];
|
|
|
|
memset(&g_cv, 0, sizeof(g_cv));
|
|
atomic_store(&g_stream_calls, 0);
|
|
atomic_store(&g_stream_last_standby_lsn, 0);
|
|
atomic_store(&g_stream_last_target_lsn, 0);
|
|
|
|
ha_config_t cfg = cv_ha_cfg();
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&g_cv.ha, &cfg, 0));
|
|
g_cv.ha.phase2_done = 1;
|
|
g_cv.ha.n_peers = n_peers;
|
|
|
|
persist_config_t pcfg = persist_config_defaults();
|
|
pcfg.wal_sync_ms = 5000; /* slow background fsync — no races */
|
|
pcfg.sync_timeout_ms = 50; /* no real standby ACKs here */
|
|
snprintf(tmpdir, sizeof(tmpdir), "/tmp/xce_cv_%s_XXXXXX", suffix);
|
|
TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir));
|
|
persist_set_workdir(&g_cv.persist, tmpdir, CV_PERSIST_NAME);
|
|
TEST_ASSERT_EQUAL_INT(0, persist_init(&g_cv.persist, &pcfg, &g_cv));
|
|
TEST_ASSERT_EQUAL_INT(0, persist_start(&g_cv.persist));
|
|
|
|
/* Prime the appended watermark: current_lsn = appended_lsn + 1, so
|
|
* last_written = appended_lsn (the replication target's upper bound). */
|
|
pthread_mutex_lock(&g_cv.persist.lock);
|
|
g_cv.persist.current_lsn = appended_lsn + 1;
|
|
pthread_mutex_unlock(&g_cv.persist.lock);
|
|
atomic_store(&g_cv.persist.committed_lsn, 0);
|
|
|
|
for (int i = 0; i < n_standbys; i++) {
|
|
snprintf(name, sizeof(name), "cvsb%d", i);
|
|
TEST_ASSERT_TRUE_MESSAGE(
|
|
repl_add_standby(&g_cv.persist.repl, &g_cv_eps[i], name,
|
|
7600 + (uint32_t)i) >= 0,
|
|
"repl_add_standby must return a slot index >= 0");
|
|
}
|
|
TEST_ASSERT_EQUAL_INT(0,
|
|
commit_engine_init(&g_cv.commit, &g_cv.persist, n_peers, n_vcpus));
|
|
repl_set_commit_engine(&g_cv.persist.repl, &g_cv.commit);
|
|
|
|
repl_broadcast_send_hook = cv_drop_send_hook;
|
|
ha_repl_stream_wal_hook = cv_stream_hook;
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, delaymgr_init(&g_cv.delaymgr, 100, NULL));
|
|
}
|
|
|
|
static void cv_teardown(void)
|
|
{
|
|
repl_broadcast_send_hook = NULL;
|
|
ha_repl_stream_wal_hook = NULL;
|
|
commit_engine_fini(&g_cv.commit);
|
|
delaymgr_fini(&g_cv.delaymgr);
|
|
persist_stop(&g_cv.persist);
|
|
persist_fini(&g_cv.persist);
|
|
repl_fini(&g_cv.persist.repl);
|
|
ha_fini(&g_cv.ha);
|
|
/* tmpdir left on disk — /tmp is ephemeral */
|
|
}
|
|
|
|
/* Count pending delaymgr tasks by walking the ordq head chain (read-only;
|
|
* single-threaded test, delaymgr never started). */
|
|
static int cv_task_count(const delaymgr_t *mgr)
|
|
{
|
|
int count = 0;
|
|
const ordq_item_t *it = mgr->tasks.head;
|
|
while (it != NULL) {
|
|
count++;
|
|
it = it->next;
|
|
}
|
|
return count;
|
|
}
|
|
|
|
/* Run the single pending delaymgr one-shot task synchronously by invoking its
|
|
* stored entry(args), then return the entry's rc. Returns CV_NO_TASK if there
|
|
* is no pending task (RED: nothing was scheduled). DEQUEUES + frees, mirroring
|
|
* the real delaymgr worker (delay.c) and test_demotion_convergence.c's
|
|
* dc_run_pending_task — required so a second call after a body that frees its own
|
|
* ctx does not re-run a freed ctx (heap-UAF). */
|
|
#define CV_NO_TASK 0x7fffffff
|
|
static int cv_run_pending_task(delaymgr_t *mgr)
|
|
{
|
|
ordq_item_t *it = ordq_get(&mgr->tasks, NULL);
|
|
if (it == NULL || it->data == NULL) {
|
|
if (it != NULL)
|
|
free(it);
|
|
return CV_NO_TASK;
|
|
}
|
|
delay_task_t *t = (delay_task_t *)it->data;
|
|
if (t->entry == NULL) {
|
|
free(t);
|
|
free(it);
|
|
return CV_NO_TASK;
|
|
}
|
|
int rc = t->entry(t->args); /* task body frees its own args ctx */
|
|
free(t);
|
|
free(it);
|
|
return rc;
|
|
}
|
|
|
|
/* Read the per-standby in_catch_up flag (repl registry slot identified by ep). */
|
|
static int cv_standby_in_catch_up(repl_ep_t ep)
|
|
{
|
|
replicator_t *r = &g_cv.persist.repl;
|
|
int result = -1;
|
|
mylock_lock(&r->lock);
|
|
for (int i = 0; i < MAX_STANDBYS; i++) {
|
|
if (r->standbys[i].ep == ep) {
|
|
result = atomic_load_explicit(&r->standbys[i].in_catch_up,
|
|
memory_order_acquire);
|
|
break;
|
|
}
|
|
}
|
|
mylock_unlock(&r->lock);
|
|
return result;
|
|
}
|
|
|
|
/* ── Tests ──────────────────────────────────────────────────────────────── */
|
|
|
|
/* 5.1: a walAck with acked_lsn < target = ha_compute_repl_target_lsn(
|
|
* committed_lsn, last_written) schedules EXACTLY ONE continuation that
|
|
* streams [acked_lsn+1, target] without any intervening heartbeat/
|
|
* stateReport dispatch.
|
|
* RED: the walAck continuation scaffold schedules nothing → no task, the stream
|
|
* hook is never called. */
|
|
static void test_walack_behind_target_schedules_one_continuation(void)
|
|
{
|
|
/* appended_lsn=9 → last_written=9; committed_lsn=9 → target=9. Kept equal
|
|
* to last_written (rather than a lower value like the historical 4) so
|
|
* this test exercises catch-up continuation scheduling only — the
|
|
* committed_lsn-vs-last_written edge case itself (bug-1478) has its own
|
|
* dedicated pure-function coverage in test_ha_role.c. */
|
|
cv_setup("behind", /*n_peers=*/3, /*n_standbys=*/1,
|
|
/*appended_lsn=*/9, /*n_vcpus=*/8);
|
|
atomic_store(&g_cv.persist.committed_lsn, 9);
|
|
|
|
const int peer_idx = 0;
|
|
const uint64_t acked_lsn = 3; /* behind target 9 */
|
|
const uint64_t target = 9;
|
|
|
|
int tasks_before = cv_task_count(&g_cv.delaymgr);
|
|
|
|
/* Act: standby acknowledges lsn=3 while target=9. */
|
|
int rc = ha_walack_continuation_for_test(&g_cv, &g_cv_eps[peer_idx],
|
|
peer_idx, acked_lsn);
|
|
(void)rc;
|
|
|
|
/* A continuation must have been scheduled OFF the ACK path (exactly one). */
|
|
int tasks_after = cv_task_count(&g_cv.delaymgr);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(tasks_before + 1, tasks_after,
|
|
"a walAck behind target must schedule exactly one catch-up continuation "
|
|
"(Req 5.1)");
|
|
|
|
/* No streaming may have happened on the ACK path itself (it defers). */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_stream_calls),
|
|
"the walAck branch must NOT stream inline — streaming runs in the "
|
|
"deferred continuation (Req 5.1, iomgr-safe)");
|
|
|
|
/* Run the scheduled continuation; it must stream [acked_lsn+1, target]. */
|
|
int trc = cv_run_pending_task(&g_cv.delaymgr);
|
|
TEST_ASSERT_NOT_EQUAL_MESSAGE(CV_NO_TASK, trc,
|
|
"no continuation task was scheduled on the walAck-behind-target edge "
|
|
"(Req 5.1)");
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_stream_calls),
|
|
"the continuation must stream exactly once (Req 5.1)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(acked_lsn,
|
|
atomic_load(&g_stream_last_standby_lsn),
|
|
"the continuation must stream starting just above the ACKed LSN "
|
|
"(ha_stream_wal_to_standby streams (standby_lsn, target] = "
|
|
"[acked_lsn+1, target]) (Req 5.1)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(target,
|
|
atomic_load(&g_stream_last_target_lsn),
|
|
"the continuation must stream up to target = "
|
|
"ha_compute_repl_target_lsn(committed_lsn, last_written) (Req 5.1)");
|
|
|
|
cv_teardown();
|
|
printf("test_walack_behind_target_schedules_one_continuation: checked\n");
|
|
}
|
|
|
|
/* 5.2 single-flight: a SECOND walAck while a continuation is already in flight
|
|
* for the same standby schedules NOTHING.
|
|
* RED: the scaffold schedules nothing for the first ACK either, so the
|
|
* "exactly one continuation across two ACKs" assertion fails (0 != 1). */
|
|
static void test_walack_single_flight_no_duplicate_continuation(void)
|
|
{
|
|
/* committed_lsn == appended_lsn (target=9 either way) — see the "behind"
|
|
* test's own note on why these don't use a lower committed_lsn. */
|
|
cv_setup("single", /*n_peers=*/3, /*n_standbys=*/1,
|
|
/*appended_lsn=*/9, /*n_vcpus=*/8);
|
|
atomic_store(&g_cv.persist.committed_lsn, 9);
|
|
|
|
const int peer_idx = 0;
|
|
|
|
int tasks_before = cv_task_count(&g_cv.delaymgr);
|
|
|
|
/* Act: first walAck (behind target) → schedules one continuation and marks
|
|
* the standby's catchup_inflight. */
|
|
(void)ha_walack_continuation_for_test(&g_cv, &g_cv_eps[peer_idx], peer_idx,
|
|
/*acked_lsn=*/3);
|
|
int tasks_after_first = cv_task_count(&g_cv.delaymgr);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(tasks_before + 1, tasks_after_first,
|
|
"the first walAck behind target must schedule one continuation (Req 5.2 "
|
|
"setup)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1,
|
|
atomic_load(&g_cv.ha.catchup_inflight[peer_idx]),
|
|
"scheduling a continuation must set the per-standby single-flight flag "
|
|
"(Req 5.2)");
|
|
|
|
/* Act: a SECOND walAck while the continuation is still in flight (the task
|
|
* has NOT been drained). */
|
|
(void)ha_walack_continuation_for_test(&g_cv, &g_cv_eps[peer_idx], peer_idx,
|
|
/*acked_lsn=*/5);
|
|
|
|
/* Assert (Req 5.2): no duplicate continuation — task count unchanged. */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(tasks_after_first,
|
|
cv_task_count(&g_cv.delaymgr),
|
|
"a second walAck while a continuation is in flight must schedule NOTHING "
|
|
"(single-flight, Req 5.2)");
|
|
|
|
/* Drain the single in-flight continuation (clears the flag). */
|
|
(void)cv_run_pending_task(&g_cv.delaymgr);
|
|
|
|
cv_teardown();
|
|
printf("test_walack_single_flight_no_duplicate_continuation: checked\n");
|
|
}
|
|
|
|
/* 5.2 release: after the in-flight continuation completes (flag cleared), the
|
|
* NEXT lagging walAck schedules a fresh continuation again.
|
|
* RED: the scaffold never schedules/clears, so neither continuation appears. */
|
|
static void test_walack_reschedules_after_continuation_completes(void)
|
|
{
|
|
/* committed_lsn == appended_lsn (target=9 either way) — see the "behind"
|
|
* test's own note on why these don't use a lower committed_lsn. */
|
|
cv_setup("respawn", /*n_peers=*/3, /*n_standbys=*/1,
|
|
/*appended_lsn=*/9, /*n_vcpus=*/8);
|
|
atomic_store(&g_cv.persist.committed_lsn, 9);
|
|
|
|
const int peer_idx = 0;
|
|
|
|
/* Act 1: first walAck → continuation #1. */
|
|
(void)ha_walack_continuation_for_test(&g_cv, &g_cv_eps[peer_idx], peer_idx,
|
|
/*acked_lsn=*/3);
|
|
int trc1 = cv_run_pending_task(&g_cv.delaymgr);
|
|
TEST_ASSERT_NOT_EQUAL_MESSAGE(CV_NO_TASK, trc1,
|
|
"first walAck must schedule a continuation (Req 5.2 release setup)");
|
|
|
|
/* After the continuation completes the single-flight flag must be clear. */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
atomic_load(&g_cv.ha.catchup_inflight[peer_idx]),
|
|
"the continuation must clear the single-flight flag on completion "
|
|
"(Req 5.2 release)");
|
|
|
|
int streams_after_first = atomic_load(&g_stream_calls);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, streams_after_first,
|
|
"the first continuation must have streamed once");
|
|
|
|
/* Act 2: a later lagging walAck (still behind target 9) → continuation #2. */
|
|
(void)ha_walack_continuation_for_test(&g_cv, &g_cv_eps[peer_idx], peer_idx,
|
|
/*acked_lsn=*/6);
|
|
int trc2 = cv_run_pending_task(&g_cv.delaymgr);
|
|
TEST_ASSERT_NOT_EQUAL_MESSAGE(CV_NO_TASK, trc2,
|
|
"a lagging walAck after the prior continuation completed must schedule "
|
|
"AGAIN (Req 5.2 release)");
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(2, atomic_load(&g_stream_calls),
|
|
"the second continuation must stream once more after the first released "
|
|
"(Req 5.2 release)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(6,
|
|
atomic_load(&g_stream_last_standby_lsn),
|
|
"the second continuation streams from the newly ACKed LSN (Req 5.2 "
|
|
"release)");
|
|
|
|
cv_teardown();
|
|
printf("test_walack_reschedules_after_continuation_completes: checked\n");
|
|
}
|
|
|
|
/* 7.1 parity: a walAck with acked_lsn >= target calls repl_complete_catchup,
|
|
* which clears the standby's in_catch_up (re-inclusion in live replication).
|
|
* RED: the scaffold never calls repl_complete_catchup, so in_catch_up — set here
|
|
* via repl_start_catchup — stays 1. */
|
|
static void test_walack_at_parity_completes_catchup_and_clears_in_catch_up(void)
|
|
{
|
|
/* appended_lsn=9 → last_written=9; committed_lsn=9 → target=9. */
|
|
cv_setup("parity", /*n_peers=*/3, /*n_standbys=*/1,
|
|
/*appended_lsn=*/9, /*n_vcpus=*/8);
|
|
atomic_store(&g_cv.persist.committed_lsn, 9);
|
|
|
|
const int peer_idx = 0;
|
|
repl_ep_t ep = &g_cv_eps[peer_idx];
|
|
const uint64_t acked_lsn = 9; /* at parity with target 9 */
|
|
|
|
/* Put the standby into catch-up so re-inclusion is observable. */
|
|
repl_start_catchup(&g_cv.persist.repl, ep, /*join_applied_lsn=*/0);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, cv_standby_in_catch_up(ep),
|
|
"fixture precondition: the standby must be in_catch_up before parity ACK");
|
|
|
|
int tasks_before = cv_task_count(&g_cv.delaymgr);
|
|
|
|
/* Act: the standby acknowledges the target → parity. */
|
|
(void)ha_walack_continuation_for_test(&g_cv, ep, peer_idx, acked_lsn);
|
|
|
|
/* Assert (Req 7.1): no continuation scheduled at parity, and the standby is
|
|
* re-included — in_catch_up cleared by repl_complete_catchup. */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(tasks_before, cv_task_count(&g_cv.delaymgr),
|
|
"a parity walAck must NOT schedule a streaming continuation (Req 7.1)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_stream_calls),
|
|
"a parity walAck must NOT stream (nothing left to send) (Req 7.1)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, cv_standby_in_catch_up(ep),
|
|
"a parity walAck must call repl_complete_catchup, clearing the standby's "
|
|
"in_catch_up so it re-enters live replication (Req 7.1)");
|
|
|
|
cv_teardown();
|
|
printf("test_walack_at_parity_completes_catchup_and_clears_in_catch_up: checked\n");
|
|
}
|
|
|
|
/* 5.3 decision pin: the heartbeat decision function ha_evaluate_repl_action is
|
|
* the fallback initiator of catch-up and the SOLE owner of the snapshot
|
|
* decision; the ACK-driven continuation must not change its verdicts. This pins
|
|
* the existing boundary so task 4.2 cannot disturb the heartbeat path.
|
|
*
|
|
* This case exercises only existing pure-function behavior, so it PASSES against
|
|
* the current tree (intended — it locks the boundary; the RED signal for task
|
|
* 4.1 comes from the continuation/single-flight/parity cases above). */
|
|
static void test_heartbeat_decision_path_unchanged(void)
|
|
{
|
|
/* HA_REPL_STREAM_WAL: tag current, lsn within the WAL window
|
|
* (oldest_wal_lsn <= standby_lsn < committed_lsn). */
|
|
ha_repl_action_t a_stream = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/3,
|
|
/*committed_tag=*/5, /*committed_lsn=*/9, /*oldest_wal_lsn=*/1);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)HA_REPL_STREAM_WAL, (int)a_stream,
|
|
"heartbeat decision: standby_lsn within the WAL window must remain "
|
|
"HA_REPL_STREAM_WAL — the heartbeat path stays the fallback catch-up "
|
|
"initiator (Req 5.3)");
|
|
|
|
/* HA_REPL_SEND_SNAPSHOT: standby_lsn genuinely GC'd behind the oldest
|
|
* retained WAL (standby_lsn + 1 < oldest_wal_lsn → 2 + 1 = 3 < 5). */
|
|
ha_repl_action_t a_snap = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/2,
|
|
/*committed_tag=*/5, /*committed_lsn=*/9, /*oldest_wal_lsn=*/5);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)HA_REPL_SEND_SNAPSHOT, (int)a_snap,
|
|
"heartbeat decision: standby GC'd behind oldest_wal_lsn must remain "
|
|
"HA_REPL_SEND_SNAPSHOT — the heartbeat path remains the SOLE snapshot "
|
|
"decision owner (Req 5.3); the ACK-driven continuation never decides "
|
|
"snapshots");
|
|
|
|
/* Strict boundary (fix-snapshot-boundary): standby_lsn == oldest_wal_lsn - 1
|
|
* (next-needed entry retained) must STREAM, not snapshot — pins the
|
|
* standby_lsn + 1 < oldest_wal_lsn comparison on the heartbeat path. */
|
|
ha_repl_action_t a_edge = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/4,
|
|
/*committed_tag=*/5, /*committed_lsn=*/9, /*oldest_wal_lsn=*/5);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)HA_REPL_STREAM_WAL, (int)a_edge,
|
|
"heartbeat decision: standby_lsn (4) == oldest_wal_lsn (5) - 1 — the "
|
|
"next-needed entry (5) is retained, so the heartbeat path must STREAM, "
|
|
"not snapshot (strict standby_lsn + 1 < oldest_wal_lsn boundary)");
|
|
|
|
printf("test_heartbeat_decision_path_unchanged: checked\n");
|
|
}
|
|
|
|
/* ── Quorum transition logging (task 5.1, Req 8.1-8.3) ───────────────────────
|
|
*
|
|
* Observable pattern: there is NO log-capture harness in this testsuite, so
|
|
* these tests use the emission-count seam ha_quorum_transition_log_hook (the
|
|
* task permits "log-capture OR counter-hook pattern"). The real
|
|
* ha_log_quorum_transition (task 5.2) invokes the hook exactly once on each
|
|
* loss/restore EDGE it WARNs about, passing the new state and the
|
|
* connected/required counts that go into the WARN line. We count invocations to
|
|
* assert "logged once per transition" and inspect the args to assert the loss
|
|
* line carries the right counts.
|
|
*
|
|
* RED status against the current tree: ha_log_quorum_transition is a no-op RED
|
|
* scaffold (ha_convergence.c) — it performs no edge detection, flips no flag,
|
|
* and never invokes the hook. Every assertion below that expects an emission
|
|
* (or an edge-flag flip) therefore FAILS until task 5.2 lands the real edge
|
|
* detector and wires the call sites.
|
|
*
|
|
* Quorum arithmetic for the fixture (ha_has_write_quorum, ha.c):
|
|
* n_peers=3, ha->repl set → connected = repl_active_standby_count(repl);
|
|
* quorum held iff connected >= n_peers/2 = 1. So 1 active standby = quorum,
|
|
* 0 active standbys = quorum lost. The required count in the WARN line is the
|
|
* threshold n_peers/2 = 1.
|
|
*/
|
|
|
|
/* Emission-count hook: records every transition WARN ha_log_quorum_transition
|
|
* emits, capturing the latest (lost, connected, required) triple. */
|
|
static atomic_int g_qt_calls;
|
|
static atomic_int g_qt_loss_calls;
|
|
static atomic_int g_qt_restore_calls;
|
|
static atomic_int g_qt_last_lost;
|
|
static atomic_int g_qt_last_connected;
|
|
static atomic_int g_qt_last_required;
|
|
|
|
static void cv_quorum_log_hook(int lost, int connected, int required)
|
|
{
|
|
atomic_fetch_add(&g_qt_calls, 1);
|
|
if (lost)
|
|
atomic_fetch_add(&g_qt_loss_calls, 1);
|
|
else
|
|
atomic_fetch_add(&g_qt_restore_calls, 1);
|
|
atomic_store(&g_qt_last_lost, lost);
|
|
atomic_store(&g_qt_last_connected, connected);
|
|
atomic_store(&g_qt_last_required, required);
|
|
}
|
|
|
|
static void cv_quorum_reset_counters(void)
|
|
{
|
|
atomic_store(&g_qt_calls, 0);
|
|
atomic_store(&g_qt_loss_calls, 0);
|
|
atomic_store(&g_qt_restore_calls, 0);
|
|
atomic_store(&g_qt_last_lost, -1);
|
|
atomic_store(&g_qt_last_connected, -1);
|
|
atomic_store(&g_qt_last_required, -1);
|
|
}
|
|
|
|
/* Build a quorum fixture: n_peers=3, registry-truth quorum accounting wired
|
|
* (ha.repl = &persist.repl) and exactly one active standby so the cluster STARTS
|
|
* with write quorum held (quorum_lost == 0). Installs the emission-count hook. */
|
|
static void cv_quorum_setup(const char *suffix)
|
|
{
|
|
cv_setup(suffix, /*n_peers=*/3, /*n_standbys=*/1,
|
|
/*appended_lsn=*/0, /*n_vcpus=*/8);
|
|
/* Wire registry-truth quorum accounting (ha-commit-engine path): with
|
|
* ha.repl set, ha_has_write_quorum counts repl_active_standby_count. */
|
|
g_cv.ha.repl = &g_cv.persist.repl;
|
|
g_cv.ha.role = CTRL_PRIMARY; /* quorum gate is a CTRL_PRIMARY concern */
|
|
|
|
cv_quorum_reset_counters();
|
|
ha_quorum_transition_log_hook = cv_quorum_log_hook;
|
|
|
|
/* Fixture precondition: one active standby → connected=1 >= required=1 →
|
|
* quorum held at the start of every quorum test. */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1,
|
|
repl_active_standby_count(&g_cv.persist.repl),
|
|
"quorum fixture precondition: exactly one active standby (quorum held)");
|
|
}
|
|
|
|
static void cv_quorum_teardown(void)
|
|
{
|
|
ha_quorum_transition_log_hook = NULL;
|
|
cv_teardown();
|
|
}
|
|
|
|
/* 8.1: dropping the only standby below majority logs the loss WARN EXACTLY ONCE,
|
|
* carrying the connected/required counts; the edge flag flips to lost.
|
|
* RED: the no-op stub never detects the edge → 0 emissions, flag stays 0. */
|
|
static void test_quorum_loss_logged_once_with_counts(void)
|
|
{
|
|
cv_quorum_setup("qloss");
|
|
|
|
/* Act: remove the only standby → connected drops 1→0 → quorum lost. */
|
|
repl_remove_standby(&g_cv.persist.repl, &g_cv_eps[0]);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
repl_active_standby_count(&g_cv.persist.repl),
|
|
"fixture: the cluster must have zero active standbys after removal");
|
|
|
|
ha_log_quorum_transition(&g_cv.ha);
|
|
|
|
/* Assert (Req 8.1): exactly one loss WARN, no restore WARN. */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_loss_calls),
|
|
"dropping below majority must log the quorum-loss WARN exactly once "
|
|
"(Req 8.1)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_qt_restore_calls),
|
|
"a quorum-loss edge must not emit a restore WARN (Req 8.1)");
|
|
|
|
/* Assert (Req 8.1): the WARN carries the connected/required counts. */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_qt_last_connected),
|
|
"the quorum-loss WARN must report connected=0 (Req 8.1)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_last_required),
|
|
"the quorum-loss WARN must report required=1 (n_peers/2 threshold) "
|
|
"(Req 8.1)");
|
|
|
|
/* The edge-detector flag must now read 'lost'. */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_cv.ha.quorum_lost),
|
|
"a quorum-loss edge must set the ha_state_t.quorum_lost flag (Req 8.1)");
|
|
|
|
cv_quorum_teardown();
|
|
printf("test_quorum_loss_logged_once_with_counts: checked\n");
|
|
}
|
|
|
|
/* 8.3: while quorum stays lost, repeated dispatch-gate rejections (modeled as
|
|
* repeated ha_log_quorum_transition calls — the gate backstop calls it on every
|
|
* rejected op) add NO further transition logs.
|
|
* RED: the no-op stub never logs the first loss either, so the "exactly one loss
|
|
* across many calls" assertion fails (0 != 1). */
|
|
static void test_quorum_loss_no_repeat_log_while_lost(void)
|
|
{
|
|
cv_quorum_setup("qstorm");
|
|
|
|
/* Cross the loss edge once. */
|
|
repl_remove_standby(&g_cv.persist.repl, &g_cv_eps[0]);
|
|
ha_log_quorum_transition(&g_cv.ha);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_loss_calls),
|
|
"setup: the first call after the loss edge must log once (Req 8.3 setup)");
|
|
|
|
/* Act: simulate many rejected operations re-invoking the backstop while
|
|
* quorum stays lost (no membership change between calls). */
|
|
for (int i = 0; i < 5; i++)
|
|
ha_log_quorum_transition(&g_cv.ha);
|
|
|
|
/* Assert (Req 8.3): still exactly one loss WARN — no log storm. */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_loss_calls),
|
|
"repeated quorum-transition calls while quorum stays lost must NOT log "
|
|
"again — once per transition, no storms (Req 8.3)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_qt_restore_calls),
|
|
"no restore WARN may appear while quorum remains lost (Req 8.3)");
|
|
|
|
cv_quorum_teardown();
|
|
printf("test_quorum_loss_no_repeat_log_while_lost: checked\n");
|
|
}
|
|
|
|
/* 8.2: restoring a standby above majority logs the restore WARN ONCE; the edge
|
|
* flag flips back to held.
|
|
* RED: the no-op stub neither sets nor clears the flag and never emits. */
|
|
static void test_quorum_restore_logged_once(void)
|
|
{
|
|
cv_quorum_setup("qrestore");
|
|
|
|
/* Drive into the lost state first. */
|
|
repl_remove_standby(&g_cv.persist.repl, &g_cv_eps[0]);
|
|
ha_log_quorum_transition(&g_cv.ha);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_loss_calls),
|
|
"setup: must be in the lost state with one loss WARN (Req 8.2 setup)");
|
|
|
|
/* Act: re-register a standby → connected 0→1 → quorum restored. */
|
|
TEST_ASSERT_TRUE(
|
|
repl_add_standby(&g_cv.persist.repl, &g_cv_eps[0], "cvsb0", 7600) >= 0);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1,
|
|
repl_active_standby_count(&g_cv.persist.repl),
|
|
"fixture: the re-registered standby must restore the active count to 1");
|
|
|
|
ha_log_quorum_transition(&g_cv.ha);
|
|
|
|
/* Assert (Req 8.2): exactly one restore WARN, loss count unchanged. */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_restore_calls),
|
|
"restoring quorum must log the quorum-restore WARN exactly once "
|
|
"(Req 8.2)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_loss_calls),
|
|
"the restore edge must not re-emit a loss WARN (Req 8.2)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_cv.ha.quorum_lost),
|
|
"a quorum-restore edge must clear the ha_state_t.quorum_lost flag "
|
|
"(Req 8.2)");
|
|
|
|
cv_quorum_teardown();
|
|
printf("test_quorum_restore_logged_once: checked\n");
|
|
}
|
|
|
|
/* 8.1+8.2 edge detector (not a latch): a SECOND loss/restore cycle logs again.
|
|
* RED: the no-op stub never logs any cycle. */
|
|
static void test_quorum_second_cycle_logs_again(void)
|
|
{
|
|
cv_quorum_setup("qcycle");
|
|
|
|
/* Cycle 1: lose then restore. */
|
|
repl_remove_standby(&g_cv.persist.repl, &g_cv_eps[0]);
|
|
ha_log_quorum_transition(&g_cv.ha);
|
|
TEST_ASSERT_TRUE(
|
|
repl_add_standby(&g_cv.persist.repl, &g_cv_eps[0], "cvsb0", 7600) >= 0);
|
|
ha_log_quorum_transition(&g_cv.ha);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_loss_calls),
|
|
"cycle 1: exactly one loss WARN (edge-detector setup)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_restore_calls),
|
|
"cycle 1: exactly one restore WARN (edge-detector setup)");
|
|
|
|
/* Cycle 2: lose then restore AGAIN — the detector is not a one-shot latch. */
|
|
repl_remove_standby(&g_cv.persist.repl, &g_cv_eps[0]);
|
|
ha_log_quorum_transition(&g_cv.ha);
|
|
TEST_ASSERT_TRUE(
|
|
repl_add_standby(&g_cv.persist.repl, &g_cv_eps[0], "cvsb0", 7600) >= 0);
|
|
ha_log_quorum_transition(&g_cv.ha);
|
|
|
|
/* Assert: each transition logged again (totals = 2 loss + 2 restore). */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(2, atomic_load(&g_qt_loss_calls),
|
|
"a SECOND quorum-loss edge must log again — the transition logger is an "
|
|
"edge detector, not a one-shot latch (Req 8.1)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(2, atomic_load(&g_qt_restore_calls),
|
|
"a SECOND quorum-restore edge must log again (Req 8.2)");
|
|
|
|
cv_quorum_teardown();
|
|
printf("test_quorum_second_cycle_logs_again: checked\n");
|
|
}
|
|
|
|
/* ════════════════════════════════════════════════════════════════════════════
|
|
* Task 6.1 — REGRESSION PINS (verification of existing behavior, NO production
|
|
* change). These encode behavior already present in the current tree; each pin
|
|
* PASSES against the code as-is. A failure here is a real regression, not a
|
|
* RED-phase placeholder.
|
|
*
|
|
* (a) repl_broadcast full fan-out — umbrella 6.1 pin (Req 6.1)
|
|
* (b) ha_evaluate_repl_action snapshot fallback boundary — umbrella 6.3 pin
|
|
* (Req 6.1, decision boundary only)
|
|
* (c) ha_dispatch_gate degraded-mode allowlist — umbrella 7.3 pin (Req 7.1)
|
|
* (d) repl_complete_catchup re-inclusion mechanism — umbrella 6.4 pin
|
|
* (Req 7.1, complements task 4's ACK-driven trigger test)
|
|
* ════════════════════════════════════════════════════════════════════════════ */
|
|
|
|
/* Drive one repl_broadcast of a single WAL_NOOP entry (no payload). */
|
|
static int cv_broadcast_one(replicator_t *r, uint64_t lsn)
|
|
{
|
|
wal_entry_hdr_t hdr;
|
|
memset(&hdr, 0, sizeof(hdr));
|
|
hdr.lsn = lsn;
|
|
hdr.type = WAL_NOOP;
|
|
hdr.payload_len = 0;
|
|
return repl_broadcast(r, &hdr, NULL, 0, 0 /* prev_log_term */);
|
|
}
|
|
|
|
/* (a) umbrella 6.1: repl_broadcast enqueues to EVERY registered active standby
|
|
* and skips ONLY empty slots and in_catch_up standbys.
|
|
*
|
|
* Fixture: three standby slots — slot 0 active, slot 1 emptied (removed), slot 2
|
|
* active but in_catch_up. A single broadcast must hit slot 0 exactly once and
|
|
* NEITHER slot 1 (empty) NOR slot 2 (catch-up). Pins existing repl.c:259-307. */
|
|
static void test_pin_repl_broadcast_full_fanout_skips_empty_and_catchup(void)
|
|
{
|
|
cv_setup("fanout", /*n_peers=*/3, /*n_standbys=*/3,
|
|
/*appended_lsn=*/0, /*n_vcpus=*/8);
|
|
|
|
/* Swap the drop hook for the counting hook and reset its tallies. */
|
|
cv_bcast_reset_counters();
|
|
repl_broadcast_send_hook = cv_count_send_hook;
|
|
|
|
/* Arrange the three slots into the three states under test. */
|
|
repl_remove_standby(&g_cv.persist.repl, &g_cv_eps[1]); /* slot 1 → empty */
|
|
repl_start_catchup(&g_cv.persist.repl, &g_cv_eps[2],
|
|
/*join_applied_lsn=*/0); /* slot 2 → catch-up */
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, cv_standby_in_catch_up(&g_cv_eps[2]),
|
|
"fixture precondition: slot 2 must be in_catch_up before broadcast");
|
|
|
|
/* Act: one broadcast. */
|
|
int rc = cv_broadcast_one(&g_cv.persist.repl, /*lsn=*/1);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "repl_broadcast must succeed");
|
|
|
|
/* Assert (Req 6.1): the active non-catch-up standby got exactly one send. */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_bcast_slot_calls[0]),
|
|
"repl_broadcast must enqueue to the active non-catch-up standby (slot 0) "
|
|
"exactly once (umbrella 6.1 full fan-out)");
|
|
|
|
/* Assert: the EMPTY slot was skipped (no ep → no send). */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_bcast_slot_calls[1]),
|
|
"repl_broadcast must skip the empty slot (slot 1) (umbrella 6.1)");
|
|
|
|
/* Assert: the in_catch_up standby was skipped. */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_bcast_slot_calls[2]),
|
|
"repl_broadcast must skip the in_catch_up standby (slot 2) "
|
|
"(umbrella 6.1)");
|
|
|
|
/* Assert: exactly one send total — no over-fan-out. */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_bcast_total_calls),
|
|
"repl_broadcast must enqueue exactly once across all slots in this "
|
|
"fixture (one active, one empty, one catch-up) (umbrella 6.1)");
|
|
|
|
cv_teardown();
|
|
printf("test_pin_repl_broadcast_full_fanout_skips_empty_and_catchup: checked\n");
|
|
}
|
|
|
|
/* (b) umbrella 6.3: ha_evaluate_repl_action returns HA_REPL_SEND_SNAPSHOT when
|
|
* standby_lsn < oldest_wal_lsn (the standby predates the oldest retained WAL, so
|
|
* streaming cannot reach it — a fresh snapshot is required). Decision boundary
|
|
* only, per the task: at/above oldest_wal_lsn the verdict is NOT snapshot.
|
|
* Pins existing ha.c:1274-1275. (The snapshot send path itself —
|
|
* ha_send_snapshot_to_standby, config_repl.c:1857 — takes a fresh checkpoint
|
|
* before sending; that send-internals behavior is out of scope for this
|
|
* decision-boundary pin.) */
|
|
static void test_pin_evaluate_repl_action_snapshot_below_oldest_wal(void)
|
|
{
|
|
/* fix-snapshot-boundary: the snapshot fallback fires iff the standby's
|
|
* NEXT-needed entry (standby_lsn + 1) is older than the oldest retained WAL
|
|
* entry — i.e. standby_lsn + 1 < oldest_wal_lsn (genuinely GC'd behind).
|
|
* tags equal (no config-push shortcut); committed_lsn high so the snapshot
|
|
* boundary is the only deciding factor. */
|
|
|
|
/* GENUINELY GC'd behind: standby_lsn=2, oldest_wal_lsn=5 → entries 3,4 were
|
|
* GC'd, the next-needed entry (3) is gone → SEND_SNAPSHOT.
|
|
* (2 + 1 = 3 < 5 → snapshot.) */
|
|
ha_repl_action_t gcd_behind = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/2,
|
|
/*committed_tag=*/5, /*committed_lsn=*/9, /*oldest_wal_lsn=*/5);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)HA_REPL_SEND_SNAPSHOT, (int)gcd_behind,
|
|
"standby_lsn (2) is GC'd behind oldest_wal_lsn (5) — its next-needed "
|
|
"entry (3) is no longer retained, so it must decide SEND_SNAPSHOT "
|
|
"(snapshot still fires when genuinely needed)");
|
|
|
|
/* BOUNDARY (streamable edge): standby_lsn == oldest_wal_lsn - 1, i.e. the
|
|
* next-needed entry is EXACTLY the oldest retained entry → STREAM, not
|
|
* snapshot. standby_lsn=4, oldest_wal_lsn=5 → 4 + 1 = 5 < 5 is false. This
|
|
* is the strict boundary: a wrong comparison (standby_lsn < oldest_wal_lsn)
|
|
* would (incorrectly) snapshot here, failing this assertion. */
|
|
ha_repl_action_t boundary = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/4,
|
|
/*committed_tag=*/5, /*committed_lsn=*/9, /*oldest_wal_lsn=*/5);
|
|
TEST_ASSERT_NOT_EQUAL_MESSAGE((int)HA_REPL_SEND_SNAPSHOT, (int)boundary,
|
|
"standby_lsn (4) == oldest_wal_lsn (5) - 1: the next-needed entry (5) IS "
|
|
"the oldest retained entry — streamable, must NOT decide SEND_SNAPSHOT "
|
|
"(strict standby_lsn + 1 < oldest_wal_lsn boundary)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)HA_REPL_STREAM_WAL, (int)boundary,
|
|
"at the snapshot boundary with lsn (4) below committed (9) the standby is "
|
|
"streamable — HA_REPL_STREAM_WAL");
|
|
|
|
/* EMPTY standby on fresh cluster: standby_lsn=0, oldest_wal_lsn=1 → next
|
|
* entry (1) retained → STREAM. 0 + 1 = 1 < 1 false. The core fix case. */
|
|
ha_repl_action_t empty_fresh = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/0,
|
|
/*committed_tag=*/5, /*committed_lsn=*/9, /*oldest_wal_lsn=*/1);
|
|
TEST_ASSERT_NOT_EQUAL_MESSAGE((int)HA_REPL_SEND_SNAPSHOT, (int)empty_fresh,
|
|
"empty standby (lsn=0), oldest_wal_lsn=1: next-needed entry (1) retained "
|
|
"→ must STREAM, not snapshot (strict boundary)");
|
|
|
|
/* Above the oldest WAL and at committed → CONVERGED, never snapshot. */
|
|
ha_repl_action_t converged = ha_evaluate_repl_action(
|
|
/*standby_tag=*/5, /*standby_lsn=*/9,
|
|
/*committed_tag=*/5, /*committed_lsn=*/9, /*oldest_wal_lsn=*/5);
|
|
TEST_ASSERT_NOT_EQUAL_MESSAGE((int)HA_REPL_SEND_SNAPSHOT, (int)converged,
|
|
"a fully caught-up standby must never decide SEND_SNAPSHOT "
|
|
"(umbrella 6.3 boundary)");
|
|
|
|
printf("test_pin_evaluate_repl_action_snapshot_below_oldest_wal: checked\n");
|
|
}
|
|
|
|
/* fix-snapshot-boundary: an EMPTY standby on a FRESH cluster must STREAM, not
|
|
* demand a snapshot.
|
|
*
|
|
* After reserve-lsn-0 the first WAL entry is lsn 1, so a fresh cluster's oldest
|
|
* retained entry is lsn 1. A freshly-joined empty standby reports applied_lsn=0.
|
|
* Its NEXT-needed entry is lsn 1, which IS retained — so streaming can serve it;
|
|
* no snapshot exists yet on a fresh cluster. The snapshot fallback must fire
|
|
* only when the standby's next-needed entry (standby_lsn + 1) has been GC'd below
|
|
* the oldest retained entry, i.e. standby_lsn + 1 < oldest_wal_lsn.
|
|
*
|
|
* RED before the fix: the old boundary (standby_lsn < oldest_wal_lsn) gives
|
|
* 0 < 1 → SEND_SNAPSHOT for the empty standby. GREEN after: 0+1=1 < 1 is
|
|
* false → STREAM_WAL streams lsn 1, the cold-start NOOP replicates + commits. */
|
|
static void test_empty_standby_fresh_cluster_streams_not_snapshot(void)
|
|
{
|
|
/* Empty standby (applied_lsn=0), fresh cluster (oldest_wal_lsn=1, NOOP at
|
|
* lsn 1), WAL tip > 0 (committed_lsn=1). Tags equal so the config-push
|
|
* shortcut does not mask the snapshot/stream decision. */
|
|
ha_repl_action_t a = ha_evaluate_repl_action(
|
|
/*standby_tag=*/0, /*standby_lsn=*/0,
|
|
/*committed_tag=*/0, /*committed_lsn=*/1, /*oldest_wal_lsn=*/1);
|
|
TEST_ASSERT_NOT_EQUAL_MESSAGE((int)HA_REPL_SEND_SNAPSHOT, (int)a,
|
|
"empty standby (lsn=0) on a fresh cluster (oldest_wal_lsn=1) needs lsn 1, "
|
|
"which is retained — it must NOT demand a snapshot (there is none on a "
|
|
"fresh cluster); the next-needed entry (lsn 1) is streamable");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)HA_REPL_STREAM_WAL, (int)a,
|
|
"empty standby (lsn=0 < committed_lsn=1) with next-needed entry retained "
|
|
"must STREAM_WAL so the cold-start NOOP (lsn 1) replicates and commits");
|
|
|
|
printf("test_empty_standby_fresh_cluster_streams_not_snapshot: checked\n");
|
|
}
|
|
|
|
/* ── Degraded-mode dispatch-gate fixture (task 6.1 (c)) ──────────────────────
|
|
* Self-contained ha_state_t with role=CTRL_PRIMARY and write quorum ABSENT
|
|
* (n_peers=3, zero connected standby endpoints → ha_has_write_quorum false via
|
|
* the standby_endpoints fallback count). Mirrors test_commit_barrier_rollout.c's
|
|
* fact_gate_setup. No persist/repl needed — the gate reads role + quorum only. */
|
|
static ha_state_t g_cvg_ha;
|
|
static int g_cvg_ep; /* dummy endpoint (zarray stores its address) */
|
|
|
|
static void cv_gate_setup(int n_peers, int n_endpoints)
|
|
{
|
|
ha_config_t cfg = cv_ha_cfg();
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&g_cvg_ha, &cfg, 0));
|
|
g_cvg_ha.n_peers = n_peers;
|
|
atomic_store_explicit(&g_cvg_ha.role, (int32_t)CTRL_PRIMARY,
|
|
memory_order_release);
|
|
g_cvg_ha.standby_endpoints = calloc(1, sizeof(zarray_t));
|
|
TEST_ASSERT_NOT_NULL(g_cvg_ha.standby_endpoints);
|
|
zarray_init(g_cvg_ha.standby_endpoints, 4, NULL);
|
|
for (int i = 0; i < n_endpoints; i++)
|
|
zarray_store(g_cvg_ha.standby_endpoints, &g_cvg_ep);
|
|
}
|
|
|
|
static void cv_gate_teardown(void)
|
|
{
|
|
if (g_cvg_ha.standby_endpoints) {
|
|
zarray_fini(g_cvg_ha.standby_endpoints);
|
|
free(g_cvg_ha.standby_endpoints);
|
|
g_cvg_ha.standby_endpoints = NULL;
|
|
}
|
|
ha_fini(&g_cvg_ha);
|
|
}
|
|
|
|
/* (c) umbrella 7.3: without write quorum, ha_dispatch_gate admits OPCode_get,
|
|
* heartbeat, the replication-opcode allowlist, and the fact-shaped opcodes
|
|
* (notify/update/setpreason/clrpreason — admitted per ha-commit-barrier-rollout's
|
|
* fact allowlist), while rejecting request-shaped opcodes
|
|
* (jobsub/jobcontrol/queuecontrol/hostcontrol/jobdispatch/join) with
|
|
* -XCE_ERR_NOT_READY. Pins existing ha.c:1516-1551. */
|
|
static void test_pin_dispatch_gate_degraded_allowlist(void)
|
|
{
|
|
/* Arrange: CTRL_PRIMARY, 3-peer cluster, zero standbys → quorum absent. */
|
|
cv_gate_setup(/*n_peers=*/3, /*n_endpoints=*/0);
|
|
|
|
/* ── ADMITTED (return 0) ───────────────────────────────────────────── */
|
|
static const struct { int opcode; const char *name; } admitted[] = {
|
|
/* read-only query + probe */
|
|
{ OPCode_get, "OPCode_get" },
|
|
{ OPCode_probeConfigTag, "OPCode_probeConfigTag" },
|
|
/* heartbeat */
|
|
{ OPCode_heartbeat, "OPCode_heartbeat" },
|
|
/* replication allowlist */
|
|
{ OPCode_replConfig, "OPCode_replConfig" },
|
|
{ OPCode_replConfigPush, "OPCode_replConfigPush" },
|
|
{ OPCode_upgradeConfigAck, "OPCode_upgradeConfigAck" },
|
|
{ OPCode_hello, "OPCode_hello" },
|
|
{ OPCode_walAck, "OPCode_walAck" },
|
|
{ OPCode_replEntry, "OPCode_replEntry" },
|
|
{ OPCode_replAck, "OPCode_replAck" },
|
|
{ OPCode_replCatchUpDone, "OPCode_replCatchUpDone" },
|
|
{ OPCode_walCatchupEntry, "OPCode_walCatchupEntry" },
|
|
{ OPCode_snapshotTransfer, "OPCode_snapshotTransfer" },
|
|
/* fact-shaped ops (admitted per the fact allowlist) */
|
|
{ OPCode_notify, "OPCode_notify" },
|
|
{ OPCode_update, "OPCode_update" },
|
|
{ OPCode_setpreason, "OPCode_setpreason" },
|
|
{ OPCode_clrpreason, "OPCode_clrpreason" },
|
|
};
|
|
for (size_t i = 0; i < sizeof(admitted) / sizeof(admitted[0]); i++) {
|
|
int rc = ha_dispatch_gate(&g_cvg_ha, admitted[i].opcode);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, admitted[i].name);
|
|
}
|
|
|
|
/* ── REJECTED with -XCE_ERR_NOT_READY (request-shaped ops only) ────── */
|
|
static const struct { int opcode; const char *name; } rejected[] = {
|
|
{ OPCode_jobsub, "OPCode_jobsub" },
|
|
{ OPCode_jobcontrol, "OPCode_jobcontrol" },
|
|
{ OPCode_queuecontrol, "OPCode_queuecontrol" },
|
|
{ OPCode_hostcontrol, "OPCode_hostcontrol" },
|
|
{ OPCode_jobdispatch, "OPCode_jobdispatch" },
|
|
{ OPCode_join, "OPCode_join" },
|
|
};
|
|
for (size_t i = 0; i < sizeof(rejected) / sizeof(rejected[0]); i++) {
|
|
int rc = ha_dispatch_gate(&g_cvg_ha, rejected[i].opcode);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(-XCE_ERR_NOT_READY, rc, rejected[i].name);
|
|
}
|
|
|
|
cv_gate_teardown();
|
|
printf("test_pin_dispatch_gate_degraded_allowlist: checked\n");
|
|
}
|
|
|
|
/* (d) umbrella 6.4 mechanism: repl_complete_catchup clears in_catch_up and the
|
|
* standby re-enters repl_broadcast iteration (i.e. it is sent to again).
|
|
* Complements task 4's ACK-driven *trigger* test by pinning the *mechanism*.
|
|
* Pins existing repl.c:602-628 + the broadcast re-inclusion at repl.c:265. */
|
|
static void test_pin_repl_complete_catchup_reincludes_in_broadcast(void)
|
|
{
|
|
cv_setup("reinc", /*n_peers=*/3, /*n_standbys=*/1,
|
|
/*appended_lsn=*/0, /*n_vcpus=*/8);
|
|
|
|
cv_bcast_reset_counters();
|
|
repl_broadcast_send_hook = cv_count_send_hook;
|
|
|
|
repl_ep_t ep = &g_cv_eps[0];
|
|
|
|
/* Arrange: put the only standby into catch-up → excluded from broadcast. */
|
|
repl_start_catchup(&g_cv.persist.repl, ep, /*join_applied_lsn=*/0);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, cv_standby_in_catch_up(ep),
|
|
"fixture precondition: the standby must be in_catch_up");
|
|
|
|
/* While in catch-up, a broadcast must NOT reach it. */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, cv_broadcast_one(&g_cv.persist.repl, 1),
|
|
"repl_broadcast must succeed");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_bcast_slot_calls[0]),
|
|
"an in_catch_up standby must be skipped by repl_broadcast (umbrella 6.4 "
|
|
"precondition)");
|
|
|
|
/* Act: complete catch-up → re-inclusion. */
|
|
repl_complete_catchup(&g_cv.persist.repl, ep);
|
|
|
|
/* Assert: in_catch_up cleared. */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, cv_standby_in_catch_up(ep),
|
|
"repl_complete_catchup must clear in_catch_up (umbrella 6.4)");
|
|
|
|
/* Assert: the standby re-enters broadcast iteration — a fresh broadcast now
|
|
* reaches it. */
|
|
cv_bcast_reset_counters();
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, cv_broadcast_one(&g_cv.persist.repl, 2),
|
|
"repl_broadcast must succeed");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_bcast_slot_calls[0]),
|
|
"after repl_complete_catchup the standby must re-enter repl_broadcast "
|
|
"iteration and receive the next entry (umbrella 6.4 re-inclusion)");
|
|
|
|
cv_teardown();
|
|
printf("test_pin_repl_complete_catchup_reincludes_in_broadcast: checked\n");
|
|
}
|
|
|
|
/* ── fix-promotion-standby-registration ─────────────────────────────────────
|
|
*
|
|
* Root cause pinned here: raft_cb_send_vote_request must set votes[i].peer
|
|
* (the candidate pointer) alongside votes[i].ep, so that the promotion path
|
|
* (raft_cb_become_leader) can call repl_add_standby for each surviving vote
|
|
* endpoint. Before the fix, votes[i].peer was NEVER set, so the promotion
|
|
* registration block (guarded by `votes[i].peer != NULL`) was skipped, the
|
|
* standbys were never added to c->persist.repl.standbys[], commit/quorum saw
|
|
* zero standbys, and every write failed with E_XCE_NOT_READY.
|
|
*
|
|
* These tests drive the REAL promotion path (raft_cb_become_leader via the
|
|
* with-cluster seam) over a real persist/repl/commit fixture, then assert the
|
|
* surviving vote endpoints are registered in the WAL replicator.
|
|
*
|
|
* The promotion tail is safe against this fixture:
|
|
* - ha_register_persist_hooks: documented safe on this cluster_t (ha.c).
|
|
* - ha_start_primary_heartbeat: no-ops because delaymgr is not started.
|
|
* - ha_arm_own_term_gate: uses the real persist + commit engine (broadcast
|
|
* drops through the stubbed repl_broadcast_send_hook).
|
|
*/
|
|
|
|
/* Build a vote endpoint entry on the election ctx: set BOTH .ep and .peer
|
|
* (the post-fix vote-send contract). Uses g_cv_eps[] for the ep identity (so
|
|
* repl_active_standby_count / broadcast slot-matching stay consistent with the
|
|
* rest of this suite) and a caller-owned xce_candidate_t for .peer. */
|
|
static void cv_set_vote(ha_elect_ctx_t *ctx, int idx, xce_candidate_t *peer,
|
|
const char *host, int port)
|
|
{
|
|
snprintf(peer->hostname, sizeof(peer->hostname), "%s", host);
|
|
peer->port = port;
|
|
ctx->votes[idx].ep = (struct endpoint_t *)&g_cv_eps[idx];
|
|
ctx->votes[idx].peer = peer;
|
|
}
|
|
|
|
/* (fix) Promotion with surviving vote endpoints carrying .peer registers every
|
|
* one of them in the WAL replicator → repl_active_standby_count == n.
|
|
*
|
|
* RED before the fix: with the production vote-send leaving votes[i].peer NULL,
|
|
* an election ctx built the SAME way the real path builds it would have
|
|
* .peer == NULL, the promotion block would skip repl_add_standby, and the count
|
|
* would be 0. This test sets .peer (simulating the fixed vote-send) and pins
|
|
* that the promotion path then registers all surviving standbys. */
|
|
static void test_promotion_registers_standbys_when_peer_set(void)
|
|
{
|
|
/* n_standbys=0: the replicator starts EMPTY; only the promotion path may
|
|
* populate it, so the count is attributable solely to registration. */
|
|
cv_setup("promreg", /*n_peers=*/2, /*n_standbys=*/0,
|
|
/*appended_lsn=*/0, /*n_vcpus=*/8);
|
|
|
|
/* Precondition: nothing registered yet. */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
repl_active_standby_count(&g_cv.persist.repl),
|
|
"fixture precondition: replicator must start empty (n_standbys=0)");
|
|
|
|
/* Build an election ctx with 2 surviving vote endpoints, each carrying a
|
|
* peer pointer (the post-fix vote-send contract). */
|
|
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + 2 * sizeof(ctx->votes[0]));
|
|
TEST_ASSERT_NOT_NULL(ctx);
|
|
ctx->n_peers = 2;
|
|
pthread_mutex_init(&ctx->lock, NULL);
|
|
sem_init(&ctx->done, 0, 0);
|
|
|
|
xce_candidate_t peer0 = {0}, peer1 = {0};
|
|
cv_set_vote(ctx, 0, &peer0, "sb0.example", 7700);
|
|
cv_set_vote(ctx, 1, &peer1, "sb1.example", 7701);
|
|
|
|
/* standby_endpoints array must exist (promotion stores eps there too). */
|
|
g_cv.ha.standby_endpoints = calloc(1, sizeof(zarray_t));
|
|
TEST_ASSERT_NOT_NULL(g_cv.ha.standby_endpoints);
|
|
zarray_init(g_cv.ha.standby_endpoints, 4, NULL);
|
|
|
|
atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx, memory_order_release);
|
|
|
|
/* Act: real promotion with a live cluster. */
|
|
raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/1);
|
|
|
|
/* Assert (Req 1.1): both surviving vote endpoints are now registered in the
|
|
* WAL replicator, so commit/quorum can advance committed_lsn. */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(2,
|
|
repl_active_standby_count(&g_cv.persist.repl),
|
|
"promotion must register every surviving vote endpoint (votes[i].peer "
|
|
"set) in the WAL replicator — repl_active_standby_count must equal the "
|
|
"number of vote endpoints (fix-promotion-standby-registration Req 1.1)");
|
|
|
|
/* Cleanup */
|
|
atomic_store_explicit(&g_cv.ha.elect_ctx, NULL, memory_order_release);
|
|
if (g_cv.ha.standby_endpoints) {
|
|
zarray_fini(g_cv.ha.standby_endpoints);
|
|
free(g_cv.ha.standby_endpoints);
|
|
g_cv.ha.standby_endpoints = NULL;
|
|
}
|
|
pthread_mutex_destroy(&ctx->lock);
|
|
sem_destroy(&ctx->done);
|
|
free(ctx);
|
|
cv_teardown();
|
|
printf("test_promotion_registers_standbys_when_peer_set: checked\n");
|
|
}
|
|
|
|
/* RED-pin: with votes[i].peer == NULL (the PRE-fix vote-send behavior), the
|
|
* promotion block must NOT register the endpoints — the count stays 0. This
|
|
* pins the exact failure the fix addresses: the registration is conditional on
|
|
* .peer being set, so leaving it NULL (as the buggy vote-send did) reproduces
|
|
* the zero-standby / no-quorum / NOT_READY condition. GREEN: the production
|
|
* vote-send now always sets .peer, so the real path takes the registered
|
|
* branch above. */
|
|
static void test_promotion_skips_registration_when_peer_null(void)
|
|
{
|
|
cv_setup("promnull", /*n_peers=*/2, /*n_standbys=*/0,
|
|
/*appended_lsn=*/0, /*n_vcpus=*/8);
|
|
|
|
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + 2 * sizeof(ctx->votes[0]));
|
|
TEST_ASSERT_NOT_NULL(ctx);
|
|
ctx->n_peers = 2;
|
|
pthread_mutex_init(&ctx->lock, NULL);
|
|
sem_init(&ctx->done, 0, 0);
|
|
|
|
/* eps present (election connected), but peer left NULL (the BUG). */
|
|
ctx->votes[0].ep = (struct endpoint_t *)&g_cv_eps[0];
|
|
ctx->votes[0].peer = NULL;
|
|
ctx->votes[1].ep = (struct endpoint_t *)&g_cv_eps[1];
|
|
ctx->votes[1].peer = NULL;
|
|
|
|
g_cv.ha.standby_endpoints = calloc(1, sizeof(zarray_t));
|
|
TEST_ASSERT_NOT_NULL(g_cv.ha.standby_endpoints);
|
|
zarray_init(g_cv.ha.standby_endpoints, 4, NULL);
|
|
|
|
atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx, memory_order_release);
|
|
|
|
raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/1);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
repl_active_standby_count(&g_cv.persist.repl),
|
|
"with votes[i].peer NULL (pre-fix vote-send), promotion must NOT register "
|
|
"standbys — this reproduces the zero-quorum / NOT_READY bug "
|
|
"(fix-promotion-standby-registration RED pin)");
|
|
|
|
atomic_store_explicit(&g_cv.ha.elect_ctx, NULL, memory_order_release);
|
|
if (g_cv.ha.standby_endpoints) {
|
|
zarray_fini(g_cv.ha.standby_endpoints);
|
|
free(g_cv.ha.standby_endpoints);
|
|
g_cv.ha.standby_endpoints = NULL;
|
|
}
|
|
pthread_mutex_destroy(&ctx->lock);
|
|
sem_destroy(&ctx->done);
|
|
free(ctx);
|
|
cv_teardown();
|
|
printf("test_promotion_skips_registration_when_peer_null: checked\n");
|
|
}
|
|
|
|
/* ── raft-apply-decoupling task 5.3 (round-3 remediation, Req 8.4) ──────────
|
|
*
|
|
* COVERAGE GAP closed here: round-2 added an `!apply_divergent` conjunct to
|
|
* BOTH the promotion_fence_timeout_fn re-alert timer (ha.c ~876) AND the
|
|
* raft_cb_become_leader pre-promotion eligibility gate (ha.c ~1130,
|
|
* `gate == RECOVER_DECISION_ELIGIBLE && !unrecoverable && !apply_divergent`).
|
|
* The round-2 reviewer proved the SECOND site had zero test coverage: reverting
|
|
* ONLY that conjunct left the entire suite (298 tests across 13 groups,
|
|
* including every test that drives raft_cb_become_leader /
|
|
* raft_cb_become_leader_with_cluster_for_test) fully green. This test targets
|
|
* exactly the scenario that conjunct alone must catch: a node that is
|
|
* structurally ELIGIBLE (ha_promotion_gate_decision returns
|
|
* RECOVER_DECISION_ELIGIBLE — not truncated, durable_floor >= the cluster
|
|
* commit high-water mark) and NOT sticky-fenced (persist_recovery_unrecoverable
|
|
* is explicitly false, i.e. the OTHER guard has nothing to contribute), whose
|
|
* apply_verdict is nonetheless APPLY_DIVERGENT — a local materialized-state
|
|
* defect that only a fresh, clean persist_recover() boot replay may clear
|
|
* (task 5.1/5.2), never a peer/HWM condition.
|
|
*
|
|
* Fixture: identical to test_promotion_registers_standbys_when_peer_set above
|
|
* (n_peers=2, n_standbys=0, appended_lsn=0) — that sibling test's own
|
|
* assertion (repl_active_standby_count == 2 after promotion) is only
|
|
* reachable if this exact fixture already drives ha_promotion_gate_decision to
|
|
* RECOVER_DECISION_ELIGIBLE with unrecoverable == false, so reusing it here
|
|
* isolates the apply_divergent conjunct as the ONLY variable changed.
|
|
*
|
|
* RED (site-2 guard reverted): the gate promotes anyway — role flips to
|
|
* CTRL_PRIMARY and the vote endpoints get registered as standbys.
|
|
* GREEN (current code): promotion is refused — role stays non-PRIMARY, the
|
|
* member is fenced (in_catch_up=1, never-self-exit — it stays alive and keeps
|
|
* retrying catch-up), and no standbys are registered.
|
|
*/
|
|
static void test_promotion_refused_when_apply_divergent_even_if_eligible(void)
|
|
{
|
|
/* n_standbys=0, appended_lsn=0: same structurally-healthy fixture as
|
|
* test_promotion_registers_standbys_when_peer_set (durable_floor=0,
|
|
* recovery_truncated=false, and — with vote_granted left at its
|
|
* calloc-zero default — ha_compute_safe_committed_lsn's only counted
|
|
* voter is self at self_lsn=0, so promotion_safe_lsn=0). durable_floor(0)
|
|
* >= promotion_safe_lsn(0) and not truncated ⇒ genuinely ELIGIBLE. */
|
|
cv_setup("applydiv", /*n_peers=*/2, /*n_standbys=*/0,
|
|
/*appended_lsn=*/0, /*n_vcpus=*/8);
|
|
|
|
/* Precondition (Req 8.4 scenario, explicit per task spec): NOT fenced by
|
|
* the sticky flag — isolates the apply_divergent conjunct from the
|
|
* already-covered `unrecoverable` conjunct. */
|
|
TEST_ASSERT_FALSE_MESSAGE(
|
|
persist_recovery_unrecoverable(&g_cv.persist),
|
|
"fixture precondition: sticky recovery_unrecoverable must be clear so "
|
|
"this test isolates the apply_divergent guard, not the sticky-flag "
|
|
"guard (Req 8.4)");
|
|
|
|
/* Force the apply-divergent verdict directly — exactly as task 5.1's
|
|
* persist_recover() replay leaves it after detecting a materialized-state
|
|
* mismatch, and exactly as
|
|
* test_promotion_fence_apply_divergent_survives_structural_catchup
|
|
* (test_commit_durability.c) sets it for the sibling (site-1) call site. */
|
|
g_cv.persist.apply_verdict = APPLY_DIVERGENT;
|
|
|
|
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + 2 * sizeof(ctx->votes[0]));
|
|
TEST_ASSERT_NOT_NULL(ctx);
|
|
ctx->n_peers = 2;
|
|
pthread_mutex_init(&ctx->lock, NULL);
|
|
sem_init(&ctx->done, 0, 0);
|
|
|
|
xce_candidate_t peer0 = {0}, peer1 = {0};
|
|
cv_set_vote(ctx, 0, &peer0, "sb0.example", 7700);
|
|
cv_set_vote(ctx, 1, &peer1, "sb1.example", 7701);
|
|
|
|
g_cv.ha.standby_endpoints = calloc(1, sizeof(zarray_t));
|
|
TEST_ASSERT_NOT_NULL(g_cv.ha.standby_endpoints);
|
|
zarray_init(g_cv.ha.standby_endpoints, 4, NULL);
|
|
|
|
atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx, memory_order_release);
|
|
|
|
/* Act: real promotion attempt with a live, structurally-healthy cluster. */
|
|
raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/1);
|
|
|
|
/* Assert (Req 8.4): refused despite genuine structural ELIGIBLE +
|
|
* unrecoverable==false — the apply_divergent conjunct alone must block
|
|
* promotion. */
|
|
TEST_ASSERT_NOT_EQUAL_MESSAGE(
|
|
(int32_t)CTRL_PRIMARY,
|
|
atomic_load_explicit(&g_cv.ha.role, memory_order_acquire),
|
|
"raft-apply-decoupling task 5.3 (round-3 coverage, Req 8.4): a "
|
|
"structurally-eligible, non-sticky-fenced node whose apply_verdict is "
|
|
"APPLY_DIVERGENT must NOT be promoted to CTRL_PRIMARY");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
|
1, atomic_load_explicit(&g_cv.ha.in_catch_up, memory_order_acquire),
|
|
"a refused apply-divergent promotion must fence the member "
|
|
"(in_catch_up=1) and keep it alive, retrying catch-up — never "
|
|
"self-exit (Req 8.4)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
repl_active_standby_count(&g_cv.persist.repl),
|
|
"promotion refusal must happen BEFORE the standby-registration block "
|
|
"— no vote endpoint may be registered when the node never became "
|
|
"primary");
|
|
|
|
/* Cleanup */
|
|
atomic_store_explicit(&g_cv.ha.elect_ctx, NULL, memory_order_release);
|
|
if (g_cv.ha.standby_endpoints) {
|
|
zarray_fini(g_cv.ha.standby_endpoints);
|
|
free(g_cv.ha.standby_endpoints);
|
|
g_cv.ha.standby_endpoints = NULL;
|
|
}
|
|
pthread_mutex_destroy(&ctx->lock);
|
|
sem_destroy(&ctx->done);
|
|
free(ctx);
|
|
cv_teardown();
|
|
printf("test_promotion_refused_when_apply_divergent_even_if_eligible: "
|
|
"checked\n");
|
|
}
|
|
|
|
/* ── job-array-single-wal-record task 4.3 (Req 9.4/9.5) ──────────────────────
|
|
* Site 2 (election-time check, raft_cb_become_leader ~ha.c:1344-1345): the
|
|
* Live Apply Resource Fence (task 4.2's persist_apply_resource_fenced /
|
|
* persist_mark_apply_resource_fenced) must gate this eligibility check the
|
|
* same way apply_divergent/self_restart_pending already do.
|
|
*
|
|
* Mirrors test_promotion_refused_when_apply_divergent_even_if_eligible above
|
|
* exactly — same structurally-healthy fixture (durable_floor(0) >=
|
|
* promotion_safe_lsn(0), not truncated => genuinely ha_promotion_gate_decision
|
|
* == ELIGIBLE, and persist_recovery_unrecoverable explicitly false so the
|
|
* sticky-flag conjunct contributes nothing) — swapping the trigger for the new
|
|
* fence primitive so the new conjunct is isolated as the ONLY variable
|
|
* changed, exactly as the round-3 apply_divergent test isolated its conjunct.
|
|
*
|
|
* RED (pre-task-4.3 code): the gate has no !persist_apply_resource_fenced
|
|
* conjunct, so a structurally-eligible node with the live fence engaged is
|
|
* promoted anyway — role flips to CTRL_PRIMARY and vote endpoints get
|
|
* registered as standbys. GREEN (post-fix): promotion is refused — role stays
|
|
* non-PRIMARY, the member is fenced (in_catch_up=1, never-self-exit), and no
|
|
* standbys are registered. */
|
|
static void test_promotion_refused_when_apply_resource_fenced_even_if_eligible(void)
|
|
{
|
|
/* Identical fixture to test_promotion_refused_when_apply_divergent_even_if_eligible:
|
|
* durable_floor=0, recovery_truncated=false, promotion_safe_lsn=0 =>
|
|
* genuinely ELIGIBLE on every OTHER conjunct. */
|
|
cv_setup("resfenced", /*n_peers=*/2, /*n_standbys=*/0,
|
|
/*appended_lsn=*/0, /*n_vcpus=*/8);
|
|
|
|
TEST_ASSERT_FALSE_MESSAGE(
|
|
persist_recovery_unrecoverable(&g_cv.persist),
|
|
"fixture precondition: sticky recovery_unrecoverable must be clear so "
|
|
"this test isolates the apply-resource-fence guard, not the "
|
|
"sticky-flag guard");
|
|
TEST_ASSERT_FALSE_MESSAGE(
|
|
g_cv.persist.apply_verdict == APPLY_DIVERGENT,
|
|
"fixture precondition: apply_verdict must not be APPLY_DIVERGENT so "
|
|
"this test isolates the apply-resource-fence guard from the sibling "
|
|
"apply_divergent guard");
|
|
|
|
/* Engage the Live Apply Resource Fence directly -- exactly as
|
|
* persist_apply_pump_run does on a resource-exhaustion apply failure
|
|
* (task 4.2), and exactly as the sibling site-1 test
|
|
* (test_commit_durability.c) sets it for that call site. */
|
|
persist_mark_apply_resource_fenced(&g_cv.persist, /*lsn=*/3);
|
|
TEST_ASSERT_TRUE(persist_apply_resource_fenced(&g_cv.persist));
|
|
|
|
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + 2 * sizeof(ctx->votes[0]));
|
|
TEST_ASSERT_NOT_NULL(ctx);
|
|
ctx->n_peers = 2;
|
|
pthread_mutex_init(&ctx->lock, NULL);
|
|
sem_init(&ctx->done, 0, 0);
|
|
|
|
xce_candidate_t peer0 = {0}, peer1 = {0};
|
|
cv_set_vote(ctx, 0, &peer0, "sb0.example", 7700);
|
|
cv_set_vote(ctx, 1, &peer1, "sb1.example", 7701);
|
|
|
|
g_cv.ha.standby_endpoints = calloc(1, sizeof(zarray_t));
|
|
TEST_ASSERT_NOT_NULL(g_cv.ha.standby_endpoints);
|
|
zarray_init(g_cv.ha.standby_endpoints, 4, NULL);
|
|
|
|
atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx, memory_order_release);
|
|
|
|
/* Act: real promotion attempt with a live, structurally-healthy cluster. */
|
|
raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/1);
|
|
|
|
/* Assert (Req 9.4/9.5): refused despite genuine structural ELIGIBLE +
|
|
* unrecoverable==false + apply_verdict!=DIVERGENT -- the new
|
|
* apply-resource-fence conjunct alone must block promotion. */
|
|
TEST_ASSERT_NOT_EQUAL_MESSAGE(
|
|
(int32_t)CTRL_PRIMARY,
|
|
atomic_load_explicit(&g_cv.ha.role, memory_order_acquire),
|
|
"job-array-single-wal-record task 4.3 (Req 9.4/9.5): a "
|
|
"structurally-eligible node with the Live Apply Resource Fence "
|
|
"engaged must NOT be promoted to CTRL_PRIMARY");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
|
1, atomic_load_explicit(&g_cv.ha.in_catch_up, memory_order_acquire),
|
|
"a refused apply-resource-fenced promotion must fence the member "
|
|
"(in_catch_up=1) and keep it alive, retrying catch-up -- never "
|
|
"self-exit");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
|
repl_active_standby_count(&g_cv.persist.repl),
|
|
"promotion refusal must happen BEFORE the standby-registration block "
|
|
"-- no vote endpoint may be registered when the node never became "
|
|
"primary");
|
|
|
|
/* Now clear the fence (the apply pump's own self-heal, task 4.2) and
|
|
* retry promotion: eligibility must return with no other action (task
|
|
* 4.3's Observable clause). */
|
|
persist_clear_apply_resource_fence(&g_cv.persist);
|
|
TEST_ASSERT_FALSE(persist_apply_resource_fenced(&g_cv.persist));
|
|
atomic_store_explicit(&g_cv.ha.in_catch_up, 0, memory_order_release);
|
|
|
|
xce_candidate_t peer0b = {0}, peer1b = {0};
|
|
ha_elect_ctx_t *ctx2 = calloc(1, sizeof(*ctx2) + 2 * sizeof(ctx2->votes[0]));
|
|
TEST_ASSERT_NOT_NULL(ctx2);
|
|
ctx2->n_peers = 2;
|
|
pthread_mutex_init(&ctx2->lock, NULL);
|
|
sem_init(&ctx2->done, 0, 0);
|
|
cv_set_vote(ctx2, 0, &peer0b, "sb0.example", 7700);
|
|
cv_set_vote(ctx2, 1, &peer1b, "sb1.example", 7701);
|
|
atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx2, memory_order_release);
|
|
|
|
raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/2);
|
|
|
|
TEST_ASSERT_EQUAL_MESSAGE(
|
|
(int32_t)CTRL_PRIMARY,
|
|
atomic_load_explicit(&g_cv.ha.role, memory_order_acquire),
|
|
"task 4.3: once the live apply resource fence clears, the node is "
|
|
"eligible for promotion again with no other action");
|
|
|
|
/* Cleanup */
|
|
atomic_store_explicit(&g_cv.ha.elect_ctx, NULL, memory_order_release);
|
|
if (g_cv.ha.standby_endpoints) {
|
|
zarray_fini(g_cv.ha.standby_endpoints);
|
|
free(g_cv.ha.standby_endpoints);
|
|
g_cv.ha.standby_endpoints = NULL;
|
|
}
|
|
pthread_mutex_destroy(&ctx->lock);
|
|
sem_destroy(&ctx->done);
|
|
free(ctx);
|
|
pthread_mutex_destroy(&ctx2->lock);
|
|
sem_destroy(&ctx2->done);
|
|
free(ctx2);
|
|
cv_teardown();
|
|
printf("test_promotion_refused_when_apply_resource_fenced_even_if_eligible: "
|
|
"checked\n");
|
|
}
|
|
|
|
#endif /* XCE_TESTING */
|
|
|
|
/* ── Suite entry point ──────────────────────────────────────────────────── */
|
|
|
|
void test_convergence_verification(void)
|
|
{
|
|
UnitySetTestFile(__FILE__);
|
|
#ifdef XCE_TESTING
|
|
printf("\n>> ACK-driven catch-up continuation tests (RED, task 4.1)\n\n");
|
|
|
|
RUN_TEST(test_walack_behind_target_schedules_one_continuation);
|
|
RUN_TEST(test_walack_single_flight_no_duplicate_continuation);
|
|
RUN_TEST(test_walack_reschedules_after_continuation_completes);
|
|
RUN_TEST(test_walack_at_parity_completes_catchup_and_clears_in_catch_up);
|
|
RUN_TEST(test_heartbeat_decision_path_unchanged);
|
|
|
|
printf("\n>> Quorum transition logging tests (RED, task 5.1)\n\n");
|
|
|
|
RUN_TEST(test_quorum_loss_logged_once_with_counts);
|
|
RUN_TEST(test_quorum_loss_no_repeat_log_while_lost);
|
|
RUN_TEST(test_quorum_restore_logged_once);
|
|
RUN_TEST(test_quorum_second_cycle_logs_again);
|
|
|
|
printf("\n>> Regression pins: fan-out / snapshot fallback / degraded gate / "
|
|
"re-inclusion (task 6.1 — PASS against current code)\n\n");
|
|
|
|
RUN_TEST(test_pin_repl_broadcast_full_fanout_skips_empty_and_catchup);
|
|
RUN_TEST(test_pin_evaluate_repl_action_snapshot_below_oldest_wal);
|
|
RUN_TEST(test_empty_standby_fresh_cluster_streams_not_snapshot);
|
|
RUN_TEST(test_pin_dispatch_gate_degraded_allowlist);
|
|
RUN_TEST(test_pin_repl_complete_catchup_reincludes_in_broadcast);
|
|
|
|
printf("\n>> Promotion standby registration "
|
|
"(fix-promotion-standby-registration: votes[].peer → repl_add_standby)\n\n");
|
|
|
|
RUN_TEST(test_promotion_registers_standbys_when_peer_set);
|
|
RUN_TEST(test_promotion_skips_registration_when_peer_null);
|
|
|
|
printf("\n>> Promotion eligibility gate site-2 coverage "
|
|
"(raft-apply-decoupling task 5.3 round-3, Req 8.4)\n\n");
|
|
|
|
RUN_TEST(test_promotion_refused_when_apply_divergent_even_if_eligible);
|
|
|
|
printf("\n>> Promotion eligibility gate site-2 coverage "
|
|
"(job-array-single-wal-record task 4.3, Req 9.4/9.5)\n\n");
|
|
|
|
RUN_TEST(test_promotion_refused_when_apply_resource_fenced_even_if_eligible);
|
|
#else
|
|
printf("test_convergence_verification: SKIP (non-test build)\n");
|
|
#endif
|
|
}
|