lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

test(ha-convergence-recovery): RED quorum transition logging tests + seam (task 5.1)

Browse source
This commit is contained in:
Rongsong Shen 2026-06-14 06:09:01 +00:00
parent ffd08c6472
commit 3218d9ffaf
4 changed files with 321 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

233
src/controller/testsuite/test_convergence_verification.c
View file

@ -60,7 +60,8 @@
* ha_repl_stream_wal_hook, HA_REPL_* */
#include "ordq.h" /* ordq_item_t (drive the scheduled task) */
#include "persist.h" /* persist_t */
#include "repl.h" /* repl_add_standby / repl_set_commit_engine / standby slots */
#include "repl.h" /* repl_add_standby / repl_remove_standby /
* repl_active_standby_count / standby slots */
#include "xce/cluster_def.h" /* full cluster_t layout */
#ifdef XCE_TESTING
@ -463,6 +464,229 @@ static void test_heartbeat_decision_path_unchanged(void)
printf("test_heartbeat_decision_path_unchanged: checked\n");
}
/* ── Quorum transition logging (task 5.1, Req 8.1-8.3) ───────────────────────
*
* Observable pattern: there is NO log-capture harness in this testsuite, so
* these tests use the emission-count seam ha_quorum_transition_log_hook (the
* task permits "log-capture OR counter-hook pattern"). The real
* ha_log_quorum_transition (task 5.2) invokes the hook exactly once on each
* loss/restore EDGE it WARNs about, passing the new state and the
* connected/required counts that go into the WARN line. We count invocations to
* assert "logged once per transition" and inspect the args to assert the loss
* line carries the right counts.
*
* RED status against the current tree: ha_log_quorum_transition is a no-op RED
* scaffold (ha_convergence.c) — it performs no edge detection, flips no flag,
* and never invokes the hook. Every assertion below that expects an emission
* (or an edge-flag flip) therefore FAILS until task 5.2 lands the real edge
* detector and wires the call sites.
*
* Quorum arithmetic for the fixture (ha_has_write_quorum, ha.c):
* n_peers=3, ha->repl set → connected = repl_active_standby_count(repl);
* quorum held iff connected >= n_peers/2 = 1. So 1 active standby = quorum,
* 0 active standbys = quorum lost. The required count in the WARN line is the
* threshold n_peers/2 = 1.
*/
/* Emission-count hook: records every transition WARN ha_log_quorum_transition
* emits, capturing the latest (lost, connected, required) triple. */
static atomic_int g_qt_calls;
static atomic_int g_qt_loss_calls;
static atomic_int g_qt_restore_calls;
static atomic_int g_qt_last_lost;
static atomic_int g_qt_last_connected;
static atomic_int g_qt_last_required;
static void cv_quorum_log_hook(int lost, int connected, int required)
{
atomic_fetch_add(&g_qt_calls, 1);
if (lost)
atomic_fetch_add(&g_qt_loss_calls, 1);
else
atomic_fetch_add(&g_qt_restore_calls, 1);
atomic_store(&g_qt_last_lost, lost);
atomic_store(&g_qt_last_connected, connected);
atomic_store(&g_qt_last_required, required);
}
static void cv_quorum_reset_counters(void)
{
atomic_store(&g_qt_calls, 0);
atomic_store(&g_qt_loss_calls, 0);
atomic_store(&g_qt_restore_calls, 0);
atomic_store(&g_qt_last_lost, -1);
atomic_store(&g_qt_last_connected, -1);
atomic_store(&g_qt_last_required, -1);
}
/* Build a quorum fixture: n_peers=3, registry-truth quorum accounting wired
* (ha.repl = &persist.repl) and exactly one active standby so the cluster STARTS
* with write quorum held (quorum_lost == 0). Installs the emission-count hook. */
static void cv_quorum_setup(const char *suffix)
{
cv_setup(suffix, /*n_peers=*/3, /*n_standbys=*/1,
/*appended_lsn=*/0, /*n_vcpus=*/8);
/* Wire registry-truth quorum accounting (ha-commit-engine path): with
* ha.repl set, ha_has_write_quorum counts repl_active_standby_count. */
g_cv.ha.repl = &g_cv.persist.repl;
g_cv.ha.role = CTRL_PRIMARY; /* quorum gate is a CTRL_PRIMARY concern */
cv_quorum_reset_counters();
ha_quorum_transition_log_hook = cv_quorum_log_hook;
/* Fixture precondition: one active standby → connected=1 >= required=1 →
* quorum held at the start of every quorum test. */
TEST_ASSERT_EQUAL_INT_MESSAGE(1,
repl_active_standby_count(&g_cv.persist.repl),
"quorum fixture precondition: exactly one active standby (quorum held)");
}
static void cv_quorum_teardown(void)
{
ha_quorum_transition_log_hook = NULL;
cv_teardown();
}
/* 8.1: dropping the only standby below majority logs the loss WARN EXACTLY ONCE,
* carrying the connected/required counts; the edge flag flips to lost.
* RED: the no-op stub never detects the edge → 0 emissions, flag stays 0. */
static void test_quorum_loss_logged_once_with_counts(void)
{
cv_quorum_setup("qloss");
/* Act: remove the only standby → connected drops 1→0 → quorum lost. */
repl_remove_standby(&g_cv.persist.repl, &g_cv_eps[0]);
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
repl_active_standby_count(&g_cv.persist.repl),
"fixture: the cluster must have zero active standbys after removal");
ha_log_quorum_transition(&g_cv.ha);
/* Assert (Req 8.1): exactly one loss WARN, no restore WARN. */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_loss_calls),
"dropping below majority must log the quorum-loss WARN exactly once "
"(Req 8.1)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_qt_restore_calls),
"a quorum-loss edge must not emit a restore WARN (Req 8.1)");
/* Assert (Req 8.1): the WARN carries the connected/required counts. */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_qt_last_connected),
"the quorum-loss WARN must report connected=0 (Req 8.1)");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_last_required),
"the quorum-loss WARN must report required=1 (n_peers/2 threshold) "
"(Req 8.1)");
/* The edge-detector flag must now read 'lost'. */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_cv.ha.quorum_lost),
"a quorum-loss edge must set the ha_state_t.quorum_lost flag (Req 8.1)");
cv_quorum_teardown();
printf("test_quorum_loss_logged_once_with_counts: checked\n");
}
/* 8.3: while quorum stays lost, repeated dispatch-gate rejections (modeled as
* repeated ha_log_quorum_transition calls — the gate backstop calls it on every
* rejected op) add NO further transition logs.
* RED: the no-op stub never logs the first loss either, so the "exactly one loss
* across many calls" assertion fails (0 != 1). */
static void test_quorum_loss_no_repeat_log_while_lost(void)
{
cv_quorum_setup("qstorm");
/* Cross the loss edge once. */
repl_remove_standby(&g_cv.persist.repl, &g_cv_eps[0]);
ha_log_quorum_transition(&g_cv.ha);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_loss_calls),
"setup: the first call after the loss edge must log once (Req 8.3 setup)");
/* Act: simulate many rejected operations re-invoking the backstop while
* quorum stays lost (no membership change between calls). */
for (int i = 0; i < 5; i++)
ha_log_quorum_transition(&g_cv.ha);
/* Assert (Req 8.3): still exactly one loss WARN — no log storm. */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_loss_calls),
"repeated quorum-transition calls while quorum stays lost must NOT log "
"again — once per transition, no storms (Req 8.3)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_qt_restore_calls),
"no restore WARN may appear while quorum remains lost (Req 8.3)");
cv_quorum_teardown();
printf("test_quorum_loss_no_repeat_log_while_lost: checked\n");
}
/* 8.2: restoring a standby above majority logs the restore WARN ONCE; the edge
* flag flips back to held.
* RED: the no-op stub neither sets nor clears the flag and never emits. */
static void test_quorum_restore_logged_once(void)
{
cv_quorum_setup("qrestore");
/* Drive into the lost state first. */
repl_remove_standby(&g_cv.persist.repl, &g_cv_eps[0]);
ha_log_quorum_transition(&g_cv.ha);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_loss_calls),
"setup: must be in the lost state with one loss WARN (Req 8.2 setup)");
/* Act: re-register a standby → connected 0→1 → quorum restored. */
TEST_ASSERT_TRUE(
repl_add_standby(&g_cv.persist.repl, &g_cv_eps[0], "cvsb0", 7600) >= 0);
TEST_ASSERT_EQUAL_INT_MESSAGE(1,
repl_active_standby_count(&g_cv.persist.repl),
"fixture: the re-registered standby must restore the active count to 1");
ha_log_quorum_transition(&g_cv.ha);
/* Assert (Req 8.2): exactly one restore WARN, loss count unchanged. */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_restore_calls),
"restoring quorum must log the quorum-restore WARN exactly once "
"(Req 8.2)");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_loss_calls),
"the restore edge must not re-emit a loss WARN (Req 8.2)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_cv.ha.quorum_lost),
"a quorum-restore edge must clear the ha_state_t.quorum_lost flag "
"(Req 8.2)");
cv_quorum_teardown();
printf("test_quorum_restore_logged_once: checked\n");
}
/* 8.1+8.2 edge detector (not a latch): a SECOND loss/restore cycle logs again.
* RED: the no-op stub never logs any cycle. */
static void test_quorum_second_cycle_logs_again(void)
{
cv_quorum_setup("qcycle");
/* Cycle 1: lose then restore. */
repl_remove_standby(&g_cv.persist.repl, &g_cv_eps[0]);
ha_log_quorum_transition(&g_cv.ha);
TEST_ASSERT_TRUE(
repl_add_standby(&g_cv.persist.repl, &g_cv_eps[0], "cvsb0", 7600) >= 0);
ha_log_quorum_transition(&g_cv.ha);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_loss_calls),
"cycle 1: exactly one loss WARN (edge-detector setup)");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_restore_calls),
"cycle 1: exactly one restore WARN (edge-detector setup)");
/* Cycle 2: lose then restore AGAIN — the detector is not a one-shot latch. */
repl_remove_standby(&g_cv.persist.repl, &g_cv_eps[0]);
ha_log_quorum_transition(&g_cv.ha);
TEST_ASSERT_TRUE(
repl_add_standby(&g_cv.persist.repl, &g_cv_eps[0], "cvsb0", 7600) >= 0);
ha_log_quorum_transition(&g_cv.ha);
/* Assert: each transition logged again (totals = 2 loss + 2 restore). */
TEST_ASSERT_EQUAL_INT_MESSAGE(2, atomic_load(&g_qt_loss_calls),
"a SECOND quorum-loss edge must log again — the transition logger is an "
"edge detector, not a one-shot latch (Req 8.1)");
TEST_ASSERT_EQUAL_INT_MESSAGE(2, atomic_load(&g_qt_restore_calls),
"a SECOND quorum-restore edge must log again (Req 8.2)");
cv_quorum_teardown();
printf("test_quorum_second_cycle_logs_again: checked\n");
}
#endif /* XCE_TESTING */
/* ── Suite entry point ──────────────────────────────────────────────────── */
@ -477,6 +701,13 @@ void test_convergence_verification(void)
RUN_TEST(test_walack_reschedules_after_continuation_completes);
RUN_TEST(test_walack_at_parity_completes_catchup_and_clears_in_catch_up);
RUN_TEST(test_heartbeat_decision_path_unchanged);
printf("\n>> Quorum transition logging tests (RED, task 5.1)\n\n");
RUN_TEST(test_quorum_loss_logged_once_with_counts);
RUN_TEST(test_quorum_loss_no_repeat_log_while_lost);
RUN_TEST(test_quorum_restore_logged_once);
RUN_TEST(test_quorum_second_cycle_logs_again);
#else
printf("test_convergence_verification: SKIP (non-test build)\n");
#endif