lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

test(ha-commit-engine): membership re-evaluation regression tests (tasks 3.1+3.2 joint)

Browse source 
Tests 17-20: catch-up ACK exclusion vs pending waiter, membership-change
wake of covered waiter, removal-below-majority fails at own deadline
(no forced early-fail), non-blocking bound. Landed GREEN: task 1.2's
commit_on_membership_change already implements the full 3.2 contract,
so RED was unachievable; reviewer confirmed mutation-killing power and
zero production changes. 20/20 group tests; suite at baseline.
This commit is contained in:
Rongsong Shen 2026-06-11 15:48:42 +00:00
parent 3bc6fd14de
commit 631a08939b
4 changed files with 167 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

157
src/controller/testsuite/test_commit_engine.c
View file

@ -30,7 +30,15 @@
* registry + commit_on_ack. All threads are joined BEFORE any assertion so
* Unity's longjmp never strands a live thread (ASAN/LSan clean).
*
* Requirements: 1.1, 1.2, 2.1, 2.2, 2.3, 4.3, 4.4, 5.1, 6.3, 7.1, 7.2, 7.3
* Task 3.1 extends the file with membership re-evaluation cases
* (commit_on_membership_change): catch-up exclusion against a pending
* waiter, catch-up completion waking a pending waiter, standby removal
* below majority dying at the waiter's own deadline (no forced early-fail —
* design §Commit Engine), and the non-blocking contract of
* commit_on_membership_change itself.
*
* Requirements: 1.1, 1.2, 2.1, 2.2, 2.3, 4.1, 4.2, 4.3, 4.4, 5.1, 6.3,
* 7.1, 7.2, 7.3
*/
#include <errno.h>
#include <pthread.h>
@ -634,6 +642,148 @@ static void test_wait_gate_never_opening_times_out_not_notready(void)
printf("test_wait_gate_never_opening_times_out_not_notready: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* Task 3.1 — membership re-evaluation (commit_on_membership_change).
* (Req 4.1, 4.2; design §Commit Engine: re-evaluate under the engine mutex
* against current registry state + broadcast; no forced early-fail.)
* ════════════════════════════════════════════════════════════════════════ */
/* ── Test 17: catch-up standby's watermark never counts for a pending entry
* A pending WAIT must not complete off an ACK from a standby still in
* catch-up — it dies at its own deadline with committed_lsn unmoved.
* (Req 4.1) */
static void test_catchup_standby_ack_does_not_count_toward_pending_entry(void)
{
/* Arrange: 3-member cluster, entry 1 appended; the ONLY ACKing standby
* is in catch-up */
engine_setup(3, 2, 1);
atomic_store(&g_p.repl.standbys[0].in_catch_up, 1);
spawn_waiters(1, 1, 250);
/* Act: the catch-up standby's watermark covers the entry */
sleep_ms(50);
standby_ack(0, 1);
join_waiters(1);
/* Assert: never success — catch-up ACKs are excluded from majority */
TEST_ASSERT_TRUE_MESSAGE(g_wargs[0].rc != 0,
"catch-up standby ACK must not commit a pending entry");
TEST_ASSERT_EQUAL_INT(-ETIMEDOUT, g_wargs[0].rc);
TEST_ASSERT_TRUE_MESSAGE(g_wargs[0].elapsed_ms >= 200,
"WAIT must hold to its deadline, not early-complete");
TEST_ASSERT_EQUAL_UINT64(0, committed());
engine_teardown();
printf("test_catchup_standby_ack_does_not_count_toward_pending_entry: PASS\n");
}
/* ── Test 18: clearing in_catch_up + membership change completes the waiter
* The standby's watermark already covers the entry; once it leaves
* catch-up, commit_on_membership_change must re-evaluate against current
* registry state and wake the pending waiter to success. (Req 4.2) */
static void test_catchup_clear_then_membership_change_completes_waiter(void)
{
int done_while_in_catchup;
/* Arrange: 3-member cluster, entry 1 appended; standby 0 in catch-up
* with a watermark that covers the entry (excluded while catching up) */
engine_setup(3, 2, 1);
atomic_store(&g_p.repl.standbys[0].in_catch_up, 1);
standby_ack(0, 1);
TEST_ASSERT_EQUAL_UINT64(0, committed());
spawn_waiters(1, 1, 2000);
/* Act: waiter stays blocked while the standby is in catch-up... */
sleep_ms(120);
done_while_in_catchup = atomic_load(&g_waiters_done);
/* ...then catch-up completes: clear the flag and signal membership */
atomic_store(&g_p.repl.standbys[0].in_catch_up, 0);
commit_on_membership_change(&g_ce);
join_waiters(1);
/* Assert (after join): held during catch-up... */
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, done_while_in_catchup,
"waiter must not complete while the covering standby is in catch-up");
/* ...then woken to success by the membership re-evaluation */
TEST_ASSERT_EQUAL_INT(0, g_wargs[0].rc);
TEST_ASSERT_TRUE_MESSAGE(g_wargs[0].elapsed_ms >= 100,
"waiter must block until the membership change");
TEST_ASSERT_EQUAL_UINT64(1, committed());
engine_teardown();
printf("test_catchup_clear_then_membership_change_completes_waiter: PASS\n");
}
/* ── Test 19: removal below majority — pending WAIT dies at ITS deadline ──
* Standby removal that makes majority unreachable must never surface
* success; per the design (no forced early-fail — reconnection within the
* deadline may still commit), the wait fails with -ETIMEDOUT at its own
* deadline, and committed_lsn never regresses. (Req 4.1, 4.2) */
static void test_standby_removal_below_majority_wait_fails_at_deadline(void)
{
/* Arrange: 3-member cluster, entries 1..2 appended; entry 1 committed */
engine_setup(3, 2, 2);
standby_ack(0, 1);
TEST_ASSERT_EQUAL_UINT64(1, committed());
spawn_waiters(1, 2, 400);
/* Act: both standbys drop — majority for entry 2 is now unreachable */
sleep_ms(50);
repl_remove_standby(&g_p.repl, &g_eps[0]);
repl_remove_standby(&g_p.repl, &g_eps[1]);
commit_on_membership_change(&g_ce);
join_waiters(1);
/* Assert: never success... */
TEST_ASSERT_TRUE_MESSAGE(
g_wargs[0].rc != 0,
"WAIT must not succeed after removal makes majority unreachable");
/* ...fails with -ETIMEDOUT at its own deadline (no forced early-fail) */
TEST_ASSERT_EQUAL_INT(-ETIMEDOUT, g_wargs[0].rc);
TEST_ASSERT_TRUE_MESSAGE(g_wargs[0].elapsed_ms >= 350,
"removal must not abort the wait before its deadline");
/* ...and the commit index never regresses on membership shrink */
TEST_ASSERT_EQUAL_UINT64(1, committed());
engine_teardown();
printf("test_standby_removal_below_majority_wait_fails_at_deadline: PASS\n");
}
/* ── Test 20: commit_on_membership_change returns without blocking ────────
* iomgr-callable contract: bounded mutex hold only — it must return
* immediately even while a WAIT is pending (it never waits for waiter
* completion or I/O). (Req 4.2; design §External Dependencies) */
static void test_membership_change_returns_without_blocking(void)
{
uint64_t t0, elapsed_ms;
/* Arrange: 3-member cluster, entry 1 appended, one pending waiter */
engine_setup(3, 2, 1);
spawn_waiters(1, 1, 2000);
sleep_ms(50);
/* Act: membership event with the waiter still blocked */
t0 = mono_ms();
commit_on_membership_change(&g_ce);
elapsed_ms = mono_ms() - t0;
/* Complete the waiter so teardown joins cleanly */
standby_ack(0, 1);
join_waiters(1);
/* Assert (after join): the membership call itself never blocked... */
TEST_ASSERT_TRUE_MESSAGE(elapsed_ms < 500,
"commit_on_membership_change must not block");
/* ...and the pending waiter still completed normally via the ACK */
TEST_ASSERT_EQUAL_INT(0, g_wargs[0].rc);
TEST_ASSERT_EQUAL_UINT64(1, committed());
engine_teardown();
printf("test_membership_change_returns_without_blocking: PASS\n");
}
/* ── Test group entry point ──────────────────────────────────────────────── */
void test_commit_engine(void)
@ -656,5 +806,10 @@ void test_commit_engine(void)
RUN_TEST(test_wait_no_gate_when_writable_lsn_unset);
RUN_TEST(test_wait_gate_blocks_until_committed_reaches_gate);
RUN_TEST(test_wait_gate_never_opening_times_out_not_notready);
/* Task 3.1 — membership re-evaluation */
RUN_TEST(test_catchup_standby_ack_does_not_count_toward_pending_entry);
RUN_TEST(test_catchup_clear_then_membership_change_completes_waiter);
RUN_TEST(test_standby_removal_below_majority_wait_fails_at_deadline);
RUN_TEST(test_membership_change_returns_without_blocking);
UNITY_END();
}