myxce/src/controller/testsuite/test_repl_unit.c
Rongsong Shen c0b0c7d1de fix(repl): harden standby slot allocation against WAL-retention guard clobber (bug-1474)
repl_add_standby()'s "find first empty slot" scan treated any ep==NULL
slot as available, with no awareness that a disconnected slot can still
hold an unexpired bug-1147 WAL-retention guard (last_disconnected_lsn,
set by repl_remove_standby, meant to be cleared only by
repl_expire_disconnect_slots once snap_interval_s genuinely elapses).
The scan could pick a guard-holding slot over a genuinely virgin one
purely by lower index, then unconditionally memset it -- destroying the
guard before trim_wal_segments/repl_trim_lsn could ever honor it.
Confirmed via 2/2 reproducible live 3-node soak runs (config-upgrade-
snapshot task 8.9's new --config-upgrade-wal-trim mode): a killed
standby's still-needed WAL segment got trimmed out from under it.

Now a two-pass scan: prefer a genuinely virgin slot; only fall back to
reusing a guard-holding slot (oldest last_disconnect_time first, to
discard the least remaining protection) when no virgin slot exists --
logged via WARN() so the discarded guard is operator-visible rather
than silent. Defense-in-depth at the slot-allocation layer, independent
of whatever triggers a given repl_add_standby call (a separate,
not-fully-root-caused ha.c reconnect-sweep defect is the specific
trigger seen in the soak, out of this fix's scope).

Independently reviewed across two rounds: round 1 flagged the
exhausted-fallback path silently discarding a guard with zero logging;
remediated with a WARN() matching this file's existing repl_broadcast
convention, empirically confirmed to fire only on the fallback path
with correct content. Suite: 1028 -> 1030 tests, 0 failures (net of
this fix; final combined total with bug-1472 landing concurrently:
1033 tests, 0 failures).
2026-07-26 04:11:27 +00:00

742 lines
30 KiB
C

#include <assert.h>
#include <errno.h>
#include <stdatomic.h>
#include <stdint.h>
#include <string.h>
#include <stdio.h>
#include <pthread.h>
#include <time.h>
/* Use the same Unity include pattern as other controller tests */
#include <unity/unity.h>
#include "repl.h"
#include "config.h"
#include "typemap.h"
#include "wal_types.h"
#include "controller.capnp.h" /* OPCode_walAck */
/* ============================================================
* Test seam for repl_broadcast send path (task 1.1)
* Declared as extern here; defined in repl.c under XCE_TESTING.
* ============================================================ */
#ifdef XCE_TESTING
extern int (*repl_broadcast_send_hook)(void *ep,
wal_catchup_entry_t **entries,
int n_entries);
#endif
static void test_repl_init_zeroes_nstandbys(void) {
replicator_t r;
TEST_ASSERT_EQUAL_INT(0, repl_init(&r));
TEST_ASSERT_EQUAL_INT(0, r.nstandbys);
repl_fini(&r);
printf("test_repl_init_zeroes_nstandbys: PASS\n");
}
static void test_repl_add_remove_standby(void) {
replicator_t r;
/* Use a dummy int as the endpoint pointer — repl.c only compares ep pointers */
int dummy_ep = 0;
repl_init(&r);
int idx = repl_add_standby(&r, (void*)&dummy_ep, "standby1", 7001);
TEST_ASSERT_TRUE(idx >= 0);
TEST_ASSERT_EQUAL_INT(1, r.nstandbys);
repl_remove_standby(&r, (void*)&dummy_ep);
TEST_ASSERT_EQUAL_INT(0, r.nstandbys);
repl_fini(&r);
printf("test_repl_add_remove_standby: PASS\n");
}
static void test_repl_add_standby_max(void) {
replicator_t r;
int eps[MAX_STANDBYS + 1];
repl_init(&r);
for (int i = 0; i < MAX_STANDBYS; i++) {
int rc = repl_add_standby(&r, (void*)&eps[i], "sb", 7000 + i);
TEST_ASSERT_TRUE(rc >= 0);
}
/* One over limit must fail */
int rc = repl_add_standby(&r, (void*)&eps[MAX_STANDBYS], "sb_extra", 9999);
TEST_ASSERT_TRUE(rc < 0);
repl_fini(&r);
printf("test_repl_add_standby_max: PASS\n");
}
static void test_repl_trim_lsn_no_standbys(void) {
replicator_t r;
repl_init(&r);
uint64_t trim = repl_trim_lsn(&r, 42, 60);
TEST_ASSERT_EQUAL_UINT64(42, trim);
repl_fini(&r);
printf("test_repl_trim_lsn_no_standbys: PASS\n");
}
static void test_repl_trim_lsn_disconnected_slot_expiry(void) {
replicator_t r;
int dummy_ep = 0;
repl_init(&r);
repl_add_standby(&r, (void*)&dummy_ep, "sb", 7001);
repl_on_ack(&r, (void*)&dummy_ep, 10);
repl_remove_standby(&r, (void*)&dummy_ep);
/* Immediately after disconnect: trim guard holds at last_disconnected_lsn=10 */
uint64_t trim = repl_trim_lsn(&r, 50, 60);
TEST_ASSERT_EQUAL_UINT64(10, trim);
/* After expiry (snap_interval_s=0 forces immediate expiry): trim == checkpoint */
repl_expire_disconnect_slots(&r, 999999, 0);
trim = repl_trim_lsn(&r, 50, 0);
TEST_ASSERT_EQUAL_UINT64(50, trim);
repl_fini(&r);
printf("test_repl_trim_lsn_disconnected_slot_expiry: PASS\n");
}
/* ============================================================
* P2-3: repl_wait_ack() tests
* ============================================================ */
/* Test 1: repl_wait_ack returns -ENODEV when no standbys connected */
static void test_wait_ack_no_standbys_returns_enodev(void) {
replicator_t r;
repl_init(&r);
int rc = repl_wait_ack(&r, /*target_lsn=*/10, /*timeout_ms=*/50);
TEST_ASSERT_EQUAL_INT(-ENODEV, rc);
repl_fini(&r);
printf("test_wait_ack_no_standbys_returns_enodev: PASS\n");
}
/* Test 2: repl_wait_ack returns 0 when standby already ACKed target LSN */
static void test_wait_ack_already_acked(void) {
replicator_t r;
int dummy_ep = 0;
repl_init(&r);
repl_add_standby(&r, (void*)&dummy_ep, "sb", 7001);
/* Pre-set acked_lsn to 20 (already past target 10) */
repl_on_ack(&r, (void*)&dummy_ep, 20);
int rc = repl_wait_ack(&r, 10, 50);
TEST_ASSERT_EQUAL_INT(0, rc);
repl_fini(&r);
printf("test_wait_ack_already_acked: PASS\n");
}
/* Test 3: repl_wait_ack times out when standby never ACKs */
static void test_wait_ack_timeout(void) {
replicator_t r;
int dummy_ep = 0;
repl_init(&r);
repl_add_standby(&r, (void*)&dummy_ep, "sb", 7001);
/* acked_lsn stays at 0 — never reaches target 100 */
struct timespec start, end;
clock_gettime(CLOCK_MONOTONIC, &start);
int rc = repl_wait_ack(&r, 100, 100 /* 100ms timeout */);
clock_gettime(CLOCK_MONOTONIC, &end);
TEST_ASSERT_EQUAL_INT(-ETIMEDOUT, rc);
/* Verify we actually waited ~100ms (not returned immediately) */
long elapsed_ms = (end.tv_sec - start.tv_sec) * 1000
+ (end.tv_nsec - start.tv_nsec) / 1000000;
TEST_ASSERT_TRUE(elapsed_ms >= 80); /* at least 80ms */
repl_fini(&r);
printf("test_wait_ack_timeout: PASS\n");
}
/* Test 4: ACK from background thread wakes repl_wait_ack */
struct ack_args {
replicator_t *r;
void *ep;
uint64_t lsn;
int delay_ms;
};
static void* ack_sender_thread(void* arg) {
struct ack_args* a = arg;
struct timespec ts = { 0, (long)a->delay_ms * 1000000L };
nanosleep(&ts, NULL);
repl_on_ack(a->r, a->ep, a->lsn);
return NULL;
}
static void test_wait_ack_woken_by_background_ack(void) {
replicator_t r;
int dummy_ep = 0;
repl_init(&r);
repl_add_standby(&r, (void*)&dummy_ep, "sb", 7001);
struct ack_args args = { &r, (void*)&dummy_ep, 15, 30 /* 30ms delay */ };
pthread_t th;
pthread_create(&th, NULL, ack_sender_thread, &args);
/* Wait for LSN 15 with 200ms timeout — ACK arrives after 30ms */
int rc = repl_wait_ack(&r, 15, 200);
pthread_join(th, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
repl_fini(&r);
printf("test_wait_ack_woken_by_background_ack: PASS\n");
}
/* ============================================================
* Task 1.1: repl_broadcast send path tests
* ============================================================ */
/* Shared hook state for send-path tests */
static int g_hook_call_count;
static void *g_hook_last_ep;
static uint64_t g_hook_last_lsn;
static uint8_t g_hook_last_entry_type;
static const void *g_hook_last_payload;
static size_t g_hook_last_payload_len;
static int g_hook_return_val; /* 0 = success, negative = error */
/* ha-term-safety task 3.4: capture the AppendEntries term + prev tuple the live
* broadcast carries, so a test can assert the leader's term is on the wire. */
static uint64_t g_hook_last_term;
static uint64_t g_hook_last_prev_log_index;
static uint64_t g_hook_last_prev_log_term;
static int test_broadcast_hook(void *ep, wal_catchup_entry_t **entries,
int n_entries)
{
g_hook_call_count++;
g_hook_last_ep = ep;
if (n_entries >= 1 && entries && entries[0]) {
g_hook_last_lsn = entries[0]->lsn;
g_hook_last_entry_type = entries[0]->entry_type;
g_hook_last_payload = entries[0]->raw_payload;
g_hook_last_payload_len = entries[0]->raw_payload_len;
g_hook_last_term = entries[0]->term;
g_hook_last_prev_log_index = entries[0]->prev_log_index;
g_hook_last_prev_log_term = entries[0]->prev_log_term;
}
return g_hook_return_val;
}
static void reset_hook_state(void)
{
g_hook_call_count = 0;
g_hook_last_ep = NULL;
g_hook_last_lsn = 0;
g_hook_last_entry_type = 0;
g_hook_last_payload = NULL;
g_hook_last_payload_len = 0;
g_hook_return_val = 0;
g_hook_last_term = 0;
g_hook_last_prev_log_index = 0;
g_hook_last_prev_log_term = 0;
}
/* Test: no standbys — broadcast returns 0, hook not called */
static void test_broadcast_no_standbys_returns_zero(void)
{
#ifdef XCE_TESTING
replicator_t r;
repl_init(&r);
reset_hook_state();
repl_broadcast_send_hook = test_broadcast_hook;
wal_entry_hdr_t hdr = { .lsn = 10, .type = WAL_JOB_SUBMIT };
uint8_t payload[] = { 0xAA, 0xBB };
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload), 0 /* prev_log_term */);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(0, g_hook_call_count);
repl_broadcast_send_hook = NULL;
repl_fini(&r);
printf("test_broadcast_no_standbys_returns_zero: PASS\n");
#else
printf("test_broadcast_no_standbys_returns_zero: SKIPPED (no XCE_TESTING)\n");
#endif
}
/* Test: one standby — hook called once with correct lsn, type, payload */
static void test_broadcast_one_standby_sends_entry(void)
{
#ifdef XCE_TESTING
replicator_t r;
int dummy_ep = 0;
repl_init(&r);
repl_add_standby(&r, (void *)&dummy_ep, "sb1", 7001);
reset_hook_state();
repl_broadcast_send_hook = test_broadcast_hook;
wal_entry_hdr_t hdr = { .lsn = 42, .type = WAL_JOB_SUBMIT };
uint8_t payload[] = { 1, 2, 3, 4 };
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload), 0 /* prev_log_term */);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(1, g_hook_call_count);
TEST_ASSERT_EQUAL_PTR((void *)&dummy_ep, g_hook_last_ep);
TEST_ASSERT_EQUAL_UINT64(42, g_hook_last_lsn);
TEST_ASSERT_EQUAL_UINT8(WAL_JOB_SUBMIT, g_hook_last_entry_type);
TEST_ASSERT_EQUAL_PTR(payload, g_hook_last_payload);
TEST_ASSERT_EQUAL_UINT(sizeof(payload), g_hook_last_payload_len);
repl_broadcast_send_hook = NULL;
repl_fini(&r);
printf("test_broadcast_one_standby_sends_entry: PASS\n");
#else
printf("test_broadcast_one_standby_sends_entry: SKIPPED (no XCE_TESTING)\n");
#endif
}
/* ha-term-safety task 3.4 (Req 1.3/1.4): the LIVE broadcast must carry the
* leader's per-entry term and the AppendEntries prev tuple, so the standby
* intake routes it through raft_on_append_entries_request and persists the
* LEADER's term — not the term-0 legacy direct-apply path. Before the cutover
* repl_broadcast sent term=0 (RED): the wire entry's term was always 0. */
static void test_broadcast_carries_leader_term_and_prev_tuple(void)
{
#ifdef XCE_TESTING
replicator_t r;
int dummy_ep = 0;
repl_init(&r);
repl_add_standby(&r, (void *)&dummy_ep, "sb1", 7001);
reset_hook_state();
repl_broadcast_send_hook = test_broadcast_hook;
/* Leader appends LSN 5 at term 7; predecessor (LSN 4) was term 7. */
wal_entry_hdr_t hdr = { .lsn = 5, .type = WAL_JOB_SUBMIT, .term = 7 };
uint8_t payload[] = { 9, 8, 7 };
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload),
7 /* prev_log_term of LSN 4 */);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(1, g_hook_call_count);
TEST_ASSERT_EQUAL_UINT64(5, g_hook_last_lsn);
/* The load-bearing assertion: the leader's real term is on the wire entry
* (NOT 0). This FAILS before the task 3.4 cutover. */
TEST_ASSERT_EQUAL_UINT64(7, g_hook_last_term);
/* prev tuple: prev_log_index == lsn - 1; prev_log_term == supplied. */
TEST_ASSERT_EQUAL_UINT64(4, g_hook_last_prev_log_index);
TEST_ASSERT_EQUAL_UINT64(7, g_hook_last_prev_log_term);
repl_broadcast_send_hook = NULL;
repl_fini(&r);
printf("test_broadcast_carries_leader_term_and_prev_tuple: PASS\n");
#else
printf("test_broadcast_carries_leader_term_and_prev_tuple: "
"SKIPPED (no XCE_TESTING)\n");
#endif
}
/* Test: standby in catch-up is skipped — hook not called for it */
static void test_broadcast_skips_catchup_standby(void)
{
#ifdef XCE_TESTING
replicator_t r;
int ep1 = 0, ep2 = 1;
repl_init(&r);
repl_add_standby(&r, (void *)&ep1, "sb1", 7001);
repl_add_standby(&r, (void *)&ep2, "sb2", 7002);
/* Mark sb1 as in catch-up */
repl_start_catchup(&r, (void *)&ep1, 0);
reset_hook_state();
repl_broadcast_send_hook = test_broadcast_hook;
wal_entry_hdr_t hdr = { .lsn = 99, .type = WAL_JOB_SUBMIT };
uint8_t payload[] = { 0xFF };
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload), 0 /* prev_log_term */);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Only sb2 should be sent to */
TEST_ASSERT_EQUAL_INT(1, g_hook_call_count);
TEST_ASSERT_EQUAL_PTR((void *)&ep2, g_hook_last_ep);
repl_broadcast_send_hook = NULL;
repl_fini(&r);
printf("test_broadcast_skips_catchup_standby: PASS\n");
#else
printf("test_broadcast_skips_catchup_standby: SKIPPED (no XCE_TESTING)\n");
#endif
}
/* Test: standby with pending_ack_count > MAX_PENDING_ACKS is evicted to catch-up */
static void test_broadcast_evicts_overloaded_standby(void)
{
#ifdef XCE_TESTING
replicator_t r;
int dummy_ep = 0;
repl_init(&r);
int idx = repl_add_standby(&r, (void *)&dummy_ep, "sb1", 7001);
/* Manually set pending_ack_count just above the limit */
r.standbys[idx].pending_ack_count = MAX_PENDING_ACKS + 1;
reset_hook_state();
repl_broadcast_send_hook = test_broadcast_hook;
wal_entry_hdr_t hdr = { .lsn = 5, .type = WAL_JOB_SUBMIT };
uint8_t payload[] = { 0x01 };
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload), 0 /* prev_log_term */);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Evicted — hook must NOT be called */
TEST_ASSERT_EQUAL_INT(0, g_hook_call_count);
/* Standby must now be in catch-up */
TEST_ASSERT_NOT_EQUAL(0,
atomic_load_explicit(&r.standbys[idx].in_catch_up, memory_order_acquire));
repl_broadcast_send_hook = NULL;
repl_fini(&r);
printf("test_broadcast_evicts_overloaded_standby: PASS\n");
#else
printf("test_broadcast_evicts_overloaded_standby: SKIPPED (no XCE_TESTING)\n");
#endif
}
/* Test (Task 2.3): repl_add_standby is idempotent for the same ep.
* Calling it twice with the same endpoint must return the same index,
* keep nstandbys==1, and not crash or corrupt state.
* Req 1.3: If repl_add_standby is called with an endpoint already
* registered, the system shall treat it as idempotent (no-op or update-in-place). */
static void test_repl_add_standby_idempotent(void) {
replicator_t r;
int dummy_ep = 0;
repl_init(&r);
int idx1 = repl_add_standby(&r, (void *)&dummy_ep, "sb1", 7001);
TEST_ASSERT_TRUE(idx1 >= 0);
TEST_ASSERT_EQUAL_INT(1, r.nstandbys);
/* Second call with the same ep — must be a no-op */
int idx2 = repl_add_standby(&r, (void *)&dummy_ep, "sb1", 7001);
TEST_ASSERT_TRUE(idx2 >= 0);
/* nstandbys must still be 1 (not 2) */
TEST_ASSERT_EQUAL_INT(1, r.nstandbys);
/* Must return the same slot index */
TEST_ASSERT_EQUAL_INT(idx1, idx2);
repl_fini(&r);
printf("test_repl_add_standby_idempotent: PASS\n");
}
/* ============================================================
* Task 4.1: WAL ACK reply handler — Req 4.3
*
* These tests verify the behaviour that repl_ack_decode_and_dispatch
* implements in ha.c: on a successful WalAckReply it calls
* repl_on_ack(&c->persist.repl, ep, acked_lsn), which must:
* a) update standby's acked_lsn
* b) signal ack_cond so repl_wait_majority_ack unblocks
*
* We test repl_on_ack directly (the exact call the handler makes).
* ============================================================ */
/* ACK sender thread args reused from above — ack_args struct already defined */
/* Test: repl_on_ack (handler ACK path) updates acked_lsn and unblocks
* repl_wait_majority_ack. Req 4.3: when the primary receives a WAL ACK
* from a standby it shall update that standby's acked_lsn in the replicator
* and signal any waiter on the ACK condition variable. */
static void test_wal_ack_reply_updates_acked_lsn_and_signals(void)
{
replicator_t r;
int dummy_ep = 0;
repl_init(&r);
repl_add_standby(&r, (void *)&dummy_ep, "sb1", 7001);
const uint64_t target_lsn = 77;
/* Background thread simulates the handler calling repl_on_ack after 30 ms */
struct ack_args args = { &r, (void *)&dummy_ep, target_lsn, 30 };
pthread_t th;
pthread_create(&th, NULL, ack_sender_thread, &args);
/* repl_wait_majority_ack: 1 active standby, quorum=(1+2)/2=1,
* acked_total = 1(self) + 1(standby) = 2 >= 1 → returns 0. */
int rc = repl_wait_majority_ack(&r, 0, 200, (uint8_t)OPCode_walAck);
pthread_join(th, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"repl_wait_majority_ack must return 0 after repl_on_ack signals (Req 4.3)");
/* Verify acked_lsn was actually updated in the slot */
int found = 0;
for (int i = 0; i < MAX_STANDBYS; i++) {
if (r.standbys[i].ep == (void *)&dummy_ep) {
uint64_t al = atomic_load_explicit(&r.standbys[i].acked_lsn,
memory_order_acquire);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(target_lsn, al,
"acked_lsn must equal the LSN passed to repl_on_ack (Req 4.3)");
found = 1;
break;
}
}
TEST_ASSERT_TRUE_MESSAGE(found, "standby slot must be present after repl_on_ack");
repl_fini(&r);
printf("test_wal_ack_reply_updates_acked_lsn_and_signals: PASS\n");
}
/* Test: NACK path — handler does NOT call repl_on_ack, so acked_lsn stays 0.
* Req 4.3 (negative): only a success==1 reply triggers the update; success==0
* is a no-op on the replicator.
*
* Quorum analysis for this scenario:
* n_active = 1 (one standby registered, not in catch-up)
* quorum = (1 + 2) / 2 = 1
* walAck: acked_total = 1 (self) + 0 (standby acked_lsn == 0, not > 0) = 1
* 1 >= quorum(1) → repl_wait_majority_ack returns 0 IMMEDIATELY.
*
* The primary self-count alone satisfies a 1-standby quorum, so the function
* does NOT time out even when the standby never ACKs. The NACK invariant is
* that acked_lsn remains 0 (not that the wait times out).
*/
static void test_wal_nack_does_not_update_acked_lsn(void)
{
replicator_t r;
int dummy_ep = 0;
repl_init(&r);
repl_add_standby(&r, (void *)&dummy_ep, "sb1", 7001);
/* No repl_on_ack call — simulates handler receiving success==0 NACK */
/* acked_lsn must remain 0 */
uint64_t al = 0;
for (int i = 0; i < MAX_STANDBYS; i++) {
if (r.standbys[i].ep == (void *)&dummy_ep) {
al = atomic_load_explicit(&r.standbys[i].acked_lsn,
memory_order_acquire);
break;
}
}
TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, al,
"acked_lsn must remain 0 when NACK path skips repl_on_ack (Req 4.3)");
/* With 1 standby and no standby ACK, primary self-count (1) already meets
* quorum (1), so repl_wait_majority_ack returns 0, not -ETIMEDOUT.
* A timeout would only occur if quorum required at least one standby ACK,
* which only happens when n_active >= 2 (e.g. 2+ standbys registered). */
int rc = repl_wait_majority_ack(&r, 0, 50, (uint8_t)OPCode_walAck);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"repl_wait_majority_ack must return 0 with 1 standby: "
"self-count(1) >= quorum(1) even without standby ACK (Req 4.3)");
repl_fini(&r);
printf("test_wal_nack_does_not_update_acked_lsn: PASS\n");
}
/* Test: send failure (hook returns -EIO) is tolerated — broadcast returns 0 */
static void test_broadcast_send_failure_continues(void)
{
#ifdef XCE_TESTING
replicator_t r;
int ep1 = 0, ep2 = 1;
repl_init(&r);
repl_add_standby(&r, (void *)&ep1, "sb1", 7001);
repl_add_standby(&r, (void *)&ep2, "sb2", 7002);
reset_hook_state();
g_hook_return_val = -EIO; /* First call fails */
repl_broadcast_send_hook = test_broadcast_hook;
wal_entry_hdr_t hdr = { .lsn = 77, .type = WAL_JOB_SUBMIT };
uint8_t payload[] = { 0xDE, 0xAD };
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload), 0 /* prev_log_term */);
/* broadcast should still return 0 — errors are WARN+continue */
TEST_ASSERT_EQUAL_INT(0, rc);
/* Both standbys attempted */
TEST_ASSERT_EQUAL_INT(2, g_hook_call_count);
repl_broadcast_send_hook = NULL;
repl_fini(&r);
printf("test_broadcast_send_failure_continues: PASS\n");
#else
printf("test_broadcast_send_failure_continues: SKIPPED (no XCE_TESTING)\n");
#endif
}
/* ============================================================
* Task 5.3: Registration lifecycle — Req 1.1, 1.2, 1.3
*
* Comprehensive test covering all three lifecycle steps in sequence:
* Step 1 (Req 1.1): Register ep → nstandbys==1, standbys[idx].ep==ep
* Step 2 (Req 1.3): Re-register same ep → idempotent, nstandbys==1, same idx
* Step 3 (Req 1.2): Deregister ep → nstandbys==0, standbys[idx].ep==NULL
* ============================================================ */
static void test_repl_registration_lifecycle(void) {
replicator_t r;
int dummy_ep = 0;
repl_init(&r);
/* Step 1: Register ep — Req 1.1 */
int idx = repl_add_standby(&r, (void *)&dummy_ep, "sb1", 7001);
TEST_ASSERT_TRUE_MESSAGE(idx >= 0, "repl_add_standby must return a valid slot index");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, r.nstandbys,
"nstandbys must be 1 after first registration (Req 1.1)");
TEST_ASSERT_EQUAL_PTR_MESSAGE((void *)&dummy_ep, r.standbys[idx].ep,
"standbys[idx].ep must equal registered ep (Req 1.1)");
/* Step 2: Re-register same ep — idempotent, Req 1.3 */
int idx2 = repl_add_standby(&r, (void *)&dummy_ep, "sb1", 7001);
TEST_ASSERT_TRUE_MESSAGE(idx2 >= 0, "duplicate repl_add_standby must return valid index");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, r.nstandbys,
"nstandbys must remain 1 after duplicate registration (Req 1.3)");
TEST_ASSERT_EQUAL_INT_MESSAGE(idx, idx2,
"duplicate registration must return the same slot index (Req 1.3)");
/* Step 3: Deregister ep — Req 1.2 */
repl_remove_standby(&r, (void *)&dummy_ep);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, r.nstandbys,
"nstandbys must be 0 after deregistration (Req 1.2)");
TEST_ASSERT_NULL_MESSAGE(r.standbys[idx].ep,
"standbys[idx].ep must be NULL after deregistration (Req 1.2)");
repl_fini(&r);
printf("test_repl_registration_lifecycle: PASS\n");
}
/* ============================================================
* bug-1474: repl_add_standby slot allocation must never clobber an
* unexpired bug-1147 disconnect guard (last_disconnected_lsn) belonging to
* a DIFFERENT, still-disconnected peer, when a genuinely virgin
* (never-used) slot is available. See .wolf/buglog.json bug-1474 for the
* full forensic trace: a spurious/duplicate reconnect for an unrelated,
* already-connected peer picked the lower-indexed, guard-holding slot over
* a higher-indexed virgin slot and memset it to zero, silently destroying
* the guard repl_trim_lsn relies on to protect WAL segments a genuinely
* disconnected-but-reconnecting standby still needs.
* ============================================================ */
/* RED (pre-fix): repl_add_standby's old "first empty slot by lowest index"
* scan picks idx_a (guard-holding, index 0) over the virgin slot (index 2)
* purely because of index order, destroying A's guard. Req: the guard must
* survive and the new peer must land in the virgin slot instead. */
static void test_repl_add_standby_prefers_virgin_slot_over_guard(void) {
replicator_t r;
int ep_a = 0, ep_b = 1, ep_c = 2;
repl_init(&r);
int idx_a = repl_add_standby(&r, (void *)&ep_a, "standby-a", 7001);
int idx_b = repl_add_standby(&r, (void *)&ep_b, "standby-b", 7002);
TEST_ASSERT_TRUE(idx_a >= 0);
TEST_ASSERT_TRUE(idx_b >= 0);
TEST_ASSERT_NOT_EQUAL(idx_a, idx_b);
/* A acks up to LSN 42, then disconnects: leaves a bug-1147 guard at
* idx_a with last_disconnected_lsn == 42. The third slot (whichever
* index neither idx_a nor idx_b used) remains genuinely virgin. */
repl_on_ack(&r, (void *)&ep_a, 42);
repl_remove_standby(&r, (void *)&ep_a);
TEST_ASSERT_NULL(r.standbys[idx_a].ep);
TEST_ASSERT_EQUAL_UINT64(42, r.standbys[idx_a].last_disconnected_lsn);
/* A third, genuinely NEW peer (unrelated to A) connects while idx_a's
* guard is still unexpired and a virgin slot is also available. */
int idx_c = repl_add_standby(&r, (void *)&ep_c, "standby-c", 7003);
TEST_ASSERT_TRUE(idx_c >= 0);
/* The new peer must land in the virgin slot, NOT the guard-holding one,
* even though idx_a may be the lower index. */
TEST_ASSERT_NOT_EQUAL_MESSAGE(idx_a, idx_c,
"new peer must not be allocated the guard-holding slot when a"
" virgin slot exists (bug-1474)");
/* Load-bearing assertion: A's guard must survive completely untouched.
* Before the fix, repl_add_standby's unconditional memset(s, 0, ...)
* on the chosen slot would have wiped last_disconnected_lsn to 0 here. */
TEST_ASSERT_NULL_MESSAGE(r.standbys[idx_a].ep,
"guard-holding slot must remain disconnected (bug-1474)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(42, r.standbys[idx_a].last_disconnected_lsn,
"bug-1147 guard LSN must survive an unrelated peer's"
" repl_add_standby call (bug-1474)");
repl_fini(&r);
printf("test_repl_add_standby_prefers_virgin_slot_over_guard: PASS\n");
}
/* RED (pre-fix, and proves the DELIBERATE fallback policy): with
* MAX_STANDBYS==3, exhaust all three slots, then disconnect two of them
* (both now guard-holding, one older-disconnected than the other) so NO
* virgin slot remains. A fourth genuinely new peer must still be admitted
* (never -ENOSPC for this case -- refusing a legitimate reconnect would be
* worse for cluster availability than an early guard loss), and the
* fallback must deterministically reuse the OLDER-disconnected guard slot
* (closest to its own natural repl_expire_disconnect_slots expiry, so
* reusing it discards the LEAST remaining WAL-retention protection) --
* not merely the lowest-indexed guard slot. */
static void test_repl_add_standby_exhausted_guards_prefers_oldest_disconnect(void) {
replicator_t r;
int ep_a = 0, ep_b = 1, ep_c = 2, ep_d = 3;
repl_init(&r);
int idx_a = repl_add_standby(&r, (void *)&ep_a, "standby-a", 7001);
int idx_b = repl_add_standby(&r, (void *)&ep_b, "standby-b", 7002);
int idx_c = repl_add_standby(&r, (void *)&ep_c, "standby-c", 7003);
TEST_ASSERT_TRUE(idx_a >= 0);
TEST_ASSERT_TRUE(idx_b >= 0);
TEST_ASSERT_TRUE(idx_c >= 0);
/* All MAX_STANDBYS(3) slots now occupied. */
/* B disconnects first; force its disconnect timestamp strictly OLDER
* than A's (below) so the two guards are unambiguously ordered by
* time, independent of their slot index (idx_a < idx_b by add order --
* deliberately the OPPOSITE of the time order, so a naive
* "lowest-index guard slot" fallback would pick the wrong one). */
repl_on_ack(&r, (void *)&ep_b, 20);
repl_remove_standby(&r, (void *)&ep_b);
r.standbys[idx_b].last_disconnect_time -= 100;
/* A disconnects second, with its real (newer) "now" timestamp. */
repl_on_ack(&r, (void *)&ep_a, 10);
repl_remove_standby(&r, (void *)&ep_a);
TEST_ASSERT_TRUE_MESSAGE(
r.standbys[idx_b].last_disconnect_time < r.standbys[idx_a].last_disconnect_time,
"test setup invariant: B must be the OLDER disconnect");
/* C is still connected; nstandbys == 1 (< MAX_STANDBYS), so the
* "too many distinct live peers" -ENOSPC guard does not trip -- but
* BOTH empty slots (idx_a, idx_b) are guard-holding: no virgin slot
* exists anywhere in the registry. */
TEST_ASSERT_EQUAL_INT(1, r.nstandbys);
/* A fourth, genuinely new peer connects into the exhausted state. */
int idx_d = repl_add_standby(&r, (void *)&ep_d, "standby-d", 7004);
TEST_ASSERT_TRUE_MESSAGE(idx_d >= 0,
"exhausted-guard fallback must still admit a legitimate new peer,"
" never -ENOSPC (bug-1474 policy)");
/* Deliberate policy: reuse the OLDER-disconnected guard slot (idx_b),
* not idx_a (which is both newer-disconnected AND lower-indexed --
* ruling out an accidental "first guard by index" fallthrough). */
TEST_ASSERT_EQUAL_INT_MESSAGE(idx_b, idx_d,
"exhausted fallback must reuse the OLDEST-disconnected guard slot,"
" not merely the lowest-indexed one (bug-1474 policy)");
/* idx_a's guard (the one NOT chosen) must remain completely untouched. */
TEST_ASSERT_NULL_MESSAGE(r.standbys[idx_a].ep,
"non-selected guard slot must remain disconnected");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(10, r.standbys[idx_a].last_disconnected_lsn,
"non-selected guard slot's LSN must survive untouched");
repl_fini(&r);
printf("test_repl_add_standby_exhausted_guards_prefers_oldest_disconnect: PASS\n");
}
void test_repl_unit(void) {
UnitySetTestFile(__FILE__);
RUN_TEST(test_repl_init_zeroes_nstandbys);
RUN_TEST(test_repl_add_remove_standby);
RUN_TEST(test_repl_add_standby_max);
RUN_TEST(test_repl_trim_lsn_no_standbys);
RUN_TEST(test_repl_trim_lsn_disconnected_slot_expiry);
RUN_TEST(test_wait_ack_no_standbys_returns_enodev);
RUN_TEST(test_wait_ack_already_acked);
RUN_TEST(test_wait_ack_timeout);
RUN_TEST(test_wait_ack_woken_by_background_ack);
/* Task 2.3: idempotent re-registration */
RUN_TEST(test_repl_add_standby_idempotent);
/* Task 1.1: broadcast send path */
RUN_TEST(test_broadcast_no_standbys_returns_zero);
RUN_TEST(test_broadcast_one_standby_sends_entry);
RUN_TEST(test_broadcast_carries_leader_term_and_prev_tuple);
RUN_TEST(test_broadcast_skips_catchup_standby);
RUN_TEST(test_broadcast_evicts_overloaded_standby);
RUN_TEST(test_broadcast_send_failure_continues);
/* Task 4.1: WAL ACK reply handler — Req 4.3 */
RUN_TEST(test_wal_ack_reply_updates_acked_lsn_and_signals);
RUN_TEST(test_wal_nack_does_not_update_acked_lsn);
/* Task 5.3: registration lifecycle — Req 1.1, 1.2, 1.3 */
RUN_TEST(test_repl_registration_lifecycle);
/* bug-1474: slot allocation must not clobber an unexpired guard */
RUN_TEST(test_repl_add_standby_prefers_virgin_slot_over_guard);
RUN_TEST(test_repl_add_standby_exhausted_guards_prefers_oldest_disconnect);
}