repl_add_standby()'s "find first empty slot" scan treated any ep==NULL slot as available, with no awareness that a disconnected slot can still hold an unexpired bug-1147 WAL-retention guard (last_disconnected_lsn, set by repl_remove_standby, meant to be cleared only by repl_expire_disconnect_slots once snap_interval_s genuinely elapses). The scan could pick a guard-holding slot over a genuinely virgin one purely by lower index, then unconditionally memset it -- destroying the guard before trim_wal_segments/repl_trim_lsn could ever honor it. Confirmed via 2/2 reproducible live 3-node soak runs (config-upgrade- snapshot task 8.9's new --config-upgrade-wal-trim mode): a killed standby's still-needed WAL segment got trimmed out from under it. Now a two-pass scan: prefer a genuinely virgin slot; only fall back to reusing a guard-holding slot (oldest last_disconnect_time first, to discard the least remaining protection) when no virgin slot exists -- logged via WARN() so the discarded guard is operator-visible rather than silent. Defense-in-depth at the slot-allocation layer, independent of whatever triggers a given repl_add_standby call (a separate, not-fully-root-caused ha.c reconnect-sweep defect is the specific trigger seen in the soak, out of this fix's scope). Independently reviewed across two rounds: round 1 flagged the exhausted-fallback path silently discarding a guard with zero logging; remediated with a WARN() matching this file's existing repl_broadcast convention, empirically confirmed to fire only on the fallback path with correct content. Suite: 1028 -> 1030 tests, 0 failures (net of this fix; final combined total with bug-1472 landing concurrently: 1033 tests, 0 failures).
742 lines
30 KiB
C
742 lines
30 KiB
C
#include <assert.h>
|
|
#include <errno.h>
|
|
#include <stdatomic.h>
|
|
#include <stdint.h>
|
|
#include <string.h>
|
|
#include <stdio.h>
|
|
#include <pthread.h>
|
|
#include <time.h>
|
|
/* Use the same Unity include pattern as other controller tests */
|
|
#include <unity/unity.h>
|
|
#include "repl.h"
|
|
#include "config.h"
|
|
#include "typemap.h"
|
|
#include "wal_types.h"
|
|
#include "controller.capnp.h" /* OPCode_walAck */
|
|
|
|
/* ============================================================
|
|
* Test seam for repl_broadcast send path (task 1.1)
|
|
* Declared as extern here; defined in repl.c under XCE_TESTING.
|
|
* ============================================================ */
|
|
#ifdef XCE_TESTING
|
|
extern int (*repl_broadcast_send_hook)(void *ep,
|
|
wal_catchup_entry_t **entries,
|
|
int n_entries);
|
|
#endif
|
|
|
|
static void test_repl_init_zeroes_nstandbys(void) {
|
|
replicator_t r;
|
|
TEST_ASSERT_EQUAL_INT(0, repl_init(&r));
|
|
TEST_ASSERT_EQUAL_INT(0, r.nstandbys);
|
|
repl_fini(&r);
|
|
printf("test_repl_init_zeroes_nstandbys: PASS\n");
|
|
}
|
|
|
|
static void test_repl_add_remove_standby(void) {
|
|
replicator_t r;
|
|
/* Use a dummy int as the endpoint pointer — repl.c only compares ep pointers */
|
|
int dummy_ep = 0;
|
|
repl_init(&r);
|
|
int idx = repl_add_standby(&r, (void*)&dummy_ep, "standby1", 7001);
|
|
TEST_ASSERT_TRUE(idx >= 0);
|
|
TEST_ASSERT_EQUAL_INT(1, r.nstandbys);
|
|
repl_remove_standby(&r, (void*)&dummy_ep);
|
|
TEST_ASSERT_EQUAL_INT(0, r.nstandbys);
|
|
repl_fini(&r);
|
|
printf("test_repl_add_remove_standby: PASS\n");
|
|
}
|
|
|
|
static void test_repl_add_standby_max(void) {
|
|
replicator_t r;
|
|
int eps[MAX_STANDBYS + 1];
|
|
repl_init(&r);
|
|
for (int i = 0; i < MAX_STANDBYS; i++) {
|
|
int rc = repl_add_standby(&r, (void*)&eps[i], "sb", 7000 + i);
|
|
TEST_ASSERT_TRUE(rc >= 0);
|
|
}
|
|
/* One over limit must fail */
|
|
int rc = repl_add_standby(&r, (void*)&eps[MAX_STANDBYS], "sb_extra", 9999);
|
|
TEST_ASSERT_TRUE(rc < 0);
|
|
repl_fini(&r);
|
|
printf("test_repl_add_standby_max: PASS\n");
|
|
}
|
|
|
|
static void test_repl_trim_lsn_no_standbys(void) {
|
|
replicator_t r;
|
|
repl_init(&r);
|
|
uint64_t trim = repl_trim_lsn(&r, 42, 60);
|
|
TEST_ASSERT_EQUAL_UINT64(42, trim);
|
|
repl_fini(&r);
|
|
printf("test_repl_trim_lsn_no_standbys: PASS\n");
|
|
}
|
|
|
|
static void test_repl_trim_lsn_disconnected_slot_expiry(void) {
|
|
replicator_t r;
|
|
int dummy_ep = 0;
|
|
repl_init(&r);
|
|
repl_add_standby(&r, (void*)&dummy_ep, "sb", 7001);
|
|
repl_on_ack(&r, (void*)&dummy_ep, 10);
|
|
repl_remove_standby(&r, (void*)&dummy_ep);
|
|
|
|
/* Immediately after disconnect: trim guard holds at last_disconnected_lsn=10 */
|
|
uint64_t trim = repl_trim_lsn(&r, 50, 60);
|
|
TEST_ASSERT_EQUAL_UINT64(10, trim);
|
|
|
|
/* After expiry (snap_interval_s=0 forces immediate expiry): trim == checkpoint */
|
|
repl_expire_disconnect_slots(&r, 999999, 0);
|
|
trim = repl_trim_lsn(&r, 50, 0);
|
|
TEST_ASSERT_EQUAL_UINT64(50, trim);
|
|
|
|
repl_fini(&r);
|
|
printf("test_repl_trim_lsn_disconnected_slot_expiry: PASS\n");
|
|
}
|
|
|
|
/* ============================================================
|
|
* P2-3: repl_wait_ack() tests
|
|
* ============================================================ */
|
|
|
|
/* Test 1: repl_wait_ack returns -ENODEV when no standbys connected */
|
|
static void test_wait_ack_no_standbys_returns_enodev(void) {
|
|
replicator_t r;
|
|
repl_init(&r);
|
|
int rc = repl_wait_ack(&r, /*target_lsn=*/10, /*timeout_ms=*/50);
|
|
TEST_ASSERT_EQUAL_INT(-ENODEV, rc);
|
|
repl_fini(&r);
|
|
printf("test_wait_ack_no_standbys_returns_enodev: PASS\n");
|
|
}
|
|
|
|
/* Test 2: repl_wait_ack returns 0 when standby already ACKed target LSN */
|
|
static void test_wait_ack_already_acked(void) {
|
|
replicator_t r;
|
|
int dummy_ep = 0;
|
|
repl_init(&r);
|
|
repl_add_standby(&r, (void*)&dummy_ep, "sb", 7001);
|
|
/* Pre-set acked_lsn to 20 (already past target 10) */
|
|
repl_on_ack(&r, (void*)&dummy_ep, 20);
|
|
int rc = repl_wait_ack(&r, 10, 50);
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
repl_fini(&r);
|
|
printf("test_wait_ack_already_acked: PASS\n");
|
|
}
|
|
|
|
/* Test 3: repl_wait_ack times out when standby never ACKs */
|
|
static void test_wait_ack_timeout(void) {
|
|
replicator_t r;
|
|
int dummy_ep = 0;
|
|
repl_init(&r);
|
|
repl_add_standby(&r, (void*)&dummy_ep, "sb", 7001);
|
|
/* acked_lsn stays at 0 — never reaches target 100 */
|
|
struct timespec start, end;
|
|
clock_gettime(CLOCK_MONOTONIC, &start);
|
|
int rc = repl_wait_ack(&r, 100, 100 /* 100ms timeout */);
|
|
clock_gettime(CLOCK_MONOTONIC, &end);
|
|
TEST_ASSERT_EQUAL_INT(-ETIMEDOUT, rc);
|
|
/* Verify we actually waited ~100ms (not returned immediately) */
|
|
long elapsed_ms = (end.tv_sec - start.tv_sec) * 1000
|
|
+ (end.tv_nsec - start.tv_nsec) / 1000000;
|
|
TEST_ASSERT_TRUE(elapsed_ms >= 80); /* at least 80ms */
|
|
repl_fini(&r);
|
|
printf("test_wait_ack_timeout: PASS\n");
|
|
}
|
|
|
|
/* Test 4: ACK from background thread wakes repl_wait_ack */
|
|
struct ack_args {
|
|
replicator_t *r;
|
|
void *ep;
|
|
uint64_t lsn;
|
|
int delay_ms;
|
|
};
|
|
|
|
static void* ack_sender_thread(void* arg) {
|
|
struct ack_args* a = arg;
|
|
struct timespec ts = { 0, (long)a->delay_ms * 1000000L };
|
|
nanosleep(&ts, NULL);
|
|
repl_on_ack(a->r, a->ep, a->lsn);
|
|
return NULL;
|
|
}
|
|
|
|
static void test_wait_ack_woken_by_background_ack(void) {
|
|
replicator_t r;
|
|
int dummy_ep = 0;
|
|
repl_init(&r);
|
|
repl_add_standby(&r, (void*)&dummy_ep, "sb", 7001);
|
|
|
|
struct ack_args args = { &r, (void*)&dummy_ep, 15, 30 /* 30ms delay */ };
|
|
pthread_t th;
|
|
pthread_create(&th, NULL, ack_sender_thread, &args);
|
|
|
|
/* Wait for LSN 15 with 200ms timeout — ACK arrives after 30ms */
|
|
int rc = repl_wait_ack(&r, 15, 200);
|
|
pthread_join(th, NULL);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
repl_fini(&r);
|
|
printf("test_wait_ack_woken_by_background_ack: PASS\n");
|
|
}
|
|
|
|
/* ============================================================
|
|
* Task 1.1: repl_broadcast send path tests
|
|
* ============================================================ */
|
|
|
|
/* Shared hook state for send-path tests */
|
|
static int g_hook_call_count;
|
|
static void *g_hook_last_ep;
|
|
static uint64_t g_hook_last_lsn;
|
|
static uint8_t g_hook_last_entry_type;
|
|
static const void *g_hook_last_payload;
|
|
static size_t g_hook_last_payload_len;
|
|
static int g_hook_return_val; /* 0 = success, negative = error */
|
|
/* ha-term-safety task 3.4: capture the AppendEntries term + prev tuple the live
|
|
* broadcast carries, so a test can assert the leader's term is on the wire. */
|
|
static uint64_t g_hook_last_term;
|
|
static uint64_t g_hook_last_prev_log_index;
|
|
static uint64_t g_hook_last_prev_log_term;
|
|
|
|
static int test_broadcast_hook(void *ep, wal_catchup_entry_t **entries,
|
|
int n_entries)
|
|
{
|
|
g_hook_call_count++;
|
|
g_hook_last_ep = ep;
|
|
if (n_entries >= 1 && entries && entries[0]) {
|
|
g_hook_last_lsn = entries[0]->lsn;
|
|
g_hook_last_entry_type = entries[0]->entry_type;
|
|
g_hook_last_payload = entries[0]->raw_payload;
|
|
g_hook_last_payload_len = entries[0]->raw_payload_len;
|
|
g_hook_last_term = entries[0]->term;
|
|
g_hook_last_prev_log_index = entries[0]->prev_log_index;
|
|
g_hook_last_prev_log_term = entries[0]->prev_log_term;
|
|
}
|
|
return g_hook_return_val;
|
|
}
|
|
|
|
static void reset_hook_state(void)
|
|
{
|
|
g_hook_call_count = 0;
|
|
g_hook_last_ep = NULL;
|
|
g_hook_last_lsn = 0;
|
|
g_hook_last_entry_type = 0;
|
|
g_hook_last_payload = NULL;
|
|
g_hook_last_payload_len = 0;
|
|
g_hook_return_val = 0;
|
|
g_hook_last_term = 0;
|
|
g_hook_last_prev_log_index = 0;
|
|
g_hook_last_prev_log_term = 0;
|
|
}
|
|
|
|
/* Test: no standbys — broadcast returns 0, hook not called */
|
|
static void test_broadcast_no_standbys_returns_zero(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
replicator_t r;
|
|
repl_init(&r);
|
|
reset_hook_state();
|
|
repl_broadcast_send_hook = test_broadcast_hook;
|
|
|
|
wal_entry_hdr_t hdr = { .lsn = 10, .type = WAL_JOB_SUBMIT };
|
|
uint8_t payload[] = { 0xAA, 0xBB };
|
|
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload), 0 /* prev_log_term */);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
TEST_ASSERT_EQUAL_INT(0, g_hook_call_count);
|
|
|
|
repl_broadcast_send_hook = NULL;
|
|
repl_fini(&r);
|
|
printf("test_broadcast_no_standbys_returns_zero: PASS\n");
|
|
#else
|
|
printf("test_broadcast_no_standbys_returns_zero: SKIPPED (no XCE_TESTING)\n");
|
|
#endif
|
|
}
|
|
|
|
/* Test: one standby — hook called once with correct lsn, type, payload */
|
|
static void test_broadcast_one_standby_sends_entry(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
replicator_t r;
|
|
int dummy_ep = 0;
|
|
repl_init(&r);
|
|
repl_add_standby(&r, (void *)&dummy_ep, "sb1", 7001);
|
|
reset_hook_state();
|
|
repl_broadcast_send_hook = test_broadcast_hook;
|
|
|
|
wal_entry_hdr_t hdr = { .lsn = 42, .type = WAL_JOB_SUBMIT };
|
|
uint8_t payload[] = { 1, 2, 3, 4 };
|
|
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload), 0 /* prev_log_term */);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
TEST_ASSERT_EQUAL_INT(1, g_hook_call_count);
|
|
TEST_ASSERT_EQUAL_PTR((void *)&dummy_ep, g_hook_last_ep);
|
|
TEST_ASSERT_EQUAL_UINT64(42, g_hook_last_lsn);
|
|
TEST_ASSERT_EQUAL_UINT8(WAL_JOB_SUBMIT, g_hook_last_entry_type);
|
|
TEST_ASSERT_EQUAL_PTR(payload, g_hook_last_payload);
|
|
TEST_ASSERT_EQUAL_UINT(sizeof(payload), g_hook_last_payload_len);
|
|
|
|
repl_broadcast_send_hook = NULL;
|
|
repl_fini(&r);
|
|
printf("test_broadcast_one_standby_sends_entry: PASS\n");
|
|
#else
|
|
printf("test_broadcast_one_standby_sends_entry: SKIPPED (no XCE_TESTING)\n");
|
|
#endif
|
|
}
|
|
|
|
/* ha-term-safety task 3.4 (Req 1.3/1.4): the LIVE broadcast must carry the
|
|
* leader's per-entry term and the AppendEntries prev tuple, so the standby
|
|
* intake routes it through raft_on_append_entries_request and persists the
|
|
* LEADER's term — not the term-0 legacy direct-apply path. Before the cutover
|
|
* repl_broadcast sent term=0 (RED): the wire entry's term was always 0. */
|
|
static void test_broadcast_carries_leader_term_and_prev_tuple(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
replicator_t r;
|
|
int dummy_ep = 0;
|
|
repl_init(&r);
|
|
repl_add_standby(&r, (void *)&dummy_ep, "sb1", 7001);
|
|
reset_hook_state();
|
|
repl_broadcast_send_hook = test_broadcast_hook;
|
|
|
|
/* Leader appends LSN 5 at term 7; predecessor (LSN 4) was term 7. */
|
|
wal_entry_hdr_t hdr = { .lsn = 5, .type = WAL_JOB_SUBMIT, .term = 7 };
|
|
uint8_t payload[] = { 9, 8, 7 };
|
|
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload),
|
|
7 /* prev_log_term of LSN 4 */);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
TEST_ASSERT_EQUAL_INT(1, g_hook_call_count);
|
|
TEST_ASSERT_EQUAL_UINT64(5, g_hook_last_lsn);
|
|
/* The load-bearing assertion: the leader's real term is on the wire entry
|
|
* (NOT 0). This FAILS before the task 3.4 cutover. */
|
|
TEST_ASSERT_EQUAL_UINT64(7, g_hook_last_term);
|
|
/* prev tuple: prev_log_index == lsn - 1; prev_log_term == supplied. */
|
|
TEST_ASSERT_EQUAL_UINT64(4, g_hook_last_prev_log_index);
|
|
TEST_ASSERT_EQUAL_UINT64(7, g_hook_last_prev_log_term);
|
|
|
|
repl_broadcast_send_hook = NULL;
|
|
repl_fini(&r);
|
|
printf("test_broadcast_carries_leader_term_and_prev_tuple: PASS\n");
|
|
#else
|
|
printf("test_broadcast_carries_leader_term_and_prev_tuple: "
|
|
"SKIPPED (no XCE_TESTING)\n");
|
|
#endif
|
|
}
|
|
|
|
/* Test: standby in catch-up is skipped — hook not called for it */
|
|
static void test_broadcast_skips_catchup_standby(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
replicator_t r;
|
|
int ep1 = 0, ep2 = 1;
|
|
repl_init(&r);
|
|
repl_add_standby(&r, (void *)&ep1, "sb1", 7001);
|
|
repl_add_standby(&r, (void *)&ep2, "sb2", 7002);
|
|
/* Mark sb1 as in catch-up */
|
|
repl_start_catchup(&r, (void *)&ep1, 0);
|
|
reset_hook_state();
|
|
repl_broadcast_send_hook = test_broadcast_hook;
|
|
|
|
wal_entry_hdr_t hdr = { .lsn = 99, .type = WAL_JOB_SUBMIT };
|
|
uint8_t payload[] = { 0xFF };
|
|
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload), 0 /* prev_log_term */);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
/* Only sb2 should be sent to */
|
|
TEST_ASSERT_EQUAL_INT(1, g_hook_call_count);
|
|
TEST_ASSERT_EQUAL_PTR((void *)&ep2, g_hook_last_ep);
|
|
|
|
repl_broadcast_send_hook = NULL;
|
|
repl_fini(&r);
|
|
printf("test_broadcast_skips_catchup_standby: PASS\n");
|
|
#else
|
|
printf("test_broadcast_skips_catchup_standby: SKIPPED (no XCE_TESTING)\n");
|
|
#endif
|
|
}
|
|
|
|
/* Test: standby with pending_ack_count > MAX_PENDING_ACKS is evicted to catch-up */
|
|
static void test_broadcast_evicts_overloaded_standby(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
replicator_t r;
|
|
int dummy_ep = 0;
|
|
repl_init(&r);
|
|
int idx = repl_add_standby(&r, (void *)&dummy_ep, "sb1", 7001);
|
|
/* Manually set pending_ack_count just above the limit */
|
|
r.standbys[idx].pending_ack_count = MAX_PENDING_ACKS + 1;
|
|
reset_hook_state();
|
|
repl_broadcast_send_hook = test_broadcast_hook;
|
|
|
|
wal_entry_hdr_t hdr = { .lsn = 5, .type = WAL_JOB_SUBMIT };
|
|
uint8_t payload[] = { 0x01 };
|
|
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload), 0 /* prev_log_term */);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
/* Evicted — hook must NOT be called */
|
|
TEST_ASSERT_EQUAL_INT(0, g_hook_call_count);
|
|
/* Standby must now be in catch-up */
|
|
TEST_ASSERT_NOT_EQUAL(0,
|
|
atomic_load_explicit(&r.standbys[idx].in_catch_up, memory_order_acquire));
|
|
|
|
repl_broadcast_send_hook = NULL;
|
|
repl_fini(&r);
|
|
printf("test_broadcast_evicts_overloaded_standby: PASS\n");
|
|
#else
|
|
printf("test_broadcast_evicts_overloaded_standby: SKIPPED (no XCE_TESTING)\n");
|
|
#endif
|
|
}
|
|
|
|
/* Test (Task 2.3): repl_add_standby is idempotent for the same ep.
|
|
* Calling it twice with the same endpoint must return the same index,
|
|
* keep nstandbys==1, and not crash or corrupt state.
|
|
* Req 1.3: If repl_add_standby is called with an endpoint already
|
|
* registered, the system shall treat it as idempotent (no-op or update-in-place). */
|
|
static void test_repl_add_standby_idempotent(void) {
|
|
replicator_t r;
|
|
int dummy_ep = 0;
|
|
repl_init(&r);
|
|
|
|
int idx1 = repl_add_standby(&r, (void *)&dummy_ep, "sb1", 7001);
|
|
TEST_ASSERT_TRUE(idx1 >= 0);
|
|
TEST_ASSERT_EQUAL_INT(1, r.nstandbys);
|
|
|
|
/* Second call with the same ep — must be a no-op */
|
|
int idx2 = repl_add_standby(&r, (void *)&dummy_ep, "sb1", 7001);
|
|
TEST_ASSERT_TRUE(idx2 >= 0);
|
|
/* nstandbys must still be 1 (not 2) */
|
|
TEST_ASSERT_EQUAL_INT(1, r.nstandbys);
|
|
/* Must return the same slot index */
|
|
TEST_ASSERT_EQUAL_INT(idx1, idx2);
|
|
|
|
repl_fini(&r);
|
|
printf("test_repl_add_standby_idempotent: PASS\n");
|
|
}
|
|
|
|
/* ============================================================
|
|
* Task 4.1: WAL ACK reply handler — Req 4.3
|
|
*
|
|
* These tests verify the behaviour that repl_ack_decode_and_dispatch
|
|
* implements in ha.c: on a successful WalAckReply it calls
|
|
* repl_on_ack(&c->persist.repl, ep, acked_lsn), which must:
|
|
* a) update standby's acked_lsn
|
|
* b) signal ack_cond so repl_wait_majority_ack unblocks
|
|
*
|
|
* We test repl_on_ack directly (the exact call the handler makes).
|
|
* ============================================================ */
|
|
|
|
/* ACK sender thread args reused from above — ack_args struct already defined */
|
|
|
|
/* Test: repl_on_ack (handler ACK path) updates acked_lsn and unblocks
|
|
* repl_wait_majority_ack. Req 4.3: when the primary receives a WAL ACK
|
|
* from a standby it shall update that standby's acked_lsn in the replicator
|
|
* and signal any waiter on the ACK condition variable. */
|
|
static void test_wal_ack_reply_updates_acked_lsn_and_signals(void)
|
|
{
|
|
replicator_t r;
|
|
int dummy_ep = 0;
|
|
repl_init(&r);
|
|
repl_add_standby(&r, (void *)&dummy_ep, "sb1", 7001);
|
|
|
|
const uint64_t target_lsn = 77;
|
|
|
|
/* Background thread simulates the handler calling repl_on_ack after 30 ms */
|
|
struct ack_args args = { &r, (void *)&dummy_ep, target_lsn, 30 };
|
|
pthread_t th;
|
|
pthread_create(&th, NULL, ack_sender_thread, &args);
|
|
|
|
/* repl_wait_majority_ack: 1 active standby, quorum=(1+2)/2=1,
|
|
* acked_total = 1(self) + 1(standby) = 2 >= 1 → returns 0. */
|
|
int rc = repl_wait_majority_ack(&r, 0, 200, (uint8_t)OPCode_walAck);
|
|
pthread_join(th, NULL);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
|
|
"repl_wait_majority_ack must return 0 after repl_on_ack signals (Req 4.3)");
|
|
|
|
/* Verify acked_lsn was actually updated in the slot */
|
|
int found = 0;
|
|
for (int i = 0; i < MAX_STANDBYS; i++) {
|
|
if (r.standbys[i].ep == (void *)&dummy_ep) {
|
|
uint64_t al = atomic_load_explicit(&r.standbys[i].acked_lsn,
|
|
memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(target_lsn, al,
|
|
"acked_lsn must equal the LSN passed to repl_on_ack (Req 4.3)");
|
|
found = 1;
|
|
break;
|
|
}
|
|
}
|
|
TEST_ASSERT_TRUE_MESSAGE(found, "standby slot must be present after repl_on_ack");
|
|
|
|
repl_fini(&r);
|
|
printf("test_wal_ack_reply_updates_acked_lsn_and_signals: PASS\n");
|
|
}
|
|
|
|
/* Test: NACK path — handler does NOT call repl_on_ack, so acked_lsn stays 0.
|
|
* Req 4.3 (negative): only a success==1 reply triggers the update; success==0
|
|
* is a no-op on the replicator.
|
|
*
|
|
* Quorum analysis for this scenario:
|
|
* n_active = 1 (one standby registered, not in catch-up)
|
|
* quorum = (1 + 2) / 2 = 1
|
|
* walAck: acked_total = 1 (self) + 0 (standby acked_lsn == 0, not > 0) = 1
|
|
* 1 >= quorum(1) → repl_wait_majority_ack returns 0 IMMEDIATELY.
|
|
*
|
|
* The primary self-count alone satisfies a 1-standby quorum, so the function
|
|
* does NOT time out even when the standby never ACKs. The NACK invariant is
|
|
* that acked_lsn remains 0 (not that the wait times out).
|
|
*/
|
|
static void test_wal_nack_does_not_update_acked_lsn(void)
|
|
{
|
|
replicator_t r;
|
|
int dummy_ep = 0;
|
|
repl_init(&r);
|
|
repl_add_standby(&r, (void *)&dummy_ep, "sb1", 7001);
|
|
|
|
/* No repl_on_ack call — simulates handler receiving success==0 NACK */
|
|
|
|
/* acked_lsn must remain 0 */
|
|
uint64_t al = 0;
|
|
for (int i = 0; i < MAX_STANDBYS; i++) {
|
|
if (r.standbys[i].ep == (void *)&dummy_ep) {
|
|
al = atomic_load_explicit(&r.standbys[i].acked_lsn,
|
|
memory_order_acquire);
|
|
break;
|
|
}
|
|
}
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, al,
|
|
"acked_lsn must remain 0 when NACK path skips repl_on_ack (Req 4.3)");
|
|
|
|
/* With 1 standby and no standby ACK, primary self-count (1) already meets
|
|
* quorum (1), so repl_wait_majority_ack returns 0, not -ETIMEDOUT.
|
|
* A timeout would only occur if quorum required at least one standby ACK,
|
|
* which only happens when n_active >= 2 (e.g. 2+ standbys registered). */
|
|
int rc = repl_wait_majority_ack(&r, 0, 50, (uint8_t)OPCode_walAck);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
|
|
"repl_wait_majority_ack must return 0 with 1 standby: "
|
|
"self-count(1) >= quorum(1) even without standby ACK (Req 4.3)");
|
|
|
|
repl_fini(&r);
|
|
printf("test_wal_nack_does_not_update_acked_lsn: PASS\n");
|
|
}
|
|
|
|
/* Test: send failure (hook returns -EIO) is tolerated — broadcast returns 0 */
|
|
static void test_broadcast_send_failure_continues(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
replicator_t r;
|
|
int ep1 = 0, ep2 = 1;
|
|
repl_init(&r);
|
|
repl_add_standby(&r, (void *)&ep1, "sb1", 7001);
|
|
repl_add_standby(&r, (void *)&ep2, "sb2", 7002);
|
|
reset_hook_state();
|
|
g_hook_return_val = -EIO; /* First call fails */
|
|
repl_broadcast_send_hook = test_broadcast_hook;
|
|
|
|
wal_entry_hdr_t hdr = { .lsn = 77, .type = WAL_JOB_SUBMIT };
|
|
uint8_t payload[] = { 0xDE, 0xAD };
|
|
int rc = repl_broadcast(&r, &hdr, payload, sizeof(payload), 0 /* prev_log_term */);
|
|
|
|
/* broadcast should still return 0 — errors are WARN+continue */
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
/* Both standbys attempted */
|
|
TEST_ASSERT_EQUAL_INT(2, g_hook_call_count);
|
|
|
|
repl_broadcast_send_hook = NULL;
|
|
repl_fini(&r);
|
|
printf("test_broadcast_send_failure_continues: PASS\n");
|
|
#else
|
|
printf("test_broadcast_send_failure_continues: SKIPPED (no XCE_TESTING)\n");
|
|
#endif
|
|
}
|
|
|
|
/* ============================================================
|
|
* Task 5.3: Registration lifecycle — Req 1.1, 1.2, 1.3
|
|
*
|
|
* Comprehensive test covering all three lifecycle steps in sequence:
|
|
* Step 1 (Req 1.1): Register ep → nstandbys==1, standbys[idx].ep==ep
|
|
* Step 2 (Req 1.3): Re-register same ep → idempotent, nstandbys==1, same idx
|
|
* Step 3 (Req 1.2): Deregister ep → nstandbys==0, standbys[idx].ep==NULL
|
|
* ============================================================ */
|
|
static void test_repl_registration_lifecycle(void) {
|
|
replicator_t r;
|
|
int dummy_ep = 0;
|
|
repl_init(&r);
|
|
|
|
/* Step 1: Register ep — Req 1.1 */
|
|
int idx = repl_add_standby(&r, (void *)&dummy_ep, "sb1", 7001);
|
|
TEST_ASSERT_TRUE_MESSAGE(idx >= 0, "repl_add_standby must return a valid slot index");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, r.nstandbys,
|
|
"nstandbys must be 1 after first registration (Req 1.1)");
|
|
TEST_ASSERT_EQUAL_PTR_MESSAGE((void *)&dummy_ep, r.standbys[idx].ep,
|
|
"standbys[idx].ep must equal registered ep (Req 1.1)");
|
|
|
|
/* Step 2: Re-register same ep — idempotent, Req 1.3 */
|
|
int idx2 = repl_add_standby(&r, (void *)&dummy_ep, "sb1", 7001);
|
|
TEST_ASSERT_TRUE_MESSAGE(idx2 >= 0, "duplicate repl_add_standby must return valid index");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, r.nstandbys,
|
|
"nstandbys must remain 1 after duplicate registration (Req 1.3)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(idx, idx2,
|
|
"duplicate registration must return the same slot index (Req 1.3)");
|
|
|
|
/* Step 3: Deregister ep — Req 1.2 */
|
|
repl_remove_standby(&r, (void *)&dummy_ep);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, r.nstandbys,
|
|
"nstandbys must be 0 after deregistration (Req 1.2)");
|
|
TEST_ASSERT_NULL_MESSAGE(r.standbys[idx].ep,
|
|
"standbys[idx].ep must be NULL after deregistration (Req 1.2)");
|
|
|
|
repl_fini(&r);
|
|
printf("test_repl_registration_lifecycle: PASS\n");
|
|
}
|
|
|
|
/* ============================================================
|
|
* bug-1474: repl_add_standby slot allocation must never clobber an
|
|
* unexpired bug-1147 disconnect guard (last_disconnected_lsn) belonging to
|
|
* a DIFFERENT, still-disconnected peer, when a genuinely virgin
|
|
* (never-used) slot is available. See .wolf/buglog.json bug-1474 for the
|
|
* full forensic trace: a spurious/duplicate reconnect for an unrelated,
|
|
* already-connected peer picked the lower-indexed, guard-holding slot over
|
|
* a higher-indexed virgin slot and memset it to zero, silently destroying
|
|
* the guard repl_trim_lsn relies on to protect WAL segments a genuinely
|
|
* disconnected-but-reconnecting standby still needs.
|
|
* ============================================================ */
|
|
|
|
/* RED (pre-fix): repl_add_standby's old "first empty slot by lowest index"
|
|
* scan picks idx_a (guard-holding, index 0) over the virgin slot (index 2)
|
|
* purely because of index order, destroying A's guard. Req: the guard must
|
|
* survive and the new peer must land in the virgin slot instead. */
|
|
static void test_repl_add_standby_prefers_virgin_slot_over_guard(void) {
|
|
replicator_t r;
|
|
int ep_a = 0, ep_b = 1, ep_c = 2;
|
|
repl_init(&r);
|
|
|
|
int idx_a = repl_add_standby(&r, (void *)&ep_a, "standby-a", 7001);
|
|
int idx_b = repl_add_standby(&r, (void *)&ep_b, "standby-b", 7002);
|
|
TEST_ASSERT_TRUE(idx_a >= 0);
|
|
TEST_ASSERT_TRUE(idx_b >= 0);
|
|
TEST_ASSERT_NOT_EQUAL(idx_a, idx_b);
|
|
|
|
/* A acks up to LSN 42, then disconnects: leaves a bug-1147 guard at
|
|
* idx_a with last_disconnected_lsn == 42. The third slot (whichever
|
|
* index neither idx_a nor idx_b used) remains genuinely virgin. */
|
|
repl_on_ack(&r, (void *)&ep_a, 42);
|
|
repl_remove_standby(&r, (void *)&ep_a);
|
|
TEST_ASSERT_NULL(r.standbys[idx_a].ep);
|
|
TEST_ASSERT_EQUAL_UINT64(42, r.standbys[idx_a].last_disconnected_lsn);
|
|
|
|
/* A third, genuinely NEW peer (unrelated to A) connects while idx_a's
|
|
* guard is still unexpired and a virgin slot is also available. */
|
|
int idx_c = repl_add_standby(&r, (void *)&ep_c, "standby-c", 7003);
|
|
TEST_ASSERT_TRUE(idx_c >= 0);
|
|
|
|
/* The new peer must land in the virgin slot, NOT the guard-holding one,
|
|
* even though idx_a may be the lower index. */
|
|
TEST_ASSERT_NOT_EQUAL_MESSAGE(idx_a, idx_c,
|
|
"new peer must not be allocated the guard-holding slot when a"
|
|
" virgin slot exists (bug-1474)");
|
|
|
|
/* Load-bearing assertion: A's guard must survive completely untouched.
|
|
* Before the fix, repl_add_standby's unconditional memset(s, 0, ...)
|
|
* on the chosen slot would have wiped last_disconnected_lsn to 0 here. */
|
|
TEST_ASSERT_NULL_MESSAGE(r.standbys[idx_a].ep,
|
|
"guard-holding slot must remain disconnected (bug-1474)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(42, r.standbys[idx_a].last_disconnected_lsn,
|
|
"bug-1147 guard LSN must survive an unrelated peer's"
|
|
" repl_add_standby call (bug-1474)");
|
|
|
|
repl_fini(&r);
|
|
printf("test_repl_add_standby_prefers_virgin_slot_over_guard: PASS\n");
|
|
}
|
|
|
|
/* RED (pre-fix, and proves the DELIBERATE fallback policy): with
|
|
* MAX_STANDBYS==3, exhaust all three slots, then disconnect two of them
|
|
* (both now guard-holding, one older-disconnected than the other) so NO
|
|
* virgin slot remains. A fourth genuinely new peer must still be admitted
|
|
* (never -ENOSPC for this case -- refusing a legitimate reconnect would be
|
|
* worse for cluster availability than an early guard loss), and the
|
|
* fallback must deterministically reuse the OLDER-disconnected guard slot
|
|
* (closest to its own natural repl_expire_disconnect_slots expiry, so
|
|
* reusing it discards the LEAST remaining WAL-retention protection) --
|
|
* not merely the lowest-indexed guard slot. */
|
|
static void test_repl_add_standby_exhausted_guards_prefers_oldest_disconnect(void) {
|
|
replicator_t r;
|
|
int ep_a = 0, ep_b = 1, ep_c = 2, ep_d = 3;
|
|
repl_init(&r);
|
|
|
|
int idx_a = repl_add_standby(&r, (void *)&ep_a, "standby-a", 7001);
|
|
int idx_b = repl_add_standby(&r, (void *)&ep_b, "standby-b", 7002);
|
|
int idx_c = repl_add_standby(&r, (void *)&ep_c, "standby-c", 7003);
|
|
TEST_ASSERT_TRUE(idx_a >= 0);
|
|
TEST_ASSERT_TRUE(idx_b >= 0);
|
|
TEST_ASSERT_TRUE(idx_c >= 0);
|
|
/* All MAX_STANDBYS(3) slots now occupied. */
|
|
|
|
/* B disconnects first; force its disconnect timestamp strictly OLDER
|
|
* than A's (below) so the two guards are unambiguously ordered by
|
|
* time, independent of their slot index (idx_a < idx_b by add order --
|
|
* deliberately the OPPOSITE of the time order, so a naive
|
|
* "lowest-index guard slot" fallback would pick the wrong one). */
|
|
repl_on_ack(&r, (void *)&ep_b, 20);
|
|
repl_remove_standby(&r, (void *)&ep_b);
|
|
r.standbys[idx_b].last_disconnect_time -= 100;
|
|
|
|
/* A disconnects second, with its real (newer) "now" timestamp. */
|
|
repl_on_ack(&r, (void *)&ep_a, 10);
|
|
repl_remove_standby(&r, (void *)&ep_a);
|
|
|
|
TEST_ASSERT_TRUE_MESSAGE(
|
|
r.standbys[idx_b].last_disconnect_time < r.standbys[idx_a].last_disconnect_time,
|
|
"test setup invariant: B must be the OLDER disconnect");
|
|
|
|
/* C is still connected; nstandbys == 1 (< MAX_STANDBYS), so the
|
|
* "too many distinct live peers" -ENOSPC guard does not trip -- but
|
|
* BOTH empty slots (idx_a, idx_b) are guard-holding: no virgin slot
|
|
* exists anywhere in the registry. */
|
|
TEST_ASSERT_EQUAL_INT(1, r.nstandbys);
|
|
|
|
/* A fourth, genuinely new peer connects into the exhausted state. */
|
|
int idx_d = repl_add_standby(&r, (void *)&ep_d, "standby-d", 7004);
|
|
TEST_ASSERT_TRUE_MESSAGE(idx_d >= 0,
|
|
"exhausted-guard fallback must still admit a legitimate new peer,"
|
|
" never -ENOSPC (bug-1474 policy)");
|
|
|
|
/* Deliberate policy: reuse the OLDER-disconnected guard slot (idx_b),
|
|
* not idx_a (which is both newer-disconnected AND lower-indexed --
|
|
* ruling out an accidental "first guard by index" fallthrough). */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(idx_b, idx_d,
|
|
"exhausted fallback must reuse the OLDEST-disconnected guard slot,"
|
|
" not merely the lowest-indexed one (bug-1474 policy)");
|
|
|
|
/* idx_a's guard (the one NOT chosen) must remain completely untouched. */
|
|
TEST_ASSERT_NULL_MESSAGE(r.standbys[idx_a].ep,
|
|
"non-selected guard slot must remain disconnected");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(10, r.standbys[idx_a].last_disconnected_lsn,
|
|
"non-selected guard slot's LSN must survive untouched");
|
|
|
|
repl_fini(&r);
|
|
printf("test_repl_add_standby_exhausted_guards_prefers_oldest_disconnect: PASS\n");
|
|
}
|
|
|
|
void test_repl_unit(void) {
|
|
UnitySetTestFile(__FILE__);
|
|
RUN_TEST(test_repl_init_zeroes_nstandbys);
|
|
RUN_TEST(test_repl_add_remove_standby);
|
|
RUN_TEST(test_repl_add_standby_max);
|
|
RUN_TEST(test_repl_trim_lsn_no_standbys);
|
|
RUN_TEST(test_repl_trim_lsn_disconnected_slot_expiry);
|
|
RUN_TEST(test_wait_ack_no_standbys_returns_enodev);
|
|
RUN_TEST(test_wait_ack_already_acked);
|
|
RUN_TEST(test_wait_ack_timeout);
|
|
RUN_TEST(test_wait_ack_woken_by_background_ack);
|
|
/* Task 2.3: idempotent re-registration */
|
|
RUN_TEST(test_repl_add_standby_idempotent);
|
|
/* Task 1.1: broadcast send path */
|
|
RUN_TEST(test_broadcast_no_standbys_returns_zero);
|
|
RUN_TEST(test_broadcast_one_standby_sends_entry);
|
|
RUN_TEST(test_broadcast_carries_leader_term_and_prev_tuple);
|
|
RUN_TEST(test_broadcast_skips_catchup_standby);
|
|
RUN_TEST(test_broadcast_evicts_overloaded_standby);
|
|
RUN_TEST(test_broadcast_send_failure_continues);
|
|
/* Task 4.1: WAL ACK reply handler — Req 4.3 */
|
|
RUN_TEST(test_wal_ack_reply_updates_acked_lsn_and_signals);
|
|
RUN_TEST(test_wal_nack_does_not_update_acked_lsn);
|
|
/* Task 5.3: registration lifecycle — Req 1.1, 1.2, 1.3 */
|
|
RUN_TEST(test_repl_registration_lifecycle);
|
|
/* bug-1474: slot allocation must not clobber an unexpired guard */
|
|
RUN_TEST(test_repl_add_standby_prefers_virgin_slot_over_guard);
|
|
RUN_TEST(test_repl_add_standby_exhausted_guards_prefers_oldest_disconnect);
|
|
}
|