6.3: Fix single-node bug — ha_compute_safe_committed_lsn now returns self_lsn instead of 0 for n_peers=0 (prevents WAL truncation on single-node re-election). Add self_lsn/self_tag to ha_elect_ctx_t, populated unconditionally in ha_start_election_async. All 3 test scenarios pass under ASAN. 6.4: Add test_snapshot_recv_128kb_two_chunks (128KB snapshot in 2x64KB chunks, verifies size and applied_lsn); 4 other test cases already existed from tasks 3.5c/3.5d. All ASAN-clean.
749 lines
29 KiB
C
749 lines
29 KiB
C
/*
|
|
* test_ha_evaluate_committing.c — Unity tests for Task 4.1:
|
|
* ha_elect_evaluate_and_signal COMMITTING path.
|
|
*
|
|
* Tests verify:
|
|
* Req 4.1: quorum reached → ha->role transitions to CTRL_PRIMARY_COMMITTING
|
|
* (not CTRL_PRIMARY) after ha_elect_evaluate_and_signal
|
|
* Req 5.2: COMMITTING path: ha->repl_ack_count reset to 0 when entering COMMITTING
|
|
* Req 9.2: cluster_init_phase2 called exactly once on first election;
|
|
* second election (RDHUP) skips it (phase2_done=1)
|
|
*
|
|
* Design ref: design.md "ha_elect_evaluate_and_signal (extended)"
|
|
* Boundary: ha.c (election evaluation)
|
|
* Requirements: 4.1, 5.2, 9.2
|
|
*/
|
|
#define _GNU_SOURCE
|
|
#include <errno.h>
|
|
#include <semaphore.h>
|
|
#include <stdatomic.h>
|
|
#include <stdbool.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <time.h>
|
|
|
|
#include <unity/unity.h>
|
|
|
|
#include "ha.h"
|
|
#include "xce/cluster_def.h" /* full cluster_t layout for committing_timeout_fn tests */
|
|
|
|
/* ── Test-only seam: cluster_init_phase2 call counter ───────────────────────
|
|
*
|
|
* In production, ha_elect_evaluate_and_signal calls cluster_init_phase2(c)
|
|
* when !ha->phase2_done. For unit tests we cannot provide a real cluster_t,
|
|
* so we hook via an XCE_TESTING seam in ha.c: the evaluate function checks
|
|
* ha_phase2_hook; if non-NULL it calls the hook instead of cluster_init_phase2.
|
|
*
|
|
* This avoids pulling in the full cluster_t dependency in the test binary.
|
|
*/
|
|
extern int (*ha_phase2_hook)(void *c);
|
|
|
|
/* Call counter for the hook */
|
|
static int g_phase2_call_count = 0;
|
|
static int g_phase2_hook_rc = 0; /* return value for hook (0 = success) */
|
|
|
|
static int stub_cluster_init_phase2(void *c)
|
|
{
|
|
(void)c;
|
|
g_phase2_call_count++;
|
|
return g_phase2_hook_rc;
|
|
}
|
|
|
|
/* ── Minimal ha_config_t for tests ─────────────────────────────────────── */
|
|
|
|
static ha_config_t g_cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 10000,
|
|
.election_timeout_ms = 2000,
|
|
.probe_connect_timeout_ms = 500,
|
|
.max_promotion_retries = 5,
|
|
.election_jitter_max_ms = 0,
|
|
.election_priority_step_ms = 500,
|
|
.committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
|
|
};
|
|
|
|
/* ── Helper: build a minimal ha_elect_ctx_t for n_other peers ───────────── */
|
|
|
|
static ha_elect_ctx_t *make_ctx(int n_other, int votes_granted)
|
|
{
|
|
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) +
|
|
(size_t)n_other * sizeof(ctx->votes[0]));
|
|
if (!ctx) return NULL;
|
|
ctx->n_peers = n_other;
|
|
ctx->votes_granted = votes_granted; /* includes self-vote */
|
|
ctx->n_replied = n_other;
|
|
ctx->max_resp_tag = 0;
|
|
ctx->max_resp_lsn = 0;
|
|
ctx->result = -1;
|
|
ctx->done_posted = 0;
|
|
ctx->c = NULL; /* no real cluster in unit tests */
|
|
pthread_mutex_init(&ctx->lock, NULL);
|
|
sem_init(&ctx->done, 0, 0);
|
|
/* Store my_tag/my_lsn in votes[0] for yield gate (only if n_other > 0) */
|
|
if (n_other > 0) {
|
|
ctx->votes[0].my_tag = 100;
|
|
ctx->votes[0].my_lsn = 50;
|
|
}
|
|
return ctx;
|
|
}
|
|
|
|
static void free_ctx(ha_elect_ctx_t *ctx)
|
|
{
|
|
if (!ctx) return;
|
|
pthread_mutex_destroy(&ctx->lock);
|
|
sem_destroy(&ctx->done);
|
|
free(ctx);
|
|
}
|
|
|
|
/* ── Test setup/teardown ─────────────────────────────────────────────────── */
|
|
|
|
static void reset_counters(void)
|
|
{
|
|
g_phase2_call_count = 0;
|
|
g_phase2_hook_rc = 0;
|
|
/* Wire up the test seam */
|
|
ha_phase2_hook = stub_cluster_init_phase2;
|
|
}
|
|
|
|
static void clear_hook(void)
|
|
{
|
|
ha_phase2_hook = NULL;
|
|
}
|
|
|
|
/* ── Test 1: winning quorum → CTRL_PRIMARY_COMMITTING (not PRIMARY) ─────── */
|
|
|
|
static void test_winning_quorum_enters_committing(void)
|
|
{
|
|
/* 3-node cluster: n_other=2, quorum=2. Self-vote + 2 peer grants = 3.
|
|
* ha_elect_evaluate_and_signal should detect quorum → COMMITTING. */
|
|
reset_counters();
|
|
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_cfg, 0);
|
|
ha.phase2_done = 0; /* first boot */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
|
|
|
|
ha_elect_ctx_t *ctx = make_ctx(2, 3); /* votes_granted=3 (self + 2 peers) */
|
|
TEST_ASSERT_NOT_NULL_MESSAGE(ctx, "make_ctx failed");
|
|
/* Set votes[0] my_tag/my_lsn for yield gate */
|
|
ctx->votes[0].my_tag = 100;
|
|
ctx->votes[0].my_lsn = 50;
|
|
|
|
ctrl_role_t result = ha_elect_evaluate_and_signal(&ha, ctx);
|
|
|
|
/* Function must return CTRL_PRIMARY (the election result) */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, (int)result,
|
|
"ha_elect_evaluate_and_signal must return CTRL_PRIMARY when quorum won");
|
|
|
|
/* But ha->role must be CTRL_PRIMARY_COMMITTING (Req 4.1) */
|
|
ctrl_role_t actual_role =
|
|
(ctrl_role_t)atomic_load_explicit(&ha.role, memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, (int)actual_role,
|
|
"ha->role must be CTRL_PRIMARY_COMMITTING after winning election (not CTRL_PRIMARY)");
|
|
|
|
free_ctx(ctx);
|
|
ha_fini(&ha);
|
|
clear_hook();
|
|
printf("test_winning_quorum_enters_committing: PASS (role=%d==CTRL_PRIMARY_COMMITTING)\n",
|
|
(int)actual_role);
|
|
}
|
|
|
|
/* ── Test 2: first boot — phase2 called exactly once ────────────────────── */
|
|
|
|
static void test_first_boot_phase2_called_once(void)
|
|
{
|
|
/* phase2_done=0: cluster_init_phase2 must be called exactly once. */
|
|
reset_counters();
|
|
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_cfg, 0);
|
|
ha.phase2_done = 0;
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
|
|
|
|
/* Single-node-like: n_other=0, quorum=1, votes_granted=1 → wins */
|
|
ha_elect_ctx_t *ctx = make_ctx(0, 1);
|
|
TEST_ASSERT_NOT_NULL_MESSAGE(ctx, "make_ctx failed");
|
|
|
|
ha_elect_evaluate_and_signal(&ha, ctx);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_phase2_call_count,
|
|
"cluster_init_phase2 must be called exactly once on first boot");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.phase2_done,
|
|
"ha->phase2_done must be set to 1 after cluster_init_phase2");
|
|
|
|
free_ctx(ctx);
|
|
ha_fini(&ha);
|
|
clear_hook();
|
|
printf("test_first_boot_phase2_called_once: PASS (calls=%d)\n", g_phase2_call_count);
|
|
}
|
|
|
|
/* ── Test 3: second election (RDHUP) skips phase2 ───────────────────────── */
|
|
|
|
static void test_rdhup_second_election_skips_phase2(void)
|
|
{
|
|
/* phase2_done=1: cluster_init_phase2 must NOT be called. */
|
|
reset_counters();
|
|
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_cfg, 0);
|
|
ha.phase2_done = 1; /* already initialized (post-RDHUP failover) */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
|
|
|
|
ha_elect_ctx_t *ctx = make_ctx(0, 1); /* single-node wins again */
|
|
TEST_ASSERT_NOT_NULL_MESSAGE(ctx, "make_ctx failed");
|
|
|
|
ha_elect_evaluate_and_signal(&ha, ctx);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_phase2_call_count,
|
|
"cluster_init_phase2 must NOT be called when phase2_done=1 (RDHUP failover)");
|
|
|
|
free_ctx(ctx);
|
|
ha_fini(&ha);
|
|
clear_hook();
|
|
printf("test_rdhup_second_election_skips_phase2: PASS (calls=%d)\n", g_phase2_call_count);
|
|
}
|
|
|
|
/* ── Test 4: COMMITTING path resets repl_ack_count ──────────────────────── */
|
|
|
|
static void test_committing_resets_repl_ack_count(void)
|
|
{
|
|
/* Req 5.2: repl_ack_count must be 0 when entering COMMITTING.
|
|
* Start with a non-zero value to verify it gets reset. */
|
|
reset_counters();
|
|
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_cfg, 0);
|
|
ha.phase2_done = 1; /* skip phase2 call */
|
|
atomic_store_explicit(&ha.repl_ack_count, 3, memory_order_release); /* stale count */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
|
|
|
|
ha_elect_ctx_t *ctx = make_ctx(0, 1); /* wins */
|
|
TEST_ASSERT_NOT_NULL_MESSAGE(ctx, "make_ctx failed");
|
|
|
|
ha_elect_evaluate_and_signal(&ha, ctx);
|
|
|
|
int ack_count = atomic_load_explicit(&ha.repl_ack_count, memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ack_count,
|
|
"ha->repl_ack_count must be reset to 0 when entering CTRL_PRIMARY_COMMITTING");
|
|
|
|
free_ctx(ctx);
|
|
ha_fini(&ha);
|
|
clear_hook();
|
|
printf("test_committing_resets_repl_ack_count: PASS (repl_ack_count=%d)\n", ack_count);
|
|
}
|
|
|
|
/* ── Test 5: losing election → CTRL_STANDBY, phase2 called on first boot ── */
|
|
|
|
static void test_losing_election_enters_standby(void)
|
|
{
|
|
/* n_other=2, quorum=2, votes_granted=1 (self only) → loses.
|
|
* On first boot, phase2 must still be called; role → CTRL_STANDBY. */
|
|
reset_counters();
|
|
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_cfg, 0);
|
|
ha.phase2_done = 0; /* first boot */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
|
|
|
|
ha_elect_ctx_t *ctx = make_ctx(2, 1); /* votes_granted=1, quorum=2 → loses */
|
|
TEST_ASSERT_NOT_NULL_MESSAGE(ctx, "make_ctx failed");
|
|
|
|
ctrl_role_t result = ha_elect_evaluate_and_signal(&ha, ctx);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY, (int)result,
|
|
"ha_elect_evaluate_and_signal must return CTRL_STANDBY when quorum not won");
|
|
|
|
ctrl_role_t actual_role =
|
|
(ctrl_role_t)atomic_load_explicit(&ha.role, memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY, (int)actual_role,
|
|
"ha->role must be CTRL_STANDBY after losing election");
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_phase2_call_count,
|
|
"cluster_init_phase2 must be called once even for losing standby (first boot)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.phase2_done,
|
|
"ha->phase2_done must be set to 1 after losing first-boot election");
|
|
|
|
free_ctx(ctx);
|
|
ha_fini(&ha);
|
|
clear_hook();
|
|
printf("test_losing_election_enters_standby: PASS (role=%d==CTRL_STANDBY, phase2=%d)\n",
|
|
(int)actual_role, g_phase2_call_count);
|
|
}
|
|
|
|
/* ── Test 6: election timeout fires evaluate → COMMITTING if wins ────────── */
|
|
|
|
static void test_election_timeout_fn_fires_evaluate(void)
|
|
{
|
|
/* election_timeout_fn calls ha_elect_evaluate_and_signal.
|
|
* Use election_timeout_fn_for_test (XCE_TESTING shim in ha.c):
|
|
* election_timeout_fn_for_test(ha, ctx, cancelled=0) → fires evaluate.
|
|
* election_timeout_fn_for_test(ha, ctx, cancelled=1) → no-op.
|
|
*/
|
|
reset_counters();
|
|
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_cfg, 0);
|
|
ha.phase2_done = 1; /* skip phase2 */
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
|
|
|
|
ha_elect_ctx_t *ctx = make_ctx(0, 1); /* single-node wins */
|
|
TEST_ASSERT_NOT_NULL_MESSAGE(ctx, "make_ctx failed");
|
|
|
|
/* Use the test-shim to simulate the timeout firing with cancelled=0 */
|
|
extern int election_timeout_fn_for_test(ha_state_t *ha, ha_elect_ctx_t *ctx,
|
|
int cancelled_val);
|
|
election_timeout_fn_for_test(&ha, ctx, 0 /* not cancelled */);
|
|
|
|
ctrl_role_t role =
|
|
(ctrl_role_t)atomic_load_explicit(&ha.role, memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, (int)role,
|
|
"election_timeout_fn (not cancelled) must drive ha->role to CTRL_PRIMARY_COMMITTING");
|
|
|
|
free_ctx(ctx);
|
|
ha_fini(&ha);
|
|
clear_hook();
|
|
printf("test_election_timeout_fn_fires_evaluate: PASS (role=%d)\n", (int)role);
|
|
}
|
|
|
|
/* ── Test 7: cancelled election timeout → no role change ────────────────── */
|
|
|
|
static void test_cancelled_timeout_is_noop(void)
|
|
{
|
|
reset_counters();
|
|
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_cfg, 0);
|
|
ha.phase2_done = 1;
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
|
|
|
|
ha_elect_ctx_t *ctx = make_ctx(0, 1);
|
|
TEST_ASSERT_NOT_NULL_MESSAGE(ctx, "make_ctx failed");
|
|
|
|
extern int election_timeout_fn_for_test(ha_state_t *ha, ha_elect_ctx_t *ctx,
|
|
int cancelled_val);
|
|
election_timeout_fn_for_test(&ha, ctx, 1 /* cancelled */);
|
|
|
|
ctrl_role_t role =
|
|
(ctrl_role_t)atomic_load_explicit(&ha.role, memory_order_acquire);
|
|
/* Role must still be CTRL_ELECTING — cancelled, nothing happened */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_ELECTING, (int)role,
|
|
"cancelled election_timeout_fn must not change ha->role");
|
|
|
|
free_ctx(ctx);
|
|
ha_fini(&ha);
|
|
clear_hook();
|
|
printf("test_cancelled_timeout_is_noop: PASS (role still CTRL_ELECTING)\n");
|
|
}
|
|
|
|
/* ── Test 8: done_posted guard prevents double-execute ───────────────────── */
|
|
|
|
static void test_done_posted_prevents_double_execute(void)
|
|
{
|
|
/* Calling ha_elect_evaluate_and_signal twice on the same ctx must be a no-op
|
|
* on the second call (done_posted guard). */
|
|
reset_counters();
|
|
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_cfg, 0);
|
|
ha.phase2_done = 0;
|
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
|
|
|
|
ha_elect_ctx_t *ctx = make_ctx(0, 1); /* wins */
|
|
TEST_ASSERT_NOT_NULL_MESSAGE(ctx, "make_ctx failed");
|
|
|
|
ha_elect_evaluate_and_signal(&ha, ctx); /* first call: phase2_done→1, role→COMMITTING */
|
|
/* Reset phase2_done to detect a second call attempt */
|
|
ha.phase2_done = 0;
|
|
ha_elect_evaluate_and_signal(&ha, ctx); /* second call: done_posted=1, should no-op */
|
|
|
|
/* phase2_call_count must still be 1 (second call was no-op) */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_phase2_call_count,
|
|
"done_posted guard must prevent cluster_init_phase2 from being called twice");
|
|
|
|
free_ctx(ctx);
|
|
ha_fini(&ha);
|
|
clear_hook();
|
|
printf("test_done_posted_prevents_double_execute: PASS (calls=%d)\n", g_phase2_call_count);
|
|
}
|
|
|
|
/* ── Test 9: committing_timeout_fn fires ha_promote_to_primary ───────────── */
|
|
|
|
static void test_committing_timeout_fn_promotes_to_primary(void)
|
|
{
|
|
/* Req 4.3, 8.4: when not cancelled and role == CTRL_PRIMARY_COMMITTING,
|
|
* committing_timeout_fn must call ha_promote_to_primary, which drives
|
|
* c->ha.role to CTRL_PRIMARY (degraded promotion). */
|
|
reset_counters();
|
|
|
|
/* Zero-init cluster: xce_add_hook returns -EINVAL harmlessly on zeroed
|
|
* hooksys, so ha_promote_to_primary is safe to call (comment at ha.c:603). */
|
|
cluster_t c;
|
|
memset(&c, 0, sizeof(c));
|
|
ha_init(&c.ha, &g_cfg, 0);
|
|
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
|
memory_order_release);
|
|
|
|
extern int committing_timeout_fn_for_test(ha_state_t *ha, cluster_t *c,
|
|
int cancelled_val);
|
|
committing_timeout_fn_for_test(&c.ha, &c, 0 /* not cancelled */);
|
|
|
|
ctrl_role_t role =
|
|
(ctrl_role_t)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, (int)role,
|
|
"committing_timeout_fn (not cancelled, COMMITTING role) must promote to CTRL_PRIMARY");
|
|
|
|
ha_fini(&c.ha);
|
|
clear_hook();
|
|
printf("test_committing_timeout_fn_promotes_to_primary: PASS (role=%d==CTRL_PRIMARY)\n",
|
|
(int)role);
|
|
}
|
|
|
|
/* ── Test 10: cancelled committing_timeout_fn → no role change ───────────── */
|
|
|
|
static void test_committing_timeout_fn_cancelled_is_noop(void)
|
|
{
|
|
/* Req 4.3: when cancelled=1, committing_timeout_fn must be a no-op;
|
|
* ha->role must remain CTRL_PRIMARY_COMMITTING. */
|
|
reset_counters();
|
|
|
|
cluster_t c;
|
|
memset(&c, 0, sizeof(c));
|
|
ha_init(&c.ha, &g_cfg, 0);
|
|
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
|
|
memory_order_release);
|
|
|
|
extern int committing_timeout_fn_for_test(ha_state_t *ha, cluster_t *c,
|
|
int cancelled_val);
|
|
committing_timeout_fn_for_test(&c.ha, &c, 1 /* cancelled */);
|
|
|
|
ctrl_role_t role =
|
|
(ctrl_role_t)atomic_load_explicit(&c.ha.role, memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, (int)role,
|
|
"cancelled committing_timeout_fn must not change ha->role");
|
|
|
|
ha_fini(&c.ha);
|
|
clear_hook();
|
|
printf("test_committing_timeout_fn_cancelled_is_noop: PASS (role still CTRL_PRIMARY_COMMITTING)\n");
|
|
}
|
|
|
|
/* ── Task 4.1 (ha-repl-hardening): ha_compute_safe_committed_lsn ────────────
|
|
*
|
|
* Computes safe_committed_lsn and safe_config_tag from the vote replies stored
|
|
* in ha_elect_ctx_t.votes[] after winning an election.
|
|
*
|
|
* Algorithm: collect LSNs from all granted voters (including self at votes[0].my_lsn),
|
|
* sort descending, take the value at index floor(n_granted/2) (i.e. the majority
|
|
* threshold element) — this is the highest LSN held by a majority.
|
|
*
|
|
* Same logic applies for config tags (peer_tag).
|
|
*
|
|
* Requirements: 2.3 (ha-repl-hardening spec)
|
|
* Design ref: design.md §Flow 0: Post-Election — Determine Safe Commit Point
|
|
*/
|
|
#ifdef XCE_TESTING
|
|
extern void ha_compute_safe_committed_lsn(ha_elect_ctx_t *ctx,
|
|
uint64_t *safe_lsn_out,
|
|
uint64_t *safe_tag_out);
|
|
#endif
|
|
|
|
/*
|
|
* Test case A (from task description):
|
|
* 3-node cluster — self at lsn=100, peer0 granted at lsn=99, peer1 granted at lsn=100
|
|
* Sorted granted LSNs desc: [100 (self), 100 (peer1), 99 (peer0)]
|
|
* Majority threshold for 3 granted = ceil(3/2) = 2; element at index 1 (0-based) = 100
|
|
* → safe_committed_lsn = 100
|
|
*/
|
|
static void test_safe_committed_lsn_two_peers_at_99_and_100(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
/* n_other=2: ctx->votes[] has 2 entries */
|
|
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + 2 * sizeof(ctx->votes[0]));
|
|
TEST_ASSERT_NOT_NULL(ctx);
|
|
ctx->n_peers = 2; /* 2 other peers */
|
|
ctx->votes_granted = 3; /* self + 2 peers granted */
|
|
ctx->n_replied = 2;
|
|
pthread_mutex_init(&ctx->lock, NULL);
|
|
sem_init(&ctx->done, 0, 0);
|
|
|
|
/* Self data: self_lsn/self_tag (task 6.3) and votes[0].my_lsn/my_tag */
|
|
ctx->self_lsn = 100;
|
|
ctx->self_tag = 5;
|
|
ctx->votes[0].my_lsn = 100;
|
|
ctx->votes[0].my_tag = 5;
|
|
ctx->votes[0].vote_granted = 1; /* peer0: granted, lsn=99 */
|
|
ctx->votes[0].peer_lsn = 99;
|
|
ctx->votes[0].peer_tag = 5;
|
|
|
|
ctx->votes[1].vote_granted = 1; /* peer1: granted, lsn=100 */
|
|
ctx->votes[1].peer_lsn = 100;
|
|
ctx->votes[1].peer_tag = 5;
|
|
|
|
uint64_t safe_lsn = 0, safe_tag = 0;
|
|
ha_compute_safe_committed_lsn(ctx, &safe_lsn, &safe_tag);
|
|
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(100, safe_lsn,
|
|
"voters [99,100] + self=100 → safe_committed_lsn must be 100");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5, safe_tag,
|
|
"voters all at tag=5 → safe_config_tag must be 5");
|
|
|
|
free_ctx(ctx);
|
|
printf("test_safe_committed_lsn_two_peers_at_99_and_100: PASS (safe_lsn=%" PRIu64
|
|
", safe_tag=%" PRIu64 ")\n", safe_lsn, safe_tag);
|
|
#else
|
|
printf("test_safe_committed_lsn_two_peers_at_99_and_100: SKIP (non-test build)\n");
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* Test case B (from task description):
|
|
* 3-node cluster — self at lsn=100, peer0 granted at lsn=50, peer1 granted at lsn=51
|
|
* Sorted granted LSNs desc: [100 (self), 51 (peer1), 50 (peer0)]
|
|
* Majority threshold for 3 granted = 2; element at index 1 = 51
|
|
* → safe_committed_lsn = 51
|
|
*/
|
|
static void test_safe_committed_lsn_two_peers_at_50_and_51(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + 2 * sizeof(ctx->votes[0]));
|
|
TEST_ASSERT_NOT_NULL(ctx);
|
|
ctx->n_peers = 2;
|
|
ctx->votes_granted = 3;
|
|
ctx->n_replied = 2;
|
|
pthread_mutex_init(&ctx->lock, NULL);
|
|
sem_init(&ctx->done, 0, 0);
|
|
|
|
ctx->self_lsn = 100;
|
|
ctx->self_tag = 5;
|
|
ctx->votes[0].my_lsn = 100;
|
|
ctx->votes[0].my_tag = 5;
|
|
ctx->votes[0].vote_granted = 1; /* peer0: granted, lsn=50 */
|
|
ctx->votes[0].peer_lsn = 50;
|
|
ctx->votes[0].peer_tag = 5;
|
|
|
|
ctx->votes[1].vote_granted = 1; /* peer1: granted, lsn=51 */
|
|
ctx->votes[1].peer_lsn = 51;
|
|
ctx->votes[1].peer_tag = 5;
|
|
|
|
uint64_t safe_lsn = 0, safe_tag = 0;
|
|
ha_compute_safe_committed_lsn(ctx, &safe_lsn, &safe_tag);
|
|
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(51, safe_lsn,
|
|
"voters [50,51] + self=100 → safe_committed_lsn must be 51");
|
|
|
|
free_ctx(ctx);
|
|
printf("test_safe_committed_lsn_two_peers_at_50_and_51: PASS (safe_lsn=%" PRIu64 ")\n",
|
|
safe_lsn);
|
|
#else
|
|
printf("test_safe_committed_lsn_two_peers_at_50_and_51: SKIP (non-test build)\n");
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* Test case C: single-node cluster (n_peers=0, no votes[] entries).
|
|
* safe_committed_lsn must equal self's own applied_lsn (self is the only majority).
|
|
* Req 2.3 / task 6.3: returning 0 would truncate all WAL on re-election — wrong.
|
|
* The fix (task 6.3): ha_elect_ctx_t gains self_lsn/self_tag fields populated
|
|
* unconditionally by ha_start_election_async; ha_compute_safe_committed_lsn
|
|
* returns ctx->self_lsn for the n_peers=0 case.
|
|
*/
|
|
static void test_safe_committed_lsn_single_node(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx));
|
|
TEST_ASSERT_NOT_NULL(ctx);
|
|
ctx->n_peers = 0;
|
|
ctx->votes_granted = 1; /* self-vote only */
|
|
ctx->n_replied = 0;
|
|
/* Task 6.3: self_lsn/self_tag populated unconditionally */
|
|
ctx->self_lsn = 77;
|
|
ctx->self_tag = 3;
|
|
pthread_mutex_init(&ctx->lock, NULL);
|
|
sem_init(&ctx->done, 0, 0);
|
|
|
|
uint64_t safe_lsn = 0xDEAD, safe_tag = 0xDEAD;
|
|
ha_compute_safe_committed_lsn(ctx, &safe_lsn, &safe_tag);
|
|
|
|
/* Single-node: safe_committed_lsn must equal self_lsn (not 0) */
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(77, safe_lsn,
|
|
"single-node (n_peers=0): safe_committed_lsn must equal self_lsn (77), not 0");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(3, safe_tag,
|
|
"single-node (n_peers=0): safe_config_tag must equal self_tag (3), not 0");
|
|
|
|
free_ctx(ctx);
|
|
printf("test_safe_committed_lsn_single_node: PASS (safe_lsn=%" PRIu64 ")\n", safe_lsn);
|
|
#else
|
|
printf("test_safe_committed_lsn_single_node: SKIP (non-test build)\n");
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* Test case D: only one peer but it didn't grant vote.
|
|
* Only self vote is in the granted set; n_granted=1, majority threshold index=0.
|
|
* safe_lsn should be from the only granted voter (self), but since self's lsn
|
|
* is in votes[0].my_lsn, result should reflect that.
|
|
* This tests the partial-grant scenario.
|
|
*/
|
|
static void test_safe_committed_lsn_one_peer_denied(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + 1 * sizeof(ctx->votes[0]));
|
|
TEST_ASSERT_NOT_NULL(ctx);
|
|
ctx->n_peers = 1; /* 1 other peer */
|
|
ctx->votes_granted = 1; /* only self-vote granted */
|
|
ctx->n_replied = 1;
|
|
pthread_mutex_init(&ctx->lock, NULL);
|
|
sem_init(&ctx->done, 0, 0);
|
|
|
|
ctx->self_lsn = 200;
|
|
ctx->self_tag = 7;
|
|
ctx->votes[0].my_lsn = 200;
|
|
ctx->votes[0].my_tag = 7;
|
|
ctx->votes[0].vote_granted = 0; /* peer denied vote */
|
|
ctx->votes[0].peer_lsn = 150;
|
|
ctx->votes[0].peer_tag = 7;
|
|
|
|
uint64_t safe_lsn = 0, safe_tag = 0;
|
|
ha_compute_safe_committed_lsn(ctx, &safe_lsn, &safe_tag);
|
|
|
|
/* Only self granted (my_lsn=200). n_granted=1, majority index=0 → lsn=200 */
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(200, safe_lsn,
|
|
"only self granted (my_lsn=200): safe_committed_lsn must be 200");
|
|
|
|
free_ctx(ctx);
|
|
printf("test_safe_committed_lsn_one_peer_denied: PASS (safe_lsn=%" PRIu64 ")\n",
|
|
safe_lsn);
|
|
#else
|
|
printf("test_safe_committed_lsn_one_peer_denied: SKIP (non-test build)\n");
|
|
#endif
|
|
}
|
|
|
|
/* ── Task 4.2 (ha-repl-hardening): committed_lsn set to safe_committed_lsn ──
|
|
*
|
|
* raft_cb_become_leader must set ha->last_known_committed_lsn to the value
|
|
* returned by ha_compute_safe_committed_lsn after WAL truncation.
|
|
*
|
|
* This ensures that in test mode (c == NULL) the decision loop reads the
|
|
* correct committed_lsn from ha->last_known_committed_lsn (see decision-loop
|
|
* XCE_TESTING path in ha.c).
|
|
*
|
|
* Requirement 2.3 (ha-repl-hardening): safe_committed_lsn is derived from
|
|
* voter replies; entries at or below it survive truncation; entries above are
|
|
* removed.
|
|
*
|
|
* The test uses raft_cb_become_leader_for_test (XCE_TESTING shim) which calls
|
|
* raft_cb_become_leader with ha->raft_c = NULL (no real cluster, no WAL I/O).
|
|
* The WAL-level truncation behaviour (file entries above safe_lsn deleted) is
|
|
* covered by test_persist_truncate.c; this test focuses on the committed_lsn
|
|
* field assignment that task 4.2 adds.
|
|
*
|
|
* Design ref: design.md §Flow 0 step 5: committed_lsn = safe_committed_lsn
|
|
*/
|
|
|
|
#ifdef XCE_TESTING
|
|
extern void raft_cb_become_leader_for_test(ha_state_t *ha, uint64_t term);
|
|
#endif
|
|
|
|
/*
|
|
* Test E: raft_cb_become_leader sets ha->last_known_committed_lsn = safe_lsn.
|
|
*
|
|
* 3-node cluster (n_other=2, votes_granted=3):
|
|
* votes[0]: my_lsn=200, peer_lsn=100, vote_granted=1
|
|
* votes[1]: peer_lsn=150, vote_granted=1
|
|
*
|
|
* Sorted granted LSNs desc: [200 (self), 150 (peer1), 100 (peer0)]
|
|
* Majority threshold for 3 granted = ceil(3/2) = 2; element at index 1 = 150
|
|
* → safe_committed_lsn = 150
|
|
*
|
|
* After raft_cb_become_leader_for_test, ha->last_known_committed_lsn must == 150.
|
|
*
|
|
* Additionally verify via ha_compute_safe_committed_lsn that entry AT safe_lsn
|
|
* (lsn=150) is the boundary: safe_lsn=150 means truncate ABOVE 150 (entries
|
|
* 151+ removed); entry at 150 is preserved.
|
|
*/
|
|
static void test_become_leader_sets_committed_lsn_to_safe_lsn(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
reset_counters();
|
|
|
|
ha_state_t ha;
|
|
ha_init(&ha, &g_cfg, 0);
|
|
ha.phase2_done = 1; /* skip phase2 in this test */
|
|
|
|
/* Build elect_ctx with 2 peers; safe_lsn will be 150 */
|
|
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + 2 * sizeof(ctx->votes[0]));
|
|
TEST_ASSERT_NOT_NULL(ctx);
|
|
ctx->n_peers = 2;
|
|
ctx->votes_granted = 3; /* self + 2 peers */
|
|
ctx->n_replied = 2;
|
|
pthread_mutex_init(&ctx->lock, NULL);
|
|
sem_init(&ctx->done, 0, 0);
|
|
|
|
/* Self data in self_lsn/self_tag (task 6.3) and votes[0].my_lsn/my_tag */
|
|
ctx->self_lsn = 200;
|
|
ctx->self_tag = 5;
|
|
/* votes[0]: self data (my_lsn=200) + peer0 granted at lsn=100 */
|
|
ctx->votes[0].my_lsn = 200;
|
|
ctx->votes[0].my_tag = 5;
|
|
ctx->votes[0].vote_granted = 1;
|
|
ctx->votes[0].peer_lsn = 100;
|
|
ctx->votes[0].peer_tag = 5;
|
|
|
|
/* votes[1]: peer1 granted at lsn=150 */
|
|
ctx->votes[1].vote_granted = 1;
|
|
ctx->votes[1].peer_lsn = 150;
|
|
ctx->votes[1].peer_tag = 5;
|
|
|
|
/* Verify the safe_lsn computation independently */
|
|
uint64_t safe_lsn = 0, safe_tag = 0;
|
|
ha_compute_safe_committed_lsn(ctx, &safe_lsn, &safe_tag);
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(150, safe_lsn,
|
|
"ha_compute_safe_committed_lsn: voters [100,150]+self=200 → safe_lsn must be 150");
|
|
|
|
/* Wire elect_ctx into ha_state so raft_cb_become_leader can read it */
|
|
atomic_store_explicit(&ha.elect_ctx, (void *)ctx, memory_order_release);
|
|
|
|
/* Drive the leader promotion path (c=NULL → no WAL I/O, no phase2) */
|
|
raft_cb_become_leader_for_test(&ha, /*term=*/1);
|
|
|
|
/* Task 4.2: ha->last_known_committed_lsn must equal safe_lsn (150) */
|
|
uint64_t committed_lsn =
|
|
atomic_load_explicit(&ha.last_known_committed_lsn, memory_order_acquire);
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(150, committed_lsn,
|
|
"raft_cb_become_leader must set ha->last_known_committed_lsn = safe_lsn (150); "
|
|
"WAL entry AT safe_lsn=150 is preserved; entries above 150 are truncated");
|
|
|
|
sem_destroy(&ctx->done);
|
|
pthread_mutex_destroy(&ctx->lock);
|
|
free(ctx);
|
|
ha_fini(&ha);
|
|
clear_hook();
|
|
printf("test_become_leader_sets_committed_lsn_to_safe_lsn: PASS "
|
|
"(committed_lsn=%" PRIu64 ")\n", committed_lsn);
|
|
#else
|
|
printf("test_become_leader_sets_committed_lsn_to_safe_lsn: SKIP (non-test build)\n");
|
|
#endif
|
|
}
|
|
|
|
/* ── Suite entry point ───────────────────────────────────────────────────── */
|
|
|
|
void test_ha_evaluate_committing(void)
|
|
{
|
|
RUN_TEST(test_winning_quorum_enters_committing);
|
|
RUN_TEST(test_first_boot_phase2_called_once);
|
|
RUN_TEST(test_rdhup_second_election_skips_phase2);
|
|
RUN_TEST(test_committing_resets_repl_ack_count);
|
|
RUN_TEST(test_losing_election_enters_standby);
|
|
RUN_TEST(test_election_timeout_fn_fires_evaluate);
|
|
RUN_TEST(test_cancelled_timeout_is_noop);
|
|
RUN_TEST(test_done_posted_prevents_double_execute);
|
|
RUN_TEST(test_committing_timeout_fn_promotes_to_primary);
|
|
RUN_TEST(test_committing_timeout_fn_cancelled_is_noop);
|
|
/* Task 4.1 (ha-repl-hardening): safe committed LSN computation */
|
|
RUN_TEST(test_safe_committed_lsn_two_peers_at_99_and_100);
|
|
RUN_TEST(test_safe_committed_lsn_two_peers_at_50_and_51);
|
|
RUN_TEST(test_safe_committed_lsn_single_node);
|
|
RUN_TEST(test_safe_committed_lsn_one_peer_denied);
|
|
/* Task 4.2 (ha-repl-hardening): committed_lsn set to safe_committed_lsn */
|
|
RUN_TEST(test_become_leader_sets_committed_lsn_to_safe_lsn);
|
|
}
|