wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity

feat(election-iomgr-refactor): COMMITTING path in ha_elect_evaluate_and_signal + committing/election timeout watchdogs — tasks 4.1, 4.3, 4.4

Browse source
This commit is contained in:
Rongsong Shen 2026-05-30 11:37:12 +00:00
parent 6de4b3ff0e
commit f4a6726f1b
6 changed files with 646 additions and 65 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/controller/testsuite/controller_test.c
View file

@ -46,6 +46,7 @@ extern void test_ha_failover(void);
extern void test_ha_vote_send(void);
extern void test_ha_start_election_async(void);
extern void test_ha_vote_gate(void);
extern void test_ha_evaluate_committing(void);
void setUp(void) { return; }
@ -161,6 +162,7 @@ int main(int argc, char* argv[])
TEST(test_ha_vote_send);
TEST(test_ha_start_election_async);
TEST(test_ha_vote_gate);
TEST(test_ha_evaluate_committing);
return UNITY_END();
}

381
src/controller/testsuite/test_ha_evaluate_committing.c Normal file
View file

@ -0,0 +1,381 @@
/*
* test_ha_evaluate_committing.c — Unity tests for Task 4.1:
* ha_elect_evaluate_and_signal COMMITTING path.
*
* Tests verify:
* Req 4.1: quorum reached → ha->role transitions to CTRL_PRIMARY_COMMITTING
* (not CTRL_PRIMARY) after ha_elect_evaluate_and_signal
* Req 5.2: COMMITTING path: ha->repl_ack_count reset to 0 when entering COMMITTING
* Req 9.2: cluster_init_phase2 called exactly once on first election;
* second election (RDHUP) skips it (phase2_done=1)
*
* Design ref: design.md "ha_elect_evaluate_and_signal (extended)"
* Boundary: ha.c (election evaluation)
* Requirements: 4.1, 5.2, 9.2
*/
#define _GNU_SOURCE
#include <errno.h>
#include <semaphore.h>
#include <stdatomic.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unity/unity.h>
#include "ha.h"
/* ── Test-only seam: cluster_init_phase2 call counter ───────────────────────
*
* In production, ha_elect_evaluate_and_signal calls cluster_init_phase2(c)
* when !ha->phase2_done. For unit tests we cannot provide a real cluster_t,
* so we hook via an XCE_TESTING seam in ha.c: the evaluate function checks
* ha_phase2_hook; if non-NULL it calls the hook instead of cluster_init_phase2.
*
* This avoids pulling in the full cluster_t dependency in the test binary.
*/
extern int (*ha_phase2_hook)(void *c);
/* Call counter for the hook */
static int g_phase2_call_count = 0;
static int g_phase2_hook_rc = 0; /* return value for hook (0 = success) */
static int stub_cluster_init_phase2(void *c)
{
(void)c;
g_phase2_call_count++;
return g_phase2_hook_rc;
}
/* ── Minimal ha_config_t for tests ─────────────────────────────────────── */
static ha_config_t g_cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.election_jitter_max_ms = 0,
.election_priority_step_ms = 500,
.committing_timeout_ms = 10000,
};
/* ── Helper: build a minimal ha_elect_ctx_t for n_other peers ───────────── */
static ha_elect_ctx_t *make_ctx(int n_other, int votes_granted)
{
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) +
(size_t)n_other * sizeof(ctx->votes[0]));
if (!ctx) return NULL;
ctx->n_peers = n_other;
ctx->votes_granted = votes_granted; /* includes self-vote */
ctx->n_replied = n_other;
ctx->max_resp_tag = 0;
ctx->max_resp_lsn = 0;
ctx->result = -1;
ctx->done_posted = 0;
ctx->c = NULL; /* no real cluster in unit tests */
pthread_mutex_init(&ctx->lock, NULL);
sem_init(&ctx->done, 0, 0);
/* Store my_tag/my_lsn in votes[0] for yield gate (only if n_other > 0) */
if (n_other > 0) {
ctx->votes[0].my_tag = 100;
ctx->votes[0].my_lsn = 50;
}
return ctx;
}
static void free_ctx(ha_elect_ctx_t *ctx)
{
if (!ctx) return;
pthread_mutex_destroy(&ctx->lock);
sem_destroy(&ctx->done);
free(ctx);
}
/* ── Test setup/teardown ─────────────────────────────────────────────────── */
static void reset_counters(void)
{
g_phase2_call_count = 0;
g_phase2_hook_rc = 0;
/* Wire up the test seam */
ha_phase2_hook = stub_cluster_init_phase2;
}
static void clear_hook(void)
{
ha_phase2_hook = NULL;
}
/* ── Test 1: winning quorum → CTRL_PRIMARY_COMMITTING (not PRIMARY) ─────── */
static void test_winning_quorum_enters_committing(void)
{
/* 3-node cluster: n_other=2, quorum=2. Self-vote + 2 peer grants = 3.
* ha_elect_evaluate_and_signal should detect quorum → COMMITTING. */
reset_counters();
ha_state_t ha;
ha_init(&ha, &g_cfg, 0);
ha.phase2_done = 0; /* first boot */
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
ha_elect_ctx_t *ctx = make_ctx(2, 3); /* votes_granted=3 (self + 2 peers) */
TEST_ASSERT_NOT_NULL_MESSAGE(ctx, "make_ctx failed");
/* Set votes[0] my_tag/my_lsn for yield gate */
ctx->votes[0].my_tag = 100;
ctx->votes[0].my_lsn = 50;
ctrl_role_t result = ha_elect_evaluate_and_signal(&ha, ctx);
/* Function must return CTRL_PRIMARY (the election result) */
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, (int)result,
"ha_elect_evaluate_and_signal must return CTRL_PRIMARY when quorum won");
/* But ha->role must be CTRL_PRIMARY_COMMITTING (Req 4.1) */
ctrl_role_t actual_role =
(ctrl_role_t)atomic_load_explicit(&ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, (int)actual_role,
"ha->role must be CTRL_PRIMARY_COMMITTING after winning election (not CTRL_PRIMARY)");
free_ctx(ctx);
ha_fini(&ha);
clear_hook();
printf("test_winning_quorum_enters_committing: PASS (role=%d==CTRL_PRIMARY_COMMITTING)\n",
(int)actual_role);
}
/* ── Test 2: first boot — phase2 called exactly once ────────────────────── */
static void test_first_boot_phase2_called_once(void)
{
/* phase2_done=0: cluster_init_phase2 must be called exactly once. */
reset_counters();
ha_state_t ha;
ha_init(&ha, &g_cfg, 0);
ha.phase2_done = 0;
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
/* Single-node-like: n_other=0, quorum=1, votes_granted=1 → wins */
ha_elect_ctx_t *ctx = make_ctx(0, 1);
TEST_ASSERT_NOT_NULL_MESSAGE(ctx, "make_ctx failed");
ha_elect_evaluate_and_signal(&ha, ctx);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_phase2_call_count,
"cluster_init_phase2 must be called exactly once on first boot");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.phase2_done,
"ha->phase2_done must be set to 1 after cluster_init_phase2");
free_ctx(ctx);
ha_fini(&ha);
clear_hook();
printf("test_first_boot_phase2_called_once: PASS (calls=%d)\n", g_phase2_call_count);
}
/* ── Test 3: second election (RDHUP) skips phase2 ───────────────────────── */
static void test_rdhup_second_election_skips_phase2(void)
{
/* phase2_done=1: cluster_init_phase2 must NOT be called. */
reset_counters();
ha_state_t ha;
ha_init(&ha, &g_cfg, 0);
ha.phase2_done = 1; /* already initialized (post-RDHUP failover) */
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
ha_elect_ctx_t *ctx = make_ctx(0, 1); /* single-node wins again */
TEST_ASSERT_NOT_NULL_MESSAGE(ctx, "make_ctx failed");
ha_elect_evaluate_and_signal(&ha, ctx);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_phase2_call_count,
"cluster_init_phase2 must NOT be called when phase2_done=1 (RDHUP failover)");
free_ctx(ctx);
ha_fini(&ha);
clear_hook();
printf("test_rdhup_second_election_skips_phase2: PASS (calls=%d)\n", g_phase2_call_count);
}
/* ── Test 4: COMMITTING path resets repl_ack_count ──────────────────────── */
static void test_committing_resets_repl_ack_count(void)
{
/* Req 5.2: repl_ack_count must be 0 when entering COMMITTING.
* Start with a non-zero value to verify it gets reset. */
reset_counters();
ha_state_t ha;
ha_init(&ha, &g_cfg, 0);
ha.phase2_done = 1; /* skip phase2 call */
atomic_store_explicit(&ha.repl_ack_count, 3, memory_order_release); /* stale count */
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
ha_elect_ctx_t *ctx = make_ctx(0, 1); /* wins */
TEST_ASSERT_NOT_NULL_MESSAGE(ctx, "make_ctx failed");
ha_elect_evaluate_and_signal(&ha, ctx);
int ack_count = atomic_load_explicit(&ha.repl_ack_count, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ack_count,
"ha->repl_ack_count must be reset to 0 when entering CTRL_PRIMARY_COMMITTING");
free_ctx(ctx);
ha_fini(&ha);
clear_hook();
printf("test_committing_resets_repl_ack_count: PASS (repl_ack_count=%d)\n", ack_count);
}
/* ── Test 5: losing election → CTRL_STANDBY, phase2 called on first boot ── */
static void test_losing_election_enters_standby(void)
{
/* n_other=2, quorum=2, votes_granted=1 (self only) → loses.
* On first boot, phase2 must still be called; role → CTRL_STANDBY. */
reset_counters();
ha_state_t ha;
ha_init(&ha, &g_cfg, 0);
ha.phase2_done = 0; /* first boot */
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
ha_elect_ctx_t *ctx = make_ctx(2, 1); /* votes_granted=1, quorum=2 → loses */
TEST_ASSERT_NOT_NULL_MESSAGE(ctx, "make_ctx failed");
ctrl_role_t result = ha_elect_evaluate_and_signal(&ha, ctx);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY, (int)result,
"ha_elect_evaluate_and_signal must return CTRL_STANDBY when quorum not won");
ctrl_role_t actual_role =
(ctrl_role_t)atomic_load_explicit(&ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY, (int)actual_role,
"ha->role must be CTRL_STANDBY after losing election");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_phase2_call_count,
"cluster_init_phase2 must be called once even for losing standby (first boot)");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.phase2_done,
"ha->phase2_done must be set to 1 after losing first-boot election");
free_ctx(ctx);
ha_fini(&ha);
clear_hook();
printf("test_losing_election_enters_standby: PASS (role=%d==CTRL_STANDBY, phase2=%d)\n",
(int)actual_role, g_phase2_call_count);
}
/* ── Test 6: election timeout fires evaluate → COMMITTING if wins ────────── */
static void test_election_timeout_fn_fires_evaluate(void)
{
/* election_timeout_fn calls ha_elect_evaluate_and_signal.
* Use election_timeout_fn_for_test (XCE_TESTING shim in ha.c):
* election_timeout_fn_for_test(ha, ctx, cancelled=0) → fires evaluate.
* election_timeout_fn_for_test(ha, ctx, cancelled=1) → no-op.
*/
reset_counters();
ha_state_t ha;
ha_init(&ha, &g_cfg, 0);
ha.phase2_done = 1; /* skip phase2 */
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
ha_elect_ctx_t *ctx = make_ctx(0, 1); /* single-node wins */
TEST_ASSERT_NOT_NULL_MESSAGE(ctx, "make_ctx failed");
/* Use the test-shim to simulate the timeout firing with cancelled=0 */
extern int election_timeout_fn_for_test(ha_state_t *ha, ha_elect_ctx_t *ctx,
int cancelled_val);
election_timeout_fn_for_test(&ha, ctx, 0 /* not cancelled */);
ctrl_role_t role =
(ctrl_role_t)atomic_load_explicit(&ha.role, memory_order_acquire);
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, (int)role,
"election_timeout_fn (not cancelled) must drive ha->role to CTRL_PRIMARY_COMMITTING");
free_ctx(ctx);
ha_fini(&ha);
clear_hook();
printf("test_election_timeout_fn_fires_evaluate: PASS (role=%d)\n", (int)role);
}
/* ── Test 7: cancelled election timeout → no role change ────────────────── */
static void test_cancelled_timeout_is_noop(void)
{
reset_counters();
ha_state_t ha;
ha_init(&ha, &g_cfg, 0);
ha.phase2_done = 1;
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
ha_elect_ctx_t *ctx = make_ctx(0, 1);
TEST_ASSERT_NOT_NULL_MESSAGE(ctx, "make_ctx failed");
extern int election_timeout_fn_for_test(ha_state_t *ha, ha_elect_ctx_t *ctx,
int cancelled_val);
election_timeout_fn_for_test(&ha, ctx, 1 /* cancelled */);
ctrl_role_t role =
(ctrl_role_t)atomic_load_explicit(&ha.role, memory_order_acquire);
/* Role must still be CTRL_ELECTING — cancelled, nothing happened */
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_ELECTING, (int)role,
"cancelled election_timeout_fn must not change ha->role");
free_ctx(ctx);
ha_fini(&ha);
clear_hook();
printf("test_cancelled_timeout_is_noop: PASS (role still CTRL_ELECTING)\n");
}
/* ── Test 8: done_posted guard prevents double-execute ───────────────────── */
static void test_done_posted_prevents_double_execute(void)
{
/* Calling ha_elect_evaluate_and_signal twice on the same ctx must be a no-op
* on the second call (done_posted guard). */
reset_counters();
ha_state_t ha;
ha_init(&ha, &g_cfg, 0);
ha.phase2_done = 0;
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
ha_elect_ctx_t *ctx = make_ctx(0, 1); /* wins */
TEST_ASSERT_NOT_NULL_MESSAGE(ctx, "make_ctx failed");
ha_elect_evaluate_and_signal(&ha, ctx); /* first call: phase2_done→1, role→COMMITTING */
/* Reset phase2_done to detect a second call attempt */
ha.phase2_done = 0;
ha_elect_evaluate_and_signal(&ha, ctx); /* second call: done_posted=1, should no-op */
/* phase2_call_count must still be 1 (second call was no-op) */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_phase2_call_count,
"done_posted guard must prevent cluster_init_phase2 from being called twice");
free_ctx(ctx);
ha_fini(&ha);
clear_hook();
printf("test_done_posted_prevents_double_execute: PASS (calls=%d)\n", g_phase2_call_count);
}
/* ── Suite entry point ───────────────────────────────────────────────────── */
void test_ha_evaluate_committing(void)
{
RUN_TEST(test_winning_quorum_enters_committing);
RUN_TEST(test_first_boot_phase2_called_once);
RUN_TEST(test_rdhup_second_election_skips_phase2);
RUN_TEST(test_committing_resets_repl_ack_count);
RUN_TEST(test_losing_election_enters_standby);
RUN_TEST(test_election_timeout_fn_fires_evaluate);
RUN_TEST(test_cancelled_timeout_is_noop);
RUN_TEST(test_done_posted_prevents_double_execute);
}