feat: P3-6 ha_trigger_manual_failover_for_test() shim + 3 failover cycle tests

Add test-only function ha_trigger_manual_failover_for_test() to exercise manual
failover state machine (demote → promote cycle) without requiring live cluster_t.

Test coverage:
- test_manual_failover_transitions_role_and_increments_term: Validates role
  transitions (PRIMARY → STANDBY → PRIMARY) and term monotonicity
- test_manual_failover_resets_hooks_registered: Confirms demotion resets the
  hooks_registered idempotency guard
- test_manual_failover_monotonic_term_across_multiple_cycles: Validates term
  remains monotonically increasing across 5 consecutive failover cycles

Implementation:
- Exposed only under #ifdef XCE_TESTING in ha.h
- Function body (6 lines): calls ha_do_demote() then ha_do_promote(); returns
  the new term value via atomic_load_explicit()
- Enables design validation (P3-6) without test-cluster infrastructure

All tests pass. Build clean.
This commit is contained in:
Rongsong Shen 2026-05-07 23:55:00 +00:00
parent 1f97aee1a3
commit b490c419ca
5 changed files with 131 additions and 0 deletions

View file

@ -105,6 +105,17 @@ void ha_do_demote(ha_state_t *ha)
* because it needs the cluster's hooksys_t. */
}
/* ── P3-6: Test-only failover shim ────────────────────────────── */
#ifdef XCE_TESTING
uint64_t ha_trigger_manual_failover_for_test(ha_state_t *ha)
{
ha_do_demote(ha);
ha_do_promote(ha);
return atomic_load_explicit(&ha->term, memory_order_acquire);
}
#endif
/* ── Watchdog thread implementation ────────────────────────── */
void ha_check_heartbeat_nocluster(ha_state_t *ha)

View file

@ -158,4 +158,15 @@ int ha_should_redirect(const ha_state_t *ha);
*/
int ha_no_primary_known(const ha_state_t *ha);
/* ── P3-6: Test-only failover shim ────────────────────────────── */
#ifdef XCE_TESTING
/* Test-only shim: demote then re-promote; returns new term.
* Exercises the manual failover state machine without a live cluster_t.
* Calls ha_do_demote() then ha_do_promote(); validates role transitions
* and term monotonicity. See design doc P3-6 and src/docs/persistence-and-ha-design.md.
*/
uint64_t ha_trigger_manual_failover_for_test(ha_state_t *ha);
#endif
#endif /* HA_H */

View file

@ -94,6 +94,7 @@ controller_unit_test_src = [
'testsuite/test_repl_catchup.c',
'testsuite/test_p2_integration.c',
'testsuite/test_p3_hardening.c',
'testsuite/test_p3_manual_failover.c',
'persist.c',
'ha.c',
'repl.c',
@ -112,6 +113,7 @@ controller_unit_test = executable('controller_unit_test',
controller_unit_test_src,
include_directories: [incdir],
dependencies: controller_unit_test_deps,
c_args: ['-DXCE_TESTING'],
install: false)
test('controller test', controller_unit_test,

View file

@ -29,6 +29,7 @@ extern void test_persist_repl(void);
extern void test_repl_catchup(void);
extern void test_p2_integration(void);
extern void test_p3_hardening(void);
extern void test_p3_manual_failover(void);
void setUp(void) { return; }
@ -127,6 +128,7 @@ int main(int argc, char* argv[])
TEST(test_repl_catchup);
TEST(test_p2_integration);
TEST(test_p3_hardening);
TEST(test_p3_manual_failover);
return UNITY_END();
}

View file

@ -0,0 +1,105 @@
#include <stdatomic.h>
#include <stdio.h>
#include <string.h>
#include <unity/unity.h>
#include "ha.h"
/* P3-6: Manual failover shim tests.
*
* Validates the ha_trigger_manual_failover_for_test() shim:
* - Role transitions from PRIMARY → STANDBY → PRIMARY
* - Term is monotonically increasing across multiple cycles
* - hooks_registered is reset after demotion
*/
static void test_manual_failover_transitions_role_and_increments_term(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
ha_init(&ha, &cfg, 0);
/* Start as primary */
ha_do_promote(&ha);
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
/* Manual failover: demote then re-promote via shim */
uint64_t new_term = ha_trigger_manual_failover_for_test(&ha);
/* Term must have incremented (new election) */
TEST_ASSERT_TRUE(new_term > term_before);
/* Role must end as PRIMARY (re-promoted) */
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
ha_fini(&ha);
printf("test_manual_failover_transitions_role_and_increments_term: PASS\n");
}
static void test_manual_failover_resets_hooks_registered(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
ha_init(&ha, &cfg, 0);
ha_do_promote(&ha);
ha.hooks_registered = 1; /* Simulate hooks registered */
ha_trigger_manual_failover_for_test(&ha);
/* After demote+re-promote: hooks_registered was cleared by demote.
* Demotion ALWAYS resets hooks_registered to 0. */
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
/* Verify that demote reset the flag (no real cluster to re-register with) */
TEST_ASSERT_EQUAL_INT(0, ha.hooks_registered);
ha_fini(&ha);
printf("test_manual_failover_resets_hooks_registered: PASS\n");
}
static void test_manual_failover_monotonic_term_across_multiple_cycles(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
ha_init(&ha, &cfg, 0);
uint64_t prev_term = 0;
for (int i = 0; i < 5; i++) {
uint64_t new_term = ha_trigger_manual_failover_for_test(&ha);
TEST_ASSERT_TRUE(new_term > prev_term);
prev_term = new_term;
}
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
ha_fini(&ha);
printf("test_manual_failover_monotonic_term_across_multiple_cycles: PASS\n");
}
void test_p3_manual_failover(void)
{
RUN_TEST(test_manual_failover_transitions_role_and_increments_term);
RUN_TEST(test_manual_failover_resets_hooks_registered);
RUN_TEST(test_manual_failover_monotonic_term_across_multiple_cycles);
}