test(ha-convergence-recovery): RED clamped rejoin reconciliation tests + intake extraction (task 3.1)
This commit is contained in:
parent
09103a3141
commit
5521720b0c
5 changed files with 300 additions and 9 deletions
|
|
@ -514,6 +514,204 @@ static void test_demotion_replay_yields_committed_only(void)
|
|||
printf("test_demotion_replay_yields_committed_only: checked\n");
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════════
|
||||
* Rejoin reconciliation (ha-convergence-recovery task 3.1 — RED)
|
||||
*
|
||||
* Requirements (this spec's IDs — see requirements.md):
|
||||
* 2.1 — when a standby learns the primary's committed boundary at (re)join and
|
||||
* its local log tip EXCEEDS that boundary, it truncates above the boundary
|
||||
* and self-restarts ONCE. A second reconcile call in the same process
|
||||
* lifetime must NOT restart again (defensive once-guard).
|
||||
* 2.2 — the standby must NOT remove entries at/below its own locally known
|
||||
* committed_lsn during reconciliation (clamp): the truncation cutoff is
|
||||
* max(primary_committed, local committed_lsn). Verified by setting the
|
||||
* local committed_lsn ABOVE the reported boundary and asserting the cutoff
|
||||
* is the higher local value, NOT the reported boundary.
|
||||
* 2.3 — while the local tip does NOT exceed the boundary, the standby joins with
|
||||
* NO truncation and NO restart, returning 0.
|
||||
* (intake integration) — delivering a CtrlHelloReply through the real cluster.c
|
||||
* intake stores last_known_committed_lsn (existing) AND triggers
|
||||
* reconciliation.
|
||||
*
|
||||
* RED status against the current tree: ha_rejoin_reconcile is a no-op scaffold
|
||||
* (ha_convergence.c) and the cluster.c CtrlHelloReply intake does NOT call it yet
|
||||
* (task 3.2 GREEN wires both). Every case below therefore FAILS until 3.2 lands.
|
||||
*
|
||||
* Harness: reuses the dc_* fixture (real persist over a tmpdir, restart/fatal
|
||||
* seam counters). The fixture comes up CTRL_PRIMARY; rejoin reconciliation is a
|
||||
* STANDBY-side concern, so each rejoin test first demotes to STANDBY (which also
|
||||
* clears the persist hooks — irrelevant to reconciliation) before exercising it.
|
||||
* AAA structure throughout.
|
||||
* ════════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
/* Set the local on-disk WAL tip to `tip_lsn` (current_lsn = tip_lsn + 1) and the
|
||||
* local committed_lsn to `committed`. Mirrors dc_setup's priming convention. */
|
||||
static void rj_set_local_state(uint64_t tip_lsn, uint64_t committed)
|
||||
{
|
||||
pthread_mutex_lock(&g_dc.persist.lock);
|
||||
g_dc.persist.current_lsn = tip_lsn + 1;
|
||||
pthread_mutex_unlock(&g_dc.persist.lock);
|
||||
atomic_store(&g_dc.persist.committed_lsn, committed);
|
||||
}
|
||||
|
||||
/* Demote the fixture to STANDBY (the role rejoin reconciliation runs in) and
|
||||
* install the restart/fatal seams so the self-restart is observable. */
|
||||
static void rj_make_standby_with_seams(void)
|
||||
{
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_demote_to_standby(&g_dc));
|
||||
TEST_ASSERT_EQUAL_INT((int)CTRL_STANDBY,
|
||||
(int)atomic_load_explicit(&g_dc.ha.role, memory_order_acquire));
|
||||
ha_restart_hook = dc_restart_hook;
|
||||
ha_fatal_hook = dc_fatal_hook;
|
||||
g_restart_hook_rc = 0; /* model a successful restart */
|
||||
}
|
||||
|
||||
/* 2.1/2.2: local tip ABOVE the primary boundary AND local committed_lsn ABOVE the
|
||||
* boundary → reconcile truncates at the CLAMP max(boundary, local committed) (the
|
||||
* higher local value, never the lower reported boundary) and self-restarts once.
|
||||
* RED: ha_rejoin_reconcile is a no-op stub → no truncation, no restart. */
|
||||
static void test_rejoin_clamp_truncates_at_local_committed_and_restarts_once(void)
|
||||
{
|
||||
dc_setup("rjclamp", /*n_peers=*/3, /*n_standbys=*/0,
|
||||
/*appended_lsn=*/9, /*n_vcpus=*/8);
|
||||
rj_make_standby_with_seams();
|
||||
|
||||
/* Local tip = 9 (entries 1..9 on disk). Local committed_lsn = 6 — ABOVE the
|
||||
* primary-reported boundary of 4. The clamp must protect [1..6] and truncate
|
||||
* only above 6 → cutoff = max(4, 6) = 6 → current_lsn collapses to 7. */
|
||||
rj_set_local_state(/*tip_lsn=*/9, /*committed=*/6);
|
||||
const uint64_t primary_boundary = 4;
|
||||
const uint64_t local_committed = 6;
|
||||
|
||||
/* Act. */
|
||||
int rc = ha_rejoin_reconcile(&g_dc, primary_boundary);
|
||||
/* Drive any deferred convergence task (the intake may defer to delaymgr,
|
||||
* same rule as demotion); harmless if 3.2 truncates inline. */
|
||||
(void)dc_run_pending_task(&g_dc.delaymgr);
|
||||
|
||||
/* Assert (Req 2.2 clamp): truncation cutoff is the LOCAL committed (6), not the
|
||||
* lower reported boundary (4) — current_lsn collapses to local_committed + 1. */
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(local_committed + 1, g_dc.persist.current_lsn,
|
||||
"rejoin reconcile must clamp the cutoff to max(boundary, local committed) "
|
||||
"(Req 2.2): with boundary=4 and local committed=6 it must truncate above 6, "
|
||||
"NOT above the lower reported boundary 4");
|
||||
|
||||
/* Assert (Req 2.1): self-restart fired exactly once, fatal seam never. */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_restart_calls),
|
||||
"rejoin reconcile with tip > boundary must self-restart exactly once "
|
||||
"(Req 2.1)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_fatal_calls),
|
||||
"a successful rejoin restart must NOT invoke the fatal seam");
|
||||
(void)rc;
|
||||
|
||||
dc_teardown();
|
||||
printf("test_rejoin_clamp_truncates_at_local_committed_and_restarts_once: checked\n");
|
||||
}
|
||||
|
||||
/* 2.3: local tip AT/BELOW the boundary → no truncation, no restart, returns 0.
|
||||
* RED: the no-op stub returns 0 with no side effects, so the no-action assertions
|
||||
* PASS but the suite as a whole is RED via the other cases; this case pins the
|
||||
* negative contract so 3.2 cannot over-truncate. (It is the one rejoin case that
|
||||
* a correct no-op stub already satisfies — kept to lock the boundary for GREEN.) */
|
||||
static void test_rejoin_tip_at_or_below_boundary_no_action(void)
|
||||
{
|
||||
dc_setup("rjnoop", /*n_peers=*/3, /*n_standbys=*/0,
|
||||
/*appended_lsn=*/4, /*n_vcpus=*/8);
|
||||
rj_make_standby_with_seams();
|
||||
|
||||
/* Local tip = 4, exactly the primary boundary; nothing to discard. */
|
||||
rj_set_local_state(/*tip_lsn=*/4, /*committed=*/4);
|
||||
uint64_t tip_before = g_dc.persist.current_lsn;
|
||||
|
||||
/* Act. */
|
||||
int rc = ha_rejoin_reconcile(&g_dc, /*primary_committed_lsn=*/4);
|
||||
|
||||
/* Assert (Req 2.3): no side effects, returns 0. */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
|
||||
"rejoin reconcile with tip <= boundary must return 0 (Req 2.3)");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(tip_before, g_dc.persist.current_lsn,
|
||||
"rejoin reconcile with tip <= boundary must NOT truncate (Req 2.3)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_restart_calls),
|
||||
"rejoin reconcile with tip <= boundary must NOT self-restart (Req 2.3)");
|
||||
|
||||
dc_teardown();
|
||||
printf("test_rejoin_tip_at_or_below_boundary_no_action: checked\n");
|
||||
}
|
||||
|
||||
/* 2.1 once-guard: a SECOND reconcile call in the same process lifetime must NOT
|
||||
* self-restart again, even if the tip still exceeds the boundary (defensive guard;
|
||||
* "once" is also structural post-restart).
|
||||
* RED: the no-op stub never restarts even once, so the "exactly one restart total"
|
||||
* assertion fails. */
|
||||
static void test_rejoin_second_call_no_second_restart(void)
|
||||
{
|
||||
dc_setup("rjonce", /*n_peers=*/3, /*n_standbys=*/0,
|
||||
/*appended_lsn=*/9, /*n_vcpus=*/8);
|
||||
rj_make_standby_with_seams();
|
||||
rj_set_local_state(/*tip_lsn=*/9, /*committed=*/4);
|
||||
|
||||
/* Act: first reconcile (tip 9 > boundary 4 → truncate + restart once). */
|
||||
(void)ha_rejoin_reconcile(&g_dc, /*primary_committed_lsn=*/4);
|
||||
(void)dc_run_pending_task(&g_dc.delaymgr);
|
||||
int restarts_after_first = atomic_load(&g_restart_calls);
|
||||
|
||||
/* Act: second reconcile in the SAME lifetime. The once-guard must suppress a
|
||||
* second restart regardless of the reported boundary. */
|
||||
(void)ha_rejoin_reconcile(&g_dc, /*primary_committed_lsn=*/4);
|
||||
(void)dc_run_pending_task(&g_dc.delaymgr);
|
||||
|
||||
/* Assert (Req 2.1): exactly ONE restart across both calls. */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, restarts_after_first,
|
||||
"the FIRST rejoin reconcile with tip > boundary must self-restart once "
|
||||
"(Req 2.1)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_restart_calls),
|
||||
"a SECOND rejoin reconcile in the same process lifetime must NOT restart "
|
||||
"again — defensive once-guard (Req 2.1)");
|
||||
|
||||
dc_teardown();
|
||||
printf("test_rejoin_second_call_no_second_restart: checked\n");
|
||||
}
|
||||
|
||||
/* Intake integration: delivering a CtrlHelloReply through the REAL cluster.c
|
||||
* intake (cluster_apply_ctrlhello_reply) stores last_known_committed_lsn AND
|
||||
* triggers reconciliation.
|
||||
* RED: the intake stores the boundary (passes) but does NOT call
|
||||
* ha_rejoin_reconcile yet (task 3.2 wires it), so the restart assertion fails. */
|
||||
static void test_ctrlhello_reply_intake_stores_and_reconciles(void)
|
||||
{
|
||||
dc_setup("rjintake", /*n_peers=*/3, /*n_standbys=*/0,
|
||||
/*appended_lsn=*/9, /*n_vcpus=*/8);
|
||||
rj_make_standby_with_seams();
|
||||
/* Local tip 9 > the boundary the primary will report (4) → reconcile must fire
|
||||
* from inside the intake. */
|
||||
rj_set_local_state(/*tip_lsn=*/9, /*committed=*/4);
|
||||
|
||||
const uint64_t reported_committed = 4;
|
||||
const uint64_t reported_tag = 77;
|
||||
|
||||
/* Act: drive the real standby-side CtrlHelloReply intake. */
|
||||
cluster_apply_ctrlhello_reply(&g_dc, reported_committed, reported_tag);
|
||||
(void)dc_run_pending_task(&g_dc.delaymgr);
|
||||
|
||||
/* Assert (existing behavior): the boundary + config tag are stored. */
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(reported_committed,
|
||||
atomic_load(&g_dc.ha.last_known_committed_lsn),
|
||||
"CtrlHelloReply intake must store last_known_committed_lsn (existing)");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(reported_tag,
|
||||
g_dc.ha.last_known_committed_config_tag,
|
||||
"CtrlHelloReply intake must store the config tag (existing)");
|
||||
|
||||
/* Assert (intake integration): the intake also TRIGGERED reconciliation —
|
||||
* tip 9 > boundary 4 → exactly one self-restart. */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_restart_calls),
|
||||
"CtrlHelloReply intake must trigger rejoin reconciliation when the local "
|
||||
"tip exceeds the reported boundary — exactly one self-restart (task 3.2 "
|
||||
"wires ha_rejoin_reconcile into the intake)");
|
||||
|
||||
dc_teardown();
|
||||
printf("test_ctrlhello_reply_intake_stores_and_reconciles: checked\n");
|
||||
}
|
||||
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
/* ── Suite entry point ──────────────────────────────────────────────────── */
|
||||
|
|
@ -530,6 +728,13 @@ void test_demotion_convergence(void)
|
|||
RUN_TEST(test_demotion_from_electing_no_truncation_no_restart);
|
||||
RUN_TEST(test_demotion_from_init_no_truncation_no_restart);
|
||||
RUN_TEST(test_demotion_replay_yields_committed_only);
|
||||
|
||||
printf("\n>> rejoin reconciliation tests (RED, task 3.1)\n\n");
|
||||
|
||||
RUN_TEST(test_rejoin_clamp_truncates_at_local_committed_and_restarts_once);
|
||||
RUN_TEST(test_rejoin_tip_at_or_below_boundary_no_action);
|
||||
RUN_TEST(test_rejoin_second_call_no_second_restart);
|
||||
RUN_TEST(test_ctrlhello_reply_intake_stores_and_reconciles);
|
||||
#else
|
||||
printf("test_demotion_convergence: SKIP (non-test build)\n");
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue