forked from lei.chen/myxce
test(ha-commit-engine): RED integration tests for ACK feed and sync-append (task 5.1)
Tests 30-34: wired repl_on_ack feeds engine, membership mutators trigger re-evaluation, per-entry sync-append contract (stale-ACK rejection, committed_lsn unmoved on timeout, n_peers=1 replaces -ENODEV). test_repl_majority walAck cases rewired to per-LSN semantics (configAck byte-identical). Sanctioned scaffolding: entry_lsn param on repl_wait_majority_ack (legacy body byte-identical, callers pass 0), inert repl_set_commit_engine stub. 9 RED failures; baseline signatures unchanged. LSN-0 edge documented for 5.2/5.3.
This commit is contained in:
parent
c9154bd5cc
commit
d1353360cc
9 changed files with 512 additions and 79 deletions
|
|
@ -42,8 +42,21 @@
|
|||
* and ha_dispatch_gate write-quorum behavior. RED against the inert repl.c
|
||||
* stub (returns -1); green with the task-4.2 implementation.
|
||||
*
|
||||
* Requirements: 1.1, 1.2, 2.1, 2.2, 2.3, 3.1, 3.2, 3.3, 4.1, 4.2, 4.3, 4.4,
|
||||
* 5.1, 6.1, 6.2, 6.3, 7.1, 7.2, 7.3
|
||||
* Task 5.1 extends the file with integration tests for the ACK feed and the
|
||||
* job-submit sync-append path: repl_on_ack with a wired engine
|
||||
* (repl_set_commit_engine) updates acked_lsn AND triggers commit evaluation;
|
||||
* the four membership mutators (repl_add_standby / repl_remove_standby /
|
||||
* repl_start_catchup / repl_complete_catchup) trigger re-evaluation; and
|
||||
* persist_wal_sync_append over a real persist_t (tmpdir fixture, pattern
|
||||
* from test_p2_integration.c) honors the POST-wiring contract: 0 only on
|
||||
* per-entry majority, -ETIMEDOUT on stale ACKs covering only an earlier
|
||||
* entry, committed_lsn unmoved past the entry on timeout, and n_peers=1
|
||||
* returning 0 with committed_lsn advanced (replaces -ENODEV degraded path).
|
||||
* RED against the inert repl_set_commit_engine stub and the unwired
|
||||
* production sync-append path; green with tasks 5.2 + 5.3.
|
||||
*
|
||||
* Requirements: 1.1, 1.2, 1.3, 2.1, 2.2, 2.3, 3.1, 3.2, 3.3, 4.1, 4.2, 4.3,
|
||||
* 4.4, 5.1, 6.1, 6.2, 6.3, 7.1, 7.2, 7.3
|
||||
*/
|
||||
#include <errno.h>
|
||||
#include <pthread.h>
|
||||
|
|
@ -62,6 +75,8 @@
|
|||
#include "ha.h" /* ha_state_t, ha_dispatch_gate fixture */
|
||||
#include "persist.h"
|
||||
#include "repl.h"
|
||||
#include "typemap.h" /* wal_catchup_entry_t (task 5.1 send hook) */
|
||||
#include "wal_types.h" /* WAL_JOB_SUBMIT (task 5.1 sync-append cases) */
|
||||
|
||||
/* ha_dispatch_gate — defined in ha.c (same pattern as test_ha_repl_ack.c) */
|
||||
int ha_dispatch_gate(ha_state_t *ha, int opcode);
|
||||
|
|
@ -1100,6 +1115,300 @@ static void test_gate_wired_registry_catchup_standby_rejects_then_admits(void)
|
|||
printf("test_gate_wired_registry_catchup_standby_rejects_then_admits: PASS\n");
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════
|
||||
* Task 5.1 — integration: ACK feed and the job-submit sync-append path.
|
||||
* (Req 1.1, 1.2, 1.3, 2.1, 4.2, 4.4, 6.3; design §ACK Feed,
|
||||
* §Job-Submit Adapter)
|
||||
*
|
||||
* Tests 30-31 wire the engine into the stub-fixture replicator via
|
||||
* repl_set_commit_engine and drive the PRODUCTION entry points
|
||||
* (repl_on_ack, membership mutators) instead of commit_on_ack directly.
|
||||
* Tests 32-34 run persist_wal_sync_append against a real persist_t
|
||||
* (tmpdir fixture, test_p2_integration.c pattern) with an engine wired
|
||||
* over it, asserting the post-task-5.3 return contract.
|
||||
* ════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
/* Test-injectable send hook (defined in repl.c under XCE_TESTING) — keeps
|
||||
* repl_broadcast from dereferencing the dummy endpoints. */
|
||||
extern int (*repl_broadcast_send_hook)(void *ep,
|
||||
wal_catchup_entry_t **entries,
|
||||
int n_entries);
|
||||
|
||||
static int ce_drop_send_hook(void *ep, wal_catchup_entry_t **entries,
|
||||
int n_entries)
|
||||
{
|
||||
(void)ep;
|
||||
(void)entries;
|
||||
(void)n_entries;
|
||||
return 0; /* pretend success, drop the data */
|
||||
}
|
||||
|
||||
/* Heap persist over a private tmpdir (test_p2_integration.c pattern).
|
||||
* Static pointer: survives Unity's longjmp on assertion failure. */
|
||||
static persist_t *g_sp;
|
||||
|
||||
static persist_t *sync_persist_setup(const char *suffix, uint32_t sync_timeout_ms)
|
||||
{
|
||||
persist_t *p;
|
||||
persist_config_t cfg;
|
||||
char tmpdir[256];
|
||||
|
||||
p = calloc(1, sizeof(persist_t));
|
||||
TEST_ASSERT_NOT_NULL(p);
|
||||
|
||||
cfg = persist_config_defaults();
|
||||
cfg.wal_sync_ms = 5000;
|
||||
cfg.sync_timeout_ms = sync_timeout_ms;
|
||||
|
||||
snprintf(tmpdir, sizeof(tmpdir), "/tmp/xce_ce51_%s_XXXXXX", suffix);
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir));
|
||||
|
||||
persist_set_workdir(p, tmpdir, "ctrl");
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_init(p, &cfg, NULL));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_start(p));
|
||||
return p;
|
||||
}
|
||||
|
||||
static void sync_persist_teardown(persist_t *p)
|
||||
{
|
||||
if (p == NULL) {
|
||||
return;
|
||||
}
|
||||
persist_stop(p);
|
||||
persist_fini(p);
|
||||
free(p);
|
||||
}
|
||||
|
||||
/* Background ACK through the PRODUCTION feed (repl_on_ack), not the
|
||||
* commit_on_ack shortcut — exercises the task-5.2 wiring. */
|
||||
typedef struct {
|
||||
replicator_t *r;
|
||||
void *ep;
|
||||
uint64_t lsn;
|
||||
uint32_t delay_ms;
|
||||
} repl_ack_arg_t;
|
||||
|
||||
static repl_ack_arg_t g_rack;
|
||||
|
||||
static void *repl_ack_thread_fn(void *arg)
|
||||
{
|
||||
repl_ack_arg_t *a = (repl_ack_arg_t *)arg;
|
||||
|
||||
if (a->delay_ms > 0) {
|
||||
sleep_ms(a->delay_ms);
|
||||
}
|
||||
repl_on_ack(a->r, a->ep, a->lsn);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* ── Test 30: repl_on_ack with a wired engine feeds commit evaluation ─────
|
||||
* The production ACK entry point must BOTH update the registry watermark
|
||||
* AND trigger engine evaluation: majority → committed_lsn advances and the
|
||||
* pending waiter wakes with success. (Req 1.1, 4.2) */
|
||||
static void test_repl_on_ack_wired_updates_watermark_and_commits(void)
|
||||
{
|
||||
uint64_t watermark;
|
||||
|
||||
/* Arrange: 3-member cluster, entry 1 appended; engine wired into the
|
||||
* replicator the way cluster init will (repl_set_commit_engine) */
|
||||
engine_setup(3, 2, 1);
|
||||
repl_set_commit_engine(&g_p.repl, &g_ce);
|
||||
append_entries(1);
|
||||
spawn_waiters(1, 1, 1000);
|
||||
|
||||
/* Act: standby 0 ACKs entry 1 through the PRODUCTION feed */
|
||||
sleep_ms(80);
|
||||
repl_on_ack(&g_p.repl, &g_eps[0], 1);
|
||||
join_waiters(1);
|
||||
|
||||
/* Assert: the registry watermark was updated (pre-existing contract)... */
|
||||
watermark = atomic_load(&g_p.repl.standbys[0].acked_lsn);
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(1, watermark,
|
||||
"repl_on_ack must update the standby's acked_lsn");
|
||||
/* ...AND the wired engine evaluated the majority: commit advanced... */
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(1, committed(),
|
||||
"wired repl_on_ack must trigger commit evaluation (majority -> advance)");
|
||||
/* ...and the pending waiter woke with success */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_wargs[0].rc,
|
||||
"waiter must complete once the wired ACK feed reaches majority");
|
||||
|
||||
engine_teardown();
|
||||
printf("test_repl_on_ack_wired_updates_watermark_and_commits: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 31: all four membership mutators trigger re-evaluation ──────────
|
||||
* For each of repl_add_standby / repl_start_catchup / repl_complete_catchup
|
||||
* / repl_remove_standby: a majority-covering watermark is planted DIRECTLY
|
||||
* in the registry (no ACK feed), so committed_lsn can only advance if the
|
||||
* mutator itself triggers commit_on_membership_change re-evaluation.
|
||||
* (Req 4.2) */
|
||||
static void test_membership_mutators_trigger_commit_reevaluation(void)
|
||||
{
|
||||
/* Arrange: 3-member cluster, entries 1..4 appended, ONE standby (sb0)
|
||||
* registered and wired; sb0's watermark alone forms majority with self */
|
||||
engine_setup(3, 1, 4);
|
||||
repl_set_commit_engine(&g_p.repl, &g_ce);
|
||||
append_entries(4);
|
||||
|
||||
/* Act + Assert per mutator. Plant the watermark silently, confirm no
|
||||
* advancement happened, then attribute the advance to the mutator. */
|
||||
|
||||
/* repl_add_standby (standby connect) re-evaluates */
|
||||
atomic_store(&g_p.repl.standbys[0].acked_lsn, 1);
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, committed(),
|
||||
"planted watermark alone must not advance committed_lsn");
|
||||
TEST_ASSERT_TRUE(repl_add_standby(&g_p.repl, &g_eps[1], "sb1", 7001) >= 0);
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(1, committed(),
|
||||
"repl_add_standby must trigger commit re-evaluation");
|
||||
|
||||
/* repl_start_catchup (catch-up entry) re-evaluates */
|
||||
atomic_store(&g_p.repl.standbys[0].acked_lsn, 2);
|
||||
TEST_ASSERT_EQUAL_UINT64(1, committed());
|
||||
repl_start_catchup(&g_p.repl, &g_eps[1], 0);
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(2, committed(),
|
||||
"repl_start_catchup must trigger commit re-evaluation");
|
||||
|
||||
/* repl_complete_catchup (catch-up completion) re-evaluates */
|
||||
atomic_store(&g_p.repl.standbys[0].acked_lsn, 3);
|
||||
TEST_ASSERT_EQUAL_UINT64(2, committed());
|
||||
repl_complete_catchup(&g_p.repl, &g_eps[1]);
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(3, committed(),
|
||||
"repl_complete_catchup must trigger commit re-evaluation");
|
||||
|
||||
/* repl_remove_standby (disconnect) re-evaluates */
|
||||
atomic_store(&g_p.repl.standbys[0].acked_lsn, 4);
|
||||
TEST_ASSERT_EQUAL_UINT64(3, committed());
|
||||
repl_remove_standby(&g_p.repl, &g_eps[1]);
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(4, committed(),
|
||||
"repl_remove_standby must trigger commit re-evaluation");
|
||||
|
||||
engine_teardown();
|
||||
printf("test_membership_mutators_trigger_commit_reevaluation: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 32: sync append returns 0 on per-entry majority (wired) ─────────
|
||||
* Contract-pinning positive case: a standby watermark covering THE entry
|
||||
* commits it — return 0, sync status 0, committed_lsn at the entry.
|
||||
* (Req 1.1, 1.2) */
|
||||
static int g_ep_sa, g_ep_sb;
|
||||
|
||||
static void test_sync_append_wired_commits_on_per_entry_majority(void)
|
||||
{
|
||||
pthread_t th;
|
||||
int rc, sync_rc;
|
||||
|
||||
/* Arrange: real persist, 2 standbys, engine n_peers=3 wired; one
|
||||
* priming async append so the sync entry lands at LSN 1 */
|
||||
g_sp = sync_persist_setup("maj", 2000);
|
||||
TEST_ASSERT_TRUE(repl_add_standby(&g_sp->repl, &g_ep_sa, "sb_a", 7100) >= 0);
|
||||
TEST_ASSERT_TRUE(repl_add_standby(&g_sp->repl, &g_ep_sb, "sb_b", 7101) >= 0);
|
||||
TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&g_ce, g_sp, 3, 8));
|
||||
repl_set_commit_engine(&g_sp->repl, &g_ce);
|
||||
|
||||
repl_broadcast_send_hook = ce_drop_send_hook;
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(g_sp, WAL_JOB_UPDATE, "p", 1));
|
||||
|
||||
/* Act: standby A's ACK covering the sync entry (LSN 1) arrives mid-wait */
|
||||
g_rack = (repl_ack_arg_t){ &g_sp->repl, &g_ep_sa, 1, 30 };
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
pthread_create(&th, NULL, repl_ack_thread_fn, &g_rack));
|
||||
sync_rc = -1;
|
||||
rc = persist_wal_sync_append(g_sp, WAL_JOB_SUBMIT, "job", 3, &sync_rc);
|
||||
pthread_join(th, NULL);
|
||||
repl_broadcast_send_hook = NULL;
|
||||
|
||||
/* Assert: per-entry majority → confirmed, commit index at the entry */
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, sync_rc,
|
||||
"sync status must be 0 on per-entry majority");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
||||
1, atomic_load(&g_sp->committed_lsn),
|
||||
"committed_lsn must advance to the majority-acked entry");
|
||||
|
||||
commit_engine_fini(&g_ce);
|
||||
sync_persist_teardown(g_sp);
|
||||
g_sp = NULL;
|
||||
printf("test_sync_append_wired_commits_on_per_entry_majority: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 33: stale-ACK regression — earlier-entry ACKs never commit ──────
|
||||
* Both standbys' watermarks cover only an EARLIER entry. The dishonest
|
||||
* `acked_lsn > 0` counting confirms the new entry off those stale ACKs;
|
||||
* the engine contract is -ETIMEDOUT with committed_lsn unmoved past the
|
||||
* entry: connectivity / old ACKs never substitute for per-entry
|
||||
* acknowledgment. (Req 1.2, 1.3, 2.1, 4.4) */
|
||||
static void test_sync_append_wired_stale_acks_time_out(void)
|
||||
{
|
||||
int rc, sync_rc;
|
||||
|
||||
/* Arrange: real persist (short sync timeout), 2 standbys, engine
|
||||
* n_peers=3 wired; entries 0..1 appended and BOTH standbys ACK
|
||||
* watermark 1 BEFORE the new sync entry exists */
|
||||
g_sp = sync_persist_setup("stale", 60);
|
||||
TEST_ASSERT_TRUE(repl_add_standby(&g_sp->repl, &g_ep_sa, "sb_a", 7110) >= 0);
|
||||
TEST_ASSERT_TRUE(repl_add_standby(&g_sp->repl, &g_ep_sb, "sb_b", 7111) >= 0);
|
||||
TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&g_ce, g_sp, 3, 8));
|
||||
repl_set_commit_engine(&g_sp->repl, &g_ce);
|
||||
|
||||
repl_broadcast_send_hook = ce_drop_send_hook;
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(g_sp, WAL_JOB_UPDATE, "p", 1));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(g_sp, WAL_JOB_UPDATE, "q", 1));
|
||||
repl_on_ack(&g_sp->repl, &g_ep_sa, 1);
|
||||
repl_on_ack(&g_sp->repl, &g_ep_sb, 1);
|
||||
|
||||
/* Act: sync append entry LSN 2 — no ACK for it ever arrives */
|
||||
sync_rc = 0;
|
||||
rc = persist_wal_sync_append(g_sp, WAL_JOB_SUBMIT, "job", 3, &sync_rc);
|
||||
repl_broadcast_send_hook = NULL;
|
||||
|
||||
/* Assert: stale ACKs (watermark 1 > 0) must NOT confirm entry 2... */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(-ETIMEDOUT, rc,
|
||||
"stale earlier-entry ACKs must never confirm a new entry");
|
||||
TEST_ASSERT_EQUAL_INT(-ETIMEDOUT, sync_rc);
|
||||
/* ...and the commit index never advances past the unacked entry */
|
||||
TEST_ASSERT_TRUE_MESSAGE(atomic_load(&g_sp->committed_lsn) < 2,
|
||||
"committed_lsn must stay unmoved past the entry on timeout");
|
||||
|
||||
commit_engine_fini(&g_ce);
|
||||
sync_persist_teardown(g_sp);
|
||||
g_sp = NULL;
|
||||
printf("test_sync_append_wired_stale_acks_time_out: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 34: n_peers=1 → sync append succeeds with committed advanced ────
|
||||
* Standalone self-quorum replaces the old `-ENODEV` degraded path: the
|
||||
* single-member primary confirms locally (sync status 0, NOT -ENODEV) and
|
||||
* committed_lsn advances to the entry. (Req 6.3) */
|
||||
static void test_sync_append_wired_single_member_commits_locally(void)
|
||||
{
|
||||
int rc, sync_rc;
|
||||
|
||||
/* Arrange: real persist, NO standbys, engine n_peers=1 wired; one
|
||||
* priming async append so the sync entry lands at LSN 1 */
|
||||
g_sp = sync_persist_setup("solo", 2000);
|
||||
TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&g_ce, g_sp, 1, 8));
|
||||
repl_set_commit_engine(&g_sp->repl, &g_ce);
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(g_sp, WAL_JOB_UPDATE, "p", 1));
|
||||
|
||||
/* Act */
|
||||
sync_rc = -1;
|
||||
rc = persist_wal_sync_append(g_sp, WAL_JOB_SUBMIT, "job", 3, &sync_rc);
|
||||
|
||||
/* Assert: local confirmation, no degraded -ENODEV signal... */
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, sync_rc,
|
||||
"n_peers=1 must self-commit (0), not report -ENODEV degraded mode");
|
||||
/* ...and the standalone commit index advances to the entry */
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
||||
1, atomic_load(&g_sp->committed_lsn),
|
||||
"n_peers=1 sync append must advance committed_lsn to the entry");
|
||||
|
||||
commit_engine_fini(&g_ce);
|
||||
sync_persist_teardown(g_sp);
|
||||
g_sp = NULL;
|
||||
printf("test_sync_append_wired_single_member_commits_locally: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test group entry point ──────────────────────────────────────────────── */
|
||||
|
||||
void test_commit_engine(void)
|
||||
|
|
@ -1138,5 +1447,11 @@ void test_commit_engine(void)
|
|||
RUN_TEST(test_gate_reaccepts_after_reconnect_without_reset);
|
||||
/* Task 4.2 — wired gate: registry truth at ha_dispatch_gate */
|
||||
RUN_TEST(test_gate_wired_registry_catchup_standby_rejects_then_admits);
|
||||
/* Task 5.1 — integration: ACK feed + job-submit sync-append path */
|
||||
RUN_TEST(test_repl_on_ack_wired_updates_watermark_and_commits);
|
||||
RUN_TEST(test_membership_mutators_trigger_commit_reevaluation);
|
||||
RUN_TEST(test_sync_append_wired_commits_on_per_entry_majority);
|
||||
RUN_TEST(test_sync_append_wired_stale_acks_time_out);
|
||||
RUN_TEST(test_sync_append_wired_single_member_commits_locally);
|
||||
UNITY_END();
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue