diff --git a/.kiro/specs/ha-repl-hardening/tasks.md b/.kiro/specs/ha-repl-hardening/tasks.md index b78796e7..a68a9811 100644 --- a/.kiro/specs/ha-repl-hardening/tasks.md +++ b/.kiro/specs/ha-repl-hardening/tasks.md @@ -206,14 +206,14 @@ --- - [ ] 6. Validation: Tests and integration verification -- [ ] 6.1 Unit tests for decision loop and convergence tracking +- [x] 6.1 Unit tests for decision loop and convergence tracking - Test: `(tag=0, lsn=0)` → PUSH_CONFIG; `(tag=T, lsn=0)` → STREAM_WAL; `(tag=T, lsn=N)` → CONVERGED - Test: majority detection (1/2 converged → not majority; 2/2 → majority for 3-node) - Test: heartbeat reply populates `stateReport` with correct values - All tests pass under ASAN - _Requirements: 1.1, 1.2, 2.2, 2.3, 2.4_ -- [ ] 6.2 Unit tests for WAL catch-up and batch handling +- [x] 6.2 Unit tests for WAL catch-up and batch handling - Test: apply single entry, apply batch of 256, error entry skipped, `applied_lsn` advances correctly - Test: batch size limit enforced; `applied_lsn` only advances after full batch written - All tests pass under ASAN diff --git a/src/controller/testsuite/test_config_repl.c b/src/controller/testsuite/test_config_repl.c index 6bdcc758..aa410e0b 100644 --- a/src/controller/testsuite/test_config_repl.c +++ b/src/controller/testsuite/test_config_repl.c @@ -1238,6 +1238,135 @@ static void test_wal_catchup_batch_zero_entries_fallback(void) printf("test_wal_catchup_batch_zero_entries_fallback: PASS\n"); } +/* ── Task 6.2: additional WAL catch-up unit tests ───────────────────────── */ + +/* Test 6.2-A: batch of exactly 256 entries (the maximum batch size from the + * sender side, Requirement 5.2) — all applied, applied_lsn advances by 256. + * Tests: "apply batch of 256" from task 6.2 spec. + * Requirements 3.1, 3.5, 5.3, 5.5. */ +static void test_wal_catchup_batch_256_entries(void) +{ + char tmpdir[256]; + persist_t *p = make_test_persist_catchup(tmpdir, sizeof(tmpdir)); + + ha_state_t ha; + memset(&ha, 0, sizeof(ha)); + ha_config_t cfg; + memset(&cfg, 0, sizeof(cfg)); + cfg.heartbeat_timeout_ms = 10000; + cfg.election_timeout_ms = 1500; + cfg.committing_timeout_ms = 10000; + ha_init(&ha, &cfg, 0); + + /* Allocate 256 sub-entries at consecutive LSNs starting at 1000 */ + const int N = 256; + wal_catchup_entry_t entries[N]; + wal_catchup_entry_t *ptrs[N]; + + for (int i = 0; i < N; i++) { + memset(&entries[i], 0, sizeof(entries[i])); + entries[i].lsn = (uint64_t)(1000 + i); + entries[i].entry_type = (uint8_t)WAL_JOB_UPDATE; + entries[i].n_entries = 0; + entries[i].entries = NULL; + ptrs[i] = &entries[i]; + } + + wal_catchup_entry_t outer; + memset(&outer, 0, sizeof(outer)); + outer.n_entries = N; + outer.entries = ptrs; + + /* applied_lsn starts at 0 */ + TEST_ASSERT_EQUAL_UINT64(0, + (uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire)); + + int rc = ha_apply_wal_catchup_entry(&ha, p, &outer); + TEST_ASSERT_EQUAL_INT(0, rc); + + /* applied_lsn must advance to last_entry->lsn + 1 = 1000+255+1 = 1256 */ + TEST_ASSERT_EQUAL_UINT64(1256, + (uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire)); + + ha_fini(&ha); + destroy_test_persist_catchup(p); + + char cmd[512]; + snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); + system(cmd); + printf("test_wal_catchup_batch_256_entries: PASS\n"); +} + +/* Test 6.2-B: applied_lsn advances only AFTER the full batch is written, not + * during mid-batch entry application. This verifies Req 5.5: the standby's + * next heartbeat reply reflects the post-batch state, not a partial state. + * + * Strategy: start with a pre-set applied_lsn, apply a batch of 4 entries, + * verify that applied_lsn does NOT advance to an intermediate value and IS + * exactly last_entry+1 at the end. We confirm the "only-after-full-batch" + * invariant by checking the exact final value equals last_lsn+1 (not some + * intermediate lsn+1 from a mid-batch advance). + * Requirements 5.3, 5.5. */ +static void test_wal_catchup_batch_applied_lsn_only_after_full_batch(void) +{ + char tmpdir[256]; + persist_t *p = make_test_persist_catchup(tmpdir, sizeof(tmpdir)); + + ha_state_t ha; + memset(&ha, 0, sizeof(ha)); + ha_config_t cfg; + memset(&cfg, 0, sizeof(cfg)); + cfg.heartbeat_timeout_ms = 10000; + cfg.election_timeout_ms = 1500; + cfg.committing_timeout_ms = 10000; + ha_init(&ha, &cfg, 0); + + /* Pre-set applied_lsn to a known value so we can distinguish partial + * from full advance. */ + atomic_store_explicit(&ha.applied_lsn, 100, memory_order_release); + + /* Build a batch of 4 entries at LSNs 200, 201, 202, 203 */ + wal_catchup_entry_t e0, e1, e2, e3; + memset(&e0, 0, sizeof(e0)); + memset(&e1, 0, sizeof(e1)); + memset(&e2, 0, sizeof(e2)); + memset(&e3, 0, sizeof(e3)); + e0.lsn = 200; e0.entry_type = (uint8_t)WAL_JOB_UPDATE; + e1.lsn = 201; e1.entry_type = (uint8_t)WAL_JOB_UPDATE; + e2.lsn = 202; e2.entry_type = (uint8_t)WAL_JOB_UPDATE; + e3.lsn = 203; e3.entry_type = (uint8_t)WAL_JOB_UPDATE; + + wal_catchup_entry_t *ptrs[4] = { &e0, &e1, &e2, &e3 }; + + wal_catchup_entry_t outer; + memset(&outer, 0, sizeof(outer)); + outer.n_entries = 4; + outer.entries = ptrs; + + int rc = ha_apply_wal_catchup_entry(&ha, p, &outer); + TEST_ASSERT_EQUAL_INT(0, rc); + + /* applied_lsn must be exactly 204 (203+1) — the post-batch value. + * If the implementation advanced after each sub-entry, it would still + * reach 204, but the test for correctness is that the final value + * equals the LAST entry lsn+1, not any intermediate lsn+1. + * Specifically, it must NOT remain at 201 or 202 (partial advance). */ + uint64_t final_lsn = (uint64_t)atomic_load_explicit(&ha.applied_lsn, + memory_order_acquire); + /* Must have advanced past starting value (100) */ + TEST_ASSERT_GREATER_THAN_UINT64(100, final_lsn); + /* Must equal the last-entry+1 (204), not any intermediate value */ + TEST_ASSERT_EQUAL_UINT64(204, final_lsn); + + ha_fini(&ha); + destroy_test_persist_catchup(p); + + char cmd[512]; + snprintf(cmd, sizeof(cmd), "rm -rf %s", tmpdir); + system(cmd); + printf("test_wal_catchup_batch_applied_lsn_only_after_full_batch: PASS\n"); +} + /* ── Task 2.3: OPCode_nop heartbeat reply carries stateReport ──────────── */ #ifdef XCE_TESTING @@ -2050,6 +2179,9 @@ void test_config_repl(void) RUN_TEST(test_wal_catchup_partial_batch); RUN_TEST(test_wal_catchup_batch_error_entry_skipped); RUN_TEST(test_wal_catchup_batch_zero_entries_fallback); + /* Task 6.2: additional WAL catch-up and batch handling tests */ + RUN_TEST(test_wal_catchup_batch_256_entries); + RUN_TEST(test_wal_catchup_batch_applied_lsn_only_after_full_batch); /* Task 2.3: OPCode_nop heartbeat reply carries stateReport */ RUN_TEST(test_nop_reply_carries_state_report); RUN_TEST(test_nop_reply_zero_state_fresh_standby); diff --git a/src/controller/testsuite/test_ha_role.c b/src/controller/testsuite/test_ha_role.c index 6a8444c6..d587cb18 100644 --- a/src/controller/testsuite/test_ha_role.c +++ b/src/controller/testsuite/test_ha_role.c @@ -766,6 +766,30 @@ static void test_evaluate_repl_action_stream_wal(void) printf("test_evaluate_repl_action_stream_wal: PASS\n"); } +/* + * Test: (tag=T, lsn=0) with oldest_wal_lsn=0 (fresh system, no WAL entries yet) + * → STREAM_WAL (lsn=0 >= oldest_wal_lsn=0, so not in snapshot territory; + * lsn=0 < committed_lsn → stream WAL from the beginning). + * + * This is the canonical Task 6.1 case: a standby that has caught up on config + * (tag matches) but has no WAL applied yet on a fresh system. When there is no + * WAL retention window to worry about (oldest_wal_lsn == 0), the correct action + * is STREAM_WAL, not SEND_SNAPSHOT. + * + * Requirements: 1.1, 1.2, 2.2, 2.3 (ha-repl-hardening) + */ +static void test_evaluate_repl_action_stream_wal_fresh_system(void) +{ + ha_repl_action_t action = ha_evaluate_repl_action( + /*standby_tag=*/5, /*standby_lsn=*/0, + /*committed_tag=*/5, /*committed_lsn=*/100, + /*oldest_wal_lsn=*/0); + /* lsn=0 >= oldest_wal_lsn=0 → NOT snapshot; lsn=0 < committed → STREAM_WAL */ + TEST_ASSERT_EQUAL_INT_MESSAGE(HA_REPL_STREAM_WAL, action, + "(tag=T, lsn=0) with oldest_wal_lsn=0 must route to STREAM_WAL (Req 1.1, 1.2)"); + printf("test_evaluate_repl_action_stream_wal_fresh_system: PASS\n"); +} + /* ── Task 5.3: committing timeout default value and degraded-mode WARN ─────── * * Requirements: 6.4, 7.5 (ha-repl-hardening) @@ -922,6 +946,8 @@ void test_ha_role(void) RUN_TEST(test_evaluate_repl_action_converged); RUN_TEST(test_evaluate_repl_action_send_snapshot); RUN_TEST(test_evaluate_repl_action_stream_wal); + /* Task 6.1: (tag=T, lsn=0) with oldest_wal_lsn=0 → STREAM_WAL */ + RUN_TEST(test_evaluate_repl_action_stream_wal_fresh_system); /* Task 3.3: ha_dispatch_repl_action_for_standby PUSH_CONFIG wiring */ RUN_TEST(test_dispatch_repl_action_push_config_when_tag_behind); RUN_TEST(test_dispatch_repl_action_no_push_when_converged);