myxce/src/controller/testsuite/test_e2e_checkpoint.c

/* P0-15b: Checkpoint E2E — round trip
 *
 * Exercises cross-phase interactions:
 *   - WAL writer (P0-B) + checkpoint (P0-C) + recovery (P0-C)
 *   - LSN continuity across checkpoint + post-checkpoint WAL entries
 *   - WAL segment trimming after checkpoint
 *   - Snapshot rotation keeping snap_keep_n files
 */
#define _GNU_SOURCE
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>

#include <unity/unity.h>

#include "config.h"
#include "persist.h"
#include "repl.h"     /* repl_add_standby / repl_on_ack for trim-guard tests
                        * (raft-drive-async task 3.2, bug-1147) */
#include "typemap.h"  /* wal_catchup_entry_t */

/* Drop hook for repl_broadcast: fixtures with dummy (int) endpoints installed
 * via repl_add_standby must install this so persist_wal_async→repl_broadcast does
 * not crash on the fake endpoint.  Pattern from test_commit_engine.c, test_p2_integration.c. */
static int ckpt_drop_broadcast(void *ep, wal_catchup_entry_t **entries, int count)
{
    (void)ep;
    (void)entries;
    (void)count;
    return 0;
}

extern int (*repl_broadcast_send_hook)(void *ep,
                                         wal_catchup_entry_t **entries,
                                         int count);

/* ------------------------------------------------------------------ */
/* Helpers                                                              */
/* ------------------------------------------------------------------ */

static int count_files_suffix(const char* dir, const char* suffix)
{
    DIR* d = opendir(dir);
    if (!d) return 0;
    int n = 0;
    size_t slen = strlen(suffix);
    struct dirent* e;
    while ((e = readdir(d)) != NULL) {
        size_t len = strlen(e->d_name);
        if (len > slen && strcmp(e->d_name + len - slen, suffix) == 0)
            n++;
    }
    closedir(d);
    return n;
}

/* Make a persist_t with a fresh tmpdir.  Caller must persist_stop/fini/free. */
static persist_t* make_ckpt_e2e(char* tmpdir, size_t sz,
                                 uint32_t snap_keep_n)
{
    snprintf(tmpdir, sz, "/tmp/xce_e2e_ckpt_XXXXXX");
    if (!mkdtemp(tmpdir)) return NULL;

    persist_t* p = calloc(1, sizeof(persist_t));
    if (!p) return NULL;

    persist_config_t cfg = persist_config_defaults();
    cfg.wal_sync_ms = 5000;
    cfg.snap_keep_n = snap_keep_n;

    persist_set_workdir(p, tmpdir, "e2e-ckpt");

    if (persist_init(p, &cfg, NULL) != 0) { free(p); return NULL; }
    if (persist_start(p) != 0) { persist_fini(p); free(p); return NULL; }
    return p;
}

static void teardown_ckpt_e2e(persist_t* p)
{
    persist_stop(p);
    persist_fini(p);
    free(p);
}

/* ------------------------------------------------------------------ */
/* Test 1: LSN continuity — 20 WAL + checkpoint + 10 WAL + recovery   */
/* ------------------------------------------------------------------ */

static void test_checkpoint_recovery_lsn_continuity(void)
{
    char tmpdir[256];
    persist_t* p = make_ckpt_e2e(tmpdir, sizeof(tmpdir), 2);
    TEST_ASSERT_NOT_NULL(p);

    /* Phase 1: 20 WAL entries */
    for (int i = 0; i < 20; i++) {
        TEST_ASSERT_EQUAL_INT(0,
            persist_wal_async(p, WAL_JOB_UPDATE, "x", 1));
    }

    /* No live apply pump runs in this cluster-less fixture, so
     * last_applied_lsn stays 0 unless simulated -- persist_checkpoint now
     * stamps checkpoint_lsn from last_applied_lsn, not current_lsn
     * (checkpoint_lsn-vs-last_applied_lsn fix); simulate apply having
     * caught up to what was appended before the checkpoint runs. */
    atomic_store_explicit(&p->last_applied_lsn, p->current_lsn - 1,
                          memory_order_relaxed);

    /* Checkpoint */
    TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
    uint64_t ckpt_lsn = p->checkpoint_lsn;
    TEST_ASSERT_TRUE(ckpt_lsn > 0);

    /* Phase 2: 10 more WAL entries after checkpoint */
    for (int i = 0; i < 10; i++) {
        TEST_ASSERT_EQUAL_INT(0,
            persist_wal_async(p, WAL_JOB_UPDATE, "y", 1));
    }
    TEST_ASSERT_EQUAL_INT(0, persist_flush(p));

    teardown_ckpt_e2e(p);

    /* Phase 3: fresh persist_t, recover */
    persist_t* p2 = calloc(1, sizeof(persist_t));
    TEST_ASSERT_NOT_NULL(p2);

    persist_config_t cfg = persist_config_defaults();
    cfg.wal_sync_ms = 5000;
    cfg.snap_keep_n = 2;
    persist_set_workdir(p2, tmpdir, "e2e-ckpt");
    TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL));
    TEST_ASSERT_EQUAL_INT(0, persist_start(p2));

    TEST_ASSERT_EQUAL_INT(0, persist_recover(p2));

    /* All 30+ entries must be accounted for */
    TEST_ASSERT_TRUE(p2->current_lsn >= 30);
    /* The snapshot was used */
    TEST_ASSERT_TRUE(p2->checkpoint_lsn > 0);

    teardown_ckpt_e2e(p2);
    printf("test_checkpoint_recovery_lsn_continuity: PASS\n");
}

/* ------------------------------------------------------------------ */
/* Test 2: WAL trim — old segments removed after two checkpoints       */
/* ------------------------------------------------------------------ */

static void test_checkpoint_wal_trim_after_checkpoint(void)
{
    char tmpdir[256];
    persist_t* p = make_ckpt_e2e(tmpdir, sizeof(tmpdir), 2);
    TEST_ASSERT_NOT_NULL(p);

    /* Write enough entries and checkpoint once. No live apply pump runs in
     * this cluster-less fixture, so last_applied_lsn stays 0 unless
     * simulated -- bug-1426's trim floor now also caps at last_applied_lsn
     * (the local apply cursor), so it must be advanced here to reflect the
     * pump having caught up, or trim_wal_segments would correctly refuse
     * to delete anything this test expects trimmed. */
    for (int i = 0; i < 5; i++)
        persist_wal_async(p, WAL_JOB_UPDATE, "trim", 4);
    atomic_store_explicit(&p->last_applied_lsn, p->current_lsn - 1,
                          memory_order_relaxed);
    TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));

    /* Write a few more entries then checkpoint again */
    for (int i = 0; i < 3; i++)
        persist_wal_async(p, WAL_JOB_UPDATE, "trim2", 5);
    atomic_store_explicit(&p->last_applied_lsn, p->current_lsn - 1,
                          memory_order_relaxed);
    TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));

    /* After two checkpoints the wal directory must still be functional
     * (at least one segment exists — the currently-active one). */
    char wal_path[512];
    snprintf(wal_path, sizeof(wal_path), "%s/e2e-ckpt/data/wal", tmpdir);
    int n = count_files_suffix(wal_path, ".wal");
    TEST_ASSERT_TRUE(n >= 1);

    teardown_ckpt_e2e(p);
    printf("test_checkpoint_wal_trim_after_checkpoint: PASS\n");
}

/* ------------------------------------------------------------------ */
/* Test 3: snapshot rotation keeps at most snap_keep_n files           */
/* ------------------------------------------------------------------ */

static void test_snapshot_rotation_keeps_snap_keep_n(void)
{
    char tmpdir[256];
    persist_t* p = make_ckpt_e2e(tmpdir, sizeof(tmpdir), 2);
    TEST_ASSERT_NOT_NULL(p);

    /* Run 4 checkpoints with snap_keep_n = 2 */
    for (int i = 0; i < 4; i++) {
        persist_wal_async(p, WAL_JOB_UPDATE, "r", 1);
        TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
    }

    char snap_path[512];
    snprintf(snap_path, sizeof(snap_path), "%s/e2e-ckpt/data/snap", tmpdir);
    int n = count_files_suffix(snap_path, ".capnp");
    TEST_ASSERT_TRUE(n <= (int)p->config.snap_keep_n);

    teardown_ckpt_e2e(p);
    printf("test_snapshot_rotation_keeps_snap_keep_n: PASS\n");
}

/* ------------------------------------------------------------------ */
/* Helper: check if a WAL segment file at a given start LSN exists.    */
/* Builds the name "seg-<lsn16>.wal" inside dir.                       */
/* ------------------------------------------------------------------ */

static int seg_file_exists(const char* wal_dir, uint64_t seg_start_lsn)
{
    char path[4096];
    snprintf(path, sizeof(path), "%s/seg-%016" PRIx64 ".wal",
             wal_dir, seg_start_lsn);
    return access(path, F_OK) == 0 ? 1 : 0;
}

/* ------------------------------------------------------------------ */
/* Test 4: standby cursor-bound trim guard — Req 1.1 (task 3.2,       */
/*         bug-1147).  With a standby whose acked_lsn is below the     */
/*         active segment, segments the standby still needs for        */
/*         catch-up are NOT deleted.                                    */
/*                                                                     */
/* RED proof: revert the repl_trim_lsn floor call in trim_wal_segments */
/* → the segment at the standby's acked LSN is deleted because it's    */
/* below the active segment → re-creates the bug-795/828 gap where     */
/* catch-up hits a missing segment → permanent committed-job loss.     */
/* ------------------------------------------------------------------ */

static void test_trim_guard_preserves_standby_segments(void)
{
    char tmpdir[256];
    persist_t* p = make_ckpt_e2e(tmpdir, sizeof(tmpdir), 2);
    TEST_ASSERT_NOT_NULL(p);

    /* Install drop hook so repl_broadcast doesn't crash on the dummy ep */
    repl_broadcast_send_hook = ckpt_drop_broadcast;

    /* Add a standby with acked_lsn = 2 (simulating a standby that has
     * caught up through LSN 2 but not beyond) */
    int dummy_ep = 0;
    int idx = repl_add_standby(&p->repl, (void*)&dummy_ep, "standby1", 7001);
    TEST_ASSERT_TRUE(idx >= 0);
    repl_on_ack(&p->repl, (void*)&dummy_ep, 2);

    /* Write entries to create segments.
     *   init: current_lsn = 1 (LSN 0 reserved, fresh store)
     *   write "x" @ LSN 1 → current_lsn = 2
     *   force → close seg-at-1, open seg-at-2
     *   write "y" @ LSN 2 → current_lsn = 3
     *   force → close seg-at-2, open seg-at-3
     *
     * Now we have 3 segments: seg-...001.wal, seg-...002.wal, seg-...003.wal.
     * Active segment = seg-at-3.  Standby acked_lsn = 2. */
    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "x", 1));
    TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));
    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "y", 1));
    TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));

    /* bug-1426: simulate the local apply pump having caught up to the
     * active segment tip -- trim_wal_segments now also caps its floor at
     * last_applied_lsn (this test targets the standby-floor guard
     * specifically, not the apply-cursor one, so the pump must not be the
     * limiting factor here). */
    atomic_store_explicit(&p->last_applied_lsn, p->current_lsn - 1,
                          memory_order_relaxed);

    /* Trigger checkpoint → trim_wal_segments runs */
    TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));

    /* floor = repl_trim_lsn(active_seg_lsn=3, ...) = min(3, acked=2) = 2.
     * So seg_start_lsn < 2 means only seg-at-1 is deleted (1 < 2).
     * seg-at-2 (the standby still needs it) is NOT deleted (2 >= 2). */
    char wal_path[512];
    snprintf(wal_path, sizeof(wal_path), "%s/e2e-ckpt/data/wal", tmpdir);

    /* The old segment below the standby's acked LSN IS trimmed */
    TEST_ASSERT_FALSE_MESSAGE(
        seg_file_exists(wal_path, 1),
        "seg-at-1: below standby acked_lsn=2, MUST be trimmed");

    /* The segment at the standby's acked LSN is PRESERVED (bug-1147 guard) */
    TEST_ASSERT_TRUE_MESSAGE(
        seg_file_exists(wal_path, 2),
        "seg-at-2: at standby acked_lsn=2, MUST be preserved (bug-1147 guard)");

    /* The active segment is never trimmed */
    TEST_ASSERT_TRUE_MESSAGE(
        seg_file_exists(wal_path, 3),
        "seg-at-3: active segment, MUST always be preserved");

    repl_broadcast_send_hook = NULL;
    teardown_ckpt_e2e(p);
    printf("test_trim_guard_preserves_standby_segments: PASS\n");
}

/* ------------------------------------------------------------------ */
/* Test 5: trim with no standbys proceeds as before — Req 1.1          */
/*         (task 3.2).  With no standbys registered, the floor         */
/*         defaults to active_seg_lsn, so all old segments are         */
/*         trimmed.  This confirms the guard does not prevent normal   */
/*         cleanup when no standbys need protection.                    */
/* ------------------------------------------------------------------ */

static void test_trim_without_standbys_proceeds_normally(void)
{
    char tmpdir[256];
    persist_t* p = make_ckpt_e2e(tmpdir, sizeof(tmpdir), 2);
    TEST_ASSERT_NOT_NULL(p);

    /* No standbys — just create segments and checkpoint */
    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
    TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));
    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
    TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));
    /* bug-1426: simulate the local apply pump having caught up (see
     * test_trim_guard_preserves_standby_segments's identical note above). */
    atomic_store_explicit(&p->last_applied_lsn, p->current_lsn - 1,
                          memory_order_relaxed);
    TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));

    char wal_path[512];
    snprintf(wal_path, sizeof(wal_path), "%s/e2e-ckpt/data/wal", tmpdir);

    /* Without standbys, floor = active_seg_lsn = 3.
     * seg_at_1 < 3 → deleted.  seg_at_2 < 3 → deleted too. */
    TEST_ASSERT_FALSE_MESSAGE(
        seg_file_exists(wal_path, 1),
        "seg-at-1: no standbys, MUST be trimmed");

    TEST_ASSERT_FALSE_MESSAGE(
        seg_file_exists(wal_path, 2),
        "seg-at-2: no standbys, MUST be trimmed");

    /* Active segment still there */
    TEST_ASSERT_TRUE_MESSAGE(
        seg_file_exists(wal_path, 3),
        "seg-at-3: active segment, MUST always be preserved");

    teardown_ckpt_e2e(p);
    printf("test_trim_without_standbys_proceeds_normally: PASS\n");
}

/* ------------------------------------------------------------------ */
/* Test 6: local apply-cursor trim guard — bug-1426. With no standbys   */
/* registered (so the standby-floor guard alone would allow trimming   */
/* everything below the active segment), a LAGGING local apply pump     */
/* (last_applied_lsn below the active segment) must still block         */
/* trimming of segments it hasn't caught up to yet.                     */
/*                                                                      */
/* RED proof: revert the last_applied_lsn floor cap in                  */
/* trim_wal_segments -> seg-at-2 (which the lagging pump hasn't read     */
/* yet) is deleted anyway, because the standby-only floor (= active_seg  */
/* _lsn=3, no standbys) never considers local apply progress -> a        */
/* concurrent live apply-pump reader relying on an already-open cached   */
/* fd into that segment (POSIX unlink-keeps-fd-alive) would silently     */
/* keep succeeding, while any FRESH read of the same range (a restart's  */
/* persist_recover() replay) would find it already gone -- the apply-    */
/* correctness gap this bug describes.                                  */
/* ------------------------------------------------------------------ */

static void test_trim_guard_preserves_unapplied_local_segments(void)
{
    char tmpdir[256];
    persist_t* p = make_ckpt_e2e(tmpdir, sizeof(tmpdir), 2);
    TEST_ASSERT_NOT_NULL(p);

    /* No standbys registered — the pre-bug-1426 floor would be
     * active_seg_lsn=3 alone, same as test_trim_without_standbys_proceeds_
     * normally above. This time, deliberately advance last_applied_lsn only
     * through the entry in seg-at-1 (LSN 1), NOT the one in seg-at-2 (LSN
     * 2) — simulating a live apply pump that is still lagging (e.g. mid
     * catch-up burst) when the checkpoint's trim fires. seg-at-1 is fully
     * consumed and safe to delete; seg-at-2 is not — the pump hasn't
     * reached its entry yet. */
    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
    TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));
    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
    TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));
    atomic_store_explicit(&p->last_applied_lsn, 1, memory_order_relaxed);

    TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));

    char wal_path[512];
    snprintf(wal_path, sizeof(wal_path), "%s/e2e-ckpt/data/wal", tmpdir);

    /* bug-1426 floor = min(active_seg_lsn=3, last_applied_lsn+1=2) = 2.
     * seg_at_1 (start=1) < 2 → deleted: its one entry (LSN 1) is fully
     * applied, safe to reclaim. seg_at_2 (start=2) < 2 is FALSE →
     * preserved: its entry (LSN 2) has NOT been applied yet — deleting it
     * would strand data the local apply pump still needs. */
    TEST_ASSERT_FALSE_MESSAGE(
        seg_file_exists(wal_path, 1),
        "seg-at-1: fully applied (last_applied_lsn=1), safe to trim even "
        "though no standby needs it either");
    TEST_ASSERT_TRUE_MESSAGE(
        seg_file_exists(wal_path, 2),
        "seg-at-2: NOT yet applied by the lagging local pump "
        "(last_applied_lsn=1 < 2), MUST be preserved (bug-1426 guard) even "
        "though no standby needs it");
    TEST_ASSERT_TRUE_MESSAGE(
        seg_file_exists(wal_path, 3),
        "seg-at-3: active segment, MUST always be preserved");

    teardown_ckpt_e2e(p);
    printf("test_trim_guard_preserves_unapplied_local_segments: PASS\n");
}

/* ------------------------------------------------------------------ */
/* Suite entry point                                                    */
/* ------------------------------------------------------------------ */

void test_e2e_checkpoint(void)
{
    UnitySetTestFile(__FILE__);
    printf("\n>> P0-15b: Checkpoint E2E — round trip\n\n");

    RUN_TEST(test_checkpoint_recovery_lsn_continuity);
    RUN_TEST(test_checkpoint_wal_trim_after_checkpoint);
    RUN_TEST(test_snapshot_rotation_keeps_snap_keep_n);
    RUN_TEST(test_trim_guard_preserves_standby_segments);
    RUN_TEST(test_trim_without_standbys_proceeds_normally);
    RUN_TEST(test_trim_guard_preserves_unapplied_local_segments);
}