myxce/src/controller/testsuite/test_persist_checkpoint.c

#define _GNU_SOURCE
#include <assert.h>
#include <dirent.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>

#include <unity/unity.h>

#include "config.h"
#include "persist.h"

/* ------------------------------------------------------------------ */
/* Test-local helpers                                                   */
/* ------------------------------------------------------------------ */

/* Count files with a given suffix in a directory */
static int count_files_with_suffix(const char* dir, const char* suffix)
{
    DIR* d = opendir(dir);
    if (!d) return 0;
    int n = 0;
    size_t slen = strlen(suffix);
    struct dirent* e;
    while ((e = readdir(d)) != NULL) {
        size_t len = strlen(e->d_name);
        if (len > slen && strcmp(e->d_name + len - slen, suffix) == 0)
            n++;
    }
    closedir(d);
    return n;
}

static persist_t* make_ckpt_persist(char* tmpdir_buf, size_t buf_sz)
{
    snprintf(tmpdir_buf, buf_sz, "/tmp/xce_ckpt_XXXXXX");
    if (!mkdtemp(tmpdir_buf)) return NULL;

    persist_t* p = calloc(1, sizeof(persist_t));
    if (!p) return NULL;

    persist_config_t cfg = persist_config_defaults();
    cfg.wal_sync_ms   = 5000; /* slow background fsync — prevent races */
    cfg.snap_keep_n   = 2;

    persist_set_workdir(p, tmpdir_buf, "ctrl-test");

    if (persist_init(p, &cfg, NULL) != 0) {
        free(p);
        return NULL;
    }
    if (persist_start(p) != 0) {
        persist_fini(p);
        free(p);
        return NULL;
    }
    return p;
}

static void cleanup_ckpt_persist(persist_t* p)
{
    persist_stop(p);
    persist_fini(p);
    free(p);
}

/* ------------------------------------------------------------------ */
/* Test 1: checkpoint creates at least one snapshot file               */
/* ------------------------------------------------------------------ */

static void test_checkpoint_creates_snapshot_file(void)
{
    char tmpdir[256];
    persist_t* p = make_ckpt_persist(tmpdir, sizeof(tmpdir));
    TEST_ASSERT_NOT_NULL(p);

    /* Write a WAL entry so there is something to checkpoint */
    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "data", 4));

    /* Trigger checkpoint */
    TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));

    /* Verify at least one .capnp snapshot file was created */
    char snap_path[512];
    snprintf(snap_path, sizeof(snap_path), "%s/ctrl-test/data/snap", tmpdir);
    int n = count_files_with_suffix(snap_path, ".capnp");
    TEST_ASSERT_TRUE(n >= 1);

    cleanup_ckpt_persist(p);
    printf("test_checkpoint_creates_snapshot_file: PASS\n");
}

/* ------------------------------------------------------------------ */
/* Test 2: checkpoint advances checkpoint_lsn                          */
/* ------------------------------------------------------------------ */

static void test_checkpoint_sets_checkpoint_lsn(void)
{
    char tmpdir[256];
    persist_t* p = make_ckpt_persist(tmpdir, sizeof(tmpdir));
    TEST_ASSERT_NOT_NULL(p);

    /* Write 3 WAL entries */
    for (int i = 0; i < 3; i++)
        persist_wal_async(p, WAL_JOB_UPDATE, "x", 1);

    uint64_t lsn_before = p->current_lsn;

    /* No live apply pump runs in this cluster-less unit-test fixture, so
     * last_applied_lsn stays 0 unless a test drives it directly — simulate
     * apply having fully caught up to what was appended (persist_checkpoint
     * now stamps checkpoint_lsn from last_applied_lsn, not current_lsn;
     * see the checkpoint_lsn-vs-last_applied_lsn fix). */
    atomic_store_explicit(&p->last_applied_lsn, lsn_before,
                           memory_order_relaxed);

    TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));

    /* checkpoint_lsn must have advanced to at least lsn_before */
    TEST_ASSERT_TRUE(p->checkpoint_lsn >= lsn_before);

    cleanup_ckpt_persist(p);
    printf("test_checkpoint_sets_checkpoint_lsn: PASS\n");
}

/* ------------------------------------------------------------------ */
/* Test 2b: checkpoint_lsn tracks the APPLIED cursor, not the WAL tip  */
/* ------------------------------------------------------------------ */

/* Regression test for the checkpoint_lsn/last_applied_lsn mislabeling bug
 * (persist.h last_applied_lsn doc, "DISTINCT from current_lsn"): when the
 * apply pump lags behind the physical WAL append tip -- the normal state
 * under raft-apply-decoupling once a commit resolves before its entry is
 * dispatched into jobmgr -- a checkpoint must label the snapshot with the
 * position jobmgr ACTUALLY reflects (last_applied_lsn), never the tip
 * (current_lsn). Stamping current_lsn here would seed the next recovery's
 * last_applied_lsn/replay-resume point past entries jobmgr never received,
 * permanently skipping them (the root cause behind bug-1419's apply-divergent
 * fence on kill-mid-apply). */
static void test_checkpoint_lsn_tracks_last_applied_not_current_lsn(void)
{
    char tmpdir[256];
    persist_t* p = make_ckpt_persist(tmpdir, sizeof(tmpdir));
    TEST_ASSERT_NOT_NULL(p);

    /* Append 5 entries, but only simulate the apply pump having dispatched
     * the first 2 of them (jobmgr reflects up through lsn_before + 1). */
    uint64_t lsn_before = p->current_lsn;
    for (int i = 0; i < 5; i++)
        persist_wal_async(p, WAL_JOB_UPDATE, "z", 1);

    TEST_ASSERT_EQUAL_UINT64(lsn_before + 5, p->current_lsn);

    const uint64_t simulated_apply_cursor = lsn_before + 1;
    atomic_store_explicit(&p->last_applied_lsn, simulated_apply_cursor,
                           memory_order_relaxed);

    TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));

    /* checkpoint_lsn must equal the applied cursor, NOT the WAL tip -- a
     * checkpoint_lsn of 5 here would mean recovery resumes replay above
     * LSN 5, silently dropping entries 3-5 that jobmgr never applied. */
    TEST_ASSERT_EQUAL_UINT64(simulated_apply_cursor, p->checkpoint_lsn);
    TEST_ASSERT_TRUE(p->checkpoint_lsn < p->current_lsn);

    cleanup_ckpt_persist(p);
    printf("test_checkpoint_lsn_tracks_last_applied_not_current_lsn: PASS\n");
}

/* ------------------------------------------------------------------ */
/* Test 3 (bug-1870): a checkpoint writes no WAL_CHECKPOINT marker and    */
/* consumes no LSN -- a local-only marker in the replicated log left a    */
/* hole standbys never had and stalled replication.                       */
/* ------------------------------------------------------------------ */

static void test_checkpoint_writes_no_marker_to_wal(void)
{
    char tmpdir[256];
    persist_t* p = make_ckpt_persist(tmpdir, sizeof(tmpdir));
    TEST_ASSERT_NOT_NULL(p);

    persist_wal_async(p, WAL_JOB_UPDATE, "z", 1);
    uint64_t lsn_before = p->current_lsn;
    atomic_store_explicit(&p->last_applied_lsn, lsn_before - 1,
                          memory_order_relaxed);
    TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));

    TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before, p->current_lsn,
        "a checkpoint must not consume a replicated-log LSN");

    /* the next real entry takes the very next LSN -- no hole */
    persist_wal_async(p, WAL_JOB_UPDATE, "z", 1);
    TEST_ASSERT_EQUAL_UINT64(lsn_before + 1, p->current_lsn);

    cleanup_ckpt_persist(p);
    printf("test_checkpoint_writes_no_marker_to_wal: PASS\n");
}

/* ------------------------------------------------------------------ */
/* bug-1870 follow-up: recovering from a snapshot at X with no WAL entry */
/* above it must resume at LSN X+1. load_snapshot seeded current_lsn = X */
/* (the NEXT lsn to write) -- hidden while every checkpoint also wrote a */
/* WAL_CHECKPOINT marker above X; without the marker a standby recovered */
/* one short, took entry X+1 for a gap and refused it forever (the       */
/* --config-upgrade soak's force-elect stall).                           */
/* ------------------------------------------------------------------ */

static void test_recover_from_snapshot_without_wal_resumes_after_it(void)
{
    char tmpdir[256];
    persist_t* p = make_ckpt_persist(tmpdir, sizeof(tmpdir));
    TEST_ASSERT_NOT_NULL(p);

    for (int i = 0; i < 3; i++)
        persist_wal_async(p, WAL_JOB_UPDATE, "z", 1);        /* lsn 1..3 */
    TEST_ASSERT_EQUAL_UINT64(4, p->current_lsn);
    atomic_store_explicit(&p->last_applied_lsn, 3, memory_order_relaxed);
    TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
    TEST_ASSERT_EQUAL_UINT64(3, p->checkpoint_lsn);
    TEST_ASSERT_EQUAL_INT(0, persist_flush(p));

    char walpath[4096];
    snprintf(walpath, sizeof(walpath), "%s", p->wal_path);
    cleanup_ckpt_persist(p);

    /* No WAL entry above (or at) the checkpoint survives. */
    char cmd[4200];
    snprintf(cmd, sizeof(cmd), "rm -f %s/seg-*.wal", walpath);
    TEST_ASSERT_EQUAL_INT(0, system(cmd));

    persist_t* p2 = calloc(1, sizeof(persist_t));
    TEST_ASSERT_NOT_NULL(p2);
    persist_config_t cfg = persist_config_defaults();
    cfg.wal_sync_ms = 5000;
    cfg.snap_keep_n = 2;
    persist_set_workdir(p2, tmpdir, "ctrl-test");
    TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL));
    TEST_ASSERT_EQUAL_INT(0, persist_start(p2));
    TEST_ASSERT_EQUAL_INT(0, persist_recover(p2));

    TEST_ASSERT_EQUAL_UINT64(3, p2->checkpoint_lsn);
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(4, p2->current_lsn,
        "after a snapshot covering 1..3, the next lsn to write is 4");
    TEST_ASSERT_EQUAL_UINT64(3, persist_durable_floor(p2));

    cleanup_ckpt_persist(p2);
}

/* ------------------------------------------------------------------ */
/* Test 4: concurrent checkpoint returns -EBUSY                        */
/* ------------------------------------------------------------------ */

static void test_checkpoint_busy_guard(void)
{
    char tmpdir[256];
    persist_t* p = make_ckpt_persist(tmpdir, sizeof(tmpdir));
    TEST_ASSERT_NOT_NULL(p);

    /* Simulate a checkpoint already in-progress */
    atomic_store(&p->checkpoint_in_progress, 1);

    int rc = persist_checkpoint(p);
    TEST_ASSERT_EQUAL_INT(-EBUSY, rc);

    /* Clean up: allow stop to proceed */
    atomic_store(&p->checkpoint_in_progress, 0);
    cleanup_ckpt_persist(p);
    printf("test_checkpoint_busy_guard: PASS\n");
}

/* ------------------------------------------------------------------ */
/* Test 5: snapshot rotation keeps at most snap_keep_n files           */
/* ------------------------------------------------------------------ */

static void test_checkpoint_rotation_keeps_snap_keep_n(void)
{
    char tmpdir[256];
    persist_t* p = make_ckpt_persist(tmpdir, sizeof(tmpdir));
    TEST_ASSERT_NOT_NULL(p);

    /* snap_keep_n is 2; run 4 checkpoints */
    for (int i = 0; i < 4; i++) {
        persist_wal_async(p, WAL_JOB_UPDATE, "y", 1);
        TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
    }

    char snap_path[512];
    snprintf(snap_path, sizeof(snap_path), "%s/ctrl-test/data/snap", tmpdir);
    int n = count_files_with_suffix(snap_path, ".capnp");
    TEST_ASSERT_TRUE(n <= (int)p->config.snap_keep_n);

    cleanup_ckpt_persist(p);
    printf("test_checkpoint_rotation_keeps_snap_keep_n: PASS\n");
}

/* ------------------------------------------------------------------ */
/* Test suite entry point                                               */
/* ------------------------------------------------------------------ */

void test_persist_checkpoint(void)
{
    UnitySetTestFile(__FILE__);
    RUN_TEST(test_checkpoint_creates_snapshot_file);
    RUN_TEST(test_checkpoint_sets_checkpoint_lsn);
    RUN_TEST(test_checkpoint_lsn_tracks_last_applied_not_current_lsn);
    RUN_TEST(test_checkpoint_writes_no_marker_to_wal);
    RUN_TEST(test_recover_from_snapshot_without_wal_resumes_after_it);
    RUN_TEST(test_checkpoint_busy_guard);
    RUN_TEST(test_checkpoint_rotation_keeps_snap_keep_n);
}