forked from lei.chen/myxce
259 lines
12 KiB
C
259 lines
12 KiB
C
#pragma once
|
|
#include <pthread.h>
|
|
#include <stdarg.h>
|
|
#include <stdatomic.h>
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
|
|
#include "config.h"
|
|
#include "wal_types.h" /* wal_entry_hdr_t, wal_entry_type — P2-4: shared with repl.h */
|
|
#include "repl.h" /* replicator_t — P2-4: embedded in persist_t */
|
|
|
|
/* Forward declaration */
|
|
typedef struct cluster_s cluster_t;
|
|
|
|
/* ============================================================
|
|
* Persistence configuration (Part 2.6)
|
|
* ============================================================ */
|
|
typedef struct {
|
|
uint32_t wal_sync_ms; /* fsync interval in ms (default 10) */
|
|
uint32_t snap_interval_s; /* checkpoint interval in seconds (default 60) */
|
|
uint32_t snap_wal_size_mb; /* checkpoint when WAL exceeds this MB (default 64) */
|
|
uint32_t snap_keep_n; /* snapshots to retain (default 2) */
|
|
uint32_t transient_job_timeout_s; /* seconds before TRANSIENT job auto-terminates (default 300) */
|
|
uint32_t finished_retain_s; /* retain finished jobs N seconds (default 3600) */
|
|
uint32_t worker_retain_s; /* retain unavailable workers N seconds (default 86400) */
|
|
uint32_t sync_timeout_ms; /* sync replication ACK timeout ms (default 50) */
|
|
uint32_t max_wal_records; /* checkpoint trigger: records since last snap (default 10000) */
|
|
} persist_config_t;
|
|
|
|
static inline persist_config_t persist_config_defaults(void)
|
|
{
|
|
persist_config_t c;
|
|
c.wal_sync_ms = DEFAULT_WAL_SYNC_MS;
|
|
c.snap_interval_s = DEFAULT_SNAP_INTERVAL_S;
|
|
c.snap_wal_size_mb = DEFAULT_SNAP_WAL_SIZE_MB;
|
|
c.snap_keep_n = DEFAULT_SNAP_KEEP_N;
|
|
c.transient_job_timeout_s = DEFAULT_TRANSIENT_JOB_TIMEOUT_S;
|
|
c.finished_retain_s = DEFAULT_FINISHED_RETAIN_S;
|
|
c.worker_retain_s = DEFAULT_WORKER_RETAIN_S;
|
|
c.sync_timeout_ms = DEFAULT_SYNC_TIMEOUT_MS;
|
|
c.max_wal_records = DEFAULT_MAX_WAL_RECORDS;
|
|
return c;
|
|
}
|
|
|
|
/*
|
|
* persist_config_set_wal_sync_ms — apply the --wal-sync-ms flush-cadence
|
|
* override (ha-commit-durability Req 3.1).
|
|
*
|
|
* Single override point shared by both persist_config_defaults() callers
|
|
* (cluster.c phase2 and myapp.c step 6), mirroring the --sync-timeout-ms
|
|
* convention: override_ms == 0 means "flag not passed" and leaves the
|
|
* default flush cadence untouched (Req 3.3 — the seam changes timing
|
|
* controllability only, not the durability contract). A positive value
|
|
* sets the cadence to exactly that, applied before persistence starts.
|
|
*/
|
|
static inline void persist_config_set_wal_sync_ms(persist_config_t *c,
|
|
uint32_t override_ms)
|
|
{
|
|
if (override_ms > 0)
|
|
c->wal_sync_ms = override_ms;
|
|
}
|
|
|
|
/* ============================================================
|
|
* Key→index mapping for checkpoint serialization (Part 3.8)
|
|
* ============================================================ */
|
|
typedef struct {
|
|
int index; /* shadow slot index in the frozen zarray */
|
|
char key[XCE_MAXNAME_LEN]; /* owned deep copy — not a borrowed kvtab pointer */
|
|
size_t key_len; /* length including null terminator */
|
|
} slot_key_t;
|
|
|
|
/* ============================================================
|
|
* persist_t — concrete definition (exposed for stack/calloc allocation)
|
|
* The opaque pattern can be restored in a later refactor once the
|
|
* full API surface is stable.
|
|
* ============================================================ */
|
|
struct persist_s {
|
|
/* Back-pointer to owning cluster (set by persist_init, used in persist_start) */
|
|
cluster_t* cluster;
|
|
|
|
/* WAL state */
|
|
int wal_fd;
|
|
uint64_t current_lsn;
|
|
uint32_t last_crc;
|
|
uint32_t last_replicated_crc;
|
|
uint64_t term;
|
|
uint32_t seg_offset;
|
|
uint64_t checkpoint_lsn;
|
|
_Atomic uint64_t committed_lsn; /* highest LSN known committed; invariant: <= current_lsn - 1 */
|
|
uint64_t total_wal_bytes;
|
|
char seg_path[4096];
|
|
|
|
/* Paths (derived from workdir + name via persist_set_workdir) */
|
|
char wal_path[4096];
|
|
char snap_path[4096];
|
|
|
|
/* Config */
|
|
persist_config_t config;
|
|
|
|
/* Concurrency */
|
|
pthread_mutex_t lock;
|
|
atomic_uint wal_record_count; /* records appended since last checkpoint */
|
|
atomic_int checkpoint_in_progress; /* CAS guard: 0=idle, 1=scheduled/running */
|
|
atomic_int fsync_stop;
|
|
atomic_int fsync_thread_running;
|
|
pthread_t fsync_thread;
|
|
pthread_mutex_t fsync_mutex; /* guards the fsync timer wait */
|
|
pthread_cond_t fsync_cond; /* signaled by persist_stop for prompt join */
|
|
|
|
/* Checkpoint (stubs for now — wired in Task 10) */
|
|
atomic_int checkpoint_requested;
|
|
pthread_cond_t checkpoint_done;
|
|
pthread_mutex_t checkpoint_mutex;
|
|
pthread_t snap_thread;
|
|
|
|
/* Fan-out replicator — P2-4 */
|
|
replicator_t repl;
|
|
};
|
|
typedef struct persist_s persist_t;
|
|
|
|
/* Set workdir and controller name before persist_init().
|
|
* Derives wal_path and snap_path as <workdir>/<name>/data/{wal,snap}. */
|
|
void persist_set_workdir(persist_t* p, const char* workdir, const char* name);
|
|
|
|
/* Lifecycle */
|
|
int persist_init(persist_t* p, persist_config_t* config, cluster_t* c);
|
|
int persist_start(persist_t* p);
|
|
void persist_stop(persist_t* p);
|
|
void persist_fini(persist_t* p);
|
|
int persist_recover(persist_t* p);
|
|
|
|
/* WAL write paths */
|
|
int persist_wal_async(persist_t* p, uint32_t type,
|
|
const void* payload, uint32_t len);
|
|
int persist_wal_sync_append(persist_t* p, uint32_t type,
|
|
const void* payload, uint32_t len,
|
|
int* sync_rc_out);
|
|
|
|
/* Write a REPLICATED entry (received from the primary) into the local WAL
|
|
* at its GIVEN lsn — used by standbys to durably persist streamed entries
|
|
* BEFORE acking them (persist-before-ACK; an ACK must imply durability).
|
|
*
|
|
* Unlike persist_wal_async it does NOT assign a new lsn (the lsn is supplied
|
|
* by the primary) and does NOT repl_broadcast (standbys never re-replicate).
|
|
* It builds a SELF-consistent CRC chain (the streamed entry carries no crc),
|
|
* fsyncs the segment before returning, and is idempotent on re-stream: if the
|
|
* lsn is already persisted (lsn+1 <= current_lsn) it is a no-op returning 0.
|
|
*
|
|
* WAL_CHECKPOINT is rejected with -EINVAL (never streamed to standbys).
|
|
* Returns 0 on success (incl. dedup no-op), negative errno on failure. */
|
|
int persist_wal_write_replicated(persist_t* p, uint64_t lsn, uint32_t type,
|
|
const void* payload, uint32_t len);
|
|
|
|
/* ============================================================
|
|
* Write-path commit classification (ha-commit-barrier-rollout)
|
|
* ============================================================ */
|
|
typedef enum {
|
|
WAL_MODE_WAIT, /* request-shaped: barrier before requester reply */
|
|
WAL_MODE_RETAIN, /* fact-shaped: retained until committed, no timeout */
|
|
WAL_MODE_LOCAL, /* not replicated (WAL_CHECKPOINT) */
|
|
} wal_commit_mode_t;
|
|
|
|
/* Pure classification lookup. detail disambiguates WAL_WORKER_CTRL only
|
|
* (HOOK_DETAIL_WORKER_JOIN/UPDATE -> WAIT, HOOK_DETAIL_WORKER_DEAD ->
|
|
* RETAIN). Unknown type: XCE_TESTING assert (fail fast); production logs
|
|
* ERROR and returns WAL_MODE_RETAIN (never dropped, never confirmed early). */
|
|
wal_commit_mode_t wal_classify(uint32_t wal_type, int detail);
|
|
|
|
/* Commit barrier — handler-callable WAIT completion on everything appended
|
|
* so far (ha-commit-barrier-rollout). Snapshots the appended watermark
|
|
* (current_lsn - 1, under p->lock — matching persist_wal_sync_append's
|
|
* read); returns 0 without blocking when committed_lsn already covers it
|
|
* (or nothing was ever appended). Otherwise delegates to
|
|
* commit_submit(COMMIT_MODE_WAIT) and returns its code unmodified:
|
|
* 0 — committed on a majority
|
|
* -ETIMEDOUT — not committed within the deadline (entries remain
|
|
* in the log; outcome converges per umbrella 2.5)
|
|
* -XCE_ERR_NOT_READY — engine admission cap exhausted (retriable)
|
|
* timeout_ms = 0 falls back to p->config.sync_timeout_ms. n_peers <= 1
|
|
* returns 0 immediately (engine self-quorum, standalone preserved).
|
|
* Call only from vtask threads with no entity locks held. */
|
|
int persist_commit_barrier(persist_t* p, uint32_t timeout_ms);
|
|
|
|
/* Force WAL fdatasync (for tests and clean shutdown) */
|
|
int persist_flush(persist_t* p);
|
|
|
|
#ifdef XCE_TESTING
|
|
/* ============================================================
|
|
* Durability-verification test seam (ha-commit-durability Req 3.2, 3.3)
|
|
*
|
|
* persist_test_durable_lsn_on_disk — re-read and CRC-validate the WAL
|
|
* segments from STABLE STORAGE and return the highest fully-persisted
|
|
* LSN, independent of the in-memory current_lsn / committed_lsn
|
|
* watermarks. Lets a test assert that an acknowledged operation is
|
|
* recoverable from disk after an immediate, ungraceful termination —
|
|
* without relying on the background flush timer (Req 3.2):
|
|
*
|
|
* persist_wal_async(p, ...); // append to page cache
|
|
* persist_flush(p); // fdatasync → durable on disk
|
|
* lsn = persist_test_durable_lsn_on_disk(p); // highest durable LSN
|
|
*
|
|
* It scans the same on-disk format and CRC chain as recovery, stopping
|
|
* at the first corruption / short read / pre-allocated zero region, so
|
|
* it reports only entries that are genuinely on stable storage. It
|
|
* never mutates persist_t state and fires no hooks (Req 3.3 — the seam
|
|
* is purely inspective; the acknowledgment contract is unchanged).
|
|
*
|
|
* Returns the highest durable LSN, or 0 when nothing is durable (empty
|
|
* first-boot WAL, no segments). Compiled out of production builds:
|
|
* the xce_controller target is built without -DXCE_TESTING and links
|
|
* without this symbol.
|
|
* ============================================================ */
|
|
uint64_t persist_test_durable_lsn_on_disk(persist_t* p);
|
|
#endif /* XCE_TESTING */
|
|
|
|
/* Checkpoint */
|
|
int persist_checkpoint(persist_t* p);
|
|
|
|
/* WAL truncation — remove all entries with LSN > cutoff_lsn.
|
|
* Idempotent; returns 0 on success, negative errno on I/O error. */
|
|
int persist_truncate_wal_above(persist_t* p, uint64_t cutoff_lsn);
|
|
|
|
/* Logging helper — never logs payload content (security invariant) */
|
|
void persist_log_entry_metadata(const wal_entry_hdr_t* hdr);
|
|
|
|
/* ============================================================
|
|
* AFTER_EVENT hook functions — registered by persist_start()
|
|
*
|
|
* Signature matches hook_fn_t: int fn(int type, int situation,
|
|
* int detail, void* data, va_list ap)
|
|
* cls/data points to the persist_t instance.
|
|
*
|
|
* P0: writes a minimal 4-byte payload (detail value) to the WAL.
|
|
* Full Cap'n Proto serialization is wired in Task 15.
|
|
* ============================================================ */
|
|
int persist_job_hook_fun(int type, int situation, int detail,
|
|
void* cls, va_list ap);
|
|
int persist_queue_hook_fun(int type, int situation, int detail,
|
|
void* cls, va_list ap);
|
|
int persist_worker_hook_fun(int type, int situation, int detail,
|
|
void* cls, va_list ap);
|
|
|
|
/* Apply a single WAL entry to in-memory state (standby replay path).
|
|
* No hooks fire — hook system is empty on standbys until promotion.
|
|
* Returns 0 on success (including skip for checkpoint/unknown types),
|
|
* negative errno on error (oversized payload, NULL arguments). */
|
|
int apply_wal_entry(persist_t* p, const wal_entry_hdr_t* hdr, const void* payload);
|
|
|
|
/* Load a snapshot file from path, set p->checkpoint_lsn and p->current_lsn.
|
|
* lsn hint is informational — the actual LSN is read from the file header.
|
|
* Returns 0 on success, -errno or -EINVAL/-EIO on error. */
|
|
int persist_load_snapshot(persist_t* p, const char* path, uint64_t lsn);
|
|
|
|
/* GC retention predicates (P0-5a / P0-8 / P0-13) */
|
|
#include <stdbool.h>
|
|
#include "xce/types.h"
|
|
bool job_is_terminal(const job_t* j);
|
|
bool job_should_retain(const job_t* j, const persist_config_t* cfg);
|
|
bool worker_should_retain(const xce_worker_t* w, const persist_config_t* cfg);
|