diff --git a/src/controller/ctrl_metrics.c b/src/controller/ctrl_metrics.c new file mode 100644 index 00000000..c63d2696 --- /dev/null +++ b/src/controller/ctrl_metrics.c @@ -0,0 +1,68 @@ +/* + * ctrl_metrics.c — P3-7: Operational metrics for the XCE controller. + * + * Defines the 7 atomic counters/gauges and the descriptor table used + * for metric discovery. All variables are initialised to 0 at program + * start (C guarantees this for file-scope variables), and are zeroed + * explicitly by ctrl_metrics_init() which is called from persist_start() + * so that tests can call it repeatedly without interference. + */ +#include "ctrl_metrics.h" + +#include +#include +#include + +/* ── Counter/gauge definitions ──────────────────────────────────────── */ +_Atomic uint64_t ctrl_repl_degraded_count = 0; +_Atomic uint64_t ctrl_split_brain_detected = 0; +_Atomic uint64_t ctrl_async_events_lost = 0; +_Atomic uint64_t ctrl_transient_jobs_stuck = 0; +_Atomic uint64_t ctrl_wal_bytes = 0; +_Atomic uint64_t ctrl_checkpoint_duration_ms = 0; +_Atomic uint64_t ctrl_repl_lag_lsn = 0; + +/* ── Descriptor table ───────────────────────────────────────────────── */ +const ctrl_metric_desc_t ctrl_metric_descriptors[] = { + { "ctrl_repl_degraded_count", + &ctrl_repl_degraded_count, + "Sync replication fallbacks to local-only durability", + 1 }, + { "ctrl_split_brain_detected", + &ctrl_split_brain_detected, + "Term conflicts indicating split-brain risk", + 1 }, + { "ctrl_async_events_lost", + &ctrl_async_events_lost, + "Async WAL entries lost during failover", + 1 }, + { "ctrl_transient_jobs_stuck", + &ctrl_transient_jobs_stuck, + "TRANSIENT jobs auto-terminated after timeout", + 1 }, + { "ctrl_wal_bytes", + &ctrl_wal_bytes, + "Current WAL size in bytes", + 0 }, + { "ctrl_checkpoint_duration_ms", + &ctrl_checkpoint_duration_ms, + "Duration of last checkpoint in milliseconds", + 0 }, + { "ctrl_repl_lag_lsn", + &ctrl_repl_lag_lsn, + "LSN gap between primary and slowest standby", + 0 }, + { NULL, NULL, NULL, 0 } /* sentinel */ +}; + +/* ── ctrl_metrics_init ──────────────────────────────────────────────── */ +void ctrl_metrics_init(void) +{ + atomic_store_explicit(&ctrl_repl_degraded_count, 0, memory_order_relaxed); + atomic_store_explicit(&ctrl_split_brain_detected, 0, memory_order_relaxed); + atomic_store_explicit(&ctrl_async_events_lost, 0, memory_order_relaxed); + atomic_store_explicit(&ctrl_transient_jobs_stuck, 0, memory_order_relaxed); + atomic_store_explicit(&ctrl_wal_bytes, 0, memory_order_relaxed); + atomic_store_explicit(&ctrl_checkpoint_duration_ms, 0, memory_order_relaxed); + atomic_store_explicit(&ctrl_repl_lag_lsn, 0, memory_order_relaxed); +} diff --git a/src/controller/meson.build b/src/controller/meson.build index 1f30b428..eb3e0deb 100644 --- a/src/controller/meson.build +++ b/src/controller/meson.build @@ -19,6 +19,7 @@ controller_src = [ 'myconf.c', 'persist.c', 'repl.c', + 'ctrl_metrics.c', ] incdir = include_directories('.', '../include', '../lib', @@ -95,9 +96,11 @@ controller_unit_test_src = [ 'testsuite/test_p2_integration.c', 'testsuite/test_p3_hardening.c', 'testsuite/test_p3_manual_failover.c', + 'testsuite/test_ctrl_metrics.c', 'persist.c', 'ha.c', 'repl.c', + 'ctrl_metrics.c', ] controller_unit_test_deps = [ diff --git a/src/controller/persist.c b/src/controller/persist.c index c97de0e8..24c2e4ec 100644 --- a/src/controller/persist.c +++ b/src/controller/persist.c @@ -1,6 +1,7 @@ #define _GNU_SOURCE #include "persist.h" #include "config.h" +#include "ctrl_metrics.h" #include "hook.h" #include "xce/cluster_def.h" #include "xce/types.h" @@ -179,6 +180,8 @@ int persist_init(persist_t* p, persist_config_t* cfg, cluster_t* c) int persist_start(persist_t* p) { + ctrl_metrics_init(); + /* Register AFTER_EVENT persistence hooks if we have a cluster back-pointer */ if (p->cluster != NULL) { xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_JOB, @@ -869,6 +872,8 @@ int persist_wal_sync_append(persist_t* p, uint32_t type, int sync_err = repl_wait_ack(&p->repl, written_lsn, p->config.sync_timeout_ms); + if (sync_err < 0) + CTRL_METRIC_INC(ctrl_repl_degraded_count); if (sync_rc_out) *sync_rc_out = sync_err; /* -ETIMEDOUT = degraded mode (local-only durability); -ENODEV = no standbys. diff --git a/src/controller/testsuite/controller_test.c b/src/controller/testsuite/controller_test.c index c3885fab..907ce3b7 100644 --- a/src/controller/testsuite/controller_test.c +++ b/src/controller/testsuite/controller_test.c @@ -30,6 +30,7 @@ extern void test_repl_catchup(void); extern void test_p2_integration(void); extern void test_p3_hardening(void); extern void test_p3_manual_failover(void); +extern void test_ctrl_metrics(void); void setUp(void) { return; } @@ -129,6 +130,7 @@ int main(int argc, char* argv[]) TEST(test_p2_integration); TEST(test_p3_hardening); TEST(test_p3_manual_failover); + TEST(test_ctrl_metrics); return UNITY_END(); } diff --git a/src/controller/testsuite/test_ctrl_metrics.c b/src/controller/testsuite/test_ctrl_metrics.c new file mode 100644 index 00000000..2c9f0956 --- /dev/null +++ b/src/controller/testsuite/test_ctrl_metrics.c @@ -0,0 +1,88 @@ +/* test_ctrl_metrics.c — P3-7: Metrics infrastructure unit tests + * + * TDD: these tests are written FIRST to drive the implementation. + * They verify: + * - ctrl_metrics_init() zeroes all 7 counters/gauges + * - CTRL_METRIC_INC increments atomically + * - CTRL_METRIC_SET writes and overwrites a gauge + * - The descriptor table has exactly 7 entries + NULL sentinel + * - ctrl_transient_jobs_stuck counter accumulates correctly + */ +#define _GNU_SOURCE +#include +#include +#include + +#include + +#include "ctrl_metrics.h" + +/* ------------------------------------------------------------------ */ +/* Tests */ +/* ------------------------------------------------------------------ */ + +static void test_metrics_init_zeros_all_counters(void) +{ + ctrl_metrics_init(); + TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&ctrl_repl_degraded_count)); + TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&ctrl_split_brain_detected)); + TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&ctrl_async_events_lost)); + TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&ctrl_transient_jobs_stuck)); + TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&ctrl_wal_bytes)); + TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&ctrl_checkpoint_duration_ms)); + TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&ctrl_repl_lag_lsn)); + printf("test_metrics_init_zeros_all_counters: PASS\n"); +} + +static void test_metrics_increment_repl_degraded(void) +{ + ctrl_metrics_init(); + CTRL_METRIC_INC(ctrl_repl_degraded_count); + CTRL_METRIC_INC(ctrl_repl_degraded_count); + TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&ctrl_repl_degraded_count)); + printf("test_metrics_increment_repl_degraded: PASS\n"); +} + +static void test_metrics_set_gauge(void) +{ + ctrl_metrics_init(); + CTRL_METRIC_SET(ctrl_wal_bytes, 1024 * 1024); + TEST_ASSERT_EQUAL_UINT64(1024 * 1024, atomic_load(&ctrl_wal_bytes)); + CTRL_METRIC_SET(ctrl_wal_bytes, 0); + TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&ctrl_wal_bytes)); + printf("test_metrics_set_gauge: PASS\n"); +} + +static void test_metrics_descriptor_table_has_sentinel(void) +{ + /* The descriptor table must end with a NULL-name sentinel + * and contain exactly 7 real entries. */ + int i = 0; + while (ctrl_metric_descriptors[i].name != NULL) + i++; + TEST_ASSERT_EQUAL_INT(7, i); + printf("test_metrics_descriptor_table_has_sentinel: PASS\n"); +} + +static void test_transient_jobs_stuck_counter(void) +{ + ctrl_metrics_init(); + /* Simulate 3 stuck transient jobs */ + for (int j = 0; j < 3; j++) + CTRL_METRIC_INC(ctrl_transient_jobs_stuck); + TEST_ASSERT_EQUAL_UINT64(3, atomic_load(&ctrl_transient_jobs_stuck)); + printf("test_transient_jobs_stuck_counter: PASS\n"); +} + +/* ------------------------------------------------------------------ */ +/* Suite entry point (called from controller_test.c) */ +/* ------------------------------------------------------------------ */ + +void test_ctrl_metrics(void) +{ + RUN_TEST(test_metrics_init_zeros_all_counters); + RUN_TEST(test_metrics_increment_repl_degraded); + RUN_TEST(test_metrics_set_gauge); + RUN_TEST(test_metrics_descriptor_table_has_sentinel); + RUN_TEST(test_transient_jobs_stuck_counter); +} diff --git a/src/include/ctrl_metrics.h b/src/include/ctrl_metrics.h new file mode 100644 index 00000000..47520bd1 --- /dev/null +++ b/src/include/ctrl_metrics.h @@ -0,0 +1,47 @@ +#pragma once +/* + * ctrl_metrics.h — P3-7: Operational metrics for the XCE controller. + * + * All metrics use C11 _Atomic uint64_t for lock-free, thread-safe access. + * Metric names use the ctrl_ prefix. + * + * Design doc: Part P3-7, review A8-007. + */ +#include +#include + +/* ── Counter/gauge globals ──────────────────────────────────────────── */ +extern _Atomic uint64_t ctrl_repl_degraded_count; /* counter */ +extern _Atomic uint64_t ctrl_split_brain_detected; /* counter */ +extern _Atomic uint64_t ctrl_async_events_lost; /* counter */ +extern _Atomic uint64_t ctrl_transient_jobs_stuck; /* counter */ +extern _Atomic uint64_t ctrl_wal_bytes; /* gauge */ +extern _Atomic uint64_t ctrl_checkpoint_duration_ms; /* gauge */ +extern _Atomic uint64_t ctrl_repl_lag_lsn; /* gauge */ + +/* ── Macros ─────────────────────────────────────────────────────────── */ + +/* Increment a monotonic counter by 1 (relaxed ordering — counters are + * sampled, not used for synchronization). */ +#define CTRL_METRIC_INC(name) \ + atomic_fetch_add_explicit(&(name), 1, memory_order_relaxed) + +/* Overwrite a gauge with a new absolute value. */ +#define CTRL_METRIC_SET(name, val) \ + atomic_store_explicit(&(name), (uint64_t)(val), memory_order_relaxed) + +/* ── Descriptor table for metric discovery ──────────────────────────── */ +typedef struct { + const char* name; + _Atomic uint64_t* value; + const char* description; + int is_counter; /* 1 = counter (monotonic), 0 = gauge */ +} ctrl_metric_desc_t; + +/* Terminated by a { NULL, NULL, NULL, 0 } sentinel entry. */ +extern const ctrl_metric_desc_t ctrl_metric_descriptors[]; + +/* ── Lifecycle ──────────────────────────────────────────────────────── */ + +/* Reset all metrics to zero. Call from persist_start() and ha_init(). */ +void ctrl_metrics_init(void);