fix: spurious errors/warnings from a pc13v HA deploy (bug-1872..1877)

- bug-1872 (worker): "Failed to find any disk statistics in
  /proc/diskstats" ERROR every 15 s on an NVMe-only host. The whole-disk
  filter guessed from the name and skipped anything ending in a digit or
  containing 'p' -- every nvmeXnY, mmcblkN, even sdp -- so io was never
  reported. diskstats.h now counts a device only when
  /sys/block/<name>/device exists (real disks; not partitions, loop, dm,
  md, zram), and a host with no disk warns once.
- bug-1873 (worker): "Failed to get worker pid" WARN on job start. MainPID
  reads 0 until systemd has forked the new transient unit; xcecd.c already
  retries, so the miss is DEBUG and the final failure names the unit.
- bug-1874 (scheduler): deprecation WARN for the legacy xce_sched_fcfs on
  every start, though builtin_fcfs is the picker. Registration still wraps
  a job_elect_fn plugin but no longer warns; selecting one warns once.
- bug-1875 (apiserver): "Not running as root -- PAM authentication will not
  work" with the default provider config_file; now only for provider pam.
- bug-1876 (controller): standbys logged "cluster_init_phase2 ... failed
  (rc=-22)" WARN every 3 s while waiting for the primary's config push,
  a state the function itself calls normal. It returns -EAGAIN for that
  wait; callers log it at INFO, other failures still WARN.
- bug-1877 (controller): "WAL/snapshot stored unencrypted" warned without
  looking. persist_path_encrypted() checks fscrypt (statx) and dm-crypt
  (sysfs dm/uuid CRYPT-, through LVM slaves); the warning stays when the
  storage is not shown to be encrypted (e.g. a plain /tmp).

Tests: worker diskstats_test (2 of 4 RED on the old filter with pc13v's
/proc/diskstats), controller test_phase2_cold_boot and
test_persist_encryption, scheduler picker tests, apiserver
TestRootWarning. Full ASAN build: all 113 suites pass (controller 319/0,
harness 141/0).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

src/controller/cluster.c

@@ -1934,7 +1934,7 @@ int cluster_init_phase2(cluster_t* c)
         INFO("cluster_init_phase2: cluster_config not loaded yet — normal "
              "during cold boot while awaiting the primary's config-push; "
              "will retry once it arrives");
-        return -EINVAL;
+        return -EAGAIN; /* bug-1876: a wait, not -EINVAL */
     }
 
     SSL_CTX* ssl_ctx = (SSL_CTX*)c->ha.ssl_ctx;

src/controller/ha.c

@@ -433,8 +433,14 @@ int ha_phase2_retry_fn(void *arg)
         INFO("HA: cluster_init_phase2 retry succeeded — commit engine now "
              "wired (bug-1477)");
     } else {
-        WARN("HA: cluster_init_phase2 retry rc=%d — member stays alive and "
-             "will retry again (bug-1477, never self-terminate)", rc);
+        if (ha_phase2_rc_is_waiting(rc)) {
+            /* bug-1876: still waiting for the primary's config push */
+            INFO("HA: cluster_init_phase2 still waiting for the primary's "
+                 "config push — will retry");
+        } else {
+            WARN("HA: cluster_init_phase2 retry rc=%d — member stays alive "
+                 "and will retry again (bug-1477, never self-terminate)", rc);
+        }
         ha_schedule_phase2_retry(c);
     }
     return rc;

src/controller/ha.h

@@ -18,6 +18,7 @@
  * The watchdog (P1-3) will include netio.h directly where it needs endpoint_t.
  */
 
+#include <errno.h>
 #include <pthread.h>
 #include <semaphore.h>
 #include <stdbool.h>
@@ -952,6 +953,14 @@ int ha_phase2_retry_fn(void *arg);
  */
 int ha_schedule_phase2_retry(cluster_t *c);
 
+/* bug-1876: cluster_init_phase2()'s "cluster_config not pushed yet" result
+ * (-EAGAIN) -- the normal cold-boot wait of a standby, retried, not a
+ * failure; callers log it at INFO, everything else at WARN. */
+static inline int ha_phase2_rc_is_waiting(int rc)
+{
+    return rc == -EAGAIN;
+}
+
 /*
  * ha_apply_wal_catchup_entry: apply a single WAL catch-up entry to local
  * persistent storage and advance ha->applied_lsn on success.

src/controller/meson.build

@@ -154,6 +154,8 @@ controller_unit_test_src = [
   'testsuite/test_wal_job_start_fact.c',
   'testsuite/test_wal_job_finish_fact.c',
   'testsuite/test_wal_job_fail_fact.c',
+  'testsuite/test_phase2_cold_boot.c',
+  'testsuite/test_persist_encryption.c',
   'testsuite/test_config_queue_reconcile.c',
   'testsuite/test_wal_job_rusage_fact.c',
   'testsuite/test_wal_job_ckpt_fact.c',

src/controller/myapp.c

@@ -438,7 +438,13 @@ skip_election:;
              * node already loaded its config from disk pre-election. */
             if (!c.ha.phase2_done) {
                 int prc = cluster_init_phase2(&c);
-                if (prc != 0) {
+                if (ha_phase2_rc_is_waiting(prc)) {
+                    /* bug-1876: the normal cold-boot wait, not a failure */
+                    INFO("myapp_run: standby waiting for the primary's "
+                         "config push before cluster_init_phase2; will "
+                         "retry");
+                    ha_schedule_phase2_retry(&c);
+                } else if (prc != 0) {
                     WARN("myapp_run: cluster_init_phase2 on standby rejoin "
                          "failed (rc=%d) — WAL not recovered; member stays "
                          "alive and will retry (bug-1477: this passive "

src/controller/persist.c

@@ -27,6 +27,7 @@
 #include <string.h>
 #include <sys/stat.h>
 #include <sys/statfs.h>
+#include <sys/sysmacros.h> /* major/minor -- bug-1877 */
 #include <time.h>
 #include <unistd.h>
 #include <zlib.h>
@@ -1147,6 +1148,85 @@ void persist_set_workdir(persist_t* p, const char* workdir, const char* name)
     snprintf(p->snap_path, sizeof(p->snap_path), "%s/data/snap", effective);
 }
 
+static int dmcrypt_walk(const char* sysfs, unsigned maj, unsigned min,
+                        int depth)
+{
+    char           path[512];
+    char           uuid[128] = "";
+    FILE*          fp;
+    DIR*           d;
+    struct dirent* de;
+    int            found = 0;
+
+    if (depth > 8) {
+        return 0;
+    }
+
+    snprintf(path, sizeof(path), "%s/dev/block/%u:%u/dm/uuid", sysfs, maj, min);
+    fp = fopen(path, "r");
+    if (fp != NULL) {
+        if (fgets(uuid, sizeof(uuid), fp) == NULL) {
+            uuid[0] = '\0';
+        }
+        fclose(fp);
+        if (strncmp(uuid, "CRYPT-", 6) == 0) {
+            return 1;
+        }
+    }
+
+    /* a stacked device (LVM, md, ...) is encrypted if what it sits on is */
+    snprintf(path, sizeof(path), "%s/dev/block/%u:%u/slaves", sysfs, maj, min);
+    d = opendir(path);
+    if (d == NULL) {
+        return 0;
+    }
+    while (!found && (de = readdir(d)) != NULL) {
+        char     devpath[512];
+        unsigned smaj, smin;
+
+        if (de->d_name[0] == '.') {
+            continue;
+        }
+        snprintf(devpath, sizeof(devpath), "%s/class/block/%s/dev", sysfs,
+                 de->d_name);
+        fp = fopen(devpath, "r");
+        if (fp == NULL) {
+            continue;
+        }
+        if (fscanf(fp, "%u:%u", &smaj, &smin) == 2) {
+            found = dmcrypt_walk(sysfs, smaj, smin, depth + 1);
+        }
+        fclose(fp);
+    }
+    closedir(d);
+    return found;
+}
+
+int persist_dev_is_dmcrypt(const char* sysfs, unsigned maj, unsigned min)
+{
+    return dmcrypt_walk(sysfs, maj, min, 0);
+}
+
+int persist_path_encrypted(const char* path)
+{
+    struct statx stx;
+    struct stat  st;
+
+#ifdef STATX_ATTR_ENCRYPTED
+    if ((statx(AT_FDCWD, path, 0, STATX_BASIC_STATS, &stx) == 0)
+        && (stx.stx_attributes_mask & STATX_ATTR_ENCRYPTED)
+        && (stx.stx_attributes & STATX_ATTR_ENCRYPTED)) {
+        return 1; /* fscrypt */
+    }
+#else
+    (void)stx;
+#endif
+    if (stat(path, &st) != 0) {
+        return 0;
+    }
+    return persist_dev_is_dmcrypt("/sys", major(st.st_dev), minor(st.st_dev));
+}
+
 int persist_init(persist_t* p, persist_config_t* cfg, cluster_t* c)
 {
     p->cluster = c;
@@ -1288,9 +1368,16 @@ int persist_init(persist_t* p, persist_config_t* cfg, cluster_t* c)
     pthread_cond_init(&p->fsync_cond, NULL);
     repl_init(&p->repl);   /* P2-4: fan-out replicator lifecycle */
 
-    WARN("WAL/snapshot stored unencrypted in %s — "
-         "ensure filesystem is encrypted if job payloads contain sensitive data",
-         p->wal_path);
+    /* bug-1877: only warn when the storage is not shown to be encrypted;
+     * this used to warn on every start without looking */
+    if (persist_path_encrypted(p->wal_path)) {
+        INFO("WAL/snapshot in %s is on encrypted storage", p->wal_path);
+    } else {
+        WARN("WAL/snapshot stored unencrypted in %s — "
+             "ensure filesystem is encrypted if job payloads contain "
+             "sensitive data",
+             p->wal_path);
+    }
     return 0;
 }
 

src/controller/testsuite/controller_test.c

@@ -85,6 +85,8 @@ extern void test_wal_job_dispatch_fact(void);
 extern void test_wal_job_start_fact(void);
 extern void test_wal_job_finish_fact(void);
 extern void test_wal_job_fail_fact(void);
+extern void test_phase2_cold_boot(void);
+extern void test_persist_encryption(void);
 extern void test_config_queue_reconcile(void);
 extern void test_wal_job_rusage_fact(void);
 extern void test_wal_job_ckpt_fact(void);
@@ -272,6 +274,8 @@ int main(int argc, char* argv[])
     TEST(test_wal_job_start_fact);
     TEST(test_wal_job_finish_fact);
     TEST(test_wal_job_fail_fact);
+    TEST(test_phase2_cold_boot);
+    TEST(test_persist_encryption);
     TEST(test_config_queue_reconcile);
     TEST(test_wal_job_rusage_fact);
     TEST(test_wal_job_ckpt_fact);

src/controller/testsuite/test_persist_encryption.c

@@ -0,0 +1,115 @@
+/*
+ * test_persist_encryption.c -- bug-1877: persist_init() warned "WAL/snapshot
+ * stored unencrypted" on every start without looking, so the warning also
+ * fired on encrypted storage. persist_dev_is_dmcrypt() now checks the WAL's
+ * block device in sysfs for a dm-crypt (LUKS) layer, directly or under LVM
+ * (via `slaves`); the warning stays for storage it cannot show is encrypted.
+ */
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include <unity/unity.h>
+
+#include "persist.h"
+
+static char g_sysfs[256];
+
+static void mkdirs(const char* rel)
+{
+    char path[512];
+    char cmd[640];
+
+    snprintf(path, sizeof(path), "%s/%s", g_sysfs, rel);
+    snprintf(cmd, sizeof(cmd), "mkdir -p '%s'", path);
+    TEST_ASSERT_EQUAL_INT(0, system(cmd));
+}
+
+static void put(const char* rel, const char* text)
+{
+    char  path[512];
+    FILE* fp;
+
+    snprintf(path, sizeof(path), "%s/%s", g_sysfs, rel);
+    fp = fopen(path, "w");
+    TEST_ASSERT_NOT_NULL(fp);
+    fputs(text, fp);
+    fclose(fp);
+}
+
+static void fake_sysfs(void)
+{
+    char cmd[300];
+
+    snprintf(g_sysfs, sizeof(g_sysfs), "/tmp/test_persist_enc.%d", (int)getpid());
+    snprintf(cmd, sizeof(cmd), "rm -rf %s", g_sysfs);
+    TEST_ASSERT_EQUAL_INT(0, system(cmd));
+
+    /* 259:2 nvme0n1p2, a plain partition */
+    mkdirs("dev/block/259:2");
+    mkdirs("class/block/nvme0n1p2");
+    put("class/block/nvme0n1p2/dev", "259:2\n");
+    /* 253:0 dm-0 = LUKS on nvme0n1p2 */
+    mkdirs("dev/block/253:0/dm");
+    mkdirs("dev/block/253:0/slaves/nvme0n1p2");
+    put("dev/block/253:0/dm/uuid", "CRYPT-LUKS2-0123abcd-luks-0123\n");
+    mkdirs("class/block/dm-0");
+    put("class/block/dm-0/dev", "253:0\n");
+    /* 253:1 dm-1 = LVM volume on dm-0 */
+    mkdirs("dev/block/253:1/dm");
+    mkdirs("dev/block/253:1/slaves/dm-0");
+    put("dev/block/253:1/dm/uuid", "LVM-abcdefabcdef\n");
+    /* 253:2 dm-2 = LVM volume straight on the plain partition */
+    mkdirs("dev/block/253:2/dm");
+    mkdirs("dev/block/253:2/slaves/nvme0n1p2");
+    put("dev/block/253:2/dm/uuid", "LVM-fedcbafedcba\n");
+}
+
+static void drop_sysfs(void)
+{
+    char cmd[300];
+
+    snprintf(cmd, sizeof(cmd), "rm -rf %s", g_sysfs);
+    TEST_ASSERT_EQUAL_INT(0, system(cmd));
+}
+
+static void test_luks_device_is_encrypted(void)
+{
+    fake_sysfs();
+    TEST_ASSERT_EQUAL_INT(1, persist_dev_is_dmcrypt(g_sysfs, 253, 0));
+    drop_sysfs();
+}
+
+static void test_lvm_on_luks_is_encrypted(void)
+{
+    fake_sysfs();
+    TEST_ASSERT_EQUAL_INT(1, persist_dev_is_dmcrypt(g_sysfs, 253, 1));
+    drop_sysfs();
+}
+
+static void test_plain_partition_and_plain_lvm_are_not(void)
+{
+    fake_sysfs();
+    TEST_ASSERT_EQUAL_INT(0, persist_dev_is_dmcrypt(g_sysfs, 259, 2));
+    TEST_ASSERT_EQUAL_INT(0, persist_dev_is_dmcrypt(g_sysfs, 253, 2));
+    drop_sysfs();
+}
+
+static void test_unknown_device_is_not_claimed_encrypted(void)
+{
+    fake_sysfs();
+    TEST_ASSERT_EQUAL_INT(0, persist_dev_is_dmcrypt(g_sysfs, 8, 99));
+    drop_sysfs();
+}
+
+void test_persist_encryption(void)
+{
+    UnitySetTestFile(__FILE__);
+    RUN_TEST(test_luks_device_is_encrypted);
+    RUN_TEST(test_lvm_on_luks_is_encrypted);
+    RUN_TEST(test_plain_partition_and_plain_lvm_are_not);
+    RUN_TEST(test_unknown_device_is_not_claimed_encrypted);
+}

src/controller/testsuite/test_phase2_cold_boot.c

@@ -0,0 +1,40 @@
+/*
+ * test_phase2_cold_boot.c -- bug-1876: while a standby waits for the
+ * primary's config push, cluster_init_phase2() has nothing to build yet.
+ * That is the normal cold-boot state, but it returned -EINVAL, and its
+ * callers logged it as a WARN ("cluster_init_phase2 on standby rejoin
+ * failed (rc=-22)") every 3 s until the config arrived. It now returns
+ * -EAGAIN, which the callers log at INFO.
+ */
+#include <errno.h>
+#include <string.h>
+
+#include <unity/unity.h>
+
+#include "cluster.h"
+#include "ha.h"
+#include "myconf.h"
+#include "xce/cluster_def.h"
+
+extern int cluster_init_phase2(cluster_t* c);
+
+static void test_phase2_without_config_is_a_wait_not_an_error(void)
+{
+    cluster_t c;
+    myconf_t  conf;
+
+    memset(&c, 0, sizeof(c));
+    memset(&conf, 0, sizeof(conf));
+    c.myconf = &conf; /* cluster_config not pushed yet */
+
+    TEST_ASSERT_EQUAL_INT(-EAGAIN, cluster_init_phase2(&c));
+    TEST_ASSERT_TRUE(ha_phase2_rc_is_waiting(-EAGAIN));
+    TEST_ASSERT_FALSE(ha_phase2_rc_is_waiting(-EINVAL));
+    TEST_ASSERT_FALSE(ha_phase2_rc_is_waiting(0));
+}
+
+void test_phase2_cold_boot(void)
+{
+    UnitySetTestFile(__FILE__);
+    RUN_TEST(test_phase2_without_config_is_a_wait_not_an_error);
+}