fix(controller): raft_cb_persist must deny vote and roll back state on persist failure (bug-1684)

ha_persist_term() had a void return -- every open/write/fsync/rename
failure was silently swallowed ("best-effort; caller must not rely on
this for correctness"). Its sole vote-granting caller, raft_cb_persist(),
always returned 0 regardless, so the raft library always granted the
vote even when the term/voted_for record never reached disk -- a real
split-vote risk if the node then restarted and forgot it voted
(Election Safety requires durably-persisted votedFor).

ha_persist_term() now returns 0 on success (including the deliberate
NULL-workdir test-mode skip) or -errno on a genuine I/O failure
(fsync's previously-unchecked return is now checked too). raft_cb_persist()
propagates that failure to the raft library (denying the vote per its
own documented contract) and rolls back ha->voted_for_term/
ha->voted_for_name to their pre-call snapshot, so the adapter's
in-memory state stays consistent with the library's own "deny without
updating voted_for" behavior. ha->term is deliberately NOT rolled back:
the raft library adopts a higher observed term unconditionally at Step 2
of raft_on_vote_request, before and independent of the vote-grant/persist
outcome, so the adapter correctly mirrors that same semantics.

Also corrects a stale claim in controller/CLAUDE.md's Protocol
Verification Checklist (found by a sibling audit dimension the same
session): term+voted_for were never written "atomically under ha->lock"
-- term uses its own atomic store outside the lock.

Found by an /xce-audit-invariants controller pass (error-handling
dimension). 3 new tests in test_ha_durability.c drive the real
production path (ha_init_raft + raft_on_vote_request, the exact
function cluster.c's dispatch calls) against a forced ENOENT write
failure; RED/GREEN independently verified.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AgYRU3zDTqNGV5Jk8aiKLg

src/controller/ha.c

@@ -2248,11 +2248,27 @@ static void raft_cb_become_follower(void *ctx, uint64_t term, int leader_idx)
  * raft_cb_persist: persist term + voted_for to disk before granting vote.
  *
  * Maps raft integer voted_for index to a candidate name using cluster config.
- * Updates ha->term, ha->voted_for_term, ha->voted_for_name then calls
+ * Updates ha->term (its own atomic store, not under ha->lock -- mirrors the
+ * raft library's own unconditional term adoption in raft_on_vote_request's
+ * Step 2, which happens regardless of whether the vote itself is later
+ * granted or denied, so this is never rolled back here either), then updates
+ * ha->voted_for_term/ha->voted_for_name under ha->lock and calls
  * ha_persist_term(ha, workdir).
  *
  * voted_for == -1 means "no vote"; voted_for >= 0 is the candidate index.
- * Returns 0 on success, non-zero on error (causes raft library to deny vote).
+ * Returns 0 on success, non-zero on error (causes raft library to deny vote,
+ * per raft_on_vote_request's Step 7 contract: "Persist failed — deny without
+ * updating voted_for").
+ *
+ * bug-1684: on a genuine ha_persist_term() failure, the voted_for_term/
+ * voted_for_name mutation above must be ROLLED BACK to its pre-call value --
+ * otherwise this adapter's in-memory state would claim a vote was cast for a
+ * candidate the raft library itself denied and never durably recorded,
+ * diverging from the library's own voted_for (which correctly stays
+ * unchanged on a persist failure). A crash-and-restart after that divergence
+ * could let this node grant a second, conflicting vote in the same term --
+ * the exact split-vote hazard Election Safety exists to prevent
+ * (controller/CLAUDE.md's TLA+ safety invariants).
  */
 static int raft_cb_persist(void *ctx, uint64_t term, int voted_for)
 {
@@ -2271,6 +2287,10 @@ static int raft_cb_persist(void *ctx, uint64_t term, int voted_for)
     /* Map voted_for index to name */
     pthread_mutex_lock(&ha->lock);
 
+    uint64_t prev_voted_for_term = ha->voted_for_term;
+    char     prev_voted_for_name[sizeof(ha->voted_for_name)];
+    memcpy(prev_voted_for_name, ha->voted_for_name, sizeof(prev_voted_for_name));
+
     if (voted_for == -1) {
         ha->voted_for_term = 0;
         memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
@@ -2295,7 +2315,19 @@ static int raft_cb_persist(void *ctx, uint64_t term, int voted_for)
 
     pthread_mutex_unlock(&ha->lock);
 
-    ha_persist_term(ha, workdir);
+    int rc = ha_persist_term(ha, workdir);
+    if (rc < 0) {
+        ERROR("raft_cb_persist: failed to durably persist term=%llu "
+              "voted_for=%d rc=%d -- denying vote, rolling back in-memory "
+              "vote state (bug-1684)",
+              (unsigned long long)term, voted_for, rc);
+        pthread_mutex_lock(&ha->lock);
+        ha->voted_for_term = prev_voted_for_term;
+        memcpy(ha->voted_for_name, prev_voted_for_name,
+               sizeof(ha->voted_for_name));
+        pthread_mutex_unlock(&ha->lock);
+        return rc;
+    }
     return 0;
 }
 
@@ -5168,13 +5200,20 @@ typedef struct {
 /*
  * ha_persist_term: atomically write ha_term_file_t to {workdir}/ha-term.bin.
  * Uses write-to-tmp + fsync + rename for durability (existing project pattern).
- * If workdir is NULL, the persist is skipped (test mode or no workdir configured).
+ * If workdir is NULL, the persist is skipped (test mode or no workdir configured)
+ * -- returns 0, a deliberate no-op, not a failure.
  * Req 9.1: must be called before any vote grant or term increment is reported.
+ * Returns 0 on success (including the NULL-workdir skip); a negative errno on
+ * a genuine open/write/fsync/rename failure (bug-1684: this used to be a void
+ * return with every failure silently swallowed, so raft_cb_persist always told
+ * the raft library the vote was durably persisted even when it never reached
+ * disk -- a real split-vote risk if the node then restarted and forgot it
+ * voted).
  */
-void ha_persist_term(ha_state_t *ha, const char *workdir)
+int ha_persist_term(ha_state_t *ha, const char *workdir)
 {
     if (!workdir)
-        return; /* test mode: skip file I/O */
+        return 0; /* test mode: skip file I/O -- not a failure */
 
     ha_term_file_t f;
     memset(&f, 0, sizeof(f));
@@ -5195,15 +5234,26 @@ void ha_persist_term(ha_state_t *ha, const char *workdir)
 
     int fd = open(tmp, O_WRONLY | O_CREAT | O_TRUNC, 0600);
     if (fd < 0)
-        return; /* best-effort; caller must not rely on this for correctness */
+        return -errno;
 
     ssize_t written = write(fd, &f, sizeof(f));
-    if (written == (ssize_t)sizeof(f))
-        fsync(fd);
+    if (written != (ssize_t)sizeof(f)) {
+        int saved_errno = (written < 0) ? errno : EIO;
+        close(fd);
+        return -saved_errno;
+    }
+
+    if (fsync(fd) < 0) {
+        int saved_errno = errno;
+        close(fd);
+        return -saved_errno;
+    }
     close(fd);
 
-    if (written == (ssize_t)sizeof(f))
-        rename(tmp, path);
+    if (rename(tmp, path) < 0)
+        return -errno;
+
+    return 0;
 }
 
 /*