feat(ha-commit-barrier-rollout): commit-first-then-reply in all four handlers (task 3.2)

Uniform pattern (ctrl fn -> persist_commit_barrier -> reply) in
do_jobcontrol_op/do_queuecontrol_op/do_hostcontrol_op; join barrier
between AFTER_EVENT and success reply (failure -> error reply, skip
on_worker_join). FIXES bug-447: key_find found-rc normalized inside
job_ctrl/queue_ctrl/worker_ctrl — queue/host control now actually
append WAL and replicate (was silently unpersisted); idempotent no-ops
now return 0. handle_jobsubmit untouched. 12/12 + 15/15; census zero
failures.

src/controller/job.c

@@ -327,6 +327,15 @@ int job_ctrl(cluster_t* c, xce_jobctrl_t* ctrinfo)
         return -ENOENT;
     }
 
+    /* bug-447: key_find returns 1 on found (binary librtti, unnormalizable
+     * upstream). Normalize the found-rc to 0 so the not-started siginfo
+     * branch below (which appends via the mark_job_* hooks but never
+     * reassigns rc) returns 0 to the handler's commit-barrier pattern
+     * instead of leaking 1 as the reply status. Real errors are
+     * untouched: rc < 0 returned above, the j == NULL miss returned
+     * -ENOENT, and every other branch sets rc itself. */
+    rc = 0;
+
     /* FIXME: eliminate held multiple locks */
     lock(j, RTTI_LOCK_READ);
     if (j->job.allocs != NULL) {
@@ -925,7 +934,20 @@ int do_jobcontrol_op(cluster_t* c, endpoint_t* ep, xce_request_t* request)
 
     init_reply_by_req(&reply, request);
 
-    rc               = job_ctrl(c, request->body.jobctrlmsg);
+    rc = job_ctrl(c, request->body.jobctrlmsg);
+
+    /* Commit barrier (ha-commit-barrier-rollout Req 1.1-1.4): the success
+     * reply is sent only after the operation's WAL entry is majority-
+     * committed; barrier failures (-ETIMEDOUT, ...) become the error
+     * reply. Errors from job_ctrl reply immediately — no barrier wait.
+     * job_ctrl released all entity locks before returning, so blocking
+     * here holds no lock. timeout 0 = configured sync_timeout_ms. */
+    if (rc == 0) {
+        rc = persist_commit_barrier(&c->persist, 0);
+        if (rc != 0)
+            WARN("%s commit barrier failed, rc %d", __func__, rc);
+    }
+
     reply.status     = rc;
     reply.data_which = RequestReply_data_nodata;
 

src/controller/join.c

@@ -7,6 +7,7 @@
 #include "cluster.h"
 #include "config_repl.h"
 #include "ha.h"
+#include "persist.h"
 #include "xce/cluster_def.h"
 #include "xce/errnox.h"
 
@@ -104,6 +105,21 @@ static int handle_worker_join(cluster_t* c, endpoint_t* ep,
     xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_WORKER, HOOK_AFTER_EVENT,
                   HOOK_DETAIL_WORKER_JOIN, c, from->name, w);
 
+    /* Commit barrier (ha-commit-barrier-rollout Req 1.1-1.4): the success
+     * reply is sent only after the join's WAL entry is majority-committed;
+     * barrier failures (-ETIMEDOUT, ...) become the error reply and skip
+     * on_worker_join below. Join errors (idx < 0) reply immediately — no
+     * barrier wait. No lock is held here (AFTER_EVENT runs outside any
+     * lock). timeout 0 = configured sync_timeout_ms. */
+    if (idx >= 0) {
+        rc = persist_commit_barrier(&c->persist, 0);
+        if (rc != 0) {
+            WARN("%s commit barrier failed for %s, rc %d", __func__,
+                 from->name, rc);
+            idx = rc;
+        }
+    }
+
     reply.status     = idx;
     reply.data_which = RequestReply_data_nodata;
     xce_reply(ep, c->myhost, &reply);

src/controller/queue.c

@@ -3,6 +3,7 @@
 #include <string.h>
 
 #include "cluster.h"
+#include "persist.h"
 #include "qobj.h"
 #include "xce/cluster_def.h"
 #include "xce/types.h"
@@ -73,6 +74,14 @@ int queue_ctrl(cluster_t* c, xce_queuectrl_t* ctrl)
         return -ENOENT;
     }
 
+    /* bug-447: key_find returns 1 on found (binary librtti, unnormalizable
+     * upstream). Normalize the found-rc to 0 here so the AFTER_EVENT hook
+     * gate below fires (WAL append + replication) and the handler's
+     * commit-barrier pattern sees 0. Real errors are untouched: rc < 0
+     * returned above, the q == NULL miss returned -ENOENT, and control-
+     * action failures set rc themselves. */
+    rc = 0;
+
     lock(q, RTTI_LOCK_WRITE);
     switch (ctrl->action) {
     case QUEUE_OPEN:
@@ -123,7 +132,20 @@ int do_queuecontrol_op(cluster_t* c, endpoint_t* ep, xce_request_t* request)
 
     init_reply_by_req(&reply, request);
 
-    rc               = queue_ctrl(c, request->body.queuectrlmsg);
+    rc = queue_ctrl(c, request->body.queuectrlmsg);
+
+    /* Commit barrier (ha-commit-barrier-rollout Req 1.1-1.4): the success
+     * reply is sent only after the operation's WAL entry is majority-
+     * committed; barrier failures (-ETIMEDOUT, ...) become the error
+     * reply. Errors from queue_ctrl reply immediately — no barrier wait.
+     * queue_ctrl released all entity locks before returning, so blocking
+     * here holds no lock. timeout 0 = configured sync_timeout_ms. */
+    if (rc == 0) {
+        rc = persist_commit_barrier(&c->persist, 0);
+        if (rc != 0)
+            WARN("%s commit barrier failed, rc %d", __func__, rc);
+    }
+
     reply.status     = rc;
     reply.data_which = RequestReply_data_nodata;
 

src/controller/worker.c

@@ -657,6 +657,14 @@ int worker_ctrl(cluster_t* c, xce_hostctrl_t* ctrl)
         return -ENOENT;
     }
 
+    /* bug-447: key_find returns 1 on found (binary librtti, unnormalizable
+     * upstream). Normalize the found-rc to 0 here so the AFTER_EVENT hook
+     * gate below fires (WAL append + replication) and the handler's
+     * commit-barrier pattern sees 0. Real errors are untouched: rc < 0
+     * returned above, the w == NULL miss returned -ENOENT, and control-
+     * action failures set rc themselves. */
+    rc = 0;
+
     lock(w, RTTI_LOCK_WRITE);
     switch (ctrl->action) {
     case HOST_OPEN:
@@ -693,7 +701,20 @@ int do_hostcontrol_op(cluster_t* c, endpoint_t* ep, xce_request_t* request)
 
     init_reply_by_req(&reply, request);
 
-    rc               = worker_ctrl(c, request->body.hostctrlmsg);
+    rc = worker_ctrl(c, request->body.hostctrlmsg);
+
+    /* Commit barrier (ha-commit-barrier-rollout Req 1.1-1.4): the success
+     * reply is sent only after the operation's WAL entry is majority-
+     * committed; barrier failures (-ETIMEDOUT, ...) become the error
+     * reply. Errors from worker_ctrl reply immediately — no barrier wait.
+     * worker_ctrl released all entity locks before returning, so blocking
+     * here holds no lock. timeout 0 = configured sync_timeout_ms. */
+    if (rc == 0) {
+        rc = persist_commit_barrier(&c->persist, 0);
+        if (rc != 0)
+            WARN("%s commit barrier failed, rc %d", __func__, rc);
+    }
+
     reply.status     = rc;
     reply.data_which = RequestReply_data_nodata;