fix(controller): raft_cb_persist must deny vote and roll back state on persist failure (bug-1684)
ha_persist_term() had a void return -- every open/write/fsync/rename
failure was silently swallowed ("best-effort; caller must not rely on
this for correctness"). Its sole vote-granting caller, raft_cb_persist(),
always returned 0 regardless, so the raft library always granted the
vote even when the term/voted_for record never reached disk -- a real
split-vote risk if the node then restarted and forgot it voted
(Election Safety requires durably-persisted votedFor).
ha_persist_term() now returns 0 on success (including the deliberate
NULL-workdir test-mode skip) or -errno on a genuine I/O failure
(fsync's previously-unchecked return is now checked too). raft_cb_persist()
propagates that failure to the raft library (denying the vote per its
own documented contract) and rolls back ha->voted_for_term/
ha->voted_for_name to their pre-call snapshot, so the adapter's
in-memory state stays consistent with the library's own "deny without
updating voted_for" behavior. ha->term is deliberately NOT rolled back:
the raft library adopts a higher observed term unconditionally at Step 2
of raft_on_vote_request, before and independent of the vote-grant/persist
outcome, so the adapter correctly mirrors that same semantics.
Also corrects a stale claim in controller/CLAUDE.md's Protocol
Verification Checklist (found by a sibling audit dimension the same
session): term+voted_for were never written "atomically under ha->lock"
-- term uses its own atomic store outside the lock.
Found by an /xce-audit-invariants controller pass (error-handling
dimension). 3 new tests in test_ha_durability.c drive the real
production path (ha_init_raft + raft_on_vote_request, the exact
function cluster.c's dispatch calls) against a forced ENOENT write
failure; RED/GREEN independently verified.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AgYRU3zDTqNGV5Jk8aiKLg
This commit is contained in:
parent
5441d1b8e6
commit
e5e6cd1817
4 changed files with 205 additions and 14 deletions
|
|
@ -34,6 +34,7 @@
|
|||
|
||||
#include "ha.h"
|
||||
#include "config.h" /* XCE_MAXNAME_LEN */
|
||||
#include "xce/cluster_def.h" /* cluster_t, for bug-1684's raft_on_vote_request test */
|
||||
|
||||
/* ── On-disk format (mirrors ha.c internals, duplicated here for white-box tests) ─
|
||||
*
|
||||
|
|
@ -381,6 +382,133 @@ static void test_persist_atomicity_tmp_cleaned_up(void)
|
|||
printf("test_persist_atomicity_tmp_cleaned_up: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 6: ha_persist_term() returns a real error on a genuine write
|
||||
* failure (bug-1684) ─────────────────────────────────────────────────────
|
||||
*
|
||||
* Before this fix, ha_persist_term() had a void return -- every open/write/
|
||||
* fsync/rename failure was silently swallowed ("best-effort; caller must
|
||||
* not rely on this for correctness"). Its sole vote-granting caller,
|
||||
* raft_cb_persist(), always returned 0 to the raft library regardless, so a
|
||||
* vote was granted even when the term/voted_for record never reached disk.
|
||||
*
|
||||
* Points workdir at a path whose PARENT directory does not exist, so
|
||||
* open(O_CREAT) on the .tmp file deterministically fails ENOENT -- no
|
||||
* permission-bit tricks needed (those misbehave when running as root,
|
||||
* common in sandboxes), and no real filesystem quota/full-disk simulation
|
||||
* required either.
|
||||
*/
|
||||
static void test_persist_term_returns_error_on_write_failure(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
ha_init(&ha, &g_durability_cfg, 0);
|
||||
|
||||
atomic_store_explicit(&ha.term, 3ULL, memory_order_release);
|
||||
ha.voted_for_term = 3;
|
||||
strncpy(ha.voted_for_name, "ctrl-node-9", sizeof(ha.voted_for_name) - 1);
|
||||
|
||||
const char *bad_workdir =
|
||||
"/tmp/xce_ha_durability_test_nonexistent_parent_xyz/subdir";
|
||||
int rc = ha_persist_term(&ha, bad_workdir);
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
rc < 0,
|
||||
"ha_persist_term must return a negative errno when the write "
|
||||
"genuinely fails (bug-1684)");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_persist_term_returns_error_on_write_failure: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 7: NULL workdir stays a no-op SUCCESS, not folded into the new
|
||||
* error path (bug-1684 regression guard) ──────────────────────────────── */
|
||||
static void test_persist_term_null_workdir_returns_success(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
ha_init(&ha, &g_durability_cfg, 0);
|
||||
|
||||
int rc = ha_persist_term(&ha, NULL);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
0, rc,
|
||||
"ha_persist_term(..., NULL) must return 0 -- test-mode skip is not "
|
||||
"a failure");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_persist_term_null_workdir_returns_success: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 8: a vote denied by a real persist failure leaves ha's in-memory
|
||||
* voted_for state UNCHANGED (bug-1684) ─────────────────────────────────────
|
||||
*
|
||||
* Drives the REAL production path: ha_init_raft() wires raft_cb_persist as
|
||||
* the raft library's on_persist callback (s_real_cbs, ha.c); the function
|
||||
* under test, raft_on_vote_request() (raft.c, the pure library), is the
|
||||
* exact function cluster.c's production dispatch calls. ha_init_raft(ha,
|
||||
* NULL, n_peers) avoids needing a real persist_t (only used for
|
||||
* initial_log_len/initial_commit_index when c is non-NULL) -- ha->raft_c is
|
||||
* then set directly afterward, the same test-shim pattern ha.h's own
|
||||
* raft_cb_become_leader_for_test/raft_cb_become_follower_for_test comments
|
||||
* document ("this shim sets ha->raft_c = c first").
|
||||
*
|
||||
* The fixture's myconf populates ONE real candidate name (index 1) so
|
||||
* raft_cb_persist's voted_for_name lookup resolves to a genuine non-empty
|
||||
* name -- not the self/empty fallback -- making the rollback assertion
|
||||
* meaningful: before this fix, ha.voted_for_name/voted_for_term would show
|
||||
* that candidate's vote despite the vote being denied and never durably
|
||||
* recorded.
|
||||
*/
|
||||
static void test_vote_denied_and_state_rolled_back_on_persist_failure(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
ha_init(&ha, &g_durability_cfg, 0);
|
||||
ha_init_raft(&ha, NULL, 2);
|
||||
|
||||
myconf_t test_myconf;
|
||||
memset(&test_myconf, 0, sizeof(test_myconf));
|
||||
test_myconf.workdir =
|
||||
(char *)"/tmp/xce_ha_durability_test_nonexistent_parent_xyz/subdir";
|
||||
test_myconf.name = (char *)"self-node";
|
||||
test_myconf.peers_cfg.candidates_count = 2;
|
||||
strncpy(test_myconf.peers_cfg.candidates[1].name, "candidate-1",
|
||||
sizeof(test_myconf.peers_cfg.candidates[1].name) - 1);
|
||||
|
||||
cluster_t test_c;
|
||||
memset(&test_c, 0, sizeof(test_c));
|
||||
test_c.myconf = &test_myconf;
|
||||
|
||||
/* Test shim: same pattern ha.h documents for the other raft_cb_*_for_test
|
||||
* helpers -- set ha->raft_c directly rather than driving a full
|
||||
* persist_t through ha_init_raft(). */
|
||||
ha.raft_c = &test_c;
|
||||
|
||||
/* Baseline: no prior vote. */
|
||||
TEST_ASSERT_EQUAL_UINT64(0ULL, ha.voted_for_term);
|
||||
TEST_ASSERT_EQUAL_STRING("", ha.voted_for_name);
|
||||
|
||||
uint64_t reply_term = 0;
|
||||
int result = raft_on_vote_request(&ha.raft, /*from_idx=*/1, /*term=*/5,
|
||||
/*cand_last_log_term=*/0,
|
||||
/*cand_last_log_index=*/0, &reply_term);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
RAFT_DENY, result,
|
||||
"vote must be DENIED when the term/vote persist genuinely fails "
|
||||
"(bug-1684)");
|
||||
|
||||
/* voted_for state must be UNCHANGED -- the vote was never durably
|
||||
* recorded, so this node must not believe it already voted. */
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
||||
0ULL, ha.voted_for_term,
|
||||
"voted_for_term must roll back to its pre-call value on persist "
|
||||
"failure (bug-1684)");
|
||||
TEST_ASSERT_EQUAL_STRING_MESSAGE(
|
||||
"", ha.voted_for_name,
|
||||
"voted_for_name must roll back to its pre-call value on persist "
|
||||
"failure (bug-1684)");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_vote_denied_and_state_rolled_back_on_persist_failure: "
|
||||
"PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test suite entry point ───────────────────────────────────────────────── */
|
||||
|
||||
void test_ha_durability(void)
|
||||
|
|
@ -400,4 +528,11 @@ void test_ha_durability(void)
|
|||
|
||||
/* Req 9.1: write+rename atomicity — no .tmp artifact left */
|
||||
RUN_TEST(test_persist_atomicity_tmp_cleaned_up);
|
||||
|
||||
/* bug-1684: ha_persist_term() must return a real error on write failure,
|
||||
* NULL-workdir must stay a no-op success, and raft_cb_persist() must
|
||||
* deny the vote AND roll back in-memory voted_for state on failure. */
|
||||
RUN_TEST(test_persist_term_returns_error_on_write_failure);
|
||||
RUN_TEST(test_persist_term_null_workdir_returns_success);
|
||||
RUN_TEST(test_vote_denied_and_state_rolled_back_on_persist_failure);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue