lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(controller): record standby state under its absolute peer index (bug-1891)

Browse source 
A promoted vote endpoint carries a peer index relative to the votes
array (which leaves out the primary); a late-joiner's heartbeat endpoint
carries the absolute index. repl_ack_decode_and_dispatch recorded state
reports under the raw index, so with ctrl-1 primary, ctrl-3's vote
endpoint (relative 1) and a rejoined ctrl-2's heartbeat endpoint
(absolute 1) shared standby_reported_*[1]. Each handler then acted on
the other's position over its own endpoint: the snapshot meant for the
empty, rejoining ctrl-2 went to the caught-up ctrl-3, and ctrl-2 never
caught up.

Resolve the absolute index with ha_repl_ack_abs_peer_idx (as the
WAL-ack branch already did) for the state report and for
ha_walack_on_ack, and size catchup_inflight by MAX_HA_CANDIDATES so
every absolute index fits.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-09-29 15:12:53 +00:00
parent 1922ac803e
commit 4922f6c844
Signed by untrusted user: rshen
GPG key ID: 80533536B0EB651C
6 changed files with 188 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/controller/testsuite/controller_test.c
View file

@ -112,6 +112,7 @@ extern void test_dispatch_fail_terminal(void);
extern void test_dispatch_fail_retry(void);
extern void test_jobdispatch_no_double_notify(void);
extern void test_worker_resync(void);
extern void test_standby_state_peer_idx(void);
void setUp(void) { return; }
@ -301,6 +302,7 @@ int main(int argc, char* argv[])
TEST(test_dispatch_fail_retry);
TEST(test_jobdispatch_no_double_notify);
TEST(test_worker_resync);
TEST(test_standby_state_peer_idx);
return UNITY_END();
}

139
src/controller/testsuite/test_standby_state_peer_idx.c Normal file
View file

@ -0,0 +1,139 @@
/*
* test_standby_state_peer_idx.c — bug-1891: a standby's heartbeat state
* report must be recorded under that standby's own ABSOLUTE peer index.
*
* Two kinds of endpoint deliver state reports to the primary: a promoted vote
* endpoint (its vote_ep_ctx_t.peer_idx is RELATIVE to the votes array, which
* leaves out the primary itself) and a heartbeat standby endpoint (peer_idx
* already absolute). repl_ack_decode_and_dispatch recorded the report under
* the raw peer_idx. With ctrl-1 primary (self 0), ctrl-3's promoted vote
* endpoint (relative 1) and a rejoined ctrl-2's heartbeat endpoint
* (absolute 1) then shared ha->standby_reported_*[1]: one handler read the
* other's position and acted on its own endpoint, so the snapshot meant for
* the empty, rejoining ctrl-2 was sent to the caught-up ctrl-3 instead.
*/
#define _GNU_SOURCE
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
#include "cluster.h"
#include "controller.capnp.h"
#include "ha.h"
#include "netio.h"
#include "request.h"
#define SENTINEL_LSN 777
#define CONFIG_TAG 42
static ha_state_t g_sp_ha;
static ha_config_t g_sp_cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.election_jitter_max_ms = 0,
.election_priority_step_ms = 500,
};
/* Primary at absolute index 0 of a 3-node cluster; every slot starts at a
* sentinel so a write to the wrong one shows. */
static void sp_setup(void)
{
int i;
TEST_ASSERT_EQUAL_INT(0, ha_init(&g_sp_ha, &g_sp_cfg, 0));
g_sp_ha.n_peers = 3;
atomic_store_explicit(&g_sp_ha.role, (int32_t)CTRL_PRIMARY,
memory_order_release);
atomic_store_explicit(&g_sp_ha.committed_config_tag, CONFIG_TAG,
memory_order_release);
atomic_store_explicit(&g_sp_ha.last_known_committed_lsn, 114,
memory_order_release);
for (i = 0; i < MAX_HA_CANDIDATES; i++) {
g_sp_ha.standby_reported_lsn[i] = SENTINEL_LSN;
g_sp_ha.standby_reported_tag[i] = CONFIG_TAG;
}
}
/* A heartbeat NOP reply carrying a state report, as the standby sends it. */
static io_packet_t* sp_state_report_pkt(uint64_t applied_lsn)
{
state_report_t sr;
xce_reply_t rep;
xce_request_t req;
io_packet_t* pkt;
memset(&sr, 0, sizeof(sr));
sr.applied_lsn = applied_lsn;
sr.config_tag = CONFIG_TAG;
memset(&rep, 0, sizeof(rep));
rep.opcode = OPCode_nop;
rep.data_which = RequestReply_data_stateReportMsg;
rep.data.state_report = &sr;
memset(&req, 0, sizeof(req));
req.opcode = OPCode_reply;
req.body_which = Request_body_replymsg;
req.body.replymsg = &rep;
pkt = calloc(1, sizeof(*pkt));
TEST_ASSERT_NOT_NULL(pkt);
TEST_ASSERT_TRUE_MESSAGE(xce_encode(&req, pkt) >= 0,
"failed to encode the state report");
return pkt;
}
/* Baseline: a heartbeat standby endpoint's index is already absolute. */
static void test_heartbeat_ep_report_lands_on_its_absolute_index(void)
{
endpoint_t ep;
sp_setup();
memset(&ep, 0, sizeof(ep));
ha_repl_ack_dispatch_for_test(&g_sp_ha, &ep, /*ep_is_vote=*/0,
/*peer_idx=*/1, sp_state_report_pkt(5));
TEST_ASSERT_EQUAL_UINT64(5, g_sp_ha.standby_reported_lsn[1]);
TEST_ASSERT_EQUAL_UINT64(SENTINEL_LSN, g_sp_ha.standby_reported_lsn[2]);
ha_fini(&g_sp_ha);
printf("test_heartbeat_ep_report_lands_on_its_absolute_index: PASS\n");
}
/* THE BUG: a promoted vote endpoint's relative index 1 is peer 2 (self is
* 0); its report must not land on peer 1's slot. */
static void test_vote_ep_report_lands_on_its_absolute_index(void)
{
endpoint_t ep;
sp_setup();
memset(&ep, 0, sizeof(ep));
ha_repl_ack_dispatch_for_test(&g_sp_ha, &ep, /*ep_is_vote=*/1,
/*peer_idx=*/1, sp_state_report_pkt(5));
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5, g_sp_ha.standby_reported_lsn[2],
"bug-1891: peer 2's state report was not recorded under peer 2");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(SENTINEL_LSN,
g_sp_ha.standby_reported_lsn[1],
"bug-1891: peer 2's state report overwrote peer 1's slot");
ha_fini(&g_sp_ha);
printf("test_vote_ep_report_lands_on_its_absolute_index: PASS\n");
}
void test_standby_state_peer_idx(void)
{
printf("\n>> standby state report peer index tests (bug-1891)\n\n");
RUN_TEST(test_heartbeat_ep_report_lands_on_its_absolute_index);
RUN_TEST(test_vote_ep_report_lands_on_its_absolute_index);
}