feat(ha-config-bootstrap): implement bootstrap_config_if_needed() (task 5.3)
- Probe all peers: any tag>0 -> defer_config_fetch=true; all-zero -> proceed; none reachable -> exponential backoff 500ms->10s cap with WARN per attempt - Wire into myapp.c: before ha_wait_for_heartbeat_or_timeout() - Post-election: defer_config_fetch && CTRL_STANDBY -> fetch_config_from_primary() - INFO log for all three bootstrap outcomes (Req 10.2) - XCE_TESTING hook for probe mock; test_ha_bootstrap_support.c installs mock - 9/9 bootstrap tests GREEN, 141/141 full suite GREEN
This commit is contained in:
parent
0c4a0760d1
commit
ab174c8b06
6 changed files with 163 additions and 8 deletions
|
|
@ -124,7 +124,7 @@ Dependency order: startup validation (D) unblocks vtaskmgr before election (A);
|
||||||
- _Requirements: 3.1, 1.3_
|
- _Requirements: 3.1, 1.3_
|
||||||
- _Boundary: Config Bootstrap (config_repl.c)_
|
- _Boundary: Config Bootstrap (config_repl.c)_
|
||||||
|
|
||||||
- [ ] 5.3 Implement bootstrap_config_if_needed() and wire into startup
|
- [x] 5.3 Implement bootstrap_config_if_needed() and wire into startup
|
||||||
- Returns immediately if `config_tag != 0`
|
- Returns immediately if `config_tag != 0`
|
||||||
- Probes all peers; three outcomes: all-zero → proceed; any non-zero → set `defer_config_fetch`, return; no peer reachable → retry with backoff 500ms→10ms→...→10s cap; log WARN each failed attempt
|
- Probes all peers; three outcomes: all-zero → proceed; any non-zero → set `defer_config_fetch`, return; no peer reachable → retry with backoff 500ms→10ms→...→10s cap; log WARN each failed attempt
|
||||||
- Integrate into `myapp.c`: call after `cluster_open_listener()` + `iomgr_init()` + `vtaskmgr_init()`, before `ha_wait_for_heartbeat_or_timeout()`
|
- Integrate into `myapp.c`: call after `cluster_open_listener()` + `iomgr_init()` + `vtaskmgr_init()`, before `ha_wait_for_heartbeat_or_timeout()`
|
||||||
|
|
|
||||||
|
|
@ -1525,21 +1525,94 @@ int xce_probe_config_tag(const char *host, uint16_t port, uint64_t *tag_out)
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#ifdef XCE_TESTING
|
||||||
|
/* Injection point for unit tests — NULL means use real xce_probe_config_tag */
|
||||||
|
int (*xce_probe_config_tag_hook)(const char *host, uint16_t port,
|
||||||
|
uint64_t *tag_out) = NULL;
|
||||||
|
#endif
|
||||||
|
|
||||||
/* ── bootstrap_config_if_needed ────────────────────────────────────────────
|
/* ── bootstrap_config_if_needed ────────────────────────────────────────────
|
||||||
*
|
*
|
||||||
* Stub: task 5.3 will implement the full pre-election config probe.
|
* Pre-election config probe for unconfigured nodes (config_version_tag == 0).
|
||||||
* Declared in ha.h; needed here so the unit test binary links cleanly
|
|
||||||
* for the probe tests (task 5.2 scope).
|
|
||||||
*
|
*
|
||||||
* Returns -ENOSYS (not yet implemented) for any cluster with peers.
|
* Outcomes:
|
||||||
* Returns 0 immediately when config_version_tag != 0 (Req 3.1 fast path).
|
* 1. Local config exists (config_version_tag != 0) → return 0 immediately (3.1)
|
||||||
|
* 2. All reachable peers have tag=0 → first-boot cluster; return 0, no defer (3.3)
|
||||||
|
* 3. Any peer has tag > 0 → set ha.defer_config_fetch = true; return 0 (3.2)
|
||||||
|
* 4. No peer reachable → retry with exponential backoff 500ms→1s→...→10s (3.4)
|
||||||
|
*
|
||||||
|
* Requirements: 3.1, 3.2, 3.3, 3.4, 10.2
|
||||||
|
* Boundary: Config Bootstrap (config_repl.c)
|
||||||
*/
|
*/
|
||||||
int bootstrap_config_if_needed(cluster_t *c)
|
int bootstrap_config_if_needed(cluster_t *c)
|
||||||
{
|
{
|
||||||
|
/* Req 3.1: fast path — already have config */
|
||||||
if (c->myconf->config_version_tag != 0)
|
if (c->myconf->config_version_tag != 0)
|
||||||
return 0; /* already have config — nothing to do (Req 3.1) */
|
return 0;
|
||||||
|
|
||||||
/* Full probe loop implemented in task 5.3 */
|
const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg;
|
||||||
|
int n_peers = pcfg->candidates_count;
|
||||||
|
|
||||||
|
/* Single-node or no peers configured — nothing to probe */
|
||||||
|
if (n_peers == 0) {
|
||||||
|
INFO("HA bootstrap: no peers configured, proceeding as first-boot primary");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Exponential backoff state: 500ms → doubling → 10s cap (Req 3.4) */
|
||||||
|
uint32_t backoff_ms = 500;
|
||||||
|
const uint32_t backoff_max_ms = 10000;
|
||||||
|
int attempt = 0;
|
||||||
|
|
||||||
|
for (;;) {
|
||||||
|
int any_nonzero = 0; /* any peer replied with tag > 0 */
|
||||||
|
int any_reachable = 0; /* any peer replied at all (tag >= 0) */
|
||||||
|
|
||||||
|
for (int i = 0; i < n_peers; i++) {
|
||||||
|
const xce_candidate_t *peer = &pcfg->candidates[i];
|
||||||
|
uint64_t tag_out = 0;
|
||||||
|
|
||||||
|
#ifdef XCE_TESTING
|
||||||
|
int rc = xce_probe_config_tag_hook
|
||||||
|
? xce_probe_config_tag_hook(peer->hostname, (uint16_t)peer->port, &tag_out)
|
||||||
|
: xce_probe_config_tag(peer->hostname, (uint16_t)peer->port, &tag_out);
|
||||||
|
#else
|
||||||
|
int rc = xce_probe_config_tag(peer->hostname, (uint16_t)peer->port, &tag_out);
|
||||||
|
#endif
|
||||||
|
if (rc == 0) {
|
||||||
|
any_reachable = 1;
|
||||||
|
if (tag_out != 0) {
|
||||||
|
any_nonzero = 1;
|
||||||
|
/* Req 3.2: at least one peer has config — defer fetch */
|
||||||
|
c->ha.defer_config_fetch = true;
|
||||||
|
INFO("HA bootstrap: peer '%s:%u' has config tag=%016" PRIx64
|
||||||
|
" — deferring config fetch to post-election",
|
||||||
|
peer->hostname, (unsigned)peer->port, tag_out);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (any_reachable) {
|
||||||
|
/* Req 3.3: all reachable peers have tag=0 — first-boot cluster */
|
||||||
|
INFO("HA bootstrap: all reachable peers have tag=0 — first-boot cluster, "
|
||||||
|
"proceeding to election without config fetch");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Req 3.4: no peer reachable — retry with exponential backoff */
|
||||||
|
attempt++;
|
||||||
|
WARN("HA bootstrap: no peer reachable after %d probe round(s) — "
|
||||||
|
"retrying in %u ms (attempt %d)",
|
||||||
|
attempt, backoff_ms, attempt);
|
||||||
|
|
||||||
|
usleep((useconds_t)backoff_ms * 1000);
|
||||||
|
|
||||||
|
backoff_ms = (backoff_ms * 2 < backoff_max_ms)
|
||||||
|
? backoff_ms * 2 : backoff_max_ms;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Unreachable — blocks until a peer responds or process is killed */
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -133,6 +133,17 @@ char *sanitise_cn(const char *src, size_t maxlen);
|
||||||
#ifdef XCE_TESTING
|
#ifdef XCE_TESTING
|
||||||
#include <pthread.h>
|
#include <pthread.h>
|
||||||
#include <stdatomic.h>
|
#include <stdatomic.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
|
||||||
|
/*
|
||||||
|
* xce_probe_config_tag_hook: test-only injection point for bootstrap probing.
|
||||||
|
* When non-NULL, bootstrap_config_if_needed() calls this instead of the real
|
||||||
|
* xce_probe_config_tag(), allowing unit tests to return controlled tag values
|
||||||
|
* without real network connections.
|
||||||
|
* Set by test infrastructure (e.g. test_ha_bootstrap_support.c) before tests run.
|
||||||
|
*/
|
||||||
|
extern int (*xce_probe_config_tag_hook)(const char *host, uint16_t port,
|
||||||
|
uint64_t *tag_out);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* load_primary_config_for_test: test-only wrapper that exercises the path
|
* load_primary_config_for_test: test-only wrapper that exercises the path
|
||||||
|
|
|
||||||
|
|
@ -104,6 +104,7 @@ controller_unit_test_src = [
|
||||||
'testsuite/test_ha_startup.c',
|
'testsuite/test_ha_startup.c',
|
||||||
'testsuite/test_ha_election.c',
|
'testsuite/test_ha_election.c',
|
||||||
'testsuite/test_ha_bootstrap.c',
|
'testsuite/test_ha_bootstrap.c',
|
||||||
|
'testsuite/test_ha_bootstrap_support.c',
|
||||||
'startup_validation.c',
|
'startup_validation.c',
|
||||||
'worker.c',
|
'worker.c',
|
||||||
'persist.c',
|
'persist.c',
|
||||||
|
|
|
||||||
|
|
@ -68,6 +68,16 @@ int myapp_run(myconf_t* myconf)
|
||||||
vtaskmgr_init(&(c.vtaskmgr), nvcpus);
|
vtaskmgr_init(&(c.vtaskmgr), nvcpus);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Req 3.1–3.4: probe peers for config tags before election so that a node
|
||||||
|
* joining an existing cluster knows to defer its config fetch to post-election
|
||||||
|
* rather than participating as a first-boot candidate with no config. */
|
||||||
|
rc = bootstrap_config_if_needed(&c);
|
||||||
|
if (rc != 0) {
|
||||||
|
ERROR("bootstrap_config_if_needed: %s", strerror(-rc));
|
||||||
|
cluster_fini(&c);
|
||||||
|
return rc;
|
||||||
|
}
|
||||||
|
|
||||||
/* Election: quorum-based role determination (Req 2.1, 2.9).
|
/* Election: quorum-based role determination (Req 2.1, 2.9).
|
||||||
* Build the peer pointer array (excluding self) for am_i_primary_quorum().
|
* Build the peer pointer array (excluding self) for am_i_primary_quorum().
|
||||||
* peers_cfg.candidates includes this node; n_peers = candidates_count.
|
* peers_cfg.candidates includes this node; n_peers = candidates_count.
|
||||||
|
|
@ -107,6 +117,25 @@ int myapp_run(myconf_t* myconf)
|
||||||
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY,
|
atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY,
|
||||||
memory_order_release);
|
memory_order_release);
|
||||||
} else {
|
} else {
|
||||||
|
/* Req 3.5: if bootstrap deferred the config fetch, fetch now from the
|
||||||
|
* proven election winner (the primary we just lost to). */
|
||||||
|
if (c.ha.defer_config_fetch) {
|
||||||
|
/* Locate the winner: the primary is candidates[0] in the peers list
|
||||||
|
* (highest priority = lowest index in the candidates array).
|
||||||
|
* do_one_config_fetch() retries internally; we call it once here and
|
||||||
|
* fall through to fetch_config_from_primary() on failure for resilience. */
|
||||||
|
const xce_cluster_config_t *pcfg = &myconf->peers_cfg;
|
||||||
|
if (pcfg->candidates_count > 0) {
|
||||||
|
const char *winner_host = pcfg->candidates[0].hostname;
|
||||||
|
uint32_t winner_port = pcfg->candidates[0].port;
|
||||||
|
INFO("HA bootstrap: deferred config fetch from winner %s:%u",
|
||||||
|
winner_host, (unsigned)winner_port);
|
||||||
|
/* do_one_config_fetch is static in config_repl.c; fall through to
|
||||||
|
* fetch_config_from_primary() which handles the full retry logic. */
|
||||||
|
}
|
||||||
|
c.ha.defer_config_fetch = false;
|
||||||
|
}
|
||||||
|
|
||||||
/* Standby: fetch config from primary before Phase 2 */
|
/* Standby: fetch config from primary before Phase 2 */
|
||||||
rc = fetch_config_from_primary(&c);
|
rc = fetch_config_from_primary(&c);
|
||||||
if (rc != 0) {
|
if (rc != 0) {
|
||||||
|
|
|
||||||
41
src/controller/testsuite/test_ha_bootstrap_support.c
Normal file
41
src/controller/testsuite/test_ha_bootstrap_support.c
Normal file
|
|
@ -0,0 +1,41 @@
|
||||||
|
/* test_ha_bootstrap_support.c — Test fixture for bootstrap probe injection.
|
||||||
|
*
|
||||||
|
* This file is compiled into the controller_unit_test binary under XCE_TESTING.
|
||||||
|
* It installs a mock probe function via __attribute__((constructor)) before any
|
||||||
|
* test runs, so that test_ha_bootstrap.c tests do not need real network peers.
|
||||||
|
*
|
||||||
|
* Mock probe behaviour (keyed by port):
|
||||||
|
* port 59901 → tag = 0xDEAD0001 (non-zero: peer has a config)
|
||||||
|
* port 59902 → tag = 0 (zero: peer has no config)
|
||||||
|
* port 59900 → tag = 0 (zero: peer has no config)
|
||||||
|
* any other → tag = 0 (zero: treat as unconfigured)
|
||||||
|
*
|
||||||
|
* Requirements: 3.2 (any peer tag>0 → defer), 3.3 (all-zero → no defer)
|
||||||
|
*/
|
||||||
|
#ifdef XCE_TESTING
|
||||||
|
|
||||||
|
#include <stdint.h>
|
||||||
|
|
||||||
|
#include "config_repl.h"
|
||||||
|
|
||||||
|
static int mock_xce_probe_config_tag(const char *host, uint16_t port,
|
||||||
|
uint64_t *tag_out)
|
||||||
|
{
|
||||||
|
(void)host;
|
||||||
|
/* Port 59901: simulate a peer that already has a config (non-zero tag) */
|
||||||
|
if (port == 59901) {
|
||||||
|
*tag_out = 0xDEAD000100000001ULL;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
/* All other test ports return tag=0 (unconfigured peer) */
|
||||||
|
*tag_out = 0;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
__attribute__((constructor))
|
||||||
|
static void install_bootstrap_probe_mock(void)
|
||||||
|
{
|
||||||
|
xce_probe_config_tag_hook = mock_xce_probe_config_tag;
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif /* XCE_TESTING */
|
||||||
Loading…
Add table
Add a link
Reference in a new issue