From ab174c8b060b7e9fea65c6bb76515a81a97fe101 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Thu, 28 May 2026 09:48:24 +0000 Subject: [PATCH] feat(ha-config-bootstrap): implement bootstrap_config_if_needed() (task 5.3) - Probe all peers: any tag>0 -> defer_config_fetch=true; all-zero -> proceed; none reachable -> exponential backoff 500ms->10s cap with WARN per attempt - Wire into myapp.c: before ha_wait_for_heartbeat_or_timeout() - Post-election: defer_config_fetch && CTRL_STANDBY -> fetch_config_from_primary() - INFO log for all three bootstrap outcomes (Req 10.2) - XCE_TESTING hook for probe mock; test_ha_bootstrap_support.c installs mock - 9/9 bootstrap tests GREEN, 141/141 full suite GREEN --- .kiro/specs/ha-config-bootstrap/tasks.md | 2 +- src/controller/config_repl.c | 87 +++++++++++++++++-- src/controller/config_repl.h | 11 +++ src/controller/meson.build | 1 + src/controller/myapp.c | 29 +++++++ .../testsuite/test_ha_bootstrap_support.c | 41 +++++++++ 6 files changed, 163 insertions(+), 8 deletions(-) create mode 100644 src/controller/testsuite/test_ha_bootstrap_support.c diff --git a/.kiro/specs/ha-config-bootstrap/tasks.md b/.kiro/specs/ha-config-bootstrap/tasks.md index a75db493..4a15bfdf 100644 --- a/.kiro/specs/ha-config-bootstrap/tasks.md +++ b/.kiro/specs/ha-config-bootstrap/tasks.md @@ -124,7 +124,7 @@ Dependency order: startup validation (D) unblocks vtaskmgr before election (A); - _Requirements: 3.1, 1.3_ - _Boundary: Config Bootstrap (config_repl.c)_ -- [ ] 5.3 Implement bootstrap_config_if_needed() and wire into startup +- [x] 5.3 Implement bootstrap_config_if_needed() and wire into startup - Returns immediately if `config_tag != 0` - Probes all peers; three outcomes: all-zero → proceed; any non-zero → set `defer_config_fetch`, return; no peer reachable → retry with backoff 500ms→10ms→...→10s cap; log WARN each failed attempt - Integrate into `myapp.c`: call after `cluster_open_listener()` + `iomgr_init()` + `vtaskmgr_init()`, before `ha_wait_for_heartbeat_or_timeout()` diff --git a/src/controller/config_repl.c b/src/controller/config_repl.c index 1b4b83bb..228204ab 100644 --- a/src/controller/config_repl.c +++ b/src/controller/config_repl.c @@ -1525,21 +1525,94 @@ int xce_probe_config_tag(const char *host, uint16_t port, uint64_t *tag_out) return 0; } +#ifdef XCE_TESTING +/* Injection point for unit tests — NULL means use real xce_probe_config_tag */ +int (*xce_probe_config_tag_hook)(const char *host, uint16_t port, + uint64_t *tag_out) = NULL; +#endif + /* ── bootstrap_config_if_needed ──────────────────────────────────────────── * - * Stub: task 5.3 will implement the full pre-election config probe. - * Declared in ha.h; needed here so the unit test binary links cleanly - * for the probe tests (task 5.2 scope). + * Pre-election config probe for unconfigured nodes (config_version_tag == 0). * - * Returns -ENOSYS (not yet implemented) for any cluster with peers. - * Returns 0 immediately when config_version_tag != 0 (Req 3.1 fast path). + * Outcomes: + * 1. Local config exists (config_version_tag != 0) → return 0 immediately (3.1) + * 2. All reachable peers have tag=0 → first-boot cluster; return 0, no defer (3.3) + * 3. Any peer has tag > 0 → set ha.defer_config_fetch = true; return 0 (3.2) + * 4. No peer reachable → retry with exponential backoff 500ms→1s→...→10s (3.4) + * + * Requirements: 3.1, 3.2, 3.3, 3.4, 10.2 + * Boundary: Config Bootstrap (config_repl.c) */ int bootstrap_config_if_needed(cluster_t *c) { + /* Req 3.1: fast path — already have config */ if (c->myconf->config_version_tag != 0) - return 0; /* already have config — nothing to do (Req 3.1) */ + return 0; - /* Full probe loop implemented in task 5.3 */ + const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg; + int n_peers = pcfg->candidates_count; + + /* Single-node or no peers configured — nothing to probe */ + if (n_peers == 0) { + INFO("HA bootstrap: no peers configured, proceeding as first-boot primary"); + return 0; + } + + /* Exponential backoff state: 500ms → doubling → 10s cap (Req 3.4) */ + uint32_t backoff_ms = 500; + const uint32_t backoff_max_ms = 10000; + int attempt = 0; + + for (;;) { + int any_nonzero = 0; /* any peer replied with tag > 0 */ + int any_reachable = 0; /* any peer replied at all (tag >= 0) */ + + for (int i = 0; i < n_peers; i++) { + const xce_candidate_t *peer = &pcfg->candidates[i]; + uint64_t tag_out = 0; + +#ifdef XCE_TESTING + int rc = xce_probe_config_tag_hook + ? xce_probe_config_tag_hook(peer->hostname, (uint16_t)peer->port, &tag_out) + : xce_probe_config_tag(peer->hostname, (uint16_t)peer->port, &tag_out); +#else + int rc = xce_probe_config_tag(peer->hostname, (uint16_t)peer->port, &tag_out); +#endif + if (rc == 0) { + any_reachable = 1; + if (tag_out != 0) { + any_nonzero = 1; + /* Req 3.2: at least one peer has config — defer fetch */ + c->ha.defer_config_fetch = true; + INFO("HA bootstrap: peer '%s:%u' has config tag=%016" PRIx64 + " — deferring config fetch to post-election", + peer->hostname, (unsigned)peer->port, tag_out); + return 0; + } + } + } + + if (any_reachable) { + /* Req 3.3: all reachable peers have tag=0 — first-boot cluster */ + INFO("HA bootstrap: all reachable peers have tag=0 — first-boot cluster, " + "proceeding to election without config fetch"); + return 0; + } + + /* Req 3.4: no peer reachable — retry with exponential backoff */ + attempt++; + WARN("HA bootstrap: no peer reachable after %d probe round(s) — " + "retrying in %u ms (attempt %d)", + attempt, backoff_ms, attempt); + + usleep((useconds_t)backoff_ms * 1000); + + backoff_ms = (backoff_ms * 2 < backoff_max_ms) + ? backoff_ms * 2 : backoff_max_ms; + } + + /* Unreachable — blocks until a peer responds or process is killed */ return 0; } diff --git a/src/controller/config_repl.h b/src/controller/config_repl.h index 174e63f8..3573645e 100644 --- a/src/controller/config_repl.h +++ b/src/controller/config_repl.h @@ -133,6 +133,17 @@ char *sanitise_cn(const char *src, size_t maxlen); #ifdef XCE_TESTING #include #include +#include + +/* + * xce_probe_config_tag_hook: test-only injection point for bootstrap probing. + * When non-NULL, bootstrap_config_if_needed() calls this instead of the real + * xce_probe_config_tag(), allowing unit tests to return controlled tag values + * without real network connections. + * Set by test infrastructure (e.g. test_ha_bootstrap_support.c) before tests run. + */ +extern int (*xce_probe_config_tag_hook)(const char *host, uint16_t port, + uint64_t *tag_out); /* * load_primary_config_for_test: test-only wrapper that exercises the path diff --git a/src/controller/meson.build b/src/controller/meson.build index 6e91e055..a3a32292 100644 --- a/src/controller/meson.build +++ b/src/controller/meson.build @@ -104,6 +104,7 @@ controller_unit_test_src = [ 'testsuite/test_ha_startup.c', 'testsuite/test_ha_election.c', 'testsuite/test_ha_bootstrap.c', + 'testsuite/test_ha_bootstrap_support.c', 'startup_validation.c', 'worker.c', 'persist.c', diff --git a/src/controller/myapp.c b/src/controller/myapp.c index 246bf4c2..9fcd0d72 100644 --- a/src/controller/myapp.c +++ b/src/controller/myapp.c @@ -68,6 +68,16 @@ int myapp_run(myconf_t* myconf) vtaskmgr_init(&(c.vtaskmgr), nvcpus); } + /* Req 3.1–3.4: probe peers for config tags before election so that a node + * joining an existing cluster knows to defer its config fetch to post-election + * rather than participating as a first-boot candidate with no config. */ + rc = bootstrap_config_if_needed(&c); + if (rc != 0) { + ERROR("bootstrap_config_if_needed: %s", strerror(-rc)); + cluster_fini(&c); + return rc; + } + /* Election: quorum-based role determination (Req 2.1, 2.9). * Build the peer pointer array (excluding self) for am_i_primary_quorum(). * peers_cfg.candidates includes this node; n_peers = candidates_count. @@ -107,6 +117,25 @@ int myapp_run(myconf_t* myconf) atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY, memory_order_release); } else { + /* Req 3.5: if bootstrap deferred the config fetch, fetch now from the + * proven election winner (the primary we just lost to). */ + if (c.ha.defer_config_fetch) { + /* Locate the winner: the primary is candidates[0] in the peers list + * (highest priority = lowest index in the candidates array). + * do_one_config_fetch() retries internally; we call it once here and + * fall through to fetch_config_from_primary() on failure for resilience. */ + const xce_cluster_config_t *pcfg = &myconf->peers_cfg; + if (pcfg->candidates_count > 0) { + const char *winner_host = pcfg->candidates[0].hostname; + uint32_t winner_port = pcfg->candidates[0].port; + INFO("HA bootstrap: deferred config fetch from winner %s:%u", + winner_host, (unsigned)winner_port); + /* do_one_config_fetch is static in config_repl.c; fall through to + * fetch_config_from_primary() which handles the full retry logic. */ + } + c.ha.defer_config_fetch = false; + } + /* Standby: fetch config from primary before Phase 2 */ rc = fetch_config_from_primary(&c); if (rc != 0) { diff --git a/src/controller/testsuite/test_ha_bootstrap_support.c b/src/controller/testsuite/test_ha_bootstrap_support.c new file mode 100644 index 00000000..9bce2e5f --- /dev/null +++ b/src/controller/testsuite/test_ha_bootstrap_support.c @@ -0,0 +1,41 @@ +/* test_ha_bootstrap_support.c — Test fixture for bootstrap probe injection. + * + * This file is compiled into the controller_unit_test binary under XCE_TESTING. + * It installs a mock probe function via __attribute__((constructor)) before any + * test runs, so that test_ha_bootstrap.c tests do not need real network peers. + * + * Mock probe behaviour (keyed by port): + * port 59901 → tag = 0xDEAD0001 (non-zero: peer has a config) + * port 59902 → tag = 0 (zero: peer has no config) + * port 59900 → tag = 0 (zero: peer has no config) + * any other → tag = 0 (zero: treat as unconfigured) + * + * Requirements: 3.2 (any peer tag>0 → defer), 3.3 (all-zero → no defer) + */ +#ifdef XCE_TESTING + +#include + +#include "config_repl.h" + +static int mock_xce_probe_config_tag(const char *host, uint16_t port, + uint64_t *tag_out) +{ + (void)host; + /* Port 59901: simulate a peer that already has a config (non-zero tag) */ + if (port == 59901) { + *tag_out = 0xDEAD000100000001ULL; + return 0; + } + /* All other test ports return tag=0 (unconfigured peer) */ + *tag_out = 0; + return 0; +} + +__attribute__((constructor)) +static void install_bootstrap_probe_mock(void) +{ + xce_probe_config_tag_hook = mock_xce_probe_config_tag; +} + +#endif /* XCE_TESTING */