Rongsong Shen
1439e2655e
Chain of fixes needed to get --force-snapshot soak to a real live PASS (raft-ha-core-safety task 5.10a / ha-entity-snapshots task 7.1): 1. WAL segment rotation was gated on a hardcoded 64MB compile-time constant; trim_wal_segments() never deletes the active (currently-open) segment. At soak job counts a segment never rotated, so nothing was ever physically trimmable and ha_evaluate_repl_action always chose STREAM_WAL over SEND_SNAPSHOT for an empty-rejoining node. Fixed by threading a wal_segment_bytes override through persist_config_t (mirrors the --max-wal-records pattern: config.h default, persist.h setter, persist.c 3 usage sites + minimum-floor enforcement, 6-point CLI wiring through main.c/myconf.h/myapp.c/cluster.c) plus a new --wal-segment-bytes CLI flag. run-ha-soak.sh's --force-snapshot mode now sets WAL_SEGMENT_BYTES=4096 to force real rotation at soak scale. 2. This unmasked a genuine heap-use-after-free in INVOKE_MSG_HANDLER (include/netio.h): the macro calls ep->msg_handler(ep) synchronously (not via the vtask dispatcher, so the bug-1298 vtask ref/unref pairing doesn't cover it) then writes ep->fsm_in.pkt = NULL. A stale vote-reply handler chain (raft_on_vote_reply -> raft_cb_become_follower -> endpoint_unref) could drop the endpoint's last ref and free it inside the handler call, so the trailing write hit freed memory. This is the first-ever live exercise of a node receiving a snapshot transfer while mid-election. Fixed by holding a ref across the synchronous call, same pattern task 5.8 established for the async path. With both fixes: --force-snapshot soak now PASSES live (committed divergence=0, job_loss=0, confirmed real "HA: snapshot" install on the restarted node) — first genuine live proof of this scenario. 3-round --chaotic regression clean. Full 217-test controller suite green before/after every fix in the chain. Builds on the delaymgr starvation fix (b9ad018) and walcompare checkpoint-floor fix (f3c86a6) from earlier in this investigation.
119 lines
4.6 KiB
C
119 lines
4.6 KiB
C
#if !defined(_CONFIG_H_)
|
|
|
|
#define _CONFIG_H_
|
|
|
|
#include <stdint.h>
|
|
|
|
#if defined(__has_c_attribute) && __has_c_attribute(gnu::constructor)
|
|
#define CONSTRUCTOR(prio) [[gnu::constructor(prio)]]
|
|
#define DESTRUCTOR(prio) [[gnu::destructor(prio)]]
|
|
#elif defined(__GNUC__) || defined(__clang__)
|
|
#define CONSTRUCTOR(prio) __attribute__((constructor(prio)))
|
|
#define DESTRUCTOR(prio) __attribute__((destructor(prio)))
|
|
#else
|
|
#define CONSTRUCTOR(prio)
|
|
#define DESTRUCTOR(prio)
|
|
#warning "No constructor support found"
|
|
#endif
|
|
|
|
typedef uint64_t jobid_t;
|
|
|
|
#define BUG_EMAIL "rshen@shenrs.eu"
|
|
#define DEFAULT_WORKDIR "/tmp/xce"
|
|
|
|
#define XCE_MAXNAME_LEN (256)
|
|
#define XCE_MAXHOSTNAME_LEN (256) /* hostname suggest to 256 */
|
|
#define XCE_MAXDESC_LEN (4096) /* enlarge to 4096 */
|
|
#define XCE_MAXKEY_LEN (256)
|
|
#define XCE_MAXMODEL_LEN (32)
|
|
#define XCE_MAXTYPE_LEN (32)
|
|
|
|
#define XCE_WORKER_HTAB_SIZE (673)
|
|
#define XCE_JOB_HTAB_SIZE (5009)
|
|
|
|
#define XCE_MAXPATH_LEN (4096) /* enlarge to 4096 */
|
|
#define XCE_MAXCMD_LEN (4096)
|
|
#define XCE_MAX_ERRMSG_LEN (4096)
|
|
|
|
#define N_JOBID_BITS (sizeof(jobid_t) * 8)
|
|
|
|
#define N_TAG_BITS (2)
|
|
#define TAG_SHIFT (N_JOBID_BITS - N_TAG_BITS)
|
|
|
|
#define TAG_JOB_ARRAY (0x01)
|
|
|
|
#define N_CLUSTER_ID_BITS (6)
|
|
#define CLUSTER_ID_SHIFT (TAG_SHIFT - N_CLUSTER_ID_BITS)
|
|
|
|
#define N_APISERVER_ID_BITS (4)
|
|
#define APISERVER_ID_SHIFT (CLUSTER_ID_SHIFT - N_APISERVER_ID_BITS)
|
|
|
|
#define N_TIMESTAMP_BITS (32)
|
|
#define TIMESTAMP_SHIFT (APISERVER_ID_SHIFT - N_TIMESTAMP_BITS)
|
|
|
|
#define N_ARRAY_IDX_BITS (15)
|
|
|
|
#define N_SEQUENCE_BITS \
|
|
(N_JOBID_BITS - N_CLUSTER_ID_BITS - N_APISERVER_ID_BITS - N_TIMESTAMP_BITS \
|
|
- N_TAG_BITS)
|
|
|
|
#define SEQUENCE_HIGH_SHIFT (N_ARRAY_IDX_BITS)
|
|
|
|
#define N_MAX_FIELDS_JOBID (5)
|
|
|
|
#define UMAX_BITS(w) ((((uint64_t)0x01) << ((int)w)) - 1)
|
|
|
|
#define MAX_CLUSTER_ID (UMAX_BITS(N_CLUSTER_ID_BITS))
|
|
#define MAX_APISERVER_ID (UMAX_BITS(N_APISERVER_ID_BITS))
|
|
#define MAX_ARRAY_IDX (UMAX_BITS(N_ARRAY_IDX_BITS))
|
|
|
|
#define REGEX_JOBID "[ \t]*([0-9]+)-([0-9]+)-([0-9]+)-([0-9]+)[ \t]*"
|
|
#define REGEX_JOBARRAY_ID \
|
|
"[ \t]*([0-9]+)-([0-9]+)-([0-9]+)-([0-9]+)\\[([0-9]+)\\][ \t]*"
|
|
|
|
/* ======================================================
|
|
* Persistence & HA system constants (Part 2.6, Appendix A.10)
|
|
* ====================================================== */
|
|
#define MAX_STANDBYS 3
|
|
#define MAX_HA_CANDIDATES 4
|
|
#define WAL_FORMAT_VERSION 1
|
|
#define WAL_ENTRY_TYPE_MAX 100 /* WAL_CHECKPOINT = 100, highest known type */
|
|
#define MAX_WAL_PAYLOAD_BYTES (1 << 20) /* 1 MB per replication entry */
|
|
#define MAX_SNAPSHOT_BYTES (256 * 1024 * 1024) /* 256 MB max snapshot size */
|
|
#define WAL_HDR_CRC_SIZE 32 /* sizeof(wal_entry_hdr_t) - 2*sizeof(uint32_t) */
|
|
#define WAL_CRC_NONE 0xFFFFFFFFu /* sentinel: no predecessor in chain */
|
|
|
|
#define DEFAULT_WAL_SYNC_MS 10
|
|
#define DEFAULT_SNAP_INTERVAL_S 60
|
|
#define DEFAULT_SNAP_WAL_SIZE_MB 64
|
|
#define DEFAULT_SNAP_KEEP_N 2
|
|
#define DEFAULT_TRANSIENT_JOB_TIMEOUT_S 300
|
|
#define DEFAULT_FINISHED_RETAIN_S 3600
|
|
#define DEFAULT_WORKER_RETAIN_S 86400
|
|
#define DEFAULT_SYNC_TIMEOUT_MS 500
|
|
#define DEFAULT_HB_INTERVAL_MS 3000
|
|
#define DEFAULT_MAX_WAL_RECORDS 10000
|
|
#define DEFAULT_WAL_SEGMENT_BYTES (64UL * 1024 * 1024) /* 64 MB per segment */
|
|
|
|
/* ======================================================
|
|
* raft-drive-async constants (task 1.1)
|
|
* Non-blocking Raft drive: bounded peer connect + slow-lane executor +
|
|
* pool-level liveness reservation. All overridable via myconf_t (0 = use
|
|
* the default below); see drive_async_liveness_reserved() in ha.h for the
|
|
* peer-scaled reservation derivation.
|
|
* ====================================================== */
|
|
|
|
/* R2.3: bound the peer connect attempt so an unreachable peer cannot stall
|
|
* the delaymgr heartbeat thread for the OS connect timeout. BoundedConnect
|
|
* polls up to this many ms after a non-blocking connect() before failing. */
|
|
#define DEFAULT_CONNECT_TIMEOUT_MS 1000
|
|
|
|
/* SlowLane executor sizing (task 1.2 consumes these). Dedicated worker
|
|
* threads isolated from the vtask request pool; sized to peer count so a
|
|
* snapshot transfer and a reconnect to distinct peers do not starve each
|
|
* other. MAX_HA_CANDIDATES (4) bounds a supported cluster, so a small
|
|
* constant covers reconnect + transfer per peer without over-allocating. */
|
|
#define DEFAULT_SLOWLANE_WORKERS 4
|
|
#define DEFAULT_SLOWLANE_MAX_INFLIGHT 8
|
|
|
|
#endif
|