Rongsong Shen
ca1d9ae42d
Live symptom: jobsub intermittently failed with 'Connection timed out' even though the cluster had quorum. Root cause: the synchronous commit barrier waits only DEFAULT_SYNC_TIMEOUT_MS=50ms for durable majority replication, and that was not configurable. 50ms is too tight for durable sync majority replication — especially since standbys now fdatasync each replicated entry before ACK — so a multi-entry jobsub's later WAL entry intermittently misses the window: commit: WAIT deadline expired without commit (lsn=3 committed_lsn=2 timeout_ms=50) -> persist hook returns -ETIMEDOUT -> handle_jobsubmit rejects -> client ETIMEDOUT. Fix: - Raise DEFAULT_SYNC_TIMEOUT_MS 50 -> 500 (config.h) — realistic for durable sync majority replication, still bounded for failure detection. - Add a --sync-timeout-ms controller flag threaded arguments -> myconf -> persist pcfg (override applied before persist_init); 0 = use default, positive overrides — so operators can tune per environment. TDD: literal-pinned persist_config_defaults().sync_timeout_ms == 500. All timeout-sensitive tests already pass an explicit timeout_ms (none rely on the default), so none are slowed; full controller suite green under ASAN. No commit-engine/barrier/replication/wire change beyond the default value + plumbing. bug-561.
97 lines
3.4 KiB
C
97 lines
3.4 KiB
C
#if !defined(_CONFIG_H_)
|
|
|
|
#define _CONFIG_H_
|
|
|
|
#include <stdint.h>
|
|
|
|
#if defined(__has_c_attribute) && __has_c_attribute(gnu::constructor)
|
|
#define CONSTRUCTOR(prio) [[gnu::constructor(prio)]]
|
|
#define DESTRUCTOR(prio) [[gnu::destructor(prio)]]
|
|
#elif defined(__GNUC__) || defined(__clang__)
|
|
#define CONSTRUCTOR(prio) __attribute__((constructor(prio)))
|
|
#define DESTRUCTOR(prio) __attribute__((destructor(prio)))
|
|
#else
|
|
#define CONSTRUCTOR(prio)
|
|
#define DESTRUCTOR(prio)
|
|
#warning "No constructor support found"
|
|
#endif
|
|
|
|
typedef uint64_t jobid_t;
|
|
|
|
#define BUG_EMAIL "rshen@shenrs.eu"
|
|
#define DEFAULT_WORKDIR "/tmp/xce"
|
|
|
|
#define XCE_MAXNAME_LEN (256)
|
|
#define XCE_MAXHOSTNAME_LEN (256) /* hostname suggest to 256 */
|
|
#define XCE_MAXDESC_LEN (4096) /* enlarge to 4096 */
|
|
#define XCE_MAXKEY_LEN (256)
|
|
#define XCE_MAXMODEL_LEN (32)
|
|
#define XCE_MAXTYPE_LEN (32)
|
|
|
|
#define XCE_WORKER_HTAB_SIZE (673)
|
|
#define XCE_JOB_HTAB_SIZE (5009)
|
|
|
|
#define XCE_MAXPATH_LEN (4096) /* enlarge to 4096 */
|
|
#define XCE_MAXCMD_LEN (4096)
|
|
#define XCE_MAX_ERRMSG_LEN (4096)
|
|
|
|
#define N_JOBID_BITS (sizeof(jobid_t) * 8)
|
|
|
|
#define N_TAG_BITS (2)
|
|
#define TAG_SHIFT (N_JOBID_BITS - N_TAG_BITS)
|
|
|
|
#define TAG_JOB_ARRAY (0x01)
|
|
|
|
#define N_CLUSTER_ID_BITS (6)
|
|
#define CLUSTER_ID_SHIFT (TAG_SHIFT - N_CLUSTER_ID_BITS)
|
|
|
|
#define N_APISERVER_ID_BITS (4)
|
|
#define APISERVER_ID_SHIFT (CLUSTER_ID_SHIFT - N_APISERVER_ID_BITS)
|
|
|
|
#define N_TIMESTAMP_BITS (32)
|
|
#define TIMESTAMP_SHIFT (APISERVER_ID_SHIFT - N_TIMESTAMP_BITS)
|
|
|
|
#define N_ARRAY_IDX_BITS (15)
|
|
|
|
#define N_SEQUENCE_BITS \
|
|
(N_JOBID_BITS - N_CLUSTER_ID_BITS - N_APISERVER_ID_BITS - N_TIMESTAMP_BITS \
|
|
- N_TAG_BITS)
|
|
|
|
#define SEQUENCE_HIGH_SHIFT (N_ARRAY_IDX_BITS)
|
|
|
|
#define N_MAX_FIELDS_JOBID (5)
|
|
|
|
#define UMAX_BITS(w) ((((uint64_t)0x01) << ((int)w)) - 1)
|
|
|
|
#define MAX_CLUSTER_ID (UMAX_BITS(N_CLUSTER_ID_BITS))
|
|
#define MAX_APISERVER_ID (UMAX_BITS(N_APISERVER_ID_BITS))
|
|
#define MAX_ARRAY_IDX (UMAX_BITS(N_ARRAY_IDX_BITS))
|
|
|
|
#define REGEX_JOBID "[ \t]*([0-9]+)-([0-9]+)-([0-9]+)-([0-9]+)[ \t]*"
|
|
#define REGEX_JOBARRAY_ID \
|
|
"[ \t]*([0-9]+)-([0-9]+)-([0-9]+)-([0-9]+)\\[([0-9]+)\\][ \t]*"
|
|
|
|
/* ======================================================
|
|
* Persistence & HA system constants (Part 2.6, Appendix A.10)
|
|
* ====================================================== */
|
|
#define MAX_STANDBYS 3
|
|
#define MAX_HA_CANDIDATES 4
|
|
#define WAL_FORMAT_VERSION 1
|
|
#define WAL_ENTRY_TYPE_MAX 100 /* WAL_CHECKPOINT = 100, highest known type */
|
|
#define MAX_WAL_PAYLOAD_BYTES (1 << 20) /* 1 MB per replication entry */
|
|
#define MAX_SNAPSHOT_BYTES (256 * 1024 * 1024) /* 256 MB max snapshot size */
|
|
#define WAL_HDR_CRC_SIZE 32 /* sizeof(wal_entry_hdr_t) - 2*sizeof(uint32_t) */
|
|
#define WAL_CRC_NONE 0xFFFFFFFFu /* sentinel: no predecessor in chain */
|
|
|
|
#define DEFAULT_WAL_SYNC_MS 10
|
|
#define DEFAULT_SNAP_INTERVAL_S 60
|
|
#define DEFAULT_SNAP_WAL_SIZE_MB 64
|
|
#define DEFAULT_SNAP_KEEP_N 2
|
|
#define DEFAULT_TRANSIENT_JOB_TIMEOUT_S 300
|
|
#define DEFAULT_FINISHED_RETAIN_S 3600
|
|
#define DEFAULT_WORKER_RETAIN_S 86400
|
|
#define DEFAULT_SYNC_TIMEOUT_MS 500
|
|
#define DEFAULT_HB_INTERVAL_MS 3000
|
|
#define DEFAULT_MAX_WAL_RECORDS 10000
|
|
|
|
#endif
|