fix(ha-commit-engine): raise commit-barrier default 50->500ms + add --sync-timeout-ms
Live symptom: jobsub intermittently failed with 'Connection timed out' even though the cluster had quorum. Root cause: the synchronous commit barrier waits only DEFAULT_SYNC_TIMEOUT_MS=50ms for durable majority replication, and that was not configurable. 50ms is too tight for durable sync majority replication — especially since standbys now fdatasync each replicated entry before ACK — so a multi-entry jobsub's later WAL entry intermittently misses the window: commit: WAIT deadline expired without commit (lsn=3 committed_lsn=2 timeout_ms=50) -> persist hook returns -ETIMEDOUT -> handle_jobsubmit rejects -> client ETIMEDOUT. Fix: - Raise DEFAULT_SYNC_TIMEOUT_MS 50 -> 500 (config.h) — realistic for durable sync majority replication, still bounded for failure detection. - Add a --sync-timeout-ms controller flag threaded arguments -> myconf -> persist pcfg (override applied before persist_init); 0 = use default, positive overrides — so operators can tune per environment. TDD: literal-pinned persist_config_defaults().sync_timeout_ms == 500. All timeout-sensitive tests already pass an explicit timeout_ms (none rely on the default), so none are slowed; full controller suite green under ASAN. No commit-engine/barrier/replication/wire change beyond the default value + plumbing. bug-561.
This commit is contained in:
parent
c4fd3e1c8f
commit
ca1d9ae42d
5 changed files with 24 additions and 1 deletions
|
|
@ -29,6 +29,7 @@ const char* argp_program_bug_address = BUG_EMAIL;
|
|||
#define OPT_CONF_STALE_TIMEOUT 258
|
||||
#define OPT_VCPUS 259
|
||||
#define OPT_CERTDIR 260
|
||||
#define OPT_SYNC_TIMEOUT 261
|
||||
|
||||
struct arguments {
|
||||
char* myname;
|
||||
|
|
@ -40,6 +41,7 @@ struct arguments {
|
|||
long conf_stale_timeout;
|
||||
int vcpus;
|
||||
char* certdir;
|
||||
uint32_t sync_timeout_ms;
|
||||
};
|
||||
|
||||
static struct argp_option options[] = {
|
||||
|
|
@ -58,6 +60,8 @@ static struct argp_option options[] = {
|
|||
"Number of virtual CPUs for the task manager (must be >= 1)" },
|
||||
{ "certdir", OPT_CERTDIR, "DIR", 0,
|
||||
"Directory containing xce.crt and xce.key (default: <workdir>/certs)" },
|
||||
{ "sync-timeout-ms", OPT_SYNC_TIMEOUT, "MS", 0,
|
||||
"Synchronous majority-replication commit barrier timeout in ms (0 = default)" },
|
||||
{ 0 }
|
||||
};
|
||||
|
||||
|
|
@ -100,6 +104,13 @@ static error_t parse_opt(int key, char* arg, struct argp_state* state)
|
|||
case OPT_CERTDIR:
|
||||
arguments->certdir = arg;
|
||||
break;
|
||||
case OPT_SYNC_TIMEOUT: {
|
||||
int v = atoi(arg);
|
||||
if (v < 0)
|
||||
argp_error(state, "--sync-timeout-ms must be >= 0 (got: %s)", arg);
|
||||
arguments->sync_timeout_ms = (uint32_t)v;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
return ARGP_ERR_UNKNOWN;
|
||||
}
|
||||
|
|
@ -151,6 +162,7 @@ int main(int argc, char* argv[])
|
|||
arguments.conf_keep_n = 2;
|
||||
arguments.conf_stale_timeout = 86400;
|
||||
arguments.vcpus = 0; /* 0 = not set; use cluster config default */
|
||||
arguments.sync_timeout_ms = 0; /* 0 = not set; use persist config default */
|
||||
|
||||
argp_parse(&argp, argc, argv, 0, 0, &arguments);
|
||||
|
||||
|
|
@ -199,6 +211,7 @@ int main(int argc, char* argv[])
|
|||
myconf.conf_keep_n = arguments.conf_keep_n;
|
||||
myconf.conf_stale_timeout = (time_t)arguments.conf_stale_timeout;
|
||||
myconf.vcpus = arguments.vcpus;
|
||||
myconf.sync_timeout_ms = arguments.sync_timeout_ms;
|
||||
memset(&myconf.peers_cfg, 0, sizeof(myconf.peers_cfg));
|
||||
|
||||
if (arguments.peers_str) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue