Merge branch 'working' into job_peek_lchen
job_peek_lchen: Add fetchfile feature (job output/error peek proxy) working: HA/Raft infrastructure (consensus, replication, config mgmt) Conflict resolution summary: - controller.capnp: fetchfile @41, fetchfilereply @18, fetchfilemsg @30 - cluster.c: bifurcated OPCode_reply for fetchfile + config-push ACK - job.c: kept full fetchfile pending-request subsystem (~183 lines) - request.c: kept both xce_fetchfile_req() and XCE_RETRY_DEFAULTS - helper.c: added missing preVoteRequest/preVoteReply mappings - All 15 new test registrations (test_cbuf.c + 14 HA tests)
This commit is contained in:
Lei Chen
commit
3a2094a461
529 changed files with 345174 additions and 5048 deletions
|
|
@ -30,7 +30,26 @@ enum OPCode {
|
|||
runtasks @19;
|
||||
setpreason @20;
|
||||
clrpreason @21;
|
||||
fetchfile @22;
|
||||
replEntry @22;
|
||||
replAck @23;
|
||||
redirect @24;
|
||||
heartbeat @25;
|
||||
steppingDown @26;
|
||||
replSnapshot @27;
|
||||
replCatchUpDone @28;
|
||||
replConfig @29;
|
||||
replConfigPush @30;
|
||||
upgradeConfig @31;
|
||||
requestVote @32;
|
||||
probeConfigTag @33;
|
||||
walAck @34;
|
||||
upgradeConfigAck @35;
|
||||
snapshotTransfer @36;
|
||||
walCatchupEntry @37;
|
||||
preVoteRequest @38;
|
||||
preVoteReply @39;
|
||||
ctrlstatus @40;
|
||||
fetchfile @41;
|
||||
}
|
||||
|
||||
struct PeerInfo $C.mapname("xce_peerinfo_t") {
|
||||
|
|
@ -115,12 +134,13 @@ struct HostAttrs $C.mapname("hostattrs_t") {
|
|||
}
|
||||
|
||||
struct Worker $C.mapname("xce_worker_t"){
|
||||
name @0: Text;
|
||||
attrs @1: HostAttrs;
|
||||
status @2: UInt32;
|
||||
name @0: Text;
|
||||
attrs @1: HostAttrs;
|
||||
status @2: UInt32;
|
||||
dispatchWindow @3: TimeWindow $C.mapname("dispatch_window_config");
|
||||
thresholds @4: List(Threshold) $C.mapname("thresholds") $C.maplistcount("nthresholds");
|
||||
resvals @5: List(ValueWithIdx) $C.mapname("resvals_to_sched") $C.maplistcount("nresvals_to_sched");
|
||||
thresholds @4: List(Threshold) $C.mapname("thresholds") $C.maplistcount("nthresholds");
|
||||
resvals @5: List(NameValue) $C.mapname("resvals_to_sched") $C.maplistcount("nresvals_to_sched");
|
||||
deadSince @6: UInt64 = 0 $C.mapname("dead_since"); # epoch seconds; set when worker becomes UNAVAILABLE
|
||||
}
|
||||
|
||||
enum HostRole {
|
||||
|
|
@ -220,7 +240,7 @@ struct Conf4Sched $C.mapname("xce_conf4sched_t"){
|
|||
queues @3: List(QueueConf) $C.maplistcount("n_queues");
|
||||
ugroups @4: List(GroupConfig) $C.maplistcount("n_ugroups");
|
||||
hgroups @5: List(GroupConfig) $C.maplistcount("n_hgroups");
|
||||
resvals @6: List(ValueWithIdx) $C.mapname("resvals") $C.maplistcount("nresvals");
|
||||
resvals @6: List(NameValue) $C.mapname("resvals") $C.maplistcount("nresvals");
|
||||
hproviders @7: List(ScalerConfig) $C.mapname("hproviders") $C.maplistcount("nhproviders");
|
||||
limits @8: List(LimitConfig) $C.mapname("limits") $C.maplistcount("nlimits");
|
||||
}
|
||||
|
|
@ -266,17 +286,18 @@ struct JobStartInfo $C.mapname("job_startinfo_t"){
|
|||
}
|
||||
|
||||
struct Job $C.mapname("xce_job_t"){
|
||||
jobid @0: UInt64;
|
||||
submitData @1: JobSubmitData $C.mapname("submit");
|
||||
subtime @2: UInt64 $C.mapname("sub_time");
|
||||
queuetime @3: UInt64 $C.mapname("queue_time");
|
||||
starttime @4: UInt64 $C.mapname("start_time");
|
||||
finishtime @5: UInt64 $C.mapname("finish_time");
|
||||
state @6: UInt32;
|
||||
allocs @7: JobAlloc $C.mapname("allocs");
|
||||
runtime @8: UInt32 $C.mapname("run_time");
|
||||
pidinfo @9: JobPIDInfo;
|
||||
pendreason @10: JobPendReason $C.mapname("pend_reasons");
|
||||
jobid @0: UInt64;
|
||||
submitData @1: JobSubmitData $C.mapname("submit");
|
||||
subtime @2: UInt64 $C.mapname("sub_time");
|
||||
queuetime @3: UInt64 $C.mapname("queue_time");
|
||||
starttime @4: UInt64 $C.mapname("start_time");
|
||||
finishtime @5: UInt64 $C.mapname("finish_time");
|
||||
state @6: UInt32;
|
||||
allocs @7: JobAlloc $C.mapname("allocs");
|
||||
runtime @8: UInt32 $C.mapname("run_time");
|
||||
pidinfo @9: JobPIDInfo;
|
||||
pendreason @10: JobPendReason $C.mapname("pend_reasons");
|
||||
transientSince @11: UInt64 = 0 $C.mapname("transient_since"); # epoch seconds; set when job enters JOB_STAT_TRANSIENT
|
||||
}
|
||||
|
||||
struct SchedJob $C.mapname("xce_job_t"){
|
||||
|
|
@ -467,6 +488,29 @@ struct NotifyData $C.mapname("xce_notify_data_t"){
|
|||
}
|
||||
}
|
||||
|
||||
struct CtrlPeer $C.mapname("xce_ctrlpeer_t") {
|
||||
name @0: Text;
|
||||
host @1: Text;
|
||||
port @2: UInt32;
|
||||
}
|
||||
|
||||
struct CtrlStatus $C.mapname("xce_ctrlstatus_t") {
|
||||
clusterName @0: Text;
|
||||
primaryName @1: Text;
|
||||
responderName @2: Text;
|
||||
responderRole @3: UInt8;
|
||||
term @4: UInt64;
|
||||
configVersion @5: UInt64;
|
||||
configChecksum @6: Text;
|
||||
committedConfigVersion @7: UInt64;
|
||||
committedLsn @8: UInt64;
|
||||
appliedLsn @9: UInt64;
|
||||
fenced @10: Bool;
|
||||
peers @11: List(CtrlPeer) $C.mapname("peers.list") $C.maplistcount("peers.count");
|
||||
peerCount @12: UInt32;
|
||||
responderTimeMs @13: UInt64 $C.mapname("responder_time_ms"); # responder's wall-clock at sample time (epoch milliseconds)
|
||||
}
|
||||
|
||||
struct ClusterConf $C.mapname("xce_clusterconf_t") {
|
||||
name @0: Text;
|
||||
resources @1: List(Resource) $C.mapname("reslist.res") $C.maplistcount("reslist.count");
|
||||
|
|
@ -536,9 +580,20 @@ struct RequestReply $C.mapname("xce_reply_t"){
|
|||
debugmsg @4: Text;
|
||||
getdata @5: Data;
|
||||
jobid @6: UInt64;
|
||||
clusterconf @7: ClusterConf;
|
||||
fetchfilereply @8: FetchFileReply;
|
||||
clusterconf @7: ClusterConf;
|
||||
replconfigreplymsg @8: ReplConfigReply;
|
||||
replconfigpushreplymsg @9: ReplConfigPushReply;
|
||||
upgradeconfigreplymsg @10: UpgradeConfigReply;
|
||||
ctrlhelloreplymsg @13: CtrlHelloReply;
|
||||
requestvotereplymsg @14: RequestVoteReply;
|
||||
stateReportMsg @15: StateReport $C.mapname("state_report"); # heartbeat reply: standby state
|
||||
walackreplymsg @16: WalAckReply $C.mapname("walackreplymsg"); # WAL apply ACK from standby
|
||||
ctrlstatus @17: CtrlStatus; # ctrlstatus reply: identity/health snapshot
|
||||
fetchfilereply @18: FetchFileReply;
|
||||
}
|
||||
|
||||
redirectHost @11: Text $C.mapname("redirect_host");
|
||||
redirectPort @12: UInt16 $C.mapname("redirect_port");
|
||||
}
|
||||
|
||||
struct TaskHost $C.mapname("xce_taskhost_t") {
|
||||
|
|
@ -609,8 +664,16 @@ struct Request $C.mapname("xce_request_t"){
|
|||
runtaskmsg @18: RunTask;
|
||||
preasonmsg @19: PendReasonSet;
|
||||
clrpreasonmsg @20: ClearPReason;
|
||||
hostctrlmsg @21: HostControl;
|
||||
fetchfilemsg @22: FetchFileMsg;
|
||||
hostctrlmsg @21: HostControl;
|
||||
replconfigrequestmsg @22: ReplConfigRequest;
|
||||
replconfigpushrequestmsg @23: ReplConfigPushRequest;
|
||||
upgradeconfigrequestmsg @24: UpgradeConfigRequest;
|
||||
heartbeatmsg @25: HeartbeatMsg;
|
||||
ctrlhellomsg @26: CtrlHelloRequest;
|
||||
requestvotemsg @27: RequestVoteRequest;
|
||||
walcatchupentrymsg @28: WalCatchupEntry;
|
||||
snapshottransfermsg @29: SnapshotTransfer;
|
||||
fetchfilemsg @30: FetchFileMsg;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -670,7 +733,7 @@ struct ClusterConfig $C.mapname("cluster_config_t") {
|
|||
inactiveHeartbeatCount @4: UInt32 $C.mapname("inactive_heartbeat_count");
|
||||
fetchJobInterval @5: UInt32 $C.mapname("fetch_job_interval");
|
||||
netioTimeout @6: UInt32 $C.mapname("netio_timeout");
|
||||
vcpus @7: UInt32 $C.mapname("vcpus");
|
||||
vcpus @7: UInt32 $C.mapname("vcpus"); # deprecated: use --vcpus CLI flag; kept for config blob backward compatibility
|
||||
resources @8: List(Resource) $C.mapname("resources") $C.maplistcount("nres");
|
||||
queues @9: List(QueueConf) $C.mapname("queues") $C.maplistcount("nqueues");
|
||||
ugroups @10: List(GroupConfig) $C.mapname("ugroups") $C.maplistcount("nugroups");
|
||||
|
|
@ -683,4 +746,198 @@ struct ClusterConfig $C.mapname("cluster_config_t") {
|
|||
xstrings @17: HashZArrayConfig;
|
||||
hproviders @18: List(ScalerConfig) $C.mapname("hproviders") $C.maplistcount("nhproviders");
|
||||
limits @19: List(LimitConfig) $C.mapname("limits") $C.maplistcount("nlimits");
|
||||
configGeneration @20: UInt64 $C.mapname("config_generation");
|
||||
}
|
||||
|
||||
# ── Persistence configuration (Part 2.6) ────────────────────────────
|
||||
# $C.mapname uses capnp_persist_cfg_t (not persist_config_t) to avoid
|
||||
# collision with the plain-C struct already defined in persist.h.
|
||||
struct PersistConfig $C.mapname("capnp_persist_cfg_t") {
|
||||
walSyncMs @0 :UInt32 = 10;
|
||||
snapIntervalS @1 :UInt32 = 60;
|
||||
snapWalSizeMb @2 :UInt32 = 64;
|
||||
finishedRetainS @3 :UInt32 = 3600;
|
||||
workerRetainS @4 :UInt32 = 86400;
|
||||
syncTimeoutMs @5 :UInt32 = 50;
|
||||
snapKeepN @6 :UInt32 = 2;
|
||||
transientJobTimeoutS @7 :UInt32 = 300;
|
||||
}
|
||||
|
||||
# ── Snapshot header (Part 2.5) ───────────────────────────────────────
|
||||
struct SnapshotHeader $C.mapname("capnp_snap_hdr_t") {
|
||||
lsn @0 :UInt64;
|
||||
term @1 :UInt64;
|
||||
timestamp @2 :UInt64;
|
||||
version @3 :UInt32 = 1;
|
||||
nJobs @4 :UInt32;
|
||||
nQueues @5 :UInt32;
|
||||
nWorkers @6 :UInt32;
|
||||
checksum @7 :UInt32;
|
||||
headerChecksum @8 :UInt32;
|
||||
}
|
||||
|
||||
# ── HA heartbeat ────────────────────────────────────────────────────
|
||||
struct HeartbeatMsg $C.mapname("heartbeat_msg_t") {
|
||||
senderPriority @0: UInt32 $C.mapname("sender_priority");
|
||||
term @1: UInt64 $C.mapname("term");
|
||||
committedLsn @2: UInt64 $C.mapname("committed_lsn");
|
||||
committedConfigTag @3: UInt64 $C.mapname("committed_config_tag");
|
||||
}
|
||||
|
||||
# ── Config replication message structs ──────────────────────────────
|
||||
struct ReplConfigRequest $C.mapname("repl_config_req_t") {
|
||||
versionTag @0: UInt64 $C.mapname("version_tag");
|
||||
}
|
||||
|
||||
struct ReplConfigReply $C.mapname("repl_config_reply_t") {
|
||||
upToDate @0: Bool $C.mapname("up_to_date");
|
||||
versionTag @1: UInt64 $C.mapname("version_tag");
|
||||
sha256 @2: Data $C.mapname("sha256");
|
||||
bytes @3: Data $C.mapname("bytes");
|
||||
}
|
||||
|
||||
struct ReplConfigPushRequest $C.mapname("repl_config_push_req_t") {
|
||||
versionTag @0: UInt64 $C.mapname("version_tag");
|
||||
sha256 @1: Data $C.mapname("sha256");
|
||||
bytes @2: Data $C.mapname("bytes");
|
||||
}
|
||||
|
||||
struct ReplConfigPushReply $C.mapname("repl_config_push_reply_t") {
|
||||
accepted @0: Bool $C.mapname("accepted");
|
||||
versionTag @1: UInt64 $C.mapname("version_tag");
|
||||
}
|
||||
|
||||
struct UpgradeConfigRequest $C.mapname("upgrade_config_req_t") {
|
||||
sha256 @0: Data $C.mapname("sha256");
|
||||
bytes @1: Data $C.mapname("bytes");
|
||||
}
|
||||
|
||||
struct UpgradeConfigReply $C.mapname("upgrade_config_reply_t") {
|
||||
versionTag @0: UInt64 $C.mapname("version_tag");
|
||||
error @1: Text $C.mapname("error");
|
||||
ackedCount @2: UInt32 $C.mapname("acked_count");
|
||||
}
|
||||
|
||||
# ── WAL replication envelope (Part 6 — NOT the on-disk format) ───────
|
||||
# Named ReplEnvelope to avoid collision with the on-disk wal_entry_hdr_t.
|
||||
struct ReplEnvelope $C.mapname("capnp_repl_envelope_t") {
|
||||
term @0 :UInt64;
|
||||
lsn @1 :UInt64;
|
||||
entryType @2 :UInt32;
|
||||
payload @3 :Data;
|
||||
crc @4 :UInt32;
|
||||
prevCrc @5 :UInt32;
|
||||
}
|
||||
|
||||
# ── HA election and replication message structs (Task 1.1) ───────────
|
||||
|
||||
# Heartbeat reply state report — standby piggybacks its replication state
|
||||
# onto every OPCode_nop reply so the primary can drive convergence decisions.
|
||||
struct StateReport $C.mapname("state_report_t") {
|
||||
appliedLsn @0 :UInt64 $C.mapname("applied_lsn"); # standby's last applied WAL LSN
|
||||
configTag @1 :UInt64 $C.mapname("config_tag"); # standby's current config version tag
|
||||
}
|
||||
|
||||
struct RequestVoteRequest $C.mapname("request_vote_req_t") {
|
||||
term @0 :UInt64 $C.mapname("term");
|
||||
candidateId @1 :Text $C.mapname("candidate_id");
|
||||
configTag @2 :UInt64 $C.mapname("config_tag");
|
||||
lastLogLsn @3 :UInt64 $C.mapname("last_log_lsn");
|
||||
# Raft (lastLogTerm, lastLogIndex) freshness tuple (ha-term-safety task 1.2,
|
||||
# Req 2.3/2.4/2.6). Distinct from the XCE freshness fields above: configTag /
|
||||
# lastLogLsn drive the application-policy freshness gate (on_vote_eligible),
|
||||
# while these two feed the library's logOk gate (raft.tla HandleRequestVoteRequest
|
||||
# logOk). lastLogIndex == candidate WAL log_len; lastLogTerm == term of that entry.
|
||||
lastLogTerm @4 :UInt64 $C.mapname("last_log_term");
|
||||
lastLogIndex @5 :UInt64 $C.mapname("last_log_index");
|
||||
}
|
||||
|
||||
struct RequestVoteReply $C.mapname("request_vote_reply_t") {
|
||||
term @0 :UInt64 $C.mapname("term");
|
||||
voteGranted @1 :Bool $C.mapname("vote_granted");
|
||||
peerConfigTag @2 :UInt64 $C.mapname("peer_config_tag");
|
||||
peerLastLogLsn @3 :UInt64 $C.mapname("peer_last_log_lsn");
|
||||
}
|
||||
|
||||
struct ProbeConfigTagRequest $C.mapname("probe_config_tag_req_t") {
|
||||
name @0 :Text $C.mapname("name");
|
||||
}
|
||||
|
||||
struct ProbeConfigTagReply $C.mapname("probe_config_tag_reply_t") {
|
||||
configTag @0 :UInt64 $C.mapname("config_tag");
|
||||
}
|
||||
|
||||
struct WalAckRequest $C.mapname("wal_ack_req_t") {
|
||||
term @0 :UInt64 $C.mapname("term");
|
||||
lsn @1 :UInt64 $C.mapname("lsn");
|
||||
}
|
||||
|
||||
struct WalAckReply $C.mapname("wal_ack_reply_t") {
|
||||
term @0 :UInt64 $C.mapname("term");
|
||||
lsn @1 :UInt64 $C.mapname("lsn");
|
||||
success @2 :Bool $C.mapname("success");
|
||||
}
|
||||
|
||||
struct UpgradeConfigAck $C.mapname("upgrade_config_ack_t") {
|
||||
term @0 :UInt64 $C.mapname("term");
|
||||
ackedTag @1 :UInt64 $C.mapname("acked_tag");
|
||||
success @2 :Bool $C.mapname("success");
|
||||
}
|
||||
|
||||
# ── Snapshot chunk transfer (Req 4.1–4.2) ───────────────────────────
|
||||
# Used for OPCode_snapshotTransfer: full state snapshot transferred as
|
||||
# a sequence of fixed-size chunks (64 KB max each). Restart-on-reconnect
|
||||
# semantics — no resumption state needed. Fields @0 and @1 are preserved
|
||||
# for wire-protocol continuity; @2–@5 carry the chunk envelope.
|
||||
struct SnapshotTransfer $C.mapname("snapshot_transfer_t") {
|
||||
snapshotLsn @0 :UInt64 $C.mapname("snapshot_lsn"); # checkpoint LSN this snapshot corresponds to
|
||||
payload @1 :Data $C.mapname("payload"); # chunk payload (64 KB max)
|
||||
chunkIndex @2 :UInt32 $C.mapname("chunk_index"); # 0-based chunk number
|
||||
chunkOffset @3 :UInt64 $C.mapname("chunk_offset"); # byte offset in the full snapshot
|
||||
totalSize @4 :UInt64 $C.mapname("total_size"); # total snapshot size in bytes
|
||||
isLast @5 :Bool $C.mapname("is_last"); # true for the final chunk
|
||||
}
|
||||
|
||||
struct WalCatchupEntry $C.mapname("wal_catchup_entry_t") {
|
||||
lsn @0 :UInt64 $C.mapname("lsn");
|
||||
entryType @1 :UInt8 $C.mapname("entry_type");
|
||||
payload @2 :Data $C.mapname("payload");
|
||||
entries @3 :List(WalCatchupEntry) $C.mapname("entries") $C.maplistcount("n_entries");
|
||||
# Raft AppendEntries term fields (ha-term-safety task 1.1, Req 1.3/1.4).
|
||||
# This envelope is the replication-carrying message (WAL catch-up); the four
|
||||
# log callbacks (tasks 2.x) need prevLog{Index,Term} for the AppendEntries
|
||||
# logOk / conflict-truncation decision and `term` to stamp the entry. lsn ==
|
||||
# this entry's 1-based log index; prevLogIndex == lsn-1; prevLogTerm == term of
|
||||
# the entry at prevLogIndex (0 if absent); term == this entry's term stamp.
|
||||
term @4 :UInt64 $C.mapname("term");
|
||||
prevLogIndex @5 :UInt64 $C.mapname("prev_log_index");
|
||||
prevLogTerm @6 :UInt64 $C.mapname("prev_log_term");
|
||||
# ha-term-safety task 3.2 (Req 2.5/4.1): the leader's commit index, carried so
|
||||
# the standby intake passes it into raft_on_append_entries_request. Append-only
|
||||
# field number (@7) — no collision with the @4/@5/@6 term fields above.
|
||||
leaderCommit @7 :UInt64 $C.mapname("leader_commit");
|
||||
# bug-1067: the streaming leader's CURRENT term — the Raft AppendEntries header
|
||||
# term (`term` arg of raft_on_append_entries_request), DISTINCT from `term` @4
|
||||
# which is the entry's STORED term. For a live append the two are equal; for
|
||||
# catch-up of a HISTORICAL entry written by a prior leader, the entry's stored
|
||||
# term is LOWER than the current leader's term. Using the stored term as the AE
|
||||
# header term made the library stale-reject every historical entry
|
||||
# (ae_reject_stale), wedging cross-term catch-up. The standby intake uses this
|
||||
# as the AE header term (UpdateTerm/stale-check) and keeps `term` @4 as the
|
||||
# per-entry term (logOk / on_persist_entry). 0 = unset (legacy sender) → the
|
||||
# intake falls back to the entry's stored term (the prior, live-only behaviour).
|
||||
leaderCurrentTerm @8 :UInt64 $C.mapname("leader_current_term");
|
||||
}
|
||||
|
||||
struct CtrlHelloRequest $C.mapname("ctrl_hello_req_t") {
|
||||
name @0 :Text $C.mapname("name");
|
||||
key @1 :Data $C.mapname("key");
|
||||
appliedLsn @2 :UInt64 $C.mapname("applied_lsn");
|
||||
configTag @3 :UInt64 $C.mapname("config_tag");
|
||||
}
|
||||
|
||||
struct CtrlHelloReply $C.mapname("ctrl_hello_reply_t") {
|
||||
committedLsn @0 :UInt64 $C.mapname("committed_lsn");
|
||||
needsSnapshot @1 :Bool $C.mapname("needs_snapshot");
|
||||
configTag @2 :UInt64 $C.mapname("config_tag");
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue