lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'working' into job_peek_lchen

Browse source 
job_peek_lchen: Add fetchfile feature (job output/error peek proxy)
working: HA/Raft infrastructure (consensus, replication, config mgmt)

Conflict resolution summary:
- controller.capnp: fetchfile @41, fetchfilereply @18, fetchfilemsg @30
- cluster.c: bifurcated OPCode_reply for fetchfile + config-push ACK
- job.c: kept full fetchfile pending-request subsystem (~183 lines)
- request.c: kept both xce_fetchfile_req() and XCE_RETRY_DEFAULTS
- helper.c: added missing preVoteRequest/preVoteReply mappings
- All 15 new test registrations (test_cbuf.c + 14 HA tests)
This commit is contained in:
Lei Chen 2026-07-22 13:57:25 +08:00
commit 3a2094a461
529 changed files with 345174 additions and 5048 deletions
Download patch file Download diff file Expand all files Collapse all files

21
src/frontend/xcecli/common/output/page_print_array_elements_jobs.go
View file

@ -4,7 +4,6 @@ import (
"encoding/json"
"fmt"
"strings"
"time"
jobfmt "xcecli/common/format"
"xcecli/models"
)
@ -96,7 +95,7 @@ func (p *ArrayElementsPagePrinter) initColumnWidths(jobs *models.JobsResponse) {
case models.HEADER_JOB_NAME:
width = len(job.JobName)
case models.HEADER_JOB_SUBMIT_TIME:
width = len(job.SubmitTime)
width = len(fmtUnixTime(job.SubmitTime))
}
if !p.wideFormat && width > 15 {
@ -166,7 +165,7 @@ func (p *ArrayElementsPagePrinter) formatRows(jobs *models.JobsResponse, truncat
mainRow = appendOrDefault(mainRow, FormatStringField(job.FromHost, width, truncatFlag, true), "-")
case models.HEADER_JOB_EXEC_HOST:
var execHosts models.ExecHosts
if err := json.Unmarshal([]byte(job.ExecHosts.String()), &execHosts); err == nil && len(execHosts.HostSlots) > 0 {
if err := json.Unmarshal([]byte(job.ExecHosts.String()), &execHosts); err == nil && len(execHosts.HostSlots) > 0 && execHosts.HostSlots[0].HostName != "" {
firstHost := fmt.Sprintf("%d*%s", execHosts.HostSlots[0].NumSlots, execHosts.HostSlots[0].HostName)
mainRow = appendOrDefault(mainRow, FormatStringField(firstHost, width, truncatFlag, true), "-")
@ -190,21 +189,7 @@ func (p *ArrayElementsPagePrinter) formatRows(jobs *models.JobsResponse, truncat
case models.HEADER_JOB_NAME:
mainRow = appendOrDefault(mainRow, FormatStringField(job.JobName, width, truncatFlag, true), "-")
case models.HEADER_JOB_SUBMIT_TIME:
// remove the timezone part and parse the time
timeStr := strings.Split(job.SubmitTime, " ")[0:2]
timeStrWithoutTZ := strings.Join(timeStr, " ")
if submitTime, err := time.Parse("2006-01-02 15:04:05", timeStrWithoutTZ); err == nil {
month := submitTime.Format("Jan")
displayTime := fmt.Sprintf("%s %02d %02d:%02d",
month,
submitTime.Day(),
submitTime.Hour(),
submitTime.Minute())
mainRow = appendOrDefault(mainRow, FormatStringField(displayTime, width, truncatFlag, true), "-")
} else {
mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, "-"), "-")
}
mainRow = appendOrDefault(mainRow, FormatStringField(fmtUnixTime(job.SubmitTime), width, truncatFlag, true), "-")
}
}
rows = append(rows, mainRow)

84
src/frontend/xcecli/common/output/page_print_hosts.go
View file

@ -146,6 +146,7 @@ func (p *HostsPagePrinter) formatRows(hosts *models.HostsResponse, truncatFlag b
truncatFlag = false
}
for _, host := range hosts.Data.Rows {
isUnavail := host.HostStatus == "UNAVAIL"
row := make([]string, 0, len(p.fieldFormats))
for _, format := range p.fieldFormats {
width := p.columnWidths[format.name]
@ -156,17 +157,41 @@ func (p *HostsPagePrinter) formatRows(hosts *models.HostsResponse, truncatFlag b
case models.HEADER_HOST_STATUS:
row = appendOrDefault(row, FormatStringField(host.HostStatus, width, truncatFlag, true), "-")
case models.HEADER_HOST_HOST_TYPE:
row = appendOrDefault(row, FormatStringField(host.HostType, width, truncatFlag, true), "-")
if isUnavail {
row = append(row, FormatStringField("UNKNOWN", width, truncatFlag, true))
} else {
row = appendOrDefault(row, FormatStringField(host.HostType, width, truncatFlag, true), "-")
}
case models.HEADER_HOST_NCPUS:
row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumCPUs), "0")
if isUnavail {
row = append(row, FormatStringField("-", width, truncatFlag, true))
} else {
row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumCPUs), "0")
}
case models.HEADER_HOST_MAX_SLOT:
row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.MaxSlot), "0")
if isUnavail {
row = append(row, FormatStringField("-", width, truncatFlag, true))
} else {
row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.MaxSlot), "0")
}
case models.HEADER_HOST_NUM_RUN:
row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumRun), "0")
if isUnavail {
row = append(row, FormatStringField("-", width, truncatFlag, true))
} else {
row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumRun), "0")
}
case models.HEADER_HOST_NUM_SSUSP:
row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumSSusp), "0")
if isUnavail {
row = append(row, FormatStringField("-", width, truncatFlag, true))
} else {
row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumSSusp), "0")
}
case models.HEADER_HOST_NUM_USUSP:
row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumUSusp), "0")
if isUnavail {
row = append(row, FormatStringField("-", width, truncatFlag, true))
} else {
row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumUSusp), "0")
}
}
}
rows = append(rows, row)
@ -188,6 +213,7 @@ func (p *HostsPagePrinter) printJsonPage(hosts *models.HostsResponse) {
func printHostsVerboseOutput(hosts *models.HostsResponse) {
for _, host := range hosts.Data.Rows {
isUnavail := host.HostStatus == "UNAVAIL"
fmt.Printf("HOST %s\n", host.HostName)
// table 1
headers := models.HostDetailHeaders
@ -199,19 +225,43 @@ func printHostsVerboseOutput(hosts *models.HostsResponse) {
case models.HEADER_HOST_STATUS:
val = host.HostStatus
case models.HEADER_HOST_CPUF:
val = fmt.Sprintf("%.2f", host.CPUFactor)
if isUnavail {
val = "-"
} else {
val = fmt.Sprintf("%.2f", host.CPUFactor)
}
case models.HEADER_HOST_JLU:
val = "-"
case models.HEADER_HOST_MAX:
val = fmt.Sprintf("%d", host.MaxSlot)
if isUnavail {
val = "-"
} else {
val = fmt.Sprintf("%d", host.MaxSlot)
}
case models.HEADER_HOST_NJOBS:
val = fmt.Sprintf("%d", host.NumRun)
if isUnavail {
val = "-"
} else {
val = fmt.Sprintf("%d", host.NumRun)
}
case models.HEADER_HOST_RUN:
val = fmt.Sprintf("%d", host.NumRun)
if isUnavail {
val = "-"
} else {
val = fmt.Sprintf("%d", host.NumRun)
}
case models.HEADER_HOST_SSUSP:
val = fmt.Sprintf("%d", host.NumSSusp)
if isUnavail {
val = "-"
} else {
val = fmt.Sprintf("%d", host.NumSSusp)
}
case models.HEADER_HOST_USUSP:
val = fmt.Sprintf("%d", host.NumUSusp)
if isUnavail {
val = "-"
} else {
val = fmt.Sprintf("%d", host.NumUSusp)
}
case models.HEADER_HOST_RSV:
val = "-"
case models.HEADER_HOST_DISPT_WINDOW:
@ -234,8 +284,14 @@ func printHostsVerboseOutput(hosts *models.HostsResponse) {
// table2
headers2 := append([]string{""}, models.HostLoadSchedulingHeaders...)
totalVals := []string{"Total", "0.0", "0.0", "0.0", "1%", "0.0", "33", "1", "1137", "51G", "3.9G", "6.8G", "8", "0.0"}
reserveVals := []string{"Reserved", "0.0", "0.0", "0.0", "0%", "0.0", "0", "0", "0", "0M", "0M", "0M", "-", "-"}
var totalVals, reserveVals []string
if isUnavail {
totalVals = []string{"Total", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"}
reserveVals = []string{"Reserved", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"}
} else {
totalVals = []string{"Total", "0.0", "0.0", "0.0", "1%", "0.0", "33", "1", "1137", "51G", "3.9G", "6.8G", "8", "0.0"}
reserveVals = []string{"Reserved", "0.0", "0.0", "0.0", "0%", "0.0", "0", "0", "0", "0M", "0M", "0M", "-", "-"}
}
widths2 := make([]int, len(headers2))
for i := range headers2 {

103
src/frontend/xcecli/common/output/page_print_hosts_test.go Normal file
View file

@ -0,0 +1,103 @@
package output
import (
"strings"
"testing"
"xcecli/models"
)
func makeHostsResponse(hosts ...models.XCEHost) *models.HostsResponse {
r := &models.HostsResponse{}
r.Data.Rows = hosts
r.Data.Count = len(hosts)
return r
}
func TestFormatRows_UnavailShowsDashes(t *testing.T) {
host := models.XCEHost{
HostName: "pc13v",
HostStatus: "UNAVAIL",
NumCPUs: 16,
MaxSlot: 32,
NumRun: 5,
NumSSusp: 2,
NumUSusp: 1,
HostType: "*6_64-linux-gnu",
}
resp := makeHostsResponse(host)
p := NewHostsPagePrinter()
p.PrepareForPrinting(resp)
rows := p.formatRows(resp, false)
if len(rows) != 1 {
t.Fatalf("expected 1 row, got %d", len(rows))
}
row := rows[0]
// Build a map from trimmed value by column header index.
headers := p.fieldFormats
vals := make(map[string]string, len(headers))
for i, f := range headers {
vals[f.name] = strings.TrimSpace(row[i])
}
for _, col := range []string{
models.HEADER_HOST_NCPUS,
models.HEADER_HOST_MAX_SLOT,
models.HEADER_HOST_NUM_RUN,
models.HEADER_HOST_NUM_SSUSP,
models.HEADER_HOST_NUM_USUSP,
} {
if vals[col] != "-" {
t.Errorf("UNAVAIL host: %s = %q, want \"-\"", col, vals[col])
}
}
if vals[models.HEADER_HOST_HOST_TYPE] != "UNKNOWN" {
t.Errorf("UNAVAIL host: HOST_TYPE = %q, want \"UNKNOWN\"", vals[models.HEADER_HOST_HOST_TYPE])
}
}
func TestFormatRows_AvailableShowsValues(t *testing.T) {
host := models.XCEHost{
HostName: "pc14v",
HostStatus: "OK",
NumCPUs: 16,
MaxSlot: 32,
NumRun: 5,
NumSSusp: 2,
NumUSusp: 1,
HostType: "linux",
}
resp := makeHostsResponse(host)
p := NewHostsPagePrinter()
p.PrepareForPrinting(resp)
rows := p.formatRows(resp, false)
if len(rows) != 1 {
t.Fatalf("expected 1 row, got %d", len(rows))
}
row := rows[0]
headers := p.fieldFormats
vals := make(map[string]string, len(headers))
for i, f := range headers {
vals[f.name] = strings.TrimSpace(row[i])
}
checks := map[string]string{
models.HEADER_HOST_NCPUS: "16",
models.HEADER_HOST_MAX_SLOT: "32",
models.HEADER_HOST_NUM_RUN: "5",
models.HEADER_HOST_NUM_SSUSP: "2",
models.HEADER_HOST_NUM_USUSP: "1",
models.HEADER_HOST_HOST_TYPE: "linux",
}
for col, want := range checks {
if vals[col] != want {
t.Errorf("OK host: %s = %q, want %q", col, vals[col], want)
}
}
}

34
src/frontend/xcecli/common/output/page_print_jobs.go
View file

@ -98,7 +98,7 @@ func (p *JobsPagePrinter) initColumnWidths(jobs *models.JobsResponse) {
case models.HEADER_JOB_NAME:
width = len(job.JobName)
case models.HEADER_JOB_SUBMIT_TIME:
width = len(job.SubmitTime)
width = len(fmtUnixTime(job.SubmitTime))
case models.HEADER_JOB_CORE_LIMIT:
width = len(fmt.Sprintf("%f", job.CoreLimit))
case models.HEADER_JOB_MEM_LIMIT:
@ -179,7 +179,7 @@ func (p *JobsPagePrinter) formatRows(jobs *models.JobsResponse, truncatFlag bool
case models.HEADER_JOB_EXEC_HOST:
var execHosts models.ExecHosts
if err := json.Unmarshal([]byte(job.ExecHosts.String()), &execHosts); err == nil && len(execHosts.HostSlots) > 0 {
if err := json.Unmarshal([]byte(job.ExecHosts.String()), &execHosts); err == nil && len(execHosts.HostSlots) > 0 && execHosts.HostSlots[0].HostName != "" {
firstHost := fmt.Sprintf("%d*%s", execHosts.HostSlots[0].NumSlots, execHosts.HostSlots[0].HostName)
mainRow = appendOrDefault(mainRow,
FormatStringField(firstHost, width, truncatFlag, false), "-")
@ -201,11 +201,7 @@ func (p *JobsPagePrinter) formatRows(jobs *models.JobsResponse, truncatFlag bool
mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, "-"), "-")
}
case models.HEADER_JOB_SUBMIT_TIME:
if submitTime, err := parseJobTime(job.SubmitTime); err == nil {
mainRow = appendOrDefault(mainRow, FormatStringField(formatJobTime(submitTime), width, truncatFlag, true), "-")
} else {
mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, "-"), "-")
}
mainRow = appendOrDefault(mainRow, FormatStringField(fmtUnixTime(job.SubmitTime), width, truncatFlag, true), "-")
// normal fields
case models.HEADER_JOB_NAME:
@ -223,9 +219,9 @@ func (p *JobsPagePrinter) formatRows(jobs *models.JobsResponse, truncatFlag bool
case models.HEADER_JOB_PROJECT_NAME:
mainRow = appendOrDefault(mainRow, FormatStringField(job.ProjectName, width, truncatFlag, true), "-")
case models.HEADER_JOB_START_TIME:
mainRow = appendOrDefault(mainRow, FormatStringField(job.StartTime, width, truncatFlag, true), "-")
mainRow = appendOrDefault(mainRow, FormatStringField(fmtUnixTime(job.StartTime), width, truncatFlag, true), "-")
case models.HEADER_JOB_END_TIME:
mainRow = appendOrDefault(mainRow, FormatStringField(job.EndTime, width, truncatFlag, true), "-")
mainRow = appendOrDefault(mainRow, FormatStringField(fmtUnixTime(job.EndTime), width, truncatFlag, true), "-")
case models.HEADER_JOB_CWD:
mainRow = appendOrDefault(mainRow, FormatStringField(job.CWD, width, truncatFlag, true), "-")
case models.HEADER_JOB_LAUNCH_TYPE:
@ -303,7 +299,7 @@ func printJobsVerboseOutput(jobs *models.JobsResponse, filter []string) {
// (2) print the submit time info
submitLine := ""
if submitTime, err := parseJobTime(job.SubmitTime); err == nil {
if submitTime, ok := unixToTime(job.SubmitTime); ok {
submitLine = submitLine + fmt.Sprintf("%s %s %02d:%02d:%02d: ",
submitTime.Format("Jan"), submitTime.Format("02"),
submitTime.Hour(), submitTime.Minute(), submitTime.Second())
@ -322,9 +318,20 @@ func printJobsVerboseOutput(jobs *models.JobsResponse, filter []string) {
}
PrintWrappedLine(w, submitLine, 80, 18)
// (3) print the start time info
// (3) print the pending reason for jobs waiting to be dispatched
if job.JobStatus == "PEND" || job.JobStatus == "PSUSP" {
if job.PendingReason != "" {
fmt.Fprintf(w, " PENDING REASONS:\n")
fmt.Fprintf(w, " %s\n", job.PendingReason)
}
}
// (4) print the start time info — skip for jobs that haven't started yet
startLine := ""
if startTime, err := parseJobTime(job.StartTime); err == nil {
if job.JobStatus == "PEND" || job.JobStatus == "PSUSP" {
goto printEndLine
}
if startTime, ok := unixToTime(job.StartTime); ok {
month := startTime.Format("Jan")
day := startTime.Format("02")
exec_hostname := ""
@ -380,10 +387,11 @@ func printJobsVerboseOutput(jobs *models.JobsResponse, filter []string) {
}
PrintWrappedLine(w, startLine, 80, 18)
printEndLine:
// (4) print end line
endLine := ""
if job.JobStatus == "DONE" || job.JobStatus == "EXIT" {
if endTime, err := parseJobTime(job.EndTime); err == nil {
if endTime, ok := unixToTime(job.EndTime); ok {
endLine = endLine + fmt.Sprintf("%s %s %02d:%02d:%02d: ",
endTime.Format("Jan"), endTime.Format("02"),
endTime.Hour(), endTime.Minute(), endTime.Second())

27
src/frontend/xcecli/common/output/time.go
View file

@ -5,17 +5,36 @@ import (
"time"
)
// parseJobTime parses a job time string supporting RFC3339 and space-separated formats.
// parseJobTime parses a time string in RFC3339 or "2006-01-02 15:04:05" format.
func parseJobTime(s string) (time.Time, error) {
if s == "" {
return time.Time{}, fmt.Errorf("empty time string")
}
for _, f := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02 15:04:05", "2006-01-02 15:04:05 MST"} {
if t, err := time.Parse(f, s); err == nil {
for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02 15:04:05"} {
if t, err := time.Parse(layout, s); err == nil {
return t, nil
}
}
return time.Time{}, fmt.Errorf("cannot parse time: %q", s)
return time.Time{}, fmt.Errorf("cannot parse time %q", s)
}
// unixToTime converts a Unix-second timestamp to time.Time.
// Returns (time, true) for ts > 0; (zero, false) for unset/zero values.
func unixToTime(ts int64) (time.Time, bool) {
if ts <= 0 {
return time.Time{}, false
}
return time.Unix(ts, 0), true
}
// fmtUnixTime formats a Unix timestamp as "Jan 02 15:04" for table display.
// Returns "-" for zero/unset values.
func fmtUnixTime(ts int64) string {
t, ok := unixToTime(ts)
if !ok {
return "-"
}
return formatJobTime(t)
}
// formatJobTime formats a time.Time as "Jan 02 15:04" for table display.