cluster info: fill xce id cluster_name/status from live ctrlstatus
`xce id` printed empty name/status: two breaks in the chain.
1. CLI decode: the clusters API returns store-backed rows in the OrderMap
wire format ({"Fields":[...],"Data":{...}}), but ClusterResponse.Rows
([]ClusterInfo) had no custom UnmarshalJSON, so every row landed
all-empty. Same gap array_stats hit; fixed the same way
(UnmarshalOrderMapRow).
2. Server data: the cluster row only comes from OnClusterUpdated (config
push), which never writes Status or ControllerName -- both stayed at
the store zero value (status 0 = DISCONNECT) forever even on a
healthy cluster. The controller's real identity/health snapshot
already exists on the wire (CtrlStatus reply, xce_ctrlstatus_t); the
apiserver just never asked for it.
Fix:
- backend.FetchCtrlStatus(): CGO wrapper of xcecli_ctrlstatus through
callBackend (failover + circuit breaker like every other read),
decoded into ClusterCtrlStatus and freed via free_CtrlStatus_ptr
(bug-1401 leak contract); no-CGO stub added alongside the others.
- GetClustersInfo merges the snapshot into each row before rendering:
cluster_name must match the responder's own (a stale-named row never
claims another cluster's identity), controller_name <- primaryName,
status <- CONNECTED. A backend that doesn't answer keeps the store's
last-known state instead of failing the query.
- CLI ClusterInfo gains the OrderMap-aware UnmarshalJSON.
t1 live: `xce id` now shows name <eda1> / status <CONNECTED> /
controller t1; submit->run->DONE job verified with all daemons healthy.
Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
parent
d028c8c315
commit
effe50a6ba
7 changed files with 231 additions and 0 deletions
|
|
@ -838,3 +838,33 @@ func FetchFile(jobid uint64, filepath string, offset uint64, maxBytes uint32) (*
|
||||||
}
|
}
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FetchCtrlStatus issues the any-node controller-status read through the
|
||||||
|
// shared backend connection (xcecli_ctrlstatus, failover-wrapped like every
|
||||||
|
// other read). Callers merge the result into the cluster-info row for
|
||||||
|
// `xce id`; a backend that never answers returns the callBackend error and
|
||||||
|
// the caller keeps the store's last-known state.
|
||||||
|
func FetchCtrlStatus() (*ClusterCtrlStatus, error) {
|
||||||
|
var status *C.xce_ctrlstatus_t
|
||||||
|
_, err := callBackend("xcecli_ctrlstatus", func() int {
|
||||||
|
return int(C.xcecli_ctrlstatus(gCtx.ctx, &status))
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if status == nil {
|
||||||
|
return nil, fmt.Errorf("xcecli_ctrlstatus returned nil reply")
|
||||||
|
}
|
||||||
|
// free_CtrlStatus_ptr frees the fields INSIDE the struct and the struct
|
||||||
|
// itself, nulling the pointer (bug-1401's leak contract).
|
||||||
|
defer C.free_CtrlStatus_ptr(&status)
|
||||||
|
|
||||||
|
return &ClusterCtrlStatus{
|
||||||
|
ClusterName: goString(status.clusterName),
|
||||||
|
PrimaryName: goString(status.primaryName),
|
||||||
|
ResponderName: goString(status.responderName),
|
||||||
|
ResponderRole: uint8(status.responderRole),
|
||||||
|
PeerCount: uint32(status.peerCount),
|
||||||
|
Fenced: status.fenced != 0,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -24,3 +24,19 @@ type FetchFileResult struct {
|
||||||
FileSize uint64
|
FileSize uint64
|
||||||
BytesRead uint32
|
BytesRead uint32
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ClusterCtrlStatus is the Go view of the controller's identity/health
|
||||||
|
// snapshot (CtrlStatus reply, xce_ctrlstatus_t) that the cluster-info query
|
||||||
|
// merges into the otherwise config-static cluster row. Only the fields the
|
||||||
|
// display needs travel across; term/LSN details stay available to the
|
||||||
|
// harness `ctrlstatus` command.
|
||||||
|
type ClusterCtrlStatus struct {
|
||||||
|
ClusterName string
|
||||||
|
PrimaryName string
|
||||||
|
ResponderName string
|
||||||
|
ResponderRole uint8
|
||||||
|
PeerCount uint32
|
||||||
|
// Fenced mirrors the reply's fenced flag: true means the responder
|
||||||
|
// answers but cannot serve writes (HA fence up).
|
||||||
|
Fenced bool
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -52,6 +52,9 @@ func FetchFile(_ uint64, _ string, _ uint64, _ uint32) (*FetchFileResult, error)
|
||||||
return nil, errNoCGO
|
return nil, errNoCGO
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FetchCtrlStatus stub — real implementation in clib.go (requires CGO).
|
||||||
|
func FetchCtrlStatus() (*ClusterCtrlStatus, error) { return nil, errNoCGO }
|
||||||
|
|
||||||
// ValidateResreqSyntax (bug-1744) needs libxce's grammar parser; without CGO
|
// ValidateResreqSyntax (bug-1744) needs libxce's grammar parser; without CGO
|
||||||
// no job can be submitted anyway (SubmitJob returns errNoCGO), so there is
|
// no job can be submitted anyway (SubmitJob returns errNoCGO), so there is
|
||||||
// nothing to check.
|
// nothing to check.
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
package controllers
|
package controllers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"apiserver/backend"
|
||||||
"apiserver/common"
|
"apiserver/common"
|
||||||
"apiserver/models"
|
"apiserver/models"
|
||||||
"apiserver/services"
|
"apiserver/services"
|
||||||
|
|
@ -176,6 +177,14 @@ func (c ClusterController) GetClustersInfo(ctx *gin.Context) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
logger.Debug("the cluster info query reply:", zap.Any("clqueryReply", clqueryReply), zap.String("pagetoken", pagetoken))
|
logger.Debug("the cluster info query reply:", zap.Any("clqueryReply", clqueryReply), zap.String("pagetoken", pagetoken))
|
||||||
|
|
||||||
|
// The store row is config-push static (OnClusterUpdated never writes
|
||||||
|
// Status/ControllerName), so the live controller identity/health snapshot
|
||||||
|
// is fetched per query and merged in. A backend that doesn't answer
|
||||||
|
// (degraded circuit, no reachable controller) keeps the store's
|
||||||
|
// last-known state instead of failing the whole query.
|
||||||
|
mergeCtrlStatus(clqueryReply)
|
||||||
|
|
||||||
//convert to readable string.
|
//convert to readable string.
|
||||||
queryResult := processClusterData(clqueryReply, pagetoken)
|
queryResult := processClusterData(clqueryReply, pagetoken)
|
||||||
|
|
||||||
|
|
@ -183,10 +192,45 @@ func (c ClusterController) GetClustersInfo(ctx *gin.Context) {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// fetchCtrlStatus is the backend call, a package var so tests can stub it
|
||||||
|
// (mirrors output.go's fetchFile seam).
|
||||||
|
var fetchCtrlStatus = backend.FetchCtrlStatus
|
||||||
|
|
||||||
|
// mergeCtrlStatus overwrites the live fields on each cluster row from the
|
||||||
|
// controller's ctrlstatus reply: cluster_name (the responder knows its own
|
||||||
|
// cluster), controller_name (current primary) and status (CONNECTED when the
|
||||||
|
// backend answered; the reply IS the liveness signal — a row whose store
|
||||||
|
// entry predates the join shows DISCONNECT until this merge). Rows the
|
||||||
|
// snapshot names differently are left untouched, so a stale-named row can
|
||||||
|
// never claim another cluster's identity.
|
||||||
|
func mergeCtrlStatus(rows []utils.OrderMap) {
|
||||||
|
st, err := fetchCtrlStatus()
|
||||||
|
if err != nil {
|
||||||
|
logger.Debug("cluster status live refresh skipped",
|
||||||
|
zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, row := range rows {
|
||||||
|
if name, ok := row.Data["cluster_name"].(string); !ok || name != st.ClusterName {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, exist := row.Data["controller_name"]; exist {
|
||||||
|
row.Data["controller_name"] = st.PrimaryName
|
||||||
|
}
|
||||||
|
if _, exist := row.Data["status"]; exist {
|
||||||
|
row.Data["status"] = models.CLUSTER_CONNECTED
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func processClusterData(rows []utils.OrderMap, page_token string) utils.QueryResult {
|
func processClusterData(rows []utils.OrderMap, page_token string) utils.QueryResult {
|
||||||
for _, row := range rows {
|
for _, row := range rows {
|
||||||
if v, exist := row.Data["status"]; exist {
|
if v, exist := row.Data["status"]; exist {
|
||||||
if v != nil {
|
if v != nil {
|
||||||
|
if strVal, ok := v.(string); ok && strVal != "" {
|
||||||
|
// already a rendered status string (live merge ran)
|
||||||
|
continue
|
||||||
|
}
|
||||||
statusVal, _ := utils.IntOrInt64(v)
|
statusVal, _ := utils.IntOrInt64(v)
|
||||||
status, err := utils.GetKeyFromValue(models.CLUSTERSTAT, statusVal)
|
status, err := utils.GetKeyFromValue(models.CLUSTERSTAT, statusVal)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
||||||
120
src/frontend/apiserver/controllers/clusters_status_test.go
Normal file
120
src/frontend/apiserver/controllers/clusters_status_test.go
Normal file
|
|
@ -0,0 +1,120 @@
|
||||||
|
package controllers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"apiserver/backend"
|
||||||
|
"apiserver/models"
|
||||||
|
"apiserver/store"
|
||||||
|
"apiserver/utils"
|
||||||
|
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The cluster-info row is config-push static (OnClusterUpdated never writes
|
||||||
|
// Status/ControllerName), so the query merges the controller's live
|
||||||
|
// ctrlstatus snapshot in. These tests pin that merge: the live fields land
|
||||||
|
// on the matching row, a differently-named row is untouched, and a backend
|
||||||
|
// that doesn't answer keeps the store's last-known state.
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
utils.Log = zap.NewNop()
|
||||||
|
}
|
||||||
|
|
||||||
|
func mkClusterRow(name string) utils.OrderMap {
|
||||||
|
return utils.OrderMap{
|
||||||
|
Fields: []string{"cluster_name", "status", "controller_name"},
|
||||||
|
Data: map[string]interface{}{
|
||||||
|
"cluster_name": name,
|
||||||
|
"status": 0, // store zero value = DISCONNECT
|
||||||
|
"controller_name": "",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeCtrlStatus_LiveFieldsLand(t *testing.T) {
|
||||||
|
store.InitStores()
|
||||||
|
orig := fetchCtrlStatus
|
||||||
|
defer func() { fetchCtrlStatus = orig }()
|
||||||
|
|
||||||
|
fetchCtrlStatus = func() (*backend.ClusterCtrlStatus, error) {
|
||||||
|
return &backend.ClusterCtrlStatus{
|
||||||
|
ClusterName: "eda1",
|
||||||
|
PrimaryName: "ctrl-a",
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
rows := []utils.OrderMap{mkClusterRow("eda1")}
|
||||||
|
mergeCtrlStatus(rows)
|
||||||
|
|
||||||
|
if got := rows[0].Data["status"]; got != models.CLUSTER_CONNECTED {
|
||||||
|
t.Errorf("status: got %v, want %d (CONNECTED)", got, models.CLUSTER_CONNECTED)
|
||||||
|
}
|
||||||
|
if got := rows[0].Data["controller_name"]; got != "ctrl-a" {
|
||||||
|
t.Errorf("controller_name: got %v, want ctrl-a", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeCtrlStatus_DifferentNameRowUntouched(t *testing.T) {
|
||||||
|
orig := fetchCtrlStatus
|
||||||
|
defer func() { fetchCtrlStatus = orig }()
|
||||||
|
|
||||||
|
fetchCtrlStatus = func() (*backend.ClusterCtrlStatus, error) {
|
||||||
|
return &backend.ClusterCtrlStatus{
|
||||||
|
ClusterName: "other",
|
||||||
|
PrimaryName: "ctrl-b",
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
rows := []utils.OrderMap{mkClusterRow("eda1")}
|
||||||
|
mergeCtrlStatus(rows)
|
||||||
|
|
||||||
|
if got := rows[0].Data["status"]; got != 0 {
|
||||||
|
t.Errorf("status: got %v, want 0 (untouched — snapshot names another cluster)", got)
|
||||||
|
}
|
||||||
|
if got := rows[0].Data["controller_name"]; got != "" {
|
||||||
|
t.Errorf("controller_name: got %v, want \"\" (untouched)", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeCtrlStatus_BackendDownKeepsStoreState(t *testing.T) {
|
||||||
|
orig := fetchCtrlStatus
|
||||||
|
defer func() { fetchCtrlStatus = orig }()
|
||||||
|
|
||||||
|
fetchCtrlStatus = func() (*backend.ClusterCtrlStatus, error) {
|
||||||
|
return nil, errBackendDownTest{}
|
||||||
|
}
|
||||||
|
|
||||||
|
rows := []utils.OrderMap{mkClusterRow("eda1")}
|
||||||
|
mergeCtrlStatus(rows)
|
||||||
|
|
||||||
|
if got := rows[0].Data["status"]; got != 0 {
|
||||||
|
t.Errorf("status: got %v, want 0 (store state kept when backend down)", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type errBackendDownTest struct{}
|
||||||
|
|
||||||
|
func (errBackendDownTest) Error() string { return "backend unavailable" }
|
||||||
|
|
||||||
|
func TestProcessClusterData_RendersMergedStatus(t *testing.T) {
|
||||||
|
orig := fetchCtrlStatus
|
||||||
|
defer func() { fetchCtrlStatus = orig }()
|
||||||
|
|
||||||
|
fetchCtrlStatus = func() (*backend.ClusterCtrlStatus, error) {
|
||||||
|
return &backend.ClusterCtrlStatus{ClusterName: "eda1", PrimaryName: "ctrl-a"}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
rows := []utils.OrderMap{mkClusterRow("eda1")}
|
||||||
|
mergeCtrlStatus(rows)
|
||||||
|
result := processClusterData(rows, "")
|
||||||
|
|
||||||
|
rendered, ok := result.Rows[0].Data["status"].(string)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("status should render as string after merge, got %T %v",
|
||||||
|
result.Rows[0].Data["status"], result.Rows[0].Data["status"])
|
||||||
|
}
|
||||||
|
if rendered != "CONNECTED" {
|
||||||
|
t.Errorf("status: got %q, want CONNECTED", rendered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -4,6 +4,11 @@ const (
|
||||||
CLUSTER_RECONFIG = 1
|
CLUSTER_RECONFIG = 1
|
||||||
CLUSTER_RESTART = 2
|
CLUSTER_RESTART = 2
|
||||||
CLUSTER_SHUTDOWN = 3
|
CLUSTER_SHUTDOWN = 3
|
||||||
|
|
||||||
|
// CLUSTER_CONNECTED is the store-level status value the cluster-info
|
||||||
|
// query stamps when the controller's ctrlstatus reply came back live.
|
||||||
|
// Its string rendering comes from CLUSTERSTAT below.
|
||||||
|
CLUSTER_CONNECTED = 0x01
|
||||||
)
|
)
|
||||||
|
|
||||||
var CLUSTERSTAT = map[string]int{
|
var CLUSTERSTAT = map[string]int{
|
||||||
|
|
|
||||||
|
|
@ -20,3 +20,16 @@ type ClusterInfo struct {
|
||||||
HostModels interface{} `json:"host_models"`
|
HostModels interface{} `json:"host_models"`
|
||||||
ClusterAdmins interface{} `json:"cluster_admins"`
|
ClusterAdmins interface{} `json:"cluster_admins"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UnmarshalJSON decodes both the OrderMap wire format the server sends for
|
||||||
|
// store-backed rows ({"Fields":[...],"Data":{...}}) and a bare object, so
|
||||||
|
// `xce id` decodes past the wrapper instead of landing all-empty.
|
||||||
|
func (c *ClusterInfo) UnmarshalJSON(data []byte) error {
|
||||||
|
type alias ClusterInfo
|
||||||
|
var a alias
|
||||||
|
if err := UnmarshalOrderMapRow(data, &a); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
*c = ClusterInfo(a)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue