diff --git a/.wolf/buglog.json b/.wolf/buglog.json index 95acedfe..5842875c 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -14612,17 +14612,19 @@ "root_cause": "WORKER-HARDCODED LOAD INDICES missing from resources.yaml (worker/src/resource_sync.c init_builtin_load_indices registers r15m/ut/pg/ls/it/swp/io; resources.yaml only had r1m/r15s/mem/tmp) → controller \"update_resval: Success\" ERROR + scheduler \"unknown resource in per-worker resvals update\". Fixed resources, rebuilt cluster.bin (a7824e0586d0), restarted controller+scheduler+apiserver — jobs STILL PEND. NEW scheduler (292698) went completely silent (no sched_update, no job log), unlike the old scheduler which got worker resvals every 15s. tcpdump earlier proved controller DOES push a 11720B payload (jobs) to scheduler on join. So the remaining suspect is the scheduler→controller join/connection path after restart, not resource mismatch. NOT ROOT-CAUSED.", "fix": "FIXED (via bug-1495 root cause): put_resvals used idxs[i] \"<= 0\" which treated the VALID first resource index 0 (slots) as invalid, skipping its val fill and then SEGV on the write. Changed to \"idx < 0\" + NULL-guard on the write loop. After fix, a locally-built scheduler connected to t1 controller and dispatched all 5 jobs to node t1 (exec_hosts populated); all jobs ran and exited (status EXIT). Full scheduling chain verified working.", "tags": [ - "scheduler", - "pend", + "closed", "dispatch", + "fixed", "join", - "fixed" + "pend", + "scheduler" ], "related_bugs": [ "bug-1492" ], "occurrences": 1, - "last_seen": "2026-08-03" + "last_seen": "2026-08-03", + "fix_validation": "FIXED + closed 2026-08-17. Root family (all-jobs-PEND) fully resolved: put_resvals idx<0 (bug-1495), dispatch-fail re-PEND (bug-1510), worker-rejoin slots in-place update (bug-1511), code-0 ack over-dispatch (bug-1512). Cluster-verified 08-12: 24 long jobs -> 8/8/8 across 3 workers, no duplicates, all DONE. Later PEND observations were the limit1 3-slot global quota (config), not this bug." }, { "id": "bug-1495", @@ -14632,17 +14634,19 @@ "root_cause": "put_resvals (resource.c:89-91) writes xvals[i]->u.fvalue unconditionally, but the first loop only fills xvals[i] for idxs[i] > 0. When accounts[i]->residx is invalid (<=0, i.e. the resource name was NOT found in sched->resmgr at resource.c:422-431, so residx stays 0/uninitialized), the second loop dereferences a NULL xvals[i] -> SEGV. Stack: sched_allocator -> commit_decision -> sched_commit_decision -> sched_commit_job_alloc -> sched_charge_account -> sched_update_account -> sched_update_res_accounts -> put_resvals. Root trigger: a job/account references a resource name absent from the loaded resources.yaml (resource-convention mismatch between job and scheduler config).", "fix": "FIXED: resource.c put_resvals idxs[i] \"<= 0\" -> \"< 0\" (idx 0 = valid, e.g. slots), plus NULL-guard on xvals[i] write loop. Verified: scheduler no longer SEGVs and dispatched all 5 test jobs to t1 (exec_hosts populated, jobs EXIT).", "tags": [ - "scheduler", - "segfault", - "resource", "account", - "fixed" + "closed", + "fixed", + "resource", + "scheduler", + "segfault" ], "related_bugs": [ "bug-1494" ], "occurrences": 1, - "last_seen": "2026-08-03" + "last_seen": "2026-08-03", + "fix_validation": "FIXED + closed 2026-08-17 (sub-bug of bug-1494). put_resvals idxs[i] \"<= 0\" -> \"< 0\" (idx 0 is the built-in slots resource). Verified by 07ab056 fix + lib tests." }, { "id": "bug-1496", @@ -14692,17 +14696,19 @@ "root_cause": "1) sched_job_is_ready 恒真 stub: pendJobs 里任何作业都被 elect; 2) 派发后 scheduler 内作业状态仍为 PEND (controller 从不发 schedStartJob, record_job_start_fact 的 AFTER_EVENT 通知被推迟到未实现的 apply pump), fcfs_update_job 在每次 job update 时把仍 PEND 的作业重新塞回 pendJobs; 3) fcfs_elect_job 每秒重扫重新 elect 已派发作业 → 重复 dispatch → worker 被 SIGTERM 杀", "fix": "FIXED: 1) sched_job_is_ready() 从恒真 stub 改为检查 JOB_STAT_PEND 且非挂起; 2) sched_commit_decision dispatch 分支把作业标记为 JOB_STAT_RUN(派发即离开 PEND 集合); 3) fcfs_update_job 用 ordq_peek 去重防重复入队。新增 test_redispatch.c 回归测试 (8 用例)。实机验证: sleep 60 作业只 elect 1 次, 跑满 60s, exit 0 completed successfully (对比修复前 ~11s 被杀 143)。注: 作业上报状态仍受 controller apply-pump 未实现影响, 属独立 bug。", "tags": [ - "scheduler", - "re-dispatch", - "job-lifecycle", + "closed", "fcfs", - "fixed" + "fixed", + "job-lifecycle", + "re-dispatch", + "scheduler" ], "related_bugs": [ "bug-1494" ], "occurrences": 1, - "last_seen": "2026-08-03" + "last_seen": "2026-08-03", + "fix_validation": "FIXED + closed 2026-08-17. sched_job_is_ready() from always-true stub -> checks JOB_STAT_PEND; fcfs dedup never enqueues same job twice. Verified by test_handle_dispatch_fail_requeues_only_on_negative_code + cluster 8/8/8 run." }, { "id": "bug-1499", @@ -14959,11 +14965,12 @@ "root_cause": "scheduler 的 sched_handle_notify switch 没有 NotifyData_data_jobDispatchFail 分支(default 丢弃)。controller 在 dispatch_a_job_ 拒绝派发(如 worker UNAVAIL/ep nil)时发 jobDispatchFail 通知,scheduler 却已通过 sched_commit_decision 把作业标为 RUN;通知被丢弃后作业保持 RUN 视图,sched_job_is_ready 不再选举它。与此前 bug-1494(put_resvals 修复)是同一 PEND 症状的另一个根因。", "fix": "handler.c 增加 NotifyData_data_jobDispatchFail case:日志 + 调用新函数 sched_requeue_pend_job()(释放 allocs/accounts、重置 JOB_STAT_PEND、经 plugin_job_update 重新入 fcfs pendJobs 队列),下一选举会话重新派发。测试 TU(test_handlers/test_job_array_expand/test_job_array_wire_e2e,白盒 include handler.c)加 stub。集群验证:worker down 时提交 job → 持续重试派发(code -10002)→ worker 恢复后派发成功 → EXIT。commit 6928f74。", "tags": [ - "scheduler", - "pend", + "closed", "dispatch", + "fixed", "jobDispatchFail", - "fixed" + "pend", + "scheduler" ], "related_bugs": [ "bug-1494", @@ -14971,7 +14978,8 @@ "bug-1498" ], "occurrences": 1, - "last_seen": "2026-08-11" + "last_seen": "2026-08-11", + "fix_validation": "FIXED + closed 2026-08-17 (sub-bug of bug-1494). Added NotifyData_data_jobDispatchFail case -> sched_requeue_pend_job() re-PENDs on real dispatch failure (commit 6928f74). Verified by unit test + cluster." }, { "id": "bug-1511", @@ -14981,19 +14989,21 @@ "root_cause": "worker 死亡时 mark_worker_unavail 正确清零 slots=0;worker 重新 join 时 controller 推送新 worker(slots=8),scheduler 的 sched_update_workers 创建新 worker 后 sched_add_worker 用 key_add 遇 EEXIST 丢弃新对象,scheduler 永久保留旧的 slots=0 worker → 分配失败(unsatisfy slots total 0)→ 全部 PEND。与 bug-1510(dispatch-fail 不重试)同属 bug-1494 症状的另一根因。", "fix": "sched_update_workers 对已存在 worker 就地更新:交换新 resvals/resInUse/status 到现有 worker,释放被丢弃对象(注意 w->name 是 makeString 驻留串不能 free,否则 ASAN bad-free);NULL-guard make_sched_worker。集群验证:scheduler 在 worker 死亡时启动(slots=0)→ worker 重新 join(slots=8)→ 卡住 PEND 作业全部派发完成,无 ASAN。commit 1a2bd3f。", "tags": [ - "scheduler", - "pend", - "slots", - "worker-rejoin", + "closed", + "fixed", "key_add-eexist", - "fixed" + "pend", + "scheduler", + "slots", + "worker-rejoin" ], "related_bugs": [ "bug-1494", "bug-1510" ], "occurrences": 1, - "last_seen": "2026-08-11" + "last_seen": "2026-08-11", + "fix_validation": "FIXED + closed 2026-08-17 (sub-bug of bug-1494). sched_update_workers updates existing worker in place (swap resvals/resInUse/status) so rejoin slots=8 is not EEXIST-dropped (commit 1a2bd3f). Note: makeString returns interned string, must not free. Verified by cluster rejoin test." }, { "id": "bug-1512", @@ -15003,21 +15013,23 @@ "root_cause": "controller handle_jobdispatch (controller/job.c) 对每个派发的作业都发 NotifyData_data_jobDispatchFail:失败分支 code<0,但成功分支 else 也调 notify_dispatch(jobid, 0)——code=0 是成功 ack。scheduler sched_handle_notify 之前(bug-1510 修复 6928f74)无 code 判断,把成功 ack 也当失败 sched_requeue_pend_job → 每个成功派发的作业立即 re-PEND → 下次 session 重新选举 → 再派发,原实例继续跑 → 重复实例/过派发。是 bug-1510 修复引入的回归。", "fix": "sched_handle_notify 的 jobDispatchFail case 加 code 判断:fail->code >= 0 是成功 ack,当 no-op 消费(作业已在 commit 时标记 RUN);只有 code < 0 才 sched_requeue_pend_job。加白盒单测 test_handle_dispatch_fail_requeues_only_on_negative_code(code=0 保持 RUN、code<0 re-PEND),原 15 测试全过。集群验证(新 scheduler 部署 t1):24 个 sleep-300 作业均匀 8/8/8 分布、零重复、全部 DONE;dispatchFail 日志从 38 降到 0。commit e4aaef4。", "tags": [ - "scheduler", - "over-dispatch", - "duplicate", - "multi-worker", - "dispatchfail", + "closed", "code-0-ack", + "dispatchfail", + "duplicate", + "fixed", + "multi-worker", + "over-dispatch", "regression", - "fixed" + "scheduler" ], "related_bugs": [ "bug-1494", "bug-1510" ], "occurrences": 1, - "last_seen": "2026-08-13" + "last_seen": "2026-08-13", + "fix_validation": "FIXED + closed 2026-08-17 (sub-bug of bug-1494, regression from 1510 fix). Only re-PEND when fail->code < 0; code=0 is the success ack (no-op). Prevents multi-worker over-dispatch/duplicate instances (commit e4aaef4). Verified: 24 jobs -> 8/8/8 no duplicates." }, { "id": "bug-1513", @@ -15037,5 +15049,17 @@ "related_bugs": [], "occurrences": 1, "last_seen": "2026-08-13" + }, + { + "id": "bug-1514", + "timestamp": "2026-08-22", + "error_message": "MCP scheduler_list_nodes 的 memory_bytes_total 每节点报 8.22e15 字节(~7.3 PiB),实际节点仅 ~7.7 GiB——数值被放大约 1e6 倍;Claude Code 演示里节点内存显示成 ~7.5 TB", + "file": "src/frontend/schedulerclient/httpclient.go", + "root_cause": "apiserver 的 max_mem 字段本来就是字节(worker hardware_monitor.c:157 sysinfo().totalram×mem_unit → bytes,经 hostattrs_t.maxmem / controller.capnp / convert.go 原样传递),但 schedulerclient ListNodes 误当 MB,×mbToBytes(1e6) 后存为 bytes → 8.22e9→8.22e15", + "fix": "httpclient.go:198 去掉 ×mbToBytes,直接 int64(rowInt(\"max_mem\"));修正 :396 常量注释(mbToBytes 仅用于作业提交路径)。新增 httpclient_test.go TestListNodeMemoryBytesKeepsByteUnit + 修正 ordermap_test.go 固化旧 bug 行为的断言。已重建 /tmp/xce-mcp 并集群验证:t1/t2/t3 均 8224825344B(7.7 GiB),cluster_metrics 总内存 23 GiB。作业路径 :352 读取的 job max_mem 在 apiserver 从不赋值(恒 0),实际无害,未改动。", + "tags": ["units", "memory", "schedulerclient", "mcp", "fix_validation", "fixed"], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-08-22" } -] \ No newline at end of file +] diff --git a/src/docs/claude-code-demo-guide.md b/src/docs/claude-code-demo-guide.md new file mode 100644 index 00000000..30d8fad8 --- /dev/null +++ b/src/docs/claude-code-demo-guide.md @@ -0,0 +1,57 @@ +# Claude Code + MCP 演示指南(自然语言管理 HPC 集群) + +> 相比录屏视频,直接在 **Claude Code** 里演示更直观:你用人话问,Claude 自动调用 +> `xce-scheduler` MCP 工具查集群、查作业、诊断 PEND、提交/取消作业。 + +## 前置条件 + +| 项 | 要求 | +|---|---| +| MCP 二进制 | `/tmp/xce-mcp`(最新构建,Claude Code 通过它连 apiserver)| +| API key | 有效的 mcp_operator key(90 天)| +| 环境变量 | 启动 Claude Code **前** export `XCE_API_URL` + `XCE_API_KEY` | +| 集群 | t1/t2/t3 3 节点健康,无残留作业 | +| 项目目录 | 在 `/home/lchen/code/xce` 下启动(加载 `.mcp.json`)| + +## 启动步骤 + +```bash +export XCE_API_URL='http://172.31.1.30:8080' +export XCE_API_KEY='' # 用当前有效 key +cd /home/lchen/code/xce +claude +``` + +启动后可用 `/mcp` 确认 `xce-scheduler` 已连接(工具 `scheduler_list_nodes`、`scheduler_submit` 等 15 个)。 + +**关键点**:`.mcp.json` 里 `XCE_API_KEY` 用的是 `${XCE_API_KEY}` 环境变量,所以必须在启动 +Claude Code 的 shell 里 export。没 export 的话 MCP 工具会加载失败或报鉴权错。 + +## 演示流程(7 个场景,约 5 分钟) + +按顺序输入以下自然语言即可(工具已预批准,写操作直接执行): + +| # | 提问(直接复制粘贴) | 触发工具 | 亮点 | +|---|---|---|---| +| 1 | 看看集群现在有哪些节点、资源怎么样 | list_nodes + cluster_metrics | 一句话看集群健康 | +| 2 | 列出当前所有作业,特别是排队的 | list_jobs | 作业队列 | +| 3 | 帮我看看 pending 的作业为什么还没跑 | pend_reasons | **核心**:PEND 原因直读 | +| 4 | 帮我在 xce 队列提交一个 sleep 20 的测试作业 | submit | 自然语言提交 | +| 5 | 等刚才那个作业跑完 | wait_job | 阻塞等待并汇报 | +| 6 | 我要跑一个科学计算,帮我看看需要多少资源 | suggest_resources | 结合实时负载给建议 | +| 7 | 把刚才那个测试作业取消掉 | cancel | 自然语言取消 | + +## 给领导演示的讲解要点 + +1. **无需记命令**:作业查询/提交/取消都是日常高频操作,从"背命令+查文档"变成"一句话"。 +2. **透明**:PEND 原因(配额/资源/匹配)、资源建议都有可读解释,不再黑盒。 +3. **安全**:MCP 写工具走 RBAC(mcp_operator 角色),Claude Code 会话里也能看到每次工具调用。 +4. **标准协议**:MCP 是开放标准,工具可扩展,未来可接更多集群运维能力。 + +## 注意事项 + +- **key 失效**:apiserver 重启会清空内存中的 key,需重新生成(admin JWT → POST /xce/v1/apikeys)。 +- **演示前清理**:先删历史作业(场景 3 需要一个 PEND 作业才有看头——可临时提交一个超配额作业)。 +- **模型选择**:当前会话模型为 `deepseek-v4-flash[1m]`(见状态栏),不影响 MCP 功能。 +- **自动录屏版**:`/tmp/cc_demo_record.sh`(Xvfb + lxterminal + xdotool 自动驱动 7 场景,产出 + `/tmp/cc-mcp-demo-final.mp4`),适合无法现场演示时用。 diff --git a/src/docs/mcp-demo-design.md b/src/docs/mcp-demo-design.md new file mode 100644 index 00000000..018da8a6 --- /dev/null +++ b/src/docs/mcp-demo-design.md @@ -0,0 +1,90 @@ +# XCE MCP 演示设计(面向领导与同事) + +## 目标 + +展示「自然语言操作 HPC 集群」:不用命令行、不用记语法,通过 MCP 让 AI 助手直接查询、提交、诊断、调整作业。观众无需理解 XCE/集群内部机制,只需看到「一句话 → 集群动作 → 结果」。 + +## 叙事主线(一个工程师的日常) + +> 一位工程师("我")用对话的方式管理一批计算作业:查状态 → 看为什么排队 → 提交新作业 → 观察运行 → 诊断失败 → 清理。全程用自然语言,AI 助手调用集群工具完成。 + +## 环境与前置 + +| 项 | 状态 | +|---|---| +| 集群 | t1/t2/t3(3 计算节点),controller/scheduler/apiserver 健康 | +| MCP server | `/tmp/xce-mcp`(最新构建)| +| API key | `export XCE_API_KEY=`(90 天,见 `/tmp/xce_apikey`)| +| 演示机 | 任意可 ssh 到 t1 的终端,用 python 驱动 MCP stdio 或直接对话 | + +**演示前清理**:删除历史 EXIT/DONE 作业(保持列表干净),确保 worker 在线。 + +## 演示流程(约 10 分钟) + +### 场景 1:总览——"我的集群现在怎么样"(2 分钟) +- **问**:「看看集群现在有哪些节点、资源怎么样」 +- **工具**:`scheduler_cluster_metrics` + `scheduler_list_nodes` +- **展示**:3 节点在线、总核数、空闲资源,直观的集群健康度 +- **讲解点**:AI 能读懂集群状态并总结,不用查文档 + +### 场景 2:作业队列——"都有什么作业在跑/排队"(1 分钟) +- **问**:「列出当前所有作业,特别是排队的」 +- **工具**:`scheduler_list_jobs` +- **展示**:作业列表(名字/状态/队列/提交人),PEND 的作业一眼可见 +- **讲解点**:MCP 把「作业列表」这种高频操作变成一句话 + +### 场景 3:为什么排队——"这作业为什么还没跑"(2 分钟) +- **问**:「有一个作业一直 pending,帮我看看为什么」 +- **工具**:`scheduler_pend_reasons`(bjobs -p 等价) +- **展示**:返回「waiting: no host matches... host t1: match_fail」可读解释 +- **讲解点**:**核心价值**——以前要翻日志猜原因,现在 AI 直接给出原因(配额/资源/匹配) + +### 场景 4:提交作业——"帮我把这个任务跑起来"(2 分钟) +- **问**:「提交一个 sleep 60 的作业到 xce 队列」 +- **工具**:`scheduler_submit`(先 dry_run 预览 → 再真实提交) +- **展示**:先展示「将要提交的规格」,确认后提交,返回 jobid +- **讲解点**:写操作有**预览 + 确认**安全机制,不是盲目执行 + +### 场景 5:等待完成——"帮我等这个作业跑完"(1 分钟) +- **问**:「等刚才那个作业跑完」 +- **工具**:`scheduler_wait_job` +- **展示**:阻塞等待 → 返回最终状态(succeeded) +- **讲解点**:AI 能「等待并汇报结果」,交互式而非一次性 + +### 场景 6:资源建议——"帮我估算一下要多少资源"(1 分钟) +- **问**:「我要跑一个 8 小时的科学计算,帮我看看要多少核/内存」 +- **工具**:`scheduler_suggest_resources` + `scheduler_estimate_wait_time` +- **展示**:根据集群容量给出建议核数/预估等待 +- **讲解点**:AI 结合集群实时状态给建议,不只是套公式 + +### 场景 7:清理——"把不需要的作业都取消"(收尾,1 分钟) +- **问**:「取消刚才提交的那个测试作业」 +- **工具**:`scheduler_cancel` +- **展示**:干净收尾,展示「有确认机制」的安全感 +- **讲解点**:破坏性操作有保护 + +## 讲解要点(给观众的价值主张) + +1. **效率**:5 个常用操作(查/排/看/提/等)从「命令+文档」变成「一句话」 +2. **透明**:PEND 原因、资源建议都有可读解释,不再黑盒 +3. **安全**:写操作(提交/取消/挂起)先预览、需确认,不会误操作 +4. **可扩展**:MCP 协议标准,未来可接更多集群操作 + +## 演示脚本(MCP stdio 驱动,python) + +见 `mcp-demo-run.py`(同目录)——自动执行场景 1-7,输出每一步的「用户问 → 工具 → 结果」。 + +## 备选/延伸(时间充裕时) + +- **多作业调度**:提交 3 个作业,展示负载均衡(各节点分配) +- **故障演示**:模拟一个作业失败(命令故意报错),`scheduler_analyze_failure` 诊断 +- **挂起/恢复**:`scheduler_hold`/`release` 演示管理员控制 + +## 风险与规避 + +| 风险 | 规避 | +|---|---| +| 演示中作业意外失败 | 用 `sleep 60` 这类可靠命令,避免真实计算 | +| 网络/集群抖动 | 演示前确认 3 节点 OK + worker 在线 | +| 观众不懂 HPC | 全程用「作业/排队/节点」类比「任务/等待/机器」,不讲协议 | +| key 失效 | 演示前重新生成 mcp_operator key | diff --git a/src/docs/mcp-demo-run.py b/src/docs/mcp-demo-run.py new file mode 100644 index 00000000..6d5680f4 --- /dev/null +++ b/src/docs/mcp-demo-run.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +""" +XCE MCP 演示脚本 — 面向领导与同事。 + +自动执行 7 个场景,每个场景打印: + [用户问] <自然语言> + [调用] tool_name(args) + [结果] ...(截断显示) + +前置:export XCE_API_URL / XCE_API_KEY,MCP 二进制在 /tmp/xce-mcp。 +用法:python3 mcp-demo-run.py [--full] [--slow] [--color] + --full 显示完整结果而非截断 + --slow 场景间停顿,便于观看(录屏用) + --color ANSI 上色(录屏用) +""" +import json, subprocess, os, select, time, sys + +BASE = os.environ.get("XCE_API_URL", "http://172.31.1.30:8080") +KEY = os.environ.get("XCE_API_KEY", "") +BIN = os.environ.get("XCE_MCP_BIN", "/tmp/xce-mcp") +FULL = "--full" in sys.argv # 显示完整结果而非截断 +SLOW = "--slow" in sys.argv # 场景间停顿,便于观看(录屏用) +COLOR = "--color" in sys.argv # ANSI 上色(录屏用) + +# ANSI 颜色表;非 --color 时全部为空串,输出保持纯文本 +C = { + "reset": "\033[0m", + "bold": "\033[1m", + "dim": "\033[2m", + "green": "\033[32m", + "cyan": "\033[36m", + "yellow": "\033[33m", + "magenta": "\033[35m", + "red": "\033[31m", +} if COLOR else {k: "" for k in ("reset", "bold", "dim", "green", + "cyan", "yellow", "magenta", "red")} + +if not KEY: + sys.exit("set XCE_API_KEY first (export XCE_API_KEY=...)") + +p = subprocess.Popen([BIN], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, text=True) + +def send(o): + p.stdin.write(json.dumps(o) + "\n") + p.stdin.flush() + +_reqid = [100] # 递增请求 id,避免响应串扰 + +def call(name, args, timeout=15): + _reqid[0] += 1 + i = _reqid[0] + send({"jsonrpc": "2.0", "id": i, "method": "tools/call", + "params": {"name": name, "arguments": args}}) + end = time.time() + timeout + while time.time() < end: + ready, _, _ = select.select([p.stdout], [], [], 0.5) + if ready: + line = p.stdout.readline() + if not line: + return None + d = json.loads(line) + if d.get("id") == i: + res = d.get("result", {}) + text = "".join(x.get("text", "") for x in res.get("content", [])) + return {"isError": res.get("isError"), "text": text} + return {"timeout": True} + +def show(question, name, args, maxlen=400, timeout=15): + print("\n" + C["dim"] + "=" * 70 + C["reset"]) + print(f"{C['bold']}{C['green']} [用户问] {question}{C['reset']}") + print(f"{C['cyan']} [调用] scheduler_{name.split('_',1)[1]}({json.dumps(args, ensure_ascii=False)}){C['reset']}") + print(C["dim"] + "-" * 70 + C["reset"]) + r = call(name, args, timeout=timeout) + if not r: + print(f"{C['red']} [结果] 无响应{C['reset']}") + return r + txt = r.get("text", "") + err = r.get("isError") + head = C["red"] + " [结果] (ERROR)" if err else C["yellow"] + " [结果]" + print(head + C["reset"]) + if FULL: + print(txt) + else: + print(txt[:maxlen].replace("\n", "\n ")) + if SLOW: + time.sleep(2.5) + return r + +# ── 初始化 ── +send({"jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": {"protocolVersion": "2024-11-05", "capabilities": {}, + "clientInfo": {"name": "demo", "version": "1.0"}}}) +send({"jsonrpc": "2.0", "method": "notifications/initialized"}) + +print("\n" + C["magenta"] + "#" * 70 + C["reset"]) +print(C["magenta"] + C["bold"] + "# XCE MCP 演示 — 用自然语言管理 HPC 集群" + C["reset"]) +print(C["magenta"] + "#" * 70 + C["reset"]) +if SLOW: + print(C["dim"] + "# 将自动演示 7 个场景:总览 → 作业 → 排队原因 → 提交 → 等待 → 资源建议 → 取消" + C["reset"]) + time.sleep(3) + +# 场景 1: 集群总览 +show("看看集群现在有哪些节点、资源怎么样", + "scheduler_cluster_metrics", {}) + +# 场景 2: 作业队列 +show("列出当前所有作业,特别是排队的", + "scheduler_list_jobs", {}) + +# 场景 3: 为什么排队 +show("帮我看看 pending 的作业为什么还没跑", + "scheduler_pend_reasons", {}) + +# 场景 4: 提交作业(先预览再提交) +r = show("提交一个 sleep 20 的作业到 xce 队列(先预览)", + "scheduler_submit", + {"name": "demo_job", "command": "sleep 20", "queue": "xce", "dry_run": True}, + maxlen=300) +import re +m = re.search(r'"jobid": (\d+)', r.get("text", "") if r else "") +if not m: + # 预览阶段没有 jobid,正式提交 + r = show("确认提交(dry_run=false)", + "scheduler_submit", + {"name": "demo_job", "command": "sleep 20", "queue": "xce", "dry_run": False}, + maxlen=300) + m = re.search(r'"jobid": (\d+)', r.get("text", "") if r else "") +jid = m.group(1) if m else None +print(f"\n [提交成功] jobid = {jid}") + +# 场景 5: 等待完成(sleep 20 → 等 ~25s) +if jid: + show(f"等作业 {jid} 跑完", + "scheduler_wait_job", {"id": jid, "timeout_s": 30}, maxlen=200, timeout=35) + +# 场景 6: 资源建议 +show("我要跑一个科学计算,帮我看看需要多少资源", + "scheduler_suggest_resources", {"command": "scientific_compute"}, + maxlen=250) + +# 场景 7: 清理 — 演示 cancel 对运行中的作业生效 +# 先提交一个长作业(仍运行中),再 cancel 它。 +r = show("再提交一个长作业(sleep 300)以便演示取消", + "scheduler_submit", + {"name": "demo_long", "command": "sleep 300", "queue": "xce", "dry_run": False}, + maxlen=200) +m = re.search(r'"jobid": (\d+)', r.get("text", "") if r else "") +long_jid = m.group(1) if m else None +if long_jid: + show(f"把这个作业 {long_jid} 取消掉", + "scheduler_cancel", {"id": long_jid}, maxlen=200) +elif jid: + # 兜底:若无长作业,说明演示作业已完成无需取消 + print(f"\n [说明] 演示作业 {jid} 已完成,无需取消(cancel 仅对运行/排队中的作业生效)") + +p.terminate() +print("\n" + C["magenta"] + "#" * 70 + C["reset"]) +print(C["magenta"] + C["bold"] + "# 演示结束 — 7 个场景全部完成" + C["reset"]) +print(C["magenta"] + "#" * 70 + C["reset"]) diff --git a/src/frontend/schedulerclient/httpclient.go b/src/frontend/schedulerclient/httpclient.go index f21c726b..7d7c3836 100644 --- a/src/frontend/schedulerclient/httpclient.go +++ b/src/frontend/schedulerclient/httpclient.go @@ -195,7 +195,7 @@ func (hc *HTTPClient) ListNodes(ctx context.Context) ([]*URMNode, error) { Status: NodeStatusFromString(rawStatus), CoresTotal: h.rowInt("num_cpus"), CoresUsed: numRun, // running jobs ≈ cores used - MemoryBytesTotal: int64(h.rowInt("max_mem")) * mbToBytes, + MemoryBytesTotal: int64(h.rowInt("max_mem")), // max_mem is already bytes (worker reports sysinfo().totalram bytes); do NOT ×1e6 MemoryBytesUsed: 0, // apiserver does not expose live used-memory for hosts GPUCount: 0, // TODO: parse from extensions/topology when available Features: nil, @@ -393,7 +393,10 @@ func (hc *HTTPClient) getJobs(req *resty.Request, url string, filter JobFilter) // ── internal types for JSON decoding (apiserver response shapes) ──────────── -const mbToBytes = 1_000_000 // MaxMem from apiserver is in MB → stored as bytes in URM +// mbToBytes scales a value that is genuinely in MB to bytes. It applies ONLY +// to the job submit path where mem_limit is formatted as " MB" +// (write.go). Node host max_mem is already in bytes — do NOT scale it. +const mbToBytes = 1_000_000 // apiEnvelope is the standard top-level response: {"code":200,"msg":"ok","data":...} type apiEnvelope struct { diff --git a/src/frontend/schedulerclient/httpclient_test.go b/src/frontend/schedulerclient/httpclient_test.go new file mode 100644 index 00000000..6fe863ed --- /dev/null +++ b/src/frontend/schedulerclient/httpclient_test.go @@ -0,0 +1,51 @@ +package schedulerclient + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" +) + +// TestListNodeMemoryBytesKeepsByteUnit verifies the node total memory is NOT +// re-scaled by the client: the apiserver's `max_mem` field is already in bytes +// (the worker reads sysinfo().totalram * mem_unit → bytes and passes it through +// unchanged), so scheduler_list_nodes must report memory_bytes_total ≈ 8.2e9 +// for a ~7.7 GiB node — not the ×1e6-inflated ~8.2e15 that httpclient.go:198 +// produced when it wrongly treated apiserver bytes as MB. +func TestListNodeMemoryBytesKeepsByteUnit(t *testing.T) { + // 8224825344 bytes = the real MemTotal of the t1 node (7.66 GiB). + const realMemBytes = 8224825344 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got == "" { + t.Errorf("expected Authorization header, got empty") + } + if got := r.URL.Path; got != "/xce/v1/hosts" { + t.Errorf("unexpected path %q", got) + } + w.Header().Set("Content-Type", "application/json") + body := `{"code":200,"msg":"ok","data":{"count":1,"rows":[` + + `{"fields":["host_name","host_status","num_cpus","num_run","max_mem"],` + + `"data":{"host_name":"t1","host_status":"idle","num_cpus":8,"num_run":0,"max_mem":8224825344}}` + + `]}}` + _, _ = w.Write([]byte(body)) + })) + defer srv.Close() + + hc := NewClient(srv.URL, "xce_ak_test") + nodes, err := hc.ListNodes(context.Background()) + if err != nil { + t.Fatalf("ListNodes: %v", err) + } + if len(nodes) != 1 { + t.Fatalf("expected 1 node, got %d", len(nodes)) + } + n := nodes[0] + if n.Name != "t1" || n.CoresTotal != 8 { + t.Fatalf("unexpected node: %+v", n) + } + if n.MemoryBytesTotal != realMemBytes { + t.Fatalf("MemoryBytesTotal = %d, want %d (apiserver max_mem is already bytes; must not be ×1e6)", n.MemoryBytesTotal, realMemBytes) + } +} diff --git a/src/frontend/schedulerclient/ordermap_test.go b/src/frontend/schedulerclient/ordermap_test.go index 44004a0e..d886de2b 100644 --- a/src/frontend/schedulerclient/ordermap_test.go +++ b/src/frontend/schedulerclient/ordermap_test.go @@ -56,8 +56,10 @@ func TestOrderMap_RowsUseColumnarShape(t *testing.T) { if n.CoresTotal != 8 { t.Errorf("node.CoresTotal = %d, want 8", n.CoresTotal) } - if n.MemoryBytesTotal != 8224821248*mbToBytes { - t.Errorf("node.MemoryBytesTotal = %d, want %d", n.MemoryBytesTotal, 8224821248*mbToBytes) + // max_mem from apiserver is already bytes (worker reports sysinfo bytes); + // the client must NOT scale it by mbToBytes (see httpclient.go ListNodes). + if n.MemoryBytesTotal != 8224821248 { + t.Errorf("node.MemoryBytesTotal = %d, want %d", n.MemoryBytesTotal, 8224821248) } }