From 2fcacac2cb551d95c9a1b22d13ecf2802919687b Mon Sep 17 00:00:00 2001 From: "lei.chen" Date: Tue, 27 Oct 2026 03:45:28 +0000 Subject: [PATCH] =?UTF-8?q?E2=20Phase=201:=20external-license-resources=20?= =?UTF-8?q?=E2=80=94=20xceesd=20daemon,=20extres=20role,=20EXTERNAL=20admi?= =?UTF-8?q?ssion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements Phase 1 of .kiro/specs/external-license-resources/ (design.md): externally-reported license pools pushed by a dedicated daemon through the existing resval pipeline, with EXTERNAL-aware admission arithmetic. Wire & controller: - controller.capnp HostRole.extres (@5, appended); xce_join_req accepts it - extres.c: HostRole_extres join table (endpoint+name, no workermgr slot), leave cleanup; join.c/cluster.c dispatch - opupd.c do_update_resvals: extres branch runs the resource hooks and update_resval_list(c, NULL, ...) — host=="" entries route through update_val_res' SHARED branch into c->sharedinst; per-entry -EPERM for non-shared targets is logged, never fails the list - worker.c update_val_res: NULL worker only writes SHARED resources - cluster.c cluster_build_resources: rebuilt on resourceApplyDef (bug-1854) — EEXIST-tolerant so definitions added by an activation register on a RUNNING controller (bug-1973), and RES_APPLY_CHANGED updates definition fields (valtype/ordtype/flags/interval) in place so a corrected config heals a stale registration (bug-1974); seed_sharedinst_from_config skips EXTERNAL pools (daemon-reported free counts only; seeding would fabricate capacity while the daemon is down) Scheduler: - core_events.c RESVALS_DELTA: NULL/"" host marks a SHARED value and lands in core->sharedInst via dup_val (bug-1585 ownership rule), mirroring snapshot_load's host=="" precedent — previously WARN-skipped as unknown worker - resource.c unsatisfy_: EXTERNAL skips the inuse term (F >= uncomit + wanted; F is already net of our checkouts) xceesd (src/esd, new component): - script collectors polled per source, debounce on unchanged values, fail-open stale policy; pushes host=="" shared resvals over xcecli - reconnect gap fix (bug-1975): with a stable value the debounce pushed nothing and reconnects only fire inside a push — after a controller restart the daemon sat on a dead connection forever; the main loop now forces a full re-push when xcecli_is_connection_broken() (bug-1760 predicate) latches Config/mapping: - resources.yaml: extlic sample (location: [external]) - mapping.py: RESOURCE_FLAG_EXTERNAL (0x10) + location:["external"] mapping; xceadmin tests Tests (TDD, red->green per fix): test_extres_resvals.c (join+resval pipeline), test_config_resource_reconcile.c +2 (register past EEXIST / hot-apply changed definition), test_core_events.c +131 lines (shared DELTA landing), mapping tests. Suites: controller 1358/0, scheduler 100/0, esd 7/0, lib 1/0; check-no-fork covers src/esd. Live t1 E2E: config gen=15 push -> xceesd push extlic=4 lands in sharedinst -> rusage[extlic=3] job RUN; second job PENDs at free=1; release re-admits it; controller restart -> forced re-push restores the value and admission resumes. Full precommit job verification passed. Co-Authored-By: Claude Code --- .wolf/buglog.json | 81 +++++ .wolf/bugs/bug-1973.md | 85 ++++++ .wolf/bugs/bug-1974.md | 84 +++++ .wolf/bugs/bug-1975.md | 88 ++++++ scripts/check-no-fork.py | 3 +- src/configs/resources.yaml | 13 + src/controller/cluster.c | 53 +++- src/controller/extres.c | 78 +++++ src/controller/include/cluster.h | 4 + src/controller/join.c | 40 +++ src/controller/meson.build | 3 + src/controller/opupd.c | 33 +- src/controller/testsuite/controller_test.c | 2 + .../test_config_resource_reconcile.c | 178 ++++++++++- .../testsuite/test_extres_resvals.c | 202 ++++++++++++ src/controller/worker.c | 7 + src/esd/include/esd.h | 90 ++++++ src/esd/meson.build | 41 +++ src/esd/src/esd.c | 288 ++++++++++++++++++ src/esd/src/esd_config.c | 286 +++++++++++++++++ src/esd/src/esd_script.c | 230 ++++++++++++++ src/esd/src/main.c | 127 ++++++++ src/esd/testsuite/test_esd_core.c | 201 ++++++++++++ src/include/xce/cluster_def.h | 14 + src/lib/request.c | 3 +- src/meson.build | 3 +- src/scheduler/src/core_events.c | 34 ++- src/scheduler/src/resource.c | 37 ++- src/scheduler/testsuite/test.c | 92 ++++++ src/scheduler/testsuite/test_core_events.c | 131 ++++++++ src/schema/controller.capnp | 5 + src/xceadmin/src/xcecommon/mapping.py | 13 + src/xceadmin/tests/test_mapping.py | 36 +++ 33 files changed, 2556 insertions(+), 29 deletions(-) create mode 100644 .wolf/bugs/bug-1973.md create mode 100644 .wolf/bugs/bug-1974.md create mode 100644 .wolf/bugs/bug-1975.md create mode 100644 src/controller/extres.c create mode 100644 src/controller/testsuite/test_extres_resvals.c create mode 100644 src/esd/include/esd.h create mode 100644 src/esd/meson.build create mode 100644 src/esd/src/esd.c create mode 100644 src/esd/src/esd_config.c create mode 100644 src/esd/src/esd_script.c create mode 100644 src/esd/src/main.c create mode 100644 src/esd/testsuite/test_esd_core.c diff --git a/.wolf/buglog.json b/.wolf/buglog.json index 7892eaad..890d9501 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -52,6 +52,87 @@ "f780a166" ], "record": ".wolf/bugs/bug-1972.md" + }, + { + "id": "bug-1973", + "status": "fixed", + "status_updated": "2026-10-27", + "status_note": "", + "component": "controller", + "severity": "wrong-result", + "timestamp": "2026-10-27T03:27:47Z", + "last_seen": "2026-10-27T03:27:47Z", + "occurrences": 1, + "error_message": "cluster_build_resources aborted whole loop on first EEXIST so new resource definitions never registered on a running controller\n\n**Observed:**\nController log on t1, after pushing a cluster.bin whose resources section ADDED\nthe new `extlic` definition (external-license-resources E2 Phase 1):\n\n```\n[ERROR] [log_update_resval_failure] update_resval `extlic': No such file or directory\n```\n\nThe xceesd daemon joined and pushed extlic values, but the controller had\nno `extlic` registration at all (update_resval key_find → -ENOENT).\n\n**Expected:** a config activation that adds a new resource definition must\nregister it in the running controller's resmgr, so daemon pushes land.\n\n**Impact:** every newly-added resource definition was silently inert until\nthe controller process was restarted (boot-time registration path did run).\nFor E2 this meant the whole extres-daemon chain was dead on a live controller.\n\n**Found by:** E2 Phase 1 live E2E on t1 (spec .kiro/specs/external-license-resources design.md §3.5 checklist).", + "file": "src/controller/cluster.c, src/controller/testsuite/test_config_resource_reconcile.c", + "root_cause": "`cluster_build_resources()` looped over the decoded config's resource list\ncalling `makeResource()` for each. `makeResource_` (src/lib/resobj.c:244-247)\nreturns NULL with errno=EEXIST when the name is already registered — which is\nthe COMMON case at activation (boot already registered some definitions). The\nold loop treated that rc as fatal and aborted the whole loop, so every\ndefinition AFTER the first already-registered one was never registered.", + "fix": "Rewrite the loop on top of bug-1854's `resourceApplyDef()` (lookup+add under\none write lock, EEXIST-tolerant): RES_APPLY_ADDED/SAME proceed normally;\nRES_APPLY_CHANGED updates the registered object's definition fields\n(valtype/ordtype/flags/interval) IN PLACE — none of them touches the\nregistration slot (idx) or accumulated accounting; mirrors\nreconcile_config_queues' in-place refresh precedent (second live bug, see\nbug-1974 — kept here as one loop). Negative rc still propagates.\n\nVerification:\n- Test: `test_reconcile_resources_registers_new_defs_past_eexist` +\n `test_reconcile_resources_hot_applies_changed_definition`\n (src/controller/testsuite/test_config_resource_reconcile.c); RED before the\n fix (lic1 unregistered / flags stayed 0x00), GREEN after.\n- Suites: controller testsuite 1358 tests, 0 failures (build.dbg).\n- Live: t1 E2E — config gen=15 push with extlic; xceesd push landed in\n sharedinst (extlic=4), rusage[extlic=3] job RUNNED; second job PENDed at\n free=1; release re-admitted it.", + "tags": [ + "config", + "reconcile", + "E2", + "external-license" + ], + "related_bugs": [ + "bug-1854" + ], + "fix_commits": [], + "record": ".wolf/bugs/bug-1973.md" + }, + { + "id": "bug-1974", + "status": "fixed", + "status_updated": "2026-10-27", + "status_note": "", + "component": "controller", + "severity": "wrong-result", + "timestamp": "2026-10-27T03:35:12Z", + "last_seen": "2026-10-27T03:35:12Z", + "occurrences": 1, + "error_message": "config activation kept stale resource definition (flags=0): daemon pushes refused with -EPERM\n\n**Observed:**\nController log on t1 after re-pushing a corrected cluster.bin (extlic mapped\nwith `location: [external]` → flags SHARED|GLOBAL|EXTERNAL):\n\n```\n[ERROR] [log_update_resval_failure] update_resval `extlic': Operation not permitted\n```\n\n**Expected:** update_val_res' SHARED branch routes a host==\"\" resval for a\nSHARED-flagged resource into c->sharedinst; with correct flags the push lands.\n\n**Impact:** the extres daemon's pushes were permanently refused on a controller\nwhose registration predated the flag fix — the stale shape was frozen for the\nprocess lifetime, so re-pushing a corrected config did not heal it.\n\n**Found by:** E2 Phase 1 live E2E on t1.", + "file": "src/controller/cluster.c, src/controller/worker.c, src/xceadmin/src/xcecommon/mapping.py", + "root_cause": "Two compounding factors:\n1. **Producer**: the DEPLOYED xceadmin venv (/opt/xce-denix/share/xce/\n xceadmin-venv) carries a stale mapping.py with NO resourcemap handling\n (83-line diff vs tree) — cluster bins it builds carry flags=0 for every\n resource. Ops issue, fixed by building with the tree venv\n (src/build.dbg/xceadmin/.venv/bin/xceadmin); the deployed venv still needs\n redeployment.\n2. **Controller**: even with a corrected bin, `cluster_build_resources()`\n treated an already-registered name as \"nothing to do\" (bug-1854's\n RES_APPLY_CHANGED was reported but never applied), so the stale flags=0\n shape survived every activation.", + "fix": "`cluster_build_resources()` now applies RES_APPLY_CHANGED by updating the\nregistered object's definition fields (valtype/ordtype/flags/interval) in\nplace under the resmgr write lock — none of them touches the registration\nslot (idx) or accumulated accounting; mirrors reconcile_config_queues'\n\"updated by configuration activation\" in-place refresh precedent. (Same loop\nas bug-1973's fix.) Producer side tracked separately as ops follow-up:\nredeploy the xceadmin venv.\n\nVerification:\n- Test: `test_reconcile_resources_hot_applies_changed_definition`\n (src/controller/testsuite/test_config_resource_reconcile.c); RED before\n (flags stayed 0x00), GREEN after (0x1C, idx preserved).\n- Suites: controller testsuite 1358 tests, 0 failures (build.dbg).\n- Live: t1 — bin rebuilt with tree venv (gen=15) + fix deployed; xceesd push\n landed (extlic=4 in `resources distribution list`), rusage[extlic=3] job RUN.", + "tags": [ + "config", + "flags", + "E2", + "external-license" + ], + "related_bugs": [ + "bug-1854", + "bug-1973" + ], + "fix_commits": [], + "record": ".wolf/bugs/bug-1974.md" + }, + { + "id": "bug-1975", + "status": "fixed", + "status_updated": "2026-10-27", + "status_note": "", + "component": "controller", + "severity": "wrong-result", + "timestamp": "2026-10-27T03:40:05Z", + "last_seen": "2026-10-27T03:40:05Z", + "occurrences": 1, + "error_message": "xceesd never reconnects after controller restart when its value is unchanged (debounce suppresses pushes)\n\n**Observed:**\nOld xceesd (pid 162609) sat idle on a dead connection for 50+ minutes after\nthe controller was restarted:\n\n```\n(no socket() attempts; strace showed only collector forks + epoll_wait;\n /proc/net socket table had no trace of the controller connection)\n```\n\nLast log line was the boot-time `esd full re-push: 0/1 source(s)` — no\nfurther activity.\n\n**Expected:** within one poll cycle of the controller's restart the daemon\nmust re-establish the connection and re-push its value (the new primary has\nno record of the daemon or its value).\n\n**Impact:** after any controller restart/failover with a stable license\nvalue, the controller restarts with extlic read 0 (seed skipped for EXTERNAL\nby design) and no daemon re-push — every rusage[extlic] job PENDs forever\nwith NO_RESOURCE until the license value happens to change.\n\n**Found by:** E2 Phase 1 live E2E on t1 (controller restart step).", + "file": "src/esd/src/esd.c, src/lib/client.c, src/lib/client_internal.h", + "root_cause": "Three mechanisms conspired:\n1. `esd_poll_round()` debounces unchanged values (`last_value == v` → skip\n push) — with a stable value it pushes NOTHING.\n2. A reconnect inside the client library only fires while INSIDE a push's\n failover call — no pushes, no reconnect.\n3. `esd_on_reconnect` (the rejoin hook) only runs on a successful rotate;\n with no push attempt it never runs.", + "fix": "`esd_run()`'s main loop now checks `xcecli_is_connection_broken()`\n(bug-1760's pure predicate — the transport's RDHUP handler sets\nXCE_CTX_ERR asynchronously) and forces one `esd_push_all()` when it latches.\nThe write attempt drives the rotate/reconnect; `esd_on_reconnect` keeps the\nfull re-push contract for the failover path.\n\nVerification:\n- Test: covered by the live contract (client-side predicate already unit\n tested by bug-1760's suite); esd-side loop is integration-shaped (the\n trigger is a transport callback), so live E2E is the proof.\n- Suites: controller testsuite 1358 tests, 0 failures (build.dbg).\n- Live: t1 — controller killed and restarted; esd logged\n `connection broken -- forcing a full re-push to reconnect`, rotated to the\n new primary, re-pushed; `xcecli resources distribution list` shows extlic=4\n and a fresh rusage[extlic=3] job RUNS.", + "tags": [ + "esd", + "external-license", + "reconnect", + "E2" + ], + "related_bugs": [ + "bug-1760", + "bug-1973", + "bug-1974" + ], + "fix_commits": [], + "record": ".wolf/bugs/bug-1975.md" } ] } diff --git a/.wolf/bugs/bug-1973.md b/.wolf/bugs/bug-1973.md new file mode 100644 index 00000000..2d913360 --- /dev/null +++ b/.wolf/bugs/bug-1973.md @@ -0,0 +1,85 @@ +--- +id: "bug-1973" +status: "fixed" +status_updated: "2026-10-27" +status_note: "" +component: "controller" +severity: "wrong-result" +timestamp: "2026-10-27T03:27:47Z" +last_seen: "2026-10-27T03:27:47Z" +occurrences: 1 +files: ["src/controller/cluster.c", "src/controller/testsuite/test_config_resource_reconcile.c"] +tags: ["config", "reconcile", "E2", "external-license"] +related: ["bug-1854"] +summary: "cluster_build_resources aborted at first EEXIST: new resource defs never registered on a running controller" +--- + +# bug-1973: cluster_build_resources aborted whole loop on first EEXIST so new resource definitions never registered on a running controller + +## Description +**Observed:** +Controller log on t1, after pushing a cluster.bin whose resources section ADDED +the new `extlic` definition (external-license-resources E2 Phase 1): + +``` +[ERROR] [log_update_resval_failure] update_resval `extlic': No such file or directory +``` + +The xceesd daemon joined and pushed extlic values, but the controller had +no `extlic` registration at all (update_resval key_find → -ENOENT). + +**Expected:** a config activation that adds a new resource definition must +register it in the running controller's resmgr, so daemon pushes land. + +**Impact:** every newly-added resource definition was silently inert until +the controller process was restarted (boot-time registration path did run). +For E2 this meant the whole extres-daemon chain was dead on a live controller. + +**Found by:** E2 Phase 1 live E2E on t1 (spec .kiro/specs/external-license-resources design.md §3.5 checklist). + +## Environment +- OS: Ubuntu 24.04.2 LTS; kernel Linux 6.8.0-147-generic x86_64 +- libc: glibc 2.42 (de-nix deployment closure) +- Toolchain: gcc (GCC) 15.2.0, meson 1.9.1 +- Build: debug (build.dbg), deployed with patchelf interpreter+RUNPATH +- Code: feature/license-resources @ ba42fdd8-dirty +- Deployment: t1 single-node (controller+scheduler+apiserver+worker), xceesd extres daemon + +## Reproduction +1. Controller running with a cluster.bin containing resources {slots, ...} +2. `xceadmin config build && xceadmin config push` with a bin that ADDS `extlic` +3. Start xceesd; it joins and pushes extlic values +4. Controller: `update_resval 'extlic': No such file or directory` (-ENOENT) +- Frequency: deterministic +- Evidence: controller log lines quoted above (t1, 2026-10-26 17:40) + +## Root cause +`cluster_build_resources()` looped over the decoded config's resource list +calling `makeResource()` for each. `makeResource_` (src/lib/resobj.c:244-247) +returns NULL with errno=EEXIST when the name is already registered — which is +the COMMON case at activation (boot already registered some definitions). The +old loop treated that rc as fatal and aborted the whole loop, so every +definition AFTER the first already-registered one was never registered. + +## Fix +Rewrite the loop on top of bug-1854's `resourceApplyDef()` (lookup+add under +one write lock, EEXIST-tolerant): RES_APPLY_ADDED/SAME proceed normally; +RES_APPLY_CHANGED updates the registered object's definition fields +(valtype/ordtype/flags/interval) IN PLACE — none of them touches the +registration slot (idx) or accumulated accounting; mirrors +reconcile_config_queues' in-place refresh precedent (second live bug, see +bug-1974 — kept here as one loop). Negative rc still propagates. + +## Verification +- Test: `test_reconcile_resources_registers_new_defs_past_eexist` + + `test_reconcile_resources_hot_applies_changed_definition` + (src/controller/testsuite/test_config_resource_reconcile.c); RED before the + fix (lic1 unregistered / flags stayed 0x00), GREEN after. +- Suites: controller testsuite 1358 tests, 0 failures (build.dbg). +- Live: t1 E2E — config gen=15 push with extlic; xceesd push landed in + sharedinst (extlic=4), rusage[extlic=3] job RUNNED; second job PENDed at + free=1; release re-admitted it. + +## References +- Related: bug-1854 (resourceApplyDef change-detection contract this loop is + built on), E2 spec .kiro/specs/external-license-resources/design.md §3.2 diff --git a/.wolf/bugs/bug-1974.md b/.wolf/bugs/bug-1974.md new file mode 100644 index 00000000..706fa8a1 --- /dev/null +++ b/.wolf/bugs/bug-1974.md @@ -0,0 +1,84 @@ +--- +id: "bug-1974" +status: "fixed" +status_updated: "2026-10-27" +status_note: "" +component: "controller" +severity: "wrong-result" +timestamp: "2026-10-27T03:35:12Z" +last_seen: "2026-10-27T03:35:12Z" +occurrences: 1 +files: ["src/controller/cluster.c", "src/controller/worker.c", "src/xceadmin/src/xcecommon/mapping.py"] +tags: ["config", "flags", "E2", "external-license"] +related: ["bug-1854", "bug-1973"] +--- + +# bug-1974: config activation kept stale resource definition (flags=0): daemon pushes refused with -EPERM + +## Description +**Observed:** +Controller log on t1 after re-pushing a corrected cluster.bin (extlic mapped +with `location: [external]` → flags SHARED|GLOBAL|EXTERNAL): + +``` +[ERROR] [log_update_resval_failure] update_resval `extlic': Operation not permitted +``` + +**Expected:** update_val_res' SHARED branch routes a host=="" resval for a +SHARED-flagged resource into c->sharedinst; with correct flags the push lands. + +**Impact:** the extres daemon's pushes were permanently refused on a controller +whose registration predated the flag fix — the stale shape was frozen for the +process lifetime, so re-pushing a corrected config did not heal it. + +**Found by:** E2 Phase 1 live E2E on t1. + +## Environment +- OS: Ubuntu 24.04.2 LTS; kernel Linux 6.8.0-147-generic x86_64 +- libc: glibc 2.42 (de-nix deployment closure) +- Toolchain: gcc (GCC) 15.2.0, meson 1.9.1 +- Build: debug (build.dbg) +- Code: feature/license-resources @ ba42fdd8-dirty +- Deployment: t1 single-node; xceesd extres daemon + +## Reproduction +1. Register extlic with flags=0 (see contributing factor below) +2. Push a corrected cluster.bin where extlic has SHARED|GLOBAL|EXTERNAL +3. xceesd pushes extlic=4 (host=="") +4. Controller: `update_resval 'extlic': Operation not permitted` (-EPERM) +- Frequency: deterministic (persists until controller restart or the bug-1973 fix) +- Evidence: controller log lines quoted above (t1, 2026-10-26 18:40/18:45) + +## Root cause +Two compounding factors: +1. **Producer**: the DEPLOYED xceadmin venv (/opt/xce-denix/share/xce/ + xceadmin-venv) carries a stale mapping.py with NO resourcemap handling + (83-line diff vs tree) — cluster bins it builds carry flags=0 for every + resource. Ops issue, fixed by building with the tree venv + (src/build.dbg/xceadmin/.venv/bin/xceadmin); the deployed venv still needs + redeployment. +2. **Controller**: even with a corrected bin, `cluster_build_resources()` + treated an already-registered name as "nothing to do" (bug-1854's + RES_APPLY_CHANGED was reported but never applied), so the stale flags=0 + shape survived every activation. + +## Fix +`cluster_build_resources()` now applies RES_APPLY_CHANGED by updating the +registered object's definition fields (valtype/ordtype/flags/interval) in +place under the resmgr write lock — none of them touches the registration +slot (idx) or accumulated accounting; mirrors reconcile_config_queues' +"updated by configuration activation" in-place refresh precedent. (Same loop +as bug-1973's fix.) Producer side tracked separately as ops follow-up: +redeploy the xceadmin venv. + +## Verification +- Test: `test_reconcile_resources_hot_applies_changed_definition` + (src/controller/testsuite/test_config_resource_reconcile.c); RED before + (flags stayed 0x00), GREEN after (0x1C, idx preserved). +- Suites: controller testsuite 1358 tests, 0 failures (build.dbg). +- Live: t1 — bin rebuilt with tree venv (gen=15) + fix deployed; xceesd push + landed (extlic=4 in `resources distribution list`), rusage[extlic=3] job RUN. + +## References +- Related: bug-1854 (resourceApplyDef change-detection), bug-1973 (same loop's + EEXIST abort), worker.c update_val_res guard (the -EPERM surface) diff --git a/.wolf/bugs/bug-1975.md b/.wolf/bugs/bug-1975.md new file mode 100644 index 00000000..03f18713 --- /dev/null +++ b/.wolf/bugs/bug-1975.md @@ -0,0 +1,88 @@ +--- +id: "bug-1975" +status: "fixed" +status_updated: "2026-10-27" +status_note: "" +component: "controller" +severity: "wrong-result" +timestamp: "2026-10-27T03:40:05Z" +last_seen: "2026-10-27T03:40:05Z" +occurrences: 1 +files: ["src/esd/src/esd.c", "src/lib/client.c", "src/lib/client_internal.h"] +tags: ["esd", "external-license", "reconnect", "E2"] +related: ["bug-1760", "bug-1973", "bug-1974"] +summary: "xceesd idle on dead connection after controller restart (stable value: no re-push)" +title: "xceesd idle on dead connection after controller restart (stable value: no re-push)" +--- + +# bug-1975: xceesd idle on dead connection after controller restart (stable value: no re-push) + +## Description +**Observed:** +Old xceesd (pid 162609) sat idle on a dead connection for 50+ minutes after +the controller was restarted: + +``` +(no socket() attempts; strace showed only collector forks + epoll_wait; + /proc/net socket table had no trace of the controller connection) +``` + +Last log line was the boot-time `esd full re-push: 0/1 source(s)` — no +further activity. + +**Expected:** within one poll cycle of the controller's restart the daemon +must re-establish the connection and re-push its value (the new primary has +no record of the daemon or its value). + +**Impact:** after any controller restart/failover with a stable license +value, the controller restarts with extlic read 0 (seed skipped for EXTERNAL +by design) and no daemon re-push — every rusage[extlic] job PENDs forever +with NO_RESOURCE until the license value happens to change. + +**Found by:** E2 Phase 1 live E2E on t1 (controller restart step). + +## Environment +- OS: Ubuntu 24.04.2 LTS; kernel Linux 6.8.0-147-generic x86_64 +- libc: glibc 2.42 (de-nix deployment closure) +- Toolchain: gcc (GCC) 15.2.0, meson 1.9.1 +- Build: debug (build.dbg) +- Code: feature/license-resources @ ba42fdd8-dirty +- Deployment: t1 single-node; xceesd extres daemon + +## Reproduction +1. xceesd running, pushing a value the collector keeps returning unchanged +2. Restart (or fail over) the controller +3. Observe the daemon: no reconnect attempts, pushes never resume +- Frequency: deterministic whenever the value is stable across the restart +- Evidence: strace captures /tmp/esd-strace*.out (t1, 2026-10-26); log silence quoted above + +## Root cause +Three mechanisms conspired: +1. `esd_poll_round()` debounces unchanged values (`last_value == v` → skip + push) — with a stable value it pushes NOTHING. +2. A reconnect inside the client library only fires while INSIDE a push's + failover call — no pushes, no reconnect. +3. `esd_on_reconnect` (the rejoin hook) only runs on a successful rotate; + with no push attempt it never runs. + +## Fix +`esd_run()`'s main loop now checks `xcecli_is_connection_broken()` +(bug-1760's pure predicate — the transport's RDHUP handler sets +XCE_CTX_ERR asynchronously) and forces one `esd_push_all()` when it latches. +The write attempt drives the rotate/reconnect; `esd_on_reconnect` keeps the +full re-push contract for the failover path. + +## Verification +- Test: covered by the live contract (client-side predicate already unit + tested by bug-1760's suite); esd-side loop is integration-shaped (the + trigger is a transport callback), so live E2E is the proof. +- Suites: controller testsuite 1358 tests, 0 failures (build.dbg). +- Live: t1 — controller killed and restarted; esd logged + `connection broken -- forcing a full re-push to reconnect`, rotated to the + new primary, re-pushed; `xcecli resources distribution list` shows extlic=4 + and a fresh rusage[extlic=3] job RUNS. + +## References +- Related: bug-1760 (the pure broken-connection predicate this reuses), + E2 spec design.md §3.5 E2E checklist (daemon kill/restart + controller + failover items) diff --git a/scripts/check-no-fork.py b/scripts/check-no-fork.py index 2a69c433..d5cf38e1 100644 --- a/scripts/check-no-fork.py +++ b/scripts/check-no-fork.py @@ -21,7 +21,8 @@ import os import re import sys -COVERED = ["src/controller", "src/scheduler", "src/lib", "src/worker"] +COVERED = ["src/controller", "src/scheduler", "src/lib", "src/worker", + "src/esd"] EXCLUDED_DIRS = {"testsuite", "test", "tests"} # Fork-family calls. uv_spawn() is libuv's fork()+exec on Linux. BANNED = re.compile( diff --git a/src/configs/resources.yaml b/src/configs/resources.yaml index 360d7a8b..4fc601b2 100644 --- a/src/configs/resources.yaml +++ b/src/configs/resources.yaml @@ -269,3 +269,16 @@ resources: - name: gstring description: string resource type: string + + # external-license-resources: cluster-wide license pool reported by + # the extres daemon (xceesd) -- config declares only the name/shape, + # the free counts come from the daemon's pushes + - name: extlic + description: externally reported license pool + type: numeric + interval: 0 + consumable: true + order: decrease + resourcemap: + - location: + - external diff --git a/src/controller/cluster.c b/src/controller/cluster.c index a2810a23..0ef66c03 100644 --- a/src/controller/cluster.c +++ b/src/controller/cluster.c @@ -105,6 +105,9 @@ static void handle_endpoint_error(endpoint_t* ep) case HostRole_apiserver: handle_apiserver_leave(c, ep); break; + case HostRole_extres: + handle_extres_leave(c, ep); + break; default: ERROR("unknown role %d for endpoint %p", priv->role, ep); break; @@ -1535,21 +1538,50 @@ int cluster_build_config(cluster_t* c) int cluster_build_resources(cluster_t* c) { int i; + int rc = 0; - if (c->myconf == NULL) { + if ((c->myconf == NULL) || (c->myconf->cluster_config == NULL)) { return -EINVAL; } for (i = 0; i < c->myconf->cluster_config->nres; i++) { - xce_resource_t* res; + int arc; - res = makeResource(c->resmgr, c->myconf->cluster_config->resources[i]); - if (res == NULL) { - return -errno; + arc = resourceApplyDef(c->resmgr, + c->myconf->cluster_config->resources[i]); + if (arc == RES_APPLY_CHANGED) { + /* The registered object was created from a DIFFERENT generation + * of the definition (bug-1854's change-detection contract). + * Keeping the stale shape would freeze the boot-time behavior + * for the daemon's lifetime -- the live sighting: extlic first + * registered with flags=0 (the building tool dropped its + * resourcemap), and every later activation kept refusing the + * extres daemon's pushes with -EPERM because + * update_val_res()'s sharedinst route gates on XCE_RESF_SHARED. + * Update the definition fields IN PLACE: none of them touches + * the registration slot (idx) or accumulated accounting, and + * this mirrors reconcile_config_queues' "updated by + * configuration activation" in-place refresh precedent. */ + xce_resource_t* cur = NULL; + xce_resource_t* def = c->myconf->cluster_config->resources[i]; + + lock(c->resmgr, RTTI_LOCK_WRITE); + if (key_find(c->resmgr, (void*)def->name, (void**)&cur) > 0 + && cur != NULL) { + cur->valtype = def->valtype; + cur->ordtype = def->ordtype; + cur->flags = def->flags; + cur->interval = def->interval; + INFO("resource %s definition updated by configuration " + "activation (flags=0x%x)", def->name, def->flags); + } + unlock(c->resmgr); + } else if (arc < 0) { + rc = arc; } } - return 0; + return rc; } static void resval_obj_killer(void* arg); @@ -1598,6 +1630,15 @@ static void seed_sharedinst_from_config(cluster_t* c) || (r->flags & XCE_RESF_GLOBAL) == 0) { continue; } + /* external-license-resources: an EXTERNAL pool's values come only + * from the extres daemon's free-count pushes -- seeding a static + * config value here would mask "daemon not yet up" as a phantom + * capacity (admitting jobs against licenses that do not exist). + * Until the daemon joins, get_resvals reads 0 and jobs PEND with a + * NO_RESOURCE detail naming the resource -- the honest signal. */ + if (r->flags & XCE_RESF_EXTERNAL) { + continue; + } if (r->ndistributions < 1 || r->distributions == NULL || r->distributions[0] == NULL || r->distributions[0]->value == NULL) { continue; diff --git a/src/controller/extres.c b/src/controller/extres.c new file mode 100644 index 00000000..0b3fb82e --- /dev/null +++ b/src/controller/extres.c @@ -0,0 +1,78 @@ +/* + * extres.c — connected extres daemons (HostRole_extres joins) + * + * external-license-resources Phase 1: the external-resources daemon + * (xceesd) joins the controller with HostRole_extres and pushes + * cluster-level (SHARED) resource values through the ordinary resval + * update pipeline (opupd.c do_update_resvals). Mirrors the apiserver + * join/leave registration in apiserver.c: endpoint + name only, no + * workermgr slot — the daemon is not a schedulable host. The controller + * never pushes TO the daemon; the daemon re-pushes its full value set on + * every (re)connect, so nothing here needs replay/WAL treatment. + */ + +#include +#include +#include + +#include "cluster.h" +#include "xstring.h" +#include "xce/cluster_def.h" + +int add_extres(cluster_t* c, char* name, endpoint_t* ep) +{ + int i; + + for (i = 0; i < N_EXTRES_MAX; i++) { + if ((c->extresds[i].name != NULL) + && (strcmp(c->extresds[i].name, name) == 0)) { + /* re-join (reconnect or failover re-join): rebind in place */ + c->extresds[i].ep = ep; + return i; + } + } + + for (i = 0; i < N_EXTRES_MAX; i++) { + if (c->extresds[i].name == NULL) { + c->extresds[i].name = makeString(name); + if (c->extresds[i].name == NULL) { + return -ENOMEM; + } + c->extresds[i].ep = ep; + return i; + } + } + + ERROR("too many extres daemons (max %d); join from %s refused", + N_EXTRES_MAX, name); + return -ENOSPC; +} + +int handle_extres_leave(cluster_t* c, endpoint_t* ep) +{ + ep_private_t* priv; + int i; + + if (ep == NULL) { + return -EINVAL; + } + + priv = (ep_private_t*)ep->data; + if (priv == NULL) { + return -EINVAL; + } + + i = priv->u.idx; + if ((i < 0) || (i >= N_EXTRES_MAX)) { + SET_ERROR(EINVAL); + return -errno; + } + + /* Only clear if it is still our binding — a re-join may have rebound + * the slot to a new endpoint before this old one's close event ran. */ + if (c->extresds[i].ep == ep) { + c->extresds[i].ep = NULL; + } + + return 0; +} diff --git a/src/controller/include/cluster.h b/src/controller/include/cluster.h index e2d22724..595f943f 100644 --- a/src/controller/include/cluster.h +++ b/src/controller/include/cluster.h @@ -439,5 +439,9 @@ extern int schedadmin_signal_disconnect(void); extern int handle_apiserver_leave(cluster_t*, endpoint_t*); extern int add_apiserver(cluster_t*, char*, endpoint_t*); extern int apiserver_init(cluster_t*); +/* external-license-resources Phase 1: HostRole_extres daemon registration + * (extres.c) — endpoint + name only, no workermgr slot. */ +extern int handle_extres_leave(cluster_t*, endpoint_t*); +extern int add_extres(cluster_t*, char*, endpoint_t*); #endif diff --git a/src/controller/join.c b/src/controller/join.c index f81b9d5a..fce86655 100644 --- a/src/controller/join.c +++ b/src/controller/join.c @@ -248,6 +248,43 @@ static int handle_apiserver_join(cluster_t* c, endpoint_t* ep, return 0; } +/* + * handle_extres_join: register the external-resources daemon (xceesd, + * external-license-resources Phase 1). Mirrors handle_apiserver_join's + * lightweight shape — endpoint + priv->u.idx only, no workermgr slot — and + * unlike the apiserver there is no on-join state push: the daemon is the + * pusher (it re-sends its full shared-value set on every reconnect). + */ +static int handle_extres_join(cluster_t* c, endpoint_t* ep, + xce_request_t* request) +{ + xce_reply_t reply; + xce_peerinfo_t* from = request->from; + int rc = 0; + + TRACE("%s extres daemon join, ep %p", __func__, ep); + + init_reply_by_req(&reply, request); + + rc = add_extres(c, from->name, ep); + if (rc >= 0) { + ep_private_t* priv = (ep_private_t*)ep->data; + + priv->u.idx = rc; + rc = 0; + } + + reply.status = rc; + reply.data_which = RequestReply_data_nodata; + xce_reply(ep, c->myhost, &reply); + + if (rc < 0) { + ERROR("extres daemon %s join failed, rc %d", from->name, rc); + return rc; + } + return 0; +} + static int handle_join_(cluster_t* c, endpoint_t* ep, xce_request_t* request) { ep_private_t* priv; @@ -319,6 +356,9 @@ static int handle_join_(cluster_t* c, endpoint_t* ep, xce_request_t* request) case HostRole_apiserver: rc = handle_apiserver_join(c, ep, request); break; + case HostRole_extres: + rc = handle_extres_join(c, ep, request); + break; default: rc = -E_XCE_BAD_ROLE; ERROR("unknown role %d from host %s", role, from->name); diff --git a/src/controller/meson.build b/src/controller/meson.build index 781b9ea0..e1f64c4d 100644 --- a/src/controller/meson.build +++ b/src/controller/meson.build @@ -5,6 +5,7 @@ controller_src = [ 'ctrlstatus.c', 'dataex.c', 'egress.c', + 'extres.c', 'ha.c', 'ha_convergence.c', 'job.c', @@ -90,6 +91,7 @@ controller_unit_test_src = [ 'testsuite/test_ctrl_metrics.c', 'testsuite/test_config_repl.c', 'testsuite/test_worker_join.c', + 'testsuite/test_extres_resvals.c', 'testsuite/test_ha_startup.c', 'testsuite/test_ha_election.c', 'testsuite/test_ha_elect_ctx_lifecycle.c', @@ -205,6 +207,7 @@ controller_unit_test_src = [ 'config_repl.c', 'slowlane.c', 'cluster.c', + 'extres.c', 'resync.c', ] diff --git a/src/controller/opupd.c b/src/controller/opupd.c index 6b46b03c..6c37ecd1 100644 --- a/src/controller/opupd.c +++ b/src/controller/opupd.c @@ -23,7 +23,38 @@ static int do_update_resvals(cluster_t* c, endpoint_t* ep, if (priv != NULL) { int idx = priv->u.idx; - if ((priv->role == HostRole_worker) && (idx >= 0)) { + if (priv->role == HostRole_extres) { + /* external-license-resources Phase 1: the extres daemon + * (xceesd) pushes cluster-level values with NO workermgr slot + * -- w==NULL routes every entry through update_val_res()'s + * "SHARED 即收" guard (shared values land in c->sharedinst; + * host-based values are refused -EPERM per entry). The reply + * status is 0 for any syntactically valid list: per-entry + * refusals are logged by log_update_resval_failure() and must + * not fail the whole update (mirrors the worker path's + * per-item convention). */ + xce_update_data_t* updatemsg = request->body.updatemsg; + xce_resval_list_t* resvals = updatemsg->data.resvals; + xce_resval_list_t applied = { .count = 0, .vals = NULL }; + + if ((resvals != NULL) && (resvals->count > 0)) { + applied.vals = (xce_resval_t**)calloc( + (size_t)resvals->count, sizeof(xce_resval_t*)); + } + + xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_RESOURCE, + HOOK_BEFORE_EVENT, HOOK_DETAIL_RESOURCE_UPDATE, c, + resvals); + update_resval_list(c, NULL, resvals, + (applied.vals != NULL) ? &applied : NULL); + if (applied.count > 0) { + xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_RESOURCE, + HOOK_AFTER_EVENT, HOOK_DETAIL_RESOURCE_UPDATE, c, + &applied); + } + free(applied.vals); + rc = 0; + } else if ((priv->role == HostRole_worker) && (idx >= 0)) { worker_t* w = NULL; rc = vget(c->workermgr, idx, (void**)&w); diff --git a/src/controller/testsuite/controller_test.c b/src/controller/testsuite/controller_test.c index d3cf9140..fa0284b1 100644 --- a/src/controller/testsuite/controller_test.c +++ b/src/controller/testsuite/controller_test.c @@ -40,6 +40,7 @@ extern void test_p3_manual_failover(void); extern void test_ctrl_metrics(void); extern void test_config_repl(void); extern void test_worker_join(void); +extern void test_extres_resvals(void); extern void test_ha_startup(void); extern void test_ha_election(void); extern void test_ha_elect_ctx_lifecycle(void); @@ -256,6 +257,7 @@ int main(int argc, char* argv[]) TEST(test_ctrl_metrics); TEST(test_config_repl); TEST(test_worker_join); + TEST(test_extres_resvals); TEST(test_ha_startup); TEST(test_ha_election); TEST(test_ha_elect_ctx_lifecycle); diff --git a/src/controller/testsuite/test_config_resource_reconcile.c b/src/controller/testsuite/test_config_resource_reconcile.c index 46ef051d..8cc4dbac 100644 --- a/src/controller/testsuite/test_config_resource_reconcile.c +++ b/src/controller/testsuite/test_config_resource_reconcile.c @@ -61,13 +61,18 @@ static xce_val_t* crr_shared_val(cluster_t* c, int idx) * res0 "slots" -- built-in hostbased (no resourcemap): registered, never seeded * res1 "lic1" -- SHARED|GLOBAL, distribution value 100: seeded * res2 "hostres"-- SHARED but NOT GLOBAL (per-host shape): NOT seeded - * res3 "dyn1" -- SHARED|GLOBAL, distribution WITHOUT a value: NOT seeded */ + * res3 "dyn1" -- SHARED|GLOBAL, distribution WITHOUT a value: NOT seeded + * res4 "extlic" -- SHARED|GLOBAL|EXTERNAL, distribution value 7: NOT + * seeded (external-license-resources: an EXTERNAL pool's + * values come only from the extres daemon's free-count + * pushes; seeding a static config value would mask "daemon + * not yet up" as a phantom capacity) */ static void crr_build_config(cluster_config_t* cfg, xce_resource_t** resources, res_distribution_t** dists, xce_val_t** vals) { - static xce_resource_t r0, r1, r2, r3; - static res_distribution_t d1, d2, d3; - static xce_val_t v1, v3; + static xce_resource_t r0, r1, r2, r3, r4; + static res_distribution_t d1, d2, d3, d4; + static xce_val_t v1, v3, v4; memset(&r0, 0, sizeof(r0)); r0.name = (char*)"slots"; @@ -108,11 +113,25 @@ static void crr_build_config(cluster_config_t* cfg, xce_resource_t** resources, dists[2] = &d3; vals[3] = NULL; + memset(&r4, 0, sizeof(r4)); + r4.name = (char*)"extlic"; + r4.flags = XCE_RESF_SHARED | XCE_RESF_GLOBAL | XCE_RESF_EXTERNAL; + memset(&d4, 0, sizeof(d4)); + memset(&v4, 0, sizeof(v4)); + v4.which = Value_u_fvalue; + v4.u.fvalue = 7.0; + d4.value = &v4; + r4.ndistributions = 1; + r4.distributions = (res_distribution_t**)&dists[3]; + dists[3] = &d4; + vals[4] = &v4; + resources[0] = &r0; resources[1] = &r1; resources[2] = &r2; resources[3] = &r3; - cfg->nres = 4; + resources[4] = &r4; + cfg->nres = 5; cfg->resources = resources; } @@ -181,6 +200,153 @@ static void test_reconcile_resources_registers_and_seeds_shared_pools(void) xce_resource_t* rb = crr_find(&c, "slots"); TEST_ASSERT_NULL(crr_shared_val(&c, rb->idx)); + /* EXTERNAL: daemon-reported free values only -- never seeded from the + * config, even with a valued distribution (external-license-resources). */ + xce_resource_t* re = crr_find(&c, "extlic"); + TEST_ASSERT_NOT_NULL(re); + TEST_ASSERT_NULL_MESSAGE(crr_shared_val(&c, re->idx), + "an EXTERNAL shared pool must not seed from its config value -- " + "only the extres daemon reports its free counts"); + + xce_hook_fini(&c.hooksys); + dereference(c.resmgr); + zarray_fini(&c.sharedinst); +} + +/* Live-activation partial registration: the resmgr already holds SOME of the + * config's resources (registered at boot from the previous generation) but + * not all (the activation adds a NEW definition -- the external-license- + * resources scenario: gen-12 config without extlic, gen-13 activation adds + * it while the 33 boot resources EEXIST). makeResource() refuses an existing + * name with NULL+EEXIST, so cluster_build_resources() must NOT abort the + * loop at the first EEXIST -- the new definition must still register and + * its shared pool (lic1 here) must still seed. Regression for the live + * sighting: every activation after boot bailed at "slots" (i=0) so NO new + * resource definition ever reached a running controller. */ +static void test_reconcile_resources_registers_new_defs_past_eexist(void) +{ + kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING, + .nslots = 16, .delta = 8 }; + cluster_t c; + myconf_t conf; + cluster_config_t cfg; + xce_resource_t* resources[4]; + res_distribution_t* dists[3]; + xce_val_t* vals[4]; + xce_resource_t* r; + xce_val_t* v; + + if (makeString((char*)"crr-probe") == NULL) + xstring_table_init(64); + + memset(&c, 0, sizeof(c)); + memset(&conf, 0, sizeof(conf)); + TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX)); + c.resmgr = makeInstance(&xkvmgr_type, &str_params); + TEST_ASSERT_NOT_NULL(c.resmgr); + zarray_init(&c.sharedinst, 16, + (zarray_item_killer_t)crr_val_killer); + + crr_build_config(&cfg, resources, dists, vals); + conf.cluster_config = &cfg; + c.myconf = &conf; + + /* Boot path: register ONLY the first definition (slots) -- a smaller + * previous generation. */ + { + xce_resource_t* boot = makeResource(c.resmgr, resources[0]); + TEST_ASSERT_NOT_NULL(boot); + } + + /* Activation: the full new config (slots EEXISTS, lic1 is new). */ + TEST_ASSERT_EQUAL_INT_MESSAGE(0, reconcile_config_resources(&c), + "an activation must tolerate EEXIST on pre-registered resources " + "and keep registering the rest"); + + r = crr_find(&c, "lic1"); + TEST_ASSERT_NOT_NULL_MESSAGE(r, + "the NEW definition must register even though an earlier resource " + "in the config already existed"); + v = crr_shared_val(&c, r->idx); + TEST_ASSERT_NOT_NULL_MESSAGE(v, + "the new definition's static shared pool must seed on activation"); + TEST_ASSERT_EQUAL_FLOAT(100.0, v->u.fvalue); + + /* the pre-existing runtime object must be kept (idempotent pass) */ + { + xce_resource_t* slots = crr_find(&c, "slots"); + + TEST_ASSERT_NOT_NULL(slots); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, slots->idx, + "the boot-registered object keeps its resmgr slot"); + } + + xce_hook_fini(&c.hooksys); + dereference(c.resmgr); + zarray_fini(&c.sharedinst); +} + +/* Hot-apply a CHANGED definition: the resmgr already holds a resource with + * one definition (registered at boot from the previous generation), and the + * activation brings a CHANGED one (flags changed -- the external-license- + * resources live sighting: extlic was first registered from a config built + * WITHOUT the `location: [external]` resourcemap, so its flags were 0; the + * corrected config carries SHARED|GLOBAL|EXTERNAL). Keeping the stale object + * would leave the daemon's pushes refused -EPERM forever (update_val_res + * gates the sharedinst route on XCE_RESF_SHARED) until a manual controller + * restart. The activation must update the definition fields in place -- + * idx (runtime state) preserved -- mirroring reconcile_config_queues' + * in-place refresh contract. */ +static void test_reconcile_resources_hot_applies_changed_definition(void) +{ + kvmgr_params_t str_params = { .key_type = KVMGR_KEY_STRING, + .nslots = 16, .delta = 8 }; + cluster_t c; + myconf_t conf; + cluster_config_t cfg; + xce_resource_t* resources[4]; + res_distribution_t* dists[3]; + xce_val_t* vals[4]; + xce_resource_t* r; + static xce_resource_t old_extlic; + + if (makeString((char*)"crr-probe") == NULL) + xstring_table_init(64); + + memset(&c, 0, sizeof(c)); + memset(&conf, 0, sizeof(conf)); + TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX)); + c.resmgr = makeInstance(&xkvmgr_type, &str_params); + TEST_ASSERT_NOT_NULL(c.resmgr); + zarray_init(&c.sharedinst, 16, + (zarray_item_killer_t)crr_val_killer); + + /* Boot path: register "extlic" WITHOUT the shared/external flags -- the + * stale gen-13 shape (resourcemap dropped by the building tool). */ + memset(&old_extlic, 0, sizeof(old_extlic)); + old_extlic.name = (char*)"extlic"; + old_extlic.flags = 0; + TEST_ASSERT_NOT_NULL(makeResource(c.resmgr, &old_extlic)); + + /* Activation: the corrected config where extlic is + * SHARED|GLOBAL|EXTERNAL (r4 in the shared builder). */ + crr_build_config(&cfg, resources, dists, vals); + conf.cluster_config = &cfg; + c.myconf = &conf; + + TEST_ASSERT_EQUAL_INT_MESSAGE(0, reconcile_config_resources(&c), + "an activation over a stale definition must succeed"); + + r = crr_find(&c, "extlic"); + TEST_ASSERT_NOT_NULL(r); + TEST_ASSERT_EQUAL_HEX32_MESSAGE( + XCE_RESF_SHARED | XCE_RESF_GLOBAL | XCE_RESF_EXTERNAL, r->flags, + "the activation must hot-apply the changed flags in place -- a " + "stale flags=0 object would refuse the daemon's pushes forever"); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, r->idx, + "the runtime idx (registration slot) must be preserved by the " + "in-place update"); + xce_hook_fini(&c.hooksys); dereference(c.resmgr); zarray_fini(&c.sharedinst); @@ -246,5 +412,7 @@ void test_config_resource_reconcile(void) { UnitySetTestFile(__FILE__); RUN_TEST(test_reconcile_resources_registers_and_seeds_shared_pools); + RUN_TEST(test_reconcile_resources_registers_new_defs_past_eexist); + RUN_TEST(test_reconcile_resources_hot_applies_changed_definition); RUN_TEST(test_reconcile_resources_is_idempotent_and_hot_applies); } diff --git a/src/controller/testsuite/test_extres_resvals.c b/src/controller/testsuite/test_extres_resvals.c new file mode 100644 index 00000000..966c9735 --- /dev/null +++ b/src/controller/testsuite/test_extres_resvals.c @@ -0,0 +1,202 @@ +/* + * test_extres_resvals.c — external-license-resources Phase 1 (E2), steps 1-2 + * + * The extres daemon (xceesd) pushes cluster-level (SHARED) resource values + * through the same resval update pipeline workers use, but it has NO + * workermgr slot: it joins as HostRole_extres and updates land on + * c->sharedinst directly. Two rules pin the daemon path: + * + * 1. update_resval_list() accepts w==NULL and applies SHARED-resource + * values into c->sharedinst (the only array the daemon can write). + * 2. The "SHARED 即收" guard: a NULL worker may only write SHARED + * resources — a non-SHARED (host-based) resource with w==NULL is + * refused (-EPERM) and NOT counted as applied (bug-1583's applied-list + * contract: downstream hooks must not hear about rejected values). + * + * This pins the opupd-side admission shape BEFORE the do_update_resvals + * role gate is extended to HostRole_extres (opupd itself needs a live + * endpoint, so the seam under test is worker.c's list applier, the same + * XCE_TESTING-exposure convention as apply_join_state in + * test_worker_join.c). + */ + +/* rtti.h must come first — it defines 'var' and 'typeinfo_t' used by kvmgr.h */ +#include + +#include +#include +#include +#include + +#include + +#include "cluster.h" +#include "controller.capnp.h" +#include "kvmgr.h" +#include "resobj.h" +#include "worker.h" +#include "xce/types.h" +#include "zarray.h" + +/* ── Forward declarations for functions under test ──────────────────── */ + +/* + * XCE_TESTING exposure in worker.c (same convention as apply_join_state): + * the daemon path calls update_resval_list() with w==NULL. + */ +int update_resval_list(cluster_t* c, worker_t* w, xce_resval_list_t* l, + xce_resval_list_t* applied); + +/* ── fixture ────────────────────────────────────────────────────────── */ + +typedef struct { + cluster_t c; + var resmgr; +} test_cluster_t; + +static test_cluster_t g_te; + +static void setup_resmgr(void) +{ + kvmgr_params_t str_params = { + .key_type = KVMGR_KEY_STRING, + .nslots = 64, + .delta = 16, + .datakiller = NULL, + }; + + memset(&g_te, 0, sizeof(g_te)); + + g_te.resmgr = makeInstance(&xkvmgr_type, &str_params); + TEST_ASSERT_NOT_NULL(g_te.resmgr); + + /* one SHARED resource (daemon-writable) and one host-based resource */ + { + static const char* names[2] = { "lic", "slots" }; + static const int shared[2] = { 1, 0 }; + xce_resource_t rdef; + int i; + + for (i = 0; i < 2; i++) { + memset(&rdef, 0, sizeof(rdef)); + rdef.name = (char*)names[i]; + rdef.valtype = XCE_VAL_NUMERIC; + rdef.flags = shared[i] ? XCE_RESF_SHARED : 0; + rdef.idx = -1; + TEST_ASSERT_NOT_NULL_MESSAGE(makeResource(g_te.resmgr, &rdef), + names[i]); + } + } + + g_te.c.resmgr = g_te.resmgr; + zarray_init(&g_te.c.sharedinst, 64, NULL); +} + +static void teardown_resmgr(void) +{ + zarray_fini(&g_te.c.sharedinst); + if (g_te.resmgr) dereference(g_te.resmgr); + memset(&g_te, 0, sizeof(g_te)); +} + +/* Build a one-entry resval list for `resname` with fvalue `value`. The + * xce_val_t and the list are statics so no teardown is needed (makeResource + * interns names; the resval name pointer must outlive the call). */ +static xce_resval_list_t g_te_list; +static xce_resval_t* g_te_vals[4]; +static xce_resval_t g_te_rv[4]; +static xce_val_t g_te_val[4]; + +static xce_resval_list_t* make_resval_list(const char** names, + const float* values, int n) +{ + int i; + + for (i = 0; i < n; i++) { + memset(&g_te_val[i], 0, sizeof(g_te_val[i])); + g_te_val[i].which = Value_u_fvalue; + g_te_val[i].u.fvalue = values[i]; + + memset(&g_te_rv[i], 0, sizeof(g_te_rv[i])); + g_te_rv[i].name = (char*)names[i]; + g_te_rv[i].host = NULL; + g_te_rv[i].val = &g_te_val[i]; + + g_te_vals[i] = &g_te_rv[i]; + } + g_te_list.count = n; + g_te_list.vals = g_te_vals; + return &g_te_list; +} + +/* ── tests ──────────────────────────────────────────────────────────── */ + +/* Rule 1: daemon (w==NULL) SHARED write lands in c->sharedinst at the + * resource's idx. */ +static void test_extres_shared_resval_lands_in_sharedinst(void) +{ + static const char* names[1] = { "lic" }; + static const float values[1] = { 4.0f }; + xce_resval_list_t* l; + xce_val_t* stored; + int rc; + + setup_resmgr(); + l = make_resval_list(names, values, 1); + rc = update_resval_list(&g_te.c, NULL, l, NULL); + TEST_ASSERT_EQUAL_INT(0, rc); + + stored = (xce_val_t*)zarray_get(&g_te.c.sharedinst, 0); + TEST_ASSERT_NOT_NULL(stored); + TEST_ASSERT_EQUAL_INT(Value_u_fvalue, stored->which); + TEST_ASSERT_EQUAL_FLOAT(4.0, stored->u.fvalue); + teardown_resmgr(); +} + +/* Rule 2 ("SHARED 即收"): w==NULL + non-SHARED resource is refused + * (-EPERM per-item, logged by the caller's failure hook), not applied, and + * not reported as applied -- the list-level call still succeeds, matching + * update_resvals()'s per-item convention. */ +static void test_extres_nonshared_resval_refused_eperm(void) +{ + static const char* names[2] = { "slots", "lic" }; + static const float values[2] = { 8.0f, 4.0f }; + xce_resval_list_t* l; + xce_resval_list_t applied = { .count = 0, .vals = NULL }; + xce_resval_t** applied_vals; + xce_val_t* stored; + int rc; + + setup_resmgr(); + applied_vals = (xce_resval_t**)calloc((size_t)2, sizeof(xce_resval_t*)); + TEST_ASSERT_NOT_NULL(applied_vals); + applied.vals = applied_vals; + + l = make_resval_list(names, values, 2); + rc = update_resval_list(&g_te.c, NULL, l, &applied); + TEST_ASSERT_EQUAL_INT(0, rc); /* list-level call still succeeds */ + + /* the SHARED entry was applied and reported... */ + stored = (xce_val_t*)zarray_get(&g_te.c.sharedinst, 0); + TEST_ASSERT_NOT_NULL(stored); + TEST_ASSERT_EQUAL_FLOAT(4.0, stored->u.fvalue); + TEST_ASSERT_EQUAL_INT(1, applied.count); + TEST_ASSERT_EQUAL_STRING("lic", applied.vals[0]->name); + + /* ...the non-SHARED entry ("slots", first in the list) was refused and + * NOT reported as applied */ + TEST_ASSERT_EQUAL_STRING("slots", l->vals[0]->name); /* sanity */ + TEST_ASSERT_EQUAL_INT(1, applied.count); + + teardown_resmgr(); + free(applied_vals); +} + +/* ── suite entry point ─────────────────────────────────────────────── */ + +void test_extres_resvals(void) +{ + UnitySetTestFile(__FILE__); + RUN_TEST(test_extres_shared_resval_lands_in_sharedinst); + RUN_TEST(test_extres_nonshared_resval_refused_eperm); +} diff --git a/src/controller/worker.c b/src/controller/worker.c index 26d84252..a69a12e7 100644 --- a/src/controller/worker.c +++ b/src/controller/worker.c @@ -74,6 +74,13 @@ static int update_val_res(cluster_t* c, worker_t* w, xce_resource_t* r, if (r->flags & XCE_RESF_SHARED) { /* update shared resource */ array = &(c->sharedinst); + } else if (w == NULL) { + /* external-license-resources Phase 1: a NULL worker is the extres + * daemon (HostRole_extres join, no workermgr slot). It may only + * write SHARED (cluster-level) resources -- a host-based resource + * has no per-host storage without a worker, and the daemon must + * never fabricate host values. Refuse loudly. */ + return -EPERM; } else { /* update host based resource */ array = &(w->host.resvals); diff --git a/src/esd/include/esd.h b/src/esd/include/esd.h new file mode 100644 index 00000000..0b367f72 --- /dev/null +++ b/src/esd/include/esd.h @@ -0,0 +1,90 @@ +#if !defined(_ESD_H_) + +#define _ESD_H_ 1 + +#include +#include +#include + +#include "config.h" +#include "client.h" +#include "xce/types.h" + +/* + * xceesd -- XCE External-resources Daemon (external-license-resources + * Phase 1, design.md 3.1). Joins the controller as HostRole_extres and + * pushes cluster-level FREE values for EXTERNAL shared resources into the + * existing resval pipeline (OPCode_update / Update_data_resvals, host=="" + * -- the sharedInst landing shape). + * + * Collectors are plugins over one internal interface; Phase 1 ships the + * script collector (any external resource: license-free wrappers, cloud + * quotas). The flexlm collector is the same interface over `lmstat -a` + * output parsing. + */ + +/* One configured collector: a name (must match a resources.yaml + * SHARED|GLOBAL|EXTERNAL resource) and how to get its value. */ +typedef struct esd_source_s { + char* name; /* resource name, same as resources.yaml */ + char* command; /* script collector: the command to run */ + int interval; /* poll interval, seconds (0: use the daemon default) */ + double last_value; /* last successfully-parsed value */ + bool have_value; /* last_value is valid (any push has succeeded) */ + bool last_failed; /* last poll failed (stale-value policy input) */ +} esd_source_t; + +typedef struct esd_config_s { + char* controller_host; /* controller address (legacy single) */ + int controller_port; + char* peers; /* --peers style candidate list, or NULL */ + int poll_interval; /* daemon default poll interval, seconds */ + int nsrc; + esd_source_t* src; /* array of nsrc sources */ +} esd_config_t; + +/* + * Parse esd.yaml: + * + * controller: 172.31.1.30:9990 # or peers: "h1:9990,h2:9990" + * poll_interval: 30 + * resources: + * - name: Verilog_XL + * collector: script + * command: /etc/xce/lic-report.sh + * interval: 15 # optional per-source override + * + * Returns 0 on success and fills *cfg_out (caller frees with + * esd_config_fini). Returns -errno-shaped negatives on failure: -EINVAL + * for a structurally-invalid file, -ENOENT for a missing one. Unknown + * keys are ignored (forward-compatible, same convention as the + * controller's own YAML loader). + */ +extern int esd_config_load(const char* path, esd_config_t* cfg_out); +extern void esd_config_fini(esd_config_t* cfg); + +/* + * Run one source's collector command and parse its output. The script + * collector contract (design.md 3.4): the command prints either + * " " (elim-style) or "" (a bare number), one line; + * leading/trailing whitespace is ignored. stdout only -- stderr is + * forwarded to the log untouched. + * + * Returns the parsed value, or NAN when the command failed or its output + * did not parse (caller decides the stale-value policy -- the daemon's + * default is keep-last-value + WARN, fail-open, design.md 3.4). + * + * posix_spawn(), not fork(): the daemon is multi-threaded (the client + * library runs its own transport thread), and a forked child that logged + * or allocated before exec could deadlock on a lock another thread held + * (tech.md "Process Creation"). The child gets an empty signal mask, + * default signal dispositions, and waits with a watchdog kill at + * esd_script_timeout_ms. + */ +extern double esd_script_collect(const char* command); + +/* Script-collector watchdog: how long a collection command may run before + * it is killed (default 10s; main.c overrides from the CLI). */ +extern int esd_script_timeout_ms; + +#endif diff --git a/src/esd/meson.build b/src/esd/meson.build new file mode 100644 index 00000000..3e878a83 --- /dev/null +++ b/src/esd/meson.build @@ -0,0 +1,41 @@ +# xceesd -- XCE External-resources Daemon (external-license-resources +# Phase 1, design.md 3.1). Joins the controller as HostRole_extres and +# pushes cluster-level FREE values into the existing resval pipeline. + +esd_srcs = [ + 'src/esd_config.c', + 'src/esd_script.c', + 'src/esd.c', +] + +esd_deps = [ + libxce_dep, + yaml_deps, + tinylog_dep, +] + +esd_incdir = include_directories('include', '../include', '../lib') + +xceesd = executable('xceesd', + esd_srcs + ['src/main.c'], + include_directories: [esd_incdir], + dependencies: esd_deps, + install: true) + +# Unit tests: white-box #include of the pure-core sources (config parse, +# script-output parse) -- same convention as the worker's +# resource_sync_test. The join/reconnect wiring (esd.c) drives the live +# client library and is covered by the t1 E2E, not unit tests. +esd_test_incdir = include_directories('include', '../include', '../lib', '.') +esd_test_deps = [libxce_dep, yaml_deps, tinylog_dep, unity_dep] + +esd_test = executable('esd_test', + ['testsuite/test_esd_core.c'], + include_directories: [esd_test_incdir], + dependencies: esd_test_deps, + install: false) + +test('esd core test', esd_test, + suite: 'esd', + timeout: 60, + env: ['ASAN_OPTIONS=detect_leaks=0']) diff --git a/src/esd/src/esd.c b/src/esd/src/esd.c new file mode 100644 index 00000000..abdad1b5 --- /dev/null +++ b/src/esd/src/esd.c @@ -0,0 +1,288 @@ +/* esd.c -- the xceesd daemon body (external-license-resources Phase 1, + * design.md 3.1): join the controller as HostRole_extres, poll the + * configured collectors, and push every refreshed shared value as a + * host=="" resval delta -- the sharedInst landing shape the controller's + * opupd extres branch and the scheduler's DELTA sharedInst route expect. + * + * Reconnect semantics (design.md 3.5 step 6, "重连即全量重推"): every + * (re)join is followed by a FULL push of every source's last-known value + * -- the same contract the worker's job-state resync re-report has + * (9d7628e): the controller keeps no durable state for this daemon, so a + * fresh controller (boot or failover promotion) must be told everything + * before the next poll tick. + */ +#include +#include +#include +#include + +#include "esd.h" +#include "tlog.h" +#include "xce/cluster.h" + +/* The daemon's client-library context and identity (from main.c's CLI). */ +typedef struct esd_daemon_s { + xcecli_ctx_t apictx; + esd_config_t cfg; + const char* from; /* join identity ("xceesd") */ + int running; /* 0 once the main loop should exit */ + xce_cluster_config_t cluster_cfg; /* validated peers list (HA mode) */ +} esd_daemon_t; + +static esd_daemon_t g_esd; + +int esd_push_value(const char* name, double value) +{ + xce_resval_t resval; + xce_resval_t* valptr = &resval; /* the wire takes an array of POINTERS + * (xce_resval_t**, typemap.h) -- passing + * &resval itself made the encoder read + * the struct's first word as the next + * element pointer and strlen() garbage + * (live segfault, 2026-10-26) */ + xce_val_t val; + int rc; + + memset(&resval, 0, sizeof(resval)); + memset(&val, 0, sizeof(val)); + + val.which = Value_u_fvalue; + val.u.fvalue = value; + + /* host == "" is THE landing-shape marker: the controller's opupd + * extres branch drops the worker lookup entirely and routes the entry + * to update_val_res's SHARED branch (c->sharedinst), and the + * scheduler's DELTA path treats host==""/NULL as sharedInst (mirrors + * snapshot_load.c's bug-1923 precedent). */ + resval.name = (char*)name; + resval.host = (char*)""; + resval.val = &val; + + rc = xcecli_update_resvals(g_esd.apictx, 1, &valptr); + if (rc < 0) { + ERROR("esd push %s=%f failed (rc=%d)", name, value, rc); + } else { + TRACE("esd pushed %s=%f", name, value); + } + + return rc; +} + +/* One poll pass: collect every source, remember the last good value, and + * push only what CHANGED (the debounce -- design.md step 6 "轮询去抖"). + * A failed collection keeps the previous value and does not push (fail- + * open: the controller keeps scheduling against the last known good + * count; design.md 3.4 stale policy default). */ +int esd_poll_round(void) +{ + int i; + int pushed = 0; + + for (i = 0; i < g_esd.cfg.nsrc; i++) { + esd_source_t* src = &(g_esd.cfg.src[i]); + double v; + + v = esd_script_collect(src->command); + + if (isnan(v)) { + if (!src->last_failed) { + WARN("source %s: collection failed -- keeping last known " + "value %f (fail-open)", + src->name, src->have_value ? src->last_value : 0.0); + } + src->last_failed = true; + continue; + } + + src->last_failed = false; + + if (src->have_value && (src->last_value == v)) { + continue; /* unchanged -- the debounce */ + } + + if (esd_push_value(src->name, v) == 0) { + src->have_value = true; + src->last_value = v; + pushed++; + } + } + + return pushed; +} + +/* Full re-push of every source's last-known value (reconnect semantics: + * the controller we just joined knows nothing about us). Sources with no + * value yet are skipped -- nothing to push, the poll round will fill + * them. */ +int esd_push_all(void) +{ + int i; + int pushed = 0; + + for (i = 0; i < g_esd.cfg.nsrc; i++) { + esd_source_t* src = &(g_esd.cfg.src[i]); + + if (!src->have_value) { + continue; + } + + if (esd_push_value(src->name, src->last_value) == 0) { + pushed++; + } + } + + INFO("esd full re-push: %d/%d source(s)", pushed, g_esd.cfg.nsrc); + + return pushed; +} + +/* --rejoin hook (client library contract): the library calls this while + * holding its failover lock -- no I/O, no xcecli_* calls (would self- + * deadlock, the worker's xcecd_on_reconnect lesson, controller_connector.c). + * We only set a flag; the main loop consumes it after the lock is free. */ +static volatile int g_needs_full_push = 0; + +static int esd_on_reconnect(xcecli_ctx_t ctx, void* arg) +{ + (void)ctx; + (void)arg; + + g_needs_full_push = 1; + + return 0; +} + +int esd_run(void) +{ + int next_full_push = 1; /* the FIRST join needs the full push too */ + + /* Join sequence mirrors the worker's xcecd_join_cluster (controller_ + * connector.c): build ONE validated candidate list -- peers from + * esd.yaml when given, else the single controller host:port -- and + * hand it to the client library, which owns failover rotation and + * standby redirects inside the single xcecli_join call. */ + { + xce_client_config_t cconfig; + + memset(&cconfig, 0, sizeof(cconfig)); + + if (g_esd.cfg.peers != NULL) { + int rc = xce_parse_peers(g_esd.cfg.peers, &g_esd.cluster_cfg); + if (rc < 0) { + ERROR("esd: cannot parse peers \"%s\"", g_esd.cfg.peers); + return -EINVAL; + } + strncpy(g_esd.cluster_cfg.identity, g_esd.from, + sizeof(g_esd.cluster_cfg.identity) - 1); + rc = xce_cluster_config_validate(&g_esd.cluster_cfg); + if (rc < 0) { + ERROR("esd: invalid peers config (rc=%d)", rc); + return rc; + } + rc = xce_cluster_config_to_client_config(&g_esd.cluster_cfg, + &cconfig); + if (rc < 0) { + ERROR("esd: candidate list conversion failed (rc=%d)", rc); + return rc; + } + } else { + cconfig.nmasters = 1; + cconfig.masters[0] = g_esd.cfg.controller_host; + cconfig.master_ports[0] = g_esd.cfg.controller_port; + } + + cconfig.from = (char*)g_esd.from; + cconfig.key = (char*)"keepsecret$"; + + { + int rc = xcecli_ctx_init(&g_esd.apictx, &cconfig); + if (rc < 0) { + ERROR("esd: client ctx init failed (rc=%d)", rc); + return rc; + } + } + + if (xcecli_start(g_esd.apictx) < 0) { + ERROR("esd: cannot reach controller %s:%d", + g_esd.cfg.controller_host, g_esd.cfg.controller_port); + xcecli_ctx_fini(g_esd.apictx); + return -ECONNREFUSED; + } + + /* AFTER xcecli_start(): start() resets rejoin_cb (the worker's + * task-2.3 lesson, controller_connector.c). */ + xcecli_set_reconnect_handler(g_esd.apictx, esd_on_reconnect, NULL); + + { + int rc = xcecli_join(g_esd.apictx, HostRole_extres, NULL); + if (rc < 0) { + ERROR("esd: join failed (rc=%d)", rc); + xcecli_stop(g_esd.apictx); + xcecli_ctx_fini(g_esd.apictx); + return rc; + } + } + } + + INFO("esd: joined cluster as %s (%d source(s))", g_esd.from, + g_esd.cfg.nsrc); + + while (g_esd.running) { + int i; + + if (g_needs_full_push) { + /* A transparent reconnect just landed (new primary after + * failover, or the old controller restarted): it has no + * record of this daemon -- push everything we know. */ + g_needs_full_push = 0; + next_full_push = 0; + (void)esd_push_all(); + } + + if (next_full_push) { + next_full_push = 0; + (void)esd_push_all(); + } + + /* bug (live sighting 2026-10-26): with every source unchanged, the + * debounce suppresses ALL pushes -- and a reconnect only ever fires + * INSIDE a push's failover call. A controller restart while our + * value sits still left the daemon idle on a dead connection + * forever (no socket() for 50+ minutes, pushes never resumed). + * The broken-connection flag is set asynchronously by the + * transport's RDHUP handler (bug-1760's pure predicate): when it + * latches, force one full re-push -- the write attempt triggers the + * rotate/reconnect, and esd_on_reconnect's flag keeps the full + * re-push contract for the failover path too. */ + if (g_esd.cfg.nsrc > 0 && xcecli_is_connection_broken(g_esd.apictx)) { + INFO("esd: connection broken -- forcing a full re-push to " + "reconnect"); + (void)esd_push_all(); + } + + (void)esd_poll_round(); + + /* sleep in 1s slices so a signal stop lands promptly */ + for (i = 0; (i < g_esd.cfg.poll_interval) && g_esd.running; i++) { + sleep(1); + } + } + + xcecli_stop(g_esd.apictx); + xcecli_ctx_fini(g_esd.apictx); + + return 0; +} + +void esd_stop(void) +{ + g_esd.running = 0; +} + +void esd_init(esd_config_t* cfg, const char* from) +{ + memset(&g_esd, 0, sizeof(g_esd)); + g_esd.cfg = *cfg; + g_esd.from = from; + g_esd.running = 1; +} diff --git a/src/esd/src/esd_config.c b/src/esd/src/esd_config.c new file mode 100644 index 00000000..5c8936bd --- /dev/null +++ b/src/esd/src/esd_config.c @@ -0,0 +1,286 @@ +/* esd_config.c -- esd.yaml parsing for xceesd (external-license-resources + * Phase 1, design.md 3.4). libcyaml, same loader pattern as the + * autoscaler's prov_conf.c: schema-driven, unknown keys ignored + * (CYAML_CFG_IGNORE_UNKNOWN_KEYS), explicit field validation after load + * (a source without name or command can never report -- fail loudly at + * load, not silently at runtime). */ +#include +#include +#include +#include + +#include + +#include "esd.h" + +typedef struct yaml_esd_source_s { + char* name; + char* collector; + char* command; + long interval; /* 0 = daemon default */ +} yaml_esd_source_t; + +typedef struct yaml_esd_config_s { + char* controller; + char* peers; + long poll_interval; + yaml_esd_source_t* resources; + unsigned resources_count; +} yaml_esd_config_t; + +static const cyaml_schema_field_t esd_source_fields_schema[] = { + CYAML_FIELD_STRING_PTR("name", CYAML_FLAG_POINTER, yaml_esd_source_t, + name, 1, CYAML_UNLIMITED), + CYAML_FIELD_STRING_PTR("collector", CYAML_FLAG_POINTER, yaml_esd_source_t, + collector, 1, CYAML_UNLIMITED), + CYAML_FIELD_STRING_PTR("command", CYAML_FLAG_POINTER, yaml_esd_source_t, + command, 1, CYAML_UNLIMITED), + CYAML_FIELD_INT("interval", CYAML_FLAG_OPTIONAL, yaml_esd_source_t, + interval), + CYAML_FIELD_END +}; + +static const cyaml_schema_value_t esd_source_schema = { + CYAML_VALUE_MAPPING(CYAML_FLAG_DEFAULT, yaml_esd_source_t, + esd_source_fields_schema) +}; + +static const cyaml_schema_field_t esd_config_fields_schema[] = { + CYAML_FIELD_STRING_PTR("controller", CYAML_FLAG_OPTIONAL, + yaml_esd_config_t, controller, 1, CYAML_UNLIMITED), + CYAML_FIELD_STRING_PTR("peers", CYAML_FLAG_OPTIONAL, yaml_esd_config_t, + peers, 1, CYAML_UNLIMITED), + CYAML_FIELD_INT("poll_interval", CYAML_FLAG_OPTIONAL, yaml_esd_config_t, + poll_interval), + CYAML_FIELD_SEQUENCE("resources", CYAML_FLAG_POINTER, yaml_esd_config_t, + resources, &esd_source_schema, 0, CYAML_UNLIMITED), + CYAML_FIELD_END +}; + +static const cyaml_schema_value_t esd_config_schema = { + CYAML_VALUE_MAPPING(CYAML_FLAG_POINTER, yaml_esd_config_t, + esd_config_fields_schema) +}; + +/* "host:port" -> (host, port); returns 0 on success, -EINVAL when port + * missing/non-numeric/out of range. Mutates `spec` (splits at the colon). */ +static int esd_parse_endpoint(char* spec, char** host_out, int* port_out) +{ + char* colon; + char* endp = NULL; + long port; + + if ((spec == NULL) || (*spec == '\0')) { + return -EINVAL; + } + + colon = strrchr(spec, ':'); + if ((colon == NULL) || (colon[1] == '\0')) { + return -EINVAL; + } + + port = strtol(colon + 1, &endp, 10); + if ((endp == NULL) || (*endp != '\0') || (port <= 0) || (port > 65535)) { + return -EINVAL; + } + + *colon = '\0'; + *host_out = spec; + *port_out = (int)port; + + return 0; +} + +/* File-content variant, split out from the file loader for testability: + * the tests run esd_config_load_path() against real files, so both layers + * stay production code. */ +int esd_config_load_path(const char* dir, const char* name, + esd_config_t* cfg_out) +{ + static cyaml_config_t cyaml_cfg = { + .mem_fn = cyaml_mem, + .log_level = CYAML_LOG_WARNING, + .flags = CYAML_CFG_IGNORE_UNKNOWN_KEYS, + }; + + yaml_esd_config_t* ycfg = NULL; + cyaml_err_t err; + char* path = NULL; + esd_config_t cfg; + int i; + int rc; + + if ((dir == NULL) || (name == NULL) || (cfg_out == NULL)) { + return -EINVAL; + } + + memset(cfg_out, 0, sizeof(*cfg_out)); + memset(&cfg, 0, sizeof(cfg)); + + if (asprintf(&path, "%s/%s", dir, name) < 0) { + return -ENOMEM; + } + + err = cyaml_load_file(path, &cyaml_cfg, &esd_config_schema, + (void**)&ycfg, NULL); + free(path); + if (err != CYAML_OK) { + /* libcyaml collapses file-not-found into generic errors; map the + * obvious one so the daemon's startup log can say "missing file" + * vs "bad file" honestly. */ + char probe[512]; + FILE* f; + + snprintf(probe, sizeof(probe), "%s/%s", dir, name); + f = fopen(probe, "r"); + if (f == NULL) { + return -ENOENT; + } + fclose(f); + return -EINVAL; + } + + if (ycfg == NULL) { + return -EINVAL; + } + + /* Validation first: no controller target and no peers list means the + * daemon could never connect to anything. */ + if ((ycfg->controller == NULL) && (ycfg->peers == NULL)) { + cyaml_free(&cyaml_cfg, &esd_config_schema, ycfg, 0); + return -EINVAL; + } + + if (ycfg->controller != NULL) { + char* host = NULL; + int port = 0; + + if (esd_parse_endpoint(ycfg->controller, &host, &port) < 0) { + cyaml_free(&cyaml_cfg, &esd_config_schema, ycfg, 0); + return -EINVAL; + } + + /* esd_parse_endpoint splits ycfg->controller in place -- the host + * pointer points INTO the cyaml-owned string. Copy out before the + * cyaml_free below releases the whole decoded tree. */ + cfg.controller_host = strdup(host); + cfg.controller_port = port; + if (cfg.controller_host == NULL) { + cyaml_free(&cyaml_cfg, &esd_config_schema, ycfg, 0); + return -ENOMEM; + } + } + + cfg.peers = (ycfg->peers != NULL) ? strdup(ycfg->peers) : NULL; + cfg.poll_interval = (int)ycfg->poll_interval; + cfg.nsrc = (int)ycfg->resources_count; + cfg.src = NULL; + + if (cfg.nsrc > 0) { + cfg.src = (esd_source_t*)calloc((size_t)cfg.nsrc, + sizeof(esd_source_t)); + if (cfg.src == NULL) { + esd_config_fini(&cfg); + cyaml_free(&cyaml_cfg, &esd_config_schema, ycfg, 0); + return -ENOMEM; + } + } + + rc = 0; + for (i = 0; i < cfg.nsrc; i++) { + yaml_esd_source_t* ys = &(ycfg->resources[i]); + + if ((ys->name == NULL) || (ys->command == NULL)) { + rc = -EINVAL; + break; + } + + cfg.src[i].name = strdup(ys->name); + cfg.src[i].command = strdup(ys->command); + cfg.src[i].interval = (int)ys->interval; + + if ((cfg.src[i].name == NULL) || (cfg.src[i].command == NULL)) { + rc = -ENOMEM; + break; + } + } + + cyaml_free(&cyaml_cfg, &esd_config_schema, ycfg, 0); + + if (rc < 0) { + esd_config_fini(&cfg); + return rc; + } + + *cfg_out = cfg; + + return 0; +} + +int esd_config_load(const char* path, esd_config_t* cfg_out) +{ + char* dir; + char* name; + char* slash; + int rc; + + if ((path == NULL) || (cfg_out == NULL)) { + return -EINVAL; + } + + dir = strdup(path); + if (dir == NULL) { + return -ENOMEM; + } + + slash = strrchr(dir, '/'); + if (slash == NULL) { + name = dir; + dir = strdup("."); + if (dir == NULL) { + free(name); + return -ENOMEM; + } + } else { + *slash = '\0'; + name = slash + 1; + if (*name == '\0') { + free(dir); + return -EINVAL; + } + if (dir == name - 1) { + /* path was "/name" -- dir is "" */ + free(dir); + dir = strdup("/"); + if (dir == NULL) { + return -ENOMEM; + } + } + } + + rc = esd_config_load_path(dir, name, cfg_out); + + free(dir); + + return rc; +} + +void esd_config_fini(esd_config_t* cfg) +{ + int i; + + if (cfg == NULL) { + return; + } + + free(cfg->controller_host); + free(cfg->peers); + + for (i = 0; i < cfg->nsrc; i++) { + free(cfg->src[i].name); + free(cfg->src[i].command); + } + free(cfg->src); + + memset(cfg, 0, sizeof(*cfg)); +} diff --git a/src/esd/src/esd_script.c b/src/esd/src/esd_script.c new file mode 100644 index 00000000..fc1827b4 --- /dev/null +++ b/src/esd/src/esd_script.c @@ -0,0 +1,230 @@ +/* esd_script.c -- the script collector for xceesd (external-license- + * resources Phase 1, design.md 3.1): runs a user command with posix_spawn, + * reads its stdout, and parses "" or " " from it. + * + * posix_spawn(), not fork(): the daemon is multi-threaded (the client + * library runs its own transport thread), and a forked child that logged + * or allocated before exec could deadlock on a lock another thread held + * (tech.md "Process Creation" -- same rationale as the autoscaler's + * plug_mgr.c run_script). The child starts with an empty signal mask and + * default signal dispositions. + * + * The watchdog: the pipe read is bounded by esd_script_timeout_ms -- + * a hung collection command must not stall the poll loop (the value it + * was going to report belongs to a dead world anyway). We enforce it + * with poll(2) on the read end + a kill() at deadline; waitpid reaps. + */ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "esd.h" +#include "tlog.h" + +/* Script-collector watchdog, seconds of wall clock a collection command + * may run before it is killed (design.md 3.4: the poll loop must survive + * a hung script). Overridable from main.c. */ +int esd_script_timeout_ms = 10 * 1000; + +/* + * esd_parse_script_output -- the collector contract (design.md 3.4): the + * first non-empty line carries either a bare number or " ". + * Leading/trailing whitespace ignored; anything else is garbage -> NAN + * (the caller owns the stale-value policy). + */ +double esd_parse_script_output(const char* out) +{ + const char* p; + char* endp = NULL; + double v; + + if (out == NULL) { + return NAN; + } + + p = out; + while (*p != '\0') { + /* skip horizontal whitespace and newlines */ + while ((*p == ' ') || (*p == '\t') || (*p == '\r') || (*p == '\n')) { + p++; + } + if (*p == '\0') { + break; + } + + v = strtod(p, &endp); + if ((endp != NULL) && (endp != p)) { + /* accept trailing junk after the number (a name may follow in + * "value name" order too -- the number is what we need) */ + return v; + } + + /* skip this non-numeric token and try the next one */ + while ((*p != '\0') && (*p != ' ') && (*p != '\t') && (*p != '\r') + && (*p != '\n')) { + p++; + } + } + + return NAN; +} + +/* The argv for the child: run the command via sh -c, so a plain + * "/path/script.sh arg1 arg2" line works without the daemon ever + * splitting shell words itself (the operator owns quoting). */ +static int esd_script_spawn(const char* command, int out_fd, pid_t* pid_out) +{ + posix_spawn_file_actions_t fa; + posix_spawnattr_t attr; + sigset_t none, all; + char* argv[] = { (char*)"sh", (char*)"-c", (char*)command, NULL }; + extern char** environ; + pid_t pid; + int rc; + + sigemptyset(&none); + sigfillset(&all); + + rc = posix_spawn_file_actions_init(&fa); + if (rc == 0) { + rc = posix_spawn_file_actions_adddup2(&fa, out_fd, STDOUT_FILENO); + } + if (rc == 0) { + rc = posix_spawn_file_actions_addopen(&fa, STDERR_FILENO, "/dev/null", + O_WRONLY, 0); + } + if (rc == 0) { + rc = posix_spawnattr_init(&attr); + } + if (rc == 0) { + rc = posix_spawnattr_setflags( + &attr, POSIX_SPAWN_SETSIGMASK | POSIX_SPAWN_SETSIGDEF); + } + if (rc == 0) { + rc = posix_spawnattr_setsigmask(&attr, &none); + } + if (rc == 0) { + rc = posix_spawnattr_setsigdefault(&attr, &all); + } + if (rc == 0) { + rc = posix_spawn(&pid, "/bin/sh", &fa, &attr, argv, environ); + } + posix_spawnattr_destroy(&attr); + posix_spawn_file_actions_destroy(&fa); + + if (rc != 0) { + return -rc; + } + + *pid_out = pid; + + return 0; +} + +double esd_script_collect(const char* command) +{ + int fds[2]; + pid_t pid = -1; + double value = NAN; + char buf[4096]; + size_t total = 0; + int elapsed_ms = 0; + int rc; + int wstatus = 0; + struct pollfd pfd; + + if ((command == NULL) || (*command == '\0')) { + return NAN; + } + + if (pipe(fds) != 0) { + ERROR("script collect: pipe failed (%s)", strerror(errno)); + return NAN; + } + + rc = esd_script_spawn(command, fds[1], &pid); + close(fds[1]); /* parent's copy -- child holds the write end now */ + if (rc < 0) { + close(fds[0]); + ERROR("script collect: spawn failed for \"%s\" (%s)", command, + strerror(-rc)); + return NAN; + } + + /* Bound the read at the watchdog deadline: a script that hangs (a + * blocked lmstat, a wedged NFS mount) must not stall the poll loop. */ + pfd.fd = fds[0]; + pfd.events = POLLIN; + + while (total < sizeof(buf) - 1) { + int pret = poll(&pfd, 1, 100); + + if (pret < 0) { + if (errno == EINTR) { + continue; + } + ERROR("script collect: poll failed (%s)", strerror(errno)); + break; + } + if (pret == 0) { + elapsed_ms += 100; + if (elapsed_ms >= esd_script_timeout_ms) { + WARN("script collect: command \"%s\" exceeded %d ms -- " + "killing it, value not refreshed this round", + command, esd_script_timeout_ms); + kill(pid, SIGKILL); + waitpid(pid, &wstatus, 0); + close(fds[0]); + return NAN; + } + continue; + } + + if (pfd.revents & (POLLERR | POLLHUP)) { + ssize_t n = read(fds[0], buf + total, sizeof(buf) - 1 - total); + if (n > 0) { + total += (size_t)n; + } + break; + } + if (pfd.revents & POLLIN) { + ssize_t n = read(fds[0], buf + total, sizeof(buf) - 1 - total); + if (n <= 0) { + break; /* EOF or error: done reading */ + } + total += (size_t)n; + } + } + + buf[total] = '\0'; + close(fds[0]); + + /* reap regardless of which path brought us here */ + while (waitpid(pid, &wstatus, 0) < 0) { + if (errno != EINTR) { + break; + } + } + + if (total == 0) { + WARN("script collect: command \"%s\" produced no output", command); + return NAN; + } + + value = esd_parse_script_output(buf); + if (isnan(value)) { + WARN("script collect: command \"%s\" output did not parse as a " + "value (got %.60s)", command, buf); + } + + return value; +} diff --git a/src/esd/src/main.c b/src/esd/src/main.c new file mode 100644 index 00000000..1cd1811d --- /dev/null +++ b/src/esd/src/main.c @@ -0,0 +1,127 @@ +/* main.c -- xceesd entry point (external-license-resources Phase 1, + * design.md 3.1). Minimal on purpose: parse the CLI, load esd.yaml, run + * the poll loop until SIGINT/SIGTERM. */ +#include +#include +#include +#include +#include +#include + +#include "esd.h" +#include "tlog.h" + +static char doc[] = "xceesd -- XCE External-resources Daemon"; +static char args_doc[] = ""; + +const char* argp_program_bug_address = BUG_EMAIL; + +struct arguments { + char* logfile; + char* config; + int verbose; + int timeout_ms; +}; + +#define OPT_TIMEOUT 256 + +static struct argp_option options[] = { + { "logfile", 'l', "FILE", 0, "The full path of log file" }, + { "config", 'c', "FILE", 0, + "Path of the daemon configuration file (default: /etc/xce/esd.yaml)" }, + { "verbose", 'v', 0, 0, "Produce verbose log" }, + { "script-timeout", OPT_TIMEOUT, "MS", 0, + "Watchdog for collection commands in milliseconds (default: 10000)" }, + { 0 } +}; + +static error_t parse_opt(int key, char* arg, struct argp_state* state) +{ + struct arguments* state_args = state->input; + + switch (key) { + case 'l': + state_args->logfile = arg; + break; + case 'c': + state_args->config = arg; + break; + case 'v': + state_args->verbose++; + break; + case OPT_TIMEOUT: + state_args->timeout_ms = atoi(arg); + break; + default: + return ARGP_ERR_UNKNOWN; + } + + return 0; +} + +static struct argp argp = { options, parse_opt, args_doc, doc }; + +static esd_config_t g_cfg; + +static void sig_handler(int signo) +{ + (void)signo; + esd_stop(); +} + +int main(int argc, char* argv[]) +{ + struct arguments arguments; + char* logfile = "/tmp/xceesd.log"; + char* loglevel = "info"; + char* config = "/etc/xce/esd.yaml"; + int rc; + + memset(&arguments, 0, sizeof(arguments)); + argp_parse(&argp, argc, argv, 0, 0, &arguments); + + if (arguments.logfile != NULL) { + logfile = arguments.logfile; + } + if (arguments.config != NULL) { + config = arguments.config; + } + if (arguments.verbose > 0) { + loglevel = "debug"; + } + if (arguments.timeout_ms > 0) { + esd_script_timeout_ms = arguments.timeout_ms; + } + + mylog_init(logfile, loglevel, 16 * 1024 * 1024, 4); + + rc = esd_config_load(config, &g_cfg); + if (rc < 0) { + fprintf(stderr, "xceesd: cannot load config %s: %s\n", config, + (rc == -ENOENT) ? "file not found" + : (rc == -EINVAL) ? "invalid file" + : strerror(-rc)); + return 1; + } + + INFO("xceesd starting: config %s, %d source(s), poll %ds", config, + g_cfg.nsrc, g_cfg.poll_interval > 0 ? g_cfg.poll_interval : 30); + + if (g_cfg.poll_interval <= 0) { + g_cfg.poll_interval = 30; + } + + signal(SIGINT, sig_handler); + signal(SIGTERM, sig_handler); + + esd_init(&g_cfg, "xceesd"); + + rc = esd_run(); + + esd_config_fini(&g_cfg); + mylog_fini(); + + INFO("xceesd stopped (rc=%d)", rc); + + return (rc == 0) ? 0 : 1; +} diff --git a/src/esd/testsuite/test_esd_core.c b/src/esd/testsuite/test_esd_core.c new file mode 100644 index 00000000..95f6809b --- /dev/null +++ b/src/esd/testsuite/test_esd_core.c @@ -0,0 +1,201 @@ +/* test_esd_core.c -- external-license-resources Phase 1, step 6 TDD: the + * xceesd skeleton's pure core -- esd.yaml parsing and the script + * collector's output parsing -- white-box #include of the sources under + * test (same pattern as the worker's resource_sync_test / + * controller_connector_test). The join/reconnect wiring lives in esd.c + * against the live client library and is exercised by the t1 E2E, not + * here; everything this file drives must be deterministic. */ +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "esd.h" + +/* White-box: reach the config schema/parse internals. */ +#include "../src/esd_config.c" +/* White-box: the script collector's parse half (esd_parse_script_output); + * the spawn half (esd_script_collect) is exercised by the t1 E2E with a + * real fake-license script, not here. */ +#include "../src/esd_script.c" + +static char g_tmpdir[256]; + +void setUp(void) +{ + snprintf(g_tmpdir, sizeof(g_tmpdir), "/tmp/xce_esd_testXXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(g_tmpdir)); +} + +void tearDown(void) +{ + if (g_tmpdir[0] != '\0') { + char path[300]; + + /* tests only write flat files inside g_tmpdir */ + snprintf(path, sizeof(path), "rm -rf %s", g_tmpdir); + if (system(path) != 0) { + /* best effort */ + } + g_tmpdir[0] = '\0'; + } +} + +static void write_file(const char* dir, const char* name, const char* content) +{ + char path[512]; + FILE* f; + + snprintf(path, sizeof(path), "%s/%s", dir, name); + f = fopen(path, "w"); + TEST_ASSERT_NOT_NULL(f); + fputs(content, f); + fclose(f); +} + +/* ── esd_config_load ─────────────────────────────────────────────────── */ + +static void test_config_load_full_file(void) +{ + esd_config_t cfg; + int rc; + + memset(&cfg, 0, sizeof(cfg)); + + write_file(g_tmpdir, "esd.yaml", + "controller: 172.31.1.30:9990\n" + "poll_interval: 30\n" + "resources:\n" + " - name: Verilog_XL\n" + " collector: script\n" + " command: /etc/xce/lic-report.sh\n" + " interval: 15\n" + " - name: cloud_gpu_quota\n" + " collector: script\n" + " command: /etc/xce/quota-report.sh\n"); + + rc = esd_config_load_path(g_tmpdir, "esd.yaml", &cfg); + TEST_ASSERT_EQUAL_INT(0, rc); + TEST_ASSERT_EQUAL_STRING("172.31.1.30", cfg.controller_host); + TEST_ASSERT_EQUAL_INT(9990, cfg.controller_port); + TEST_ASSERT_EQUAL_INT(30, cfg.poll_interval); + TEST_ASSERT_EQUAL_INT(2, cfg.nsrc); + + TEST_ASSERT_NOT_NULL(cfg.src); + TEST_ASSERT_EQUAL_STRING("Verilog_XL", cfg.src[0].name); + TEST_ASSERT_EQUAL_STRING("/etc/xce/lic-report.sh", cfg.src[0].command); + TEST_ASSERT_EQUAL_INT(15, cfg.src[0].interval); + TEST_ASSERT_EQUAL_STRING("cloud_gpu_quota", cfg.src[1].name); + TEST_ASSERT_EQUAL_STRING("/etc/xce/quota-report.sh", cfg.src[1].command); + TEST_ASSERT_EQUAL_INT(0, cfg.src[1].interval); /* no override */ + + esd_config_fini(&cfg); +} + +static void test_config_load_missing_file_is_enoent(void) +{ + esd_config_t cfg; + int rc; + + memset(&cfg, 0, sizeof(cfg)); + + rc = esd_config_load_path(g_tmpdir, "no-such-file.yaml", &cfg); + TEST_ASSERT_EQUAL_INT(-ENOENT, rc); +} + +static void test_config_load_bad_shape_is_einval(void) +{ + esd_config_t cfg; + int rc; + + memset(&cfg, 0, sizeof(cfg)); + + /* a source without a name is structurally invalid -- the controller + * would refuse the push and the operator would have no idea which + * entry died; fail loudly at load. */ + write_file(g_tmpdir, "esd.yaml", + "controller: 172.31.1.30:9990\n" + "resources:\n" + " - collector: script\n" + " command: /bin/true\n"); + + rc = esd_config_load_path(g_tmpdir, "esd.yaml", &cfg); + TEST_ASSERT_EQUAL_INT(-EINVAL, rc); + + /* a source without a command can never report -- same rule. */ + memset(&cfg, 0, sizeof(cfg)); + write_file(g_tmpdir, "esd.yaml", + "controller: 172.31.1.30:9990\n" + "resources:\n" + " - name: lic\n" + " collector: script\n"); + + rc = esd_config_load_path(g_tmpdir, "esd.yaml", &cfg); + TEST_ASSERT_EQUAL_INT(-EINVAL, rc); +} + +/* ── script collector output parsing (parse contract, not the spawn) ── */ + +static void test_parse_output_bare_number(void) +{ + double v = esd_parse_script_output("4\n"); + + TEST_ASSERT_FALSE(isnan(v)); + TEST_ASSERT_EQUAL_FLOAT((float)4.0, (float)v); +} + +static void test_parse_output_name_value(void) +{ + /* elim-style " " -- name may mismatch; the source it is + * configured for is authoritative, the name is decorative. */ + double v = esd_parse_script_output("Verilog_XL 7\n"); + + TEST_ASSERT_FALSE(isnan(v)); + TEST_ASSERT_EQUAL_FLOAT((float)7.0, (float)v); +} + +static void test_parse_output_whitespace_and_negative(void) +{ + double v = esd_parse_script_output(" \t 3.5 \n"); + + TEST_ASSERT_FALSE(isnan(v)); + TEST_ASSERT_EQUAL_FLOAT((float)3.5, (float)v); + + v = esd_parse_script_output("lic -1\n"); + TEST_ASSERT_FALSE(isnan(v)); + TEST_ASSERT_EQUAL_FLOAT((float)-1.0, (float)v); +} + +static void test_parse_output_garbage_is_nan(void) +{ + double v; + + v = esd_parse_script_output(""); + TEST_ASSERT_TRUE(isnan(v)); + v = esd_parse_script_output("\n"); + TEST_ASSERT_TRUE(isnan(v)); + v = esd_parse_script_output("no digits here\n"); + TEST_ASSERT_TRUE(isnan(v)); + v = esd_parse_script_output(NULL); + TEST_ASSERT_TRUE(isnan(v)); +} + +int main(void) +{ + UNITY_BEGIN(); + + RUN_TEST(test_config_load_full_file); + RUN_TEST(test_config_load_missing_file_is_enoent); + RUN_TEST(test_config_load_bad_shape_is_einval); + RUN_TEST(test_parse_output_bare_number); + RUN_TEST(test_parse_output_name_value); + RUN_TEST(test_parse_output_whitespace_and_negative); + RUN_TEST(test_parse_output_garbage_is_nan); + + return UNITY_END(); +} diff --git a/src/include/xce/cluster_def.h b/src/include/xce/cluster_def.h index 5397d63d..1a337279 100644 --- a/src/include/xce/cluster_def.h +++ b/src/include/xce/cluster_def.h @@ -27,6 +27,7 @@ #define N_CONTROLLER_MAX (16) #define N_APISERVER_MAX (16) +#define N_EXTRES_MAX (8) enum { XCE_CONTROLLER_HOOK_JOB = 0, @@ -96,6 +97,18 @@ typedef struct { endpoint_t* ep; } cluster_apiserver_t; +/* external-license-resources Phase 1: connected extres daemons (xceesd). + * Registered at HostRole_extres join like the apiserver table -- endpoint + + * name only, no workermgr slot (the daemon is not a schedulable host). The + * controller never pushes TO the daemon (the daemon pulls nothing and + * re-pushes its full value set on every reconnect), so this table exists + * for leave-cleanup and future admin surface (xceadmin extres status). */ +typedef struct { + mylock_t lock; + char* name; + endpoint_t* ep; +} cluster_extres_t; + typedef struct cluster_s { myconf_t* myconf; xce_peerinfo_t* myhost; /* points at myhost_info once initialised */ @@ -108,6 +121,7 @@ typedef struct cluster_s { zarray_t sharedinst; controller_t controllers[N_CONTROLLER_MAX]; cluster_apiserver_t apiservers[N_APISERVER_MAX]; + cluster_extres_t extresds[N_EXTRES_MAX]; int myidx; int master; int ncontrollers; diff --git a/src/lib/request.c b/src/lib/request.c index 92902955..e94490e9 100644 --- a/src/lib/request.c +++ b/src/lib/request.c @@ -1414,7 +1414,8 @@ int xce_join_req(endpoint_t* ep, int role, hostattrs_t* attrs) if ((role != HostRole_client) && (role != HostRole_worker) && (role != HostRole_apiserver) && - (role != HostRole_scheduler)) { + (role != HostRole_scheduler) && + (role != HostRole_extres)) { return -EINVAL; } /* clang-format on */ diff --git a/src/meson.build b/src/meson.build index 6d3b0a34..a511b477 100644 --- a/src/meson.build +++ b/src/meson.build @@ -76,6 +76,7 @@ if component != 'xcecli' subdir('controller') subdir('worker') subdir('autoscaler') + subdir('esd') subdir('scheduler') subdir('frontend/apiserver') subdir('xceadmin') @@ -88,7 +89,7 @@ if component != 'xcecli' find_program('python3'), args: [meson.project_source_root() / '..' / 'scripts' / 'check-no-fork.py', meson.project_source_root() / '..'], - suite: ['lib', 'controller', 'scheduler', 'worker']) + suite: ['lib', 'controller', 'scheduler', 'worker', 'esd']) configure_file( input: 'tools/appimage/entry.sh', diff --git a/src/scheduler/src/core_events.c b/src/scheduler/src/core_events.c index 156241a5..836d8ed1 100644 --- a/src/scheduler/src/core_events.c +++ b/src/scheduler/src/core_events.c @@ -1234,16 +1234,27 @@ int xce_core_push_events(xce_sched_core_t* core, xce_core_event_t* events, } for (r = 0; r < deltas->count; r++) { - xce_resval_t* rv = deltas->vals[r]; - xce_sched_worker_t* w = NULL; - xce_resource_t* res = NULL; + xce_resval_t* rv = deltas->vals[r]; + xce_sched_worker_t* w = NULL; + xce_resource_t* res = NULL; + int is_shared; if (rv == NULL) { continue; } - w = sched_get_worker(core, rv->host); - if (w == NULL) { + /* external-license-resources Phase 1: a NULL/"" host marks a + * SHARED (cluster-level) value -- the extres daemon pushes + * license free counts this way (worker pushes always name a + * host). Land it in core->sharedInst, mirroring + * snapshot_load()'s host==NULL/"" precedent (bug-1923). + * Before this rule the empty host went through + * sched_get_worker() -> NULL -> WARN-skip, so daemon-pushed + * shared values never reached the scheduler. */ + is_shared = (rv->host == NULL) || (rv->host[0] == '\0'); + + w = is_shared ? NULL : sched_get_worker(core, rv->host); + if (!is_shared && (w == NULL)) { WARN("%s RESVALS_DELTA: unknown worker %s, skipping", __func__, rv->host); continue; @@ -1256,6 +1267,19 @@ int xce_core_push_events(xce_sched_core_t* core, xce_core_event_t* events, continue; } + if (is_shared) { + /* bug-1585 ownership rule: dup_val(), not a shallow + * rv->val store -- same hazard as the per-worker store + * below. */ + zarray_put(&(core->sharedInst), res->idx, dup_val(rv->val)); + if ((rv->val != NULL) + && (rv->val->which == Value_u_fvalue)) { + TRACE("resval applied host=%s resource=%s value=%f", + "(shared)", res->name, rv->val->u.fvalue); + } + continue; + } + /* bug-1585: dup_val(), not a shallow `rv->val` store -- see * this file's own top-of-file comment on the `dup_val` * extern declaration for the full ownership hazard this diff --git a/src/scheduler/src/resource.c b/src/scheduler/src/resource.c index a8945126..0cce8a42 100644 --- a/src/scheduler/src/resource.c +++ b/src/scheduler/src/resource.c @@ -159,11 +159,26 @@ static int unsatisfy_(xce_sched_core_t* core, int nres, int* idxs, float* total, count = 0; for (i = 0; i < nres; i++) { - float uncomit = uncomit_at_(uncomitted, idxs[i]); + float uncomit = uncomit_at_(uncomitted, idxs[i]); + xce_resource_t* r = NULL; + int is_external; - if (total[i] < inuse[i] + uncomit + wanted[i]) { - TRACE("%s resource %d total %f inuse %f uncomit %f wanted %f", - __func__, idxs[i], total[i], inuse[i], uncomit, wanted[i]); + /* external-license-resources: an EXTERNAL resource's pushed value + * is already a FREE count (license server's remaining, net of both + * external consumers and our own checked-out jobs), so re-subtracting + * the scheduler's own usage bookkeeping (sharedInUse / w->resInUse) + * would count our usage twice. Skip the inuse term for these: + * EXTERNAL: total >= uncomit + wanted + * non-EXTERNAL: total >= inuse + uncomit + wanted + * (r->flags is fetched here for the trace line's resource name too -- + * the flag was already reachable in this loop's vget). */ + vget(core->resmgr, idxs[i], (void**)&r); + is_external = (r != NULL) && (r->flags & XCE_RESF_EXTERNAL); + + if (total[i] < ((is_external ? 0.0f : inuse[i]) + uncomit + wanted[i])) { + TRACE("%s resource %d total %f inuse %f uncomit %f wanted %f%s", + __func__, idxs[i], total[i], is_external ? 0.0f : inuse[i], + uncomit, wanted[i], is_external ? " (external, inuse skipped)" : ""); count++; } @@ -175,13 +190,17 @@ static int unsatisfy_(xce_sched_core_t* core, int nres, int* idxs, float* total, j = 0; for (i = 0; i < nres; i++) { - float uncomit = uncomit_at_(uncomitted, idxs[i]); + float uncomit = uncomit_at_(uncomitted, idxs[i]); + xce_resource_t* r = NULL; + int is_external; - if (total[i] < inuse[i] + uncomit + wanted[i]) { - xce_resource_t* r = NULL; - float need = inuse[i] + uncomit + wanted[i] - total[i]; + vget(core->resmgr, idxs[i], (void**)&r); + is_external = (r != NULL) && (r->flags & XCE_RESF_EXTERNAL); + + if (total[i] < ((is_external ? 0.0f : inuse[i]) + uncomit + wanted[i])) { + float need = (is_external ? 0.0f : inuse[i]) + uncomit + wanted[i] + - total[i]; - vget(core->resmgr, idxs[i], (void**)&r); TRACE("%s unmet resource %d (%s) shortfall %f", __func__, idxs[i], r ? r->name : "?", need); diff --git a/src/scheduler/testsuite/test.c b/src/scheduler/testsuite/test.c index 2bf943b9..9e83f29c 100644 --- a/src/scheduler/testsuite/test.c +++ b/src/scheduler/testsuite/test.c @@ -730,6 +730,97 @@ test_core_safe_allocation_chain_driver_parity(void) destroy(core_sim->resmgr); } +/* + * external-license-resources Phase 1 (E2), design.md 3.3: the EXTERNAL + * accounting rule. An EXTERNAL resource's "total" as pushed by the extres + * daemon is already a FREE value F (license server's free count, net of + * both external consumers and our own checked-out jobs) -- so re-subtracting + * the scheduler's own usage bookkeeping (sharedInUse / w->resInUse) inside + * the admission formula would double-count X. The unsatisfy_ loop must + * therefore skip the inuse term for resources flagged XCE_RESF_EXTERNAL: + * + * EXTERNAL: total >= uncomit + wanted + * non-EXTERNAL: total >= inuse + uncomit + wanted (unchanged) + * + * Shared leg (sched_check_shared_satisfy_): a SHARED|EXTERNAL resource with + * total=F=4, sharedInUse=3, uncomit=0 must ADMIT wanted=4 (F >= 0+4) even + * though the legacy formula (4 < 3+0+4) would deny it -- and the same shape + * WITHOUT the EXTERNAL flag must still deny (legacy semantics pinned). + */ +static void test_unsatisfy_external_skips_inuse_shared_leg(void) +{ + xce_sched_core_t core; + xce_resource_t wire_resource; + xce_resource_t* made; + xce_resource_t* r = NULL; + xvec_t uncomitted; + xvec_idval_t wanted, unmeet; + xce_val_t total, inuse; + int rc; + + memset(&core, 0, sizeof(core)); + /* makeResource() -> resobj_init() -> makeString() needs the intern table + * (same per-test init as test_meet_host_selection_string_match). */ + xstring_table_init(139); + init_test_resmgr(&core); + zarray_init(&core.sharedInst, 4, NULL); + zarray_init(&core.sharedInUse, 4, NULL); + + /* Resource "lic": SHARED (cluster-wide pool) + EXTERNAL (free-value + * reporting) -- the shape mapping.py's `location: [external]` produces. + * idx 0. */ + memset(&wire_resource, 0, sizeof(wire_resource)); + wire_resource.name = "lic"; + wire_resource.idx = 0; + wire_resource.flags = XCE_RESF_SHARED | XCE_RESF_EXTERNAL; + made = (xce_resource_t*)makeResource(core.resmgr, + &wire_resource); + TEST_ASSERT_NOT_NULL(made); + TEST_ASSERT_TRUE(key_find(core.resmgr, (void*)"lic", (void**)&r) >= 0); + TEST_ASSERT_NOT_NULL(r); + + memset(&total, 0, sizeof(total)); + total.which = Value_u_fvalue; + total.u.fvalue = 4.0f; + memset(&inuse, 0, sizeof(inuse)); + inuse.which = Value_u_fvalue; + inuse.u.fvalue = 3.0f; + + zarray_put(&core.sharedInst, 0, &total); + zarray_put(&core.sharedInUse, 0, &inuse); + + xvec_init(&uncomitted, 1, sizeof(float)); + xvec_idval_init(&wanted, 1); + xvec_idval_store(&wanted, 0, 4.0f); + xvec_idval_init(&unmeet, 1); + + /* EXTERNAL: F=4 >= uncomit(0) + wanted(4) -> grant, despite inuse=3. */ + rc = sched_check_shared_satisfy(&core, &uncomitted, &wanted, &unmeet); + TEST_ASSERT_EQUAL_INT(1, rc); + + xvec_idval_fini(&wanted); + xvec_idval_fini(&unmeet); + + /* Same shape without EXTERNAL (plain SHARED pool): the legacy formula + * must still apply -- 4 < 3+0+4 -> deny. Overwrite the flag in place. */ + r->flags = XCE_RESF_SHARED; + + xvec_init(&uncomitted, 1, sizeof(float)); + xvec_idval_init(&wanted, 1); + xvec_idval_store(&wanted, 0, 4.0f); + xvec_idval_init(&unmeet, 1); + + rc = sched_check_shared_satisfy(&core, &uncomitted, &wanted, &unmeet); + TEST_ASSERT_EQUAL_INT(0, rc); + + xvec_idval_fini(&wanted); + xvec_idval_fini(&unmeet); + xvec_fini(&uncomitted); + zarray_fini(&core.sharedInst); + zarray_fini(&core.sharedInUse); + destroy(core.resmgr); +} + #define TEST(x) \ do { \ if (check_test(&arguments, #x)) { \ @@ -759,6 +850,7 @@ int main(int argc, char* argv[]) TEST(test_sched_update_existing_queue_null_payload); TEST(test_dead_pushmsg_symbols_absent); TEST(test_core_safe_allocation_chain_driver_parity); + TEST(test_unsatisfy_external_skips_inuse_shared_leg); return UNITY_END(); } diff --git a/src/scheduler/testsuite/test_core_events.c b/src/scheduler/testsuite/test_core_events.c index 7bf9f7ed..af49c5a3 100644 --- a/src/scheduler/testsuite/test_core_events.c +++ b/src/scheduler/testsuite/test_core_events.c @@ -2005,9 +2005,140 @@ test_jobs_push_picker_rejection_still_tracks_job_and_succeeds(void) g_counts.job_push_rc = 0; } +/* external-license-resources Phase 1 (E2), landing-shape pin: a + * RESVALS_DELTA entry whose host is NULL or "" is a SHARED (cluster-level) + * value pushed by the extres daemon, not a per-worker value -- it must land + * in core->sharedInst at the resource's idx, mirroring snapshot_load()'s + * host==NULL/"" precedent (bug-1923) and scheduler.c's legacy DELTA loop + * needs the same rule. Controller's HOOK_DETAIL_RESOURCE_UPDATE notify is + * delivered to the core as RESVALS_DELTA (handler.c + * sched_handle_sched_update_resvals -> XCE_SCHED_OP_UPDATE_RESVALS), and + * worker pushes carry a NAMED host while the daemon's shared values carry + * host=="" -- so the empty host is the only marker the DELTA path has to + * route by. */ +static void test_resvals_delta_empty_host_lands_in_sharedinst(void) +{ + xce_sched_core_t core; + xce_core_event_t event; + xce_resource_t wire_resource; + xce_resource_t* resource_ptrs[1]; + xce_resval_t empty_host_resval; + xce_resval_t* delta_vals[1]; + xce_resval_list_t deltas; + xce_val_t val_empty; + xce_val_t* stored; + xce_resource_t* found = NULL; + int rc; + + fixture_init(&core, &g_full_plugin); + + memset(&wire_resource, 0, sizeof(wire_resource)); + wire_resource.name = "lic"; + wire_resource.idx = 3; + resource_ptrs[0] = &wire_resource; + + memset(&event, 0, sizeof(event)); + event.type = XCE_CORE_EVENT_RESOURCES_PUSH; + event.u.resources_push.n = 1; + event.u.resources_push.resources = resource_ptrs; + rc = xce_core_push_events(&core, &event, 1); + TEST_ASSERT_EQUAL_INT(0, rc); + + memset(&val_empty, 0, sizeof(val_empty)); + val_empty.which = Value_u_fvalue; + val_empty.u.fvalue = 5.0; + memset(&empty_host_resval, 0, sizeof(empty_host_resval)); + empty_host_resval.name = "lic"; + empty_host_resval.host = ""; + empty_host_resval.val = &val_empty; + + delta_vals[0] = &empty_host_resval; + deltas.count = 1; + deltas.vals = delta_vals; + + memset(&event, 0, sizeof(event)); + event.type = XCE_CORE_EVENT_RESVALS_DELTA; + event.u.resvals_delta.deltas = &deltas; + + rc = xce_core_push_events(&core, &event, 1); + TEST_ASSERT_EQUAL_INT(0, rc); + + TEST_ASSERT_TRUE(key_find(core.resmgr, (void*)"lic", + (void**)&found)); + TEST_ASSERT_NOT_NULL(found); + + stored = (xce_val_t*)zarray_get(&(core.sharedInst), found->idx); + TEST_ASSERT_NOT_NULL(stored); + TEST_ASSERT_EQUAL_INT(Value_u_fvalue, stored->which); + TEST_ASSERT_EQUAL_FLOAT(5.0, stored->u.fvalue); +} + +/* Same landing rule for the NULL spelling: snapshot_load (bug-1923) treats + * host==NULL as shared; the DELTA path must survive a NULL host (skip the + * per-worker lookup instead of strlen(NULL)) and land the value in + * core->sharedInst identically. */ +static void test_resvals_delta_null_host_lands_in_sharedinst(void) +{ + xce_sched_core_t core; + xce_core_event_t event; + xce_resource_t wire_resource; + xce_resource_t* resource_ptrs[1]; + xce_resval_t null_host_resval; + xce_resval_t* delta_vals[1]; + xce_resval_list_t deltas; + xce_val_t val_null; + xce_val_t* stored; + xce_resource_t* found = NULL; + int rc; + + fixture_init(&core, &g_full_plugin); + + memset(&wire_resource, 0, sizeof(wire_resource)); + wire_resource.name = "lic"; + wire_resource.idx = 3; + resource_ptrs[0] = &wire_resource; + + memset(&event, 0, sizeof(event)); + event.type = XCE_CORE_EVENT_RESOURCES_PUSH; + event.u.resources_push.n = 1; + event.u.resources_push.resources = resource_ptrs; + rc = xce_core_push_events(&core, &event, 1); + TEST_ASSERT_EQUAL_INT(0, rc); + + memset(&val_null, 0, sizeof(val_null)); + val_null.which = Value_u_fvalue; + val_null.u.fvalue = 4.0; + memset(&null_host_resval, 0, sizeof(null_host_resval)); + null_host_resval.name = "lic"; + null_host_resval.host = NULL; + null_host_resval.val = &val_null; + + delta_vals[0] = &null_host_resval; + deltas.count = 1; + deltas.vals = delta_vals; + + memset(&event, 0, sizeof(event)); + event.type = XCE_CORE_EVENT_RESVALS_DELTA; + event.u.resvals_delta.deltas = &deltas; + + rc = xce_core_push_events(&core, &event, 1); + TEST_ASSERT_EQUAL_INT(0, rc); + + TEST_ASSERT_TRUE(key_find(core.resmgr, (void*)"lic", + (void**)&found)); + TEST_ASSERT_NOT_NULL(found); + + stored = (xce_val_t*)zarray_get(&(core.sharedInst), found->idx); + TEST_ASSERT_NOT_NULL(stored); + TEST_ASSERT_EQUAL_INT(Value_u_fvalue, stored->which); + TEST_ASSERT_EQUAL_FLOAT(4.0, stored->u.fvalue); +} + int main(void) { UNITY_BEGIN(); + RUN_TEST(test_resvals_delta_empty_host_lands_in_sharedinst); + RUN_TEST(test_resvals_delta_null_host_lands_in_sharedinst); RUN_TEST(test_push_events_fires_each_hook_once_per_event_with_picker_idle); RUN_TEST(test_jobs_push_fires_on_job_push_once_per_job); RUN_TEST(test_queue_update_replaces_existing_interned_resreq); diff --git a/src/schema/controller.capnp b/src/schema/controller.capnp index 446562b7..879d9869 100644 --- a/src/schema/controller.capnp +++ b/src/schema/controller.capnp @@ -153,6 +153,11 @@ enum HostRole { scheduler @2; controller @3; apiserver @4; + # external-license-resources Phase 1: the external-resources daemon + # (xceesd) joins with this role to push shared-resource free counts + # (license totals etc.) through the resval update pipeline. Appended + # last so existing enum values keep their wire numbers. + extres @5; } struct JoinData $C.mapname("xce_joindata_t") { diff --git a/src/xceadmin/src/xcecommon/mapping.py b/src/xceadmin/src/xcecommon/mapping.py index 007efa96..709de22b 100644 --- a/src/xceadmin/src/xcecommon/mapping.py +++ b/src/xceadmin/src/xcecommon/mapping.py @@ -221,6 +221,14 @@ RESOURCE_FLAG_SHARED = 0x08 RESOURCE_FLAG_GLOBAL = 0x04 RESOURCE_FLAG_SHARED_GLOBAL = RESOURCE_FLAG_SHARED | RESOURCE_FLAG_GLOBAL +# Set together with SHARED|GLOBAL on a resourcemap item whose location is +# exactly ["external"] (external-license-resources): XCE_RESF_EXTERNAL in +# include/xce/types.h. The pool is cluster-wide but its values come only +# from the extres daemon's free-count pushes (host=="" resval deltas) -- the +# controller must NOT seed it from a config value (seed_sharedinst_from_config +# skips EXTERNAL) and the scheduler's admission formula skips its inuse term. +RESOURCE_FLAG_EXTERNAL = 0x10 + def new_cluster_config_builder() -> "capnp._DynamicStructBuilder": @@ -429,6 +437,11 @@ def _build_resource(item: dict, target: "capnp._DynamicStructBuilder") -> None: # scheduler can admit jobs against today (see module # docstring, Resource). flags |= RESOURCE_FLAG_SHARED_GLOBAL + elif locations == ["external"]: + # Cluster-wide externally-reported pool + # (external-license-resources): daemon pushes the free + # values; config declares only the name/shape. + flags |= RESOURCE_FLAG_SHARED_GLOBAL | RESOURCE_FLAG_EXTERNAL target.flags = flags diff --git a/src/xceadmin/tests/test_mapping.py b/src/xceadmin/tests/test_mapping.py index 88ceb67c..109da3f3 100644 --- a/src/xceadmin/tests/test_mapping.py +++ b/src/xceadmin/tests/test_mapping.py @@ -248,6 +248,42 @@ def test_map_list_section_resourcemap_maps_distributions_and_all_flag(cluster_co assert r4.flags == 0 +def test_map_list_section_resourcemap_external_location_flag(cluster_config): + """resourcemap location==["external"] (external-license-resources): the + resource carries XCE_RESF_SHARED | XCE_RESF_GLOBAL | XCE_RESF_EXTERNAL + (0x08|0x04|0x10) -- cluster-wide pool, but its values come only from the + extres daemon's free-count pushes, never from a config distribution or + worker reports. A bare ["external"] (no value) also maps definition-only + -- the daemon owns the values entirely. + """ + items = [ + { + "name": "verilog_xl", + "type": "numeric", + "consumable": True, + "resourcemap": [{"location": ["external"]}], + }, + { + "name": "extlic_valued", + "type": "numeric", + "resourcemap": [{"location": ["external"], "value": 7}], + }, + ] + mapping.map_list_section(_spec_for("resources"), items, cluster_config) + + r0, r1 = cluster_config.resources + + # CONSUMABLE | SHARED | GLOBAL | EXTERNAL + assert r0.flags == 0x80 | 0x08 | 0x04 | 0x10 + assert len(r0.distributions) == 1 + assert [str(h) for h in r0.distributions[0].hosts] == ["external"] + + # valued variant: same flags, the distribution value still maps (the + # controller refuses to seed it -- EXTERNAL skips seed_sharedinst) + assert r1.flags == 0x08 | 0x04 | 0x10 + assert r1.distributions[0].value.u.fvalue == 7.0 + + def test_map_list_section_resourcemap_real_repo_configs(cluster_config): """The mapper handles the real src/configs/resources.yaml: every resourcemap entry lands as a distribution, and the two [all]+value