merge: sync Rongsong's scheduler-framework (c5b172c7..3442a57d, 781 commits)
Merge origin/scheduler-framework into merge-sf-20260903 (97 commits on ours: bug-1490/1494/1498/1509/1513/1521/1522 fixes, PEND-reason P1, scheduler-agent Phase 2, bug-924 owner-pid shim). Bookkeeping excluded per policy: .kiro/, .wolf/, CHANGES.md, reports/ (92 files dropped); .claude/ + CLAUDE.md keep ours. Key theirs-side changes absorbed: - bug-1938/1960: QUEUE_STAT_CLOSED no longer gates dispatch (closed queue refuses new submissions only; pending jobs still run) -- supersedes our QUEUE_CLOSED preason on the dispatch path; sched_queue_status_reason drops that branch - bug-1846: config push now stashes and applies between scheduling rounds (sched_stash_config/sched_apply_pending_config), applied at the same main-thread round boundary that preserves bug-1513 serialization - bug-1760: sched_check_pending_rejoin exposed + xcecli_is_connection_broken public predicate - bug-1936: execuser/exechome/execcwd/exdata carried on submission (superset of our bug-1500 execuser fallback; our job-typed resolveExecUser removed as dead code) - bug-1945~1962: apiserver auth hardening (HS256 pinning, default- secret rejection, logout revocation, bcrypt, RBAC) -- our API-key branch folded into controllers.ParseTokenClaims so both coexist - bug-1736: rawRunUsage job run-usage fields in the Go model - xce_test relocated src/controller/testsuite/test.c -> src/harness/test.c; controller suite split into controller_test.c + fixtures; their clitest fixtures spawn xce_test mid-suite under `meson test -j` Ours preserved across the merge: - PEND-reason P1 (codes 1-9, preason.h, job_preason machinery) -- gate re-ported into their relocated allocator (sched_election.c) after plugin_job_election, with attach on the allocator thread only (bug-1521 race discipline) - PEND-reason P0 main_reason semantics RECONCILED to theirs: the merge initially re-carried our first-detail-add-promotes-main_reason fallback in add_job_preason (4e2ac86), but their whole trace/verify stack treats "detail-only => main_reason 0" as a contract -- sched_trace_capture counts main_reason as its own occurrence ("Expected 2 Was 3" in test_trace_capture), the outcome renderer prints "main reason: none" for detail-only PENDs (test_repl_trace_e2e), and the committed .replay/.sim_verify baselines encode main_reason=0 (test_repl_trace_baseline byte compare, test_trace_equivalence off-vs-traced). The fallback is dropped from the shared lib; the user-visible "overall reason" survives as an API-layer view: rawJobToModel (convert_pure.go) promotes the first detail row to main_reason_name when main_reason is 0 (main_reason VALUE marshals unchanged). add_job_preason keeps only the NULL-reasons guard - bug-1498: sched_job_is_ready rejects non-PEND/suspended jobs - bug-1490/1781: xce_jobctrl_t.action as capnp union discriminant (their jobCtrlUnionDiscriminant, same semantics) - xcecli typed PendingReason field (53c00834), bug-1522 reconnect resend, xcecli_force_reconnect + their is_connection_broken both exported in client.h/libclient - t1/t2/t3 test-node hostconfig in src/configs/hosts.yaml (bug-1494), /xce/v1/events + /xce/v1/apikeys whitelist routes (agent Phase 2) - lib ASAN_OPTIONS=detect_leaks=0 test env + their oom-test block - drop-audit: our bug-1504 fix covered by their bug-1680 (no action) bug-924 follow-up (replaces the67f3f6adscratch-dir wipe): the take-theirs audit's one "drop" was re-evaluated and REVERTED instead of re-applied -- a global `find /tmp -name 'xce_*' -exec rm -rf` at suite start/exit races the LIVE /tmp/xce_* fixtures of sibling suites under `meson test -j` (harness_unit_test's clitest fixtures spawn xce_test as a child mid-suite; the wipe destroyed the parent's fixtures -> test_test_cli failures). Coverage instead: - mkdtemp_clean shim gains mkdtemp_clean_mark/sweep_since (testsuite_support/mkdtemp_clean.[ch]) - new case_scratch_sweep.[ch]: aggregate runners (controller_test.c, harness_test.c -- hundreds of cases per process) call case_scratch_sweep_begin/_end_ok from setUp/tearDown; a PASSED case's dirs are swept immediately in tearDown (after engine reaping), FAILED cases keep their dirs for post-mortem; XCE_MKDTEMP_KEEP_ALL=1 disables. One full controller_unit_test run was observed to leak 12G+ to ENOSPC without this, cascading into unrelated "Expected Non-NULL" failures - harness_unit_test_deps links the shim library as well Test reconciliations on adopt: - core-ready-gates: sched_queue_is_ready now gates only INACT_ADMIN|INACT_WINDOW (bug-1938/1960); CLOSED queues elect -- covered end-to-end by test_queue_admin_control.c - test_worker_resync: the 4 restart-replay RUN assertions predate the bug-1509 B3 marking (8858afed) landing on the same lineage; replayed RUN jobs now assert RUN|UNKWN|TRANSIENT (TWR_ASSERT_REPLAYED_RUN); terminal DONE/EXIT assertions unchanged - convert_pure.go: our job-typed resolveExecUser deleted (superseded by their bug-1780/1936 two-arg version) - test_preason: the 3 P0-promotion assertions reverted to the detail-keeps-main-0 contract; new tests cover limit-code detail rows and explicit-main-wins-over-later-details; apiserver convert_test.go gains the detail-only name-promotion cases Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
lei.chen
commit
0c8607345e
663 changed files with 139848 additions and 13614 deletions
|
|
@ -18,6 +18,14 @@ func Router() *gin.Engine {
|
|||
logger.Debug("Enter Router() function")
|
||||
r := gin.Default()
|
||||
|
||||
// Honour X-Forwarded-For / X-Real-IP only from configured reverse proxies
|
||||
// (security.trusted_proxies, default none), so a client cannot choose the
|
||||
// IP seen by the IP whitelist and the JWT IP binding (bug-1948).
|
||||
if err := r.SetTrustedProxies(utils.TrustedProxiesYamlConfig); err != nil {
|
||||
logger.Error("Invalid security.trusted_proxies, trusting no proxy", zap.Error(err))
|
||||
_ = r.SetTrustedProxies(nil)
|
||||
}
|
||||
|
||||
// check access URL whitelist
|
||||
r.Use(middleware.WhitelistURLMiddleware())
|
||||
|
||||
|
|
|
|||
77
src/frontend/apiserver/router/routers_test.go
Normal file
77
src/frontend/apiserver/router/routers_test.go
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
package router
|
||||
|
||||
import (
|
||||
"apiserver/utils"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// setRouterTestConfig installs the minimal config Router() reads, with an IP
|
||||
// whitelist of 10.0.0.0/8, and restores the previous values afterwards.
|
||||
func setRouterTestConfig(t *testing.T) {
|
||||
t.Helper()
|
||||
prevLog, prevTop := utils.Log, utils.Global_TopDir
|
||||
prevWL, prevRL, prevCORS := utils.WhitelistYamlConfig, utils.RateLimitYamlConfig, utils.CORSYamlConfig
|
||||
t.Cleanup(func() {
|
||||
utils.Log, utils.Global_TopDir = prevLog, prevTop
|
||||
utils.WhitelistYamlConfig, utils.RateLimitYamlConfig, utils.CORSYamlConfig = prevWL, prevRL, prevCORS
|
||||
})
|
||||
|
||||
gin.SetMode(gin.TestMode)
|
||||
utils.Log = zap.NewNop()
|
||||
utils.Global_TopDir = t.TempDir()
|
||||
utils.WhitelistYamlConfig = &utils.WhitelistConfig{Whitelist: []string{"10.0.0.0/8"}}
|
||||
utils.RateLimitYamlConfig = &utils.RateLimitConfig{
|
||||
Global: utils.EndpointLimit{Rate: 1000, Burst: 2000, Window: "second"},
|
||||
Endpoints: map[string]utils.EndpointLimit{},
|
||||
}
|
||||
utils.CORSYamlConfig = nil
|
||||
}
|
||||
|
||||
func pingFrom(r *gin.Engine, remoteAddr, forwardedFor string) int {
|
||||
req := httptest.NewRequest(http.MethodGet, "/ping", nil)
|
||||
req.RemoteAddr = remoteAddr
|
||||
if forwardedFor != "" {
|
||||
req.Header.Set("X-Forwarded-For", forwardedFor)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
r.ServeHTTP(w, req)
|
||||
return w.Code
|
||||
}
|
||||
|
||||
// bug-1948: with no trusted proxies configured, X-Forwarded-For must not
|
||||
// change the client IP seen by the IP whitelist (or the JWT IP binding).
|
||||
func TestRouter_IgnoresForwardedForByDefault(t *testing.T) {
|
||||
setRouterTestConfig(t)
|
||||
r := Router()
|
||||
|
||||
if got := pingFrom(r, "10.1.1.1:4000", ""); got != http.StatusOK {
|
||||
t.Fatalf("whitelisted peer: status = %d, want 200", got)
|
||||
}
|
||||
if got := pingFrom(r, "192.0.2.10:4000", "10.1.1.1"); got != http.StatusForbidden {
|
||||
t.Errorf("non-whitelisted peer spoofing X-Forwarded-For: status = %d, want 403", got)
|
||||
}
|
||||
}
|
||||
|
||||
// bug-1948: X-Forwarded-For is honoured only from a configured trusted proxy.
|
||||
func TestRouter_HonoursForwardedForFromTrustedProxy(t *testing.T) {
|
||||
setRouterTestConfig(t)
|
||||
prev := utils.TrustedProxiesYamlConfig
|
||||
t.Cleanup(func() { utils.TrustedProxiesYamlConfig = prev })
|
||||
utils.TrustedProxiesYamlConfig = []string{"192.0.2.1"}
|
||||
r := Router()
|
||||
|
||||
if got := pingFrom(r, "192.0.2.1:4000", "10.1.1.1"); got != http.StatusOK {
|
||||
t.Errorf("trusted proxy forwarding a whitelisted client: status = %d, want 200", got)
|
||||
}
|
||||
if got := pingFrom(r, "192.0.2.1:4000", "198.51.100.7"); got != http.StatusForbidden {
|
||||
t.Errorf("trusted proxy forwarding a non-whitelisted client: status = %d, want 403", got)
|
||||
}
|
||||
if got := pingFrom(r, "192.0.2.99:4000", "10.1.1.1"); got != http.StatusForbidden {
|
||||
t.Errorf("untrusted peer spoofing X-Forwarded-For: status = %d, want 403", got)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue