diff --git a/.kiro/specs/job-array-one-based-index/tasks.md b/.kiro/specs/job-array-one-based-index/tasks.md index f3636903..fd02bcdb 100644 --- a/.kiro/specs/job-array-one-based-index/tasks.md +++ b/.kiro/specs/job-array-one-based-index/tasks.md @@ -108,7 +108,7 @@ - _Requirements: 1.6, 1.7, 1.8, 1.9_ - _Boundary: ApiConfig_ -- [ ] 5.3 (P) Evaluate row predicates in the query pipeline +- [x] 5.3 (P) Evaluate row predicates in the query pipeline - Add row predicates to the query options and apply them after the plain conditions and before sorting and pagination - Done when the store query tests show predicates filtering before pagination, with page boundaries and bookmarks consistent - _Requirements: 5.2, 5.11_ diff --git a/src/frontend/apiserver/store/query.go b/src/frontend/apiserver/store/query.go index 6e00d927..f6007fa2 100644 --- a/src/frontend/apiserver/store/query.go +++ b/src/frontend/apiserver/store/query.go @@ -12,6 +12,8 @@ import ( const PER_PAGE_LIMIT = 100 // ExecuteQuery runs the full filter → sort → paginate → select pipeline. +// Filtering applies the plain Conditions, then the row Predicates, so that +// sort, pagination and the page token see only matching rows. func ExecuteQuery(rows []utils.OrderMap, opts *utils.QueryOptions, fields []string) ([]utils.OrderMap, string) { if opts == nil { return SelectFields(rows, fields), "" @@ -21,6 +23,9 @@ func ExecuteQuery(rows []utils.OrderMap, opts *utils.QueryOptions, fields []stri if len(opts.Conditions) > 0 { rows = FilterRows(rows, opts.Conditions) } + if len(opts.Predicates) > 0 { + rows = FilterRowsByPredicates(rows, opts.Predicates) + } // 2. Sort if len(opts.Orders) > 0 { @@ -134,6 +139,25 @@ func FilterRows(rows []utils.OrderMap, conditions []utils.Condition) []utils.Ord return result } +// FilterRowsByPredicates keeps the rows for which every predicate returns +// true (AND logic). It returns a new slice; the input is not modified. +func FilterRowsByPredicates(rows []utils.OrderMap, preds []func(utils.OrderMap) bool) []utils.OrderMap { + result := make([]utils.OrderMap, 0, len(rows)) + for _, row := range rows { + match := true + for _, pred := range preds { + if !pred(row) { + match = false + break + } + } + if match { + result = append(result, row) + } + } + return result +} + // SortRows sorts by multiple fields with asc/desc. func SortRows(rows []utils.OrderMap, orders []utils.SortOrder) { sort.SliceStable(rows, func(i, j int) bool { diff --git a/src/frontend/apiserver/store/query_test.go b/src/frontend/apiserver/store/query_test.go index bdd9dc7a..2c26bdf6 100644 --- a/src/frontend/apiserver/store/query_test.go +++ b/src/frontend/apiserver/store/query_test.go @@ -16,9 +16,9 @@ import ( // regardless of how many jobs actually exist. func TestFilterRows_JobStatusNotNull_MatchesRealRows(t *testing.T) { rows := []utils.OrderMap{ - {Fields: []string{"job_status"}, Data: map[string]interface{}{"job_status": int64(1)}}, // PEND - {Fields: []string{"job_status"}, Data: map[string]interface{}{"job_status": int64(2)}}, // RUN - {Fields: []string{"job_status"}, Data: map[string]interface{}{"job_status": int64(8)}}, // DONE + {Fields: []string{"job_status"}, Data: map[string]interface{}{"job_status": int64(1)}}, // PEND + {Fields: []string{"job_status"}, Data: map[string]interface{}{"job_status": int64(2)}}, // RUN + {Fields: []string{"job_status"}, Data: map[string]interface{}{"job_status": int64(8)}}, // DONE } // Mirrors exactly what buildIntCondition's null/notnull scalar path // produces for `{job_status:notnull:}` (types.go): Operator: OpNOTNULL, @@ -61,3 +61,146 @@ func TestFilterRows_JobStatusBitmaskEQ_StillWorks(t *testing.T) { t.Errorf("matched row job_status = %v, want 1 (PEND)", result[0].Data["job_status"]) } } + +// --- QueryOptions.Predicates (job-array-one-based-index task 5.3) --- +// +// Element selections (bug-1898) become row predicates evaluated by +// ExecuteQuery after the plain Conditions and before sort and pagination, so +// a page holds only matching rows and the bookmark stays consistent. + +func predTestRows(n int) []utils.OrderMap { + rows := make([]utils.OrderMap, 0, n) + for i := 1; i <= n; i++ { + rows = append(rows, utils.OrderMap{ + Fields: []string{"job_id", "job_status"}, + Data: map[string]interface{}{"job_id": int64(i), "job_status": int64(1 << uint(i%2))}, + }) + } + return rows +} + +func predJobIDs(rows []utils.OrderMap) []int64 { + ids := make([]int64, 0, len(rows)) + for _, r := range rows { + ids = append(ids, r.Data["job_id"].(int64)) + } + return ids +} + +func predEqualIDs(t *testing.T, label string, got []utils.OrderMap, want []int64) { + t.Helper() + ids := predJobIDs(got) + if len(ids) != len(want) { + t.Fatalf("%s: job_ids = %v, want %v", label, ids, want) + } + for i := range ids { + if ids[i] != want[i] { + t.Fatalf("%s: job_ids = %v, want %v", label, ids, want) + } + } +} + +func predEven(row utils.OrderMap) bool { return row.Data["job_id"].(int64)%2 == 0 } + +func TestExecuteQuery_Predicates_FilterBeforePagination(t *testing.T) { + asc := []utils.SortOrder{{Column: "job_id", Ascending: true}} + opts := &utils.QueryOptions{ + Orders: asc, + Limit: 2, + Predicates: []func(utils.OrderMap) bool{predEven}, + } + + page1, tok1 := ExecuteQuery(predTestRows(10), opts, nil) + predEqualIDs(t, "page 1", page1, []int64{2, 4}) + if tok1 == "" { + t.Fatal("page 1: expected a next-page token") + } + + opts.Bookmark = tok1 + page2, tok2 := ExecuteQuery(predTestRows(10), opts, nil) + predEqualIDs(t, "page 2", page2, []int64{6, 8}) + if tok2 == "" { + t.Fatal("page 2: expected a next-page token") + } + + opts.Bookmark = tok2 + page3, tok3 := ExecuteQuery(predTestRows(10), opts, nil) + predEqualIDs(t, "page 3", page3, []int64{10}) + if tok3 != "" { + t.Fatalf("page 3: next-page token = %q, want none (last matching row)", tok3) + } +} + +func TestExecuteQuery_Predicates_ExactPageNoTrailingToken(t *testing.T) { + // 4 matching rows, limit 2: the second page ends exactly at the last + // match, so no token -- a non-matching tail row must not produce one. + opts := &utils.QueryOptions{ + Orders: []utils.SortOrder{{Column: "job_id", Ascending: true}}, + Limit: 2, + Predicates: []func(utils.OrderMap) bool{predEven}, + } + page1, tok1 := ExecuteQuery(predTestRows(9), opts, nil) + predEqualIDs(t, "page 1", page1, []int64{2, 4}) + opts.Bookmark = tok1 + page2, tok2 := ExecuteQuery(predTestRows(9), opts, nil) + predEqualIDs(t, "page 2", page2, []int64{6, 8}) + if tok2 != "" { + t.Fatalf("page 2: next-page token = %q, want none", tok2) + } +} + +func TestExecuteQuery_Predicates_ANDedWithEachOtherAndConditions(t *testing.T) { + gt3 := func(row utils.OrderMap) bool { return row.Data["job_id"].(int64) > 3 } + opts := &utils.QueryOptions{ + // job_id <= 9 via a plain condition + Conditions: []utils.Condition{{ + Column: "job_id", + Operator: utils.OpLTE, + Value: utils.Value{Type: utils.TypeInt, Value: int64(9)}, + }}, + Orders: []utils.SortOrder{{Column: "job_id", Ascending: false}}, + Predicates: []func(utils.OrderMap) bool{predEven, gt3}, + } + got, tok := ExecuteQuery(predTestRows(12), opts, nil) + predEqualIDs(t, "desc, cond+2 preds", got, []int64{8, 6, 4}) + if tok != "" { + t.Fatalf("next-page token = %q, want none", tok) + } +} + +func TestExecuteQuery_Predicates_RejectAllListsNothing(t *testing.T) { + none := func(utils.OrderMap) bool { return false } + opts := &utils.QueryOptions{Limit: 3, Predicates: []func(utils.OrderMap) bool{none}} + got, tok := ExecuteQuery(predTestRows(5), opts, nil) + if len(got) != 0 || tok != "" { + t.Fatalf("got %v rows, token %q; want 0 rows, no token", predJobIDs(got), tok) + } +} + +func TestExecuteQuery_Predicates_NilOrEmptyUnchanged(t *testing.T) { + for _, preds := range [][]func(utils.OrderMap) bool{nil, {}} { + opts := &utils.QueryOptions{ + Orders: []utils.SortOrder{{Column: "job_id", Ascending: true}}, + Limit: 3, + Predicates: preds, + } + got, tok := ExecuteQuery(predTestRows(5), opts, nil) + predEqualIDs(t, "no predicates", got, []int64{1, 2, 3}) + if tok == "" { + t.Fatal("no predicates: expected a next-page token") + } + } +} + +func TestExecuteQuery_Predicates_SetAfterBuildQueryOptions(t *testing.T) { + opts, err := utils.BuildQueryOptions(nil, utils.CompCondition{}, nil, 0, "") + if err != nil { + t.Fatalf("BuildQueryOptions: %v", err) + } + opts.Predicates = append(opts.Predicates, predEven) + got, _ := ExecuteQuery(predTestRows(6), opts, []string{"job_id"}) + predEqualIDs(t, "built opts", got, []int64{2, 4, 6}) + if len(got[0].Data) != 1 { + t.Fatalf("field selection after predicates: row = %v, want only job_id", got[0].Data) + } +} diff --git a/src/frontend/apiserver/utils/types.go b/src/frontend/apiserver/utils/types.go index 36c44ed4..62d6dbea 100644 --- a/src/frontend/apiserver/utils/types.go +++ b/src/frontend/apiserver/utils/types.go @@ -97,6 +97,12 @@ type QueryOptions struct { Orders []SortOrder Limit int Bookmark string + // Predicates are row predicates (e.g. job element selections) that + // store.ExecuteQuery evaluates after Conditions and before sort and + // pagination. A row is kept only if every predicate returns true; + // nil or empty means no extra filtering. Set them on the result of + // BuildQueryOptions; the builder does not take them. + Predicates []func(OrderMap) bool } // OrderMap represents a row of data with ordered field names. @@ -480,7 +486,6 @@ func FormatTimestamp(ts int64) string { return t.Local().Format("2006-01-02 15:04:05") } - // Supported query parameter lists for validation. var ( SupportedQueryParams = []string{"filter", "sort", "limit", "page_token", "fields"}