diff --git a/src/frontend/apiserver/store/query.go b/src/frontend/apiserver/store/query.go index f6007fa2..2dfbb1e8 100644 --- a/src/frontend/apiserver/store/query.go +++ b/src/frontend/apiserver/store/query.go @@ -2,6 +2,7 @@ package store import ( "apiserver/utils" + "bytes" "encoding/base64" "encoding/json" "fmt" @@ -378,10 +379,23 @@ func findPageTokenIndex(rows []utils.OrderMap, pageToken string, orders []utils. if err != nil { return 0 } + // bug-1903: decode numbers exactly; a float64 cannot hold a 63-bit + // unique_job_id (array element IDs set bit 62). var tokenData map[string]interface{} - if err := json.Unmarshal(jsonBytes, &tokenData); err != nil { + dec := json.NewDecoder(bytes.NewReader(jsonBytes)) + dec.UseNumber() + if err := dec.Decode(&tokenData); err != nil { return 0 } + for k, v := range tokenData { + if num, ok := v.(json.Number); ok { + if i, err := num.Int64(); err == nil { + tokenData[k] = i + } else if f, err := num.Float64(); err == nil { + tokenData[k] = f + } + } + } for i, row := range rows { match := true diff --git a/src/frontend/apiserver/store/query_test.go b/src/frontend/apiserver/store/query_test.go index 2c26bdf6..a4c0913b 100644 --- a/src/frontend/apiserver/store/query_test.go +++ b/src/frontend/apiserver/store/query_test.go @@ -204,3 +204,42 @@ func TestExecuteQuery_Predicates_SetAfterBuildQueryOptions(t *testing.T) { t.Fatalf("field selection after predicates: row = %v, want only job_id", got[0].Data) } } + +// bug-1903: the page token carried unique_job_id through JSON into a +// float64. Array element IDs set tag bit 62, where float64 spacing is +// 1024, so the token no longer named the page's last row: pages skipped +// rows or restarted at row 0 (an endless loop for callers that page until +// the token is empty, e.g. control of a >1000-element array). +func TestPaginateRowsArrayElementIDsRoundTripPageToken(t *testing.T) { + const n = 2500 + base := int64(1)<<62 | int64(1700000000)<<20 | int64(32768) + rows := make([]utils.OrderMap, n) + for k := 1; k <= n; k++ { + rows[k-1] = utils.OrderMap{Data: map[string]interface{}{"unique_job_id": base + int64(k)}} + } + orders := []utils.SortOrder{{Column: "unique_job_id", Ascending: true}} + + var got []int64 + token := "" + for page := 0; ; page++ { + if page > 10 { + t.Fatalf("paging did not terminate after %d pages (%d rows seen)", page, len(got)) + } + res, next := PaginateRows(rows, 1000, token, orders) + for _, r := range res { + got = append(got, r.Data["unique_job_id"].(int64)-base) + } + if next == "" { + break + } + token = next + } + if len(got) != n { + t.Fatalf("got %d rows, want %d", len(got), n) + } + for i, k := range got { + if k != int64(i+1) { + t.Fatalf("row %d is element %d, want %d", i, k, i+1) + } + } +}