From 90fdd1804c09abe9ffab4ed4ad7f4a2049649ac8 Mon Sep 17 00:00:00 2001 From: Preetam Dwivedi Date: Tue, 20 Oct 2026 14:36:08 -0700 Subject: [PATCH] feat(gateway): make List pagination token-only, drop web cursor secret MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary ### Why? The gateway's `List` token already carries the queue and window, yet callers had to resend both bounds on every page. So `web/submitqueue` wrapped the window plus the token in a second, HMAC-signed cursor, and every deployment needed a signing secret that protected nothing the gateway does not already validate. Stovepipe's `List` already does this right. ### What? - `ListRequest` bounds become optional (`oneof`, same field numbers 2/3, so the wire format is unchanged). The first page still requires both; with a `page_token`, the window comes from the token and any bound supplied must match it. - `ListResponse` echoes the resolved window (`received_at_or_after_ms`, `received_before_ms`), so clients can show it without decoding the token. - Tokens are versioned base64url JSON, checked field by field, including that the last receipt lies inside the window. Unversioned tokens still decode for one release (`TODO(list-token)`). - The Go client sends bounds on the first page only. - `web/submitqueue` links the raw gateway token as `?page=` and calls `list` without bounds when one is present; a token the gateway rejects redirects to the live page. `CursorCodec`, `extension/cursor` (HMAC) and the `cursors` option are removed, along with `SUBMITQUEUE_WEB_CURSOR_SECRET` from the reference host, Makefile, compose and e2e harness. - Breaking for web hosts: `GatewayReader.list` takes the `oneof` bound shape and must return the window. A Connect client satisfies it as-is; hand-written adapters need updating, and callers must drop `cursors`. ## Test Plan ✅ `./tool/bazel test //submitqueue/gateway/controller/... //service/submitqueue/gateway/... //submitqueue/client/...` — first page needs both bounds, token-only continuation, matching/mismatched bounds, tampered/wrong-version/out-of-range/legacy tokens, proto presence round-trip ✅ `./tool/bazel test //web:check` — raw token in links, token-only `list` on `?page=`, rejected-token redirect, no `CursorCodec` ✅ `./tool/bazel build //...` (integration and e2e suites compile); `make test` passes except `//tool/compose:compose_test`, a macOS `/private/var` tmp-path mismatch unrelated to this change ⚠️ Gateway integration suite (`//test/integration/submitqueue/gateway:go_default_test`) not run locally — needs Docker; CI covers it --- Makefile | 1 - api/submitqueue/gateway/proto/gateway.proto | 18 ++- api/submitqueue/gateway/protopb/gateway.pb.go | 102 ++++++++++-- .../gateway/protopb/gateway.pb.yarpc.go | 150 +++++++++--------- doc/rfc/submitqueue/status-list-api.md | 4 +- doc/rfc/web-ui.md | 4 +- service/submitqueue/docker-compose.web.yml | 1 - .../gateway/server/mapper/BUILD.bazel | 1 + .../submitqueue/gateway/server/mapper/list.go | 18 ++- .../gateway/server/mapper/list_test.go | 85 ++++++++-- submitqueue/client/query.go | 22 +-- submitqueue/client/query_test.go | 19 ++- submitqueue/entity/list.go | 14 +- submitqueue/gateway/controller/BUILD.bazel | 1 + submitqueue/gateway/controller/list.go | 95 ++--------- .../controller/list_consistency_test.go | 4 +- .../gateway/controller/list_pagination.go | 131 +++++++++++++++ submitqueue/gateway/controller/list_test.go | 129 ++++++++++----- test/e2e/submitqueue/fake_demo_test.go | 3 +- test/e2e/submitqueue/suite_test.go | 13 +- .../submitqueue/gateway/suite_test.go | 20 +-- web/api/src/api.test.ts | 9 +- .../submitqueue/gateway/proto/gateway_pb.ts | 46 ++++-- web/service/submitqueue/README.md | 1 - .../submitqueue/src/app/healthz/route.ts | 3 +- .../submitqueue/src/server/config.test.ts | 9 +- web/service/submitqueue/src/server/config.ts | 9 -- web/service/submitqueue/src/server/host.ts | 3 - .../src/server/instrumentation.test.ts | 3 +- web/submitqueue/README.md | 7 +- web/submitqueue/controller/change.test.ts | 61 ++++--- web/submitqueue/controller/change.ts | 80 ++++++---- web/submitqueue/controller/controller.test.ts | 35 ++-- web/submitqueue/controller/mapping.ts | 9 +- web/submitqueue/controller/page.ts | 2 - web/submitqueue/controller/queue.ts | 84 +++++----- web/submitqueue/core/window.test.ts | 36 +---- web/submitqueue/core/window.ts | 39 +---- web/submitqueue/extension/cursor/cursor.ts | 10 -- .../extension/cursor/hmac/hmac.test.ts | 26 --- web/submitqueue/extension/cursor/hmac/hmac.ts | 36 ----- .../extension/cursor/hmac/index.ts | 1 - web/submitqueue/extension/cursor/index.ts | 1 - web/submitqueue/extension/gateway/gateway.ts | 7 +- .../extension/gateway/mock/fake.ts | 8 +- web/submitqueue/module.test.ts | 75 ++++----- web/submitqueue/module.ts | 5 +- web/submitqueue/package.json | 7 +- web/submitqueue/server.ts | 1 - web/submitqueue/view/pages.test.tsx | 3 +- web/test/e2e/submitqueue/run_e2e.py | 1 - web/test/e2e/submitqueue/submitqueue.spec.ts | 7 +- web/test/package/consumer-runtime.mjs | 2 - web/test/package/consumer.ts | 3 +- web/test/package/pack_contents_test.py | 1 - web/test/package/react18/react18-runtime.mjs | 2 - 56 files changed, 805 insertions(+), 662 deletions(-) create mode 100644 submitqueue/gateway/controller/list_pagination.go delete mode 100644 web/submitqueue/extension/cursor/cursor.ts delete mode 100644 web/submitqueue/extension/cursor/hmac/hmac.test.ts delete mode 100644 web/submitqueue/extension/cursor/hmac/hmac.ts delete mode 100644 web/submitqueue/extension/cursor/hmac/index.ts delete mode 100644 web/submitqueue/extension/cursor/index.ts diff --git a/Makefile b/Makefile index 6699fbfe8..0861212ac 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,6 @@ export REPO_ROOT := $(shell pwd) # Local-only credential for the reference web host. Override it for manual # testing; production authentication is deliberately outside this demo. export SUBMITQUEUE_WEB_TOKEN ?= test -export SUBMITQUEUE_WEB_CURSOR_SECRET ?= local-demo-cursor-secret # Which provider the demo stack targets, and the only difference between a free # local run and a live one. Selects a configuration directory rather than a code diff --git a/api/submitqueue/gateway/proto/gateway.proto b/api/submitqueue/gateway/proto/gateway.proto index 86a29f68b..d3b387a7e 100644 --- a/api/submitqueue/gateway/proto/gateway.proto +++ b/api/submitqueue/gateway/proto/gateway.proto @@ -139,13 +139,17 @@ message GetRequestSummaryByChangeURIResponse { message ListRequest { // Exact queue to query. string queue = 1; - // Inclusive lower receipt-time bound in Unix milliseconds. - int64 received_at_or_after_ms = 2; - // Exclusive upper receipt-time bound in Unix milliseconds. - int64 received_before_ms = 3; + oneof received_lower_bound { + // Inclusive receipt-time bound in Unix milliseconds. Required without page_token; with page_token, omitted takes the token's bound and supplied must equal it. + int64 received_at_or_after_ms = 2; + } + oneof received_upper_bound { + // Exclusive receipt-time bound in Unix milliseconds. Required without page_token; with page_token, omitted takes the token's bound and supplied must equal it. + int64 received_before_ms = 3; + } // Maximum results to return. Zero selects the server default; values above the server maximum are invalid. int32 page_size = 4; - // Opaque continuation token from a previous response. Empty selects the first page. + // Opaque continuation token from a previous response, bound to its queue and receipt window. Empty selects the first page. string page_token = 5; } @@ -155,6 +159,10 @@ message ListResponse { repeated RequestSummary requests = 1; // Opaque continuation token. Empty when this is the last page. string next_page_token = 2; + // Inclusive lower receipt-time bound of the window this page was read from, in Unix milliseconds. + int64 received_at_or_after_ms = 3; + // Exclusive upper receipt-time bound of the window this page was read from, in Unix milliseconds. + int64 received_before_ms = 4; } // Queue identifies a configured submit queue. diff --git a/api/submitqueue/gateway/protopb/gateway.pb.go b/api/submitqueue/gateway/protopb/gateway.pb.go index e18c3ccae..9711be27e 100644 --- a/api/submitqueue/gateway/protopb/gateway.pb.go +++ b/api/submitqueue/gateway/protopb/gateway.pb.go @@ -14,7 +14,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.11 +// protoc-gen-go v1.36.10 // protoc v5.29.3 // source: gateway.proto @@ -688,13 +688,17 @@ type ListRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // Exact queue to query. Queue string `protobuf:"bytes,1,opt,name=queue,proto3" json:"queue,omitempty"` - // Inclusive lower receipt-time bound in Unix milliseconds. - ReceivedAtOrAfterMs int64 `protobuf:"varint,2,opt,name=received_at_or_after_ms,json=receivedAtOrAfterMs,proto3" json:"received_at_or_after_ms,omitempty"` - // Exclusive upper receipt-time bound in Unix milliseconds. - ReceivedBeforeMs int64 `protobuf:"varint,3,opt,name=received_before_ms,json=receivedBeforeMs,proto3" json:"received_before_ms,omitempty"` + // Types that are valid to be assigned to ReceivedLowerBound: + // + // *ListRequest_ReceivedAtOrAfterMs + ReceivedLowerBound isListRequest_ReceivedLowerBound `protobuf_oneof:"received_lower_bound"` + // Types that are valid to be assigned to ReceivedUpperBound: + // + // *ListRequest_ReceivedBeforeMs + ReceivedUpperBound isListRequest_ReceivedUpperBound `protobuf_oneof:"received_upper_bound"` // Maximum results to return. Zero selects the server default; values above the server maximum are invalid. PageSize int32 `protobuf:"varint,4,opt,name=page_size,json=pageSize,proto3" json:"page_size,omitempty"` - // Opaque continuation token from a previous response. Empty selects the first page. + // Opaque continuation token from a previous response, bound to its queue and receipt window. Empty selects the first page. PageToken string `protobuf:"bytes,5,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -737,16 +741,34 @@ func (x *ListRequest) GetQueue() string { return "" } +func (x *ListRequest) GetReceivedLowerBound() isListRequest_ReceivedLowerBound { + if x != nil { + return x.ReceivedLowerBound + } + return nil +} + func (x *ListRequest) GetReceivedAtOrAfterMs() int64 { if x != nil { - return x.ReceivedAtOrAfterMs + if x, ok := x.ReceivedLowerBound.(*ListRequest_ReceivedAtOrAfterMs); ok { + return x.ReceivedAtOrAfterMs + } } return 0 } +func (x *ListRequest) GetReceivedUpperBound() isListRequest_ReceivedUpperBound { + if x != nil { + return x.ReceivedUpperBound + } + return nil +} + func (x *ListRequest) GetReceivedBeforeMs() int64 { if x != nil { - return x.ReceivedBeforeMs + if x, ok := x.ReceivedUpperBound.(*ListRequest_ReceivedBeforeMs); ok { + return x.ReceivedBeforeMs + } } return 0 } @@ -765,6 +787,28 @@ func (x *ListRequest) GetPageToken() string { return "" } +type isListRequest_ReceivedLowerBound interface { + isListRequest_ReceivedLowerBound() +} + +type ListRequest_ReceivedAtOrAfterMs struct { + // Inclusive receipt-time bound in Unix milliseconds. Required without page_token; with page_token, omitted takes the token's bound and supplied must equal it. + ReceivedAtOrAfterMs int64 `protobuf:"varint,2,opt,name=received_at_or_after_ms,json=receivedAtOrAfterMs,proto3,oneof"` +} + +func (*ListRequest_ReceivedAtOrAfterMs) isListRequest_ReceivedLowerBound() {} + +type isListRequest_ReceivedUpperBound interface { + isListRequest_ReceivedUpperBound() +} + +type ListRequest_ReceivedBeforeMs struct { + // Exclusive receipt-time bound in Unix milliseconds. Required without page_token; with page_token, omitted takes the token's bound and supplied must equal it. + ReceivedBeforeMs int64 `protobuf:"varint,3,opt,name=received_before_ms,json=receivedBeforeMs,proto3,oneof"` +} + +func (*ListRequest_ReceivedBeforeMs) isListRequest_ReceivedUpperBound() {} + // ListResponse defines one page of queue receipt history. type ListResponse struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -772,8 +816,12 @@ type ListResponse struct { Requests []*RequestSummary `protobuf:"bytes,1,rep,name=requests,proto3" json:"requests,omitempty"` // Opaque continuation token. Empty when this is the last page. NextPageToken string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken,proto3" json:"next_page_token,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Inclusive lower receipt-time bound of the window this page was read from, in Unix milliseconds. + ReceivedAtOrAfterMs int64 `protobuf:"varint,3,opt,name=received_at_or_after_ms,json=receivedAtOrAfterMs,proto3" json:"received_at_or_after_ms,omitempty"` + // Exclusive upper receipt-time bound of the window this page was read from, in Unix milliseconds. + ReceivedBeforeMs int64 `protobuf:"varint,4,opt,name=received_before_ms,json=receivedBeforeMs,proto3" json:"received_before_ms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ListResponse) Reset() { @@ -820,6 +868,20 @@ func (x *ListResponse) GetNextPageToken() string { return "" } +func (x *ListResponse) GetReceivedAtOrAfterMs() int64 { + if x != nil { + return x.ReceivedAtOrAfterMs + } + return 0 +} + +func (x *ListResponse) GetReceivedBeforeMs() int64 { + if x != nil { + return x.ReceivedBeforeMs + } + return 0 +} + // Queue identifies a configured submit queue. type Queue struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1512,17 +1574,21 @@ const file_gateway_proto_rawDesc = "" + "change_uri\x18\x01 \x01(\tR\tchangeUri\x12\x14\n" + "\x05queue\x18\x02 \x01(\tR\x05queue\"l\n" + "$GetRequestSummaryByChangeURIResponse\x12D\n" + - "\brequests\x18\x01 \x03(\v2(.uber.submitqueue.gateway.RequestSummaryR\brequests\"\xc3\x01\n" + + "\brequests\x18\x01 \x03(\v2(.uber.submitqueue.gateway.RequestSummaryR\brequests\"\xf7\x01\n" + "\vListRequest\x12\x14\n" + - "\x05queue\x18\x01 \x01(\tR\x05queue\x124\n" + - "\x17received_at_or_after_ms\x18\x02 \x01(\x03R\x13receivedAtOrAfterMs\x12,\n" + - "\x12received_before_ms\x18\x03 \x01(\x03R\x10receivedBeforeMs\x12\x1b\n" + + "\x05queue\x18\x01 \x01(\tR\x05queue\x126\n" + + "\x17received_at_or_after_ms\x18\x02 \x01(\x03H\x00R\x13receivedAtOrAfterMs\x12.\n" + + "\x12received_before_ms\x18\x03 \x01(\x03H\x01R\x10receivedBeforeMs\x12\x1b\n" + "\tpage_size\x18\x04 \x01(\x05R\bpageSize\x12\x1d\n" + "\n" + - "page_token\x18\x05 \x01(\tR\tpageToken\"|\n" + + "page_token\x18\x05 \x01(\tR\tpageTokenB\x16\n" + + "\x14received_lower_boundB\x16\n" + + "\x14received_upper_bound\"\xe0\x01\n" + "\fListResponse\x12D\n" + "\brequests\x18\x01 \x03(\v2(.uber.submitqueue.gateway.RequestSummaryR\brequests\x12&\n" + - "\x0fnext_page_token\x18\x02 \x01(\tR\rnextPageToken\"\x1b\n" + + "\x0fnext_page_token\x18\x02 \x01(\tR\rnextPageToken\x124\n" + + "\x17received_at_or_after_ms\x18\x03 \x01(\x03R\x13receivedAtOrAfterMs\x12,\n" + + "\x12received_before_ms\x18\x04 \x01(\x03R\x10receivedBeforeMs\"\x1b\n" + "\x05Queue\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\"\x13\n" + "\x11ListQueuesRequest\"M\n" + @@ -1664,6 +1730,10 @@ func file_gateway_proto_init() { if File_gateway_proto != nil { return } + file_gateway_proto_msgTypes[11].OneofWrappers = []any{ + (*ListRequest_ReceivedAtOrAfterMs)(nil), + (*ListRequest_ReceivedBeforeMs)(nil), + } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ diff --git a/api/submitqueue/gateway/protopb/gateway.pb.yarpc.go b/api/submitqueue/gateway/protopb/gateway.pb.yarpc.go index 8654d566a..7b0b580cd 100644 --- a/api/submitqueue/gateway/protopb/gateway.pb.yarpc.go +++ b/api/submitqueue/gateway/protopb/gateway.pb.yarpc.go @@ -626,80 +626,82 @@ var ( var yarpcFileDescriptorClosuref1a937782ebbded5 = [][]byte{ // gateway.proto []byte{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xcb, 0x6e, 0xdb, 0x46, - 0x17, 0x36, 0x75, 0xb3, 0x74, 0x24, 0xfb, 0xf7, 0x3f, 0x49, 0x53, 0x42, 0xb1, 0x11, 0x9b, 0x4d, - 0x1d, 0x01, 0x0d, 0xa4, 0xc0, 0x4d, 0x9b, 0xa2, 0x41, 0x02, 0xc4, 0x89, 0x73, 0x01, 0xea, 0xd4, - 0xa1, 0x6b, 0x04, 0x08, 0x50, 0x08, 0x23, 0xe9, 0x44, 0x26, 0x6c, 0x92, 0xf2, 0xcc, 0xc8, 0xad, - 0x8c, 0x2e, 0xba, 0xe9, 0xa6, 0xeb, 0xee, 0x8b, 0x3e, 0x4b, 0x17, 0x45, 0xdf, 0xaa, 0x98, 0x0b, - 0x29, 0xd2, 0x16, 0x19, 0xc5, 0xc9, 0x4a, 0x9c, 0x99, 0xf3, 0x9d, 0xfb, 0x7c, 0x73, 0x04, 0x4b, - 0x43, 0x2a, 0xf0, 0x27, 0x3a, 0x69, 0x8f, 0x58, 0x28, 0x42, 0x62, 0x8f, 0x7b, 0xc8, 0xda, 0x7c, - 0xdc, 0xf3, 0x3d, 0x71, 0x32, 0xc6, 0x31, 0xb6, 0xcd, 0x79, 0xd3, 0xa1, 0x23, 0xaf, 0xd3, 0xa3, - 0x1c, 0x3b, 0xfd, 0x43, 0x1a, 0x0c, 0xb1, 0xa3, 0x00, 0x66, 0xa1, 0xd1, 0xcd, 0x3b, 0xb1, 0x8c, - 0x8f, 0x6c, 0x88, 0x5c, 0x30, 0x2a, 0x70, 0x38, 0x31, 0xa2, 0xa9, 0x3d, 0x8d, 0x70, 0x6e, 0x41, - 0x7d, 0xcf, 0x0b, 0x86, 0x2e, 0x9e, 0x8c, 0x91, 0x0b, 0x62, 0xc3, 0xa2, 0x8f, 0x9c, 0xd3, 0x21, - 0xda, 0xd6, 0xba, 0xd5, 0xaa, 0xb9, 0xd1, 0xd2, 0xf9, 0xcd, 0x82, 0x86, 0x96, 0xe4, 0xa3, 0x30, - 0xe0, 0x98, 0x2d, 0x4a, 0x36, 0xa0, 0xc1, 0x91, 0x9d, 0x7a, 0x7d, 0xec, 0x06, 0xd4, 0x47, 0xbb, - 0xa0, 0x8e, 0xeb, 0x66, 0xef, 0x25, 0xf5, 0x91, 0xac, 0x42, 0x4d, 0x78, 0x3e, 0x72, 0x41, 0xfd, - 0x91, 0x5d, 0x5c, 0xb7, 0x5a, 0x45, 0x77, 0xba, 0x41, 0x9a, 0x50, 0x3d, 0x0c, 0xb9, 0x50, 0xe0, - 0x92, 0x02, 0xc7, 0x6b, 0xe7, 0x0f, 0x0b, 0xea, 0xdf, 0xd1, 0x60, 0x10, 0x79, 0x7c, 0x15, 0xca, - 0x2a, 0x4f, 0xc6, 0x09, 0xbd, 0x20, 0x77, 0xa0, 0xa2, 0x13, 0xa3, 0x8c, 0xd7, 0xb7, 0xec, 0xb6, - 0xca, 0xab, 0x4c, 0x4d, 0xdb, 0x64, 0xec, 0xb1, 0xfa, 0x71, 0x8d, 0x1c, 0x79, 0x00, 0xd5, 0x28, - 0x35, 0xca, 0xe6, 0xf2, 0xd6, 0x46, 0x02, 0x93, 0x4e, 0xdd, 0xbe, 0xf9, 0x70, 0x63, 0x88, 0xe3, - 0x40, 0x43, 0x7b, 0x65, 0xb2, 0x43, 0xa0, 0xc4, 0x4f, 0xbc, 0x81, 0xf1, 0x4a, 0x7d, 0x3b, 0xaf, - 0x60, 0xe9, 0x31, 0x0d, 0xfa, 0x78, 0x1c, 0xf9, 0x3e, 0x43, 0x88, 0x5c, 0x83, 0x0a, 0x43, 0xca, - 0xc3, 0xc0, 0xa4, 0xcd, 0xac, 0xa6, 0x71, 0x16, 0x13, 0x71, 0x3a, 0x2b, 0xb0, 0x1c, 0xa9, 0xd4, - 0x86, 0x9d, 0x7f, 0x0b, 0xb0, 0x6c, 0xf4, 0xef, 0x8f, 0x7d, 0x9f, 0xb2, 0xc9, 0x4c, 0x33, 0xb1, - 0xba, 0x42, 0x32, 0x6d, 0x37, 0xa0, 0xae, 0xd3, 0xd1, 0x1d, 0x33, 0x8f, 0xdb, 0xc5, 0xf5, 0x62, - 0xab, 0xe6, 0x82, 0xde, 0x3a, 0x60, 0x1e, 0x27, 0x37, 0x61, 0x99, 0x61, 0x1f, 0xbd, 0x53, 0x1c, - 0x74, 0xa9, 0xe8, 0xfa, 0x5c, 0xe5, 0xaa, 0xe8, 0x36, 0xa2, 0xdd, 0x47, 0x62, 0x97, 0xcb, 0x18, - 0xb8, 0xa0, 0x62, 0xcc, 0xed, 0xb2, 0x8e, 0x41, 0xaf, 0xc8, 0x1a, 0xc0, 0x31, 0xe5, 0xa2, 0x8b, - 0x8c, 0x85, 0xcc, 0xae, 0xa8, 0xb3, 0x9a, 0xdc, 0xd9, 0x91, 0x1b, 0xc4, 0x85, 0xaa, 0x8f, 0x82, - 0x0e, 0xa8, 0xa0, 0xf6, 0xe2, 0x7a, 0xb1, 0x55, 0xdf, 0xfa, 0xba, 0x9d, 0x75, 0x1d, 0xda, 0xe9, - 0x18, 0xdb, 0xbb, 0x06, 0xb8, 0x13, 0x08, 0x36, 0x71, 0x63, 0x3d, 0xcd, 0xfb, 0xb0, 0x94, 0x3a, - 0x22, 0x2b, 0x50, 0x3c, 0xc2, 0x89, 0xc9, 0x85, 0xfc, 0x94, 0xa9, 0x38, 0xa5, 0xc7, 0xd3, 0x54, - 0xa8, 0xc5, 0xb7, 0x85, 0x6f, 0x2c, 0xe7, 0x39, 0xac, 0x3e, 0x43, 0x91, 0xb6, 0xb4, 0x3d, 0x79, - 0xf1, 0x24, 0xaf, 0x7e, 0x33, 0x13, 0xeb, 0xf4, 0x61, 0x2d, 0x43, 0x93, 0xe9, 0x97, 0x6d, 0x58, - 0x64, 0xfa, 0x54, 0x69, 0xab, 0x6f, 0xb5, 0xe6, 0x0d, 0xdd, 0x8d, 0x80, 0xce, 0x1b, 0xf8, 0x6c, - 0x86, 0x11, 0xdd, 0xe7, 0x07, 0xee, 0x8b, 0xc8, 0xeb, 0x35, 0x80, 0x69, 0x91, 0x8d, 0xef, 0xb5, - 0xb8, 0xc6, 0x19, 0x01, 0x1c, 0xc3, 0xcd, 0x7c, 0xdd, 0x26, 0x8e, 0x27, 0x50, 0x35, 0xee, 0x70, - 0xdb, 0x52, 0x35, 0x9c, 0x3f, 0x90, 0x18, 0xe9, 0xfc, 0x2d, 0x2f, 0xb9, 0xc7, 0x45, 0xfe, 0x25, - 0xbf, 0x0b, 0x9f, 0x26, 0x9b, 0x31, 0x64, 0x5d, 0xfa, 0x56, 0x20, 0x93, 0x5d, 0x59, 0x50, 0x5d, - 0x79, 0x65, 0xda, 0x95, 0xdf, 0xb3, 0x47, 0xf2, 0x6c, 0x97, 0x93, 0xdb, 0x40, 0x62, 0x54, 0x0f, - 0xdf, 0x86, 0x0c, 0x25, 0x40, 0x73, 0xd0, 0x4a, 0x74, 0xb2, 0xad, 0x0e, 0x76, 0x39, 0xb9, 0x0e, - 0xb5, 0x11, 0x1d, 0x62, 0x97, 0x7b, 0x67, 0x9a, 0x8b, 0xca, 0x6e, 0x55, 0x6e, 0xec, 0x7b, 0x67, - 0x28, 0x33, 0xa9, 0x0e, 0x45, 0x78, 0x84, 0x81, 0xe9, 0x75, 0x25, 0xfe, 0x83, 0xdc, 0x70, 0x7e, - 0x81, 0x86, 0x0e, 0xe2, 0x63, 0xe6, 0x86, 0x6c, 0xc2, 0xff, 0x02, 0xfc, 0x59, 0x74, 0x13, 0x96, - 0x75, 0xa5, 0x96, 0xe4, 0xf6, 0x5e, 0x6c, 0xfd, 0x3a, 0x94, 0x5f, 0xa9, 0x34, 0x11, 0x28, 0x29, - 0x26, 0x35, 0x5d, 0xaa, 0x58, 0xf4, 0x0a, 0xfc, 0x5f, 0xba, 0xa6, 0x04, 0xb8, 0x31, 0xe5, 0xec, - 0x02, 0x49, 0x6e, 0x1a, 0xaf, 0xef, 0x41, 0x45, 0x79, 0x16, 0xf9, 0x7c, 0x23, 0xdb, 0x67, 0x85, - 0x74, 0x8d, 0x78, 0xfa, 0xf6, 0x3c, 0xf7, 0xb8, 0x08, 0x2f, 0x7b, 0x7b, 0xfe, 0x2a, 0x40, 0xc3, - 0x28, 0xd8, 0x39, 0xc5, 0x40, 0xc8, 0x17, 0x26, 0x7e, 0x2d, 0x64, 0xf5, 0x2c, 0x55, 0xbd, 0x7a, - 0xbc, 0x97, 0xe2, 0xa0, 0x42, 0x0e, 0x07, 0x15, 0xcf, 0x73, 0xd0, 0x5e, 0x82, 0x83, 0x4a, 0x2a, - 0xde, 0xbb, 0xd9, 0xf1, 0x26, 0x7d, 0xca, 0x62, 0x20, 0x19, 0xa6, 0x98, 0x8c, 0xd0, 0xb4, 0x87, - 0xfa, 0x96, 0x61, 0xa2, 0x04, 0x19, 0x0e, 0xd4, 0x8b, 0x0f, 0xe3, 0xaa, 0x6e, 0x92, 0x61, 0x52, - 0xd9, 0x36, 0x75, 0x7c, 0x08, 0x15, 0x65, 0x26, 0xaa, 0xe3, 0xe6, 0x7c, 0x71, 0xb9, 0x06, 0x95, - 0x66, 0x97, 0xd8, 0xc0, 0xc7, 0x61, 0x97, 0x41, 0xfc, 0x66, 0x19, 0xc5, 0x33, 0x9b, 0x63, 0x1a, - 0x41, 0xe1, 0x52, 0x11, 0x04, 0x49, 0x0e, 0x9b, 0x15, 0x81, 0xc9, 0xd4, 0x53, 0xa8, 0x1d, 0xaa, - 0x53, 0x0f, 0xe7, 0xbf, 0xa8, 0x46, 0x9f, 0x3b, 0x85, 0x3a, 0x1b, 0x50, 0xd6, 0x4d, 0x95, 0x3d, - 0x55, 0x21, 0x5c, 0x3b, 0x08, 0x18, 0xf6, 0xc3, 0x61, 0xe0, 0x9d, 0xe1, 0x40, 0x5d, 0x20, 0x8d, - 0xf9, 0x0a, 0xca, 0xba, 0x45, 0xf5, 0x73, 0x90, 0x73, 0xeb, 0x94, 0xbc, 0xab, 0xa5, 0x33, 0xf2, - 0xfb, 0xab, 0x05, 0x57, 0x8d, 0x9f, 0x2f, 0x43, 0xf1, 0x34, 0x1c, 0x07, 0x83, 0x0f, 0xb2, 0x12, - 0x55, 0xa7, 0x90, 0xa8, 0x4e, 0xba, 0xf0, 0xc5, 0x73, 0x85, 0xdf, 0xfa, 0xa7, 0x0a, 0x64, 0x5f, - 0xe9, 0x55, 0x41, 0x3e, 0xd3, 0x6a, 0xc9, 0x6b, 0x28, 0xc9, 0xa9, 0x92, 0x7c, 0x9e, 0x6d, 0x39, - 0x31, 0x9f, 0x36, 0x37, 0xdf, 0x25, 0x66, 0xa6, 0xa0, 0x05, 0xe2, 0x01, 0x4c, 0xc9, 0x8c, 0x7c, - 0x91, 0x8d, 0xbb, 0xc0, 0x83, 0xcd, 0xdb, 0xf3, 0x09, 0xc7, 0xa6, 0x5e, 0x43, 0x49, 0xce, 0x7e, - 0x79, 0x31, 0x24, 0x26, 0xd6, 0xbc, 0x18, 0x92, 0x23, 0xa4, 0xb3, 0x40, 0x7e, 0x84, 0x8a, 0x9e, - 0xee, 0xc8, 0xad, 0x6c, 0x4c, 0x6a, 0xa4, 0x6c, 0xb6, 0xde, 0x2d, 0x18, 0xab, 0xff, 0xdd, 0x82, - 0x4f, 0x66, 0x4e, 0x25, 0x24, 0x67, 0xee, 0xca, 0x1b, 0x88, 0x9a, 0xf7, 0xde, 0x1b, 0x17, 0x3b, - 0xf3, 0xa7, 0x35, 0x73, 0xd8, 0x8a, 0x6f, 0x27, 0x79, 0xf0, 0x5e, 0xba, 0xcf, 0xf3, 0x52, 0xf3, - 0xe1, 0x65, 0xe1, 0xa9, 0x32, 0x7b, 0x5c, 0xe4, 0x96, 0x79, 0x3a, 0xb3, 0xe4, 0x96, 0x39, 0x31, - 0x15, 0x5c, 0xa8, 0x43, 0x82, 0xbb, 0xe7, 0xab, 0xc3, 0xc5, 0xa7, 0x75, 0xbe, 0x3a, 0xcc, 0x78, - 0x24, 0x2e, 0xd4, 0xe1, 0x22, 0x4b, 0xce, 0x57, 0x87, 0xcc, 0xf7, 0x61, 0xbe, 0x3a, 0x64, 0x93, - 0xb3, 0xb3, 0xb0, 0x7d, 0x04, 0xab, 0xfd, 0xd0, 0xcf, 0x54, 0xb3, 0xdd, 0x30, 0xdc, 0xb2, 0x27, - 0xff, 0xe0, 0xee, 0x59, 0x6f, 0xee, 0x0f, 0x3d, 0x71, 0x38, 0xee, 0xb5, 0xfb, 0xa1, 0xdf, 0x91, - 0xa0, 0x4e, 0x02, 0xd4, 0x91, 0x7f, 0x98, 0x93, 0x6b, 0xa3, 0x44, 0xff, 0x63, 0x1e, 0xf5, 0x7a, - 0x15, 0xf5, 0xf1, 0xe5, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x4d, 0x90, 0x8f, 0x2e, 0xa7, 0x0f, - 0x00, 0x00, + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x5d, 0x6f, 0x13, 0x47, + 0x17, 0xce, 0xfa, 0x2b, 0xf6, 0xb1, 0x93, 0x37, 0xef, 0x90, 0xa6, 0x96, 0x49, 0x44, 0xb2, 0xa5, + 0xc1, 0x52, 0x91, 0x8d, 0x52, 0x0a, 0x55, 0x11, 0x48, 0x18, 0x02, 0x41, 0x6a, 0x68, 0xd8, 0x34, + 0x42, 0x42, 0xaa, 0xac, 0xb1, 0x7d, 0x70, 0x56, 0xc9, 0xee, 0x3a, 0x3b, 0xe3, 0x50, 0xe7, 0xaa, + 0x37, 0xbd, 0xe9, 0x75, 0xef, 0xab, 0xfe, 0x9a, 0xaa, 0xff, 0xa2, 0xff, 0xa2, 0xb7, 0xd5, 0x7c, + 0xec, 0x7a, 0x37, 0xde, 0x5d, 0x4c, 0xe0, 0xca, 0x3b, 0x67, 0xce, 0xe7, 0x73, 0x66, 0x9e, 0x39, + 0x86, 0xa5, 0x21, 0xe5, 0xf8, 0x8e, 0x4e, 0x5a, 0x23, 0xdf, 0xe3, 0x1e, 0xa9, 0x8f, 0x7b, 0xe8, + 0xb7, 0xd8, 0xb8, 0xe7, 0xd8, 0xfc, 0x6c, 0x8c, 0x63, 0x6c, 0xe9, 0xfd, 0x86, 0x49, 0x47, 0x76, + 0xbb, 0x47, 0x19, 0xb6, 0xfb, 0xc7, 0xd4, 0x1d, 0x62, 0x5b, 0x1a, 0xe8, 0x85, 0xb2, 0x6e, 0xdc, + 0x09, 0x75, 0x1c, 0xf4, 0x87, 0xc8, 0xb8, 0x4f, 0x39, 0x0e, 0x27, 0x5a, 0x35, 0x26, 0x53, 0x16, + 0xe6, 0x2d, 0xa8, 0x1e, 0xd8, 0xee, 0xd0, 0xc2, 0xb3, 0x31, 0x32, 0x4e, 0xea, 0xb0, 0xe8, 0x20, + 0x63, 0x74, 0x88, 0x75, 0x63, 0xd3, 0x68, 0x56, 0xac, 0x60, 0x69, 0xfe, 0x6a, 0x40, 0x4d, 0x69, + 0xb2, 0x91, 0xe7, 0x32, 0x4c, 0x57, 0x25, 0x5b, 0x50, 0x63, 0xe8, 0x9f, 0xdb, 0x7d, 0xec, 0xba, + 0xd4, 0xc1, 0x7a, 0x4e, 0x6e, 0x57, 0xb5, 0xec, 0x25, 0x75, 0x90, 0xac, 0x43, 0x85, 0xdb, 0x0e, + 0x32, 0x4e, 0x9d, 0x51, 0x3d, 0xbf, 0x69, 0x34, 0xf3, 0xd6, 0x54, 0x40, 0x1a, 0x50, 0x3e, 0xf6, + 0x18, 0x97, 0xc6, 0x05, 0x69, 0x1c, 0xae, 0xcd, 0xdf, 0x0d, 0xa8, 0x7e, 0x4f, 0xdd, 0x41, 0x90, + 0xf1, 0x2a, 0x14, 0x25, 0x4e, 0x3a, 0x09, 0xb5, 0x20, 0x77, 0xa0, 0xa4, 0x80, 0x91, 0xc1, 0xab, + 0x3b, 0xf5, 0x96, 0xc4, 0x55, 0x40, 0xd3, 0xd2, 0x88, 0x3d, 0x91, 0x3f, 0x96, 0xd6, 0x23, 0x0f, + 0xa1, 0x1c, 0x40, 0x23, 0x63, 0x2e, 0xef, 0x6c, 0x45, 0x6c, 0xe2, 0xd0, 0x1d, 0xea, 0x0f, 0x2b, + 0x34, 0x31, 0x4d, 0xa8, 0xa9, 0xac, 0x34, 0x3a, 0x04, 0x0a, 0xec, 0xcc, 0x1e, 0xe8, 0xac, 0xe4, + 0xb7, 0xf9, 0x0a, 0x96, 0x9e, 0x50, 0xb7, 0x8f, 0xa7, 0x41, 0xee, 0x09, 0x4a, 0x64, 0x0d, 0x4a, + 0x3e, 0x52, 0xe6, 0xb9, 0x1a, 0x36, 0xbd, 0x9a, 0xd6, 0x99, 0x8f, 0xd4, 0x69, 0xae, 0xc0, 0x72, + 0xe0, 0x52, 0x05, 0x36, 0xff, 0xce, 0xc1, 0xb2, 0xf6, 0x7f, 0x38, 0x76, 0x1c, 0xea, 0x4f, 0x12, + 0xc3, 0x84, 0xee, 0x72, 0x51, 0xd8, 0x6e, 0x40, 0x55, 0xc1, 0xd1, 0x1d, 0xfb, 0x36, 0xab, 0xe7, + 0x37, 0xf3, 0xcd, 0x8a, 0x05, 0x4a, 0x74, 0xe4, 0xdb, 0x8c, 0xdc, 0x84, 0x65, 0x1f, 0xfb, 0x68, + 0x9f, 0xe3, 0xa0, 0x4b, 0x79, 0xd7, 0x61, 0x12, 0xab, 0xbc, 0x55, 0x0b, 0xa4, 0x8f, 0xf9, 0x3e, + 0x13, 0x35, 0x30, 0x4e, 0xf9, 0x98, 0xd5, 0x8b, 0xaa, 0x06, 0xb5, 0x22, 0x1b, 0x00, 0xa7, 0x94, + 0xf1, 0x2e, 0xfa, 0xbe, 0xe7, 0xd7, 0x4b, 0x72, 0xaf, 0x22, 0x24, 0xbb, 0x42, 0x40, 0x2c, 0x28, + 0x3b, 0xc8, 0xe9, 0x80, 0x72, 0x5a, 0x5f, 0xdc, 0xcc, 0x37, 0xab, 0x3b, 0xf7, 0x5a, 0x69, 0xd7, + 0xa1, 0x15, 0xaf, 0xb1, 0xb5, 0xaf, 0x0d, 0x77, 0x5d, 0xee, 0x4f, 0xac, 0xd0, 0x4f, 0xe3, 0x01, + 0x2c, 0xc5, 0xb6, 0xc8, 0x0a, 0xe4, 0x4f, 0x70, 0xa2, 0xb1, 0x10, 0x9f, 0x02, 0x8a, 0x73, 0x7a, + 0x3a, 0x85, 0x42, 0x2e, 0xbe, 0xcb, 0x7d, 0x6b, 0x98, 0x7b, 0xb0, 0xfe, 0x1c, 0x79, 0x3c, 0x52, + 0x67, 0xf2, 0xe2, 0x69, 0x56, 0xff, 0x12, 0x81, 0x35, 0xfb, 0xb0, 0x91, 0xe2, 0x49, 0x9f, 0x97, + 0x0e, 0x2c, 0xfa, 0x6a, 0x57, 0x7a, 0xab, 0xee, 0x34, 0xe7, 0x2d, 0xdd, 0x0a, 0x0c, 0xcd, 0x37, + 0xf0, 0x45, 0x42, 0x10, 0x75, 0xce, 0x8f, 0xac, 0x17, 0x41, 0xd6, 0x1b, 0x00, 0xd3, 0x26, 0xeb, + 0xdc, 0x2b, 0x61, 0x8f, 0x53, 0x0a, 0x38, 0x85, 0x9b, 0xd9, 0xbe, 0x75, 0x1d, 0x4f, 0xa1, 0xac, + 0xd3, 0x61, 0x75, 0x43, 0xf6, 0x70, 0xfe, 0x42, 0x42, 0x4b, 0xf3, 0x5f, 0x71, 0xc9, 0x6d, 0xc6, + 0xb3, 0x2f, 0xf9, 0x3d, 0xf8, 0x3c, 0x7a, 0x18, 0x3d, 0xbf, 0x4b, 0xdf, 0x72, 0xf4, 0xc5, 0xa9, + 0x14, 0xb9, 0xe7, 0xf7, 0x16, 0xac, 0x6b, 0xd3, 0x73, 0xf9, 0x83, 0xff, 0x58, 0xec, 0xee, 0x33, + 0xd2, 0x02, 0x12, 0xda, 0xf5, 0xf0, 0xad, 0xe7, 0xa3, 0x30, 0x91, 0x2c, 0xb4, 0x67, 0x58, 0x2b, + 0xc1, 0x5e, 0x47, 0x6e, 0xed, 0x33, 0x72, 0x1d, 0x2a, 0x23, 0x3a, 0xc4, 0x2e, 0xb3, 0x2f, 0x14, + 0x1f, 0x15, 0xad, 0xb2, 0x10, 0x1c, 0xda, 0x17, 0x28, 0xd0, 0x94, 0x9b, 0xdc, 0x3b, 0x41, 0x57, + 0x9f, 0x77, 0xa9, 0xfe, 0xa3, 0x10, 0x74, 0xd6, 0x60, 0x35, 0x8c, 0x75, 0xea, 0xbd, 0x43, 0xbf, + 0xdb, 0xf3, 0xc6, 0xee, 0x20, 0x26, 0x1f, 0x8f, 0x46, 0x81, 0xdc, 0xfc, 0xc7, 0x80, 0x9a, 0xaa, + 0xfc, 0x53, 0x02, 0x4a, 0xb6, 0xe1, 0x7f, 0x2e, 0xfe, 0xcc, 0xbb, 0x91, 0x54, 0x55, 0x7b, 0x97, + 0x84, 0xf8, 0x20, 0x48, 0x97, 0xdc, 0x4d, 0x87, 0x54, 0xb1, 0x74, 0x22, 0xa0, 0xb7, 0x13, 0x01, + 0x55, 0xcc, 0x30, 0x03, 0xa7, 0x79, 0x1d, 0x8a, 0xaf, 0x64, 0xff, 0x08, 0x14, 0x24, 0xc5, 0xeb, + 0xeb, 0x23, 0xe9, 0xfd, 0x1a, 0xfc, 0x5f, 0x94, 0x2f, 0x15, 0x98, 0x2e, 0xc7, 0xdc, 0x07, 0x12, + 0x15, 0x6a, 0x64, 0xee, 0x43, 0x49, 0x56, 0x1f, 0xe0, 0x72, 0x23, 0x1d, 0x17, 0x69, 0x69, 0x69, + 0xf5, 0xf8, 0xb5, 0xde, 0xb3, 0x19, 0xf7, 0xae, 0x7a, 0xad, 0xff, 0xcc, 0x41, 0x4d, 0x3b, 0xd8, + 0x3d, 0x47, 0x97, 0x8b, 0xa7, 0x2f, 0x7c, 0xc6, 0x04, 0x06, 0x86, 0xc4, 0xa0, 0x1a, 0xca, 0x62, + 0xe4, 0x98, 0xcb, 0x20, 0xc7, 0xfc, 0x65, 0x72, 0x3c, 0x88, 0x90, 0x63, 0x41, 0xd6, 0x7b, 0x37, + 0xbd, 0xde, 0x68, 0x4e, 0x69, 0xd4, 0x28, 0xca, 0xe4, 0x93, 0x11, 0xea, 0x33, 0x2b, 0xbf, 0x45, + 0x99, 0x28, 0x8c, 0x34, 0x39, 0xab, 0xc5, 0xc7, 0x91, 0x68, 0x37, 0x4a, 0x7d, 0x31, 0xb4, 0x75, + 0x1f, 0x1f, 0x41, 0x49, 0x86, 0x09, 0xfa, 0xb8, 0x3d, 0x5f, 0x5d, 0x96, 0xb6, 0x8a, 0xd3, 0x5e, + 0x18, 0xe0, 0xd3, 0xd0, 0xde, 0x20, 0x7c, 0x4c, 0xb5, 0xe3, 0xc4, 0xc3, 0x31, 0xad, 0x20, 0x77, + 0xa5, 0x0a, 0xdc, 0x28, 0xb9, 0x26, 0x55, 0xa0, 0x91, 0x7a, 0x06, 0x95, 0x63, 0xb9, 0x6b, 0xe3, + 0xfc, 0x64, 0xa0, 0xfd, 0x59, 0x53, 0x53, 0x73, 0x0b, 0x8a, 0xea, 0x50, 0xa5, 0x8f, 0x7b, 0x08, + 0x6b, 0x47, 0xae, 0x8f, 0x7d, 0x6f, 0xe8, 0xda, 0x17, 0x38, 0x90, 0x17, 0x48, 0xd9, 0x7c, 0x03, + 0x45, 0x75, 0x44, 0xd5, 0x3b, 0x95, 0x71, 0xeb, 0xa4, 0xbe, 0xa5, 0xb4, 0x53, 0xf0, 0xfd, 0xc5, + 0x80, 0x55, 0x9d, 0xe7, 0x4b, 0x8f, 0x3f, 0x13, 0x14, 0xf8, 0x51, 0x51, 0x82, 0xee, 0xe4, 0x22, + 0xdd, 0x89, 0x37, 0x3e, 0x7f, 0xa9, 0xf1, 0x3b, 0x7f, 0x95, 0x81, 0x1c, 0x4a, 0xbf, 0xb2, 0xc8, + 0xe7, 0xca, 0x2d, 0x79, 0x0d, 0x05, 0x31, 0xee, 0x92, 0x2f, 0xd3, 0x23, 0x47, 0x06, 0xe7, 0xc6, + 0xf6, 0xfb, 0xd4, 0xf4, 0x78, 0xb6, 0x40, 0x6c, 0x80, 0x29, 0x99, 0x91, 0xaf, 0xd2, 0xed, 0x66, + 0x78, 0xb0, 0x71, 0x7b, 0x3e, 0xe5, 0x30, 0xd4, 0x6b, 0x28, 0x88, 0xa1, 0x34, 0xab, 0x86, 0xc8, + 0x28, 0x9d, 0x55, 0x43, 0x74, 0xb6, 0x35, 0x17, 0xc8, 0x4f, 0x50, 0x52, 0x63, 0x27, 0xb9, 0x95, + 0x6e, 0x13, 0x9b, 0x75, 0x1b, 0xcd, 0xf7, 0x2b, 0x86, 0xee, 0x7f, 0x33, 0xe0, 0xb3, 0xc4, 0x71, + 0x89, 0x64, 0x0c, 0x84, 0x59, 0x93, 0x5a, 0xe3, 0xfe, 0x07, 0xdb, 0x85, 0xc9, 0xfc, 0x61, 0x24, + 0x4e, 0x81, 0xe1, 0xed, 0x24, 0x0f, 0x3f, 0xc8, 0xf7, 0x65, 0x5e, 0x6a, 0x3c, 0xba, 0xaa, 0x79, + 0xac, 0xcd, 0x36, 0xe3, 0x99, 0x6d, 0x9e, 0x0e, 0x53, 0x99, 0x6d, 0x8e, 0x4c, 0x1e, 0x33, 0x7d, + 0x88, 0x70, 0xf7, 0x7c, 0x7d, 0x98, 0x7d, 0x5a, 0xe7, 0xeb, 0x43, 0xc2, 0x23, 0x31, 0xd3, 0x87, + 0x59, 0x96, 0x9c, 0xaf, 0x0f, 0xa9, 0xef, 0xc3, 0x7c, 0x7d, 0x48, 0x27, 0x67, 0x73, 0xa1, 0x73, + 0x02, 0xeb, 0x7d, 0xcf, 0x49, 0x75, 0xd3, 0xa9, 0x69, 0x6e, 0x39, 0x10, 0xff, 0xbc, 0x0f, 0x8c, + 0x37, 0x0f, 0x86, 0x36, 0x3f, 0x1e, 0xf7, 0x5a, 0x7d, 0xcf, 0x69, 0x0b, 0xa3, 0x76, 0xc4, 0xa8, + 0x2d, 0xfe, 0xc9, 0x47, 0xd7, 0xda, 0x89, 0xfa, 0x2b, 0x3f, 0xea, 0xf5, 0x4a, 0xf2, 0xe3, 0xeb, + 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0x9c, 0x8d, 0x26, 0x2d, 0x40, 0x10, 0x00, 0x00, }, // api/base/change/proto/change.proto []byte{ diff --git a/doc/rfc/submitqueue/status-list-api.md b/doc/rfc/submitqueue/status-list-api.md index 5c48aafc4..922d9032d 100644 --- a/doc/rfc/submitqueue/status-list-api.md +++ b/doc/rfc/submitqueue/status-list-api.md @@ -59,9 +59,9 @@ The URI result is intentionally unpaginated and has a hard maximum of 100 reques ### List -`List` is a queue-scoped request receipt history query. It requires a queue and a bounded half-open receipt-time interval. Both bounds are required, and the start must be less than the end. +`List` is a queue-scoped request receipt history query. It requires a queue and a bounded half-open receipt-time interval. The first page must supply both bounds, and the start must be less than the end. -Results are ordered by `(received_at_ms DESC, sqid DESC)`, so the newest requests are returned first. Pagination uses an opaque token bound to the original queue and time range plus the last returned ordering tuple. Reusing a token with different query parameters is invalid. +Results are ordered by `(received_at_ms DESC, sqid DESC)`, so the newest requests are returned first. Pagination uses an opaque, versioned token bound to the original queue and time range plus the last returned ordering tuple. A continuation needs only the queue and the token: the window comes from the token, and a bound the caller also supplies must match it. Each response echoes the window it was read from, so a client can show it without decoding the token. The token is not signed; the gateway validates every field, and a forged token can only select a window the caller could have asked for directly. Page size is optional and subject to a server default and maximum. diff --git a/doc/rfc/web-ui.md b/doc/rfc/web-ui.md index 49b7dead9..3f24897d5 100644 --- a/doc/rfc/web-ui.md +++ b/doc/rfc/web-ui.md @@ -62,7 +62,7 @@ Summary is the default request view, and `?view=history` makes History shareable ## Behavior -**Windows and pagination.** `List` takes a queue and a half-open receipt window. The module recalculates the trailing 24-hour window on refresh and keeps timestamps out of the URL. Older pages carry the original bounds in an opaque cursor, signed under a scope that names the page, so a cursor cannot be replayed elsewhere. The reference host signs with HMAC under a key separate from any viewer credential. Refreshing an older page returns to the live first page. +**Windows and pagination.** `List` takes a queue and a half-open receipt window. The module recalculates the trailing 24-hour window on refresh and keeps timestamps out of the URL. Older pages carry the gateway's continuation token verbatim as `?page=`; the token holds the original bounds, and the page shows the window the gateway echoes back. The module neither wraps nor signs the token, so hosts need no secret, and a token the gateway rejects redirects to the live first page. Refreshing an older page returns to the live first page. **Refresh and polling.** A refresh does not schedule the next until it has finished rendering. The wait is the poll interval plus jitter, grows across consecutive transport failures, and pauses while the document is hidden or the browser is offline. A request view stops once the summary is terminal and loaded history contains the same terminal status, or when a terminal summary pairs with a non-retryable history error that polling cannot fix. A queue list keeps polling for the life of its window. A change page that scans the receipt window refreshes only on request, because each render reads up to ten queue pages; a pinned version, found by one exact lookup, keeps polling. @@ -92,7 +92,7 @@ The workspace lives under `web/` and repeats the Go root layout; the UI is a gat ## Acceptance -- Unit tests cover route round-trips, readable paths, timestamp conversion, stable list bounds, cursor scoping and tampering, error classification and host classifiers, catalog caching, request-scoped logging, gateway and page metrics, mount paths and authorization, client navigation, and polling (transition-aware single flight, progressive backoff, hidden and offline pause, history-aware terminal stop). +- Unit tests cover route round-trips, readable paths, timestamp conversion, stable list bounds, token-only continuation and rejected tokens, error classification and host classifiers, catalog caching, request-scoped logging, gateway and page metrics, mount paths and authorization, client navigation, and polling (transition-aware single flight, progressive backoff, hidden and offline pause, history-aware terminal stop). - Every protected host layout and route repeats the authentication check rather than relying only on the Next.js proxy. - `//web/test/e2e/submitqueue:web_test` runs the Bazel-built Go services, the Bazel-built web image, and a digest-pinned MySQL image with pulls and Dockerfile builds disabled. It covers the Basic-auth challenge, explicit h2c, newest-first lists, client-side navigation, a slash-containing request ID, lifecycle and build history, paginated change history, and axe checks. - Built package tarballs are inspected for contents and forbidden imports, typechecked by a consumer written only against public entry points, rendered by a plain Node consumer through `handle`, and server-rendered and hydrated under React 18. diff --git a/service/submitqueue/docker-compose.web.yml b/service/submitqueue/docker-compose.web.yml index ad8450557..e9a3b0962 100644 --- a/service/submitqueue/docker-compose.web.yml +++ b/service/submitqueue/docker-compose.web.yml @@ -16,7 +16,6 @@ services: - SUBMITQUEUE_GATEWAY_URL=http://gateway-service:8080 - SUBMITQUEUE_GATEWAY_ALLOW_PLAINTEXT=true - SUBMITQUEUE_WEB_TOKEN=${SUBMITQUEUE_WEB_TOKEN:?set SUBMITQUEUE_WEB_TOKEN} - - SUBMITQUEUE_WEB_CURSOR_SECRET=${SUBMITQUEUE_WEB_CURSOR_SECRET:?set SUBMITQUEUE_WEB_CURSOR_SECRET} depends_on: gateway-service: condition: service_started diff --git a/service/submitqueue/gateway/server/mapper/BUILD.bazel b/service/submitqueue/gateway/server/mapper/BUILD.bazel index 69bb3e2d6..b014aa3ea 100644 --- a/service/submitqueue/gateway/server/mapper/BUILD.bazel +++ b/service/submitqueue/gateway/server/mapper/BUILD.bazel @@ -40,5 +40,6 @@ go_test( "//submitqueue/entity:go_default_library", "@com_github_stretchr_testify//assert:go_default_library", "@com_github_stretchr_testify//require:go_default_library", + "@org_golang_google_protobuf//proto:go_default_library", ], ) diff --git a/service/submitqueue/gateway/server/mapper/list.go b/service/submitqueue/gateway/server/mapper/list.go index a99a741f4..f16175578 100644 --- a/service/submitqueue/gateway/server/mapper/list.go +++ b/service/submitqueue/gateway/server/mapper/list.go @@ -22,18 +22,22 @@ import ( // ProtoToListRequest maps the wire request to the entity request the controller operates on. func ProtoToListRequest(req *pb.ListRequest) entity.ListRequest { return entity.ListRequest{ - Queue: req.GetQueue(), - ReceivedAtOrAfterMs: req.GetReceivedAtOrAfterMs(), - ReceivedBeforeMs: req.GetReceivedBeforeMs(), - PageSize: req.GetPageSize(), - PageToken: req.GetPageToken(), + Queue: req.GetQueue(), + ReceivedAtOrAfterMs: req.GetReceivedAtOrAfterMs(), + HasReceivedAtOrAfterMs: req.GetReceivedLowerBound() != nil, + ReceivedBeforeMs: req.GetReceivedBeforeMs(), + HasReceivedBeforeMs: req.GetReceivedUpperBound() != nil, + PageSize: req.GetPageSize(), + PageToken: req.GetPageToken(), } } // ListResultToProto maps the domain result to the wire response. func ListResultToProto(result entity.ListResult) *pb.ListResponse { return &pb.ListResponse{ - Requests: RequestSummariesToProto(result.Requests), - NextPageToken: result.NextPageToken, + Requests: RequestSummariesToProto(result.Requests), + NextPageToken: result.NextPageToken, + ReceivedAtOrAfterMs: result.ReceivedAtOrAfterMs, + ReceivedBeforeMs: result.ReceivedBeforeMs, } } diff --git a/service/submitqueue/gateway/server/mapper/list_test.go b/service/submitqueue/gateway/server/mapper/list_test.go index 1768e6001..721dc35cf 100644 --- a/service/submitqueue/gateway/server/mapper/list_test.go +++ b/service/submitqueue/gateway/server/mapper/list_test.go @@ -18,26 +18,77 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" pb "github.com/uber/submitqueue/api/submitqueue/gateway/protopb" "github.com/uber/submitqueue/submitqueue/entity" + "google.golang.org/protobuf/proto" ) func TestProtoToListRequest(t *testing.T) { - req := &pb.ListRequest{ - Queue: "q", - ReceivedAtOrAfterMs: 100, - ReceivedBeforeMs: 200, - PageSize: 25, - PageToken: "token", + for _, tt := range []struct { + name string + req *pb.ListRequest + want entity.ListRequest + }{ + {name: "nil request"}, + { + name: "explicit zero bounds keep presence", + req: &pb.ListRequest{ + ReceivedLowerBound: &pb.ListRequest_ReceivedAtOrAfterMs{}, + ReceivedUpperBound: &pb.ListRequest_ReceivedBeforeMs{}, + }, + want: entity.ListRequest{HasReceivedAtOrAfterMs: true, HasReceivedBeforeMs: true}, + }, + { + name: "explicit window and pagination", + req: &pb.ListRequest{ + Queue: "q", PageSize: 25, PageToken: "token", + ReceivedLowerBound: &pb.ListRequest_ReceivedAtOrAfterMs{ReceivedAtOrAfterMs: 100}, + ReceivedUpperBound: &pb.ListRequest_ReceivedBeforeMs{ReceivedBeforeMs: 200}, + }, + want: entity.ListRequest{ + Queue: "q", PageSize: 25, PageToken: "token", + ReceivedAtOrAfterMs: 100, HasReceivedAtOrAfterMs: true, + ReceivedBeforeMs: 200, HasReceivedBeforeMs: true, + }, + }, + { + name: "continuation with omitted bounds", + req: &pb.ListRequest{Queue: "q", PageToken: "token"}, + want: entity.ListRequest{Queue: "q", PageToken: "token"}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, ProtoToListRequest(tt.req)) + }) } +} - assert.Equal(t, entity.ListRequest{ - Queue: "q", - ReceivedAtOrAfterMs: 100, - ReceivedBeforeMs: 200, - PageSize: 25, - PageToken: "token", - }, ProtoToListRequest(req)) +func TestListRequestBoundPresenceSurvivesWire(t *testing.T) { + for _, tt := range []struct { + name string + req *pb.ListRequest + want entity.ListRequest + }{ + {name: "omitted bounds", req: &pb.ListRequest{Queue: "q"}, want: entity.ListRequest{Queue: "q"}}, + { + name: "explicit zero bounds", + req: &pb.ListRequest{ + Queue: "q", + ReceivedLowerBound: &pb.ListRequest_ReceivedAtOrAfterMs{}, + ReceivedUpperBound: &pb.ListRequest_ReceivedBeforeMs{}, + }, + want: entity.ListRequest{Queue: "q", HasReceivedAtOrAfterMs: true, HasReceivedBeforeMs: true}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + wire, err := proto.Marshal(tt.req) + require.NoError(t, err) + var decoded pb.ListRequest + require.NoError(t, proto.Unmarshal(wire, &decoded)) + assert.Equal(t, tt.want, ProtoToListRequest(&decoded)) + }) + } } func TestListResultToProto(t *testing.T) { @@ -51,7 +102,9 @@ func TestListResultToProto(t *testing.T) { LastError: "build failed", Metadata: map[string]string{"build": "url"}, }}, - NextPageToken: "next", + NextPageToken: "next", + ReceivedAtOrAfterMs: 50, + ReceivedBeforeMs: 250, } response := ListResultToProto(result) @@ -62,6 +115,8 @@ func TestListResultToProto(t *testing.T) { ReceivedAtMs: 100, Status: string(entity.RequestStatusError), LastError: "build failed", Metadata: map[string]string{"build": "url"}, }}, - NextPageToken: "next", + NextPageToken: "next", + ReceivedAtOrAfterMs: 50, + ReceivedBeforeMs: 250, }, response) } diff --git a/submitqueue/client/query.go b/submitqueue/client/query.go index 3b4ea02ca..dd61d79fe 100644 --- a/submitqueue/client/query.go +++ b/submitqueue/client/query.go @@ -61,18 +61,18 @@ func (c *Client) List(ctx context.Context, q ListQuery) ([]*pb.RequestSummary, e return nil, fmt.Errorf("queue must not be empty") } - // Both bounds are fixed before the first page. The gateway requires - // received_at_or_after_ms < received_before_ms, so an unset upper bound - // rejects every call, and its continuation token pins both bounds, so one - // recomputed per page would be rejected from the second page on. + // The gateway requires both bounds on the first page; its continuation + // token carries the window, so later pages send only the token. now := time.Now() - req := &pb.ListRequest{ - Queue: q.Queue, - PageSize: int32(q.PageSize), - ReceivedBeforeMs: now.UnixMilli(), - } + var receivedAtOrAfterMs int64 if q.Since > 0 { - req.ReceivedAtOrAfterMs = now.Add(-q.Since).UnixMilli() + receivedAtOrAfterMs = now.Add(-q.Since).UnixMilli() + } + req := &pb.ListRequest{ + Queue: q.Queue, + PageSize: int32(q.PageSize), + ReceivedLowerBound: &pb.ListRequest_ReceivedAtOrAfterMs{ReceivedAtOrAfterMs: receivedAtOrAfterMs}, + ReceivedUpperBound: &pb.ListRequest_ReceivedBeforeMs{ReceivedBeforeMs: now.UnixMilli()}, } var out []*pb.RequestSummary @@ -92,7 +92,7 @@ func (c *Client) List(ctx context.Context, q ListQuery) ([]*pb.RequestSummary, e if resp.GetNextPageToken() == "" || len(resp.GetRequests()) == 0 { return out, nil } - req.PageToken = resp.GetNextPageToken() + req = &pb.ListRequest{Queue: q.Queue, PageSize: int32(q.PageSize), PageToken: resp.GetNextPageToken()} } } diff --git a/submitqueue/client/query_test.go b/submitqueue/client/query_test.go index 92c9c5012..9a61936ad 100644 --- a/submitqueue/client/query_test.go +++ b/submitqueue/client/query_test.go @@ -162,8 +162,8 @@ func TestListWithoutSinceLeavesTheWindowOpen(t *testing.T) { _, err := sq.List(context.Background(), ListQuery{Queue: "q"}) require.NoError(t, err) - assert.Zero(t, gw.lastRequest.GetReceivedAtOrAfterMs(), - "no window means all retained history, not a bound of zero-time") + require.NotNil(t, gw.lastRequest.GetReceivedLowerBound(), "the gateway requires an explicit lower bound on the first page") + assert.Zero(t, gw.lastRequest.GetReceivedAtOrAfterMs(), "no window means all retained history") } func TestListClosesTheWindowAtTheCallTime(t *testing.T) { @@ -185,20 +185,23 @@ func TestListClosesTheWindowAtTheCallTime(t *testing.T) { "the gateway requires received_at_or_after_ms < received_before_ms") } -func TestListKeepsTheWindowFixedAcrossPages(t *testing.T) { - // The continuation token pins both bounds, so a bound recomputed per page - // would be rejected from the second page on. +func TestListContinuesWithOnlyTheToken(t *testing.T) { gw := &pagingGateway{pages: [][]string{{"1"}, {"2"}, {"3"}}} sq, stop := dial(t, gw) defer stop() - _, err := sq.List(context.Background(), ListQuery{Queue: "q", Since: time.Hour}) + _, err := sq.List(context.Background(), ListQuery{Queue: "q", Since: time.Hour, PageSize: 1}) require.NoError(t, err) require.Len(t, gw.requests, 3) + assert.NotNil(t, gw.requests[0].GetReceivedLowerBound()) + assert.NotNil(t, gw.requests[0].GetReceivedUpperBound()) for _, req := range gw.requests[1:] { - assert.Equal(t, gw.requests[0].GetReceivedAtOrAfterMs(), req.GetReceivedAtOrAfterMs()) - assert.Equal(t, gw.requests[0].GetReceivedBeforeMs(), req.GetReceivedBeforeMs()) + assert.Equal(t, "q", req.GetQueue()) + assert.Equal(t, int32(1), req.GetPageSize()) + assert.NotEmpty(t, req.GetPageToken()) + assert.Nil(t, req.GetReceivedLowerBound()) + assert.Nil(t, req.GetReceivedUpperBound()) } } diff --git a/submitqueue/entity/list.go b/submitqueue/entity/list.go index fd10ebc3a..d867b7de0 100644 --- a/submitqueue/entity/list.go +++ b/submitqueue/entity/list.go @@ -18,13 +18,17 @@ package entity type ListRequest struct { // Queue is the exact queue to query. Queue string - // ReceivedAtOrAfterMs is the inclusive lower receipt-time bound in Unix milliseconds. + // ReceivedAtOrAfterMs is the inclusive lower receipt-time bound in Unix milliseconds; meaningful only when HasReceivedAtOrAfterMs. ReceivedAtOrAfterMs int64 - // ReceivedBeforeMs is the exclusive upper receipt-time bound in Unix milliseconds. + // HasReceivedAtOrAfterMs reports whether ReceivedAtOrAfterMs was supplied. + HasReceivedAtOrAfterMs bool + // ReceivedBeforeMs is the exclusive upper receipt-time bound in Unix milliseconds; meaningful only when HasReceivedBeforeMs. ReceivedBeforeMs int64 + // HasReceivedBeforeMs reports whether ReceivedBeforeMs was supplied. + HasReceivedBeforeMs bool // PageSize is the maximum number of results to return. Zero selects the server default. PageSize int32 - // PageToken is an opaque continuation token from a previous result. + // PageToken is an opaque continuation token from a previous result, bound to its queue and receipt window. PageToken string } @@ -34,4 +38,8 @@ type ListResult struct { Requests []RequestSummary // NextPageToken is an opaque continuation token. Empty means this is the last page. NextPageToken string + // ReceivedAtOrAfterMs is the inclusive lower receipt-time bound of the window the page was read from, in Unix milliseconds. + ReceivedAtOrAfterMs int64 + // ReceivedBeforeMs is the exclusive upper receipt-time bound of the window the page was read from, in Unix milliseconds. + ReceivedBeforeMs int64 } diff --git a/submitqueue/gateway/controller/BUILD.bazel b/submitqueue/gateway/controller/BUILD.bazel index 53f91cc4d..3281e0c2c 100644 --- a/submitqueue/gateway/controller/BUILD.bazel +++ b/submitqueue/gateway/controller/BUILD.bazel @@ -6,6 +6,7 @@ go_library( "cancel.go", "land.go", "list.go", + "list_pagination.go", "list_queues.go", "ping.go", "read_errors.go", diff --git a/submitqueue/gateway/controller/list.go b/submitqueue/gateway/controller/list.go index bbc86fc41..1587323f9 100644 --- a/submitqueue/gateway/controller/list.go +++ b/submitqueue/gateway/controller/list.go @@ -16,11 +16,8 @@ package controller import ( "context" - "encoding/base64" "errors" "fmt" - "net/url" - "strconv" "github.com/uber-go/tally" "github.com/uber/submitqueue/platform/errs" @@ -37,14 +34,6 @@ const ( maxListPageSize = 200 ) -type listPageToken struct { - Queue string - ReceivedAtOrAfterMs int64 - ReceivedBeforeMs int64 - LastReceivedAtMs int64 - LastRequestID string -} - // ListController handles bounded queue receipt-history queries. type ListController interface { List(ctx context.Context, req entity.ListRequest) (entity.ListResult, error) @@ -69,7 +58,8 @@ func NewListController(logger *zap.SugaredLogger, scope tally.Scope, stores stor } } -// List returns one page of requests received for a queue in the supplied half-open time range. +// List returns one page of requests received for a queue in a half-open time range, taken from the request on the +// first page and from the page token afterwards. func (c *listController) List(ctx context.Context, req entity.ListRequest) (result entity.ListResult, retErr error) { op := metrics.Begin(c.metricsScope, "list", metrics.StorageLatencyBuckets) defer func() { op.Complete(retErr) }() @@ -83,9 +73,6 @@ func (c *listController) List(ctx context.Context, req entity.ListRequest) (resu } return entity.ListResult{}, fmt.Errorf("failed to look up queue %q: %w", req.Queue, err) } - if req.ReceivedAtOrAfterMs >= req.ReceivedBeforeMs { - return entity.ListResult{}, fmt.Errorf("requires received_at_or_after_ms < received_before_ms: %w", ErrInvalidRequest) - } pageSize := int(req.PageSize) if pageSize == 0 { pageSize = defaultListPageSize @@ -93,34 +80,24 @@ func (c *listController) List(ctx context.Context, req entity.ListRequest) (resu if pageSize < 0 || pageSize > maxListPageSize { return entity.ListResult{}, fmt.Errorf("page_size must be between 0 and %d: %w", maxListPageSize, ErrInvalidRequest) } + query, err := resolveListRange(req, pageSize) + if err != nil { + return entity.ListResult{}, err + } store, err := c.stores.For(storage.Config{QueueName: req.Queue}) if err != nil { return entity.ListResult{}, fmt.Errorf("failed to resolve storage for queue %q: %w", req.Queue, err) } - query := basestorage.RequestReceiptRange{ - ReceivedAtOrAfterMs: req.ReceivedAtOrAfterMs, - ReceivedBeforeMs: req.ReceivedBeforeMs, - Limit: pageSize + 1, - } - if req.PageToken != "" { - token, err := decodeListPageToken(req.PageToken) - if err != nil { - return entity.ListResult{}, fmt.Errorf("invalid page token: %w", ErrInvalidRequest) - } - if token.Queue != req.Queue || token.ReceivedAtOrAfterMs != req.ReceivedAtOrAfterMs || token.ReceivedBeforeMs != req.ReceivedBeforeMs { - return entity.ListResult{}, fmt.Errorf("page token does not match query: %w", ErrInvalidRequest) - } - query.Before = basestorage.RequestReceiptCursor{ReceivedAtMs: token.LastReceivedAtMs, RequestID: token.LastRequestID} - } - receipts, err := store.GetRequestReceiptStore().List(ctx, query) if err != nil { return entity.ListResult{}, fmt.Errorf("failed to list request receipts queue=%q: %w", req.Queue, err) } visible := receipts[:min(len(receipts), pageSize)] + result.ReceivedAtOrAfterMs = query.ReceivedAtOrAfterMs + result.ReceivedBeforeMs = query.ReceivedBeforeMs result.Requests = make([]entity.RequestSummary, 0, len(visible)) if len(visible) > 0 { summaries := store.GetRequestSummaryStore() @@ -134,13 +111,18 @@ func (c *listController) List(ctx context.Context, req entity.ListRequest) (resu } if len(receipts) > pageSize { last := visible[len(visible)-1] - result.NextPageToken = encodeListPageToken(listPageToken{ + token, err := encodeListPageToken(listPageToken{ + Version: listPageTokenVersion, Queue: req.Queue, - ReceivedAtOrAfterMs: req.ReceivedAtOrAfterMs, - ReceivedBeforeMs: req.ReceivedBeforeMs, + ReceivedAtOrAfterMs: query.ReceivedAtOrAfterMs, + ReceivedBeforeMs: query.ReceivedBeforeMs, LastReceivedAtMs: last.ReceivedAtMs, LastRequestID: last.RequestID, }) + if err != nil { + return entity.ListResult{}, fmt.Errorf("failed to encode page token queue=%q: %w", req.Queue, err) + } + result.NextPageToken = token } c.logger.Debugw("queue requests listed", "queue", req.Queue, "request_count", len(result.Requests), "has_next_page", result.NextPageToken != "") return result, nil @@ -162,48 +144,3 @@ func readListSummary(ctx context.Context, summaries basestorage.RequestSummarySt } return summary, nil } - -func encodeListPageToken(token listPageToken) string { - values := url.Values{ - "queue": {token.Queue}, - "received_at_or_after_ms": {strconv.FormatInt(token.ReceivedAtOrAfterMs, 10)}, - "received_before_ms": {strconv.FormatInt(token.ReceivedBeforeMs, 10)}, - "last_received_at_ms": {strconv.FormatInt(token.LastReceivedAtMs, 10)}, - "last_request_id": {token.LastRequestID}, - } - return base64.RawURLEncoding.EncodeToString([]byte(values.Encode())) -} - -func decodeListPageToken(encoded string) (listPageToken, error) { - data, err := base64.RawURLEncoding.DecodeString(encoded) - if err != nil { - return listPageToken{}, err - } - values, err := url.ParseQuery(string(data)) - if err != nil { - return listPageToken{}, err - } - receivedAtOrAfterMs, err := strconv.ParseInt(values.Get("received_at_or_after_ms"), 10, 64) - if err != nil { - return listPageToken{}, err - } - receivedBeforeMs, err := strconv.ParseInt(values.Get("received_before_ms"), 10, 64) - if err != nil { - return listPageToken{}, err - } - lastReceivedAtMs, err := strconv.ParseInt(values.Get("last_received_at_ms"), 10, 64) - if err != nil { - return listPageToken{}, err - } - token := listPageToken{ - Queue: values.Get("queue"), - ReceivedAtOrAfterMs: receivedAtOrAfterMs, - ReceivedBeforeMs: receivedBeforeMs, - LastReceivedAtMs: lastReceivedAtMs, - LastRequestID: values.Get("last_request_id"), - } - if token.Queue == "" || token.LastRequestID == "" || token.LastReceivedAtMs <= 0 || token.ReceivedAtOrAfterMs >= token.ReceivedBeforeMs { - return listPageToken{}, fmt.Errorf("invalid token fields") - } - return token, nil -} diff --git a/submitqueue/gateway/controller/list_consistency_test.go b/submitqueue/gateway/controller/list_consistency_test.go index 3d3a56e26..8e1c84bdd 100644 --- a/submitqueue/gateway/controller/list_consistency_test.go +++ b/submitqueue/gateway/controller/list_consistency_test.go @@ -61,7 +61,7 @@ func TestList_ConsistencyErrors(t *testing.T) { f.queueConfigs.EXPECT().Get(gomock.Any(), "q").Return(entity.QueueConfig{}, nil) f.receipts.EXPECT().List(gomock.Any(), gomock.Any()).Return([]entity.RequestReceipt{badReceipt}, nil) - result, err := f.controller.List(context.Background(), entity.ListRequest{Queue: "q", ReceivedAtOrAfterMs: 1, ReceivedBeforeMs: 200}) + result, err := f.controller.List(context.Background(), boundedListRequest("q", 1, 200)) require.Error(t, err) assert.True(t, IsInternalConsistency(err)) @@ -83,7 +83,7 @@ func TestList_DoesNotReturnPartialPage(t *testing.T) { }, nil) f.summaries.EXPECT().Get(gomock.Any(), "1").Return(entity.RequestSummary{}, basestorage.ErrNotFound) - result, err := f.controller.List(context.Background(), entity.ListRequest{Queue: "q", ReceivedAtOrAfterMs: 1, ReceivedBeforeMs: 200}) + result, err := f.controller.List(context.Background(), boundedListRequest("q", 1, 200)) assert.True(t, IsInternalConsistency(err)) assert.Equal(t, entity.ListResult{}, result) diff --git a/submitqueue/gateway/controller/list_pagination.go b/submitqueue/gateway/controller/list_pagination.go new file mode 100644 index 000000000..e28f15c1d --- /dev/null +++ b/submitqueue/gateway/controller/list_pagination.go @@ -0,0 +1,131 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "net/url" + "strconv" + + "github.com/uber/submitqueue/submitqueue/entity" + basestorage "github.com/uber/submitqueue/submitqueue/extension/storage" +) + +const listPageTokenVersion = 1 + +type listPageToken struct { + Version int `json:"version"` + Queue string `json:"queue"` + ReceivedAtOrAfterMs int64 `json:"received_at_or_after_ms"` + ReceivedBeforeMs int64 `json:"received_before_ms"` + LastReceivedAtMs int64 `json:"last_received_at_ms"` + LastRequestID string `json:"last_request_id"` +} + +// resolveListRange takes the window from the page token when one is supplied, so a continuation needs only the +// queue and the token; a bound the caller also supplies must match the token's. +func resolveListRange(req entity.ListRequest, pageSize int) (basestorage.RequestReceiptRange, error) { + query := basestorage.RequestReceiptRange{Limit: pageSize + 1} + if req.PageToken != "" { + token, err := decodeListPageToken(req.PageToken) + if err != nil { + return basestorage.RequestReceiptRange{}, fmt.Errorf("invalid page token: %w", ErrInvalidRequest) + } + if token.Queue != req.Queue || + (req.HasReceivedAtOrAfterMs && req.ReceivedAtOrAfterMs != token.ReceivedAtOrAfterMs) || + (req.HasReceivedBeforeMs && req.ReceivedBeforeMs != token.ReceivedBeforeMs) { + return basestorage.RequestReceiptRange{}, fmt.Errorf("page token does not match the queue and time bounds: %w", ErrInvalidRequest) + } + query.ReceivedAtOrAfterMs = token.ReceivedAtOrAfterMs + query.ReceivedBeforeMs = token.ReceivedBeforeMs + query.Before = basestorage.RequestReceiptCursor{ReceivedAtMs: token.LastReceivedAtMs, RequestID: token.LastRequestID} + return query, nil + } + if !req.HasReceivedAtOrAfterMs || !req.HasReceivedBeforeMs || req.ReceivedAtOrAfterMs >= req.ReceivedBeforeMs { + return basestorage.RequestReceiptRange{}, fmt.Errorf("first page requires received_at_or_after_ms < received_before_ms: %w", ErrInvalidRequest) + } + query.ReceivedAtOrAfterMs = req.ReceivedAtOrAfterMs + query.ReceivedBeforeMs = req.ReceivedBeforeMs + return query, nil +} + +func encodeListPageToken(token listPageToken) (string, error) { + contents, err := json.Marshal(token) + if err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(contents), nil +} + +func decodeListPageToken(encoded string) (listPageToken, error) { + contents, err := base64.RawURLEncoding.Strict().DecodeString(encoded) + if err != nil { + return listPageToken{}, err + } + token, err := parseListPageTokenContents(contents) + if err != nil { + return listPageToken{}, err + } + if token.Queue == "" || token.LastRequestID == "" || token.LastReceivedAtMs <= 0 || + token.ReceivedAtOrAfterMs >= token.ReceivedBeforeMs || + token.LastReceivedAtMs < token.ReceivedAtOrAfterMs || token.LastReceivedAtMs >= token.ReceivedBeforeMs { + return listPageToken{}, fmt.Errorf("invalid token fields") + } + return token, nil +} + +func parseListPageTokenContents(contents []byte) (listPageToken, error) { + // TODO(list-token): drop the unversioned query-string format once tokens minted before versioning have expired. + if !bytes.HasPrefix(contents, []byte("{")) { + return parseUnversionedListPageToken(contents) + } + var token listPageToken + if err := json.Unmarshal(contents, &token); err != nil { + return listPageToken{}, err + } + if token.Version != listPageTokenVersion { + return listPageToken{}, fmt.Errorf("unsupported token version %d", token.Version) + } + return token, nil +} + +func parseUnversionedListPageToken(contents []byte) (listPageToken, error) { + values, err := url.ParseQuery(string(contents)) + if err != nil { + return listPageToken{}, err + } + receivedAtOrAfterMs, err := strconv.ParseInt(values.Get("received_at_or_after_ms"), 10, 64) + if err != nil { + return listPageToken{}, err + } + receivedBeforeMs, err := strconv.ParseInt(values.Get("received_before_ms"), 10, 64) + if err != nil { + return listPageToken{}, err + } + lastReceivedAtMs, err := strconv.ParseInt(values.Get("last_received_at_ms"), 10, 64) + if err != nil { + return listPageToken{}, err + } + return listPageToken{ + Queue: values.Get("queue"), + ReceivedAtOrAfterMs: receivedAtOrAfterMs, + ReceivedBeforeMs: receivedBeforeMs, + LastReceivedAtMs: lastReceivedAtMs, + LastRequestID: values.Get("last_request_id"), + }, nil +} diff --git a/submitqueue/gateway/controller/list_test.go b/submitqueue/gateway/controller/list_test.go index 79cb652f5..03ecfbf63 100644 --- a/submitqueue/gateway/controller/list_test.go +++ b/submitqueue/gateway/controller/list_test.go @@ -57,6 +57,20 @@ func newListTestFixture(t *testing.T) listTestFixture { return f } +func boundedListRequest(queue string, after, before int64) entity.ListRequest { + return entity.ListRequest{ + Queue: queue, + ReceivedAtOrAfterMs: after, HasReceivedAtOrAfterMs: true, + ReceivedBeforeMs: before, HasReceivedBeforeMs: true, + } +} + +func mustEncodeListPageToken(t *testing.T, token listPageToken) string { + encoded, err := encodeListPageToken(token) + require.NoError(t, err) + return encoded +} + func TestList_ReceiptPages(t *testing.T) { receipts := []entity.RequestReceipt{ {RequestID: "9", Queue: "q", ReceivedAtMs: 190}, @@ -91,13 +105,15 @@ func TestList_ReceiptPages(t *testing.T) { f.summaries.EXPECT().Get(gomock.Any(), receipt.RequestID).Return(summary, nil) want = append(want, summary) } + req := boundedListRequest("q", 100, 200) + req.PageSize = 2 - result, err := f.controller.List(context.Background(), entity.ListRequest{ - Queue: "q", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, PageSize: 2, - }) + result, err := f.controller.List(context.Background(), req) require.NoError(t, err) assert.Equal(t, want, result.Requests) + assert.Equal(t, int64(100), result.ReceivedAtOrAfterMs) + assert.Equal(t, int64(200), result.ReceivedBeforeMs) ids := make([]string, 0, len(result.Requests)) for _, summary := range result.Requests { ids = append(ids, summary.RequestID) @@ -110,37 +126,68 @@ func TestList_ReceiptPages(t *testing.T) { token, err := decodeListPageToken(result.NextPageToken) require.NoError(t, err) assert.Equal(t, listPageToken{ - Queue: "q", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, + Version: listPageTokenVersion, Queue: "q", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, LastReceivedAtMs: 190, LastRequestID: "10", }, token) }) } } -func TestList_UsesExistingCursor(t *testing.T) { - f := newListTestFixture(t) - f.queueConfigs.EXPECT().Get(gomock.Any(), "q").Return(entity.QueueConfig{}, nil) - token := encodeListPageToken(listPageToken{Queue: "q", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, LastReceivedAtMs: 190, LastRequestID: "10"}) - f.receipts.EXPECT().List(gomock.Any(), basestorage.RequestReceiptRange{ - ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, Limit: 51, - Before: basestorage.RequestReceiptCursor{ReceivedAtMs: 190, RequestID: "10"}, - }).Return([]entity.RequestReceipt{{Queue: "q", ReceivedAtMs: 170, RequestID: "1"}}, nil) - summary := entity.RequestSummary{Queue: "q", ReceivedAtMs: 170, RequestID: "1", Status: entity.RequestStatusLanded} - f.summaries.EXPECT().Get(gomock.Any(), "1").Return(summary, nil) +func TestList_ContinuationResolvesWindowFromToken(t *testing.T) { + token := mustEncodeListPageToken(t, listPageToken{ + Version: listPageTokenVersion, Queue: "q", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, LastReceivedAtMs: 190, LastRequestID: "10", + }) + legacyToken := base64.RawURLEncoding.EncodeToString([]byte( + "queue=q&received_at_or_after_ms=100&received_before_ms=200&last_received_at_ms=190&last_request_id=10")) + withBounds := boundedListRequest("q", 100, 200) + withBounds.PageToken = token + lowerOnly := entity.ListRequest{Queue: "q", ReceivedAtOrAfterMs: 100, HasReceivedAtOrAfterMs: true, PageToken: token} + tests := []struct { + name string + request entity.ListRequest + }{ + {name: "token only", request: entity.ListRequest{Queue: "q", PageToken: token}}, + {name: "matching bounds", request: withBounds}, + {name: "one matching bound", request: lowerOnly}, + {name: "unversioned token", request: entity.ListRequest{Queue: "q", PageToken: legacyToken}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := newListTestFixture(t) + f.queueConfigs.EXPECT().Get(gomock.Any(), "q").Return(entity.QueueConfig{}, nil) + f.receipts.EXPECT().List(gomock.Any(), basestorage.RequestReceiptRange{ + ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, Limit: 51, + Before: basestorage.RequestReceiptCursor{ReceivedAtMs: 190, RequestID: "10"}, + }).Return([]entity.RequestReceipt{{Queue: "q", ReceivedAtMs: 170, RequestID: "1"}}, nil) + summary := entity.RequestSummary{Queue: "q", ReceivedAtMs: 170, RequestID: "1", Status: entity.RequestStatusLanded} + f.summaries.EXPECT().Get(gomock.Any(), "1").Return(summary, nil) - result, err := f.controller.List(context.Background(), entity.ListRequest{Queue: "q", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, PageToken: token}) + result, err := f.controller.List(context.Background(), tt.request) - require.NoError(t, err) - assert.Equal(t, []entity.RequestSummary{summary}, result.Requests) - assert.Empty(t, result.NextPageToken) + require.NoError(t, err) + assert.Equal(t, entity.ListResult{ + Requests: []entity.RequestSummary{summary}, ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, + }, result) + }) + } } func TestList_Errors(t *testing.T) { - validToken := encodeListPageToken(listPageToken{Queue: "other", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, LastReceivedAtMs: 150, LastRequestID: "1"}) + validToken := func(edit func(*listPageToken)) string { + token := listPageToken{ + Version: listPageTokenVersion, Queue: "q", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, LastReceivedAtMs: 150, LastRequestID: "1", + } + edit(&token) + return mustEncodeListPageToken(t, token) + } + withToken := func(req entity.ListRequest, token string) entity.ListRequest { + req.PageToken = token + return req + } + goodToken := validToken(func(*listPageToken) {}) + tamperedToken := base64.RawURLEncoding.EncodeToString([]byte(`{"version":1,"queue":"q","received_at_or_after_ms":"x"}`)) invalidFieldsToken := base64.RawURLEncoding.EncodeToString([]byte("queue=q&received_at_or_after_ms=100&received_before_ms=200&last_received_at_ms=150")) invalidNumberToken := base64.RawURLEncoding.EncodeToString([]byte("queue=q&received_at_or_after_ms=x&received_before_ms=200&last_received_at_ms=150&last_request_id=q%2F1")) - zeroTimeToken := encodeListPageToken(listPageToken{Queue: "q", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, LastRequestID: "1"}) - negativeTimeToken := encodeListPageToken(listPageToken{Queue: "q", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, LastReceivedAtMs: -1, LastRequestID: "1"}) backendErr := errors.New("store down") tests := []struct { name string @@ -151,26 +198,36 @@ func TestList_Errors(t *testing.T) { wantInvalid bool wantUnknown bool }{ - {name: "empty queue", request: entity.ListRequest{ReceivedAtOrAfterMs: 1, ReceivedBeforeMs: 2}, wantInvalid: true}, - {name: "unknown queue", request: entity.ListRequest{Queue: "missing", ReceivedAtOrAfterMs: 1, ReceivedBeforeMs: 2}, queueErr: queueconfig.ErrNotFound, wantUnknown: true, wantInvalid: true}, - {name: "queue config failure", request: entity.ListRequest{Queue: "q", ReceivedAtOrAfterMs: 1, ReceivedBeforeMs: 2}, queueErr: backendErr, wantErr: backendErr}, - {name: "invalid range", request: entity.ListRequest{Queue: "q", ReceivedAtOrAfterMs: 2, ReceivedBeforeMs: 2}, wantInvalid: true}, - {name: "negative page size", request: entity.ListRequest{Queue: "q", ReceivedAtOrAfterMs: 1, ReceivedBeforeMs: 2, PageSize: -1}, wantInvalid: true}, - {name: "page size above maximum", request: entity.ListRequest{Queue: "q", ReceivedAtOrAfterMs: 1, ReceivedBeforeMs: 2, PageSize: 201}, wantInvalid: true}, - {name: "malformed token", request: entity.ListRequest{Queue: "q", ReceivedAtOrAfterMs: 1, ReceivedBeforeMs: 2, PageToken: "%%%"}, wantInvalid: true}, - {name: "invalid token number", request: entity.ListRequest{Queue: "q", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, PageToken: invalidNumberToken}, wantInvalid: true}, - {name: "invalid token fields", request: entity.ListRequest{Queue: "q", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, PageToken: invalidFieldsToken}, wantInvalid: true}, - {name: "zero cursor time", request: entity.ListRequest{Queue: "q", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, PageToken: zeroTimeToken}, wantInvalid: true}, - {name: "negative cursor time", request: entity.ListRequest{Queue: "q", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, PageToken: negativeTimeToken}, wantInvalid: true}, - {name: "token query mismatch", request: entity.ListRequest{Queue: "q", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, PageToken: validToken}, wantInvalid: true}, + {name: "empty queue", request: boundedListRequest("", 1, 2), wantInvalid: true}, + {name: "unknown queue", request: boundedListRequest("missing", 1, 2), queueErr: queueconfig.ErrNotFound, wantUnknown: true, wantInvalid: true}, + {name: "queue config failure", request: boundedListRequest("q", 1, 2), queueErr: backendErr, wantErr: backendErr}, + {name: "first page without bounds", request: entity.ListRequest{Queue: "q"}, wantInvalid: true}, + {name: "first page without lower bound", request: entity.ListRequest{Queue: "q", ReceivedBeforeMs: 2, HasReceivedBeforeMs: true}, wantInvalid: true}, + {name: "first page without upper bound", request: entity.ListRequest{Queue: "q", ReceivedAtOrAfterMs: 1, HasReceivedAtOrAfterMs: true}, wantInvalid: true}, + {name: "invalid range", request: boundedListRequest("q", 2, 2), wantInvalid: true}, + {name: "negative page size", request: entity.ListRequest{Queue: "q", PageSize: -1, ReceivedAtOrAfterMs: 1, HasReceivedAtOrAfterMs: true, ReceivedBeforeMs: 2, HasReceivedBeforeMs: true}, wantInvalid: true}, + {name: "page size above maximum", request: entity.ListRequest{Queue: "q", PageSize: 201, ReceivedAtOrAfterMs: 1, HasReceivedAtOrAfterMs: true, ReceivedBeforeMs: 2, HasReceivedBeforeMs: true}, wantInvalid: true}, + {name: "malformed token", request: entity.ListRequest{Queue: "q", PageToken: "%%%"}, wantInvalid: true}, + {name: "tampered token", request: entity.ListRequest{Queue: "q", PageToken: tamperedToken}, wantInvalid: true}, + {name: "unknown token version", request: entity.ListRequest{Queue: "q", PageToken: validToken(func(tok *listPageToken) { tok.Version = 2 })}, wantInvalid: true}, + {name: "missing token version", request: entity.ListRequest{Queue: "q", PageToken: validToken(func(tok *listPageToken) { tok.Version = 0 })}, wantInvalid: true}, + {name: "invalid unversioned token number", request: entity.ListRequest{Queue: "q", PageToken: invalidNumberToken}, wantInvalid: true}, + {name: "invalid unversioned token fields", request: entity.ListRequest{Queue: "q", PageToken: invalidFieldsToken}, wantInvalid: true}, + {name: "zero cursor time", request: entity.ListRequest{Queue: "q", PageToken: validToken(func(tok *listPageToken) { tok.LastReceivedAtMs = 0 })}, wantInvalid: true}, + {name: "cursor before window", request: entity.ListRequest{Queue: "q", PageToken: validToken(func(tok *listPageToken) { tok.LastReceivedAtMs = 99 })}, wantInvalid: true}, + {name: "cursor at window end", request: entity.ListRequest{Queue: "q", PageToken: validToken(func(tok *listPageToken) { tok.LastReceivedAtMs = 200 })}, wantInvalid: true}, + {name: "token with empty window", request: entity.ListRequest{Queue: "q", PageToken: validToken(func(tok *listPageToken) { tok.ReceivedBeforeMs = 100 })}, wantInvalid: true}, + {name: "token for another queue", request: entity.ListRequest{Queue: "q", PageToken: validToken(func(tok *listPageToken) { tok.Queue = "other" })}, wantInvalid: true}, + {name: "mismatched lower bound", request: withToken(boundedListRequest("q", 101, 200), goodToken), wantInvalid: true}, + {name: "mismatched upper bound", request: withToken(entity.ListRequest{Queue: "q", ReceivedBeforeMs: 0, HasReceivedBeforeMs: true}, goodToken), wantInvalid: true}, { - name: "receipt scan failure", request: entity.ListRequest{Queue: "q", ReceivedAtOrAfterMs: 1, ReceivedBeforeMs: 2}, wantErr: backendErr, + name: "receipt scan failure", request: boundedListRequest("q", 1, 2), wantErr: backendErr, setup: func(f listTestFixture) { f.receipts.EXPECT().List(gomock.Any(), basestorage.RequestReceiptRange{ReceivedAtOrAfterMs: 1, ReceivedBeforeMs: 2, Limit: 51}).Return(nil, backendErr) }, }, { - name: "summary read failure", request: entity.ListRequest{Queue: "q", ReceivedAtOrAfterMs: 1, ReceivedBeforeMs: 2}, wantErr: backendErr, + name: "summary read failure", request: boundedListRequest("q", 1, 2), wantErr: backendErr, setup: func(f listTestFixture) { f.receipts.EXPECT().List(gomock.Any(), gomock.Any()).Return([]entity.RequestReceipt{{Queue: "q", ReceivedAtMs: 1, RequestID: "1"}}, nil) f.summaries.EXPECT().Get(gomock.Any(), "1").Return(entity.RequestSummary{}, backendErr) @@ -209,7 +266,7 @@ func TestList_StorageResolutionFailure(t *testing.T) { queueConfigs.EXPECT().Get(gomock.Any(), "q").Return(entity.QueueConfig{}, nil) c := NewListController(zap.NewNop().Sugar(), tally.NoopScope, factory, queueConfigs) - result, err := c.List(context.Background(), entity.ListRequest{Queue: "q", ReceivedAtOrAfterMs: 1, ReceivedBeforeMs: 2}) + result, err := c.List(context.Background(), boundedListRequest("q", 1, 2)) assert.ErrorIs(t, err, backendErr) assert.Equal(t, entity.ListResult{}, result) diff --git a/test/e2e/submitqueue/fake_demo_test.go b/test/e2e/submitqueue/fake_demo_test.go index 921951df5..1a2e79991 100644 --- a/test/e2e/submitqueue/fake_demo_test.go +++ b/test/e2e/submitqueue/fake_demo_test.go @@ -45,7 +45,8 @@ func (s *E2EIntegrationSuite) TestFakeDemo_IndependentAndStackedChanges() { var summaries []*gatewaypb.RequestSummary pollUntil(persistPollInterval, func() bool { resp, err := s.gatewayClient.List(s.ctx, &gatewaypb.ListRequest{ - Queue: queue, ReceivedAtOrAfterMs: started, ReceivedBeforeMs: time.Now().UnixMilli() + 1, PageSize: 10, + Queue: queue, PageSize: 10, + ReceivedLowerBound: &gatewaypb.ListRequest_ReceivedAtOrAfterMs{ReceivedAtOrAfterMs: started}, ReceivedUpperBound: &gatewaypb.ListRequest_ReceivedBeforeMs{ReceivedBeforeMs: time.Now().UnixMilli() + 1}, }) require.NoError(t, err) summaries = resp.Requests diff --git a/test/e2e/submitqueue/suite_test.go b/test/e2e/submitqueue/suite_test.go index 44d04c937..c5b1acb4a 100644 --- a/test/e2e/submitqueue/suite_test.go +++ b/test/e2e/submitqueue/suite_test.go @@ -614,13 +614,12 @@ func (s *E2EIntegrationSuite) TestReadAPIs() { var listedSqids []string var pageToken string for { - listResponse, listErr := s.gatewayClient.List(s.ctx, &gatewaypb.ListRequest{ - Queue: queue, - ReceivedAtOrAfterMs: receivedAtOrAfterMs, - ReceivedBeforeMs: receivedBeforeMs, - PageSize: 1, - PageToken: pageToken, - }) + listRequest := &gatewaypb.ListRequest{Queue: queue, PageSize: 1, PageToken: pageToken} + if pageToken == "" { + listRequest.ReceivedLowerBound = &gatewaypb.ListRequest_ReceivedAtOrAfterMs{ReceivedAtOrAfterMs: receivedAtOrAfterMs} + listRequest.ReceivedUpperBound = &gatewaypb.ListRequest_ReceivedBeforeMs{ReceivedBeforeMs: receivedBeforeMs} + } + listResponse, listErr := s.gatewayClient.List(s.ctx, listRequest) require.NoError(t, listErr) for _, request := range listResponse.Requests { if request.Sqid == firstSqid || request.Sqid == secondSqid { diff --git a/test/integration/submitqueue/gateway/suite_test.go b/test/integration/submitqueue/gateway/suite_test.go index cadaa69f9..a0144000a 100644 --- a/test/integration/submitqueue/gateway/suite_test.go +++ b/test/integration/submitqueue/gateway/suite_test.go @@ -189,7 +189,7 @@ func (s *GatewayIntegrationSuite) TestLandAPI() { summary, err := s.client.GetRequestSummaryByID(s.ctx, &pb.GetRequestSummaryByIDRequest{Sqid: resp.Sqid, Queue: req.Queue}) require.NoError(t, err) listed, err := s.client.List(s.ctx, &pb.ListRequest{ - Queue: req.Queue, ReceivedAtOrAfterMs: summary.Request.ReceivedAtMs, ReceivedBeforeMs: summary.Request.ReceivedAtMs + 1, + Queue: req.Queue, ReceivedLowerBound: &pb.ListRequest_ReceivedAtOrAfterMs{ReceivedAtOrAfterMs: summary.Request.ReceivedAtMs}, ReceivedUpperBound: &pb.ListRequest_ReceivedBeforeMs{ReceivedBeforeMs: summary.Request.ReceivedAtMs + 1}, }) require.NoError(t, err) assert.Contains(t, listed.Requests, summary.Request) @@ -225,7 +225,7 @@ func (s *GatewayIntegrationSuite) TestListAPI() { RequestID: "904", Queue: "test-queue", ReceivedAtMs: 180, Status: entity.RequestStatusAccepting, Version: 1, })) - resp, err := s.client.List(s.ctx, &pb.ListRequest{Queue: "test-queue", ReceivedAtOrAfterMs: 50, ReceivedBeforeMs: 250, PageSize: 1}) + resp, err := s.client.List(s.ctx, &pb.ListRequest{Queue: "test-queue", ReceivedLowerBound: &pb.ListRequest_ReceivedAtOrAfterMs{ReceivedAtOrAfterMs: 50}, ReceivedUpperBound: &pb.ListRequest_ReceivedBeforeMs{ReceivedBeforeMs: 250}, PageSize: 1}) require.NoError(t, err) require.Len(t, resp.Requests, 1) assert.Equal(t, "903", resp.Requests[0].Sqid) @@ -242,10 +242,12 @@ func (s *GatewayIntegrationSuite) TestListAPI() { updated.Metadata = map[string]string{"build": "url"} require.NoError(t, queueStore.GetRequestSummaryStore().Update(s.ctx, updated, current.Version, current.Version+1)) - resp, err = s.client.List(s.ctx, &pb.ListRequest{Queue: "test-queue", ReceivedAtOrAfterMs: 50, ReceivedBeforeMs: 250, PageSize: 1, PageToken: resp.NextPageToken}) + resp, err = s.client.List(s.ctx, &pb.ListRequest{Queue: "test-queue", PageSize: 1, PageToken: resp.NextPageToken}) require.NoError(t, err) require.Len(t, resp.Requests, 1) assert.Equal(t, "902", resp.Requests[0].Sqid) + assert.Equal(t, int64(50), resp.ReceivedAtOrAfterMs) + assert.Equal(t, int64(250), resp.ReceivedBeforeMs) summary, err := s.client.GetRequestSummaryByID(s.ctx, &pb.GetRequestSummaryByIDRequest{Sqid: "902", Queue: "test-queue"}) require.NoError(t, err) assert.Equal(t, summary.Request, resp.Requests[0]) @@ -254,14 +256,14 @@ func (s *GatewayIntegrationSuite) TestListAPI() { assert.Equal(t, map[string]string{"build": "url"}, resp.Requests[0].Metadata) require.NotEmpty(t, resp.NextPageToken) - resp, err = s.client.List(s.ctx, &pb.ListRequest{Queue: "test-queue", ReceivedAtOrAfterMs: 50, ReceivedBeforeMs: 250, PageSize: 1, PageToken: resp.NextPageToken}) + resp, err = s.client.List(s.ctx, &pb.ListRequest{Queue: "test-queue", ReceivedLowerBound: &pb.ListRequest_ReceivedAtOrAfterMs{ReceivedAtOrAfterMs: 50}, ReceivedUpperBound: &pb.ListRequest_ReceivedBeforeMs{ReceivedBeforeMs: 250}, PageSize: 1, PageToken: resp.NextPageToken}) require.NoError(t, err) require.Len(t, resp.Requests, 1) assert.Equal(t, "901", resp.Requests[0].Sqid) assert.Equal(t, string(entity.RequestStatusAccepted), resp.Requests[0].Status) assert.Empty(t, resp.NextPageToken) - resp, err = s.client.List(s.ctx, &pb.ListRequest{Queue: "test-queue", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200}) + resp, err = s.client.List(s.ctx, &pb.ListRequest{Queue: "test-queue", ReceivedLowerBound: &pb.ListRequest_ReceivedAtOrAfterMs{ReceivedAtOrAfterMs: 100}, ReceivedUpperBound: &pb.ListRequest_ReceivedBeforeMs{ReceivedBeforeMs: 200}}) require.NoError(t, err) require.Len(t, resp.Requests, 1) assert.Equal(t, "901", resp.Requests[0].Sqid) @@ -276,9 +278,9 @@ func (s *GatewayIntegrationSuite) TestReadAPIErrorCodes() { assert.Equal(t, codes.NotFound, status.Code(err)) _, err = s.client.List(s.ctx, &pb.ListRequest{ - Queue: "missing-queue", - ReceivedAtOrAfterMs: 1, - ReceivedBeforeMs: 2, + Queue: "missing-queue", + ReceivedLowerBound: &pb.ListRequest_ReceivedAtOrAfterMs{ReceivedAtOrAfterMs: 1}, + ReceivedUpperBound: &pb.ListRequest_ReceivedBeforeMs{ReceivedBeforeMs: 2}, }) require.Error(t, err) assert.Equal(t, codes.InvalidArgument, status.Code(err)) @@ -323,7 +325,7 @@ func (s *GatewayIntegrationSuite) TestReadAPIErrorCodes() { require.NoError(t, receiptStore.GetRequestReceiptStore().Create(s.ctx, entity.RequestReceipt{ Queue: "test-queue", RequestID: "999", ReceivedAtMs: 500, })) - _, err = s.client.List(s.ctx, &pb.ListRequest{Queue: "test-queue", ReceivedAtOrAfterMs: 500, ReceivedBeforeMs: 501}) + _, err = s.client.List(s.ctx, &pb.ListRequest{Queue: "test-queue", ReceivedLowerBound: &pb.ListRequest_ReceivedAtOrAfterMs{ReceivedAtOrAfterMs: 500}, ReceivedUpperBound: &pb.ListRequest_ReceivedBeforeMs{ReceivedBeforeMs: 501}}) require.Error(t, err) assert.Equal(t, codes.Internal, status.Code(err)) } diff --git a/web/api/src/api.test.ts b/web/api/src/api.test.ts index 74439b1d8..24793257a 100644 --- a/web/api/src/api.test.ts +++ b/web/api/src/api.test.ts @@ -10,14 +10,15 @@ describe("generated gateway API", () => { it("constructs list requests without losing int64 values", () => { const request = create(ListRequestSchema, { queue: "demo-queue", - receivedAtOrAfterMs: 1_700_000_000_000n, - receivedBeforeMs: 1_700_086_400_000n, + receivedLowerBound: { case: "receivedAtOrAfterMs", value: 1_700_000_000_000n }, + receivedUpperBound: { case: "receivedBeforeMs", value: 1_700_086_400_000n }, pageSize: 50, }); expect(request.queue).toBe("demo-queue"); - expect(request.receivedAtOrAfterMs).toBe(1_700_000_000_000n); - expect(request.receivedBeforeMs).toBe(1_700_086_400_000n); + expect(request.receivedLowerBound).toEqual({ case: "receivedAtOrAfterMs", value: 1_700_000_000_000n }); + expect(request.receivedUpperBound).toEqual({ case: "receivedBeforeMs", value: 1_700_086_400_000n }); + expect(create(ListRequestSchema, { queue: "demo-queue", pageToken: "token" }).receivedLowerBound.case).toBeUndefined(); expect(SubmitQueueGateway.methods.some((method) => method.localName === "list")).toBe(true); }); }); diff --git a/web/api/src/gen/api/submitqueue/gateway/proto/gateway_pb.ts b/web/api/src/gen/api/submitqueue/gateway/proto/gateway_pb.ts index f687bf6a9..6212a6a2f 100644 --- a/web/api/src/gen/api/submitqueue/gateway/proto/gateway_pb.ts +++ b/web/api/src/gen/api/submitqueue/gateway/proto/gateway_pb.ts @@ -28,7 +28,7 @@ import type { Message } from "@bufbuild/protobuf"; * Describes the file api/submitqueue/gateway/proto/gateway.proto. */ export const file_api_submitqueue_gateway_proto_gateway: GenFile = /*@__PURE__*/ - fileDesc("CithcGkvc3VibWl0cXVldWUvZ2F0ZXdheS9wcm90by9nYXRld2F5LnByb3RvEhh1YmVyLnN1Ym1pdHF1ZXVlLmdhdGV3YXkiHgoLUGluZ1JlcXVlc3QSDwoHbWVzc2FnZRgBIAEoCSJaCgxQaW5nUmVzcG9uc2USDwoHbWVzc2FnZRgBIAEoCRIUCgxzZXJ2aWNlX25hbWUYAiABKAkSEQoJdGltZXN0YW1wGAMgASgDEhAKCGhvc3RuYW1lGAQgASgJInsKC0xhbmRSZXF1ZXN0Eg0KBXF1ZXVlGAEgASgJEigKBmNoYW5nZRgCIAEoCzIYLnViZXIuYmFzZS5jaGFuZ2UuQ2hhbmdlEjMKCHN0cmF0ZWd5GAQgASgOMiEudWJlci5iYXNlLm1lcmdlc3RyYXRlZ3kuU3RyYXRlZ3kiHAoMTGFuZFJlc3BvbnNlEgwKBHNxaWQYASABKAkiPAoNQ2FuY2VsUmVxdWVzdBIMCgRzcWlkGAEgASgJEg4KBnJlYXNvbhgCIAEoCRINCgVxdWV1ZRgDIAEoCSIQCg5DYW5jZWxSZXNwb25zZSL5AQoOUmVxdWVzdFN1bW1hcnkSDAoEc3FpZBgBIAEoCRINCgVxdWV1ZRgCIAEoCRITCgtjaGFuZ2VfdXJpcxgDIAMoCRIWCg5yZWNlaXZlZF9hdF9tcxgEIAEoAxIOCgZzdGF0dXMYBSABKAkSEgoKbGFzdF9lcnJvchgGIAEoCRJICghtZXRhZGF0YRgHIAMoCzI2LnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5SZXF1ZXN0U3VtbWFyeS5NZXRhZGF0YUVudHJ5Gi8KDU1ldGFkYXRhRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4ASI7ChxHZXRSZXF1ZXN0U3VtbWFyeUJ5SURSZXF1ZXN0EgwKBHNxaWQYASABKAkSDQoFcXVldWUYAiABKAkiWgodR2V0UmVxdWVzdFN1bW1hcnlCeUlEUmVzcG9uc2USOQoHcmVxdWVzdBgBIAEoCzIoLnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5SZXF1ZXN0U3VtbWFyeSJICiNHZXRSZXF1ZXN0U3VtbWFyeUJ5Q2hhbmdlVVJJUmVxdWVzdBISCgpjaGFuZ2VfdXJpGAEgASgJEg0KBXF1ZXVlGAIgASgJImIKJEdldFJlcXVlc3RTdW1tYXJ5QnlDaGFuZ2VVUklSZXNwb25zZRI6CghyZXF1ZXN0cxgBIAMoCzIoLnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5SZXF1ZXN0U3VtbWFyeSKAAQoLTGlzdFJlcXVlc3QSDQoFcXVldWUYASABKAkSHwoXcmVjZWl2ZWRfYXRfb3JfYWZ0ZXJfbXMYAiABKAMSGgoScmVjZWl2ZWRfYmVmb3JlX21zGAMgASgDEhEKCXBhZ2Vfc2l6ZRgEIAEoBRISCgpwYWdlX3Rva2VuGAUgASgJImMKDExpc3RSZXNwb25zZRI6CghyZXF1ZXN0cxgBIAMoCzIoLnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5SZXF1ZXN0U3VtbWFyeRIXCg9uZXh0X3BhZ2VfdG9rZW4YAiABKAkiFQoFUXVldWUSDAoEbmFtZRgBIAEoCSITChFMaXN0UXVldWVzUmVxdWVzdCJFChJMaXN0UXVldWVzUmVzcG9uc2USLwoGcXVldWVzGAEgAygLMh8udWJlci5zdWJtaXRxdWV1ZS5nYXRld2F5LlF1ZXVlIjsKHEdldFJlcXVlc3RIaXN0b3J5QnlJRFJlcXVlc3QSDAoEc3FpZBgBIAEoCRINCgVxdWV1ZRgCIAEoCSLeAQoMSGlzdG9yeUV2ZW50EhQKDHRpbWVzdGFtcF9tcxgBIAEoAxIOCgZzdGF0dXMYAiABKAkSEgoKbGFzdF9lcnJvchgDIAEoCRJGCghtZXRhZGF0YRgEIAMoCzI0LnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5IaXN0b3J5RXZlbnQuTWV0YWRhdGFFbnRyeRIMCgR0eXBlGAUgASgJEg0KBWV2ZW50GAYgASgJGi8KDU1ldGFkYXRhRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4ASJXCh1HZXRSZXF1ZXN0SGlzdG9yeUJ5SURSZXNwb25zZRI2CgZldmVudHMYASADKAsyJi51YmVyLnN1Ym1pdHF1ZXVlLmdhdGV3YXkuSGlzdG9yeUV2ZW50IkgKI0dldFJlcXVlc3RIaXN0b3J5QnlDaGFuZ2VVUklSZXF1ZXN0EhIKCmNoYW5nZV91cmkYASABKAkSDQoFcXVldWUYAiABKAkiVgoOUmVxdWVzdEhpc3RvcnkSDAoEc3FpZBgBIAEoCRI2CgZldmVudHMYAiADKAsyJi51YmVyLnN1Ym1pdHF1ZXVlLmdhdGV3YXkuSGlzdG9yeUV2ZW50ImMKJEdldFJlcXVlc3RIaXN0b3J5QnlDaGFuZ2VVUklSZXNwb25zZRI7CgloaXN0b3JpZXMYASADKAsyKC51YmVyLnN1Ym1pdHF1ZXVlLmdhdGV3YXkuUmVxdWVzdEhpc3RvcnkiGAoFRXJyb3ISDwoHbWVzc2FnZRgBIAEoCSJXChZVbnJlY29nbml6ZWRRdWV1ZUVycm9yEi4KBWVycm9yGAEgASgLMh8udWJlci5zdWJtaXRxdWV1ZS5nYXRld2F5LkVycm9yEg0KBXF1ZXVlGAIgASgJImgKFFJlcXVlc3ROb3RGb3VuZEVycm9yEi4KBWVycm9yGAEgASgLMh8udWJlci5zdWJtaXRxdWV1ZS5nYXRld2F5LkVycm9yEgwKBHNxaWQYAiABKAkSEgoKY2hhbmdlX3VyaRgDIAEoCTLHCAoSU3VibWl0UXVldWVHYXRld2F5ElcKBFBpbmcSJS51YmVyLnN1Ym1pdHF1ZXVlLmdhdGV3YXkuUGluZ1JlcXVlc3QaJi51YmVyLnN1Ym1pdHF1ZXVlLmdhdGV3YXkuUGluZ1Jlc3BvbnNlIgASaQoKTGlzdFF1ZXVlcxIrLnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5MaXN0UXVldWVzUmVxdWVzdBosLnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5MaXN0UXVldWVzUmVzcG9uc2UiABJXCgRMYW5kEiUudWJlci5zdWJtaXRxdWV1ZS5nYXRld2F5LkxhbmRSZXF1ZXN0GiYudWJlci5zdWJtaXRxdWV1ZS5nYXRld2F5LkxhbmRSZXNwb25zZSIAEl0KBkNhbmNlbBInLnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5DYW5jZWxSZXF1ZXN0GigudWJlci5zdWJtaXRxdWV1ZS5nYXRld2F5LkNhbmNlbFJlc3BvbnNlIgASigEKFUdldFJlcXVlc3RTdW1tYXJ5QnlJRBI2LnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5HZXRSZXF1ZXN0U3VtbWFyeUJ5SURSZXF1ZXN0GjcudWJlci5zdWJtaXRxdWV1ZS5nYXRld2F5LkdldFJlcXVlc3RTdW1tYXJ5QnlJRFJlc3BvbnNlIgASnwEKHEdldFJlcXVlc3RTdW1tYXJ5QnlDaGFuZ2VVUkkSPS51YmVyLnN1Ym1pdHF1ZXVlLmdhdGV3YXkuR2V0UmVxdWVzdFN1bW1hcnlCeUNoYW5nZVVSSVJlcXVlc3QaPi51YmVyLnN1Ym1pdHF1ZXVlLmdhdGV3YXkuR2V0UmVxdWVzdFN1bW1hcnlCeUNoYW5nZVVSSVJlc3BvbnNlIgASVwoETGlzdBIlLnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5MaXN0UmVxdWVzdBomLnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5MaXN0UmVzcG9uc2UiABKKAQoVR2V0UmVxdWVzdEhpc3RvcnlCeUlEEjYudWJlci5zdWJtaXRxdWV1ZS5nYXRld2F5LkdldFJlcXVlc3RIaXN0b3J5QnlJRFJlcXVlc3QaNy51YmVyLnN1Ym1pdHF1ZXVlLmdhdGV3YXkuR2V0UmVxdWVzdEhpc3RvcnlCeUlEUmVzcG9uc2UiABKfAQocR2V0UmVxdWVzdEhpc3RvcnlCeUNoYW5nZVVSSRI9LnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5HZXRSZXF1ZXN0SGlzdG9yeUJ5Q2hhbmdlVVJJUmVxdWVzdBo+LnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5HZXRSZXF1ZXN0SGlzdG9yeUJ5Q2hhbmdlVVJJUmVzcG9uc2UiAEJrChxjb20udWJlci5zdWJtaXRxdWV1ZS5nYXRld2F5QgxHYXRld2F5UHJvdG9QAVo7Z2l0aHViLmNvbS91YmVyL3N1Ym1pdHF1ZXVlL2FwaS9zdWJtaXRxdWV1ZS9nYXRld2F5L3Byb3RvcGJiBnByb3RvMw", [file_api_base_change_proto_change, file_api_base_mergestrategy_proto_mergestrategy]); + fileDesc("CithcGkvc3VibWl0cXVldWUvZ2F0ZXdheS9wcm90by9nYXRld2F5LnByb3RvEhh1YmVyLnN1Ym1pdHF1ZXVlLmdhdGV3YXkiHgoLUGluZ1JlcXVlc3QSDwoHbWVzc2FnZRgBIAEoCSJaCgxQaW5nUmVzcG9uc2USDwoHbWVzc2FnZRgBIAEoCRIUCgxzZXJ2aWNlX25hbWUYAiABKAkSEQoJdGltZXN0YW1wGAMgASgDEhAKCGhvc3RuYW1lGAQgASgJInsKC0xhbmRSZXF1ZXN0Eg0KBXF1ZXVlGAEgASgJEigKBmNoYW5nZRgCIAEoCzIYLnViZXIuYmFzZS5jaGFuZ2UuQ2hhbmdlEjMKCHN0cmF0ZWd5GAQgASgOMiEudWJlci5iYXNlLm1lcmdlc3RyYXRlZ3kuU3RyYXRlZ3kiHAoMTGFuZFJlc3BvbnNlEgwKBHNxaWQYASABKAkiPAoNQ2FuY2VsUmVxdWVzdBIMCgRzcWlkGAEgASgJEg4KBnJlYXNvbhgCIAEoCRINCgVxdWV1ZRgDIAEoCSIQCg5DYW5jZWxSZXNwb25zZSL5AQoOUmVxdWVzdFN1bW1hcnkSDAoEc3FpZBgBIAEoCRINCgVxdWV1ZRgCIAEoCRITCgtjaGFuZ2VfdXJpcxgDIAMoCRIWCg5yZWNlaXZlZF9hdF9tcxgEIAEoAxIOCgZzdGF0dXMYBSABKAkSEgoKbGFzdF9lcnJvchgGIAEoCRJICghtZXRhZGF0YRgHIAMoCzI2LnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5SZXF1ZXN0U3VtbWFyeS5NZXRhZGF0YUVudHJ5Gi8KDU1ldGFkYXRhRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4ASI7ChxHZXRSZXF1ZXN0U3VtbWFyeUJ5SURSZXF1ZXN0EgwKBHNxaWQYASABKAkSDQoFcXVldWUYAiABKAkiWgodR2V0UmVxdWVzdFN1bW1hcnlCeUlEUmVzcG9uc2USOQoHcmVxdWVzdBgBIAEoCzIoLnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5SZXF1ZXN0U3VtbWFyeSJICiNHZXRSZXF1ZXN0U3VtbWFyeUJ5Q2hhbmdlVVJJUmVxdWVzdBISCgpjaGFuZ2VfdXJpGAEgASgJEg0KBXF1ZXVlGAIgASgJImIKJEdldFJlcXVlc3RTdW1tYXJ5QnlDaGFuZ2VVUklSZXNwb25zZRI6CghyZXF1ZXN0cxgBIAMoCzIoLnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5SZXF1ZXN0U3VtbWFyeSK0AQoLTGlzdFJlcXVlc3QSDQoFcXVldWUYASABKAkSIQoXcmVjZWl2ZWRfYXRfb3JfYWZ0ZXJfbXMYAiABKANIABIcChJyZWNlaXZlZF9iZWZvcmVfbXMYAyABKANIARIRCglwYWdlX3NpemUYBCABKAUSEgoKcGFnZV90b2tlbhgFIAEoCUIWChRyZWNlaXZlZF9sb3dlcl9ib3VuZEIWChRyZWNlaXZlZF91cHBlcl9ib3VuZCKgAQoMTGlzdFJlc3BvbnNlEjoKCHJlcXVlc3RzGAEgAygLMigudWJlci5zdWJtaXRxdWV1ZS5nYXRld2F5LlJlcXVlc3RTdW1tYXJ5EhcKD25leHRfcGFnZV90b2tlbhgCIAEoCRIfChdyZWNlaXZlZF9hdF9vcl9hZnRlcl9tcxgDIAEoAxIaChJyZWNlaXZlZF9iZWZvcmVfbXMYBCABKAMiFQoFUXVldWUSDAoEbmFtZRgBIAEoCSITChFMaXN0UXVldWVzUmVxdWVzdCJFChJMaXN0UXVldWVzUmVzcG9uc2USLwoGcXVldWVzGAEgAygLMh8udWJlci5zdWJtaXRxdWV1ZS5nYXRld2F5LlF1ZXVlIjsKHEdldFJlcXVlc3RIaXN0b3J5QnlJRFJlcXVlc3QSDAoEc3FpZBgBIAEoCRINCgVxdWV1ZRgCIAEoCSLeAQoMSGlzdG9yeUV2ZW50EhQKDHRpbWVzdGFtcF9tcxgBIAEoAxIOCgZzdGF0dXMYAiABKAkSEgoKbGFzdF9lcnJvchgDIAEoCRJGCghtZXRhZGF0YRgEIAMoCzI0LnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5IaXN0b3J5RXZlbnQuTWV0YWRhdGFFbnRyeRIMCgR0eXBlGAUgASgJEg0KBWV2ZW50GAYgASgJGi8KDU1ldGFkYXRhRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4ASJXCh1HZXRSZXF1ZXN0SGlzdG9yeUJ5SURSZXNwb25zZRI2CgZldmVudHMYASADKAsyJi51YmVyLnN1Ym1pdHF1ZXVlLmdhdGV3YXkuSGlzdG9yeUV2ZW50IkgKI0dldFJlcXVlc3RIaXN0b3J5QnlDaGFuZ2VVUklSZXF1ZXN0EhIKCmNoYW5nZV91cmkYASABKAkSDQoFcXVldWUYAiABKAkiVgoOUmVxdWVzdEhpc3RvcnkSDAoEc3FpZBgBIAEoCRI2CgZldmVudHMYAiADKAsyJi51YmVyLnN1Ym1pdHF1ZXVlLmdhdGV3YXkuSGlzdG9yeUV2ZW50ImMKJEdldFJlcXVlc3RIaXN0b3J5QnlDaGFuZ2VVUklSZXNwb25zZRI7CgloaXN0b3JpZXMYASADKAsyKC51YmVyLnN1Ym1pdHF1ZXVlLmdhdGV3YXkuUmVxdWVzdEhpc3RvcnkiGAoFRXJyb3ISDwoHbWVzc2FnZRgBIAEoCSJXChZVbnJlY29nbml6ZWRRdWV1ZUVycm9yEi4KBWVycm9yGAEgASgLMh8udWJlci5zdWJtaXRxdWV1ZS5nYXRld2F5LkVycm9yEg0KBXF1ZXVlGAIgASgJImgKFFJlcXVlc3ROb3RGb3VuZEVycm9yEi4KBWVycm9yGAEgASgLMh8udWJlci5zdWJtaXRxdWV1ZS5nYXRld2F5LkVycm9yEgwKBHNxaWQYAiABKAkSEgoKY2hhbmdlX3VyaRgDIAEoCTLHCAoSU3VibWl0UXVldWVHYXRld2F5ElcKBFBpbmcSJS51YmVyLnN1Ym1pdHF1ZXVlLmdhdGV3YXkuUGluZ1JlcXVlc3QaJi51YmVyLnN1Ym1pdHF1ZXVlLmdhdGV3YXkuUGluZ1Jlc3BvbnNlIgASaQoKTGlzdFF1ZXVlcxIrLnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5MaXN0UXVldWVzUmVxdWVzdBosLnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5MaXN0UXVldWVzUmVzcG9uc2UiABJXCgRMYW5kEiUudWJlci5zdWJtaXRxdWV1ZS5nYXRld2F5LkxhbmRSZXF1ZXN0GiYudWJlci5zdWJtaXRxdWV1ZS5nYXRld2F5LkxhbmRSZXNwb25zZSIAEl0KBkNhbmNlbBInLnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5DYW5jZWxSZXF1ZXN0GigudWJlci5zdWJtaXRxdWV1ZS5nYXRld2F5LkNhbmNlbFJlc3BvbnNlIgASigEKFUdldFJlcXVlc3RTdW1tYXJ5QnlJRBI2LnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5HZXRSZXF1ZXN0U3VtbWFyeUJ5SURSZXF1ZXN0GjcudWJlci5zdWJtaXRxdWV1ZS5nYXRld2F5LkdldFJlcXVlc3RTdW1tYXJ5QnlJRFJlc3BvbnNlIgASnwEKHEdldFJlcXVlc3RTdW1tYXJ5QnlDaGFuZ2VVUkkSPS51YmVyLnN1Ym1pdHF1ZXVlLmdhdGV3YXkuR2V0UmVxdWVzdFN1bW1hcnlCeUNoYW5nZVVSSVJlcXVlc3QaPi51YmVyLnN1Ym1pdHF1ZXVlLmdhdGV3YXkuR2V0UmVxdWVzdFN1bW1hcnlCeUNoYW5nZVVSSVJlc3BvbnNlIgASVwoETGlzdBIlLnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5MaXN0UmVxdWVzdBomLnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5MaXN0UmVzcG9uc2UiABKKAQoVR2V0UmVxdWVzdEhpc3RvcnlCeUlEEjYudWJlci5zdWJtaXRxdWV1ZS5nYXRld2F5LkdldFJlcXVlc3RIaXN0b3J5QnlJRFJlcXVlc3QaNy51YmVyLnN1Ym1pdHF1ZXVlLmdhdGV3YXkuR2V0UmVxdWVzdEhpc3RvcnlCeUlEUmVzcG9uc2UiABKfAQocR2V0UmVxdWVzdEhpc3RvcnlCeUNoYW5nZVVSSRI9LnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5HZXRSZXF1ZXN0SGlzdG9yeUJ5Q2hhbmdlVVJJUmVxdWVzdBo+LnViZXIuc3VibWl0cXVldWUuZ2F0ZXdheS5HZXRSZXF1ZXN0SGlzdG9yeUJ5Q2hhbmdlVVJJUmVzcG9uc2UiAEJrChxjb20udWJlci5zdWJtaXRxdWV1ZS5nYXRld2F5QgxHYXRld2F5UHJvdG9QAVo7Z2l0aHViLmNvbS91YmVyL3N1Ym1pdHF1ZXVlL2FwaS9zdWJtaXRxdWV1ZS9nYXRld2F5L3Byb3RvcGJiBnByb3RvMw", [file_api_base_change_proto_change, file_api_base_mergestrategy_proto_mergestrategy]); /** * PingRequest is the request for the Ping method @@ -387,18 +387,30 @@ export type ListRequest = Message<"uber.submitqueue.gateway.ListRequest"> & { queue: string; /** - * Inclusive lower receipt-time bound in Unix milliseconds. - * - * @generated from field: int64 received_at_or_after_ms = 2; + * @generated from oneof uber.submitqueue.gateway.ListRequest.received_lower_bound */ - receivedAtOrAfterMs: bigint; + receivedLowerBound: { + /** + * Inclusive receipt-time bound in Unix milliseconds. Required without page_token; with page_token, omitted takes the token's bound and supplied must equal it. + * + * @generated from field: int64 received_at_or_after_ms = 2; + */ + value: bigint; + case: "receivedAtOrAfterMs"; + } | { case: undefined; value?: undefined }; /** - * Exclusive upper receipt-time bound in Unix milliseconds. - * - * @generated from field: int64 received_before_ms = 3; + * @generated from oneof uber.submitqueue.gateway.ListRequest.received_upper_bound */ - receivedBeforeMs: bigint; + receivedUpperBound: { + /** + * Exclusive receipt-time bound in Unix milliseconds. Required without page_token; with page_token, omitted takes the token's bound and supplied must equal it. + * + * @generated from field: int64 received_before_ms = 3; + */ + value: bigint; + case: "receivedBeforeMs"; + } | { case: undefined; value?: undefined }; /** * Maximum results to return. Zero selects the server default; values above the server maximum are invalid. @@ -408,7 +420,7 @@ export type ListRequest = Message<"uber.submitqueue.gateway.ListRequest"> & { pageSize: number; /** - * Opaque continuation token from a previous response. Empty selects the first page. + * Opaque continuation token from a previous response, bound to its queue and receipt window. Empty selects the first page. * * @generated from field: string page_token = 5; */ @@ -441,6 +453,20 @@ export type ListResponse = Message<"uber.submitqueue.gateway.ListResponse"> & { * @generated from field: string next_page_token = 2; */ nextPageToken: string; + + /** + * Inclusive lower receipt-time bound of the window this page was read from, in Unix milliseconds. + * + * @generated from field: int64 received_at_or_after_ms = 3; + */ + receivedAtOrAfterMs: bigint; + + /** + * Exclusive upper receipt-time bound of the window this page was read from, in Unix milliseconds. + * + * @generated from field: int64 received_before_ms = 4; + */ + receivedBeforeMs: bigint; }; /** diff --git a/web/service/submitqueue/README.md b/web/service/submitqueue/README.md index 263211a94..0bb5f6be9 100644 --- a/web/service/submitqueue/README.md +++ b/web/service/submitqueue/README.md @@ -16,7 +16,6 @@ Two Next.js specifics: `next.config.ts` sets `htmlLimitedBots: /.*/` so page tit | `SUBMITQUEUE_GATEWAY_URL` | Gateway URL (TLS unless plaintext is allowed) | | `SUBMITQUEUE_GATEWAY_ALLOW_PLAINTEXT` | `true` to allow local h2c | | `SUBMITQUEUE_WEB_TOKEN` | Basic-auth password for user `test` (demo default `test`) | -| `SUBMITQUEUE_WEB_CURSOR_SECRET` | Key that signs pagination cursors (demo default `local-demo-cursor-secret`) | | `SUBMITQUEUE_WEB_METRICS_PORT` | Optional Prometheus port | | `LOG_LEVEL` | pino level, default `info` | diff --git a/web/service/submitqueue/src/app/healthz/route.ts b/web/service/submitqueue/src/app/healthz/route.ts index ede5dfec3..531c19b92 100644 --- a/web/service/submitqueue/src/app/healthz/route.ts +++ b/web/service/submitqueue/src/app/healthz/route.ts @@ -1,6 +1,6 @@ import { createNextHealthRoute } from "@submitqueue/web-service-shared/health"; import { loadAuthConfiguration } from "../../server/auth"; -import { loadCursorSecret, loadGatewayConfiguration } from "../../server/config"; +import { loadGatewayConfiguration } from "../../server/config"; import { pingReferenceGateway } from "../../server/gateway"; import { resolveReferenceLogCallback } from "../../server/observability"; @@ -9,7 +9,6 @@ export const dynamic = "force-dynamic"; export const GET = createNextHealthRoute({ configuration: () => { loadAuthConfiguration(); - loadCursorSecret(); loadGatewayConfiguration(); }, gateway: pingReferenceGateway, diff --git a/web/service/submitqueue/src/server/config.test.ts b/web/service/submitqueue/src/server/config.test.ts index ca7be7c03..3547453a0 100644 --- a/web/service/submitqueue/src/server/config.test.ts +++ b/web/service/submitqueue/src/server/config.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { loadCursorSecret, loadGatewayConfiguration, loadMetricsPort } from "./config"; +import { loadGatewayConfiguration, loadMetricsPort } from "./config"; describe("loadGatewayConfiguration", () => { it("uses TLS by default", () => { @@ -39,13 +39,6 @@ describe("loadGatewayConfiguration", () => { }); }); -describe("loadCursorSecret", () => { - it("requires its own secret", () => { - expect(loadCursorSecret({ SUBMITQUEUE_WEB_CURSOR_SECRET: "cursor-secret" })).toBe("cursor-secret"); - expect(() => loadCursorSecret({ SUBMITQUEUE_WEB_TOKEN: "viewer-password" })).toThrow(); - }); -}); - describe("loadMetricsPort", () => { it.each([ ["unset", {}, null], diff --git a/web/service/submitqueue/src/server/config.ts b/web/service/submitqueue/src/server/config.ts index aadcc21b3..5b9140d1a 100644 --- a/web/service/submitqueue/src/server/config.ts +++ b/web/service/submitqueue/src/server/config.ts @@ -21,15 +21,6 @@ export function loadGatewayConfiguration(environment: Environment = process.env) return { baseUrl: parseServiceBaseUrl(rawBaseUrl, allowPlaintext), allowPlaintext }; } -/** The key that signs pagination cursors; independent of any viewer credential so either can rotate alone. */ -export function loadCursorSecret(environment: Environment = process.env): string { - const secret = environment.SUBMITQUEUE_WEB_CURSOR_SECRET ?? ""; - if (secret.length === 0) { - throw new Error("SUBMITQUEUE_WEB_CURSOR_SECRET must be set"); - } - return secret; -} - /** Port for the Prometheus metrics endpoint, or `null` when metrics export is disabled. */ export function loadMetricsPort(environment: Environment = process.env): number | null { const raw = environment.SUBMITQUEUE_WEB_METRICS_PORT; diff --git a/web/service/submitqueue/src/server/host.ts b/web/service/submitqueue/src/server/host.ts index 44825cd9d..d914eba0a 100644 --- a/web/service/submitqueue/src/server/host.ts +++ b/web/service/submitqueue/src/server/host.ts @@ -1,8 +1,6 @@ import "server-only"; -import { newHmacCursorCodec } from "@submitqueue/web-submitqueue/extension/cursor/hmac"; import { createSubmitQueueWeb, type SubmitQueueWeb } from "@submitqueue/web-submitqueue/server"; -import { loadCursorSecret } from "./config"; import { resolveReferenceGateway } from "./gateway"; import { instrumentSubmitQueueWeb } from "./instrumentation"; import { referenceTracer, resolveReferenceMeter } from "./observability"; @@ -17,7 +15,6 @@ let web: SubmitQueueWeb | undefined; export function resolveReferenceWeb(): SubmitQueueWeb { web ??= instrumentSubmitQueueWeb(createSubmitQueueWeb({ gateway: resolveReferenceGateway, - cursors: newHmacCursorCodec(loadCursorSecret()), }), resolveReferenceMeter(), referenceTracer); return web; } diff --git a/web/service/submitqueue/src/server/instrumentation.test.ts b/web/service/submitqueue/src/server/instrumentation.test.ts index 0255999a4..1281839ca 100644 --- a/web/service/submitqueue/src/server/instrumentation.test.ts +++ b/web/service/submitqueue/src/server/instrumentation.test.ts @@ -2,13 +2,12 @@ import { Code, ConnectError, type UnaryRequest, type UnaryResponse } from "@conn import { SpanStatusCode, type Meter, type Tracer } from "@opentelemetry/api"; import { createSubmitQueueWeb } from "@submitqueue/web-submitqueue/server"; import { createFakeGatewayReader } from "@submitqueue/web-submitqueue/extension/gateway/mock"; -import { newHmacCursorCodec } from "@submitqueue/web-submitqueue/extension/cursor/hmac"; import { describe, expect, it, vi } from "vitest"; import { instrumentSubmitQueueWeb, measureGatewayRequests } from "./instrumentation"; function createHostFixture() { const gateway = createFakeGatewayReader(); - const web = createSubmitQueueWeb({ gateway, cursors: newHmacCursorCodec("test-only-secret") }); + const web = createSubmitQueueWeb({ gateway }); const span = { setAttribute: vi.fn(), setStatus: vi.fn(), recordException: vi.fn(), end: vi.fn() }; const record = vi.fn(); const createHistogram = vi.fn(() => ({ record })); diff --git a/web/submitqueue/README.md b/web/submitqueue/README.md index 230f7bf1e..5f4888df1 100644 --- a/web/submitqueue/README.md +++ b/web/submitqueue/README.md @@ -7,20 +7,19 @@ The complete read-only SubmitQueue UI as a mountable module. Folders follow the | [`entity/`](entity/) | Serializable view models (requests, history, queues, changes, pages) and load results | | [`core/`](core/) | Paths and `parseRoute`, change references, status table, receipt windows | | [`extension/gateway/`](extension/gateway/) | The structural gateway contract, the gRPC-status classifier, and test fakes in `mock/` | -| [`extension/cursor/`](extension/cursor/) | The page-cursor contract and its HMAC implementation in `hmac/` | | [`controller/`](controller/) | One loader per page: directory, queue, request, change | | [`view/`](view/) | `SubmitQueueApp`, pages, the shell and small components, navigation, polling, and the stylesheet | | [`module.ts`](module.ts) | `createSubmitQueueWeb`, which serves one page request end to end | -Entry points: the root is client-safe (`SubmitQueueApp`, page frames, `WebNavigationProvider`, model types); `./server` has `createSubmitQueueWeb`, the gateway and cursor contracts, and `WebPaths`; `./extension/cursor/hmac` signs cursors; `./extension/gateway/mock` has test fakes; `./styles.css` is the full stylesheet. Controllers and helpers stay internal. +Entry points: the root is client-safe (`SubmitQueueApp`, page frames, `WebNavigationProvider`, model types); `./server` has `createSubmitQueueWeb`, the gateway contract, and `WebPaths`; `./extension/gateway/mock` has test fakes; `./styles.css` is the full stylesheet. Controllers and helpers stay internal. -**Hosting it.** Give `createSubmitQueueWeb` a gateway client (any client matching the structural contract, so the module ships no generated code), a `CursorCodec`, and optionally a mount path, authorization check, error classifier, or cached queue catalog. Route every page path to `handle`, render `SubmitQueueApp` with each page model, and wrap it in a `WebNavigationProvider` for client-side navigation. Hosts must load `./styles.css`, either by importing it or by serving it and linking to it; theme with the `--sq-*` properties. +**Hosting it.** Give `createSubmitQueueWeb` a gateway client (any client matching the structural contract, so the module ships no generated code) and optionally a mount path, authorization check, error classifier, or cached queue catalog. Route every page path to `handle`, render `SubmitQueueApp` with each page model, and wrap it in a `WebNavigationProvider` for client-side navigation. Hosts must load `./styles.css`, either by importing it or by serving it and linking to it; theme with the `--sq-*` properties. Pass `onLog` on each `handle` request to deliver severity, message, and structured metadata to the host's request-scoped logger. Errors retain their original value under `error`, and logs include the correlation `requestId` and page `path`. The callback must be synchronous; omitting it is silent, and callback exceptions do not change page results. The module has no logging-library dependency. Instrumentation belongs in the host: wrap `handle` to observe page results and instrument the concrete gateway client for RPC traces and timings. A `render` result can contain a failed load, so check `model.result.ok` before reporting success. The reference host uses native OpenTelemetry APIs; other hosts can use their own telemetry without an adapter. The reusable module has no telemetry dependency or instrumentation API. -**Behavior worth knowing.** Expected outcomes (missing resources, stale cursors) are results, never exceptions. Raw gateway errors are logged, never rendered. Views keep the last good data on transient failures. Request IDs stay opaque in URLs; dot-only segments get a visible `~` escape. +**Behavior worth knowing.** Older pages put the gateway's `List` continuation token in the URL as `?page=` unchanged; the token carries its own window, so continuing needs only the queue and token, and hosts hold no pagination secret. Expected outcomes (missing resources, page tokens the gateway rejects) are results, never exceptions. Raw gateway errors are logged, never rendered. Views keep the last good data on transient failures. Request IDs stay opaque in URLs; dot-only segments get a visible `~` escape. ```bash ./tool/bazel test //web/submitqueue:all diff --git a/web/submitqueue/controller/change.test.ts b/web/submitqueue/controller/change.test.ts index 5920cca76..460e9f96d 100644 --- a/web/submitqueue/controller/change.test.ts +++ b/web/submitqueue/controller/change.test.ts @@ -5,36 +5,46 @@ import { readChangeSubmissions } from "./change.js"; const uri = "github://github.com/uber/submitqueue/pull/123/0123456789abcdef0123456789abcdef01234567"; const request = (sqid: string) => ({ sqid, changeUris: [uri] }); -const window = { fromMs: 100, toMs: 200 }; +const start = { window: { fromMs: 100, toMs: 200 } }; +const listPage = (requests: unknown[], nextPageToken: string) => ({ + requests, nextPageToken, receivedAtOrAfterMs: 100n, receivedBeforeMs: 200n, +}); describe("bounded change history reader", () => { it("loads all queue pages and keeps repeated submissions distinct", async () => { const list = vi.fn() - .mockResolvedValueOnce({ requests: [request("42")], nextPageToken: "next" }) - .mockResolvedValueOnce({ requests: [request("38")], nextPageToken: "" }); + .mockResolvedValueOnce(listPage([request("42")], "next")) + .mockResolvedValueOnce(listPage([request("38")], "")); const client = { list } as unknown as SubmitQueueGateway; const reference = parseChangePath(["github", "github.com", "uber", "submitqueue", "pull", "123"])!; - const result = await readChangeSubmissions(client, "demo-queue", reference, window); + const result = await readChangeSubmissions(client, "demo-queue", reference, start); expect(result.submissions.map(value => value.request.sqid)).toEqual(["42", "38"]); expect(result.nextPageToken).toBeNull(); - expect(list).toHaveBeenLastCalledWith(expect.objectContaining({ pageToken: "next", receivedAtOrAfterMs: 100n, receivedBeforeMs: 200n })); + expect(result.window).toEqual({ fromMs: 100, toMs: 200 }); + expect(list).toHaveBeenNthCalledWith(1, expect.objectContaining({ + pageToken: "", + receivedLowerBound: { case: "receivedAtOrAfterMs", value: 100n }, + receivedUpperBound: { case: "receivedBeforeMs", value: 200n }, + })); + expect(list).toHaveBeenLastCalledWith({ queue: "demo-queue", pageSize: 50, pageToken: "next" }); }); it("uses exact-URI lookup rather than the receipt window for a pinned version", async () => { const getRequestSummaryByChangeURI = vi.fn().mockResolvedValue({ requests: [request("42")] }); const list = vi.fn(); const client = { list, getRequestSummaryByChangeURI } as unknown as SubmitQueueGateway; - await readChangeSubmissions(client, "demo-queue", parseChangeUri(uri)!, window); + const result = await readChangeSubmissions(client, "demo-queue", parseChangeUri(uri)!, start); expect(getRequestSummaryByChangeURI).toHaveBeenCalledWith({ queue: "demo-queue", changeUri: uri }); expect(list).not.toHaveBeenCalled(); + expect(result.window).toBeNull(); }); it("finds demo git changes by clean identity while retaining raw file hints in backend data", async () => { const uri = "git://git.example.com/demo/refs%2Fheads%2Fmain/0123456789abcdef0123456789abcdef01234567?sq-files=demo%2F00%2Ffile.txt"; - const list = vi.fn().mockResolvedValue({ requests: [{ sqid: "42", changeUris: [uri] }], nextPageToken: "" }); + const list = vi.fn().mockResolvedValue(listPage([{ sqid: "42", changeUris: [uri] }], "")); const getRequestSummaryByChangeURI = vi.fn(); const client = { list, getRequestSummaryByChangeURI } as unknown as SubmitQueueGateway; - const result = await readChangeSubmissions(client, "demo-queue", parseChangeUri(uri)!, window); + const result = await readChangeSubmissions(client, "demo-queue", parseChangeUri(uri)!, start); expect(result.submissions[0]?.request.sqid).toBe("42"); expect(result.submissions[0]?.versionHref).not.toContain("?"); expect(getRequestSummaryByChangeURI).not.toHaveBeenCalled(); @@ -43,39 +53,46 @@ describe("bounded change history reader", () => { it.each([true, false])("continues a bounded scan without losing older matches (first page has matches: %s)", async (hasMatches) => { const list = vi.fn().mockImplementation(async ({ pageToken }: { pageToken: string }) => { const page = pageToken ? Number(pageToken.slice(5)) : 0; - return { - requests: page === 10 ? [request("38")] : page === 0 && hasMatches ? [request("42")] : [], - nextPageToken: page < 10 ? `page-${page + 1}` : "", - }; + return listPage( + page === 10 ? [request("38")] : page === 0 && hasMatches ? [request("42")] : [], + page < 10 ? `page-${page + 1}` : "", + ); }); const client = { list } as unknown as SubmitQueueGateway; const reference = parseChangePath(["github", "github.com", "uber", "submitqueue", "pull", "123"])!; - const first = await readChangeSubmissions(client, "demo-queue", reference, window); + const first = await readChangeSubmissions(client, "demo-queue", reference, start); expect(first.submissions.map(value => value.request.sqid)).toEqual(hasMatches ? ["42"] : []); expect(first.nextPageToken).toBe("page-10"); expect(list).toHaveBeenCalledTimes(10); - const second = await readChangeSubmissions(client, "demo-queue", reference, { ...window, pageToken: first.nextPageToken! }); + const second = await readChangeSubmissions(client, "demo-queue", reference, { pageToken: first.nextPageToken! }); expect(second.submissions.map(value => value.request.sqid)).toEqual(["38"]); expect(second.nextPageToken).toBeNull(); - expect(list).toHaveBeenLastCalledWith(expect.objectContaining({ - pageToken: "page-10", receivedAtOrAfterMs: 100n, receivedBeforeMs: 200n, - })); + expect(second.window).toEqual({ fromMs: 100, toMs: 200 }); + expect(list).toHaveBeenLastCalledWith({ queue: "demo-queue", pageSize: 50, pageToken: "page-10" }); }); - it("rejects a cursor that loops back to the starting page", async () => { + it("rejects a token that loops back to the starting page", async () => { const client = { - list: vi.fn().mockResolvedValue({ requests: [], nextPageToken: "start" }), + list: vi.fn().mockResolvedValue(listPage([], "start")), } as unknown as SubmitQueueGateway; const reference = parseChangePath(["github", "github.com", "uber", "submitqueue", "pull", "123"])!; - await expect(readChangeSubmissions(client, "demo-queue", reference, { ...window, pageToken: "start" })) + await expect(readChangeSubmissions(client, "demo-queue", reference, { pageToken: "start" })) .rejects.toBeInstanceOf(Error); }); it("fails a looping scan instead of presenting duplicate history", async () => { const client = { - list: vi.fn().mockResolvedValue({ requests: [request("42")], nextPageToken: "repeated" }), + list: vi.fn().mockResolvedValue(listPage([request("42")], "repeated")), + } as unknown as SubmitQueueGateway; + const reference = parseChangePath(["github", "github.com", "uber", "submitqueue", "pull", "123"])!; + await expect(readChangeSubmissions(client, "demo-queue", reference, start)).rejects.toBeInstanceOf(Error); + }); + + it("fails a scan whose gateway reports no usable window", async () => { + const client = { + list: vi.fn().mockResolvedValue({ requests: [], nextPageToken: "", receivedAtOrAfterMs: 0n, receivedBeforeMs: 0n }), } as unknown as SubmitQueueGateway; const reference = parseChangePath(["github", "github.com", "uber", "submitqueue", "pull", "123"])!; - await expect(readChangeSubmissions(client, "demo-queue", reference, window)).rejects.toBeInstanceOf(Error); + await expect(readChangeSubmissions(client, "demo-queue", reference, start)).rejects.toBeInstanceOf(Error); }); }); diff --git a/web/submitqueue/controller/change.ts b/web/submitqueue/controller/change.ts index cd5a44e0c..bf40ed6a0 100644 --- a/web/submitqueue/controller/change.ts +++ b/web/submitqueue/controller/change.ts @@ -4,10 +4,11 @@ import type { ChangeDetailModel } from "../entity/index.js"; import type { GatewayChangeReader, GatewayReader, GatewayRequestSummary } from "../extension/gateway/index.js"; import { changeHref, parseChangePath, parseChangeUri, type ChangeReference } from "../core/change.js"; import { WebPaths } from "../core/paths.js"; -import { decodeWindowCursor, defaultRequestWindow, encodeWindowCursor, REQUEST_PAGE_SIZE, type RequestWindow } from "../core/window.js"; +import { defaultRequestWindow, type RequestPageStart, type RequestWindow } from "../core/window.js"; import { INTERNAL_ERROR, INVALID_INPUT } from "./error.js"; import { callGateway, type GatewayLoadOptions } from "./gateway-call.js"; -import { mapRequestSummary } from "./mapping.js"; +import { mapRequestSummary, mapResponseWindow } from "./mapping.js"; +import { gatewayListInput } from "./queue.js"; import { notFound, redirect, refreshModel, render, type PageDependencies, type SubmitQueueWebResult } from "./page.js"; const MAX_CHANGE_SCAN_PAGES = 10; @@ -21,23 +22,27 @@ export interface GatewayChangeSubmission { export interface ChangeSubmissionPage { submissions: GatewayChangeSubmission[]; nextPageToken: string | null; + /** Receipt window the gateway scanned, or `null` for an exact lookup. */ + window: RequestWindow | null; } export interface GatewayChangeSubmissionPage { submissions: readonly GatewayChangeSubmission[]; pagination?: ChangeDetailModel["pagination"]; + /** Replaces the context's window once the gateway reports the one it scanned. */ + window?: RequestWindow | null; } /** * Finds the submissions of a change: by exact URI when the reference pins a * reviewable version, otherwise by scanning at most - * {@link MAX_CHANGE_SCAN_PAGES} queue pages of the receipt window. + * {@link MAX_CHANGE_SCAN_PAGES} queue pages from `start`. */ export async function readChangeSubmissions( client: GatewayReader & GatewayChangeReader, queue: string, reference: ChangeReference, - window: RequestWindow, + start: RequestPageStart, paths = new WebPaths(), ): Promise { const submissions: GatewayChangeSubmission[] = []; @@ -57,30 +62,29 @@ export async function readChangeSubmissions( if (reference.uri !== null && reference.scheme !== "git") { const response = await client.getRequestSummaryByChangeURI({ queue, changeUri: reference.uri }); appendSubmissions(response.requests); - return { submissions, nextPageToken: null }; + return { submissions, nextPageToken: null, window: null }; } - let pageToken = window.pageToken ?? ""; - const seenTokens = new Set(pageToken ? [pageToken] : []); + let pageStart = start; + let window: RequestWindow | null = null; + const seenTokens = new Set("pageToken" in start ? [start.pageToken] : []); for (let page = 0; page < MAX_CHANGE_SCAN_PAGES; page += 1) { - const response = await client.list({ - queue, - receivedAtOrAfterMs: BigInt(window.fromMs), - receivedBeforeMs: BigInt(window.toMs), - pageSize: REQUEST_PAGE_SIZE, - pageToken, - }); + const response = await client.list(gatewayListInput(queue, pageStart)); + window ??= mapResponseWindow(response); + if (window === null) { + throw new Error("The gateway returned an invalid receipt window"); + } appendSubmissions(response.requests); if (!response.nextPageToken) { - return { submissions, nextPageToken: null }; + return { submissions, nextPageToken: null, window }; } if (seenTokens.has(response.nextPageToken)) { // A gateway contract violation; continuing would present duplicate history. throw new Error("The gateway repeated a queue page token"); } - pageToken = response.nextPageToken; - seenTokens.add(pageToken); + pageStart = { pageToken: response.nextPageToken }; + seenTokens.add(response.nextPageToken); } - return { submissions, nextPageToken: pageToken }; + return { submissions, nextPageToken: "pageToken" in pageStart ? pageStart.pageToken : null, window }; } export async function loadChangeSubmissions( @@ -105,7 +109,14 @@ export async function loadChangeSubmissions( } submissions.push({ request, version: item.version, versionHref: item.versionHref }); } - return { ok: true, data: { ...context, submissions, ...(page.pagination ? { pagination: page.pagination } : {}) } }; + return { + ok: true, + data: { + ...context, submissions, + ...(page.window !== undefined ? { window: page.window } : {}), + ...(page.pagination ? { pagination: page.pagination } : {}), + }, + }; } const PROVIDER_NAMES: Readonly> = { @@ -127,35 +138,36 @@ export async function loadChangePage( const latestHref = changeHref(queue, reference, reference.version !== null, deps.paths); const scansQueue = reference.version === null || reference.scheme === "git"; if (search.from !== undefined || search.to !== undefined || - (search.page !== undefined && (typeof search.page !== "string" || !scansQueue))) { - return redirect(latestHref); - } - const scope = `submitqueue:change:${JSON.stringify([queue, reference.logicalPath, reference.version])}`; - const pageWindow = typeof search.page === "string" ? await decodeWindowCursor(deps.cursors, scope, search.page) : undefined; - if (search.page !== undefined && pageWindow === undefined) { + (search.page !== undefined && (typeof search.page !== "string" || search.page === "" || !scansQueue))) { return redirect(latestHref); } - const window = pageWindow ?? defaultRequestWindow(deps.now()); + const pageToken = search.page; + const liveWindow = defaultRequestWindow(deps.now()); const result = deps.catalog.ok ? await loadChangeSubmissions(async () => { - const page = await readChangeSubmissions(deps.gateway, queue, reference, window, deps.paths); + const page = await readChangeSubmissions( + deps.gateway, queue, reference, pageToken === undefined ? { window: liveWindow } : { pageToken }, deps.paths); return { submissions: page.submissions, - pagination: page.nextPageToken || pageWindow ? { - nextHref: page.nextPageToken - ? `${latestHref}?page=${await encodeWindowCursor(deps.cursors, scope, window, page.nextPageToken)}` : null, - latestHref: pageWindow ? latestHref : null, + window: page.window, + pagination: page.nextPageToken || pageToken !== undefined ? { + nextHref: page.nextPageToken ? `${latestHref}?page=${encodeURIComponent(page.nextPageToken)}` : null, + latestHref: pageToken !== undefined ? latestHref : null, } : undefined, }; }, { queue, provider: PROVIDER_NAMES[reference.scheme], host: reference.host, repository: reference.repository, review: reference.review, pinnedVersion: reference.version, logicalHref: changeHref(queue, reference, false, deps.paths), - window: scansQueue ? { fromMs: window.fromMs, toMs: window.toMs } : null, + window: scansQueue ? liveWindow : null, }, deps.gatewayOptions) : deps.catalog; - const refresh = refreshModel(result, pageWindow ? latestHref : null); + // The gateway rejects a stale or foreign token; restart from the live scan rather than show an error. + if (pageToken !== undefined && !result.ok && result.error === INVALID_INPUT) { + return redirect(latestHref); + } + const refresh = refreshModel(result, pageToken !== undefined ? latestHref : null); return render({ kind: "change", - key: `${queue}:${reference.logicalPath}:${reference.version ?? "all"}:${typeof search.page === "string" ? search.page : "live"}`, + key: `${queue}:${reference.logicalPath}:${reference.version ?? "all"}:${pageToken ?? "live"}`, title: reference.review, basePath: deps.paths.basePath, result, // Each render of a scanning page reads up to MAX_CHANGE_SCAN_PAGES queue pages, too costly to poll. refresh: scansQueue ? { ...refresh, terminal: true } : refresh, diff --git a/web/submitqueue/controller/controller.test.ts b/web/submitqueue/controller/controller.test.ts index 7733da6ac..35ead506a 100644 --- a/web/submitqueue/controller/controller.test.ts +++ b/web/submitqueue/controller/controller.test.ts @@ -65,15 +65,12 @@ describe("server presentation mapping", () => { } }); - it("maps a request page while preserving its bounds", async () => { - const reader = createFakeGatewayReader({ nextPageToken: "next" }); - const result = await loadRequestList(reader, { - queue: "demo-queue", - receivedAtOrAfterMs: 100, - receivedBeforeMs: 200, - pageToken: "current", - }); + it("maps a continuation page with the window its token carries", async () => { + const reader = createFakeGatewayReader({ nextPageToken: "next", window: { fromMs: 100n, toMs: 200n } }); + const list = vi.spyOn(reader, "list"); + const result = await loadRequestList(reader, { queue: "demo-queue", pageToken: "current" }); + expect(list).toHaveBeenCalledWith({ queue: "demo-queue", pageSize: 50, pageToken: "current" }); expect(result).toMatchObject({ ok: true, data: { @@ -93,11 +90,7 @@ describe("server presentation mapping", () => { const getRequestHistoryByID = vi.fn(fake.getRequestHistoryByID); const reader = { list, getRequestSummaryByID, getRequestHistoryByID }; - await loadRequestList(reader, { - queue: "demo-queue", - receivedAtOrAfterMs: 100, - receivedBeforeMs: 200, - }); + await loadRequestList(reader, { queue: "demo-queue", window: { fromMs: 100, toMs: 200 } }); await loadRequestDetail(reader, { queue: "demo-queue", sqid: "demo-queue/1", @@ -111,11 +104,7 @@ describe("server presentation mapping", () => { it("returns an explicit invalid-input result before calling the gateway", async () => { const gateway = createFakeGatewayReader(); const list = vi.spyOn(gateway, "list"); - const result = await loadRequestList(gateway, { - queue: "demo-queue", - receivedAtOrAfterMs: 200, - receivedBeforeMs: 100, - }); + const result = await loadRequestList(gateway, { queue: "demo-queue", window: { fromMs: 200, toMs: 100 } }); expect(result).toMatchObject({ ok: false, error: { kind: "user" } }); expect(list).not.toHaveBeenCalled(); @@ -257,7 +246,7 @@ describe("gateway error classification", () => { const onLog = vi.fn(); const result = await loadRequestList( createFakeGatewayReader({ listError: new Error("custom transport: upstream timed out") }), - { queue: "demo-queue", receivedAtOrAfterMs: 100, receivedBeforeMs: 200 }, + { queue: "demo-queue", window: { fromMs: 100, toMs: 200 } }, { classifyError: () => "unavailable", onLog }, ); expect(result).toMatchObject({ ok: false, error: { kind: "transient", retryable: true } }); @@ -268,7 +257,7 @@ describe("gateway error classification", () => { const onLog = vi.fn(); const result = await loadRequestList( createFakeGatewayReader({ listError: new Error("upstream") }), - { queue: "demo-queue", receivedAtOrAfterMs: 100, receivedBeforeMs: 200 }, + { queue: "demo-queue", window: { fromMs: 100, toMs: 200 } }, { classifyError: () => { throw new Error("classifier bug"); }, onLog }, ); expect(result).toMatchObject({ ok: false, error: { kind: "internal", retryable: false } }); @@ -287,11 +276,7 @@ describe("gateway error classification", () => { const onLog = vi.fn(); const result = await loadRequestList( createFakeGatewayReader({ listError: cause }), - { - queue: "demo-queue", - receivedAtOrAfterMs: 100, - receivedBeforeMs: 200, - }, + { queue: "demo-queue", window: { fromMs: 100, toMs: 200 } }, { onLog }, ); diff --git a/web/submitqueue/controller/mapping.ts b/web/submitqueue/controller/mapping.ts index daa30a429..1f884c7f3 100644 --- a/web/submitqueue/controller/mapping.ts +++ b/web/submitqueue/controller/mapping.ts @@ -1,4 +1,5 @@ -import { safeInt64ToNumber } from "../core/int64.js"; +import { safeInt64ToNumber, type Int64Value } from "../core/int64.js"; +import type { RequestWindow } from "../core/window.js"; import type { HistoryEventModel, RequestSummaryModel } from "../entity/index.js"; import type { GatewayHistoryEvent, GatewayRequestSummary } from "../extension/gateway/index.js"; @@ -18,6 +19,12 @@ export function mapRequestSummary(value: GatewayRequestSummary): RequestSummaryM }; } +export function mapResponseWindow(value: { receivedAtOrAfterMs: Int64Value; receivedBeforeMs: Int64Value }): RequestWindow | null { + const fromMs = safeInt64ToNumber(value.receivedAtOrAfterMs); + const toMs = safeInt64ToNumber(value.receivedBeforeMs); + return fromMs === null || toMs === null || fromMs >= toMs ? null : { fromMs, toMs }; +} + export function mapHistoryEvent(value: GatewayHistoryEvent): HistoryEventModel | null { const timestampMs = safeInt64ToNumber(value.timestampMs); if (timestampMs === null) { diff --git a/web/submitqueue/controller/page.ts b/web/submitqueue/controller/page.ts index e34119a3e..9c019c670 100644 --- a/web/submitqueue/controller/page.ts +++ b/web/submitqueue/controller/page.ts @@ -1,7 +1,6 @@ import type { LoadResult, QueueModel, SubmitQueuePageModel, WebRefreshModel } from "../entity/index.js"; import type { SubmitQueueGateway } from "../extension/gateway/index.js"; import type { WebPaths } from "../core/paths.js"; -import type { CursorCodec } from "../extension/cursor/index.js"; import type { GatewayLoadOptions } from "./gateway-call.js"; /** Outcome of one page request; only `render` carries a page. */ @@ -14,7 +13,6 @@ export type SubmitQueueWebResult = /** Dependencies of one page load, resolved for the current request. */ export interface PageDependencies { gateway: SubmitQueueGateway; - cursors: CursorCodec; paths: WebPaths; now(): number; gatewayOptions: GatewayLoadOptions; diff --git a/web/submitqueue/controller/queue.ts b/web/submitqueue/controller/queue.ts index 3b64bf242..b8fa12a5c 100644 --- a/web/submitqueue/controller/queue.ts +++ b/web/submitqueue/controller/queue.ts @@ -3,30 +3,35 @@ import type { LoadResult } from "../entity/index.js"; import type { RequestListModel, RequestSummaryModel } from "../entity/index.js"; import type { GatewayReader } from "../extension/gateway/index.js"; import { requestChangeLabels, requestChangeLinks } from "../core/change.js"; -import { decodeWindowCursor, defaultRequestWindow, encodeWindowCursor, REQUEST_PAGE_SIZE } from "../core/window.js"; +import { defaultRequestWindow, REQUEST_PAGE_SIZE, type RequestPageStart } from "../core/window.js"; import { INTERNAL_ERROR, INVALID_INPUT } from "./error.js"; import { callGateway, type GatewayLoadOptions } from "./gateway-call.js"; -import { mapRequestSummary } from "./mapping.js"; +import { mapRequestSummary, mapResponseWindow } from "./mapping.js"; import { redirect, refreshModel, render, type PageDependencies, type SubmitQueueWebResult } from "./page.js"; -export interface LoadRequestListInput { +export type LoadRequestListInput = RequestPageStart & { queue: string; - receivedAtOrAfterMs: number; - receivedBeforeMs: number; pageSize?: number; - pageToken?: string; -} +}; function validListInput(input: LoadRequestListInput): boolean { const pageSize = input.pageSize ?? REQUEST_PAGE_SIZE; - return ( - input.queue !== "" && - Number.isSafeInteger(input.receivedAtOrAfterMs) && - Number.isSafeInteger(input.receivedBeforeMs) && - input.receivedAtOrAfterMs < input.receivedBeforeMs && - Number.isInteger(pageSize) && - pageSize > 0 - ); + const validStart = "pageToken" in input + ? input.pageToken !== "" + : Number.isSafeInteger(input.window.fromMs) && Number.isSafeInteger(input.window.toMs) && input.window.fromMs < input.window.toMs; + return input.queue !== "" && validStart && Number.isInteger(pageSize) && pageSize > 0; +} + +/** The gateway `list` input for one page; bounds are sent only when no token carries them. */ +export function gatewayListInput(queue: string, start: RequestPageStart, pageSize = REQUEST_PAGE_SIZE): Parameters[0] { + if ("pageToken" in start) { + return { queue, pageSize, pageToken: start.pageToken }; + } + return { + queue, pageSize, pageToken: "", + receivedLowerBound: { case: "receivedAtOrAfterMs", value: BigInt(start.window.fromMs) }, + receivedUpperBound: { case: "receivedBeforeMs", value: BigInt(start.window.toMs) }, + }; } export async function loadRequestList( @@ -37,17 +42,16 @@ export async function loadRequestList( if (!validListInput(input)) { return { ok: false, error: INVALID_INPUT }; } - const call = await callGateway(options, "list", { queue: input.queue }, () => gateway.list({ - queue: input.queue, - receivedAtOrAfterMs: BigInt(input.receivedAtOrAfterMs), - receivedBeforeMs: BigInt(input.receivedBeforeMs), - pageSize: input.pageSize ?? REQUEST_PAGE_SIZE, - pageToken: input.pageToken ?? "", - })); + const call = await callGateway(options, "list", { queue: input.queue }, () => + gateway.list(gatewayListInput(input.queue, input, input.pageSize))); if (!call.ok) { return call; } const response = call.value; + const window = mapResponseWindow(response); + if (window === null) { + return { ok: false, error: INTERNAL_ERROR }; + } const requests: RequestSummaryModel[] = []; for (const value of response.requests) { const request = mapRequestSummary(value); @@ -60,19 +64,14 @@ export async function loadRequestList( ok: true, data: { queue: input.queue, - receivedAtOrAfterMs: input.receivedAtOrAfterMs, - receivedBeforeMs: input.receivedBeforeMs, + receivedAtOrAfterMs: window.fromMs, + receivedBeforeMs: window.toMs, requests, nextPageToken: response.nextPageToken === "" ? null : response.nextPageToken, }, }; } -// Namespaced so a codec shared with other pages or modules never accepts this page's cursors elsewhere. -function queueCursorScope(queue: string): string { - return `submitqueue:queue:${queue}`; -} - export async function loadQueuePage( deps: PageDependencies, queue: string, @@ -80,26 +79,23 @@ export async function loadQueuePage( ): Promise { const latestHref = deps.paths.requests(queue); if (search.from !== undefined || search.to !== undefined || - (search.page !== undefined && typeof search.page !== "string")) { + (search.page !== undefined && (typeof search.page !== "string" || search.page === ""))) { return redirect(latestHref); } - const pageWindow = typeof search.page === "string" ? await decodeWindowCursor(deps.cursors, queueCursorScope(queue), search.page) : undefined; - if (search.page !== undefined && pageWindow === undefined) { + const pageToken = search.page; + const liveWindow = defaultRequestWindow(deps.now()); + const result = deps.catalog.ok ? await loadRequestList(deps.gateway, { + queue, pageSize: REQUEST_PAGE_SIZE, ...(pageToken === undefined ? { window: liveWindow } : { pageToken }), + }, deps.gatewayOptions) : deps.catalog; + // The gateway rejects a stale or foreign token; restart from the live window rather than show an error. + if (pageToken !== undefined && !result.ok && result.error === INVALID_INPUT) { return redirect(latestHref); } - const window = pageWindow ?? defaultRequestWindow(deps.now()); - const loaded = deps.catalog.ok ? await loadRequestList(deps.gateway, { - queue, receivedAtOrAfterMs: window.fromMs, receivedBeforeMs: window.toMs, - pageSize: REQUEST_PAGE_SIZE, ...(window.pageToken ? { pageToken: window.pageToken } : {}), - }, deps.gatewayOptions) : deps.catalog; - const result = loaded.ok && loaded.data.nextPageToken ? { - ...loaded, - data: { ...loaded.data, nextPageToken: await encodeWindowCursor(deps.cursors, queueCursorScope(queue), window, loaded.data.nextPageToken) }, - } : loaded; + const window = result.ok ? { fromMs: result.data.receivedAtOrAfterMs, toMs: result.data.receivedBeforeMs } : liveWindow; return render({ - kind: "queue", key: `${queue}:${typeof search.page === "string" ? search.page : "live"}`, title: queue, - basePath: deps.paths.basePath, queue, window: { fromMs: window.fromMs, toMs: window.toMs }, - paged: pageWindow !== undefined, latestHref, result, refresh: refreshModel(result, pageWindow ? latestHref : null), + kind: "queue", key: `${queue}:${pageToken ?? "live"}`, title: queue, + basePath: deps.paths.basePath, queue, window, + paged: pageToken !== undefined, latestHref, result, refresh: refreshModel(result, pageToken !== undefined ? latestHref : null), changeLinks: result.ok ? requestChangeLinks(queue, result.data.requests, deps.paths) : {}, changeLabels: result.ok ? requestChangeLabels(result.data.requests) : {}, }); diff --git a/web/submitqueue/core/window.test.ts b/web/submitqueue/core/window.test.ts index b4effcf4e..f52291c5a 100644 --- a/web/submitqueue/core/window.test.ts +++ b/web/submitqueue/core/window.test.ts @@ -1,41 +1,9 @@ -import { newHmacCursorCodec } from "../extension/cursor/hmac/index.js"; import { describe, expect, it } from "vitest"; -import { decodeWindowCursor, defaultRequestWindow, encodeWindowCursor, REQUEST_WINDOW_MS } from "./window.js"; +import { defaultRequestWindow, REQUEST_WINDOW_MS } from "./window.js"; -const codec = newHmacCursorCodec("test-pagination-secret"); -const queue = "demo-queue"; -const window = defaultRequestWindow(REQUEST_WINDOW_MS + 100); - -describe("live request window and snapshot pagination", () => { +describe("live request window", () => { it("advances the default window each time it is refreshed", () => { expect(defaultRequestWindow(REQUEST_WINDOW_MS + 100)).toEqual({ fromMs: 100, toMs: REQUEST_WINDOW_MS + 100 }); expect(defaultRequestWindow(REQUEST_WINDOW_MS + 200)).toEqual({ fromMs: 200, toMs: REQUEST_WINDOW_MS + 200 }); }); - - it("round-trips stable gateway paging bounds inside one opaque cursor", async () => { - const cursor = await encodeWindowCursor(codec, queue, window, "opaque/token+=\nvalue"); - expect(await decodeWindowCursor(codec, queue, cursor)).toEqual({ ...window, pageToken: "opaque/token+=\nvalue" }); - }); - - it("rejects altered, cross-queue, or differently signed cursors", async () => { - const cursor = await encodeWindowCursor(codec, queue, window, "next"); - expect(await decodeWindowCursor(codec, "other-queue", cursor)).toBeUndefined(); - expect(await decodeWindowCursor(codec, queue, `${cursor}x`)).toBeUndefined(); - expect(await decodeWindowCursor(newHmacCursorCodec("other-secret"), queue, cursor)).toBeUndefined(); - }); - - it.each(["", "not-a-cursor", ".invalid", "a".repeat(16_385)])("rejects malformed cursor", async (cursor) => { - expect(await decodeWindowCursor(codec, queue, cursor)).toBeUndefined(); - }); - - it.each([ - ["a missing token", window, ""], - ["inverted bounds", { fromMs: 200, toMs: 100 }, "next"], - ])("rejects a signed cursor with %s", async (_name, bounds, token) => { - expect(await decodeWindowCursor(codec, queue, await encodeWindowCursor(codec, queue, bounds, token))).toBeUndefined(); - }); - - it("rejects a signed payload from another cursor format", async () => { - expect(await decodeWindowCursor(codec, queue, await codec.encode(queue, "v0\n1\n2\nnext"))).toBeUndefined(); - }); }); diff --git a/web/submitqueue/core/window.ts b/web/submitqueue/core/window.ts index 7c42a0782..ba5ac55ab 100644 --- a/web/submitqueue/core/window.ts +++ b/web/submitqueue/core/window.ts @@ -1,46 +1,15 @@ -import type { CursorCodec } from "../extension/cursor/index.js"; - export const REQUEST_WINDOW_MS = 24 * 60 * 60 * 1_000; export const REQUEST_PAGE_SIZE = 50; -const CURSOR_VERSION = "v1"; - +/** Half-open receipt window, Unix epoch milliseconds. */ export type RequestWindow = Readonly<{ fromMs: number; toMs: number; - pageToken?: string; }>; +/** Where a queue read starts: a new window, or a gateway continuation token, which carries its own window. */ +export type RequestPageStart = Readonly<{ window: RequestWindow }> | Readonly<{ pageToken: string }>; + export function defaultRequestWindow(nowMs = Date.now()): RequestWindow { return { fromMs: nowMs - REQUEST_WINDOW_MS, toMs: nowMs }; } - -function parseMillisecond(value: string | undefined): number | undefined { - if (!value || !/^\d+$/u.test(value)) { - return undefined; - } - const parsed = Number(value); - return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : undefined; -} - -/** Encodes a snapshot page — the original window bounds plus the gateway's token — as a scoped cursor. */ -export async function encodeWindowCursor(codec: CursorCodec, scope: string, window: RequestWindow, token: string): Promise { - return codec.encode(scope, [CURSOR_VERSION, window.fromMs, window.toMs, token].join("\n")); -} - -/** Decodes a cursor from {@link encodeWindowCursor}; `undefined` when it is foreign, tampered, or malformed. */ -export async function decodeWindowCursor(codec: CursorCodec, scope: string, cursor: string): Promise { - const payload = await codec.decode(scope, cursor); - if (payload === undefined) { - return undefined; - } - const [version, from, to, ...tokenParts] = payload.split("\n"); - const fromMs = parseMillisecond(from); - const toMs = parseMillisecond(to); - const pageToken = tokenParts.join("\n"); - if (version !== CURSOR_VERSION || fromMs === undefined || toMs === undefined || - toMs - fromMs !== REQUEST_WINDOW_MS || !pageToken) { - return undefined; - } - return { fromMs, toMs, pageToken }; -} diff --git a/web/submitqueue/extension/cursor/cursor.ts b/web/submitqueue/extension/cursor/cursor.ts deleted file mode 100644 index 37ed20d29..000000000 --- a/web/submitqueue/extension/cursor/cursor.ts +++ /dev/null @@ -1,10 +0,0 @@ -/** - * Turns server-side continuation state into an opaque, tamper-evident URL - * value and back. A cursor is bound to `scope`: decoding it under any other - * scope, or decoding a modified cursor, yields `undefined`. Either method may - * be asynchronous so a codec can use remote key management or WebCrypto. - */ -export interface CursorCodec { - encode(scope: string, payload: string): string | Promise; - decode(scope: string, cursor: string): string | undefined | Promise; -} diff --git a/web/submitqueue/extension/cursor/hmac/hmac.test.ts b/web/submitqueue/extension/cursor/hmac/hmac.test.ts deleted file mode 100644 index 9db35d132..000000000 --- a/web/submitqueue/extension/cursor/hmac/hmac.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { newHmacCursorCodec } from "./index.js"; - -describe("newHmacCursorCodec", () => { - const codec = newHmacCursorCodec("test-secret"); - - it("round-trips a payload within its scope", () => { - const cursor = codec.encode("demo-queue", "v1\n100\n200\ntoken/with\nnewline") as string; - expect(cursor).toMatch(/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/u); - expect(codec.decode("demo-queue", cursor)).toBe("v1\n100\n200\ntoken/with\nnewline"); - }); - - it.each([ - ["another scope", codec, "other-queue", (cursor: string) => cursor], - ["a modified payload", codec, "demo-queue", (cursor: string) => `A${cursor}`], - ["another secret", newHmacCursorCodec("other-secret"), "demo-queue", (cursor: string) => cursor], - ["malformed input", codec, "demo-queue", () => "not a cursor"], - ])("rejects %s", (_name, decoder, scope, alter) => { - expect(decoder.decode(scope, alter(codec.encode("demo-queue", "payload") as string))).toBeUndefined(); - }); - - it("refuses an empty secret", () => { - expect(() => newHmacCursorCodec("")).toThrow(); - }); -}); diff --git a/web/submitqueue/extension/cursor/hmac/hmac.ts b/web/submitqueue/extension/cursor/hmac/hmac.ts deleted file mode 100644 index 68173dc80..000000000 --- a/web/submitqueue/extension/cursor/hmac/hmac.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { createHmac, timingSafeEqual } from "node:crypto"; -import type { CursorCodec } from "../cursor.js"; - -const MAX_CURSOR_LENGTH = 16_384; -const CURSOR_PATTERN = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/u; - -/** - * An HMAC-SHA256 {@link CursorCodec} for Node. Cursors are readable base64url - * payloads with a signature over scope and payload, so they are tamper-evident - * but not confidential; never put secrets in a payload. - */ -export function newHmacCursorCodec(secret: string): CursorCodec { - if (!secret) { - throw new Error("The cursor signing secret must be nonempty"); - } - const sign = (scope: string, body: string) => - createHmac("sha256", secret).update(`${encodeURIComponent(scope)}\n${body}`).digest(); - return { - encode(scope, payload) { - const body = Buffer.from(payload, "utf8").toString("base64url"); - return `${body}.${sign(scope, body).toString("base64url")}`; - }, - decode(scope, cursor) { - if (cursor.length > MAX_CURSOR_LENGTH || !CURSOR_PATTERN.test(cursor)) { - return undefined; - } - const [body, signature] = cursor.split(".") as [string, string]; - const actual = Buffer.from(signature, "base64url"); - const expected = sign(scope, body); - if (actual.length !== expected.length || !timingSafeEqual(actual, expected)) { - return undefined; - } - return Buffer.from(body, "base64url").toString("utf8"); - }, - }; -} diff --git a/web/submitqueue/extension/cursor/hmac/index.ts b/web/submitqueue/extension/cursor/hmac/index.ts deleted file mode 100644 index f1119ce31..000000000 --- a/web/submitqueue/extension/cursor/hmac/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { newHmacCursorCodec } from "./hmac.js"; diff --git a/web/submitqueue/extension/cursor/index.ts b/web/submitqueue/extension/cursor/index.ts deleted file mode 100644 index ae4a3220e..000000000 --- a/web/submitqueue/extension/cursor/index.ts +++ /dev/null @@ -1 +0,0 @@ -export type { CursorCodec } from "./cursor.js"; diff --git a/web/submitqueue/extension/gateway/gateway.ts b/web/submitqueue/extension/gateway/gateway.ts index eb86d464c..7a4edecdf 100644 --- a/web/submitqueue/extension/gateway/gateway.ts +++ b/web/submitqueue/extension/gateway/gateway.ts @@ -33,11 +33,12 @@ export interface GatewayHistoryEvent { } export interface GatewayReader { + /** Bounds are sent on a first page only; a `pageToken` carries its own window. */ list( input: { queue: string; - receivedAtOrAfterMs: bigint; - receivedBeforeMs: bigint; + receivedLowerBound?: { case: "receivedAtOrAfterMs"; value: bigint }; + receivedUpperBound?: { case: "receivedBeforeMs"; value: bigint }; pageSize: number; pageToken: string; }, @@ -45,6 +46,8 @@ export interface GatewayReader { ): Promise<{ requests: readonly GatewayRequestSummary[]; nextPageToken: string; + receivedAtOrAfterMs: Int64Value; + receivedBeforeMs: Int64Value; }>; getRequestSummaryByID( input: { sqid: string; queue: string }, diff --git a/web/submitqueue/extension/gateway/mock/fake.ts b/web/submitqueue/extension/gateway/mock/fake.ts index 83da90a66..3ab712b2a 100644 --- a/web/submitqueue/extension/gateway/mock/fake.ts +++ b/web/submitqueue/extension/gateway/mock/fake.ts @@ -1,6 +1,7 @@ import type { GatewayHistoryEvent, GatewayRequestSummary, SubmitQueueGateway } from "../gateway.js"; const DEFAULT_TIMESTAMP = 1_700_000_000_000n; +const DEFAULT_WINDOW_MS = 86_400_000n; export function gatewayRequestFixture( overrides: Partial = {}, @@ -36,6 +37,8 @@ export interface FakeGatewayReaderOptions { queuesError?: unknown; requests?: readonly GatewayRequestSummary[]; nextPageToken?: string; + /** Window reported for a continuation, which sends no bounds; a first page echoes its own bounds. */ + window?: { fromMs: bigint; toMs: bigint }; summary?: GatewayRequestSummary; history?: readonly GatewayHistoryEvent[]; listError?: unknown; @@ -54,13 +57,16 @@ export function createFakeGatewayReader(options: FakeGatewayReaderOptions = {}): } return { queues: options.queues ?? [{ name: "demo-queue" }] }; }, - async list() { + async list(input) { if (options.listError !== undefined) { throw options.listError; } + const window = options.window ?? { fromMs: DEFAULT_TIMESTAMP - DEFAULT_WINDOW_MS, toMs: DEFAULT_TIMESTAMP }; return { requests: options.requests ?? [summary], nextPageToken: options.nextPageToken ?? "", + receivedAtOrAfterMs: input.receivedLowerBound?.value ?? window.fromMs, + receivedBeforeMs: input.receivedUpperBound?.value ?? window.toMs, }; }, async getRequestSummaryByID() { diff --git a/web/submitqueue/module.test.ts b/web/submitqueue/module.test.ts index 20067e9b7..1c8587e05 100644 --- a/web/submitqueue/module.test.ts +++ b/web/submitqueue/module.test.ts @@ -3,12 +3,10 @@ import { describe, expect, it, vi } from "vitest"; import { createSubmitQueueWeb, type SubmitQueueWebOptions, type SubmitQueueWebRequest } from "./module.js"; import { createFakeGatewayReader, gatewayHistoryFixture, gatewayRequestFixture } from "./extension/gateway/mock/index.js"; import type { SubmitQueueGateway } from "./extension/gateway/index.js"; -import { newHmacCursorCodec } from "./extension/cursor/hmac/index.js"; import { WebPaths } from "./core/paths.js"; import { REQUEST_WINDOW_MS } from "./core/window.js"; const now = 1_700_000_000_000; -const cursors = newHmacCursorCodec("host-owned-test-secret"); const paths = new WebPaths({ basePath: "/sq" }); const sha = "0123456789abcdef0123456789abcdef01234567"; const gitUri = `git://git.example.com/demo/refs%2Fheads%2Fmain/${sha}`; @@ -16,7 +14,7 @@ const changePath = "/sq/demo-queue/change/git/git.example.com/demo/refs%2Fheads% const githubPath = "/sq/demo-queue/change/github/github.com/uber/submitqueue/pull/123"; function newWeb(options: Partial = {}) { - const web = createSubmitQueueWeb({ gateway: createFakeGatewayReader(), cursors, paths, ...options }); + const web = createSubmitQueueWeb({ gateway: createFakeGatewayReader(), paths, ...options }); return (path: string, search: Record = {}, extra: Pick = {}) => web.handle({ path, search, ...extra }); } @@ -47,16 +45,23 @@ describe("createSubmitQueueWeb", () => { expect(await newWeb()(path)).toEqual({ kind: "not-found" }); }); - it("loads live pages and resumes signed snapshot cursors without moving their bounds", async () => { + it("links the raw gateway token and resumes it without resending bounds", async () => { let clock = now; - const gateway = createFakeGatewayReader({ nextPageToken: "opaque/gateway-token" }); + const gateway = createFakeGatewayReader({ + nextPageToken: "opaque/gateway-token", window: { fromMs: BigInt(now - REQUEST_WINDOW_MS), toMs: BigInt(now) }, + }); const list = vi.spyOn(gateway, "list"); const get = newWeb({ gateway, now: () => clock }); const first = await get("/sq/demo-queue"); expect(first).toMatchObject({ kind: "render", model: { kind: "queue", paged: false, latestHref: "/sq/demo-queue", refresh: { terminal: false } }, }); + expect(list).toHaveBeenLastCalledWith(expect.objectContaining({ + receivedLowerBound: { case: "receivedAtOrAfterMs", value: BigInt(now - REQUEST_WINDOW_MS) }, + receivedUpperBound: { case: "receivedBeforeMs", value: BigInt(now) }, + })); const page = nextPageToken(first); + expect(page).toBe("opaque/gateway-token"); clock += 10_000; expect(await get("/sq/demo-queue", { page })).toMatchObject({ kind: "render", model: { @@ -64,20 +69,26 @@ describe("createSubmitQueueWeb", () => { refresh: { terminal: true, refreshHref: "/sq/demo-queue" }, }, }); - expect(list).toHaveBeenLastCalledWith(expect.objectContaining({ - pageToken: "opaque/gateway-token", receivedAtOrAfterMs: BigInt(now - REQUEST_WINDOW_MS), receivedBeforeMs: BigInt(now), - })); + expect(list).toHaveBeenLastCalledWith({ queue: "demo-queue", pageSize: 50, pageToken: "opaque/gateway-token" }); expect(await get("/sq/demo-queue")).toMatchObject({ kind: "render", model: { window: { toMs: clock } } }); }); it.each([ { from: "1", to: "2" }, - { page: ["multiple", "cursors"] }, - { page: "tampered" }, + { page: ["multiple", "tokens"] }, + { page: "" }, ])("normalizes stale or invalid queue inputs to a live URL: %j", async search => { expect(await newWeb()("/sq/demo-queue", search)).toEqual({ kind: "redirect", href: "/sq/demo-queue" }); }); + it.each([ + ["queue", "/sq/demo-queue"], + ["change", changePath], + ])("restarts a %s page from the live window when the gateway rejects its token", async (_name, path) => { + const gateway = createFakeGatewayReader({ listError: new ConnectError("bad token", Code.InvalidArgument) }); + expect(await newWeb({ gateway })(path, { page: "tampered" })).toEqual({ kind: "redirect", href: path }); + }); + it("keeps request IDs opaque and provides complete history navigation and refresh policy", async () => { const sqid = "legacy/id%2Fvalue"; const gateway = createFakeGatewayReader({ @@ -96,20 +107,6 @@ describe("createSubmitQueueWeb", () => { }); }); - it("accepts asynchronous cursor infrastructure without putting promises in page URLs", async () => { - const get = newWeb({ - gateway: createFakeGatewayReader({ nextPageToken: "gateway-continuation" }), - cursors: { - encode: async (...args) => cursors.encode(...args), - decode: async (...args) => cursors.decode(...args), - }, - now: () => now, - }); - const page = nextPageToken(await get("/sq/demo-queue")); - expect(typeof page).toBe("string"); - expect(await get("/sq/demo-queue", { page })).toMatchObject({ kind: "render", model: { paged: true, window: { toMs: now } } }); - }); - it("returns explicit not-found decisions without exceptions for absent resources", async () => { const get = newWeb(); expect(await get("/sq/unknown")).toEqual({ kind: "not-found" }); @@ -146,11 +143,14 @@ describe("createSubmitQueueWeb", () => { }); }); - it("scopes change continuation to queue, reference, and version while preserving the snapshot", async () => { + it("continues a change scan from the raw gateway token and shows the token's window", async () => { const gateway = createFakeGatewayReader({ requests: [gatewayRequestFixture({ changeUris: [gitUri] })] }); const list = vi.spyOn(gateway, "list").mockImplementation(async input => { const page = input.pageToken ? Number(input.pageToken.slice(5)) : 0; - return { requests: [gatewayRequestFixture({ sqid: String(page + 100), changeUris: [gitUri] })], nextPageToken: page < 10 ? `page-${page + 1}` : "" }; + return { + requests: [gatewayRequestFixture({ sqid: String(page + 100), changeUris: [gitUri] })], nextPageToken: page < 10 ? `page-${page + 1}` : "", + receivedAtOrAfterMs: BigInt(now - REQUEST_WINDOW_MS), receivedBeforeMs: BigInt(now), + }; }); const get = newWeb({ gateway, now: () => now }); const first = await get(changePath); @@ -158,26 +158,17 @@ describe("createSubmitQueueWeb", () => { throw new Error("Expected a change page"); } const page = new URL(first.model.result.data.pagination!.nextHref!, "http://host.invalid").searchParams.get("page")!; + expect(page).toBe("page-10"); expect(first.model.result.data.submissions[0]?.versionHref).toBe(`${changePath}/${sha}`); - expect(await get(`${changePath}/${sha}`, { page })).toEqual({ kind: "redirect", href: `${changePath}/${sha}` }); expect(list).toHaveBeenCalledTimes(10); - expect(await get(changePath, { page })).toMatchObject({ + expect(await newWeb({ gateway, now: () => now + 10_000 })(changePath, { page })).toMatchObject({ kind: "render", - model: { refresh: { terminal: true, refreshHref: changePath }, result: { ok: true, data: { pagination: { nextHref: null } } } }, - }); - expect(list).toHaveBeenLastCalledWith(expect.objectContaining({ - pageToken: "page-10", receivedAtOrAfterMs: BigInt(now - REQUEST_WINDOW_MS), receivedBeforeMs: BigInt(now), - })); - }); - - it("refuses a queue-page cursor on a change page", async () => { - const get = newWeb({ - gateway: createFakeGatewayReader({ nextPageToken: "gateway-continuation", requests: [gatewayRequestFixture({ changeUris: [gitUri] })] }), - now: () => now, + model: { + refresh: { terminal: true, refreshHref: changePath }, + result: { ok: true, data: { window: { fromMs: now - REQUEST_WINDOW_MS, toMs: now }, pagination: { nextHref: null } } }, + }, }); - const queuePageHref = paths.requests("demo-queue", { pageToken: nextPageToken(await get("/sq/demo-queue")) }); - const page = new URL(queuePageHref, "https://host.invalid").searchParams.get("page")!; - expect(await get(changePath, { page })).toEqual({ kind: "redirect", href: changePath }); + expect(list).toHaveBeenLastCalledWith({ queue: "demo-queue", pageSize: 50, pageToken: "page-10" }); }); it("resolves the gateway per request, so authorization-dependent visibility is never cached", async () => { diff --git a/web/submitqueue/module.ts b/web/submitqueue/module.ts index 24aeca5c2..be3876767 100644 --- a/web/submitqueue/module.ts +++ b/web/submitqueue/module.ts @@ -8,7 +8,6 @@ import { notFound, type PageDependencies, type SubmitQueueWebResult } from "./co import { loadQueuePage } from "./controller/queue.js"; import { loadRequestPage } from "./controller/request.js"; import type { LoadResult, QueueModel } from "./entity/index.js"; -import type { CursorCodec } from "./extension/cursor/index.js"; import type { GatewayErrorClassifier, SubmitQueueGateway } from "./extension/gateway/index.js"; export type { SubmitQueueWebResult } from "./controller/page.js"; @@ -47,8 +46,6 @@ export interface QueueCatalogCache { export interface SubmitQueueWebOptions { /** The gateway, or a resolver called once per load; resolve per request when visibility depends on the viewer. */ gateway: SubmitQueueGateway | ((context: SubmitQueueRequestContext) => SubmitQueueGateway); - /** Signs snapshot-page cursors. */ - cursors: CursorCodec; /** Builds links; its base path is where the host mounts the UI. */ paths?: WebPaths; /** Decides whether the principal may see a route; `false` is reported as `forbidden`. */ @@ -123,7 +120,7 @@ export function createSubmitQueueWeb(options: SubmitQueueWebOptions): SubmitQueu ...(context.onLog ? { onLog: context.onLog } : {}), }; const catalog = await loadQueues(gateway, context); - const deps: PageDependencies = { gateway, cursors: options.cursors, paths, now, gatewayOptions, catalog }; + const deps: PageDependencies = { gateway, paths, now, gatewayOptions, catalog }; if (route.kind === "queues") { return loadDirectoryPage(deps); } diff --git a/web/submitqueue/package.json b/web/submitqueue/package.json index d4e6e24ed..e4dcaa2ee 100644 --- a/web/submitqueue/package.json +++ b/web/submitqueue/package.json @@ -7,8 +7,7 @@ "typesVersions": { "*": { "server": ["dist/server.d.ts"], - "extension/gateway/mock": ["dist/extension/gateway/mock/index.d.ts"], - "extension/cursor/hmac": ["dist/extension/cursor/hmac/index.d.ts"] + "extension/gateway/mock": ["dist/extension/gateway/mock/index.d.ts"] } }, "sideEffects": [ @@ -31,10 +30,6 @@ "./extension/gateway/mock": { "types": "./dist/extension/gateway/mock/index.d.ts", "import": "./dist/extension/gateway/mock/index.js" - }, - "./extension/cursor/hmac": { - "types": "./dist/extension/cursor/hmac/index.d.ts", - "import": "./dist/extension/cursor/hmac/index.js" } }, "scripts": { diff --git a/web/submitqueue/server.ts b/web/submitqueue/server.ts index 7fef45406..f92686233 100644 --- a/web/submitqueue/server.ts +++ b/web/submitqueue/server.ts @@ -6,7 +6,6 @@ export { } from "./module.js"; export * from "./extension/gateway/index.js"; export { grpcStatusErrorCode } from "./extension/gateway/grpcstatus/index.js"; -export * from "./extension/cursor/index.js"; export type * from "./entity/index.js"; export type { SubmitQueueWebRoute, WebSearchParams } from "./core/route.js"; export { WebPaths, type RequestPathOptions, type RequestsPathOptions, type WebPathsOptions } from "./core/paths.js"; diff --git a/web/submitqueue/view/pages.test.tsx b/web/submitqueue/view/pages.test.tsx index 164faea8d..72941cc51 100644 --- a/web/submitqueue/view/pages.test.tsx +++ b/web/submitqueue/view/pages.test.tsx @@ -4,7 +4,6 @@ import { WebNavigationProvider } from "@submitqueue/web-platform"; import { SubmitQueueApp } from "./app.js"; import { SubmitQueuePage, SubmitQueueShell, SubmitQueueStatePage } from "./pages.js"; import { createSubmitQueueWeb } from "../module.js"; -import { newHmacCursorCodec } from "../extension/cursor/hmac/index.js"; import { createFakeGatewayReader, gatewayHistoryFixture, gatewayRequestFixture } from "../extension/gateway/mock/index.js"; import { WebPaths } from "../core/paths.js"; import { webErrorForGatewayCode } from "../controller/error.js"; @@ -16,7 +15,7 @@ const gateway = createFakeGatewayReader({ history: [gatewayHistoryFixture()], }); const web = createSubmitQueueWeb({ - gateway, cursors: newHmacCursorCodec("host-secret"), paths: new WebPaths({ basePath: "/sq" }), + gateway, paths: new WebPaths({ basePath: "/sq" }), }); describe("default reusable application views", () => { diff --git a/web/test/e2e/submitqueue/run_e2e.py b/web/test/e2e/submitqueue/run_e2e.py index 92f62f667..2d30b7e4d 100644 --- a/web/test/e2e/submitqueue/run_e2e.py +++ b/web/test/e2e/submitqueue/run_e2e.py @@ -202,7 +202,6 @@ def handle_signal(signum, _frame): "SQ_CONSUMER_GATE_DIR": str(gate_dir), "SQ_DOCKER_IMAGE_PREFIX": _PROJECT, "SUBMITQUEUE_WEB_TOKEN": _TOKEN, - "SUBMITQUEUE_WEB_CURSOR_SECRET": "e2e-cursor-secret", "SQ_MYSQL_INITDB_SKIP_TZINFO": "1", } ) diff --git a/web/test/e2e/submitqueue/submitqueue.spec.ts b/web/test/e2e/submitqueue/submitqueue.spec.ts index 4c83fef8f..f12a2cbe1 100644 --- a/web/test/e2e/submitqueue/submitqueue.spec.ts +++ b/web/test/e2e/submitqueue/submitqueue.spec.ts @@ -220,9 +220,10 @@ test("paginates change history through more than 500 queue requests", async ({ p const toMs = BigInt(Date.now()); let pageToken = ""; do { - const response = await gateway.list({ - queue, receivedAtOrAfterMs: toMs - 86_400_000n, receivedBeforeMs: toMs, - pageSize: 50, pageToken, + const response = await gateway.list(pageToken ? { queue, pageSize: 50, pageToken } : { + queue, pageSize: 50, + receivedLowerBound: { case: "receivedAtOrAfterMs", value: toMs - 86_400_000n }, + receivedUpperBound: { case: "receivedBeforeMs", value: toMs }, }); response.requests.forEach(request => ids.add(request.sqid)); pageToken = response.nextPageToken; diff --git a/web/test/package/consumer-runtime.mjs b/web/test/package/consumer-runtime.mjs index ee52b037d..f796355f0 100644 --- a/web/test/package/consumer-runtime.mjs +++ b/web/test/package/consumer-runtime.mjs @@ -4,12 +4,10 @@ import { createElement } from "react"; import { renderToStaticMarkup } from "react-dom/server"; import { SubmitQueueApp, SubmitQueueShell } from "@submitqueue/web-submitqueue"; import { createSubmitQueueWeb, WebPaths } from "@submitqueue/web-submitqueue/server"; -import { newHmacCursorCodec } from "@submitqueue/web-submitqueue/extension/cursor/hmac"; import { createFakeGatewayReader } from "@submitqueue/web-submitqueue/extension/gateway/mock"; const web = createSubmitQueueWeb({ gateway: createFakeGatewayReader(), - cursors: newHmacCursorCodec("ordinary-server-signing-secret"), paths: new WebPaths({ basePath: "/queues" }), }); const result = await web.handle({ path: "/queues/demo-queue", search: {} }); diff --git a/web/test/package/consumer.ts b/web/test/package/consumer.ts index 880779762..82feb6017 100644 --- a/web/test/package/consumer.ts +++ b/web/test/package/consumer.ts @@ -7,10 +7,9 @@ import { createStovepipeWeb, StovepipePaths, type StovepipeService } from "@subm import { SubmitQueueApp, SubmitQueueShell } from "@submitqueue/web-submitqueue"; import { createSubmitQueueWeb, type SubmitQueueGateway } from "@submitqueue/web-submitqueue/server"; import { createFakeGatewayReader } from "@submitqueue/web-submitqueue/extension/gateway/mock"; -import { newHmacCursorCodec } from "@submitqueue/web-submitqueue/extension/cursor/hmac"; const gateway: SubmitQueueGateway = createFakeGatewayReader(); -const web = createSubmitQueueWeb({ gateway, cursors: newHmacCursorCodec("consumer-owned-secret") }); +const web = createSubmitQueueWeb({ gateway }); export async function render() { const result = await web.handle({ diff --git a/web/test/package/pack_contents_test.py b/web/test/package/pack_contents_test.py index 940b27156..f24967206 100644 --- a/web/test/package/pack_contents_test.py +++ b/web/test/package/pack_contents_test.py @@ -46,7 +46,6 @@ def _client_entry_violations(archive: tarfile.TarFile) -> list[str]: "package/view/styles.css", "package/dist/index.d.ts", "package/dist/server.d.ts", - "package/dist/extension/cursor/hmac/index.d.ts", "package/dist/extension/gateway/mock/index.d.ts", "package/README.md", } diff --git a/web/test/package/react18/react18-runtime.mjs b/web/test/package/react18/react18-runtime.mjs index 7e2a6991e..52c912146 100644 --- a/web/test/package/react18/react18-runtime.mjs +++ b/web/test/package/react18/react18-runtime.mjs @@ -4,7 +4,6 @@ import React, { createElement } from "react"; import { renderToString } from "react-dom/server"; import { SubmitQueueApp, SubmitQueueShell } from "@submitqueue/web-submitqueue"; import { createSubmitQueueWeb } from "@submitqueue/web-submitqueue/server"; -import { newHmacCursorCodec } from "@submitqueue/web-submitqueue/extension/cursor/hmac"; import { createFakeGatewayReader } from "@submitqueue/web-submitqueue/extension/gateway/mock"; import { StovepipeApp } from "@submitqueue/web-stovepipe"; @@ -14,7 +13,6 @@ assert.match(React.version, /^18\./u); const web = createSubmitQueueWeb({ gateway: createFakeGatewayReader(), - cursors: newHmacCursorCodec("react-18-consumer-secret"), }); const routes = [ { path: "/", search: {} },