From cb939a32cda19180ca62f39b4207b7a480871b8e Mon Sep 17 00:00:00 2001 From: Andy Ford Date: Wed, 11 Nov 2026 10:21:21 +0000 Subject: [PATCH 1/5] Send ably-pubsub-server in the Ably-Agent header To identify as a server SDK, the agent string needs to include `ably-pubsub-server`. We only sent `ably-pubsub-python`. Co-Authored-By: Claude Opus 5.5 Drop the README note on token auth and 40167 --- ably/pubsub/request/http.py | 6 +- ably/pubsub/request/httputils.py | 24 +++-- ably/pubsub/server/__init__.py | 9 +- ably/pubsub/server/sync.py | 3 +- ably/pubsub/transport/websockettransport.py | 2 +- ably/pubsub/types/options.py | 7 +- ably/pubsub/util/side.py | 34 ++++++++ test/ably/http/httphttp_test.py | 2 +- test/unit/server_agent_test.py | 97 +++++++++++++++++++++ 9 files changed, 170 insertions(+), 14 deletions(-) create mode 100644 ably/pubsub/util/side.py create mode 100644 test/unit/server_agent_test.py diff --git a/ably/pubsub/request/http.py b/ably/pubsub/request/http.py index a3627d72..7a68f743 100644 --- a/ably/pubsub/request/http.py +++ b/ably/pubsub/request/http.py @@ -178,9 +178,11 @@ async def make_request(self, method, path, version=None, headers=None, body=None body = self.dump_body(body) if body: - all_headers = HttpUtils.default_post_headers(self.options.use_binary_protocol, version=version) + all_headers = HttpUtils.default_post_headers(self.options.use_binary_protocol, version=version, + agents=self.options.agents) else: - all_headers = HttpUtils.default_get_headers(self.options.use_binary_protocol, version=version) + all_headers = HttpUtils.default_get_headers(self.options.use_binary_protocol, version=version, + agents=self.options.agents) params = HttpUtils.get_query_params(self.options) diff --git a/ably/pubsub/request/httputils.py b/ably/pubsub/request/httputils.py index f24a2f00..47a1e9fe 100644 --- a/ably/pubsub/request/httputils.py +++ b/ably/pubsub/request/httputils.py @@ -17,8 +17,8 @@ class HttpUtils: } @staticmethod - def default_get_headers(binary=False, version=None): - headers = HttpUtils.default_headers(version=version) + def default_get_headers(binary=False, version=None, agents=None): + headers = HttpUtils.default_headers(version=version, agents=agents) if binary: headers["Accept"] = HttpUtils.mime_types['binary'] else: @@ -26,8 +26,8 @@ def default_get_headers(binary=False, version=None): return headers @staticmethod - def default_post_headers(binary=False, version=None): - headers = HttpUtils.default_get_headers(binary=binary, version=version) + def default_post_headers(binary=False, version=None, agents=None): + headers = HttpUtils.default_get_headers(binary=binary, version=version, agents=agents) headers["Content-Type"] = headers["Accept"] return headers @@ -51,14 +51,26 @@ def get_host_header(host): } @staticmethod - def default_headers(version=None): + def default_headers(version=None, agents=None): if version is None: version = api_version return { "X-Ably-Version": version, - "Ably-Agent": f'ably-pubsub-python/{lib_version} python/{platform.python_version()}' + "Ably-Agent": HttpUtils.agent_string(agents), } + @staticmethod + def agent_string(agents=None): + """The `Ably-Agent` value: the library and platform, then each of `agents` (RSC7d6). + + An entry with no version is emitted bare: versionless identifiers are flags, such + as the side a package declares, and `name/None` is not a valid Ably-Agent token. + """ + agent = f'ably-pubsub-python/{lib_version} python/{platform.python_version()}' + for name, version in (agents or {}).items(): + agent += f' {name}/{version}' if version else f' {name}' + return agent + @staticmethod def get_query_params(options): params = {} diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index 2e46a5c0..6ba52623 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -71,6 +71,7 @@ IncompatibleClientIdException, InvalidArgumentException, ) +from ably.pubsub.util.side import SERVER_AGENT_IDENTIFIER, options_with_side_agent from ably.pubsub.vcdiff.defaultvcdiffdecoder import AblyVCDiffDecoder from ably.pubsub.version import api_version, lib_version @@ -184,6 +185,10 @@ def create_http_client(**kwargs) -> PubSubHttpClient: The largest total size of the messages one publish may carry, counting each message's name, data, client ID and extras. A larger publish raises an AblyException with code 40009 and is not sent. The default is 65536 (64 KiB). + agents: dict[str, str | None] + Additional entries for the `Ably-Agent` header, mapping a product name to its + version, or to None for a versionless entry. For an SDK built on this one to + identify itself. The client always adds `ably-pubsub-server` as well. Returns ------- @@ -204,7 +209,7 @@ def create_http_client(**kwargs) -> PubSubHttpClient: option's 3.x name, which the message names the replacement for. """ with factory_construction(): - return _DefaultPubSubHttpClient(**kwargs) + return _DefaultPubSubHttpClient(**options_with_side_agent(kwargs, SERVER_AGENT_IDENTIFIER)) def create_realtime_client(**kwargs) -> PubSubRealtimeClient: @@ -280,7 +285,7 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: ValueError. """ with factory_construction(): - return _DefaultPubSubRealtimeClient(**kwargs) + return _DefaultPubSubRealtimeClient(**options_with_side_agent(kwargs, SERVER_AGENT_IDENTIFIER)) __all__ = [ diff --git a/ably/pubsub/server/sync.py b/ably/pubsub/server/sync.py index 88bd7aaa..5698d806 100644 --- a/ably/pubsub/server/sync.py +++ b/ably/pubsub/server/sync.py @@ -56,6 +56,7 @@ IncompatibleClientIdException, InvalidArgumentException, ) +from ably.pubsub.sync.util.side import SERVER_AGENT_IDENTIFIER, options_with_side_agent from ably.pubsub.sync.vcdiff.defaultvcdiffdecoder import AblyVCDiffDecoder from ably.pubsub.version import api_version, lib_version @@ -82,7 +83,7 @@ def create_http_client(**kwargs) -> PubSubHttpClient: ValueError. """ with factory_construction(): - return _DefaultPubSubHttpClientSync(**kwargs) + return _DefaultPubSubHttpClientSync(**options_with_side_agent(kwargs, SERVER_AGENT_IDENTIFIER)) __all__ = [ diff --git a/ably/pubsub/transport/websockettransport.py b/ably/pubsub/transport/websockettransport.py index 75f55a44..d5d071a3 100644 --- a/ably/pubsub/transport/websockettransport.py +++ b/ably/pubsub/transport/websockettransport.py @@ -88,7 +88,7 @@ def __select_connect_func(options): return ws_connect def connect(self): - headers = HttpUtils.default_headers() + headers = HttpUtils.default_headers(agents=self.options.agents) query_params = urllib.parse.urlencode(self.params) scheme = 'wss' if self.options.tls else 'ws' port = Defaults.get_port(self.options) diff --git a/ably/pubsub/types/options.py b/ably/pubsub/types/options.py index cacf205f..662233f6 100644 --- a/ably/pubsub/types/options.py +++ b/ably/pubsub/types/options.py @@ -117,7 +117,7 @@ def __init__(self, client_id=None, tls=True, port=0, suspended_retry_timeout_seconds=None, connectivity_check_url=None, channel_retry_timeout_seconds=None, add_request_ids=False, vcdiff_decoder: VCDiffDecoder = None, transport_params=None, echo_messages=True, - max_message_size=None, _test_options=None, **kwargs): + max_message_size=None, agents=None, _test_options=None, **kwargs): _reject_renamed_timing_options(kwargs) super().__init__(**kwargs) @@ -210,6 +210,7 @@ def __init__(self, client_id=None, tls=True, port=0, self.__vcdiff_decoder = vcdiff_decoder self.__transport_params = transport_params or {} self.__max_message_size = max_message_size + self.__agents = dict(agents) if agents else {} self.__test_options = _test_options self.__hosts = self.__get_hosts() @@ -393,6 +394,10 @@ def transport_params(self): def max_message_size(self): return self.__max_message_size + @property + def agents(self): + return self.__agents + @property def _test_options(self): return self.__test_options diff --git a/ably/pubsub/util/side.py b/ably/pubsub/util/side.py new file mode 100644 index 00000000..c7ecc6f7 --- /dev/null +++ b/ably/pubsub/util/side.py @@ -0,0 +1,34 @@ +"""The agent entry a package stamps to declare which side it runs on. + +Nothing here may grow into a general abstraction over the client: it exists only to +stamp the side a package's factories declare. +""" + +# The `-server` suffix on this identifier is load-bearing, not cosmetic. On API-key auth +# the realtime system grants the server exemption from monthly-active-user counting by +# matching an agent entry ending in `-server`, and an identifier that is not yet in the +# ably-common registry is classified by that suffix alone. Renaming it without preserving +# its suffix silently reclassifies every client the package constructs as a device. +# +# It matches the identifier the other Pub/Sub SDKs' server packages send, and lives here +# rather than in `ably.pubsub.server` so the naming scheme can be changed in one place. +SERVER_AGENT_IDENTIFIER = 'ably-pubsub-server' + + +def options_with_side_agent(kwargs, identifier): + """A copy of the caller's client options carrying the agent entry that declares the side. + + Only `agents` is replaced, with a new dict rather than by mutation, so the caller's + options and their own `agents` dict are both left untouched. Every other option is + copied by reference. + + The caller's `agents` entries are kept alongside the side stamp, so an SDK layered on + top of this package keeps its attribution. The side stamp is applied last and so wins + a collision on its own identifier: which side the package declares is the package's to + state, not the caller's to redefine. + + The side entry is versionless — a flag — because the identifier is shared by every + Pub/Sub SDK, so a version could not say what it versions, and would only duplicate the + `ably-pubsub-python` entry sent alongside it. + """ + return {**kwargs, 'agents': {**(kwargs.get('agents') or {}), identifier: None}} diff --git a/test/ably/http/httphttp_test.py b/test/ably/http/httphttp_test.py index 5490c149..a41fc904 100644 --- a/test/ably/http/httphttp_test.py +++ b/test/ably/http/httphttp_test.py @@ -195,7 +195,7 @@ async def test_request_headers(self): # Agent assert 'Ably-Agent' in r.request.headers - expr = r"^ably-pubsub-python\/\d+.\d+.\d+(-beta\.\d+)? python\/\d.\d+.\d+$" + expr = r"^ably-pubsub-python\/\d+.\d+.\d+(-beta\.\d+)? python\/\d.\d+.\d+ ably-pubsub-server$" assert re.search(expr, r.request.headers['Ably-Agent']) await ably.close() diff --git a/test/unit/server_agent_test.py b/test/unit/server_agent_test.py new file mode 100644 index 00000000..c8530130 --- /dev/null +++ b/test/unit/server_agent_test.py @@ -0,0 +1,97 @@ +"""The server package declares its side in the Ably-Agent header. + +On API-key auth the realtime system exempts a connection from monthly-active-user +counting only when an agent entry ends in `-server`, so every request and connection +a server factory builds must carry `ably-pubsub-server`. +""" + +import httpx +import pytest + +import ably.pubsub.server.sync as server_sync +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.server import create_http_client +from ably.pubsub.sync.types.testoptions import TestOptions as SyncTestOptions +from ably.pubsub.types.testoptions import TestOptions +from test.uts.helpers.client import await_connection_state, close_open_clients, realtime_client +from test.uts.helpers.mock_http import MockHttpClient +from test.uts.helpers.mock_websocket import CONNECTED_MESSAGE, MockWebSocket + +KEY = 'app.key:secret' +TIME_RESPONSE = [1234567890000] + + +@pytest.fixture(autouse=True) +async def close_clients(): + yield + await close_open_clients() + + +def agent_entries(header): + return header.split(' ') + + +async def http_agent(**kwargs): + mock = MockHttpClient(on_connection_attempt=lambda conn: conn.respond_with_success()) + mock.queue_response(200, TIME_RESPONSE) + client = create_http_client(key=KEY, _test_options=TestOptions(http_transport=mock.as_transport()), **kwargs) + try: + await client.time() + finally: + await client.close() + return mock.captured_requests[0].headers['Ably-Agent'] + + +async def test_http_requests_declare_the_server_side(): + agents = agent_entries(await http_agent()) + assert agents[0].startswith('ably-pubsub-python/') + assert agents[-1] == 'ably-pubsub-server' + + +async def test_the_realtime_connection_declares_the_server_side(): + mock = MockWebSocket( + on_connection_attempt=lambda connection: connection.respond_with_success(CONNECTED_MESSAGE)) + client = realtime_client(mock, disconnected_retry_timeout_seconds=60) + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED) + + agents = agent_entries(mock.connection_attempts[0].headers['Ably-Agent']) + assert agents[0].startswith('ably-pubsub-python/') + assert agents[-1] == 'ably-pubsub-server' + + +def test_sync_http_requests_declare_the_server_side(): + requests = [] + + def handler(request): + requests.append(request) + return httpx.Response(200, json=TIME_RESPONSE) + + client = server_sync.create_http_client( + key=KEY, _test_options=SyncTestOptions(http_transport=httpx.MockTransport(handler))) + try: + client.time() + finally: + client.close() + + agents = agent_entries(requests[0].headers['Ably-Agent']) + assert agents[0].startswith('ably-pubsub-python/') + assert agents[-1] == 'ably-pubsub-server' + + +async def test_caller_agents_are_kept_alongside_the_server_side(): + caller_agents = {'my-sdk': '1.2.3', 'my-flag': None} + + agents = agent_entries(await http_agent(agents=caller_agents)) + + assert 'my-sdk/1.2.3' in agents + assert 'my-flag' in agents + assert agents[-1] == 'ably-pubsub-server' + assert caller_agents == {'my-sdk': '1.2.3', 'my-flag': None} + + +async def test_a_caller_cannot_redeclare_the_side(): + agents = agent_entries(await http_agent(agents={'ably-pubsub-server': '9.9.9'})) + + assert agents.count('ably-pubsub-server') == 1 + assert 'ably-pubsub-server/9.9.9' not in agents From 1f18dc2c8573c3d2621d4924f68b83de79fa4746 Mon Sep 17 00:00:00 2001 From: Andy Ford Date: Wed, 11 Nov 2026 11:09:02 +0000 Subject: [PATCH 2/5] Authenticate token tests with server-claim JWTs A server client declares itself a server, and under token auth Ably accepts that only from a signed x-ably-clientType JWT claim; native Ably tokens can't carry it and are rejected with 40167. Tests of general token behaviour now use JWTs carrying the claim. Tests whose subject is native tokens are skipped until Ably can accept them from a server. Co-Authored-By: Claude Opus 5.5 Port the two renewal tests that still used native tokens to JWTs, and say why the publish error tests call time() --- test/ably/http/httpauth_test.py | 21 +++- test/ably/http/httpchannelpublish_test.py | 32 +++-- test/ably/realtime/realtimeauth_test.py | 114 +++++------------- test/ably/realtime/realtimeconnection_test.py | 7 +- test/ably/testapp.py | 16 +++ test/ably/utils.py | 10 ++ test/uts/helpers/sandbox.py | 12 ++ .../integration/auth/token_request_test.py | 6 +- .../integration/proxy/rest_faults_test.py | 11 +- .../integration/proxy/rest_fallback_test.py | 18 ++- test/uts/rest/integration/publish_test.py | 13 +- 11 files changed, 147 insertions(+), 113 deletions(-) diff --git a/test/ably/http/httpauth_test.py b/test/ably/http/httpauth_test.py index 5857536b..c73bfbdf 100644 --- a/test/ably/http/httpauth_test.py +++ b/test/ably/http/httpauth_test.py @@ -14,7 +14,12 @@ from ably.pubsub.server import AblyAuthException, Auth, create_http_client from ably.pubsub.types.tokendetails import TokenDetails from test.ably.testapp import TestApp -from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol +from test.ably.utils import ( + NATIVE_TOKEN_AUTH_UNSUPPORTED, + BaseAsyncTestCase, + VaryByProtocolTestsMetaclass, + dont_vary_protocol, +) log = logging.getLogger(__name__) @@ -184,6 +189,7 @@ async def test_if_authorize_changes_auth_mechanism_to_token(self): # RSA10a @dont_vary_protocol + @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_authorize_always_creates_new_token(self): await self.ably.auth.authorize({'capability': {'test': ['publish']}}) await self.ably.channels.get('test').publish('event', 'data') @@ -218,6 +224,7 @@ async def test_authorize_adheres_to_request_token(self): for arg, value in auth_params.items(): assert auth_called[arg] == value, f"{arg} called with wrong value: {value}" + @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_with_token_str_https(self): token = await self.ably.auth.authorize() token = token.token @@ -226,6 +233,7 @@ async def test_with_token_str_https(self): await ably.channels.get('test_auth_with_token_str').publish('event', 'foo_bar') await ably.close() + @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_with_token_str_http(self): token = await self.ably.auth.authorize() token = token.token @@ -301,12 +309,18 @@ async def test_timestamp_is_not_stored(self): async def test_client_id_precedence(self): client_id = uuid.uuid4().hex overridden_client_id = uuid.uuid4().hex + + # The client's clientId is what the library passes to the callback, ahead of the default token params + async def auth_callback(token_params): + assert token_params['client_id'] == client_id + return await TestApp.get_jwt(**token_params) + ably = await TestApp.get_ably_rest( use_binary_protocol=self.use_binary_protocol, client_id=client_id, + auth_callback=auth_callback, default_token_params={'client_id': overridden_client_id}) - token = await ably.auth.authorize() - assert token.client_id == client_id + await ably.auth.authorize() assert ably.auth.client_id == client_id channel = ably.channels[ @@ -326,6 +340,7 @@ async def setup(self): def per_protocol_setup(self, use_binary_protocol): self.use_binary_protocol = use_binary_protocol + @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_with_key(self): ably = await TestApp.get_ably_rest(use_binary_protocol=self.use_binary_protocol) diff --git a/test/ably/http/httpchannelpublish_test.py b/test/ably/http/httpchannelpublish_test.py index 1edff7be..07225c66 100644 --- a/test/ably/http/httpchannelpublish_test.py +++ b/test/ably/http/httpchannelpublish_test.py @@ -31,7 +31,12 @@ async def setup(self): self.test_vars = await TestApp.get_test_vars() self.ably = await TestApp.get_ably_rest() self.client_id = uuid.uuid4().hex - self.ably_with_client_id = await TestApp.get_ably_rest(client_id=self.client_id, use_token_auth=True) + + async def auth_callback(token_params): + return await TestApp.get_jwt(client_id=self.client_id) + + self.ably_with_client_id = await TestApp.get_ably_rest(client_id=self.client_id, + auth_callback=auth_callback) yield await self.ably.close() await self.ably_with_client_id.close() @@ -118,9 +123,13 @@ async def test_message_list_generate_one_request(self): assert message['data'] == str(i) async def test_publish_error(self): - ably = await TestApp.get_ably_rest(use_binary_protocol=self.use_binary_protocol) - await ably.auth.authorize( - token_params={'capability': {"only_subscribe": ["subscribe"]}}) + async def auth_callback(token_params): + return await TestApp.get_jwt(capability={"only_subscribe": ["subscribe"]}) + + ably = await TestApp.get_ably_rest(use_binary_protocol=self.use_binary_protocol, + auth_callback=auth_callback) + # The protocol harness needs one successful response, and the publish below fails. + await ably.time() with pytest.raises(AblyException) as excinfo: await ably.channels["only_subscribe"].publish() @@ -235,7 +244,7 @@ async def test_token_is_bound_to_options_client_id_after_publish(self): # defined after publish assert isinstance(self.ably_with_client_id.auth.token_details, TokenDetails) - assert self.ably_with_client_id.auth.token_details.client_id == self.client_id + assert self.ably_with_client_id.auth.client_id == self.client_id assert self.ably_with_client_id.auth.auth_mechanism == Auth.Method.TOKEN history = await channel.history() assert history.items[0].client_id == self.client_id @@ -254,7 +263,7 @@ async def test_publish_message_without_client_id_on_identified_client(self): assert messages is not None, "Expected non-None messages" assert len(messages) == 1, "Expected 1 message" - assert post_mock.call_count == 2 + assert post_mock.call_count == 1 if self.use_binary_protocol: posted_body = msgpack.unpackb( @@ -295,10 +304,12 @@ async def test_publish_message_with_client_id_on_identified_client(self): await channel.publish(message) async def test_publish_message_with_wrong_client_id_on_implicit_identified_client(self): - new_token = await self.ably.auth.authorize(token_params={'client_id': uuid.uuid4().hex}) + token = await TestApp.get_jwt(client_id=uuid.uuid4().hex) new_ably = await TestApp.get_ably_rest(key=None, - token=new_token.token, + token=token, use_binary_protocol=self.use_binary_protocol) + # The protocol harness needs one successful response, and the publish below fails. + await new_ably.time() channel = new_ably.channels[ self.get_channel_name('persisted:wrong_client_id_implicit_client')] @@ -313,10 +324,11 @@ async def test_publish_message_with_wrong_client_id_on_implicit_identified_clien # RSA15b async def test_wildcard_client_id_can_publish_as_others(self): - wildcard_token_details = await self.ably.auth.request_token({'client_id': '*'}) + wildcard_token = await TestApp.get_jwt(client_id='*') wildcard_ably = await TestApp.get_ably_rest( key=None, - token_details=wildcard_token_details, + token=wildcard_token, + client_id='*', use_binary_protocol=self.use_binary_protocol) assert wildcard_ably.auth.client_id == '*' diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index 4e8bebba..147a27a3 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -1,8 +1,8 @@ import asyncio import json +import time import urllib.parse -import httpx import pytest from ably.pubsub.realtime.connection import ConnectionState @@ -12,7 +12,7 @@ from ably.pubsub.types.tokendetails import TokenDetails from ably.pubsub.util.exceptions import IncompatibleClientIdException from test.ably.testapp import TestApp -from test.ably.utils import BaseAsyncTestCase, random_string +from test.ably.utils import NATIVE_TOKEN_AUTH_UNSUPPORTED, BaseAsyncTestCase, random_string echo_url = 'https://echo.ably.io' @@ -51,6 +51,7 @@ async def test_auth_wrong_api_key(self): assert state_change.reason.status_code == 401 await ably.close() + @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_auth_with_token_string(self): rest = await TestApp.get_ably_rest() token_details = await rest.auth.request_token() @@ -69,6 +70,7 @@ async def test_auth_with_invalid_token_string(self): assert state_change.reason.status_code == 401 await ably.close() + @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_auth_with_token_details(self): rest = await TestApp.get_ably_rest() token_details = await rest.auth.request_token() @@ -87,6 +89,7 @@ async def test_auth_with_invalid_token_details(self): assert state_change.reason.status_code == 401 await ably.close() + @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_auth_with_auth_callback_with_token_request(self): rest = await TestApp.get_ably_rest() @@ -101,6 +104,7 @@ async def callback(params): assert ably.connection.error_reason is None await ably.close() + @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_auth_with_auth_callback_token_with_details(self): rest = await TestApp.get_ably_rest() @@ -116,11 +120,8 @@ async def callback(params): await ably.close() async def test_auth_with_auth_callback_with_token_string(self): - rest = await TestApp.get_ably_rest() - async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt(**params) ably = await TestApp.get_ably_realtime(auth_callback=callback) await asyncio.wait_for(ably.connection.once_async(ConnectionState.CONNECTED), timeout=5) @@ -139,6 +140,7 @@ async def callback(params): assert state_change.reason.status_code == 401 await ably.close() + @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_auth_with_auth_url_json(self): rest = await TestApp.get_ably_rest() token_details = await rest.auth.request_token() @@ -156,9 +158,8 @@ async def test_auth_with_auth_url_json(self): await ably.close() async def test_auth_with_auth_url_text_plain(self): - rest = await TestApp.get_ably_rest() - token_details = await rest.auth.request_token() - url_path = f"{echo_url}/?type=text&body={token_details.token}" + token = await TestApp.get_jwt() + url_path = f"{echo_url}/?type=text&body={token}" ably = await TestApp.get_ably_realtime(auth_url=url_path) await asyncio.wait_for(ably.connection.once_async(ConnectionState.CONNECTED), timeout=5) @@ -168,12 +169,11 @@ async def test_auth_with_auth_url_text_plain(self): await ably.close() async def test_auth_with_auth_url_post(self): - rest = await TestApp.get_ably_rest() - token_details = await rest.auth.request_token() + token = await TestApp.get_jwt() url_path = f"{echo_url}/?type=json&" ably = await TestApp.get_ably_realtime(auth_url=url_path, auth_method='POST', - auth_params=token_details) + auth_params={'token': token}) await asyncio.wait_for(ably.connection.once_async(ConnectionState.CONNECTED), timeout=5) response_time_ms = await ably.connection.ping() assert response_time_ms is not None @@ -181,11 +181,8 @@ async def test_auth_with_auth_url_post(self): await ably.close() async def test_reauth_while_connected(self): - rest = await TestApp.get_ably_rest() - async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt(**params) ably = await TestApp.get_ably_realtime(auth_callback=callback) await asyncio.wait_for(ably.connection.once_async(ConnectionState.CONNECTED), timeout=5) @@ -221,11 +218,8 @@ def on_update(state_change): await ably.close() async def test_reauth_while_connecting(self): - rest = await TestApp.get_ably_rest() - async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt(**params) ably = await TestApp.get_ably_realtime(auth_callback=callback) original_transport = await ably.connection.connection_manager.once_async('transport.pending') @@ -247,11 +241,8 @@ async def withhold_protocol_message(msg): await ably.close() async def test_reauth_immediately(self): - rest = await TestApp.get_ably_rest() - async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt(**params) ably = await TestApp.get_ably_realtime(auth_callback=callback) await ably.auth.authorize() @@ -260,12 +251,10 @@ async def callback(params): await ably.close() async def test_capability_change_without_loss_of_continuity(self): - rest = await TestApp.get_ably_rest() channel_name = random_string(5) async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt(**params) ably = await TestApp.get_ably_realtime(auth_callback=callback) @@ -280,12 +269,10 @@ async def callback(params): await ably.close() async def test_capability_downgrade(self): - rest = await TestApp.get_ably_rest() channel_name = random_string(5) async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt(**params) ably = await TestApp.get_ably_realtime(auth_callback=callback) @@ -312,11 +299,8 @@ def on_channel_state_change(state_change): await ably.close() async def test_reauth_inbound_auth_protocol_msg(self): - rest = await TestApp.get_ably_rest() - async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt(**params) ably = await TestApp.get_ably_realtime(auth_callback=callback) msg = { @@ -336,17 +320,8 @@ def on_update(state_change): # RSC8a4 async def test_jwt_reauth(self): - test_vars = await TestApp.get_test_vars() - key = test_vars["keys"][0] - key_name = key["key_name"] - key_secret = key["key_secret"] - async def auth_callback(_): - response = httpx.get( - echo_url + '/createJWT', - params={"keyName": key_name, "keySecret": key_secret, "expiresIn": 35} - ) - return response.text + return await TestApp.get_jwt(ttl=35 * 1000) ably = await TestApp.get_ably_realtime(auth_callback=auth_callback) @@ -359,11 +334,8 @@ async def auth_callback(_): # RTN14b async def test_renew_token_single_attempt(self): - rest = await TestApp.get_ably_rest() - async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt() ably = await TestApp.get_ably_realtime(auth_callback=callback) msg = { @@ -379,19 +351,15 @@ async def callback(params): await transport.on_protocol_message(msg) assert ably.auth.token_details is not original_token_details await ably.close() - await rest.close() # RTN14b async def test_renew_token_connection_attempt_fails(self): - rest = await TestApp.get_ably_rest() call_count = 0 async def callback(params): nonlocal call_count call_count += 1 - params = {"ttl": 1} - token_details = await rest.auth.request_token(token_params=params) - return token_details + return await TestApp.get_jwt(expires_at=time.time() - 60) ably = await TestApp.get_ably_realtime(auth_callback=callback) @@ -401,7 +369,6 @@ async def callback(params): assert ably.connection.error_reason.status_code == 401 await ably.close() - await rest.close() # RSA4a async def test_renew_token_no_renew_means_provided(self): @@ -497,11 +464,8 @@ async def test_auth_url_403_custom_error(self): # RTN15h2 async def test_renew_token_single_attempt_upon_disconnection(self): - rest = await TestApp.get_ably_rest() - async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt(**params) ably = await TestApp.get_ably_realtime(auth_callback=callback) msg = { @@ -518,14 +482,12 @@ async def callback(params): await ably.connection.connection_manager.transport.on_protocol_message(msg) assert ably.auth.token_details is not original_token_details await ably.close() - await rest.close() # RTN15h1 async def test_renew_token_no_renew_means_provided_upon_disconnection(self): - rest = await TestApp.get_ably_rest() - token_details = await rest.auth.request_token() + token = await TestApp.get_jwt() - ably = await TestApp.get_ably_realtime(token_details=token_details) + ably = await TestApp.get_ably_realtime(token=token) state_change = await asyncio.wait_for(ably.connection.once_async(ConnectionState.CONNECTED), timeout=5) msg = { @@ -542,14 +504,10 @@ async def test_renew_token_no_renew_means_provided_upon_disconnection(self): assert state_change.reason.code == 40171 assert state_change.reason.status_code == 403 await ably.close() - await rest.close() async def test_renew_token_single_attempt_on_resume(self): - rest = await TestApp.get_ably_rest() - async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt(**params) ably = await TestApp.get_ably_realtime(auth_callback=callback) msg = { @@ -572,13 +530,11 @@ async def callback(params): await transport.on_protocol_message(msg) assert ably.auth.token_details is not original_token_details await ably.close() - await rest.close() async def test_renew_token_no_renew_means_provided_on_resume(self): - rest = await TestApp.get_ably_rest() - token_details = await rest.auth.request_token() + token = await TestApp.get_jwt() - ably = await TestApp.get_ably_realtime(token_details=token_details) + ably = await TestApp.get_ably_realtime(token=token) msg = { "action": ProtocolMessageAction.DISCONNECTED, @@ -603,16 +559,14 @@ async def test_renew_token_no_renew_means_provided_on_resume(self): assert state_change.reason.code == 40171 assert state_change.reason.status_code == 403 await ably.close() - await rest.close() # Request a token using client_id, then initialize a connection without one, # and check that the connection inherits the client_id from the token_details async def test_auth_client_id_inheritance_auth_callback(self): - rest = await TestApp.get_ably_rest() client_id = 'test_client_id' async def auth_callback(_): - return await rest.auth.request_token({"client_id": client_id}) + return await TestApp.get_jwt(client_id=client_id) realtime = await TestApp.get_ably_realtime(auth_callback=auth_callback) @@ -624,7 +578,6 @@ async def auth_callback(_): assert realtime.auth.client_id == client_id await realtime.close() - await rest.close() # Rest token generation with client_id, then constructing a client with a # different client_id, should fail with a library-generated error before @@ -645,12 +598,11 @@ async def test_auth_client_id_mismatch(self): # Rest token generation with clientId '*', then connecting with just the # token string and a different clientId, should succeed (RSA15b) async def test_auth_client_id_wildcard_token(self): - rest = await TestApp.get_ably_rest() client_id = 'test_client_id' - token_details = await rest.auth.request_token({"client_id": "*"}) + token = await TestApp.get_jwt(client_id="*") - realtime = await TestApp.get_ably_realtime(token_details=token_details, client_id=client_id) + realtime = await TestApp.get_ably_realtime(token=token, client_id=client_id) # RTC17: the configured clientId holds before CONNECTED assert realtime.auth.client_id == client_id @@ -660,15 +612,14 @@ async def test_auth_client_id_wildcard_token(self): assert realtime.auth.client_id == client_id await realtime.close() - await rest.close() # Request a token using clientId, then initialize a connection using just the token string, # and check that the connection inherits the clientId from the connectionDetails async def test_auth_client_id_inheritance_token(self): - rest = await TestApp.get_ably_rest() client_id = 'test_client_id' - token_details = await rest.auth.request_token({"client_id": client_id}) + token = await TestApp.get_jwt(client_id=client_id) + token_details = TokenDetails(token=token, client_id=client_id) realtime = await TestApp.get_ably_realtime(token_details=token_details) @@ -680,4 +631,3 @@ async def test_auth_client_id_inheritance_token(self): assert realtime.auth.client_id == client_id await realtime.close() - await rest.close() diff --git a/test/ably/realtime/realtimeconnection_test.py b/test/ably/realtime/realtimeconnection_test.py index 46261815..fbea8559 100644 --- a/test/ably/realtime/realtimeconnection_test.py +++ b/test/ably/realtime/realtimeconnection_test.py @@ -523,18 +523,15 @@ async def test_fallback_host_disconnected_protocol_msg(self): # RTN2d async def test_connection_null_client_id_query_params(self): - rest = await TestApp.get_ably_rest() + token = await TestApp.get_jwt() - token_details = await rest.auth.request_token() - - realtime = await TestApp.get_ably_realtime(token_details=token_details) + realtime = await TestApp.get_ably_realtime(token=token) await realtime.connection.once_async(ConnectionState.CONNECTED) assert realtime.connection.connection_manager.transport.params.get("client_id") is None assert realtime.auth.client_id is None await realtime.close() - await rest.close() async def test_connection_client_id_query_params(self): client_id = 'test_client_id' diff --git a/test/ably/testapp.py b/test/ably/testapp.py index 2a4c1efb..e952b09a 100644 --- a/test/ably/testapp.py +++ b/test/ably/testapp.py @@ -11,6 +11,7 @@ from ably.pubsub.types.capability import Capability from ably.pubsub.types.options import Options from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.sandbox import generate_jwt log = logging.getLogger(__name__) @@ -66,6 +67,21 @@ async def get_test_vars(): return TestApp.__test_vars + @staticmethod + async def get_jwt(client_id=None, capability=None, ttl=3600000, expires_at=None): + """An Ably JWT signed with the full-access key, in place of a native token. + + `capability` may be a dict, as token params give it. Use this where a test needs a token + string or an auth_callback result; the server takes its server declaration only from the + signed claim this carries. + """ + test_vars = await TestApp.get_test_vars() + key = test_vars["keys"][0] + if capability is not None and not isinstance(capability, str): + capability = str(Capability(capability)) + return generate_jwt(key["key_name"], key["key_secret"], ttl=ttl, + client_id=client_id, capability=capability, expires_at=expires_at) + @staticmethod async def get_ably_rest(**kw): test_vars = await TestApp.get_test_vars() diff --git a/test/ably/utils.py b/test/ably/utils.py index 6715ced7..6e045a13 100644 --- a/test/ably/utils.py +++ b/test/ably/utils.py @@ -13,6 +13,14 @@ from ably.pubsub.request.http import Http +# Skip reason for tests whose subject is a native Ably token. A server client cannot authenticate +# with one: Ably accepts the server declaration under token auth only from a signed x-ably-clientType +# JWT claim (40167). +NATIVE_TOKEN_AUTH_UNSUPPORTED = ( + "A server client cannot authenticate with a native Ably token: Ably accepts the server declaration " + "under token auth only from a signed x-ably-clientType JWT claim (40167)" +) + class BaseTestCase: @@ -150,6 +158,8 @@ async def wrapper(self): await old_func(self) wrapper.__name__ = old_name + '_' + ttype + # carry marks such as pytest.mark.skip over to the protocol-specific copy + wrapper.pytestmark = getattr(old_func, 'pytestmark', []) return wrapper diff --git a/test/uts/helpers/sandbox.py b/test/uts/helpers/sandbox.py index a29236c3..0ba6c92f 100644 --- a/test/uts/helpers/sandbox.py +++ b/test/uts/helpers/sandbox.py @@ -61,6 +61,15 @@ CIPHER_FIXTURE = __app_setup['cipher'] +# Skip reason for specifications whose subject is a native Ably token. A server client cannot +# authenticate with one: Ably accepts the server declaration under token auth only from a signed +# x-ably-clientType JWT claim (40167). +NATIVE_TOKEN_AUTH_UNSUPPORTED = ( + "A server client cannot authenticate with a native Ably token: Ably accepts the server declaration " + "under token auth only from a signed x-ably-clientType JWT claim (40167)" +) + + def random_id(length=6): """A short unique suffix, for the names the specifications build. @@ -205,6 +214,9 @@ def generate_jwt(key_name, key_secret, ttl=3600000, client_id=None, capability=N 'iat': issued_at, 'exp': issued_at + seconds, 'x-ably-capability': capability if capability is not None else '{"*":["*"]}', + # The clients are ably-pubsub-server clients, which declare the server side, + # and on token auth Ably accepts that only from this signed claim (40167). + 'x-ably-clientType': 'server', } if client_id is not None: claims['x-ably-clientId'] = client_id diff --git a/test/uts/realtime/integration/auth/token_request_test.py b/test/uts/realtime/integration/auth/token_request_test.py index 9729b8c0..05632aed 100644 --- a/test/uts/realtime/integration/auth/token_request_test.py +++ b/test/uts/realtime/integration/auth/token_request_test.py @@ -15,9 +15,11 @@ case, so `TokenParams(clientId: x)` is `{'client_id': x}`. """ +import pytest + from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, sandbox_rest_client -from test.uts.helpers.sandbox import random_id +from test.uts.helpers.sandbox import NATIVE_TOKEN_AUTH_UNSUPPORTED, random_id # The wait the specification gives each connection. CONNECT_TIMEOUT = 15.0 @@ -29,6 +31,7 @@ def connection_id(client): # UTS: realtime/integration/RSA9a/token-request-server-accepted-0 +@pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_rsa9a_token_request_server_accepted(realtime_sandbox): creator = sandbox_rest_client(realtime_sandbox.key_str) @@ -46,6 +49,7 @@ async def auth_callback(params): # UTS: realtime/integration/RSA9/token-request-with-clientid-0 +@pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_rsa9_token_request_with_clientid(realtime_sandbox): test_client_id = 'token-request-client-' + random_id() diff --git a/test/uts/realtime/integration/proxy/rest_faults_test.py b/test/uts/realtime/integration/proxy/rest_faults_test.py index dcbf0f38..dee19be3 100644 --- a/test/uts/realtime/integration/proxy/rest_faults_test.py +++ b/test/uts/realtime/integration/proxy/rest_faults_test.py @@ -76,9 +76,14 @@ async def auth_callback(params): return auth_callback -def jwt_auth_callback(api_key): - """The specification's JWT `authCallback`, which makes no request of its own.""" +def jwt_auth_callback(api_key, invocations=None): + """The specification's JWT `authCallback`, which makes no request of its own. + + `invocations` is appended to once per call where a test counts renewals. + """ async def auth_callback(params): + if invocations is not None: + invocations.append(params) return generate_jwt(extract_key_name(api_key), extract_key_secret(api_key)) return auth_callback @@ -111,7 +116,7 @@ async def test_rsc10_token_renewal_on_401(realtime_sandbox, proxy_session): }]) client = sandbox_rest_client( - auth_callback=token_auth_callback(realtime_sandbox.key_str, auth_callback_invocations), + auth_callback=jwt_auth_callback(realtime_sandbox.key_str, auth_callback_invocations), endpoint='localhost', port=session.proxy_port, tls=False, diff --git a/test/uts/rest/integration/proxy/rest_fallback_test.py b/test/uts/rest/integration/proxy/rest_fallback_test.py index 1e684c94..2e5488bd 100644 --- a/test/uts/rest/integration/proxy/rest_fallback_test.py +++ b/test/uts/rest/integration/proxy/rest_fallback_test.py @@ -22,7 +22,13 @@ from ably.pubsub.server import create_http_client from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import sandbox_rest_client, wall_clock_poll_until -from test.uts.helpers.sandbox import SANDBOX_ENDPOINT, random_id +from test.uts.helpers.sandbox import ( + SANDBOX_ENDPOINT, + extract_key_name, + extract_key_secret, + generate_jwt, + random_id, +) # The specification's `non_listening_port`: a port nothing is bound to, so that # connecting to it is refused rather than answered. @@ -32,6 +38,14 @@ HTTP_REQUEST_TIMEOUT_SECONDS = 3 +def jwt_auth_callback(api_key): + """An authCallback that signs an Ably JWT locally, so it makes no request the proxy could see.""" + async def auth_callback(params): + return generate_jwt(extract_key_name(api_key), extract_key_secret(api_key)) + + return auth_callback + + def token_auth_callback(api_key): """The specification's `token_auth_callback(api_key)`. @@ -321,7 +335,7 @@ async def test_rsl1k4_idempotent_retry_dedup(sandbox, proxy_session): }]) client = sandbox_rest_client( - auth_callback=token_auth_callback(sandbox.key_str), + auth_callback=jwt_auth_callback(sandbox.key_str), endpoint='localhost', fallback_hosts=['localhost'], port=session.proxy_port, diff --git a/test/uts/rest/integration/publish_test.py b/test/uts/rest/integration/publish_test.py index 30b817bc..63db0f42 100644 --- a/test/uts/rest/integration/publish_test.py +++ b/test/uts/rest/integration/publish_test.py @@ -8,7 +8,7 @@ from ably.pubsub.types.message import Message from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import sandbox_rest_client, wall_clock_poll_until -from test.uts.helpers.sandbox import random_id +from test.uts.helpers.sandbox import extract_key_name, extract_key_secret, generate_jwt, random_id # UTS: rest/integration/RSL1d/publish-failure-error-0 @@ -107,11 +107,10 @@ async def test_rsl1l1_publish_params_force_nack(sandbox, use_binary_protocol): # UTS: rest/integration/RSL1m4/clientid-mismatch-rejected-0 async def test_rsl1m4_clientid_mismatch_rejected(sandbox, use_binary_protocol): - key_client = sandbox_rest_client(sandbox.key_str, use_binary_protocol=use_binary_protocol) - - # request_token takes its token params as a positional dict; every keyword on it is an - # auth option. - token_details = await key_client.auth.request_token({'client_id': 'authenticated-client-id'}) + # A JWT carrying the clientId stands in for the native token the spec requests with + # request_token, which a server client cannot authenticate with. + token = generate_jwt(extract_key_name(sandbox.key_str), extract_key_secret(sandbox.key_str), + client_id='authenticated-client-id') # The spec passes the bare token string, not the TokenDetails, so the client does not # know the clientId the token carries and cannot rule the publish out locally. The @@ -119,7 +118,7 @@ async def test_rsl1m4_clientid_mismatch_rejected(sandbox, use_binary_protocol): # `token_details=` instead would have the library reject it in # `Channel.__publish_request_body` with the same 400/40012 before any request left. token_client = sandbox_rest_client( - token=token_details.token, use_binary_protocol=use_binary_protocol) + token=token, use_binary_protocol=use_binary_protocol) channel_name = 'clientid-mismatch-' + random_id() channel = token_client.channels.get(channel_name) From af7c2c6308fc311f04d779496f4e49bc5c5e12a2 Mon Sep 17 00:00:00 2001 From: Andy Ford Date: Wed, 11 Nov 2026 11:57:20 +0000 Subject: [PATCH 3/5] squash! Send ably-pubsub-server in the Ably-Agent header Validate the agents option, keep Options.agents from altering the header, and correct the README's TLS-off auth advice --- README.md | 2 +- ably/pubsub/request/http.py | 4 ++-- ably/pubsub/transport/websockettransport.py | 2 +- ably/pubsub/types/options.py | 15 ++++++++++++++- ably/pubsub/util/side.py | 5 ++++- test/unit/server_agent_test.py | 14 +++++++++++++- 6 files changed, 35 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index f3bbaea1..94911bc9 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ realtime = create_realtime_client(key=ably_api_key, endpoint='nonprod:sandbox') - Never put a port in `endpoint`: `localhost:8080` is not a hostname and does not resolve. - Set the port with `tls_port`, or with `port` when TLS is off. -- With TLS off (`tls=False`), use token authentication. +- With TLS off (`tls=False`), authenticate with a JWT carrying the `x-ably-clientType=server` claim: an API key is refused without TLS. See [UPDATING.md](https://github.com/ably/ably-pubsub-python/blob/main/UPDATING.md#deprecated-and-duplicate-options-are-removed) for how the old options map to `endpoint`. diff --git a/ably/pubsub/request/http.py b/ably/pubsub/request/http.py index 7a68f743..1141caef 100644 --- a/ably/pubsub/request/http.py +++ b/ably/pubsub/request/http.py @@ -179,10 +179,10 @@ async def make_request(self, method, path, version=None, headers=None, body=None if body: all_headers = HttpUtils.default_post_headers(self.options.use_binary_protocol, version=version, - agents=self.options.agents) + agents=self.options._agents) else: all_headers = HttpUtils.default_get_headers(self.options.use_binary_protocol, version=version, - agents=self.options.agents) + agents=self.options._agents) params = HttpUtils.get_query_params(self.options) diff --git a/ably/pubsub/transport/websockettransport.py b/ably/pubsub/transport/websockettransport.py index d5d071a3..c64bdc48 100644 --- a/ably/pubsub/transport/websockettransport.py +++ b/ably/pubsub/transport/websockettransport.py @@ -88,7 +88,7 @@ def __select_connect_func(options): return ws_connect def connect(self): - headers = HttpUtils.default_headers(agents=self.options.agents) + headers = HttpUtils.default_headers(agents=self.options._agents) query_params = urllib.parse.urlencode(self.params) scheme = 'wss' if self.options.tls else 'ws' port = Defaults.get_port(self.options) diff --git a/ably/pubsub/types/options.py b/ably/pubsub/types/options.py index 662233f6..f27f778e 100644 --- a/ably/pubsub/types/options.py +++ b/ably/pubsub/types/options.py @@ -210,7 +210,15 @@ def __init__(self, client_id=None, tls=True, port=0, self.__vcdiff_decoder = vcdiff_decoder self.__transport_params = transport_params or {} self.__max_message_size = max_message_size - self.__agents = dict(agents) if agents else {} + if agents is not None and not isinstance(agents, dict): + raise InvalidArgumentException( + f'agents must be a dict of product name to version or None, not {type(agents).__name__}', + 400, 40000) + for name, version in (agents or {}).items(): + if not name or any(c in f'{name}{version or ""}' for c in ' /'): + raise InvalidArgumentException( + f'agents entry {name!r}: {version!r} must not contain spaces or "/"', 400, 40000) + self.__agents = dict(agents or {}) self.__test_options = _test_options self.__hosts = self.__get_hosts() @@ -396,6 +404,11 @@ def max_message_size(self): @property def agents(self): + return dict(self.__agents) + + @property + def _agents(self): + """The agents for building the Ably-Agent header, without a defensive copy per request.""" return self.__agents @property diff --git a/ably/pubsub/util/side.py b/ably/pubsub/util/side.py index c7ecc6f7..f657d570 100644 --- a/ably/pubsub/util/side.py +++ b/ably/pubsub/util/side.py @@ -31,4 +31,7 @@ def options_with_side_agent(kwargs, identifier): Pub/Sub SDK, so a version could not say what it versions, and would only duplicate the `ably-pubsub-python` entry sent alongside it. """ - return {**kwargs, 'agents': {**(kwargs.get('agents') or {}), identifier: None}} + agents = kwargs.get('agents') + if agents is not None and not isinstance(agents, dict): + return kwargs # left for Options to reject with a message naming the option + return {**kwargs, 'agents': {**(agents or {}), identifier: None}} diff --git a/test/unit/server_agent_test.py b/test/unit/server_agent_test.py index c8530130..bcf765d2 100644 --- a/test/unit/server_agent_test.py +++ b/test/unit/server_agent_test.py @@ -10,7 +10,7 @@ import ably.pubsub.server.sync as server_sync from ably.pubsub.realtime.connection import ConnectionState -from ably.pubsub.server import create_http_client +from ably.pubsub.server import InvalidArgumentException, create_http_client from ably.pubsub.sync.types.testoptions import TestOptions as SyncTestOptions from ably.pubsub.types.testoptions import TestOptions from test.uts.helpers.client import await_connection_state, close_open_clients, realtime_client @@ -95,3 +95,15 @@ async def test_a_caller_cannot_redeclare_the_side(): assert agents.count('ably-pubsub-server') == 1 assert 'ably-pubsub-server/9.9.9' not in agents + + +@pytest.mark.parametrize('agents', [['my-sdk'], {'my sdk': '1.0'}, {'my-sdk': '1/0'}, {'': None}]) +def test_malformed_agents_are_rejected(agents): + with pytest.raises(InvalidArgumentException): + create_http_client(key=KEY, agents=agents) + + +def test_the_agents_property_cannot_change_the_header(): + client = create_http_client(key=KEY) + client.options.agents.pop('ably-pubsub-server') + assert 'ably-pubsub-server' in client.options.agents From e169306e7e6bcc54ee432de613e05abf9eaa5a70 Mon Sep 17 00:00:00 2001 From: Andy Ford Date: Wed, 11 Nov 2026 11:57:20 +0000 Subject: [PATCH 4/5] squash! Authenticate token tests with server-claim JWTs Check token clientIds from the JWT claims, count failed requests in the protocol harness instead of calling time(), and share the native-token skip reason --- test/ably/http/httpauth_test.py | 6 ++-- test/ably/http/httpchannelpublish_test.py | 12 +++----- test/ably/realtime/realtimeauth_test.py | 18 ++++++------ test/ably/testapp.py | 6 ++++ test/ably/utils.py | 35 ++++++++++++++--------- test/uts/helpers/sandbox.py | 6 ++++ 6 files changed, 50 insertions(+), 33 deletions(-) diff --git a/test/ably/http/httpauth_test.py b/test/ably/http/httpauth_test.py index c73bfbdf..8fa4dc0c 100644 --- a/test/ably/http/httpauth_test.py +++ b/test/ably/http/httpauth_test.py @@ -20,6 +20,7 @@ VaryByProtocolTestsMetaclass, dont_vary_protocol, ) +from test.uts.helpers.sandbox import jwt_claims log = logging.getLogger(__name__) @@ -313,14 +314,15 @@ async def test_client_id_precedence(self): # The client's clientId is what the library passes to the callback, ahead of the default token params async def auth_callback(token_params): assert token_params['client_id'] == client_id - return await TestApp.get_jwt(**token_params) + return await TestApp.get_jwt_for(token_params) ably = await TestApp.get_ably_rest( use_binary_protocol=self.use_binary_protocol, client_id=client_id, auth_callback=auth_callback, default_token_params={'client_id': overridden_client_id}) - await ably.auth.authorize() + token = await ably.auth.authorize() + assert jwt_claims(token.token)['x-ably-clientId'] == client_id assert ably.auth.client_id == client_id channel = ably.channels[ diff --git a/test/ably/http/httpchannelpublish_test.py b/test/ably/http/httpchannelpublish_test.py index 07225c66..eb26e298 100644 --- a/test/ably/http/httpchannelpublish_test.py +++ b/test/ably/http/httpchannelpublish_test.py @@ -18,6 +18,7 @@ from test.ably import utils from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, assert_waiter, dont_vary_protocol +from test.uts.helpers.sandbox import jwt_claims log = logging.getLogger(__name__) @@ -128,8 +129,6 @@ async def auth_callback(token_params): ably = await TestApp.get_ably_rest(use_binary_protocol=self.use_binary_protocol, auth_callback=auth_callback) - # The protocol harness needs one successful response, and the publish below fails. - await ably.time() with pytest.raises(AblyException) as excinfo: await ably.channels["only_subscribe"].publish() @@ -243,8 +242,9 @@ async def test_token_is_bound_to_options_client_id_after_publish(self): await channel.publish(name='publish', data='test') # defined after publish - assert isinstance(self.ably_with_client_id.auth.token_details, TokenDetails) - assert self.ably_with_client_id.auth.client_id == self.client_id + token_details = self.ably_with_client_id.auth.token_details + assert isinstance(token_details, TokenDetails) + assert jwt_claims(token_details.token)['x-ably-clientId'] == self.client_id assert self.ably_with_client_id.auth.auth_mechanism == Auth.Method.TOKEN history = await channel.history() assert history.items[0].client_id == self.client_id @@ -308,8 +308,6 @@ async def test_publish_message_with_wrong_client_id_on_implicit_identified_clien new_ably = await TestApp.get_ably_rest(key=None, token=token, use_binary_protocol=self.use_binary_protocol) - # The protocol harness needs one successful response, and the publish below fails. - await new_ably.time() channel = new_ably.channels[ self.get_channel_name('persisted:wrong_client_id_implicit_client')] @@ -328,10 +326,8 @@ async def test_wildcard_client_id_can_publish_as_others(self): wildcard_ably = await TestApp.get_ably_rest( key=None, token=wildcard_token, - client_id='*', use_binary_protocol=self.use_binary_protocol) - assert wildcard_ably.auth.client_id == '*' channel = wildcard_ably.channels[ self.get_channel_name('persisted:wildcard_client_id')] await channel.publish(name='publish1', data='no client_id') diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index 147a27a3..32a2f375 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -121,7 +121,7 @@ async def callback(params): async def test_auth_with_auth_callback_with_token_string(self): async def callback(params): - return await TestApp.get_jwt(**params) + return await TestApp.get_jwt_for(params) ably = await TestApp.get_ably_realtime(auth_callback=callback) await asyncio.wait_for(ably.connection.once_async(ConnectionState.CONNECTED), timeout=5) @@ -182,7 +182,7 @@ async def test_auth_with_auth_url_post(self): async def test_reauth_while_connected(self): async def callback(params): - return await TestApp.get_jwt(**params) + return await TestApp.get_jwt_for(params) ably = await TestApp.get_ably_realtime(auth_callback=callback) await asyncio.wait_for(ably.connection.once_async(ConnectionState.CONNECTED), timeout=5) @@ -219,7 +219,7 @@ def on_update(state_change): async def test_reauth_while_connecting(self): async def callback(params): - return await TestApp.get_jwt(**params) + return await TestApp.get_jwt_for(params) ably = await TestApp.get_ably_realtime(auth_callback=callback) original_transport = await ably.connection.connection_manager.once_async('transport.pending') @@ -242,7 +242,7 @@ async def withhold_protocol_message(msg): async def test_reauth_immediately(self): async def callback(params): - return await TestApp.get_jwt(**params) + return await TestApp.get_jwt_for(params) ably = await TestApp.get_ably_realtime(auth_callback=callback) await ably.auth.authorize() @@ -254,7 +254,7 @@ async def test_capability_change_without_loss_of_continuity(self): channel_name = random_string(5) async def callback(params): - return await TestApp.get_jwt(**params) + return await TestApp.get_jwt_for(params) ably = await TestApp.get_ably_realtime(auth_callback=callback) @@ -272,7 +272,7 @@ async def test_capability_downgrade(self): channel_name = random_string(5) async def callback(params): - return await TestApp.get_jwt(**params) + return await TestApp.get_jwt_for(params) ably = await TestApp.get_ably_realtime(auth_callback=callback) @@ -300,7 +300,7 @@ def on_channel_state_change(state_change): async def test_reauth_inbound_auth_protocol_msg(self): async def callback(params): - return await TestApp.get_jwt(**params) + return await TestApp.get_jwt_for(params) ably = await TestApp.get_ably_realtime(auth_callback=callback) msg = { @@ -465,7 +465,7 @@ async def test_auth_url_403_custom_error(self): # RTN15h2 async def test_renew_token_single_attempt_upon_disconnection(self): async def callback(params): - return await TestApp.get_jwt(**params) + return await TestApp.get_jwt_for(params) ably = await TestApp.get_ably_realtime(auth_callback=callback) msg = { @@ -507,7 +507,7 @@ async def test_renew_token_no_renew_means_provided_upon_disconnection(self): async def test_renew_token_single_attempt_on_resume(self): async def callback(params): - return await TestApp.get_jwt(**params) + return await TestApp.get_jwt_for(params) ably = await TestApp.get_ably_realtime(auth_callback=callback) msg = { diff --git a/test/ably/testapp.py b/test/ably/testapp.py index e952b09a..c9203d9f 100644 --- a/test/ably/testapp.py +++ b/test/ably/testapp.py @@ -82,6 +82,12 @@ async def get_jwt(client_id=None, capability=None, ttl=3600000, expires_at=None) return generate_jwt(key["key_name"], key["key_secret"], ttl=ttl, client_id=client_id, capability=capability, expires_at=expires_at) + @staticmethod + async def get_jwt_for(token_params): + """A JWT for an auth_callback, carrying the token params a JWT can express.""" + return await TestApp.get_jwt(client_id=token_params.get('client_id'), + capability=token_params.get('capability')) + @staticmethod async def get_ably_rest(**kw): test_vars = await TestApp.get_test_vars() diff --git a/test/ably/utils.py b/test/ably/utils.py index 6e045a13..83335974 100644 --- a/test/ably/utils.py +++ b/test/ably/utils.py @@ -12,14 +12,8 @@ from httpx import Response from ably.pubsub.request.http import Http - -# Skip reason for tests whose subject is a native Ably token. A server client cannot authenticate -# with one: Ably accepts the server declaration under token auth only from a signed x-ably-clientType -# JWT claim (40167). -NATIVE_TOKEN_AUTH_UNSUPPORTED = ( - "A server client cannot authenticate with a native Ably token: Ably accepts the server declaration " - "under token auth only from a signed x-ably-clientType JWT claim (40167)" -) +from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.sandbox import NATIVE_TOKEN_AUTH_UNSUPPORTED # noqa: F401 - re-exported for test/ably class BaseTestCase: @@ -73,6 +67,7 @@ def test_something(self): supports json and msgpack """ responses = [] + error_responses = [] def patch(): original = Http.make_request @@ -82,12 +77,24 @@ async def fake_make_request(self, *args, **kwargs): responses.append(response) return response - patcher = mock.patch.object(Http, 'make_request', fake_make_request) - patcher.start() - return patcher + original_raise = AblyException.raise_for_response + + # A request that fails raises before make_request returns. It still counts as a + # request made, but its error body isn't held to the protocol. + def fake_raise_for_response(response): + if not 200 <= response.status_code < 300: + error_responses.append(response) + original_raise(response) + + patchers = [mock.patch.object(Http, 'make_request', fake_make_request), + mock.patch.object(AblyException, 'raise_for_response', staticmethod(fake_raise_for_response))] + for patcher in patchers: + patcher.start() + return patchers - def unpatch(patcher): - patcher.stop() + def unpatch(patchers): + for patcher in patchers: + patcher.stop() def test_decorator(fn): @functools.wraps(fn) @@ -100,7 +107,7 @@ async def test_decorated(self, *args, **kwargs): # left as it was found even when an assertion or a transport error escapes. unpatch(patcher) - assert len(responses) >= 1, \ + assert len(responses) + len(error_responses) >= 1, \ "If your test doesn't make any requests, use the @dont_vary_protocol decorator" for response in responses: diff --git a/test/uts/helpers/sandbox.py b/test/uts/helpers/sandbox.py index 0ba6c92f..1631dfd7 100644 --- a/test/uts/helpers/sandbox.py +++ b/test/uts/helpers/sandbox.py @@ -226,6 +226,12 @@ def generate_jwt(key_name, key_secret, ttl=3600000, client_id=None, capability=N return b'.'.join((signing_input, __base64url(signature))).decode('ascii') +def jwt_claims(token): + """The claims of an Ably JWT, for a test to check what a token it was handed carries.""" + payload = token.split('.')[1] + return json.loads(base64.urlsafe_b64decode(payload + '=' * (-len(payload) % 4))) + + def __jwt_segment(part): return __base64url(json.dumps(part, separators=(',', ':')).encode('utf-8')) From 3fe963c8a603e45f65cccaf1f983957082d03dfa Mon Sep 17 00:00:00 2001 From: Andy Ford Date: Wed, 11 Nov 2026 13:41:29 +0000 Subject: [PATCH 5/5] Run lint on ubuntu-22.04, which still has Python 3.9 ubuntu-latest now resolves to 26.04, where setup-python has no 3.9 build, so the lint job fails before running. check.yml already pins ubuntu-22.04 for the same reason. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b3a5a113..01098c6c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,7 +12,7 @@ jobs: lint: permissions: contents: read - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: