diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b3a5a113..01098c6c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,7 +12,7 @@ jobs: lint: permissions: contents: read - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: diff --git a/README.md b/README.md index f3bbaea1..94911bc9 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ realtime = create_realtime_client(key=ably_api_key, endpoint='nonprod:sandbox') - Never put a port in `endpoint`: `localhost:8080` is not a hostname and does not resolve. - Set the port with `tls_port`, or with `port` when TLS is off. -- With TLS off (`tls=False`), use token authentication. +- With TLS off (`tls=False`), authenticate with a JWT carrying the `x-ably-clientType=server` claim: an API key is refused without TLS. See [UPDATING.md](https://github.com/ably/ably-pubsub-python/blob/main/UPDATING.md#deprecated-and-duplicate-options-are-removed) for how the old options map to `endpoint`. diff --git a/ably/pubsub/request/http.py b/ably/pubsub/request/http.py index a3627d72..1141caef 100644 --- a/ably/pubsub/request/http.py +++ b/ably/pubsub/request/http.py @@ -178,9 +178,11 @@ async def make_request(self, method, path, version=None, headers=None, body=None body = self.dump_body(body) if body: - all_headers = HttpUtils.default_post_headers(self.options.use_binary_protocol, version=version) + all_headers = HttpUtils.default_post_headers(self.options.use_binary_protocol, version=version, + agents=self.options._agents) else: - all_headers = HttpUtils.default_get_headers(self.options.use_binary_protocol, version=version) + all_headers = HttpUtils.default_get_headers(self.options.use_binary_protocol, version=version, + agents=self.options._agents) params = HttpUtils.get_query_params(self.options) diff --git a/ably/pubsub/request/httputils.py b/ably/pubsub/request/httputils.py index f24a2f00..47a1e9fe 100644 --- a/ably/pubsub/request/httputils.py +++ b/ably/pubsub/request/httputils.py @@ -17,8 +17,8 @@ class HttpUtils: } @staticmethod - def default_get_headers(binary=False, version=None): - headers = HttpUtils.default_headers(version=version) + def default_get_headers(binary=False, version=None, agents=None): + headers = HttpUtils.default_headers(version=version, agents=agents) if binary: headers["Accept"] = HttpUtils.mime_types['binary'] else: @@ -26,8 +26,8 @@ def default_get_headers(binary=False, version=None): return headers @staticmethod - def default_post_headers(binary=False, version=None): - headers = HttpUtils.default_get_headers(binary=binary, version=version) + def default_post_headers(binary=False, version=None, agents=None): + headers = HttpUtils.default_get_headers(binary=binary, version=version, agents=agents) headers["Content-Type"] = headers["Accept"] return headers @@ -51,14 +51,26 @@ def get_host_header(host): } @staticmethod - def default_headers(version=None): + def default_headers(version=None, agents=None): if version is None: version = api_version return { "X-Ably-Version": version, - "Ably-Agent": f'ably-pubsub-python/{lib_version} python/{platform.python_version()}' + "Ably-Agent": HttpUtils.agent_string(agents), } + @staticmethod + def agent_string(agents=None): + """The `Ably-Agent` value: the library and platform, then each of `agents` (RSC7d6). + + An entry with no version is emitted bare: versionless identifiers are flags, such + as the side a package declares, and `name/None` is not a valid Ably-Agent token. + """ + agent = f'ably-pubsub-python/{lib_version} python/{platform.python_version()}' + for name, version in (agents or {}).items(): + agent += f' {name}/{version}' if version else f' {name}' + return agent + @staticmethod def get_query_params(options): params = {} diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index 2e46a5c0..6ba52623 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -71,6 +71,7 @@ IncompatibleClientIdException, InvalidArgumentException, ) +from ably.pubsub.util.side import SERVER_AGENT_IDENTIFIER, options_with_side_agent from ably.pubsub.vcdiff.defaultvcdiffdecoder import AblyVCDiffDecoder from ably.pubsub.version import api_version, lib_version @@ -184,6 +185,10 @@ def create_http_client(**kwargs) -> PubSubHttpClient: The largest total size of the messages one publish may carry, counting each message's name, data, client ID and extras. A larger publish raises an AblyException with code 40009 and is not sent. The default is 65536 (64 KiB). + agents: dict[str, str | None] + Additional entries for the `Ably-Agent` header, mapping a product name to its + version, or to None for a versionless entry. For an SDK built on this one to + identify itself. The client always adds `ably-pubsub-server` as well. Returns ------- @@ -204,7 +209,7 @@ def create_http_client(**kwargs) -> PubSubHttpClient: option's 3.x name, which the message names the replacement for. """ with factory_construction(): - return _DefaultPubSubHttpClient(**kwargs) + return _DefaultPubSubHttpClient(**options_with_side_agent(kwargs, SERVER_AGENT_IDENTIFIER)) def create_realtime_client(**kwargs) -> PubSubRealtimeClient: @@ -280,7 +285,7 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: ValueError. """ with factory_construction(): - return _DefaultPubSubRealtimeClient(**kwargs) + return _DefaultPubSubRealtimeClient(**options_with_side_agent(kwargs, SERVER_AGENT_IDENTIFIER)) __all__ = [ diff --git a/ably/pubsub/server/sync.py b/ably/pubsub/server/sync.py index 88bd7aaa..5698d806 100644 --- a/ably/pubsub/server/sync.py +++ b/ably/pubsub/server/sync.py @@ -56,6 +56,7 @@ IncompatibleClientIdException, InvalidArgumentException, ) +from ably.pubsub.sync.util.side import SERVER_AGENT_IDENTIFIER, options_with_side_agent from ably.pubsub.sync.vcdiff.defaultvcdiffdecoder import AblyVCDiffDecoder from ably.pubsub.version import api_version, lib_version @@ -82,7 +83,7 @@ def create_http_client(**kwargs) -> PubSubHttpClient: ValueError. """ with factory_construction(): - return _DefaultPubSubHttpClientSync(**kwargs) + return _DefaultPubSubHttpClientSync(**options_with_side_agent(kwargs, SERVER_AGENT_IDENTIFIER)) __all__ = [ diff --git a/ably/pubsub/transport/websockettransport.py b/ably/pubsub/transport/websockettransport.py index 75f55a44..c64bdc48 100644 --- a/ably/pubsub/transport/websockettransport.py +++ b/ably/pubsub/transport/websockettransport.py @@ -88,7 +88,7 @@ def __select_connect_func(options): return ws_connect def connect(self): - headers = HttpUtils.default_headers() + headers = HttpUtils.default_headers(agents=self.options._agents) query_params = urllib.parse.urlencode(self.params) scheme = 'wss' if self.options.tls else 'ws' port = Defaults.get_port(self.options) diff --git a/ably/pubsub/types/options.py b/ably/pubsub/types/options.py index cacf205f..f27f778e 100644 --- a/ably/pubsub/types/options.py +++ b/ably/pubsub/types/options.py @@ -117,7 +117,7 @@ def __init__(self, client_id=None, tls=True, port=0, suspended_retry_timeout_seconds=None, connectivity_check_url=None, channel_retry_timeout_seconds=None, add_request_ids=False, vcdiff_decoder: VCDiffDecoder = None, transport_params=None, echo_messages=True, - max_message_size=None, _test_options=None, **kwargs): + max_message_size=None, agents=None, _test_options=None, **kwargs): _reject_renamed_timing_options(kwargs) super().__init__(**kwargs) @@ -210,6 +210,15 @@ def __init__(self, client_id=None, tls=True, port=0, self.__vcdiff_decoder = vcdiff_decoder self.__transport_params = transport_params or {} self.__max_message_size = max_message_size + if agents is not None and not isinstance(agents, dict): + raise InvalidArgumentException( + f'agents must be a dict of product name to version or None, not {type(agents).__name__}', + 400, 40000) + for name, version in (agents or {}).items(): + if not name or any(c in f'{name}{version or ""}' for c in ' /'): + raise InvalidArgumentException( + f'agents entry {name!r}: {version!r} must not contain spaces or "/"', 400, 40000) + self.__agents = dict(agents or {}) self.__test_options = _test_options self.__hosts = self.__get_hosts() @@ -393,6 +402,15 @@ def transport_params(self): def max_message_size(self): return self.__max_message_size + @property + def agents(self): + return dict(self.__agents) + + @property + def _agents(self): + """The agents for building the Ably-Agent header, without a defensive copy per request.""" + return self.__agents + @property def _test_options(self): return self.__test_options diff --git a/ably/pubsub/util/side.py b/ably/pubsub/util/side.py new file mode 100644 index 00000000..f657d570 --- /dev/null +++ b/ably/pubsub/util/side.py @@ -0,0 +1,37 @@ +"""The agent entry a package stamps to declare which side it runs on. + +Nothing here may grow into a general abstraction over the client: it exists only to +stamp the side a package's factories declare. +""" + +# The `-server` suffix on this identifier is load-bearing, not cosmetic. On API-key auth +# the realtime system grants the server exemption from monthly-active-user counting by +# matching an agent entry ending in `-server`, and an identifier that is not yet in the +# ably-common registry is classified by that suffix alone. Renaming it without preserving +# its suffix silently reclassifies every client the package constructs as a device. +# +# It matches the identifier the other Pub/Sub SDKs' server packages send, and lives here +# rather than in `ably.pubsub.server` so the naming scheme can be changed in one place. +SERVER_AGENT_IDENTIFIER = 'ably-pubsub-server' + + +def options_with_side_agent(kwargs, identifier): + """A copy of the caller's client options carrying the agent entry that declares the side. + + Only `agents` is replaced, with a new dict rather than by mutation, so the caller's + options and their own `agents` dict are both left untouched. Every other option is + copied by reference. + + The caller's `agents` entries are kept alongside the side stamp, so an SDK layered on + top of this package keeps its attribution. The side stamp is applied last and so wins + a collision on its own identifier: which side the package declares is the package's to + state, not the caller's to redefine. + + The side entry is versionless — a flag — because the identifier is shared by every + Pub/Sub SDK, so a version could not say what it versions, and would only duplicate the + `ably-pubsub-python` entry sent alongside it. + """ + agents = kwargs.get('agents') + if agents is not None and not isinstance(agents, dict): + return kwargs # left for Options to reject with a message naming the option + return {**kwargs, 'agents': {**(agents or {}), identifier: None}} diff --git a/test/ably/http/httpauth_test.py b/test/ably/http/httpauth_test.py index 5857536b..8fa4dc0c 100644 --- a/test/ably/http/httpauth_test.py +++ b/test/ably/http/httpauth_test.py @@ -14,7 +14,13 @@ from ably.pubsub.server import AblyAuthException, Auth, create_http_client from ably.pubsub.types.tokendetails import TokenDetails from test.ably.testapp import TestApp -from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol +from test.ably.utils import ( + NATIVE_TOKEN_AUTH_UNSUPPORTED, + BaseAsyncTestCase, + VaryByProtocolTestsMetaclass, + dont_vary_protocol, +) +from test.uts.helpers.sandbox import jwt_claims log = logging.getLogger(__name__) @@ -184,6 +190,7 @@ async def test_if_authorize_changes_auth_mechanism_to_token(self): # RSA10a @dont_vary_protocol + @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_authorize_always_creates_new_token(self): await self.ably.auth.authorize({'capability': {'test': ['publish']}}) await self.ably.channels.get('test').publish('event', 'data') @@ -218,6 +225,7 @@ async def test_authorize_adheres_to_request_token(self): for arg, value in auth_params.items(): assert auth_called[arg] == value, f"{arg} called with wrong value: {value}" + @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_with_token_str_https(self): token = await self.ably.auth.authorize() token = token.token @@ -226,6 +234,7 @@ async def test_with_token_str_https(self): await ably.channels.get('test_auth_with_token_str').publish('event', 'foo_bar') await ably.close() + @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_with_token_str_http(self): token = await self.ably.auth.authorize() token = token.token @@ -301,12 +310,19 @@ async def test_timestamp_is_not_stored(self): async def test_client_id_precedence(self): client_id = uuid.uuid4().hex overridden_client_id = uuid.uuid4().hex + + # The client's clientId is what the library passes to the callback, ahead of the default token params + async def auth_callback(token_params): + assert token_params['client_id'] == client_id + return await TestApp.get_jwt_for(token_params) + ably = await TestApp.get_ably_rest( use_binary_protocol=self.use_binary_protocol, client_id=client_id, + auth_callback=auth_callback, default_token_params={'client_id': overridden_client_id}) token = await ably.auth.authorize() - assert token.client_id == client_id + assert jwt_claims(token.token)['x-ably-clientId'] == client_id assert ably.auth.client_id == client_id channel = ably.channels[ @@ -326,6 +342,7 @@ async def setup(self): def per_protocol_setup(self, use_binary_protocol): self.use_binary_protocol = use_binary_protocol + @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_with_key(self): ably = await TestApp.get_ably_rest(use_binary_protocol=self.use_binary_protocol) diff --git a/test/ably/http/httpchannelpublish_test.py b/test/ably/http/httpchannelpublish_test.py index 1edff7be..eb26e298 100644 --- a/test/ably/http/httpchannelpublish_test.py +++ b/test/ably/http/httpchannelpublish_test.py @@ -18,6 +18,7 @@ from test.ably import utils from test.ably.testapp import TestApp from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, assert_waiter, dont_vary_protocol +from test.uts.helpers.sandbox import jwt_claims log = logging.getLogger(__name__) @@ -31,7 +32,12 @@ async def setup(self): self.test_vars = await TestApp.get_test_vars() self.ably = await TestApp.get_ably_rest() self.client_id = uuid.uuid4().hex - self.ably_with_client_id = await TestApp.get_ably_rest(client_id=self.client_id, use_token_auth=True) + + async def auth_callback(token_params): + return await TestApp.get_jwt(client_id=self.client_id) + + self.ably_with_client_id = await TestApp.get_ably_rest(client_id=self.client_id, + auth_callback=auth_callback) yield await self.ably.close() await self.ably_with_client_id.close() @@ -118,9 +124,11 @@ async def test_message_list_generate_one_request(self): assert message['data'] == str(i) async def test_publish_error(self): - ably = await TestApp.get_ably_rest(use_binary_protocol=self.use_binary_protocol) - await ably.auth.authorize( - token_params={'capability': {"only_subscribe": ["subscribe"]}}) + async def auth_callback(token_params): + return await TestApp.get_jwt(capability={"only_subscribe": ["subscribe"]}) + + ably = await TestApp.get_ably_rest(use_binary_protocol=self.use_binary_protocol, + auth_callback=auth_callback) with pytest.raises(AblyException) as excinfo: await ably.channels["only_subscribe"].publish() @@ -234,8 +242,9 @@ async def test_token_is_bound_to_options_client_id_after_publish(self): await channel.publish(name='publish', data='test') # defined after publish - assert isinstance(self.ably_with_client_id.auth.token_details, TokenDetails) - assert self.ably_with_client_id.auth.token_details.client_id == self.client_id + token_details = self.ably_with_client_id.auth.token_details + assert isinstance(token_details, TokenDetails) + assert jwt_claims(token_details.token)['x-ably-clientId'] == self.client_id assert self.ably_with_client_id.auth.auth_mechanism == Auth.Method.TOKEN history = await channel.history() assert history.items[0].client_id == self.client_id @@ -254,7 +263,7 @@ async def test_publish_message_without_client_id_on_identified_client(self): assert messages is not None, "Expected non-None messages" assert len(messages) == 1, "Expected 1 message" - assert post_mock.call_count == 2 + assert post_mock.call_count == 1 if self.use_binary_protocol: posted_body = msgpack.unpackb( @@ -295,9 +304,9 @@ async def test_publish_message_with_client_id_on_identified_client(self): await channel.publish(message) async def test_publish_message_with_wrong_client_id_on_implicit_identified_client(self): - new_token = await self.ably.auth.authorize(token_params={'client_id': uuid.uuid4().hex}) + token = await TestApp.get_jwt(client_id=uuid.uuid4().hex) new_ably = await TestApp.get_ably_rest(key=None, - token=new_token.token, + token=token, use_binary_protocol=self.use_binary_protocol) channel = new_ably.channels[ @@ -313,13 +322,12 @@ async def test_publish_message_with_wrong_client_id_on_implicit_identified_clien # RSA15b async def test_wildcard_client_id_can_publish_as_others(self): - wildcard_token_details = await self.ably.auth.request_token({'client_id': '*'}) + wildcard_token = await TestApp.get_jwt(client_id='*') wildcard_ably = await TestApp.get_ably_rest( key=None, - token_details=wildcard_token_details, + token=wildcard_token, use_binary_protocol=self.use_binary_protocol) - assert wildcard_ably.auth.client_id == '*' channel = wildcard_ably.channels[ self.get_channel_name('persisted:wildcard_client_id')] await channel.publish(name='publish1', data='no client_id') diff --git a/test/ably/http/httphttp_test.py b/test/ably/http/httphttp_test.py index 5490c149..a41fc904 100644 --- a/test/ably/http/httphttp_test.py +++ b/test/ably/http/httphttp_test.py @@ -195,7 +195,7 @@ async def test_request_headers(self): # Agent assert 'Ably-Agent' in r.request.headers - expr = r"^ably-pubsub-python\/\d+.\d+.\d+(-beta\.\d+)? python\/\d.\d+.\d+$" + expr = r"^ably-pubsub-python\/\d+.\d+.\d+(-beta\.\d+)? python\/\d.\d+.\d+ ably-pubsub-server$" assert re.search(expr, r.request.headers['Ably-Agent']) await ably.close() diff --git a/test/ably/realtime/realtimeauth_test.py b/test/ably/realtime/realtimeauth_test.py index 4e8bebba..32a2f375 100644 --- a/test/ably/realtime/realtimeauth_test.py +++ b/test/ably/realtime/realtimeauth_test.py @@ -1,8 +1,8 @@ import asyncio import json +import time import urllib.parse -import httpx import pytest from ably.pubsub.realtime.connection import ConnectionState @@ -12,7 +12,7 @@ from ably.pubsub.types.tokendetails import TokenDetails from ably.pubsub.util.exceptions import IncompatibleClientIdException from test.ably.testapp import TestApp -from test.ably.utils import BaseAsyncTestCase, random_string +from test.ably.utils import NATIVE_TOKEN_AUTH_UNSUPPORTED, BaseAsyncTestCase, random_string echo_url = 'https://echo.ably.io' @@ -51,6 +51,7 @@ async def test_auth_wrong_api_key(self): assert state_change.reason.status_code == 401 await ably.close() + @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_auth_with_token_string(self): rest = await TestApp.get_ably_rest() token_details = await rest.auth.request_token() @@ -69,6 +70,7 @@ async def test_auth_with_invalid_token_string(self): assert state_change.reason.status_code == 401 await ably.close() + @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_auth_with_token_details(self): rest = await TestApp.get_ably_rest() token_details = await rest.auth.request_token() @@ -87,6 +89,7 @@ async def test_auth_with_invalid_token_details(self): assert state_change.reason.status_code == 401 await ably.close() + @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_auth_with_auth_callback_with_token_request(self): rest = await TestApp.get_ably_rest() @@ -101,6 +104,7 @@ async def callback(params): assert ably.connection.error_reason is None await ably.close() + @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_auth_with_auth_callback_token_with_details(self): rest = await TestApp.get_ably_rest() @@ -116,11 +120,8 @@ async def callback(params): await ably.close() async def test_auth_with_auth_callback_with_token_string(self): - rest = await TestApp.get_ably_rest() - async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt_for(params) ably = await TestApp.get_ably_realtime(auth_callback=callback) await asyncio.wait_for(ably.connection.once_async(ConnectionState.CONNECTED), timeout=5) @@ -139,6 +140,7 @@ async def callback(params): assert state_change.reason.status_code == 401 await ably.close() + @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_auth_with_auth_url_json(self): rest = await TestApp.get_ably_rest() token_details = await rest.auth.request_token() @@ -156,9 +158,8 @@ async def test_auth_with_auth_url_json(self): await ably.close() async def test_auth_with_auth_url_text_plain(self): - rest = await TestApp.get_ably_rest() - token_details = await rest.auth.request_token() - url_path = f"{echo_url}/?type=text&body={token_details.token}" + token = await TestApp.get_jwt() + url_path = f"{echo_url}/?type=text&body={token}" ably = await TestApp.get_ably_realtime(auth_url=url_path) await asyncio.wait_for(ably.connection.once_async(ConnectionState.CONNECTED), timeout=5) @@ -168,12 +169,11 @@ async def test_auth_with_auth_url_text_plain(self): await ably.close() async def test_auth_with_auth_url_post(self): - rest = await TestApp.get_ably_rest() - token_details = await rest.auth.request_token() + token = await TestApp.get_jwt() url_path = f"{echo_url}/?type=json&" ably = await TestApp.get_ably_realtime(auth_url=url_path, auth_method='POST', - auth_params=token_details) + auth_params={'token': token}) await asyncio.wait_for(ably.connection.once_async(ConnectionState.CONNECTED), timeout=5) response_time_ms = await ably.connection.ping() assert response_time_ms is not None @@ -181,11 +181,8 @@ async def test_auth_with_auth_url_post(self): await ably.close() async def test_reauth_while_connected(self): - rest = await TestApp.get_ably_rest() - async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt_for(params) ably = await TestApp.get_ably_realtime(auth_callback=callback) await asyncio.wait_for(ably.connection.once_async(ConnectionState.CONNECTED), timeout=5) @@ -221,11 +218,8 @@ def on_update(state_change): await ably.close() async def test_reauth_while_connecting(self): - rest = await TestApp.get_ably_rest() - async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt_for(params) ably = await TestApp.get_ably_realtime(auth_callback=callback) original_transport = await ably.connection.connection_manager.once_async('transport.pending') @@ -247,11 +241,8 @@ async def withhold_protocol_message(msg): await ably.close() async def test_reauth_immediately(self): - rest = await TestApp.get_ably_rest() - async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt_for(params) ably = await TestApp.get_ably_realtime(auth_callback=callback) await ably.auth.authorize() @@ -260,12 +251,10 @@ async def callback(params): await ably.close() async def test_capability_change_without_loss_of_continuity(self): - rest = await TestApp.get_ably_rest() channel_name = random_string(5) async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt_for(params) ably = await TestApp.get_ably_realtime(auth_callback=callback) @@ -280,12 +269,10 @@ async def callback(params): await ably.close() async def test_capability_downgrade(self): - rest = await TestApp.get_ably_rest() channel_name = random_string(5) async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt_for(params) ably = await TestApp.get_ably_realtime(auth_callback=callback) @@ -312,11 +299,8 @@ def on_channel_state_change(state_change): await ably.close() async def test_reauth_inbound_auth_protocol_msg(self): - rest = await TestApp.get_ably_rest() - async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt_for(params) ably = await TestApp.get_ably_realtime(auth_callback=callback) msg = { @@ -336,17 +320,8 @@ def on_update(state_change): # RSC8a4 async def test_jwt_reauth(self): - test_vars = await TestApp.get_test_vars() - key = test_vars["keys"][0] - key_name = key["key_name"] - key_secret = key["key_secret"] - async def auth_callback(_): - response = httpx.get( - echo_url + '/createJWT', - params={"keyName": key_name, "keySecret": key_secret, "expiresIn": 35} - ) - return response.text + return await TestApp.get_jwt(ttl=35 * 1000) ably = await TestApp.get_ably_realtime(auth_callback=auth_callback) @@ -359,11 +334,8 @@ async def auth_callback(_): # RTN14b async def test_renew_token_single_attempt(self): - rest = await TestApp.get_ably_rest() - async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt() ably = await TestApp.get_ably_realtime(auth_callback=callback) msg = { @@ -379,19 +351,15 @@ async def callback(params): await transport.on_protocol_message(msg) assert ably.auth.token_details is not original_token_details await ably.close() - await rest.close() # RTN14b async def test_renew_token_connection_attempt_fails(self): - rest = await TestApp.get_ably_rest() call_count = 0 async def callback(params): nonlocal call_count call_count += 1 - params = {"ttl": 1} - token_details = await rest.auth.request_token(token_params=params) - return token_details + return await TestApp.get_jwt(expires_at=time.time() - 60) ably = await TestApp.get_ably_realtime(auth_callback=callback) @@ -401,7 +369,6 @@ async def callback(params): assert ably.connection.error_reason.status_code == 401 await ably.close() - await rest.close() # RSA4a async def test_renew_token_no_renew_means_provided(self): @@ -497,11 +464,8 @@ async def test_auth_url_403_custom_error(self): # RTN15h2 async def test_renew_token_single_attempt_upon_disconnection(self): - rest = await TestApp.get_ably_rest() - async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt_for(params) ably = await TestApp.get_ably_realtime(auth_callback=callback) msg = { @@ -518,14 +482,12 @@ async def callback(params): await ably.connection.connection_manager.transport.on_protocol_message(msg) assert ably.auth.token_details is not original_token_details await ably.close() - await rest.close() # RTN15h1 async def test_renew_token_no_renew_means_provided_upon_disconnection(self): - rest = await TestApp.get_ably_rest() - token_details = await rest.auth.request_token() + token = await TestApp.get_jwt() - ably = await TestApp.get_ably_realtime(token_details=token_details) + ably = await TestApp.get_ably_realtime(token=token) state_change = await asyncio.wait_for(ably.connection.once_async(ConnectionState.CONNECTED), timeout=5) msg = { @@ -542,14 +504,10 @@ async def test_renew_token_no_renew_means_provided_upon_disconnection(self): assert state_change.reason.code == 40171 assert state_change.reason.status_code == 403 await ably.close() - await rest.close() async def test_renew_token_single_attempt_on_resume(self): - rest = await TestApp.get_ably_rest() - async def callback(params): - token_details = await rest.auth.request_token(token_params=params) - return token_details.token + return await TestApp.get_jwt_for(params) ably = await TestApp.get_ably_realtime(auth_callback=callback) msg = { @@ -572,13 +530,11 @@ async def callback(params): await transport.on_protocol_message(msg) assert ably.auth.token_details is not original_token_details await ably.close() - await rest.close() async def test_renew_token_no_renew_means_provided_on_resume(self): - rest = await TestApp.get_ably_rest() - token_details = await rest.auth.request_token() + token = await TestApp.get_jwt() - ably = await TestApp.get_ably_realtime(token_details=token_details) + ably = await TestApp.get_ably_realtime(token=token) msg = { "action": ProtocolMessageAction.DISCONNECTED, @@ -603,16 +559,14 @@ async def test_renew_token_no_renew_means_provided_on_resume(self): assert state_change.reason.code == 40171 assert state_change.reason.status_code == 403 await ably.close() - await rest.close() # Request a token using client_id, then initialize a connection without one, # and check that the connection inherits the client_id from the token_details async def test_auth_client_id_inheritance_auth_callback(self): - rest = await TestApp.get_ably_rest() client_id = 'test_client_id' async def auth_callback(_): - return await rest.auth.request_token({"client_id": client_id}) + return await TestApp.get_jwt(client_id=client_id) realtime = await TestApp.get_ably_realtime(auth_callback=auth_callback) @@ -624,7 +578,6 @@ async def auth_callback(_): assert realtime.auth.client_id == client_id await realtime.close() - await rest.close() # Rest token generation with client_id, then constructing a client with a # different client_id, should fail with a library-generated error before @@ -645,12 +598,11 @@ async def test_auth_client_id_mismatch(self): # Rest token generation with clientId '*', then connecting with just the # token string and a different clientId, should succeed (RSA15b) async def test_auth_client_id_wildcard_token(self): - rest = await TestApp.get_ably_rest() client_id = 'test_client_id' - token_details = await rest.auth.request_token({"client_id": "*"}) + token = await TestApp.get_jwt(client_id="*") - realtime = await TestApp.get_ably_realtime(token_details=token_details, client_id=client_id) + realtime = await TestApp.get_ably_realtime(token=token, client_id=client_id) # RTC17: the configured clientId holds before CONNECTED assert realtime.auth.client_id == client_id @@ -660,15 +612,14 @@ async def test_auth_client_id_wildcard_token(self): assert realtime.auth.client_id == client_id await realtime.close() - await rest.close() # Request a token using clientId, then initialize a connection using just the token string, # and check that the connection inherits the clientId from the connectionDetails async def test_auth_client_id_inheritance_token(self): - rest = await TestApp.get_ably_rest() client_id = 'test_client_id' - token_details = await rest.auth.request_token({"client_id": client_id}) + token = await TestApp.get_jwt(client_id=client_id) + token_details = TokenDetails(token=token, client_id=client_id) realtime = await TestApp.get_ably_realtime(token_details=token_details) @@ -680,4 +631,3 @@ async def test_auth_client_id_inheritance_token(self): assert realtime.auth.client_id == client_id await realtime.close() - await rest.close() diff --git a/test/ably/realtime/realtimeconnection_test.py b/test/ably/realtime/realtimeconnection_test.py index 46261815..fbea8559 100644 --- a/test/ably/realtime/realtimeconnection_test.py +++ b/test/ably/realtime/realtimeconnection_test.py @@ -523,18 +523,15 @@ async def test_fallback_host_disconnected_protocol_msg(self): # RTN2d async def test_connection_null_client_id_query_params(self): - rest = await TestApp.get_ably_rest() + token = await TestApp.get_jwt() - token_details = await rest.auth.request_token() - - realtime = await TestApp.get_ably_realtime(token_details=token_details) + realtime = await TestApp.get_ably_realtime(token=token) await realtime.connection.once_async(ConnectionState.CONNECTED) assert realtime.connection.connection_manager.transport.params.get("client_id") is None assert realtime.auth.client_id is None await realtime.close() - await rest.close() async def test_connection_client_id_query_params(self): client_id = 'test_client_id' diff --git a/test/ably/testapp.py b/test/ably/testapp.py index 2a4c1efb..c9203d9f 100644 --- a/test/ably/testapp.py +++ b/test/ably/testapp.py @@ -11,6 +11,7 @@ from ably.pubsub.types.capability import Capability from ably.pubsub.types.options import Options from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.sandbox import generate_jwt log = logging.getLogger(__name__) @@ -66,6 +67,27 @@ async def get_test_vars(): return TestApp.__test_vars + @staticmethod + async def get_jwt(client_id=None, capability=None, ttl=3600000, expires_at=None): + """An Ably JWT signed with the full-access key, in place of a native token. + + `capability` may be a dict, as token params give it. Use this where a test needs a token + string or an auth_callback result; the server takes its server declaration only from the + signed claim this carries. + """ + test_vars = await TestApp.get_test_vars() + key = test_vars["keys"][0] + if capability is not None and not isinstance(capability, str): + capability = str(Capability(capability)) + return generate_jwt(key["key_name"], key["key_secret"], ttl=ttl, + client_id=client_id, capability=capability, expires_at=expires_at) + + @staticmethod + async def get_jwt_for(token_params): + """A JWT for an auth_callback, carrying the token params a JWT can express.""" + return await TestApp.get_jwt(client_id=token_params.get('client_id'), + capability=token_params.get('capability')) + @staticmethod async def get_ably_rest(**kw): test_vars = await TestApp.get_test_vars() diff --git a/test/ably/utils.py b/test/ably/utils.py index 6715ced7..83335974 100644 --- a/test/ably/utils.py +++ b/test/ably/utils.py @@ -12,6 +12,8 @@ from httpx import Response from ably.pubsub.request.http import Http +from ably.pubsub.util.exceptions import AblyException +from test.uts.helpers.sandbox import NATIVE_TOKEN_AUTH_UNSUPPORTED # noqa: F401 - re-exported for test/ably class BaseTestCase: @@ -65,6 +67,7 @@ def test_something(self): supports json and msgpack """ responses = [] + error_responses = [] def patch(): original = Http.make_request @@ -74,12 +77,24 @@ async def fake_make_request(self, *args, **kwargs): responses.append(response) return response - patcher = mock.patch.object(Http, 'make_request', fake_make_request) - patcher.start() - return patcher + original_raise = AblyException.raise_for_response - def unpatch(patcher): - patcher.stop() + # A request that fails raises before make_request returns. It still counts as a + # request made, but its error body isn't held to the protocol. + def fake_raise_for_response(response): + if not 200 <= response.status_code < 300: + error_responses.append(response) + original_raise(response) + + patchers = [mock.patch.object(Http, 'make_request', fake_make_request), + mock.patch.object(AblyException, 'raise_for_response', staticmethod(fake_raise_for_response))] + for patcher in patchers: + patcher.start() + return patchers + + def unpatch(patchers): + for patcher in patchers: + patcher.stop() def test_decorator(fn): @functools.wraps(fn) @@ -92,7 +107,7 @@ async def test_decorated(self, *args, **kwargs): # left as it was found even when an assertion or a transport error escapes. unpatch(patcher) - assert len(responses) >= 1, \ + assert len(responses) + len(error_responses) >= 1, \ "If your test doesn't make any requests, use the @dont_vary_protocol decorator" for response in responses: @@ -150,6 +165,8 @@ async def wrapper(self): await old_func(self) wrapper.__name__ = old_name + '_' + ttype + # carry marks such as pytest.mark.skip over to the protocol-specific copy + wrapper.pytestmark = getattr(old_func, 'pytestmark', []) return wrapper diff --git a/test/unit/server_agent_test.py b/test/unit/server_agent_test.py new file mode 100644 index 00000000..bcf765d2 --- /dev/null +++ b/test/unit/server_agent_test.py @@ -0,0 +1,109 @@ +"""The server package declares its side in the Ably-Agent header. + +On API-key auth the realtime system exempts a connection from monthly-active-user +counting only when an agent entry ends in `-server`, so every request and connection +a server factory builds must carry `ably-pubsub-server`. +""" + +import httpx +import pytest + +import ably.pubsub.server.sync as server_sync +from ably.pubsub.realtime.connection import ConnectionState +from ably.pubsub.server import InvalidArgumentException, create_http_client +from ably.pubsub.sync.types.testoptions import TestOptions as SyncTestOptions +from ably.pubsub.types.testoptions import TestOptions +from test.uts.helpers.client import await_connection_state, close_open_clients, realtime_client +from test.uts.helpers.mock_http import MockHttpClient +from test.uts.helpers.mock_websocket import CONNECTED_MESSAGE, MockWebSocket + +KEY = 'app.key:secret' +TIME_RESPONSE = [1234567890000] + + +@pytest.fixture(autouse=True) +async def close_clients(): + yield + await close_open_clients() + + +def agent_entries(header): + return header.split(' ') + + +async def http_agent(**kwargs): + mock = MockHttpClient(on_connection_attempt=lambda conn: conn.respond_with_success()) + mock.queue_response(200, TIME_RESPONSE) + client = create_http_client(key=KEY, _test_options=TestOptions(http_transport=mock.as_transport()), **kwargs) + try: + await client.time() + finally: + await client.close() + return mock.captured_requests[0].headers['Ably-Agent'] + + +async def test_http_requests_declare_the_server_side(): + agents = agent_entries(await http_agent()) + assert agents[0].startswith('ably-pubsub-python/') + assert agents[-1] == 'ably-pubsub-server' + + +async def test_the_realtime_connection_declares_the_server_side(): + mock = MockWebSocket( + on_connection_attempt=lambda connection: connection.respond_with_success(CONNECTED_MESSAGE)) + client = realtime_client(mock, disconnected_retry_timeout_seconds=60) + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED) + + agents = agent_entries(mock.connection_attempts[0].headers['Ably-Agent']) + assert agents[0].startswith('ably-pubsub-python/') + assert agents[-1] == 'ably-pubsub-server' + + +def test_sync_http_requests_declare_the_server_side(): + requests = [] + + def handler(request): + requests.append(request) + return httpx.Response(200, json=TIME_RESPONSE) + + client = server_sync.create_http_client( + key=KEY, _test_options=SyncTestOptions(http_transport=httpx.MockTransport(handler))) + try: + client.time() + finally: + client.close() + + agents = agent_entries(requests[0].headers['Ably-Agent']) + assert agents[0].startswith('ably-pubsub-python/') + assert agents[-1] == 'ably-pubsub-server' + + +async def test_caller_agents_are_kept_alongside_the_server_side(): + caller_agents = {'my-sdk': '1.2.3', 'my-flag': None} + + agents = agent_entries(await http_agent(agents=caller_agents)) + + assert 'my-sdk/1.2.3' in agents + assert 'my-flag' in agents + assert agents[-1] == 'ably-pubsub-server' + assert caller_agents == {'my-sdk': '1.2.3', 'my-flag': None} + + +async def test_a_caller_cannot_redeclare_the_side(): + agents = agent_entries(await http_agent(agents={'ably-pubsub-server': '9.9.9'})) + + assert agents.count('ably-pubsub-server') == 1 + assert 'ably-pubsub-server/9.9.9' not in agents + + +@pytest.mark.parametrize('agents', [['my-sdk'], {'my sdk': '1.0'}, {'my-sdk': '1/0'}, {'': None}]) +def test_malformed_agents_are_rejected(agents): + with pytest.raises(InvalidArgumentException): + create_http_client(key=KEY, agents=agents) + + +def test_the_agents_property_cannot_change_the_header(): + client = create_http_client(key=KEY) + client.options.agents.pop('ably-pubsub-server') + assert 'ably-pubsub-server' in client.options.agents diff --git a/test/uts/helpers/sandbox.py b/test/uts/helpers/sandbox.py index a29236c3..1631dfd7 100644 --- a/test/uts/helpers/sandbox.py +++ b/test/uts/helpers/sandbox.py @@ -61,6 +61,15 @@ CIPHER_FIXTURE = __app_setup['cipher'] +# Skip reason for specifications whose subject is a native Ably token. A server client cannot +# authenticate with one: Ably accepts the server declaration under token auth only from a signed +# x-ably-clientType JWT claim (40167). +NATIVE_TOKEN_AUTH_UNSUPPORTED = ( + "A server client cannot authenticate with a native Ably token: Ably accepts the server declaration " + "under token auth only from a signed x-ably-clientType JWT claim (40167)" +) + + def random_id(length=6): """A short unique suffix, for the names the specifications build. @@ -205,6 +214,9 @@ def generate_jwt(key_name, key_secret, ttl=3600000, client_id=None, capability=N 'iat': issued_at, 'exp': issued_at + seconds, 'x-ably-capability': capability if capability is not None else '{"*":["*"]}', + # The clients are ably-pubsub-server clients, which declare the server side, + # and on token auth Ably accepts that only from this signed claim (40167). + 'x-ably-clientType': 'server', } if client_id is not None: claims['x-ably-clientId'] = client_id @@ -214,6 +226,12 @@ def generate_jwt(key_name, key_secret, ttl=3600000, client_id=None, capability=N return b'.'.join((signing_input, __base64url(signature))).decode('ascii') +def jwt_claims(token): + """The claims of an Ably JWT, for a test to check what a token it was handed carries.""" + payload = token.split('.')[1] + return json.loads(base64.urlsafe_b64decode(payload + '=' * (-len(payload) % 4))) + + def __jwt_segment(part): return __base64url(json.dumps(part, separators=(',', ':')).encode('utf-8')) diff --git a/test/uts/realtime/integration/auth/token_request_test.py b/test/uts/realtime/integration/auth/token_request_test.py index 9729b8c0..05632aed 100644 --- a/test/uts/realtime/integration/auth/token_request_test.py +++ b/test/uts/realtime/integration/auth/token_request_test.py @@ -15,9 +15,11 @@ case, so `TokenParams(clientId: x)` is `{'client_id': x}`. """ +import pytest + from ably.pubsub.realtime.connection import ConnectionState from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, sandbox_rest_client -from test.uts.helpers.sandbox import random_id +from test.uts.helpers.sandbox import NATIVE_TOKEN_AUTH_UNSUPPORTED, random_id # The wait the specification gives each connection. CONNECT_TIMEOUT = 15.0 @@ -29,6 +31,7 @@ def connection_id(client): # UTS: realtime/integration/RSA9a/token-request-server-accepted-0 +@pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_rsa9a_token_request_server_accepted(realtime_sandbox): creator = sandbox_rest_client(realtime_sandbox.key_str) @@ -46,6 +49,7 @@ async def auth_callback(params): # UTS: realtime/integration/RSA9/token-request-with-clientid-0 +@pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED) async def test_rsa9_token_request_with_clientid(realtime_sandbox): test_client_id = 'token-request-client-' + random_id() diff --git a/test/uts/realtime/integration/proxy/rest_faults_test.py b/test/uts/realtime/integration/proxy/rest_faults_test.py index dcbf0f38..dee19be3 100644 --- a/test/uts/realtime/integration/proxy/rest_faults_test.py +++ b/test/uts/realtime/integration/proxy/rest_faults_test.py @@ -76,9 +76,14 @@ async def auth_callback(params): return auth_callback -def jwt_auth_callback(api_key): - """The specification's JWT `authCallback`, which makes no request of its own.""" +def jwt_auth_callback(api_key, invocations=None): + """The specification's JWT `authCallback`, which makes no request of its own. + + `invocations` is appended to once per call where a test counts renewals. + """ async def auth_callback(params): + if invocations is not None: + invocations.append(params) return generate_jwt(extract_key_name(api_key), extract_key_secret(api_key)) return auth_callback @@ -111,7 +116,7 @@ async def test_rsc10_token_renewal_on_401(realtime_sandbox, proxy_session): }]) client = sandbox_rest_client( - auth_callback=token_auth_callback(realtime_sandbox.key_str, auth_callback_invocations), + auth_callback=jwt_auth_callback(realtime_sandbox.key_str, auth_callback_invocations), endpoint='localhost', port=session.proxy_port, tls=False, diff --git a/test/uts/rest/integration/proxy/rest_fallback_test.py b/test/uts/rest/integration/proxy/rest_fallback_test.py index 1e684c94..2e5488bd 100644 --- a/test/uts/rest/integration/proxy/rest_fallback_test.py +++ b/test/uts/rest/integration/proxy/rest_fallback_test.py @@ -22,7 +22,13 @@ from ably.pubsub.server import create_http_client from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import sandbox_rest_client, wall_clock_poll_until -from test.uts.helpers.sandbox import SANDBOX_ENDPOINT, random_id +from test.uts.helpers.sandbox import ( + SANDBOX_ENDPOINT, + extract_key_name, + extract_key_secret, + generate_jwt, + random_id, +) # The specification's `non_listening_port`: a port nothing is bound to, so that # connecting to it is refused rather than answered. @@ -32,6 +38,14 @@ HTTP_REQUEST_TIMEOUT_SECONDS = 3 +def jwt_auth_callback(api_key): + """An authCallback that signs an Ably JWT locally, so it makes no request the proxy could see.""" + async def auth_callback(params): + return generate_jwt(extract_key_name(api_key), extract_key_secret(api_key)) + + return auth_callback + + def token_auth_callback(api_key): """The specification's `token_auth_callback(api_key)`. @@ -321,7 +335,7 @@ async def test_rsl1k4_idempotent_retry_dedup(sandbox, proxy_session): }]) client = sandbox_rest_client( - auth_callback=token_auth_callback(sandbox.key_str), + auth_callback=jwt_auth_callback(sandbox.key_str), endpoint='localhost', fallback_hosts=['localhost'], port=session.proxy_port, diff --git a/test/uts/rest/integration/publish_test.py b/test/uts/rest/integration/publish_test.py index 30b817bc..63db0f42 100644 --- a/test/uts/rest/integration/publish_test.py +++ b/test/uts/rest/integration/publish_test.py @@ -8,7 +8,7 @@ from ably.pubsub.types.message import Message from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import sandbox_rest_client, wall_clock_poll_until -from test.uts.helpers.sandbox import random_id +from test.uts.helpers.sandbox import extract_key_name, extract_key_secret, generate_jwt, random_id # UTS: rest/integration/RSL1d/publish-failure-error-0 @@ -107,11 +107,10 @@ async def test_rsl1l1_publish_params_force_nack(sandbox, use_binary_protocol): # UTS: rest/integration/RSL1m4/clientid-mismatch-rejected-0 async def test_rsl1m4_clientid_mismatch_rejected(sandbox, use_binary_protocol): - key_client = sandbox_rest_client(sandbox.key_str, use_binary_protocol=use_binary_protocol) - - # request_token takes its token params as a positional dict; every keyword on it is an - # auth option. - token_details = await key_client.auth.request_token({'client_id': 'authenticated-client-id'}) + # A JWT carrying the clientId stands in for the native token the spec requests with + # request_token, which a server client cannot authenticate with. + token = generate_jwt(extract_key_name(sandbox.key_str), extract_key_secret(sandbox.key_str), + client_id='authenticated-client-id') # The spec passes the bare token string, not the TokenDetails, so the client does not # know the clientId the token carries and cannot rule the publish out locally. The @@ -119,7 +118,7 @@ async def test_rsl1m4_clientid_mismatch_rejected(sandbox, use_binary_protocol): # `token_details=` instead would have the library reject it in # `Channel.__publish_request_body` with the same 400/40012 before any request left. token_client = sandbox_rest_client( - token=token_details.token, use_binary_protocol=use_binary_protocol) + token=token, use_binary_protocol=use_binary_protocol) channel_name = 'clientid-mismatch-' + random_id() channel = token_client.channels.get(channel_name)