Send ably-pubsub-server in the Ably-Agent header

.github/workflows/lint.yml

@@ -12,7 +12,7 @@ jobs:
   lint:
     permissions:
       contents: read
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
         with:

README.md

@@ -160,7 +160,7 @@ realtime = create_realtime_client(key=ably_api_key, endpoint='nonprod:sandbox')
 
 - Never put a port in `endpoint`: `localhost:8080` is not a hostname and does not resolve.
 - Set the port with `tls_port`, or with `port` when TLS is off.
-- With TLS off (`tls=False`), use token authentication.
+- With TLS off (`tls=False`), authenticate with a JWT carrying the `x-ably-clientType=server` claim: an API key is refused without TLS.
 
 See [UPDATING.md](https://github.com/ably/ably-pubsub-python/blob/main/UPDATING.md#deprecated-and-duplicate-options-are-removed) for how the old
 options map to `endpoint`.

ably/pubsub/request/http.py

@@ -178,9 +178,11 @@ async def make_request(self, method, path, version=None, headers=None, body=None
             body = self.dump_body(body)
 
         if body:
-            all_headers = HttpUtils.default_post_headers(self.options.use_binary_protocol, version=version)
+            all_headers = HttpUtils.default_post_headers(self.options.use_binary_protocol, version=version,
+                                                       agents=self.options._agents)
         else:
-            all_headers = HttpUtils.default_get_headers(self.options.use_binary_protocol, version=version)
+            all_headers = HttpUtils.default_get_headers(self.options.use_binary_protocol, version=version,
+                                                      agents=self.options._agents)
 
         params = HttpUtils.get_query_params(self.options)
 

ably/pubsub/request/httputils.py

@@ -17,17 +17,17 @@ class HttpUtils:
     }
 
     @staticmethod
-    def default_get_headers(binary=False, version=None):
-        headers = HttpUtils.default_headers(version=version)
+    def default_get_headers(binary=False, version=None, agents=None):
+        headers = HttpUtils.default_headers(version=version, agents=agents)
         if binary:
             headers["Accept"] = HttpUtils.mime_types['binary']
         else:
             headers["Accept"] = HttpUtils.mime_types['json']
         return headers
 
     @staticmethod
-    def default_post_headers(binary=False, version=None):
-        headers = HttpUtils.default_get_headers(binary=binary, version=version)
+    def default_post_headers(binary=False, version=None, agents=None):
+        headers = HttpUtils.default_get_headers(binary=binary, version=version, agents=agents)
         headers["Content-Type"] = headers["Accept"]
         return headers
 
@@ -51,14 +51,26 @@ def get_host_header(host):
         }
 
     @staticmethod
-    def default_headers(version=None):
+    def default_headers(version=None, agents=None):
         if version is None:
             version = api_version
         return {
             "X-Ably-Version": version,
-            "Ably-Agent": f'ably-pubsub-python/{lib_version} python/{platform.python_version()}'
+            "Ably-Agent": HttpUtils.agent_string(agents),
         }
 
+    @staticmethod
+    def agent_string(agents=None):
+        """The `Ably-Agent` value: the library and platform, then each of `agents` (RSC7d6).
+
+        An entry with no version is emitted bare: versionless identifiers are flags, such
+        as the side a package declares, and `name/None` is not a valid Ably-Agent token.
+        """
+        agent = f'ably-pubsub-python/{lib_version} python/{platform.python_version()}'
+        for name, version in (agents or {}).items():
+            agent += f' {name}/{version}' if version else f' {name}'
+        return agent
+
     @staticmethod
     def get_query_params(options):
         params = {}

ably/pubsub/server/__init__.py

@@ -71,6 +71,7 @@
     IncompatibleClientIdException,
     InvalidArgumentException,
 )
+from ably.pubsub.util.side import SERVER_AGENT_IDENTIFIER, options_with_side_agent
 from ably.pubsub.vcdiff.defaultvcdiffdecoder import AblyVCDiffDecoder
 from ably.pubsub.version import api_version, lib_version
 
@@ -184,6 +185,10 @@ def create_http_client(**kwargs) -> PubSubHttpClient:
             The largest total size of the messages one publish may carry, counting each
             message's name, data, client ID and extras. A larger publish raises an
             AblyException with code 40009 and is not sent. The default is 65536 (64 KiB).
+        agents: dict[str, str | None]
+            Additional entries for the `Ably-Agent` header, mapping a product name to its
+            version, or to None for a versionless entry. For an SDK built on this one to
+            identify itself. The client always adds `ably-pubsub-server` as well.
 
     Returns
     -------
@@ -204,7 +209,7 @@ def create_http_client(**kwargs) -> PubSubHttpClient:
         option's 3.x name, which the message names the replacement for.
     """
     with factory_construction():
-        return _DefaultPubSubHttpClient(**kwargs)
+        return _DefaultPubSubHttpClient(**options_with_side_agent(kwargs, SERVER_AGENT_IDENTIFIER))
 
 
 def create_realtime_client(**kwargs) -> PubSubRealtimeClient:
@@ -280,7 +285,7 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient:
         ValueError.
     """
     with factory_construction():
-        return _DefaultPubSubRealtimeClient(**kwargs)
+        return _DefaultPubSubRealtimeClient(**options_with_side_agent(kwargs, SERVER_AGENT_IDENTIFIER))
 
 
 __all__ = [

ably/pubsub/server/sync.py

@@ -56,6 +56,7 @@
     IncompatibleClientIdException,
     InvalidArgumentException,
 )
+from ably.pubsub.sync.util.side import SERVER_AGENT_IDENTIFIER, options_with_side_agent
 from ably.pubsub.sync.vcdiff.defaultvcdiffdecoder import AblyVCDiffDecoder
 from ably.pubsub.version import api_version, lib_version
 
@@ -82,7 +83,7 @@ def create_http_client(**kwargs) -> PubSubHttpClient:
         ValueError.
     """
     with factory_construction():
-        return _DefaultPubSubHttpClientSync(**kwargs)
+        return _DefaultPubSubHttpClientSync(**options_with_side_agent(kwargs, SERVER_AGENT_IDENTIFIER))
 
 
 __all__ = [

ably/pubsub/transport/websockettransport.py

@@ -88,7 +88,7 @@ def __select_connect_func(options):
         return ws_connect
 
     def connect(self):
-        headers = HttpUtils.default_headers()
+        headers = HttpUtils.default_headers(agents=self.options._agents)
         query_params = urllib.parse.urlencode(self.params)
         scheme = 'wss' if self.options.tls else 'ws'
         port = Defaults.get_port(self.options)

ably/pubsub/types/options.py

@@ -117,7 +117,7 @@ def __init__(self, client_id=None, tls=True, port=0,
                  suspended_retry_timeout_seconds=None, connectivity_check_url=None,
                  channel_retry_timeout_seconds=None, add_request_ids=False,
                  vcdiff_decoder: VCDiffDecoder = None, transport_params=None, echo_messages=True,
-                 max_message_size=None, _test_options=None, **kwargs):
+                 max_message_size=None, agents=None, _test_options=None, **kwargs):
 
         _reject_renamed_timing_options(kwargs)
         super().__init__(**kwargs)
@@ -210,6 +210,15 @@ def __init__(self, client_id=None, tls=True, port=0,
         self.__vcdiff_decoder = vcdiff_decoder
         self.__transport_params = transport_params or {}
         self.__max_message_size = max_message_size
+        if agents is not None and not isinstance(agents, dict):
+            raise InvalidArgumentException(
+                f'agents must be a dict of product name to version or None, not {type(agents).__name__}',
+                400, 40000)
+        for name, version in (agents or {}).items():
+            if not name or any(c in f'{name}{version or ""}' for c in ' /'):
+                raise InvalidArgumentException(
+                    f'agents entry {name!r}: {version!r} must not contain spaces or "/"', 400, 40000)
+        self.__agents = dict(agents or {})
         self.__test_options = _test_options
         self.__hosts = self.__get_hosts()
 
@@ -393,6 +402,15 @@ def transport_params(self):
     def max_message_size(self):
         return self.__max_message_size
 
+    @property
+    def agents(self):
+        return dict(self.__agents)
+
+    @property
+    def _agents(self):
+        """The agents for building the Ably-Agent header, without a defensive copy per request."""
+        return self.__agents
+
     @property
     def _test_options(self):
         return self.__test_options

ably/pubsub/util/side.py

@@ -0,0 +1,37 @@
+"""The agent entry a package stamps to declare which side it runs on.
+
+Nothing here may grow into a general abstraction over the client: it exists only to
+stamp the side a package's factories declare.
+"""
+
+# The `-server` suffix on this identifier is load-bearing, not cosmetic. On API-key auth
+# the realtime system grants the server exemption from monthly-active-user counting by
+# matching an agent entry ending in `-server`, and an identifier that is not yet in the
+# ably-common registry is classified by that suffix alone. Renaming it without preserving
+# its suffix silently reclassifies every client the package constructs as a device.
+#
+# It matches the identifier the other Pub/Sub SDKs' server packages send, and lives here
+# rather than in `ably.pubsub.server` so the naming scheme can be changed in one place.
+SERVER_AGENT_IDENTIFIER = 'ably-pubsub-server'
+
+
+def options_with_side_agent(kwargs, identifier):
+    """A copy of the caller's client options carrying the agent entry that declares the side.
+
+    Only `agents` is replaced, with a new dict rather than by mutation, so the caller's
+    options and their own `agents` dict are both left untouched. Every other option is
+    copied by reference.
+
+    The caller's `agents` entries are kept alongside the side stamp, so an SDK layered on
+    top of this package keeps its attribution. The side stamp is applied last and so wins
+    a collision on its own identifier: which side the package declares is the package's to
+    state, not the caller's to redefine.
+
+    The side entry is versionless — a flag — because the identifier is shared by every
+    Pub/Sub SDK, so a version could not say what it versions, and would only duplicate the
+    `ably-pubsub-python` entry sent alongside it.
+    """
+    agents = kwargs.get('agents')
+    if agents is not None and not isinstance(agents, dict):
+        return kwargs  # left for Options to reject with a message naming the option
+    return {**kwargs, 'agents': {**(agents or {}), identifier: None}}

test/ably/http/httpauth_test.py

@@ -14,7 +14,13 @@
 from ably.pubsub.server import AblyAuthException, Auth, create_http_client
 from ably.pubsub.types.tokendetails import TokenDetails
 from test.ably.testapp import TestApp
-from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, dont_vary_protocol
+from test.ably.utils import (
+    NATIVE_TOKEN_AUTH_UNSUPPORTED,
+    BaseAsyncTestCase,
+    VaryByProtocolTestsMetaclass,
+    dont_vary_protocol,
+)
+from test.uts.helpers.sandbox import jwt_claims
 
 log = logging.getLogger(__name__)
 
@@ -184,6 +190,7 @@ async def test_if_authorize_changes_auth_mechanism_to_token(self):
 
     # RSA10a
     @dont_vary_protocol
+    @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED)
     async def test_authorize_always_creates_new_token(self):
         await self.ably.auth.authorize({'capability': {'test': ['publish']}})
         await self.ably.channels.get('test').publish('event', 'data')
@@ -218,6 +225,7 @@ async def test_authorize_adheres_to_request_token(self):
         for arg, value in auth_params.items():
             assert auth_called[arg] == value, f"{arg} called with wrong value: {value}"
 
+    @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED)
     async def test_with_token_str_https(self):
         token = await self.ably.auth.authorize()
         token = token.token
@@ -226,6 +234,7 @@ async def test_with_token_str_https(self):
         await ably.channels.get('test_auth_with_token_str').publish('event', 'foo_bar')
         await ably.close()
 
+    @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED)
     async def test_with_token_str_http(self):
         token = await self.ably.auth.authorize()
         token = token.token
@@ -301,12 +310,19 @@ async def test_timestamp_is_not_stored(self):
     async def test_client_id_precedence(self):
         client_id = uuid.uuid4().hex
         overridden_client_id = uuid.uuid4().hex
+
+        # The client's clientId is what the library passes to the callback, ahead of the default token params
+        async def auth_callback(token_params):
+            assert token_params['client_id'] == client_id
+            return await TestApp.get_jwt_for(token_params)
+
         ably = await TestApp.get_ably_rest(
             use_binary_protocol=self.use_binary_protocol,
             client_id=client_id,
+            auth_callback=auth_callback,
             default_token_params={'client_id': overridden_client_id})
         token = await ably.auth.authorize()
-        assert token.client_id == client_id
+        assert jwt_claims(token.token)['x-ably-clientId'] == client_id
         assert ably.auth.client_id == client_id
 
         channel = ably.channels[
@@ -326,6 +342,7 @@ async def setup(self):
     def per_protocol_setup(self, use_binary_protocol):
         self.use_binary_protocol = use_binary_protocol
 
+    @pytest.mark.skip(reason=NATIVE_TOKEN_AUTH_UNSUPPORTED)
     async def test_with_key(self):
         ably = await TestApp.get_ably_rest(use_binary_protocol=self.use_binary_protocol)
 

test/ably/http/httpchannelpublish_test.py

@@ -18,6 +18,7 @@
 from test.ably import utils
 from test.ably.testapp import TestApp
 from test.ably.utils import BaseAsyncTestCase, VaryByProtocolTestsMetaclass, assert_waiter, dont_vary_protocol
+from test.uts.helpers.sandbox import jwt_claims
 
 log = logging.getLogger(__name__)
 
@@ -31,7 +32,12 @@ async def setup(self):
         self.test_vars = await TestApp.get_test_vars()
         self.ably = await TestApp.get_ably_rest()
         self.client_id = uuid.uuid4().hex
-        self.ably_with_client_id = await TestApp.get_ably_rest(client_id=self.client_id, use_token_auth=True)
+
+        async def auth_callback(token_params):
+            return await TestApp.get_jwt(client_id=self.client_id)
+
+        self.ably_with_client_id = await TestApp.get_ably_rest(client_id=self.client_id,
+                                                               auth_callback=auth_callback)
         yield
         await self.ably.close()
         await self.ably_with_client_id.close()
@@ -118,9 +124,11 @@ async def test_message_list_generate_one_request(self):
             assert message['data'] == str(i)
 
     async def test_publish_error(self):
-        ably = await TestApp.get_ably_rest(use_binary_protocol=self.use_binary_protocol)
-        await ably.auth.authorize(
-            token_params={'capability': {"only_subscribe": ["subscribe"]}})
+        async def auth_callback(token_params):
+            return await TestApp.get_jwt(capability={"only_subscribe": ["subscribe"]})
+
+        ably = await TestApp.get_ably_rest(use_binary_protocol=self.use_binary_protocol,
+                                           auth_callback=auth_callback)
 
         with pytest.raises(AblyException) as excinfo:
             await ably.channels["only_subscribe"].publish()
@@ -234,8 +242,9 @@ async def test_token_is_bound_to_options_client_id_after_publish(self):
         await channel.publish(name='publish', data='test')
 
         # defined after publish
-        assert isinstance(self.ably_with_client_id.auth.token_details, TokenDetails)
-        assert self.ably_with_client_id.auth.token_details.client_id == self.client_id
+        token_details = self.ably_with_client_id.auth.token_details
+        assert isinstance(token_details, TokenDetails)
+        assert jwt_claims(token_details.token)['x-ably-clientId'] == self.client_id
         assert self.ably_with_client_id.auth.auth_mechanism == Auth.Method.TOKEN
         history = await channel.history()
         assert history.items[0].client_id == self.client_id
@@ -254,7 +263,7 @@ async def test_publish_message_without_client_id_on_identified_client(self):
             assert messages is not None, "Expected non-None messages"
             assert len(messages) == 1, "Expected 1 message"
 
-            assert post_mock.call_count == 2
+            assert post_mock.call_count == 1
 
             if self.use_binary_protocol:
                 posted_body = msgpack.unpackb(
@@ -295,9 +304,9 @@ async def test_publish_message_with_client_id_on_identified_client(self):
             await channel.publish(message)
 
     async def test_publish_message_with_wrong_client_id_on_implicit_identified_client(self):
-        new_token = await self.ably.auth.authorize(token_params={'client_id': uuid.uuid4().hex})
+        token = await TestApp.get_jwt(client_id=uuid.uuid4().hex)
         new_ably = await TestApp.get_ably_rest(key=None,
-                                               token=new_token.token,
+                                               token=token,
                                                use_binary_protocol=self.use_binary_protocol)
 
         channel = new_ably.channels[
@@ -313,13 +322,12 @@ async def test_publish_message_with_wrong_client_id_on_implicit_identified_clien
 
     # RSA15b
     async def test_wildcard_client_id_can_publish_as_others(self):
-        wildcard_token_details = await self.ably.auth.request_token({'client_id': '*'})
+        wildcard_token = await TestApp.get_jwt(client_id='*')
         wildcard_ably = await TestApp.get_ably_rest(
             key=None,
-            token_details=wildcard_token_details,
+            token=wildcard_token,
             use_binary_protocol=self.use_binary_protocol)
 
-        assert wildcard_ably.auth.client_id == '*'
         channel = wildcard_ably.channels[
             self.get_channel_name('persisted:wildcard_client_id')]
         await channel.publish(name='publish1', data='no client_id')

test/ably/http/httphttp_test.py

@@ -195,7 +195,7 @@ async def test_request_headers(self):
 
         # Agent
         assert 'Ably-Agent' in r.request.headers
-        expr = r"^ably-pubsub-python\/\d+.\d+.\d+(-beta\.\d+)? python\/\d.\d+.\d+$"
+        expr = r"^ably-pubsub-python\/\d+.\d+.\d+(-beta\.\d+)? python\/\d.\d+.\d+ ably-pubsub-server$"
         assert re.search(expr, r.request.headers['Ably-Agent'])
         await ably.close()