feat: ruleset_list(sort=) for the server's active-first order (4.5.0)

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "polyswarm_api"
-version = "4.4.0"
+version = "4.5.0"
 description = "Client library to simplify interacting with the PolySwarm consumer API"
 readme = "README.md"
 requires-python = ">=3.10,<4"
@@ -55,7 +55,7 @@ package-dir = { "" = "src" }
 where = ["src"]
 
 [tool.bumpversion]
-current_version = "4.4.0"
+current_version = "4.5.0"
 commit = true
 tag = false
 sign_tags = true

specs/03-endpoints.md

@@ -190,7 +190,7 @@ refusal.
 | `live_feed(since=None, …, livescan_id=None, max_results=None)` | `LiveHuntResult.list` — `livescan_id` scopes the feed to one live hunt (the hunt-page per-ruleset feed); `since` is in **SECONDS** (the server converts with `timedelta(seconds=since)`; the 3.x/4.x docstring said minutes and was wrong), and absent-or-`0` means no time filter at all — the server applies it on a truthiness test; `max_results` bounds how many results the generator yields — `None`/`0`/negative means no bound; it does not alter the request |
 | `historical_list(since=None)` | `HistoricalHunt.list` |
 | `historical_results(hunt=None, …)` | `HistoricalHuntResultList.get` |
-| `ruleset_list(name=None, status=None, favorites_only=None, has_new_results=None)` | `YaraRuleset.list` — the hunt-page filters, conjunctive and optional; unset filters are omitted from the query so the no-filter request is byte-compatible with the old contract. `has_new_results` selects on the server's STORED counter (no window parameter — the window belongs to the server's scheduled refresh; rows carry `new_results_count` + `new_results_counted_at`) |
+| `ruleset_list(name=None, status=None, favorites_only=None, has_new_results=None, sort=None, exclude_favorites=None)` | `YaraRuleset.list` — the hunt-page filters, conjunctive and optional; unset filters are omitted from the query so the no-filter request is byte-compatible with the old contract. `exclude_favorites=True` is the inverse of `favorites_only` and refused together with it — it exists for clients that render the favorites as their own list, where leaving them in the paginated list too makes a page repeat a row or come back short. Appended to the signature rather than placed beside `favorites_only`, so a positional caller keeps working. `has_new_results` selects on the server's STORED counter (no window parameter — the window belongs to the server's scheduled refresh; rows carry `new_results_count` + `new_results_counted_at`). `sort='active_first'` (4.5.0) asks the SERVER for the hunt page's order — rulesets carrying a live hunt link first, newest first within each block — as an opt-in token; unset sends no `sort`, keeping the default newest-first. The SDK never re-orders rows: the list is keyset-paginated, so a client-side sort would reorder one page and lie about the rest. The rank is the stored link, a WIDER predicate than the one `livescan_id` is rendered under, so a legacy row whose hunt was stopped without clearing the link leads the list while serializing `livescan_id` as `null` — read the field, not the position. The rendered `id` is unique but UNORDERED (the server renders a random `number`, and orders on its own insertion key), so dedupe with it and never resume or bound a walk with it. The key is also MUTABLE, unlike that default — a ruleset whose live hunt stops mid-walk is yielded twice, one started mid-walk is skipped, in any walk including a fresh one — and the generator does not dedupe; callers consuming more than one page dedupe by `id`. |
 | `tag_list()` | `Tag.list` |
 | `family_list()` | `MalwareFamily.list` |
 | `assertions_list(engine_id)` | `AssertionsJob.list` |

src/polyswarm_api/__init__.py

@@ -1,5 +1,5 @@
 # https://www.python.org/dev/peps/pep-0008/#module-level-dunder-names
-__version__ = '4.4.0'
+__version__ = '4.5.0'
 __release_url__ = 'https://api.github.com/repos/polyswarm/polyswarm-api/releases/latest'
 
 from . import api

src/polyswarm_api/aio/api.py

@@ -697,7 +697,8 @@ async def ruleset_delete(self, ruleset_id):
         return await self._single(resources.YaraRuleset.delete(self, id=ruleset_id, community=self.community))
 
     async def ruleset_list(self, name=None, status=None, favorites_only=None,
-                           has_new_results=None):
+                           has_new_results=None, sort=None,
+                           exclude_favorites=None):
         """
         List all YaraRulesets for the current account.
 
@@ -706,17 +707,53 @@ async def ruleset_list(self, name=None, status=None, favorites_only=None,
         :param status: 'active' returns only rulesets whose live hunt is
             currently running.
         :param favorites_only: True returns only favorited rulesets.
+        :param exclude_favorites: True returns only the rulesets that are NOT
+            favorited — the inverse of ``favorites_only``, and refused together
+            with it (a contradiction, answered with an error rather than an
+            empty list). It exists for clients that render the favorites as
+            their own list: the favorites are a separate, unpaginated fetch
+            bounded by the account's budget, so leaving them in the paginated
+            list too makes a page either repeat a row or come back short.
+            Appended to the signature rather than placed beside
+            ``favorites_only`` so a positional caller keeps working.
         :param has_new_results: True returns only rulesets whose stored
             new-results counter is positive. The counter (and its window) is
             maintained server-side by a scheduled refresh; rows carry it as
             ``new_results_count`` with ``new_results_counted_at`` marking when
             it was last refreshed. There is no per-request window parameter.
+        :param sort: ``'active_first'`` returns the rulesets that carry a live
+            hunt link first, newest first within each block. Default (None) is
+            newest first. "Newest first" is the server's own insertion key, NOT
+            the ``id`` on the rows you get back — that one is unique but
+            unordered, so dedupe with it and never resume or bound a walk with
+            it. Applied SERVER-side, across pages — the list is
+            keyset-paginated, so a client-side sort would only ever reorder one
+            page; the SDK never re-orders rows. Reuse a page's ``offset`` only
+            with the same ``sort``: the server refuses a cursor minted under
+            the other order.
+
+            Two server-side properties of that key, neither of them SDK
+            behaviour. It ranks on the stored link, which is a WIDER predicate
+            than the one ``livescan_id`` is rendered under: a legacy row whose
+            hunt was stopped without clearing the link ranks in the leading
+            block while still serializing ``livescan_id`` as ``None``. Read the
+            field to decide whether a ruleset is running; never the position.
+
+            And the key is MUTABLE, unlike that default: a ruleset whose
+            live hunt stops mid-walk falls back into the idle block below the
+            cursor and is yielded twice, and one started mid-walk moves above
+            the cursor and is skipped for the rest of that walk. That is a
+            property of the walk, so starting fresh from the first page does
+            not avoid it. This generator streams pages and does not dedupe —
+            dedupe by ``id`` if you consume more than one page.
         :return: A generator of YaraRuleset resources
         """
         logger.info('List rulesets')
         async for item in self._paginate(resources.YaraRuleset.list(
                 self, name=name, status=status, favorites_only=favorites_only,
-                has_new_results=has_new_results, community=self.community)):
+                has_new_results=has_new_results, sort=sort,
+                exclude_favorites=exclude_favorites,
+                community=self.community)):
             yield item
 
     async def ruleset_favorite(self, ruleset_id, favorite=True):

src/polyswarm_api/api.py

@@ -837,7 +837,13 @@ def ruleset_delete(self, ruleset_id):
         )
 
     def ruleset_list(
-        self, name=None, status=None, favorites_only=None, has_new_results=None
+        self,
+        name=None,
+        status=None,
+        favorites_only=None,
+        has_new_results=None,
+        sort=None,
+        exclude_favorites=None,
     ):
         """
         List all YaraRulesets for the current account.
@@ -847,11 +853,45 @@ def ruleset_list(
         :param status: 'active' returns only rulesets whose live hunt is
             currently running.
         :param favorites_only: True returns only favorited rulesets.
+        :param exclude_favorites: True returns only the rulesets that are NOT
+            favorited — the inverse of ``favorites_only``, and refused together
+            with it (a contradiction, answered with an error rather than an
+            empty list). It exists for clients that render the favorites as
+            their own list: the favorites are a separate, unpaginated fetch
+            bounded by the account's budget, so leaving them in the paginated
+            list too makes a page either repeat a row or come back short.
+            Appended to the signature rather than placed beside
+            ``favorites_only`` so a positional caller keeps working.
         :param has_new_results: True returns only rulesets whose stored
             new-results counter is positive. The counter (and its window) is
             maintained server-side by a scheduled refresh; rows carry it as
             ``new_results_count`` with ``new_results_counted_at`` marking when
             it was last refreshed. There is no per-request window parameter.
+        :param sort: ``'active_first'`` returns the rulesets that carry a live
+            hunt link first, newest first within each block. Default (None) is
+            newest first. "Newest first" is the server's own insertion key, NOT
+            the ``id`` on the rows you get back — that one is unique but
+            unordered, so dedupe with it and never resume or bound a walk with
+            it. Applied SERVER-side, across pages — the list is
+            keyset-paginated, so a client-side sort would only ever reorder one
+            page; the SDK never re-orders rows. Reuse a page's ``offset`` only
+            with the same ``sort``: the server refuses a cursor minted under
+            the other order.
+
+            Two server-side properties of that key, neither of them SDK
+            behaviour. It ranks on the stored link, which is a WIDER predicate
+            than the one ``livescan_id`` is rendered under: a legacy row whose
+            hunt was stopped without clearing the link ranks in the leading
+            block while still serializing ``livescan_id`` as ``None``. Read the
+            field to decide whether a ruleset is running; never the position.
+
+            And the key is MUTABLE, unlike that default: a ruleset whose
+            live hunt stops mid-walk falls back into the idle block below the
+            cursor and is yielded twice, and one started mid-walk moves above
+            the cursor and is skipped for the rest of that walk. That is a
+            property of the walk, so starting fresh from the first page does
+            not avoid it. This generator streams pages and does not dedupe —
+            dedupe by ``id`` if you consume more than one page.
         :return: A generator of YaraRuleset resources
         """
         logger.info("List rulesets")
@@ -862,6 +902,8 @@ def ruleset_list(
                 status=status,
                 favorites_only=favorites_only,
                 has_new_results=has_new_results,
+                sort=sort,
+                exclude_favorites=exclude_favorites,
                 community=self.community,
             )
         ):

test/async_client_test.py

@@ -607,6 +607,47 @@ async def test_async_sample(self, uid):
 
     # ── YARA Rulesets ─────────────────────────────────────────────────────────
 
+    @vcr.use_cassette()
+    async def test_async_rules_sort_active_first(self, uid):
+        """Async twin of the sync ``test_rules_sort_active_first``: the
+        canonical transport must send the same token and read the same
+        server-applied order."""
+        async with self._api() as api:
+            running = await api.ruleset_create(f'{uid}-running', uid_yara(f'{uid}-running'))
+            idle = None
+            try:
+                idle = await api.ruleset_create(f'{uid}-idle', uid_yara(f'{uid}-idle'))
+                await api.live_start(int(running.id))
+                try:
+                    async def _enabled():
+                        return (await api.ruleset_get(running.id)).livescan_id is not None
+                    assert await poll_equals_async(_enabled, True)
+
+                    async def _running_precedes_idle(**kwargs):
+                        # Membership-tolerant on purpose — see the sync twin:
+                        # a replica missing `idle` must read as "not yet true"
+                        # and be retried, not raise out of the poll.
+                        ids = [r.id async for r in api.ruleset_list(**kwargs)]
+                        if running.id not in ids or idle.id not in ids:
+                            return None
+                        return ids.index(running.id) < ids.index(idle.id)
+
+                    async def _sorted():
+                        return await _running_precedes_idle(sort='active_first')
+                    assert await poll_equals_async(_sorted, True)
+                    # Polled like the sorted arm — see the sync twin.
+                    async def _unsorted():
+                        return await _running_precedes_idle()
+                    assert await poll_equals_async(_unsorted, False) is False
+                    with pytest.raises(exceptions.RequestException):
+                        _ = [r async for r in api.ruleset_list(sort='bogus')]
+                finally:
+                    await api.live_stop(int(running.id))
+            finally:
+                await api.ruleset_delete(int(running.id))
+                if idle is not None:
+                    await api.ruleset_delete(int(idle.id))
+
     @vcr.use_cassette()
     async def test_async_rules(self, uid):
         async with self._api() as api:

test/client_scan_test.py

@@ -601,6 +601,74 @@ def test_historical_results(self):
         except (exceptions.NotFoundException, exceptions.NoResultsException):
             pass
 
+    @vcr.use_cassette()
+    # NO e2e arm for `exclude_favorites`, deliberately and with a cost.
+    # specs/04 invariant 1 wants endpoint behaviour tested against the real
+    # server, and the reason is spelled out below: the server ignores unknown
+    # query args, so a renamed token leaves builder tests green and the list
+    # unfiltered. What covers it instead:
+    #   * `TestRulesetListSortOnTheWire` drives BOTH client methods and fails if
+    #     either stops forwarding the keyword (verified by deleting the
+    #     pass-through: one test fails, the rest stay green);
+    #   * the server side pins the filter itself, and the 400 for the
+    #     contradictory pair, in its own HTTP suite against a real database.
+    # What stays uncovered is a rename that both sides make in lockstep with
+    # the server's spelling — the case only a live request catches. Recording
+    # the cassette needs a stack whose AKM carries the fixture account; ours
+    # answers 500 for a hand-seeded one, so it is honest to say this is
+    # missing rather than to fake a recording.
+    def test_rules_sort_active_first(self):
+        """``sort='active_first'`` is an order the SERVER applies: two rulesets
+        owned by this test, the older one with a live hunt running, the newer
+        one idle. Newest-first (the default) puts the idle one ahead; the
+        active-first order puts the running one ahead — a relation the server
+        must actually satisfy, which no pure-unit test can express (the server
+        ignores unknown query args, so a renamed token would leave the builder
+        tests green and the list unsorted). Relative positions only: the
+        shared stack carries other tests' rulesets."""
+        api = PolyswarmAPI(self.test_api_key, uri=f'http://ai:9696/{self.api_version}', community='gamma')
+        uid = self._testMethodName
+        running = api.ruleset_create(f'{uid}-running', uid_yara(f'{uid}-running'))
+        idle = None
+        try:
+            idle = api.ruleset_create(f'{uid}-idle', uid_yara(f'{uid}-idle'))
+            api.live_start(int(running.id))
+            try:
+                # the enable lands asynchronously and reads come off the
+                # replica — poll (specs/04)
+                assert poll_equals(
+                    lambda: api.ruleset_get(running.id).livescan_id is not None, True)
+
+                def _running_precedes_idle(**kwargs):
+                    # Membership-tolerant on purpose: this is polled, and a
+                    # replica that has not applied `idle` yet must read as "not
+                    # yet true" and be retried. `.index()` would raise
+                    # ValueError, which poll_equals does not absorb, and the
+                    # lag the poll exists for would surface as an error on the
+                    # first attempt instead.
+                    ids = [r.id for r in api.ruleset_list(**kwargs)]
+                    if running.id not in ids or idle.id not in ids:
+                        return None
+                    return ids.index(running.id) < ids.index(idle.id)
+
+                assert poll_equals(lambda: _running_precedes_idle(sort='active_first'), True)
+                # The default order is untouched: the newer (idle) ruleset
+                # first. Polled like the sorted arm above — the helper returns
+                # None while either row is missing, so an unpolled read would
+                # assert `None is False` on a lagging replica instead of
+                # retrying. `want=False` is not None, so poll_equals accepts it.
+                assert poll_equals(_running_precedes_idle, False) is False
+                # a sort the server does not know is refused, never ignored
+                with self.assertRaises(exceptions.RequestException):
+                    list(api.ruleset_list(sort='bogus'))
+            finally:
+                # a running live hunt blocks deletion server-side
+                api.live_stop(int(running.id))
+        finally:
+            api.ruleset_delete(int(running.id))
+            if idle is not None:
+                api.ruleset_delete(int(idle.id))
+
     @vcr.use_cassette()
     def test_rules(self):
         api = PolyswarmAPI(self.test_api_key, uri=f'http://ai:9696/{self.api_version}', community='gamma')