From 9ab77b3071d7f0cc48398d507467846c6d3579d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mart=C3=ADnez?= Date: Tue, 1 Sep 2026 20:54:28 -0400 Subject: [PATCH 1/9] =?UTF-8?q?feat:=20ruleset=5Flist(sort=3D)=20=E2=80=94?= =?UTF-8?q?=20the=20hunt=20page's=20active-first=20order,=20server-side?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ruleset_list(sort='active_first')` asks the server for rulesets with a running live hunt first — as recorded by the server's live-hunt link, the same one `livescan_id` renders from — newest first within each block (`GET /v3/hunt/rule/list?sort=active_first`). Unset sends no `sort`, so the request stays byte-compatible with the pre-sort contract and the list keeps its newest-first default. The SDK never re-orders rows: the list is keyset-paginated, so a client-side sort would reorder one page and misrepresent the rest; a page's `offset` is only valid under the same `sort`, and the server refuses a cursor minted under the other order. Canonical change in aio/api.py; api.py is the regenerated unasync mirror (scripts/regenerate_sync.py, ruff on PATH). YaraRuleset.list already forwards arbitrary keywords through core._params, so the resource needs no change. Tests on two tiers. Pure-unit pins the wire shape on both transports, the omitted default, composition with the filters, and that `_next_page` carries `sort` onto page 2. Live-e2e (sync + async, cassettes recorded against a stack running the server branch) pins what no builder test can: the server actually applies the order — this test's running ruleset precedes its newer idle one under `sort='active_first'` and follows it under the default — and refuses an unknown sort rather than ignoring it. --- specs/03-endpoints.md | 2 +- src/polyswarm_api/aio/api.py | 13 +- src/polyswarm_api/api.py | 16 +- test/async_client_test.py | 33 ++ test/client_scan_test.py | 41 ++ test/hunt_tracking_builder_test.py | 94 ++++ .../test_async_rules_sort_active_first.vcr | 460 ++++++++++++++++++ test/vcr/test_rules_sort_active_first.vcr | 460 ++++++++++++++++++ 8 files changed, 1115 insertions(+), 4 deletions(-) create mode 100644 test/vcr/test_async_rules_sort_active_first.vcr create mode 100644 test/vcr/test_rules_sort_active_first.vcr diff --git a/specs/03-endpoints.md b/specs/03-endpoints.md index c7762522..9dca1067 100644 --- a/specs/03-endpoints.md +++ b/specs/03-endpoints.md @@ -190,7 +190,7 @@ refusal. | `live_feed(since=None, …, livescan_id=None, max_results=None)` | `LiveHuntResult.list` — `livescan_id` scopes the feed to one live hunt (the hunt-page per-ruleset feed); `since` is in **SECONDS** (the server converts with `timedelta(seconds=since)`; the 3.x/4.x docstring said minutes and was wrong), and absent-or-`0` means no time filter at all — the server applies it on a truthiness test; `max_results` bounds how many results the generator yields — `None`/`0`/negative means no bound; it does not alter the request | | `historical_list(since=None)` | `HistoricalHunt.list` | | `historical_results(hunt=None, …)` | `HistoricalHuntResultList.get` | -| `ruleset_list(name=None, status=None, favorites_only=None, has_new_results=None)` | `YaraRuleset.list` — the hunt-page filters, conjunctive and optional; unset filters are omitted from the query so the no-filter request is byte-compatible with the old contract. `has_new_results` selects on the server's STORED counter (no window parameter — the window belongs to the server's scheduled refresh; rows carry `new_results_count` + `new_results_counted_at`) | +| `ruleset_list(name=None, status=None, favorites_only=None, has_new_results=None, sort=None)` | `YaraRuleset.list` — the hunt-page filters, conjunctive and optional; unset filters are omitted from the query so the no-filter request is byte-compatible with the old contract. `has_new_results` selects on the server's STORED counter (no window parameter — the window belongs to the server's scheduled refresh; rows carry `new_results_count` + `new_results_counted_at`) `sort='active_first'` (4.5.0) asks the SERVER for the hunt page's order — rulesets with a running live hunt first, newest first within each block — as an opt-in token; unset sends no `sort`, keeping the default newest-first. The SDK never re-orders rows: the list is keyset-paginated, so a client-side sort would reorder one page and lie about the rest. | | `tag_list()` | `Tag.list` | | `family_list()` | `MalwareFamily.list` | | `assertions_list(engine_id)` | `AssertionsJob.list` | diff --git a/src/polyswarm_api/aio/api.py b/src/polyswarm_api/aio/api.py index 22a3ad0a..9f935676 100644 --- a/src/polyswarm_api/aio/api.py +++ b/src/polyswarm_api/aio/api.py @@ -697,7 +697,7 @@ async def ruleset_delete(self, ruleset_id): return await self._single(resources.YaraRuleset.delete(self, id=ruleset_id, community=self.community)) async def ruleset_list(self, name=None, status=None, favorites_only=None, - has_new_results=None): + has_new_results=None, sort=None): """ List all YaraRulesets for the current account. @@ -711,12 +711,21 @@ async def ruleset_list(self, name=None, status=None, favorites_only=None, maintained server-side by a scheduled refresh; rows carry it as ``new_results_count`` with ``new_results_counted_at`` marking when it was last refreshed. There is no per-request window parameter. + :param sort: ``'active_first'`` returns the rulesets with a running + live hunt first — as recorded by the server's live-hunt link, the + same link ``livescan_id`` renders from — newest first within each + block. Default (None) is newest first. Applied SERVER-side, across + pages — the list is keyset-paginated, so a client-side sort would + only ever reorder one page; the SDK never re-orders rows. Reuse a + page's ``offset`` only with the same ``sort``: the server refuses + a cursor minted under the other order. :return: A generator of YaraRuleset resources """ logger.info('List rulesets') async for item in self._paginate(resources.YaraRuleset.list( self, name=name, status=status, favorites_only=favorites_only, - has_new_results=has_new_results, community=self.community)): + has_new_results=has_new_results, sort=sort, + community=self.community)): yield item async def ruleset_favorite(self, ruleset_id, favorite=True): diff --git a/src/polyswarm_api/api.py b/src/polyswarm_api/api.py index 3e67d983..c589b721 100644 --- a/src/polyswarm_api/api.py +++ b/src/polyswarm_api/api.py @@ -837,7 +837,12 @@ def ruleset_delete(self, ruleset_id): ) def ruleset_list( - self, name=None, status=None, favorites_only=None, has_new_results=None + self, + name=None, + status=None, + favorites_only=None, + has_new_results=None, + sort=None, ): """ List all YaraRulesets for the current account. @@ -852,6 +857,14 @@ def ruleset_list( maintained server-side by a scheduled refresh; rows carry it as ``new_results_count`` with ``new_results_counted_at`` marking when it was last refreshed. There is no per-request window parameter. + :param sort: ``'active_first'`` returns the rulesets with a running + live hunt first — as recorded by the server's live-hunt link, the + same link ``livescan_id`` renders from — newest first within each + block. Default (None) is newest first. Applied SERVER-side, across + pages — the list is keyset-paginated, so a client-side sort would + only ever reorder one page; the SDK never re-orders rows. Reuse a + page's ``offset`` only with the same ``sort``: the server refuses + a cursor minted under the other order. :return: A generator of YaraRuleset resources """ logger.info("List rulesets") @@ -862,6 +875,7 @@ def ruleset_list( status=status, favorites_only=favorites_only, has_new_results=has_new_results, + sort=sort, community=self.community, ) ): diff --git a/test/async_client_test.py b/test/async_client_test.py index 958cc768..3f3c0774 100644 --- a/test/async_client_test.py +++ b/test/async_client_test.py @@ -607,6 +607,39 @@ async def test_async_sample(self, uid): # ── YARA Rulesets ───────────────────────────────────────────────────────── + @vcr.use_cassette() + async def test_async_rules_sort_active_first(self, uid): + """Async twin of the sync ``test_rules_sort_active_first``: the + canonical transport must send the same token and read the same + server-applied order.""" + async with self._api() as api: + running = await api.ruleset_create(f'{uid}-running', uid_yara(f'{uid}-running')) + idle = None + try: + idle = await api.ruleset_create(f'{uid}-idle', uid_yara(f'{uid}-idle')) + await api.live_start(int(running.id)) + try: + async def _enabled(): + return (await api.ruleset_get(running.id)).livescan_id is not None + assert await poll_equals_async(_enabled, True) + + async def _running_precedes_idle(**kwargs): + ids = [r.id async for r in api.ruleset_list(**kwargs)] + return ids.index(running.id) < ids.index(idle.id) + + async def _sorted(): + return await _running_precedes_idle(sort='active_first') + assert await poll_equals_async(_sorted, True) + assert await _running_precedes_idle() is False + with pytest.raises(exceptions.RequestException): + _ = [r async for r in api.ruleset_list(sort='bogus')] + finally: + await api.live_stop(int(running.id)) + finally: + await api.ruleset_delete(int(running.id)) + if idle is not None: + await api.ruleset_delete(int(idle.id)) + @vcr.use_cassette() async def test_async_rules(self, uid): async with self._api() as api: diff --git a/test/client_scan_test.py b/test/client_scan_test.py index 36efc1fa..f6d3db51 100644 --- a/test/client_scan_test.py +++ b/test/client_scan_test.py @@ -601,6 +601,47 @@ def test_historical_results(self): except (exceptions.NotFoundException, exceptions.NoResultsException): pass + @vcr.use_cassette() + def test_rules_sort_active_first(self): + """``sort='active_first'`` is an order the SERVER applies: two rulesets + owned by this test, the older one with a live hunt running, the newer + one idle. Newest-first (the default) puts the idle one ahead; the + active-first order puts the running one ahead — a relation the server + must actually satisfy, which no pure-unit test can express (the server + ignores unknown query args, so a renamed token would leave the builder + tests green and the list unsorted). Relative positions only: the + shared stack carries other tests' rulesets.""" + api = PolyswarmAPI(self.test_api_key, uri=f'http://ai:9696/{self.api_version}', community='gamma') + uid = self._testMethodName + running = api.ruleset_create(f'{uid}-running', uid_yara(f'{uid}-running')) + idle = None + try: + idle = api.ruleset_create(f'{uid}-idle', uid_yara(f'{uid}-idle')) + api.live_start(int(running.id)) + try: + # the enable lands asynchronously and reads come off the + # replica — poll (specs/04) + assert poll_equals( + lambda: api.ruleset_get(running.id).livescan_id is not None, True) + + def _running_precedes_idle(**kwargs): + ids = [r.id for r in api.ruleset_list(**kwargs)] + return ids.index(running.id) < ids.index(idle.id) + + assert poll_equals(lambda: _running_precedes_idle(sort='active_first'), True) + # the default order is untouched: the newer (idle) ruleset first + assert _running_precedes_idle() is False + # a sort the server does not know is refused, never ignored + with self.assertRaises(exceptions.RequestException): + list(api.ruleset_list(sort='bogus')) + finally: + # a running live hunt blocks deletion server-side + api.live_stop(int(running.id)) + finally: + api.ruleset_delete(int(running.id)) + if idle is not None: + api.ruleset_delete(int(idle.id)) + @vcr.use_cassette() def test_rules(self): api = PolyswarmAPI(self.test_api_key, uri=f'http://ai:9696/{self.api_version}', community='gamma') diff --git a/test/hunt_tracking_builder_test.py b/test/hunt_tracking_builder_test.py index 1aefe79d..e14f8e74 100644 --- a/test/hunt_tracking_builder_test.py +++ b/test/hunt_tracking_builder_test.py @@ -238,6 +238,100 @@ def test_zero_is_sent_and_absent_is_omitted(self): omitted = resources.LiveHuntResult.list(_FakeApi(), since=None, community='gamma') assert 'since' not in omitted.params + +class TestRulesetListSortOnTheWire: + """``ruleset_list(sort='active_first')`` — the hunt page's active-first + order is an opt-in server token, and it must REACH the server + exactly as such: the unsorted call sends no ``sort`` at all (the request + stays byte-compatible with the pre-sort contract and the list keeps its + id-desc order), and the SDK never re-orders client-side — the list is + keyset-paginated, so a local sort would only ever reorder one page. + + Both transports are driven: the sync mirror (what ``polyswarm-cli`` + calls) and the canonical async source unasync generates it from.""" + + @staticmethod + def _sync_params(**kwargs): + api = PolyswarmAPI.__new__(PolyswarmAPI) + api.uri = _FakeApi.uri + api.community = _FakeApi.community + captured = {} + + def capture(request, *a, **kw): + captured.update(request.params) + captured['__url__'] = request.url + return iter(()) + + api._paginate = capture + list(api.ruleset_list(**kwargs)) + return captured + + @staticmethod + def _async_params(**kwargs): + api = PolySwarmAsyncAPI.__new__(PolySwarmAsyncAPI) + api.uri = _FakeApi.uri + api.community = _FakeApi.community + captured = {} + + async def paginate(request, *a, **kw): + captured.update(request.params) + return + yield # pragma: no cover — makes this an async generator + + api._paginate = paginate + + async def run(): + return [item async for item in api.ruleset_list(**kwargs)] + + asyncio.run(run()) + return captured + + def test_sync_sends_the_server_token_and_nothing_else_new(self): + sent = self._sync_params(sort='active_first') + assert sent['__url__'] == f'{_FakeApi.uri}/hunt/rule/list' + assert {k: v for k, v in sent.items() if k != '__url__'} == { + 'sort': 'active_first', 'community': 'gamma'} + + def test_sync_default_sends_no_sort(self): + sent = self._sync_params() + assert 'sort' not in sent + + def test_sort_composes_with_the_filters(self): + sent = self._sync_params(sort='active_first', status='active', + favorites_only=True) + assert sent['sort'] == 'active_first' + assert sent['status'] == 'active' + assert sent['favorites_only'] == 1 + + def test_async_canonical_sends_the_same_token(self): + sent = self._async_params(sort='active_first') + assert sent == {'sort': 'active_first', 'community': 'gamma'} + assert 'sort' not in self._async_params() + + def test_sort_survives_onto_the_next_page(self): + # The order is only meaningful across pages, and page 2 is built by + # _next_page cloning the descriptor's params — the one place a + # rewrite could rebuild them from scratch and drop `sort` silently. + # The _paginate stubs above never reach it, so drive it directly. + api = PolySwarmAsyncAPI.__new__(PolySwarmAsyncAPI) + api.uri = _FakeApi.uri + api.community = _FakeApi.community + dispatched = [] + + class _Session: + async def execute(self, request, *a, **kw): + dispatched.append(request) + return request + + api.session = _Session() + first = resources.YaraRuleset.list(api, sort='active_first', community='gamma') + first.offset, first.limit = 'opaque-cursor', 25 + asyncio.run(api._next_page(first)) + assert len(dispatched) == 1 + assert dispatched[0].params == {'sort': 'active_first', 'community': 'gamma', + 'offset': 'opaque-cursor', 'limit': 25} + + class TestAsyncLiveFeedMaxResults: """The CANONICAL async bound loop, not the generated mirror. diff --git a/test/vcr/test_async_rules_sort_active_first.vcr b/test/vcr/test_async_rules_sort_active_first.vcr new file mode 100644 index 00000000..dddfcdee --- /dev/null +++ b/test/vcr/test_async_rules_sort_active_first.vcr @@ -0,0 +1,460 @@ +interactions: +- request: + body: '{"yara":"rule sdk_test_async_rules_sort_active_first_running { strings: + $u = \"test_async_rules_sort_active_first-running\" condition: $u }","name":"test_async_rules_sort_active_first-running"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + content-length: + - '193' + content-type: + - application/json + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: POST + uri: http://ai:9696/v3/hunt/rule + response: + body: + string: '{"result":{"created":"2026-09-02T00:53:16.616869+00:00","deleted":false,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"43837104550931486","livescan_created":null,"livescan_id":null,"modified":"2026-09-02T00:53:16.616869+00:00","name":"test_async_rules_sort_active_first-running","rule_count":1,"yara":"rule + sdk_test_async_rules_sort_active_first_running { strings: $u = \"test_async_rules_sort_active_first-running\" + condition: $u }"},"status":"OK"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '491' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:16 GMT + server: + - gunicorn + x-billing-id: + - '111' + status: + code: 200 + message: OK +- request: + body: '{"yara":"rule sdk_test_async_rules_sort_active_first_idle { strings: $u + = \"test_async_rules_sort_active_first-idle\" condition: $u }","name":"test_async_rules_sort_active_first-idle"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + content-length: + - '184' + content-type: + - application/json + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: POST + uri: http://ai:9696/v3/hunt/rule + response: + body: + string: '{"result":{"created":"2026-09-02T00:53:17.281514+00:00","deleted":false,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"11708616356758131","livescan_created":null,"livescan_id":null,"modified":"2026-09-02T00:53:17.281514+00:00","name":"test_async_rules_sort_active_first-idle","rule_count":1,"yara":"rule + sdk_test_async_rules_sort_active_first_idle { strings: $u = \"test_async_rules_sort_active_first-idle\" + condition: $u }"},"status":"OK"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '482' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:17 GMT + server: + - gunicorn + x-billing-id: + - '111' + status: + code: 200 + message: OK +- request: + body: '{"rule_id":"43837104550931486"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + content-length: + - '31' + content-type: + - application/json + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: POST + uri: http://ai:9696/v3/hunt/rule/live + response: + body: + string: '{"result":{"created":"2026-09-02T00:53:16.616869+00:00","deleted":false,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"43837104550931486","livescan_created":"2026-09-02T00:53:18.269679+00:00","livescan_id":"20350915266052341","modified":"2026-09-02T00:53:17.891619+00:00","name":"test_async_rules_sort_active_first-running","rule_count":1,"yara":"rule + sdk_test_async_rules_sort_active_first_running { strings: $u = \"test_async_rules_sort_active_first-running\" + condition: $u }"},"status":"OK"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '536' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:18 GMT + server: + - gunicorn + x-billing-id: + - '111' + status: + code: 200 + message: OK +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: GET + uri: http://ai:9696/v3/hunt/rule?id=43837104550931486&community=gamma + response: + body: + string: '{"result":{"created":"2026-09-02T00:53:16.616869+00:00","deleted":false,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"43837104550931486","livescan_created":"2026-09-02T00:53:18.269679+00:00","livescan_id":"20350915266052341","modified":"2026-09-02T00:53:17.891619+00:00","name":"test_async_rules_sort_active_first-running","rule_count":1,"yara":"rule + sdk_test_async_rules_sort_active_first_running { strings: $u = \"test_async_rules_sort_active_first-running\" + condition: $u }"},"status":"OK"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '536' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:18 GMT + server: + - gunicorn + x-billing-id: + - '111' + status: + code: 200 + message: OK +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: GET + uri: http://ai:9696/v3/hunt/rule/list?sort=active_first&community=gamma + response: + body: + string: '{"has_more":false,"limit":50,"result":[{"created":"2026-09-02T00:53:16.616869+00:00","deleted":false,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"43837104550931486","livescan_created":"2026-09-02T00:53:18.269679+00:00","livescan_id":"20350915266052341","modified":"2026-09-02T00:53:17.891619+00:00","name":"test_async_rules_sort_active_first-running","new_results_count":null,"new_results_counted_at":null,"rule_count":1,"yara":null},{"created":"2026-09-02T00:53:17.281514+00:00","deleted":false,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"11708616356758131","livescan_created":null,"livescan_id":null,"modified":"2026-09-02T00:53:17.281514+00:00","name":"test_async_rules_sort_active_first-idle","new_results_count":null,"new_results_counted_at":null,"rule_count":1,"yara":null}],"status":"OK"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '883' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:18 GMT + server: + - gunicorn + x-billing-id: + - '111' + status: + code: 200 + message: OK +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: GET + uri: http://ai:9696/v3/hunt/rule/list?community=gamma + response: + body: + string: '{"has_more":false,"limit":50,"result":[{"created":"2026-09-02T00:53:17.281514+00:00","deleted":false,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"11708616356758131","livescan_created":null,"livescan_id":null,"modified":"2026-09-02T00:53:17.281514+00:00","name":"test_async_rules_sort_active_first-idle","new_results_count":null,"new_results_counted_at":null,"rule_count":1,"yara":null},{"created":"2026-09-02T00:53:16.616869+00:00","deleted":false,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"43837104550931486","livescan_created":"2026-09-02T00:53:18.269679+00:00","livescan_id":"20350915266052341","modified":"2026-09-02T00:53:17.891619+00:00","name":"test_async_rules_sort_active_first-running","new_results_count":null,"new_results_counted_at":null,"rule_count":1,"yara":null}],"status":"OK"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '883' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:19 GMT + server: + - gunicorn + x-billing-id: + - '111' + status: + code: 200 + message: OK +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: GET + uri: http://ai:9696/v3/hunt/rule/list?sort=bogus&community=gamma + response: + body: + string: '{"errors":null,"result":"Invalid sort: only ''active_first'' is supported.","status":"error"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '92' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:19 GMT + server: + - gunicorn + status: + code: 400 + message: BAD REQUEST +- request: + body: '{"rule_id":"43837104550931486"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + content-length: + - '31' + content-type: + - application/json + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: DELETE + uri: http://ai:9696/v3/hunt/rule/live + response: + body: + string: '{"result":{"created":"2026-09-02T00:53:16.616869+00:00","deleted":false,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"43837104550931486","livescan_created":null,"livescan_id":null,"modified":"2026-09-02T00:53:19.285097+00:00","name":"test_async_rules_sort_active_first-running","rule_count":1,"yara":"rule + sdk_test_async_rules_sort_active_first_running { strings: $u = \"test_async_rules_sort_active_first-running\" + condition: $u }"},"status":"OK"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '491' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:19 GMT + server: + - gunicorn + x-billing-id: + - '111' + status: + code: 200 + message: OK +- request: + body: '{"community":"gamma"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + content-length: + - '21' + content-type: + - application/json + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: DELETE + uri: http://ai:9696/v3/hunt/rule?id=43837104550931486 + response: + body: + string: '{"result":{"created":"2026-09-02T00:53:16.616869+00:00","deleted":true,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"43837104550931486","livescan_created":null,"livescan_id":null,"modified":"2026-09-02T00:53:19.472057+00:00","name":"test_async_rules_sort_active_first-running","rule_count":1,"yara":"rule + sdk_test_async_rules_sort_active_first_running { strings: $u = \"test_async_rules_sort_active_first-running\" + condition: $u }"},"status":"OK"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '490' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:19 GMT + server: + - gunicorn + x-billing-id: + - '111' + status: + code: 200 + message: OK +- request: + body: '{"community":"gamma"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + content-length: + - '21' + content-type: + - application/json + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: DELETE + uri: http://ai:9696/v3/hunt/rule?id=11708616356758131 + response: + body: + string: '{"result":{"created":"2026-09-02T00:53:17.281514+00:00","deleted":true,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"11708616356758131","livescan_created":null,"livescan_id":null,"modified":"2026-09-02T00:53:19.616902+00:00","name":"test_async_rules_sort_active_first-idle","rule_count":1,"yara":"rule + sdk_test_async_rules_sort_active_first_idle { strings: $u = \"test_async_rules_sort_active_first-idle\" + condition: $u }"},"status":"OK"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '481' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:19 GMT + server: + - gunicorn + x-billing-id: + - '111' + status: + code: 200 + message: OK +version: 1 diff --git a/test/vcr/test_rules_sort_active_first.vcr b/test/vcr/test_rules_sort_active_first.vcr new file mode 100644 index 00000000..d2de7789 --- /dev/null +++ b/test/vcr/test_rules_sort_active_first.vcr @@ -0,0 +1,460 @@ +interactions: +- request: + body: '{"yara":"rule sdk_test_rules_sort_active_first_running { strings: $u = + \"test_rules_sort_active_first-running\" condition: $u }","name":"test_rules_sort_active_first-running"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + content-length: + - '175' + content-type: + - application/json + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: POST + uri: http://ai:9696/v3/hunt/rule + response: + body: + string: '{"result":{"created":"2026-09-02T00:53:09.506285+00:00","deleted":false,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"91100246556341871","livescan_created":null,"livescan_id":null,"modified":"2026-09-02T00:53:09.506285+00:00","name":"test_rules_sort_active_first-running","rule_count":1,"yara":"rule + sdk_test_rules_sort_active_first_running { strings: $u = \"test_rules_sort_active_first-running\" + condition: $u }"},"status":"OK"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '473' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:09 GMT + server: + - gunicorn + x-billing-id: + - '111' + status: + code: 200 + message: OK +- request: + body: '{"yara":"rule sdk_test_rules_sort_active_first_idle { strings: $u = \"test_rules_sort_active_first-idle\" + condition: $u }","name":"test_rules_sort_active_first-idle"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + content-length: + - '166' + content-type: + - application/json + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: POST + uri: http://ai:9696/v3/hunt/rule + response: + body: + string: '{"result":{"created":"2026-09-02T00:53:09.975229+00:00","deleted":false,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"70979412008168996","livescan_created":null,"livescan_id":null,"modified":"2026-09-02T00:53:09.975229+00:00","name":"test_rules_sort_active_first-idle","rule_count":1,"yara":"rule + sdk_test_rules_sort_active_first_idle { strings: $u = \"test_rules_sort_active_first-idle\" + condition: $u }"},"status":"OK"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '464' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:10 GMT + server: + - gunicorn + x-billing-id: + - '111' + status: + code: 200 + message: OK +- request: + body: '{"rule_id":"91100246556341871"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + content-length: + - '31' + content-type: + - application/json + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: POST + uri: http://ai:9696/v3/hunt/rule/live + response: + body: + string: '{"result":{"created":"2026-09-02T00:53:09.506285+00:00","deleted":false,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"91100246556341871","livescan_created":"2026-09-02T00:53:12.090276+00:00","livescan_id":"69497058204233968","modified":"2026-09-02T00:53:10.279078+00:00","name":"test_rules_sort_active_first-running","rule_count":1,"yara":"rule + sdk_test_rules_sort_active_first_running { strings: $u = \"test_rules_sort_active_first-running\" + condition: $u }"},"status":"OK"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '518' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:12 GMT + server: + - gunicorn + x-billing-id: + - '111' + status: + code: 200 + message: OK +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: GET + uri: http://ai:9696/v3/hunt/rule?id=91100246556341871&community=gamma + response: + body: + string: '{"result":{"created":"2026-09-02T00:53:09.506285+00:00","deleted":false,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"91100246556341871","livescan_created":"2026-09-02T00:53:12.090276+00:00","livescan_id":"69497058204233968","modified":"2026-09-02T00:53:10.279078+00:00","name":"test_rules_sort_active_first-running","rule_count":1,"yara":"rule + sdk_test_rules_sort_active_first_running { strings: $u = \"test_rules_sort_active_first-running\" + condition: $u }"},"status":"OK"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '518' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:12 GMT + server: + - gunicorn + x-billing-id: + - '111' + status: + code: 200 + message: OK +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: GET + uri: http://ai:9696/v3/hunt/rule/list?sort=active_first&community=gamma + response: + body: + string: '{"has_more":false,"limit":50,"result":[{"created":"2026-09-02T00:53:09.506285+00:00","deleted":false,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"91100246556341871","livescan_created":"2026-09-02T00:53:12.090276+00:00","livescan_id":"69497058204233968","modified":"2026-09-02T00:53:10.279078+00:00","name":"test_rules_sort_active_first-running","new_results_count":null,"new_results_counted_at":null,"rule_count":1,"yara":null},{"created":"2026-09-02T00:53:09.975229+00:00","deleted":false,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"70979412008168996","livescan_created":null,"livescan_id":null,"modified":"2026-09-02T00:53:09.975229+00:00","name":"test_rules_sort_active_first-idle","new_results_count":null,"new_results_counted_at":null,"rule_count":1,"yara":null}],"status":"OK"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '871' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:12 GMT + server: + - gunicorn + x-billing-id: + - '111' + status: + code: 200 + message: OK +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: GET + uri: http://ai:9696/v3/hunt/rule/list?community=gamma + response: + body: + string: '{"has_more":false,"limit":50,"result":[{"created":"2026-09-02T00:53:09.975229+00:00","deleted":false,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"70979412008168996","livescan_created":null,"livescan_id":null,"modified":"2026-09-02T00:53:09.975229+00:00","name":"test_rules_sort_active_first-idle","new_results_count":null,"new_results_counted_at":null,"rule_count":1,"yara":null},{"created":"2026-09-02T00:53:09.506285+00:00","deleted":false,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"91100246556341871","livescan_created":"2026-09-02T00:53:12.090276+00:00","livescan_id":"69497058204233968","modified":"2026-09-02T00:53:10.279078+00:00","name":"test_rules_sort_active_first-running","new_results_count":null,"new_results_counted_at":null,"rule_count":1,"yara":null}],"status":"OK"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '871' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:12 GMT + server: + - gunicorn + x-billing-id: + - '111' + status: + code: 200 + message: OK +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: GET + uri: http://ai:9696/v3/hunt/rule/list?sort=bogus&community=gamma + response: + body: + string: '{"errors":null,"result":"Invalid sort: only ''active_first'' is supported.","status":"error"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '92' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:12 GMT + server: + - gunicorn + status: + code: 400 + message: BAD REQUEST +- request: + body: '{"rule_id":"91100246556341871"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + content-length: + - '31' + content-type: + - application/json + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: DELETE + uri: http://ai:9696/v3/hunt/rule/live + response: + body: + string: '{"result":{"created":"2026-09-02T00:53:09.506285+00:00","deleted":false,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"91100246556341871","livescan_created":null,"livescan_id":null,"modified":"2026-09-02T00:53:12.776051+00:00","name":"test_rules_sort_active_first-running","rule_count":1,"yara":"rule + sdk_test_rules_sort_active_first_running { strings: $u = \"test_rules_sort_active_first-running\" + condition: $u }"},"status":"OK"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '473' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:12 GMT + server: + - gunicorn + x-billing-id: + - '111' + status: + code: 200 + message: OK +- request: + body: '{"community":"gamma"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + content-length: + - '21' + content-type: + - application/json + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: DELETE + uri: http://ai:9696/v3/hunt/rule?id=91100246556341871 + response: + body: + string: '{"result":{"created":"2026-09-02T00:53:09.506285+00:00","deleted":true,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"91100246556341871","livescan_created":null,"livescan_id":null,"modified":"2026-09-02T00:53:12.974978+00:00","name":"test_rules_sort_active_first-running","rule_count":1,"yara":"rule + sdk_test_rules_sort_active_first_running { strings: $u = \"test_rules_sort_active_first-running\" + condition: $u }"},"status":"OK"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '472' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:13 GMT + server: + - gunicorn + x-billing-id: + - '111' + status: + code: 200 + message: OK +- request: + body: '{"community":"gamma"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + authorization: + - '11111111111111111111111111111111' + connection: + - keep-alive + content-length: + - '21' + content-type: + - application/json + host: + - ai:9696 + user-agent: + - polyswarm_api/4.5.0 (x86_64-Darwin-CPython-3.11.3) + method: DELETE + uri: http://ai:9696/v3/hunt/rule?id=70979412008168996 + response: + body: + string: '{"result":{"created":"2026-09-02T00:53:09.975229+00:00","deleted":true,"description":null,"favorite":false,"favorited_at":null,"historical_hunt_count":0,"id":"70979412008168996","livescan_created":null,"livescan_id":null,"modified":"2026-09-02T00:53:13.214903+00:00","name":"test_rules_sort_active_first-idle","rule_count":1,"yara":"rule + sdk_test_rules_sort_active_first_idle { strings: $u = \"test_rules_sort_active_first-idle\" + condition: $u }"},"status":"OK"} + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - Authorization + connection: + - keep-alive + content-length: + - '463' + content-type: + - application/json + date: + - Wed, 02 Sep 2026 00:53:13 GMT + server: + - gunicorn + x-billing-id: + - '111' + status: + code: 200 + message: OK +version: 1 From 6fa413c7cc6b780b984d6fca2eb01989c6b870ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mart=C3=ADnez?= Date: Tue, 1 Sep 2026 20:54:29 -0400 Subject: [PATCH 2/9] feat: release 4.5.0, the floor the CLI now pins The CLI client adopts `ruleset_list(sort=)` in the paired change set and expresses that as `polyswarm_api>=4.5.0` rather than probing the installed SDK (the workspace's cross-repo dependency standard; AGENTS.md's standing exception). A floor cannot name a version this repo has not declared, so the bump lands here, in the feature PR, not at the release step. Minor, not major: one new optional keyword with a default that preserves today's behaviour. Bumped with bump-my-version; the emitted string is a clean `4.5.0` (a `.devN` form would sort below the floor and send the CLI's CI to PyPI for a version that does not exist). Order is forced as before: this repo releases before the CLI can. --- pyproject.toml | 4 ++-- src/polyswarm_api/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2af7a2d0..03817648 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "polyswarm_api" -version = "4.4.0" +version = "4.5.0" description = "Client library to simplify interacting with the PolySwarm consumer API" readme = "README.md" requires-python = ">=3.10,<4" @@ -55,7 +55,7 @@ package-dir = { "" = "src" } where = ["src"] [tool.bumpversion] -current_version = "4.4.0" +current_version = "4.5.0" commit = true tag = false sign_tags = true diff --git a/src/polyswarm_api/__init__.py b/src/polyswarm_api/__init__.py index dadcc971..d9cf1695 100644 --- a/src/polyswarm_api/__init__.py +++ b/src/polyswarm_api/__init__.py @@ -1,5 +1,5 @@ # https://www.python.org/dev/peps/pep-0008/#module-level-dunder-names -__version__ = '4.4.0' +__version__ = '4.5.0' __release_url__ = 'https://api.github.com/repos/polyswarm/polyswarm-api/releases/latest' from . import api From b065421cb48dde2b418b6b12277cb7ccc75ae7cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mart=C3=ADnez?= Date: Mon, 14 Sep 2026 13:10:22 -0300 Subject: [PATCH 3/9] docs: separate the sort sentence from the has_new_results clause in the endpoints table --- specs/03-endpoints.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/03-endpoints.md b/specs/03-endpoints.md index 9dca1067..9c7503e4 100644 --- a/specs/03-endpoints.md +++ b/specs/03-endpoints.md @@ -190,7 +190,7 @@ refusal. | `live_feed(since=None, …, livescan_id=None, max_results=None)` | `LiveHuntResult.list` — `livescan_id` scopes the feed to one live hunt (the hunt-page per-ruleset feed); `since` is in **SECONDS** (the server converts with `timedelta(seconds=since)`; the 3.x/4.x docstring said minutes and was wrong), and absent-or-`0` means no time filter at all — the server applies it on a truthiness test; `max_results` bounds how many results the generator yields — `None`/`0`/negative means no bound; it does not alter the request | | `historical_list(since=None)` | `HistoricalHunt.list` | | `historical_results(hunt=None, …)` | `HistoricalHuntResultList.get` | -| `ruleset_list(name=None, status=None, favorites_only=None, has_new_results=None, sort=None)` | `YaraRuleset.list` — the hunt-page filters, conjunctive and optional; unset filters are omitted from the query so the no-filter request is byte-compatible with the old contract. `has_new_results` selects on the server's STORED counter (no window parameter — the window belongs to the server's scheduled refresh; rows carry `new_results_count` + `new_results_counted_at`) `sort='active_first'` (4.5.0) asks the SERVER for the hunt page's order — rulesets with a running live hunt first, newest first within each block — as an opt-in token; unset sends no `sort`, keeping the default newest-first. The SDK never re-orders rows: the list is keyset-paginated, so a client-side sort would reorder one page and lie about the rest. | +| `ruleset_list(name=None, status=None, favorites_only=None, has_new_results=None, sort=None)` | `YaraRuleset.list` — the hunt-page filters, conjunctive and optional; unset filters are omitted from the query so the no-filter request is byte-compatible with the old contract. `has_new_results` selects on the server's STORED counter (no window parameter — the window belongs to the server's scheduled refresh; rows carry `new_results_count` + `new_results_counted_at`). `sort='active_first'` (4.5.0) asks the SERVER for the hunt page's order — rulesets with a running live hunt first, newest first within each block — as an opt-in token; unset sends no `sort`, keeping the default newest-first. The SDK never re-orders rows: the list is keyset-paginated, so a client-side sort would reorder one page and lie about the rest. | | `tag_list()` | `Tag.list` | | `family_list()` | `MalwareFamily.list` | | `assertions_list(engine_id)` | `AssertionsJob.list` | From dd43c14925cbb750d04c5bc74761acdadec89ccd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mart=C3=ADnez?= Date: Mon, 14 Sep 2026 13:22:57 -0300 Subject: [PATCH 4/9] docs: the active-first sort key is mutable, so callers paging it dedupe by id The server assigns that obligation to clients and pins it with boundary tests; it appeared nowhere on this side. Documented on the async method (the canonical source), regenerated into the sync mirror, and recorded in the endpoints table. No behaviour change: dedupe does not belong in the shared streaming generator, which every list endpoint uses and which must not grow an unbounded id set for one of them. --- specs/03-endpoints.md | 2 +- src/polyswarm_api/aio/api.py | 8 +++++++- src/polyswarm_api/api.py | 8 +++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/specs/03-endpoints.md b/specs/03-endpoints.md index 9c7503e4..7e8a6b14 100644 --- a/specs/03-endpoints.md +++ b/specs/03-endpoints.md @@ -190,7 +190,7 @@ refusal. | `live_feed(since=None, …, livescan_id=None, max_results=None)` | `LiveHuntResult.list` — `livescan_id` scopes the feed to one live hunt (the hunt-page per-ruleset feed); `since` is in **SECONDS** (the server converts with `timedelta(seconds=since)`; the 3.x/4.x docstring said minutes and was wrong), and absent-or-`0` means no time filter at all — the server applies it on a truthiness test; `max_results` bounds how many results the generator yields — `None`/`0`/negative means no bound; it does not alter the request | | `historical_list(since=None)` | `HistoricalHunt.list` | | `historical_results(hunt=None, …)` | `HistoricalHuntResultList.get` | -| `ruleset_list(name=None, status=None, favorites_only=None, has_new_results=None, sort=None)` | `YaraRuleset.list` — the hunt-page filters, conjunctive and optional; unset filters are omitted from the query so the no-filter request is byte-compatible with the old contract. `has_new_results` selects on the server's STORED counter (no window parameter — the window belongs to the server's scheduled refresh; rows carry `new_results_count` + `new_results_counted_at`). `sort='active_first'` (4.5.0) asks the SERVER for the hunt page's order — rulesets with a running live hunt first, newest first within each block — as an opt-in token; unset sends no `sort`, keeping the default newest-first. The SDK never re-orders rows: the list is keyset-paginated, so a client-side sort would reorder one page and lie about the rest. | +| `ruleset_list(name=None, status=None, favorites_only=None, has_new_results=None, sort=None)` | `YaraRuleset.list` — the hunt-page filters, conjunctive and optional; unset filters are omitted from the query so the no-filter request is byte-compatible with the old contract. `has_new_results` selects on the server's STORED counter (no window parameter — the window belongs to the server's scheduled refresh; rows carry `new_results_count` + `new_results_counted_at`). `sort='active_first'` (4.5.0) asks the SERVER for the hunt page's order — rulesets with a running live hunt first, newest first within each block — as an opt-in token; unset sends no `sort`, keeping the default newest-first. The SDK never re-orders rows: the list is keyset-paginated, so a client-side sort would reorder one page and lie about the rest. The key is MUTABLE, unlike the id-desc default — a ruleset whose live hunt stops mid-walk is yielded twice, one started mid-walk is skipped — and the generator does not dedupe; callers consuming more than one page dedupe by `id`. | | `tag_list()` | `Tag.list` | | `family_list()` | `MalwareFamily.list` | | `assertions_list(engine_id)` | `AssertionsJob.list` | diff --git a/src/polyswarm_api/aio/api.py b/src/polyswarm_api/aio/api.py index 9f935676..2531c049 100644 --- a/src/polyswarm_api/aio/api.py +++ b/src/polyswarm_api/aio/api.py @@ -718,7 +718,13 @@ async def ruleset_list(self, name=None, status=None, favorites_only=None, pages — the list is keyset-paginated, so a client-side sort would only ever reorder one page; the SDK never re-orders rows. Reuse a page's ``offset`` only with the same ``sort``: the server refuses - a cursor minted under the other order. + a cursor minted under the other order. The key is MUTABLE, unlike + the id-desc default: a ruleset whose live hunt stops mid-walk falls + back into the idle block below the cursor and is yielded twice, + and one started mid-walk moves above the cursor and is skipped for + the rest of that walk. This generator streams pages and does not + dedupe — dedupe by ``id`` if you consume more than one page; a + fresh walk from the first page is always self-consistent. :return: A generator of YaraRuleset resources """ logger.info('List rulesets') diff --git a/src/polyswarm_api/api.py b/src/polyswarm_api/api.py index c589b721..e8921b51 100644 --- a/src/polyswarm_api/api.py +++ b/src/polyswarm_api/api.py @@ -864,7 +864,13 @@ def ruleset_list( pages — the list is keyset-paginated, so a client-side sort would only ever reorder one page; the SDK never re-orders rows. Reuse a page's ``offset`` only with the same ``sort``: the server refuses - a cursor minted under the other order. + a cursor minted under the other order. The key is MUTABLE, unlike + the id-desc default: a ruleset whose live hunt stops mid-walk falls + back into the idle block below the cursor and is yielded twice, + and one started mid-walk moves above the cursor and is skipped for + the rest of that walk. This generator streams pages and does not + dedupe — dedupe by ``id`` if you consume more than one page; a + fresh walk from the first page is always self-consistent. :return: A generator of YaraRuleset resources """ logger.info("List rulesets") From c4c3b6a8561ed1fa4a4b2988903f458949d48299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mart=C3=ADnez?= Date: Mon, 14 Sep 2026 16:25:59 -0300 Subject: [PATCH 5/9] docs: the active-first rank is the stored hunt link, wider than what livescan_id renders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three separate things the sort docstring got wrong or over-promised: The rank is not "the same link livescan_id renders from". The server orders on the stored link and serializes the id under a stricter predicate, so a legacy row whose hunt was stopped without clearing the link leads the list while rendering a null id. A caller reading the leading block as "running" — or taking rows until the first null id — reads it backwards. The docstring now says to read the field and never the position. "A fresh walk from the first page is always self-consistent" was wrong in the same breath as the duplicate it describes: the mutable key repeats and skips rows WITHIN a walk, so starting fresh does not avoid it. Retracted. And the live ordering test's poll called list.index() on a row a lagging replica may not have returned yet. poll_equals absorbs NotFound/NoResults, not ValueError, so the lag the poll exists for would have errored the test on its first attempt instead of retrying. Both twins now read as "not yet". --- specs/03-endpoints.md | 2 +- src/polyswarm_api/aio/api.py | 36 ++++++++++++++++++++++-------------- src/polyswarm_api/api.py | 36 ++++++++++++++++++++++-------------- test/async_client_test.py | 5 +++++ test/client_scan_test.py | 8 ++++++++ 5 files changed, 58 insertions(+), 29 deletions(-) diff --git a/specs/03-endpoints.md b/specs/03-endpoints.md index 7e8a6b14..f7b9a51c 100644 --- a/specs/03-endpoints.md +++ b/specs/03-endpoints.md @@ -190,7 +190,7 @@ refusal. | `live_feed(since=None, …, livescan_id=None, max_results=None)` | `LiveHuntResult.list` — `livescan_id` scopes the feed to one live hunt (the hunt-page per-ruleset feed); `since` is in **SECONDS** (the server converts with `timedelta(seconds=since)`; the 3.x/4.x docstring said minutes and was wrong), and absent-or-`0` means no time filter at all — the server applies it on a truthiness test; `max_results` bounds how many results the generator yields — `None`/`0`/negative means no bound; it does not alter the request | | `historical_list(since=None)` | `HistoricalHunt.list` | | `historical_results(hunt=None, …)` | `HistoricalHuntResultList.get` | -| `ruleset_list(name=None, status=None, favorites_only=None, has_new_results=None, sort=None)` | `YaraRuleset.list` — the hunt-page filters, conjunctive and optional; unset filters are omitted from the query so the no-filter request is byte-compatible with the old contract. `has_new_results` selects on the server's STORED counter (no window parameter — the window belongs to the server's scheduled refresh; rows carry `new_results_count` + `new_results_counted_at`). `sort='active_first'` (4.5.0) asks the SERVER for the hunt page's order — rulesets with a running live hunt first, newest first within each block — as an opt-in token; unset sends no `sort`, keeping the default newest-first. The SDK never re-orders rows: the list is keyset-paginated, so a client-side sort would reorder one page and lie about the rest. The key is MUTABLE, unlike the id-desc default — a ruleset whose live hunt stops mid-walk is yielded twice, one started mid-walk is skipped — and the generator does not dedupe; callers consuming more than one page dedupe by `id`. | +| `ruleset_list(name=None, status=None, favorites_only=None, has_new_results=None, sort=None)` | `YaraRuleset.list` — the hunt-page filters, conjunctive and optional; unset filters are omitted from the query so the no-filter request is byte-compatible with the old contract. `has_new_results` selects on the server's STORED counter (no window parameter — the window belongs to the server's scheduled refresh; rows carry `new_results_count` + `new_results_counted_at`). `sort='active_first'` (4.5.0) asks the SERVER for the hunt page's order — rulesets carrying a live hunt link first, newest first within each block — as an opt-in token; unset sends no `sort`, keeping the default newest-first. The SDK never re-orders rows: the list is keyset-paginated, so a client-side sort would reorder one page and lie about the rest. The rank is the stored link, a WIDER predicate than the one `livescan_id` is rendered under, so a legacy row whose hunt was stopped without clearing the link leads the list while serializing `livescan_id` as `null` — read the field, not the position. The key is also MUTABLE, unlike the id-desc default — a ruleset whose live hunt stops mid-walk is yielded twice, one started mid-walk is skipped, in any walk including a fresh one — and the generator does not dedupe; callers consuming more than one page dedupe by `id`. | | `tag_list()` | `Tag.list` | | `family_list()` | `MalwareFamily.list` | | `assertions_list(engine_id)` | `AssertionsJob.list` | diff --git a/src/polyswarm_api/aio/api.py b/src/polyswarm_api/aio/api.py index 2531c049..fd9bb9ba 100644 --- a/src/polyswarm_api/aio/api.py +++ b/src/polyswarm_api/aio/api.py @@ -711,20 +711,28 @@ async def ruleset_list(self, name=None, status=None, favorites_only=None, maintained server-side by a scheduled refresh; rows carry it as ``new_results_count`` with ``new_results_counted_at`` marking when it was last refreshed. There is no per-request window parameter. - :param sort: ``'active_first'`` returns the rulesets with a running - live hunt first — as recorded by the server's live-hunt link, the - same link ``livescan_id`` renders from — newest first within each - block. Default (None) is newest first. Applied SERVER-side, across - pages — the list is keyset-paginated, so a client-side sort would - only ever reorder one page; the SDK never re-orders rows. Reuse a - page's ``offset`` only with the same ``sort``: the server refuses - a cursor minted under the other order. The key is MUTABLE, unlike - the id-desc default: a ruleset whose live hunt stops mid-walk falls - back into the idle block below the cursor and is yielded twice, - and one started mid-walk moves above the cursor and is skipped for - the rest of that walk. This generator streams pages and does not - dedupe — dedupe by ``id`` if you consume more than one page; a - fresh walk from the first page is always self-consistent. + :param sort: ``'active_first'`` returns the rulesets that carry a live + hunt link first, newest first within each block. Default (None) is + newest first. Applied SERVER-side, across pages — the list is + keyset-paginated, so a client-side sort would only ever reorder one + page; the SDK never re-orders rows. Reuse a page's ``offset`` only + with the same ``sort``: the server refuses a cursor minted under + the other order. + + Two server-side properties of that key, neither of them SDK + behaviour. It ranks on the stored link, which is a WIDER predicate + than the one ``livescan_id`` is rendered under: a legacy row whose + hunt was stopped without clearing the link ranks in the leading + block while still serializing ``livescan_id`` as ``None``. Read the + field to decide whether a ruleset is running; never the position. + + And the key is MUTABLE, unlike the id-desc default: a ruleset whose + live hunt stops mid-walk falls back into the idle block below the + cursor and is yielded twice, and one started mid-walk moves above + the cursor and is skipped for the rest of that walk. That is a + property of the walk, so starting fresh from the first page does + not avoid it. This generator streams pages and does not dedupe — + dedupe by ``id`` if you consume more than one page. :return: A generator of YaraRuleset resources """ logger.info('List rulesets') diff --git a/src/polyswarm_api/api.py b/src/polyswarm_api/api.py index e8921b51..152e6211 100644 --- a/src/polyswarm_api/api.py +++ b/src/polyswarm_api/api.py @@ -857,20 +857,28 @@ def ruleset_list( maintained server-side by a scheduled refresh; rows carry it as ``new_results_count`` with ``new_results_counted_at`` marking when it was last refreshed. There is no per-request window parameter. - :param sort: ``'active_first'`` returns the rulesets with a running - live hunt first — as recorded by the server's live-hunt link, the - same link ``livescan_id`` renders from — newest first within each - block. Default (None) is newest first. Applied SERVER-side, across - pages — the list is keyset-paginated, so a client-side sort would - only ever reorder one page; the SDK never re-orders rows. Reuse a - page's ``offset`` only with the same ``sort``: the server refuses - a cursor minted under the other order. The key is MUTABLE, unlike - the id-desc default: a ruleset whose live hunt stops mid-walk falls - back into the idle block below the cursor and is yielded twice, - and one started mid-walk moves above the cursor and is skipped for - the rest of that walk. This generator streams pages and does not - dedupe — dedupe by ``id`` if you consume more than one page; a - fresh walk from the first page is always self-consistent. + :param sort: ``'active_first'`` returns the rulesets that carry a live + hunt link first, newest first within each block. Default (None) is + newest first. Applied SERVER-side, across pages — the list is + keyset-paginated, so a client-side sort would only ever reorder one + page; the SDK never re-orders rows. Reuse a page's ``offset`` only + with the same ``sort``: the server refuses a cursor minted under + the other order. + + Two server-side properties of that key, neither of them SDK + behaviour. It ranks on the stored link, which is a WIDER predicate + than the one ``livescan_id`` is rendered under: a legacy row whose + hunt was stopped without clearing the link ranks in the leading + block while still serializing ``livescan_id`` as ``None``. Read the + field to decide whether a ruleset is running; never the position. + + And the key is MUTABLE, unlike the id-desc default: a ruleset whose + live hunt stops mid-walk falls back into the idle block below the + cursor and is yielded twice, and one started mid-walk moves above + the cursor and is skipped for the rest of that walk. That is a + property of the walk, so starting fresh from the first page does + not avoid it. This generator streams pages and does not dedupe — + dedupe by ``id`` if you consume more than one page. :return: A generator of YaraRuleset resources """ logger.info("List rulesets") diff --git a/test/async_client_test.py b/test/async_client_test.py index 3f3c0774..24320457 100644 --- a/test/async_client_test.py +++ b/test/async_client_test.py @@ -624,7 +624,12 @@ async def _enabled(): assert await poll_equals_async(_enabled, True) async def _running_precedes_idle(**kwargs): + # Membership-tolerant on purpose — see the sync twin: + # a replica missing `idle` must read as "not yet true" + # and be retried, not raise out of the poll. ids = [r.id async for r in api.ruleset_list(**kwargs)] + if running.id not in ids or idle.id not in ids: + return None return ids.index(running.id) < ids.index(idle.id) async def _sorted(): diff --git a/test/client_scan_test.py b/test/client_scan_test.py index f6d3db51..9eded3c3 100644 --- a/test/client_scan_test.py +++ b/test/client_scan_test.py @@ -625,7 +625,15 @@ def test_rules_sort_active_first(self): lambda: api.ruleset_get(running.id).livescan_id is not None, True) def _running_precedes_idle(**kwargs): + # Membership-tolerant on purpose: this is polled, and a + # replica that has not applied `idle` yet must read as "not + # yet true" and be retried. `.index()` would raise + # ValueError, which poll_equals does not absorb, and the + # lag the poll exists for would surface as an error on the + # first attempt instead. ids = [r.id for r in api.ruleset_list(**kwargs)] + if running.id not in ids or idle.id not in ids: + return None return ids.index(running.id) < ids.index(idle.id) assert poll_equals(lambda: _running_precedes_idle(sort='active_first'), True) From 9459f7c374eaf397f6ee21c1b4bef9d55ed1b30c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mart=C3=ADnez?= Date: Mon, 14 Sep 2026 19:01:40 -0300 Subject: [PATCH 6/9] docs: the id on a returned ruleset is unique but unordered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sort paragraph called the unsorted list "the id-desc default", which reads as a promise about the id callers can see. It is not one: the server orders on its own insertion key and renders a random 17-digit number as id, so a caller who recorded the smallest id yielded and resumed below it would silently skip or repeat rows. The dedupe advice in the same paragraph stands — that only needs uniqueness — and now says so explicitly. --- specs/03-endpoints.md | 2 +- src/polyswarm_api/aio/api.py | 7 +++++-- src/polyswarm_api/api.py | 7 +++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/specs/03-endpoints.md b/specs/03-endpoints.md index f7b9a51c..23db895c 100644 --- a/specs/03-endpoints.md +++ b/specs/03-endpoints.md @@ -190,7 +190,7 @@ refusal. | `live_feed(since=None, …, livescan_id=None, max_results=None)` | `LiveHuntResult.list` — `livescan_id` scopes the feed to one live hunt (the hunt-page per-ruleset feed); `since` is in **SECONDS** (the server converts with `timedelta(seconds=since)`; the 3.x/4.x docstring said minutes and was wrong), and absent-or-`0` means no time filter at all — the server applies it on a truthiness test; `max_results` bounds how many results the generator yields — `None`/`0`/negative means no bound; it does not alter the request | | `historical_list(since=None)` | `HistoricalHunt.list` | | `historical_results(hunt=None, …)` | `HistoricalHuntResultList.get` | -| `ruleset_list(name=None, status=None, favorites_only=None, has_new_results=None, sort=None)` | `YaraRuleset.list` — the hunt-page filters, conjunctive and optional; unset filters are omitted from the query so the no-filter request is byte-compatible with the old contract. `has_new_results` selects on the server's STORED counter (no window parameter — the window belongs to the server's scheduled refresh; rows carry `new_results_count` + `new_results_counted_at`). `sort='active_first'` (4.5.0) asks the SERVER for the hunt page's order — rulesets carrying a live hunt link first, newest first within each block — as an opt-in token; unset sends no `sort`, keeping the default newest-first. The SDK never re-orders rows: the list is keyset-paginated, so a client-side sort would reorder one page and lie about the rest. The rank is the stored link, a WIDER predicate than the one `livescan_id` is rendered under, so a legacy row whose hunt was stopped without clearing the link leads the list while serializing `livescan_id` as `null` — read the field, not the position. The key is also MUTABLE, unlike the id-desc default — a ruleset whose live hunt stops mid-walk is yielded twice, one started mid-walk is skipped, in any walk including a fresh one — and the generator does not dedupe; callers consuming more than one page dedupe by `id`. | +| `ruleset_list(name=None, status=None, favorites_only=None, has_new_results=None, sort=None)` | `YaraRuleset.list` — the hunt-page filters, conjunctive and optional; unset filters are omitted from the query so the no-filter request is byte-compatible with the old contract. `has_new_results` selects on the server's STORED counter (no window parameter — the window belongs to the server's scheduled refresh; rows carry `new_results_count` + `new_results_counted_at`). `sort='active_first'` (4.5.0) asks the SERVER for the hunt page's order — rulesets carrying a live hunt link first, newest first within each block — as an opt-in token; unset sends no `sort`, keeping the default newest-first. The SDK never re-orders rows: the list is keyset-paginated, so a client-side sort would reorder one page and lie about the rest. The rank is the stored link, a WIDER predicate than the one `livescan_id` is rendered under, so a legacy row whose hunt was stopped without clearing the link leads the list while serializing `livescan_id` as `null` — read the field, not the position. The rendered `id` is unique but UNORDERED (the server renders a random `number`, and orders on its own insertion key), so dedupe with it and never resume or bound a walk with it. The key is also MUTABLE, unlike that default — a ruleset whose live hunt stops mid-walk is yielded twice, one started mid-walk is skipped, in any walk including a fresh one — and the generator does not dedupe; callers consuming more than one page dedupe by `id`. | | `tag_list()` | `Tag.list` | | `family_list()` | `MalwareFamily.list` | | `assertions_list(engine_id)` | `AssertionsJob.list` | diff --git a/src/polyswarm_api/aio/api.py b/src/polyswarm_api/aio/api.py index fd9bb9ba..f78b8677 100644 --- a/src/polyswarm_api/aio/api.py +++ b/src/polyswarm_api/aio/api.py @@ -713,7 +713,10 @@ async def ruleset_list(self, name=None, status=None, favorites_only=None, it was last refreshed. There is no per-request window parameter. :param sort: ``'active_first'`` returns the rulesets that carry a live hunt link first, newest first within each block. Default (None) is - newest first. Applied SERVER-side, across pages — the list is + newest first. "Newest first" is the server's own insertion key, NOT + the ``id`` on the rows you get back — that one is unique but + unordered, so dedupe with it and never resume or bound a walk with + it. Applied SERVER-side, across pages — the list is keyset-paginated, so a client-side sort would only ever reorder one page; the SDK never re-orders rows. Reuse a page's ``offset`` only with the same ``sort``: the server refuses a cursor minted under @@ -726,7 +729,7 @@ async def ruleset_list(self, name=None, status=None, favorites_only=None, block while still serializing ``livescan_id`` as ``None``. Read the field to decide whether a ruleset is running; never the position. - And the key is MUTABLE, unlike the id-desc default: a ruleset whose + And the key is MUTABLE, unlike that default: a ruleset whose live hunt stops mid-walk falls back into the idle block below the cursor and is yielded twice, and one started mid-walk moves above the cursor and is skipped for the rest of that walk. That is a diff --git a/src/polyswarm_api/api.py b/src/polyswarm_api/api.py index 152e6211..f7dcec31 100644 --- a/src/polyswarm_api/api.py +++ b/src/polyswarm_api/api.py @@ -859,7 +859,10 @@ def ruleset_list( it was last refreshed. There is no per-request window parameter. :param sort: ``'active_first'`` returns the rulesets that carry a live hunt link first, newest first within each block. Default (None) is - newest first. Applied SERVER-side, across pages — the list is + newest first. "Newest first" is the server's own insertion key, NOT + the ``id`` on the rows you get back — that one is unique but + unordered, so dedupe with it and never resume or bound a walk with + it. Applied SERVER-side, across pages — the list is keyset-paginated, so a client-side sort would only ever reorder one page; the SDK never re-orders rows. Reuse a page's ``offset`` only with the same ``sort``: the server refuses a cursor minted under @@ -872,7 +875,7 @@ def ruleset_list( block while still serializing ``livescan_id`` as ``None``. Read the field to decide whether a ruleset is running; never the position. - And the key is MUTABLE, unlike the id-desc default: a ruleset whose + And the key is MUTABLE, unlike that default: a ruleset whose live hunt stops mid-walk falls back into the idle block below the cursor and is yielded twice, and one started mid-walk moves above the cursor and is skipped for the rest of that walk. That is a From f46b8af5b6360207d15b13e79d0ffd03fa24a81d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mart=C3=ADnez?= Date: Mon, 14 Sep 2026 19:12:00 -0300 Subject: [PATCH 7/9] test: poll the default-order read, and stop teaching id-desc in the builder test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two leftovers from the ordering correction, both caught in review. The default-order assertion was the one unpolled call of a helper that now returns None while either row is missing — deliberate, so the poll can retry. Unpolled, a lagging read replica turns that into 'None is False' instead of a retry, against the invariant that these pass on the live stack with VCR off. It is polled now, with want=False, which the helper's own guard accepts. And the builder test's class docstring still called the unsorted list id-desc, the exact claim the rest of this branch exists to correct — the cassette shows the default page returning the lower visible id first. --- test/async_client_test.py | 5 ++++- test/client_scan_test.py | 8 ++++++-- test/hunt_tracking_builder_test.py | 8 +++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/test/async_client_test.py b/test/async_client_test.py index 24320457..9dd98348 100644 --- a/test/async_client_test.py +++ b/test/async_client_test.py @@ -635,7 +635,10 @@ async def _running_precedes_idle(**kwargs): async def _sorted(): return await _running_precedes_idle(sort='active_first') assert await poll_equals_async(_sorted, True) - assert await _running_precedes_idle() is False + # Polled like the sorted arm — see the sync twin. + async def _unsorted(): + return await _running_precedes_idle() + assert await poll_equals_async(_unsorted, False) is False with pytest.raises(exceptions.RequestException): _ = [r async for r in api.ruleset_list(sort='bogus')] finally: diff --git a/test/client_scan_test.py b/test/client_scan_test.py index 9eded3c3..5b098c92 100644 --- a/test/client_scan_test.py +++ b/test/client_scan_test.py @@ -637,8 +637,12 @@ def _running_precedes_idle(**kwargs): return ids.index(running.id) < ids.index(idle.id) assert poll_equals(lambda: _running_precedes_idle(sort='active_first'), True) - # the default order is untouched: the newer (idle) ruleset first - assert _running_precedes_idle() is False + # The default order is untouched: the newer (idle) ruleset + # first. Polled like the sorted arm above — the helper returns + # None while either row is missing, so an unpolled read would + # assert `None is False` on a lagging replica instead of + # retrying. `want=False` is not None, so poll_equals accepts it. + assert poll_equals(_running_precedes_idle, False) is False # a sort the server does not know is refused, never ignored with self.assertRaises(exceptions.RequestException): list(api.ruleset_list(sort='bogus')) diff --git a/test/hunt_tracking_builder_test.py b/test/hunt_tracking_builder_test.py index e14f8e74..14534f2c 100644 --- a/test/hunt_tracking_builder_test.py +++ b/test/hunt_tracking_builder_test.py @@ -243,9 +243,11 @@ class TestRulesetListSortOnTheWire: """``ruleset_list(sort='active_first')`` — the hunt page's active-first order is an opt-in server token, and it must REACH the server exactly as such: the unsorted call sends no ``sort`` at all (the request - stays byte-compatible with the pre-sort contract and the list keeps its - id-desc order), and the SDK never re-orders client-side — the list is - keyset-paginated, so a local sort would only ever reorder one page. + stays byte-compatible with the pre-sort contract and the list keeps the + server's default newest-first order — by the server's own insertion key, + NOT by the ``id`` on the rows, which is unique but unordered), and the SDK + never re-orders client-side — the list is keyset-paginated, so a local sort + would only ever reorder one page. Both transports are driven: the sync mirror (what ``polyswarm-cli`` calls) and the canonical async source unasync generates it from.""" From 029af67f96d77c200ec973f3240fbc02ed357202 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mart=C3=ADnez?= Date: Mon, 14 Sep 2026 22:45:27 -0300 Subject: [PATCH 8/9] feat: ruleset_list(exclude_favorites=) for clients that group favorites separately The server gained the inverse of favorites_only: a paginated list with the favorites taken out. It exists because the favorites are a separate, unpaginated fetch bounded by the account's budget, so leaving them in the page too makes a client either render a row twice or render a short page. Appended to the signature rather than placed beside favorites_only, so a caller passing the later filters positionally keeps working. --- specs/03-endpoints.md | 2 +- src/polyswarm_api/aio/api.py | 13 ++++++++++++- src/polyswarm_api/api.py | 11 +++++++++++ test/hunt_tracking_builder_test.py | 13 ++++++++++++- 4 files changed, 36 insertions(+), 3 deletions(-) diff --git a/specs/03-endpoints.md b/specs/03-endpoints.md index 23db895c..4eb0c86d 100644 --- a/specs/03-endpoints.md +++ b/specs/03-endpoints.md @@ -190,7 +190,7 @@ refusal. | `live_feed(since=None, …, livescan_id=None, max_results=None)` | `LiveHuntResult.list` — `livescan_id` scopes the feed to one live hunt (the hunt-page per-ruleset feed); `since` is in **SECONDS** (the server converts with `timedelta(seconds=since)`; the 3.x/4.x docstring said minutes and was wrong), and absent-or-`0` means no time filter at all — the server applies it on a truthiness test; `max_results` bounds how many results the generator yields — `None`/`0`/negative means no bound; it does not alter the request | | `historical_list(since=None)` | `HistoricalHunt.list` | | `historical_results(hunt=None, …)` | `HistoricalHuntResultList.get` | -| `ruleset_list(name=None, status=None, favorites_only=None, has_new_results=None, sort=None)` | `YaraRuleset.list` — the hunt-page filters, conjunctive and optional; unset filters are omitted from the query so the no-filter request is byte-compatible with the old contract. `has_new_results` selects on the server's STORED counter (no window parameter — the window belongs to the server's scheduled refresh; rows carry `new_results_count` + `new_results_counted_at`). `sort='active_first'` (4.5.0) asks the SERVER for the hunt page's order — rulesets carrying a live hunt link first, newest first within each block — as an opt-in token; unset sends no `sort`, keeping the default newest-first. The SDK never re-orders rows: the list is keyset-paginated, so a client-side sort would reorder one page and lie about the rest. The rank is the stored link, a WIDER predicate than the one `livescan_id` is rendered under, so a legacy row whose hunt was stopped without clearing the link leads the list while serializing `livescan_id` as `null` — read the field, not the position. The rendered `id` is unique but UNORDERED (the server renders a random `number`, and orders on its own insertion key), so dedupe with it and never resume or bound a walk with it. The key is also MUTABLE, unlike that default — a ruleset whose live hunt stops mid-walk is yielded twice, one started mid-walk is skipped, in any walk including a fresh one — and the generator does not dedupe; callers consuming more than one page dedupe by `id`. | +| `ruleset_list(name=None, status=None, favorites_only=None, has_new_results=None, sort=None, exclude_favorites=None)` | `YaraRuleset.list` — the hunt-page filters, conjunctive and optional; unset filters are omitted from the query so the no-filter request is byte-compatible with the old contract. `exclude_favorites=True` is the inverse of `favorites_only` and refused together with it — it exists for clients that render the favorites as their own list, where leaving them in the paginated list too makes a page repeat a row or come back short. Appended to the signature rather than placed beside `favorites_only`, so a positional caller keeps working. `has_new_results` selects on the server's STORED counter (no window parameter — the window belongs to the server's scheduled refresh; rows carry `new_results_count` + `new_results_counted_at`). `sort='active_first'` (4.5.0) asks the SERVER for the hunt page's order — rulesets carrying a live hunt link first, newest first within each block — as an opt-in token; unset sends no `sort`, keeping the default newest-first. The SDK never re-orders rows: the list is keyset-paginated, so a client-side sort would reorder one page and lie about the rest. The rank is the stored link, a WIDER predicate than the one `livescan_id` is rendered under, so a legacy row whose hunt was stopped without clearing the link leads the list while serializing `livescan_id` as `null` — read the field, not the position. The rendered `id` is unique but UNORDERED (the server renders a random `number`, and orders on its own insertion key), so dedupe with it and never resume or bound a walk with it. The key is also MUTABLE, unlike that default — a ruleset whose live hunt stops mid-walk is yielded twice, one started mid-walk is skipped, in any walk including a fresh one — and the generator does not dedupe; callers consuming more than one page dedupe by `id`. | | `tag_list()` | `Tag.list` | | `family_list()` | `MalwareFamily.list` | | `assertions_list(engine_id)` | `AssertionsJob.list` | diff --git a/src/polyswarm_api/aio/api.py b/src/polyswarm_api/aio/api.py index f78b8677..d9377401 100644 --- a/src/polyswarm_api/aio/api.py +++ b/src/polyswarm_api/aio/api.py @@ -697,7 +697,8 @@ async def ruleset_delete(self, ruleset_id): return await self._single(resources.YaraRuleset.delete(self, id=ruleset_id, community=self.community)) async def ruleset_list(self, name=None, status=None, favorites_only=None, - has_new_results=None, sort=None): + has_new_results=None, sort=None, + exclude_favorites=None): """ List all YaraRulesets for the current account. @@ -706,6 +707,15 @@ async def ruleset_list(self, name=None, status=None, favorites_only=None, :param status: 'active' returns only rulesets whose live hunt is currently running. :param favorites_only: True returns only favorited rulesets. + :param exclude_favorites: True returns only the rulesets that are NOT + favorited — the inverse of ``favorites_only``, and refused together + with it (a contradiction, answered with an error rather than an + empty list). It exists for clients that render the favorites as + their own list: the favorites are a separate, unpaginated fetch + bounded by the account's budget, so leaving them in the paginated + list too makes a page either repeat a row or come back short. + Appended to the signature rather than placed beside + ``favorites_only`` so a positional caller keeps working. :param has_new_results: True returns only rulesets whose stored new-results counter is positive. The counter (and its window) is maintained server-side by a scheduled refresh; rows carry it as @@ -742,6 +752,7 @@ async def ruleset_list(self, name=None, status=None, favorites_only=None, async for item in self._paginate(resources.YaraRuleset.list( self, name=name, status=status, favorites_only=favorites_only, has_new_results=has_new_results, sort=sort, + exclude_favorites=exclude_favorites, community=self.community)): yield item diff --git a/src/polyswarm_api/api.py b/src/polyswarm_api/api.py index f7dcec31..3f98cc5d 100644 --- a/src/polyswarm_api/api.py +++ b/src/polyswarm_api/api.py @@ -843,6 +843,7 @@ def ruleset_list( favorites_only=None, has_new_results=None, sort=None, + exclude_favorites=None, ): """ List all YaraRulesets for the current account. @@ -852,6 +853,15 @@ def ruleset_list( :param status: 'active' returns only rulesets whose live hunt is currently running. :param favorites_only: True returns only favorited rulesets. + :param exclude_favorites: True returns only the rulesets that are NOT + favorited — the inverse of ``favorites_only``, and refused together + with it (a contradiction, answered with an error rather than an + empty list). It exists for clients that render the favorites as + their own list: the favorites are a separate, unpaginated fetch + bounded by the account's budget, so leaving them in the paginated + list too makes a page either repeat a row or come back short. + Appended to the signature rather than placed beside + ``favorites_only`` so a positional caller keeps working. :param has_new_results: True returns only rulesets whose stored new-results counter is positive. The counter (and its window) is maintained server-side by a scheduled refresh; rows carry it as @@ -893,6 +903,7 @@ def ruleset_list( favorites_only=favorites_only, has_new_results=has_new_results, sort=sort, + exclude_favorites=exclude_favorites, community=self.community, ) ): diff --git a/test/hunt_tracking_builder_test.py b/test/hunt_tracking_builder_test.py index 14534f2c..e3215267 100644 --- a/test/hunt_tracking_builder_test.py +++ b/test/hunt_tracking_builder_test.py @@ -65,12 +65,23 @@ def test_list_routes_filters_to_the_query_with_int_bools(self): 'name': 'alpha', 'status': 'active', 'favorites_only': 1, 'has_new_results': 1, 'community': 'gamma'} + def test_exclude_favorites_rides_the_query_as_an_int_bool(self): + """The inverse filter, for clients that render the favorites as their + own list: leaving them in the paginated list too makes a page repeat a + row or come back short. Same int-bool coercion as its sibling.""" + api = _FakeApi() + req = resources.YaraRuleset.list( + api, exclude_favorites=True, sort='active_first', + community=api.community) + assert req.params == { + 'exclude_favorites': 1, 'sort': 'active_first', 'community': 'gamma'} + def test_list_omits_every_unset_filter(self): # The no-filter request is byte-compatible with the pre-filter # contract: nothing but community rides the query string. req = resources.YaraRuleset.list( _FakeApi(), name=None, status=None, favorites_only=None, - has_new_results=None, community='gamma') + has_new_results=None, exclude_favorites=None, community='gamma') assert req.params == {'community': 'gamma'} From e4e3d304a09cf02e948ceaacabcb75f1e8c542d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mart=C3=ADnez?= Date: Tue, 15 Sep 2026 00:18:28 -0300 Subject: [PATCH 9/9] test: drive exclude_favorites through both clients, and say what is still untested MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The only coverage the parameter shipped with called the generic resource builder, which this change does not touch — dropping the keyword from both transports left the whole suite green. The new cases drive the sync and async client methods, and deleting the pass-through now fails exactly one test. The e2e arm specs/04 asks for is still missing, and the comment at the live sort test says so plainly, along with what compensates and what does not: a rename made in lockstep with the server's spelling is the case only a live request catches. Recording it needs a stack whose key-management service carries the fixture account. --- test/client_scan_test.py | 15 +++++++++++++++ test/hunt_tracking_builder_test.py | 24 +++++++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/test/client_scan_test.py b/test/client_scan_test.py index 5b098c92..26f6e6aa 100644 --- a/test/client_scan_test.py +++ b/test/client_scan_test.py @@ -602,6 +602,21 @@ def test_historical_results(self): pass @vcr.use_cassette() + # NO e2e arm for `exclude_favorites`, deliberately and with a cost. + # specs/04 invariant 1 wants endpoint behaviour tested against the real + # server, and the reason is spelled out below: the server ignores unknown + # query args, so a renamed token leaves builder tests green and the list + # unfiltered. What covers it instead: + # * `TestRulesetListSortOnTheWire` drives BOTH client methods and fails if + # either stops forwarding the keyword (verified by deleting the + # pass-through: one test fails, the rest stay green); + # * the server side pins the filter itself, and the 400 for the + # contradictory pair, in its own HTTP suite against a real database. + # What stays uncovered is a rename that both sides make in lockstep with + # the server's spelling — the case only a live request catches. Recording + # the cassette needs a stack whose AKM carries the fixture account; ours + # answers 500 for a hand-seeded one, so it is honest to say this is + # missing rather than to fake a recording. def test_rules_sort_active_first(self): """``sort='active_first'`` is an order the SERVER applies: two rulesets owned by this test, the older one with a live hunt running, the newer diff --git a/test/hunt_tracking_builder_test.py b/test/hunt_tracking_builder_test.py index e3215267..6d95b6ea 100644 --- a/test/hunt_tracking_builder_test.py +++ b/test/hunt_tracking_builder_test.py @@ -68,7 +68,12 @@ def test_list_routes_filters_to_the_query_with_int_bools(self): def test_exclude_favorites_rides_the_query_as_an_int_bool(self): """The inverse filter, for clients that render the favorites as their own list: leaving them in the paginated list too makes a page repeat a - row or come back short. Same int-bool coercion as its sibling.""" + row or come back short. Same int-bool coercion as its sibling. + + NOTE this exercises the generic builder, which this change does not + touch — the test that actually covers the new code is + ``TestRulesetListSortOnTheWire`` below, which drives both CLIENT + methods and would fail if either stopped forwarding the keyword.""" api = _FakeApi() req = resources.YaraRuleset.list( api, exclude_favorites=True, sort='active_first', @@ -321,6 +326,23 @@ def test_async_canonical_sends_the_same_token(self): assert sent == {'sort': 'active_first', 'community': 'gamma'} assert 'sort' not in self._async_params() + def test_exclude_favorites_reaches_the_server_on_both_clients(self): + """The keyword the hunt page pairs with the sort, driven through the + CLIENT methods rather than the shared builder: dropping it from either + transport's signature or its pass-through fails here, which is what the + builder-level test cannot see.""" + sent = self._sync_params(sort='active_first', exclude_favorites=True) + assert sent['exclude_favorites'] == 1 + assert sent['sort'] == 'active_first' + assert self._async_params(sort='active_first', exclude_favorites=True) == { + 'exclude_favorites': 1, 'sort': 'active_first', 'community': 'gamma'} + + def test_exclude_favorites_is_omitted_when_unset(self): + # Same rule as every other filter: the unfiltered request stays + # byte-compatible with the pre-change contract. + assert 'exclude_favorites' not in self._sync_params() + assert 'exclude_favorites' not in self._async_params(sort='active_first') + def test_sort_survives_onto_the_next_page(self): # The order is only meaningful across pages, and page 2 is built by # _next_page cloning the descriptor's params — the one place a