From ac1f8d46b30eda5a7197af364eea8cbf766ef794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mart=C3=ADnez?= Date: Tue, 1 Sep 2026 19:51:05 -0400 Subject: [PATCH 1/9] feat(rules): list --sort active-first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Forwards `ruleset_list(sort='active_first')` — the hunt page's order, rulesets with a running live hunt first (as recorded by the server's live-hunt link, the same one Livescan Id renders from), newest first within each block. Server-side like the filters: the list is keyset-paginated, so a local sort would only ever reorder one page. The option is a closed click.Choice (hyphenated CLI spelling, underscored server token) and is forwarded only when given, so the unsorted default request is unchanged. The autospec tests double as the signature check against the installed SDK, alone and combined with the filters. --- specs/02-commands.md | 2 +- src/polyswarm/client/rules.py | 15 ++++++++--- tests/formatter_hunt_fields_test.py | 40 +++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/specs/02-commands.md b/specs/02-commands.md index fd1aeb7e..de7358b6 100644 --- a/specs/02-commands.md +++ b/specs/02-commands.md @@ -30,7 +30,7 @@ The top-level command groups, what each is for, and the primary `polyswarm-api` | `tag` (`tags.py`) | Tag CRUD | `tag_{create,delete,get,list}` | | `link` (`links.py`) | Tag/family links on artifacts | `tag_link_multiple`, `tag_link_get`, `tag_link_list` | | `family` (`families.py`) | Malware-family CRUD | `family_{create,update,delete,get,list}` | -| `rules` (`rules.py`) | YARA ruleset CRUD plus `favorite [--unfavorite]` (the star toggle: renders the new state + the server-owned "N of M used" budget, and converts the machine-readable `FAVORITE_LIMIT` refusal into a clean actionable message at exit 2, never 1 — 1 is reserved for no-results/not-found; 2 is the broad bucket `ExceptionHandlingGroup` maps the PolyswarmException hierarchies to. **2 does not identify a server refusal:** click exits 2 for a `UsageError` too, so a scripted caller cannot tell “the favorite budget is full” from “you passed a bad flag” without reading the message). `list` takes the server-side filters `--name` / `--status active` / `--favorites-only` / `--has-new-results` (conjunctive; the list is keyset-paginated, so filtering locally would mean walking every page). `rules favorite` and the `rules list` filters need SDK 4.4.0, which the pin requires (see [05-sdk-contract.md](./05-sdk-contract.md) §Current floor), so they are called directly. The formatters read the hunt-page fields directly: the pin guarantees the SDK parses them, so `None` means the *server* had no answer | `ruleset_{create,delete,update,get,list,favorite}` | +| `rules` (`rules.py`) | YARA ruleset CRUD (`list` takes the server-side filters plus `--sort active-first`, the hunt page's running-hunts-first order — ordering is the server's, never a local re-sort of one keyset page) plus `favorite [--unfavorite]` (the star toggle: renders the new state + the server-owned "N of M used" budget, and converts the machine-readable `FAVORITE_LIMIT` refusal into a clean actionable message at exit 2, never 1 — 1 is reserved for no-results/not-found; 2 is the broad bucket `ExceptionHandlingGroup` maps the PolyswarmException hierarchies to. **2 does not identify a server refusal:** click exits 2 for a `UsageError` too, so a scripted caller cannot tell “the favorite budget is full” from “you passed a bad flag” without reading the message). `list` takes the server-side filters `--name` / `--status active` / `--favorites-only` / `--has-new-results` (conjunctive; the list is keyset-paginated, so filtering locally would mean walking every page). `rules favorite` and the `rules list` filters need SDK 4.4.0 and `rules list --sort` needs 4.5.0, which the pin requires (see [05-sdk-contract.md](./05-sdk-contract.md) §Current floor), so they are called directly. The formatters read the hunt-page fields directly: the pin guarantees the SDK parses them, so `None` means the *server* had no answer | `ruleset_{create,delete,update,get,list,favorite}` | | `metadata` (`metadata.py`) | Rerun metadata; scan lookup; IP/URL analysis | `rerun_metadata`, `scan_lookup`, `submit_url` | | `activity` (`event.py`) | List account activity/events | `event_list` | | `account` (`account.py`) | Account whois / features | `account_whois`, `account_features` | diff --git a/src/polyswarm/client/rules.py b/src/polyswarm/client/rules.py index 9f8fa7e2..d305aa47 100644 --- a/src/polyswarm/client/rules.py +++ b/src/polyswarm/client/rules.py @@ -38,19 +38,26 @@ def delete(ctx, rule_id): @click.option('--favorites-only', is_flag=True, help='Only favorited (starred) rulesets.') @click.option('--has-new-results', is_flag=True, help='Only rulesets whose stored new-results counter is positive.') +@click.option('--sort', type=click.Choice(['active-first']), + help='Order: rulesets with a running live hunt first (as recorded by the ' + "server's live-hunt link, the same one Livescan Id renders from), " + 'newest first within each block. Default is newest first.') @click.pass_context -def list_rules(ctx, name, status, favorites_only, has_new_results): +def list_rules(ctx, name, status, favorites_only, has_new_results, sort): """List rulesets, optionally filtered. All filters are conjunctive. - Filtering is applied SERVER-side: the list is keyset-paginated, so a - client filtering locally would have to walk every page to find matches. + Filtering and ordering are applied SERVER-side: the list is + keyset-paginated, so a client filtering or sorting locally would have to + walk every page to get it right. """ api = ctx.obj['api'] output = ctx.obj['output'] # A False flag is not a filter: send only what the caller actually asked for. + # The CLI spells the sort with a hyphen; the server token is 'active_first'. kwargs = {k: v for k, v in (('name', name), ('status', status), ('favorites_only', favorites_only or None), - ('has_new_results', has_new_results or None)) + ('has_new_results', has_new_results or None), + ('sort', sort.replace('-', '_') if sort else None)) if v is not None} for ruleset in api.ruleset_list(**kwargs): output.ruleset(ruleset) diff --git a/tests/formatter_hunt_fields_test.py b/tests/formatter_hunt_fields_test.py index 1d0675da..187b8faa 100644 --- a/tests/formatter_hunt_fields_test.py +++ b/tests/formatter_hunt_fields_test.py @@ -168,6 +168,46 @@ def test_filters_are_forwarded_only_when_given(self): mock.ANY, name='alpha', status='active', favorites_only=True, has_new_results=True) + def test_sort_active_first_is_forwarded_as_the_server_token(self): + """`--sort active-first` (CLI spelling, hyphen) reaches the SDK as + the server's `sort='active_first'` token — and, like the filters, only + when given: the unsorted default sends no `sort` at all, so the list + keeps its id-desc order and the request stays byte-compatible.""" + with mock.patch('polyswarm_api.api.PolyswarmAPI.ruleset_list', + autospec=True, return_value=iter(())) as ruleset_list: + result = CliRunner().invoke( + client.polyswarm_cli, + ['-a', '1' * 32, '-u', 'http://ai:9696/v3', '-c', 'gamma', + 'rules', 'list', '--sort', 'active-first'], + catch_exceptions=False) + assert result.exit_code == 0, result.output + ruleset_list.assert_called_once_with(mock.ANY, sort='active_first') + + def test_sort_composes_with_the_filters(self): + # The kwargs comprehension is the one site that could drop or + # mistranslate the sort when filters ride along. + with mock.patch('polyswarm_api.api.PolyswarmAPI.ruleset_list', + autospec=True, return_value=iter(())) as ruleset_list: + result = CliRunner().invoke( + client.polyswarm_cli, + ['-a', '1' * 32, '-u', 'http://ai:9696/v3', '-c', 'gamma', + 'rules', 'list', '--status', 'active', '--favorites-only', + '--sort', 'active-first'], + catch_exceptions=False) + assert result.exit_code == 0, result.output + ruleset_list.assert_called_once_with( + mock.ANY, status='active', favorites_only=True, sort='active_first') + + def test_sort_rejects_an_unknown_order(self): + # A closed choice on the CLI side too: the server would 400 an unknown + # sort, but the CLI should not have to make the round trip to say so. + result = CliRunner().invoke( + client.polyswarm_cli, + ['-a', '1' * 32, '-u', 'http://ai:9696/v3', '-c', 'gamma', + 'rules', 'list', '--sort', 'newest']) + assert result.exit_code == 2, result.output + assert 'active-first' in result.output + class LiveFeedOptionsTest(TestCase): From 5e07b7ccd28478488da265f3461185cc03157373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mart=C3=ADnez?= Date: Tue, 1 Sep 2026 19:51:05 -0400 Subject: [PATCH 2/9] chore: raise the SDK floor to polyswarm_api>=4.5.0 for ruleset_list(sort=) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The floor names the SDK version that adds the keyword `rules list --sort` forwards (specs/05 §Current floor follows the pin). Mergeable once the SDK's develop declares 4.5.0; releasable once that version is on PyPI — the SDK releases first. CI resolves the SDK from source by branch name, falling back to develop, so the paired SDK branch must carry the identical name. --- pyproject.toml | 2 +- specs/05-sdk-contract.md | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 941682c8..84c00fc3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ classifiers = [ ] dependencies = [ - "polyswarm_api>=4.4.0,<5.0.0", + "polyswarm_api>=4.5.0,<5.0.0", "click>=7.1", "colorama>=0.4.6", "click-log>=0.4.0", diff --git a/specs/05-sdk-contract.md b/specs/05-sdk-contract.md index 7a0749b4..3eb39c27 100644 --- a/specs/05-sdk-contract.md +++ b/specs/05-sdk-contract.md @@ -79,7 +79,7 @@ When a CLI feature needs an SDK surface that doesn't exist yet: **Read the declared version off the archive's own tree, and mind pre-release suffixes.** PEP 440 orders `4.2.0.dev1 < 4.2.0`, so a `develop` head carrying a dev suffix (the SDK's `pyproject.toml` has a `[tool.bumpversion.parts.dev]`) would *not* satisfy a `>=4.2.0` floor even though it looks like 4.2.0 — and the archive build would be silently replaced from PyPI. Check the version string in the SDK branch's `pyproject.toml` / `__init__.py`, not the last release tag. When the floor was last verified this way both were read from `origin/develop` as `4.2.0`, no suffix; the pin has since moved on (§Current floor is the one authoritative statement of its value), and every bump should be re-checked the same way. -### Current floor — `polyswarm_api>=4.4.0` +### Current floor — `polyswarm_api>=4.5.0` The floor is whatever `pyproject.toml` pins; this header follows it. It lives in ONE authoritative place for a reason — a copy here drifted behind the pin once already. The 4.2.0 rationale below still holds transitively; on 4.1.0 both behaviours fail *silently*, which is why the floor is a hard requirement rather than a preference: @@ -99,7 +99,11 @@ that is not supported rather than against a version the floor permits.) The hunt formatters render — are what moved the floor to 4.4.0, together with `matched_strings` / `matched_strings_dropped` on the four hunt-result classes (the yara evidence behind a hit; see [`03-formatters.md`](./03-formatters.md) -§Matched strings on hunt results). Code and tests use them directly. +§Matched strings on hunt results). `rules list --sort active-first` forwards +`ruleset_list(sort='active_first')`, a keyword 4.5.0 adds, and that is what moved the +floor to 4.5.0 (the `tests/formatter_hunt_fields_test.py` autospec assertion is the +signature check: against a 4.4.0 SDK it fails at the mock, not at the server). Code and +tests use them directly. **Raising the floor is the whole procedure** when this repo needs something new from the SDK: From 20e3c03f41d696cfad26a8f80b900b7ffd197247 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mart=C3=ADnez?= Date: Mon, 14 Sep 2026 13:10:25 -0300 Subject: [PATCH 3/9] docs(rules): name the label the formatter actually prints in --sort help --- src/polyswarm/client/rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/polyswarm/client/rules.py b/src/polyswarm/client/rules.py index d305aa47..5fdb0f80 100644 --- a/src/polyswarm/client/rules.py +++ b/src/polyswarm/client/rules.py @@ -40,7 +40,7 @@ def delete(ctx, rule_id): help='Only rulesets whose stored new-results counter is positive.') @click.option('--sort', type=click.Choice(['active-first']), help='Order: rulesets with a running live hunt first (as recorded by the ' - "server's live-hunt link, the same one Livescan Id renders from), " + "server's live-hunt link, the same one Live Hunt Id renders from), " 'newest first within each block. Default is newest first.') @click.pass_context def list_rules(ctx, name, status, favorites_only, has_new_results, sort): From fbd40cb1915cd7f02bc985fa2c5e0573318945b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mart=C3=ADnez?= Date: Mon, 14 Sep 2026 16:25:48 -0300 Subject: [PATCH 4/9] docs(rules): --sort ranks on the stored hunt link, which is wider than Live Hunt Id The help text said the order came from "the same one Live Hunt Id renders from". The server ranks on the raw link and renders the id under a stricter predicate, so a legacy row whose hunt was stopped without clearing the link leads the list with an empty Live Hunt Id. Someone reading the list top-down for what is running would have stopped at that row. --- specs/02-commands.md | 2 +- src/polyswarm/client/rules.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/specs/02-commands.md b/specs/02-commands.md index de7358b6..873af124 100644 --- a/specs/02-commands.md +++ b/specs/02-commands.md @@ -30,7 +30,7 @@ The top-level command groups, what each is for, and the primary `polyswarm-api` | `tag` (`tags.py`) | Tag CRUD | `tag_{create,delete,get,list}` | | `link` (`links.py`) | Tag/family links on artifacts | `tag_link_multiple`, `tag_link_get`, `tag_link_list` | | `family` (`families.py`) | Malware-family CRUD | `family_{create,update,delete,get,list}` | -| `rules` (`rules.py`) | YARA ruleset CRUD (`list` takes the server-side filters plus `--sort active-first`, the hunt page's running-hunts-first order — ordering is the server's, never a local re-sort of one keyset page) plus `favorite [--unfavorite]` (the star toggle: renders the new state + the server-owned "N of M used" budget, and converts the machine-readable `FAVORITE_LIMIT` refusal into a clean actionable message at exit 2, never 1 — 1 is reserved for no-results/not-found; 2 is the broad bucket `ExceptionHandlingGroup` maps the PolyswarmException hierarchies to. **2 does not identify a server refusal:** click exits 2 for a `UsageError` too, so a scripted caller cannot tell “the favorite budget is full” from “you passed a bad flag” without reading the message). `list` takes the server-side filters `--name` / `--status active` / `--favorites-only` / `--has-new-results` (conjunctive; the list is keyset-paginated, so filtering locally would mean walking every page). `rules favorite` and the `rules list` filters need SDK 4.4.0 and `rules list --sort` needs 4.5.0, which the pin requires (see [05-sdk-contract.md](./05-sdk-contract.md) §Current floor), so they are called directly. The formatters read the hunt-page fields directly: the pin guarantees the SDK parses them, so `None` means the *server* had no answer | `ruleset_{create,delete,update,get,list,favorite}` | +| `rules` (`rules.py`) | YARA ruleset CRUD (`list` takes the server-side filters plus `--sort active-first`, the hunt page's order: rulesets carrying a live hunt link first — the rank is that stored link, wider than what Live Hunt Id renders from, so a legacy stopped-but-unlinked row leads the list with an empty Live Hunt Id; ordering is the server's, never a local re-sort of one keyset page) plus `favorite [--unfavorite]` (the star toggle: renders the new state + the server-owned "N of M used" budget, and converts the machine-readable `FAVORITE_LIMIT` refusal into a clean actionable message at exit 2, never 1 — 1 is reserved for no-results/not-found; 2 is the broad bucket `ExceptionHandlingGroup` maps the PolyswarmException hierarchies to. **2 does not identify a server refusal:** click exits 2 for a `UsageError` too, so a scripted caller cannot tell “the favorite budget is full” from “you passed a bad flag” without reading the message). `list` takes the server-side filters `--name` / `--status active` / `--favorites-only` / `--has-new-results` (conjunctive; the list is keyset-paginated, so filtering locally would mean walking every page). `rules favorite` and the `rules list` filters need SDK 4.4.0 and `rules list --sort` needs 4.5.0, which the pin requires (see [05-sdk-contract.md](./05-sdk-contract.md) §Current floor), so they are called directly. The formatters read the hunt-page fields directly: the pin guarantees the SDK parses them, so `None` means the *server* had no answer | `ruleset_{create,delete,update,get,list,favorite}` | | `metadata` (`metadata.py`) | Rerun metadata; scan lookup; IP/URL analysis | `rerun_metadata`, `scan_lookup`, `submit_url` | | `activity` (`event.py`) | List account activity/events | `event_list` | | `account` (`account.py`) | Account whois / features | `account_whois`, `account_features` | diff --git a/src/polyswarm/client/rules.py b/src/polyswarm/client/rules.py index 5fdb0f80..ca01f15c 100644 --- a/src/polyswarm/client/rules.py +++ b/src/polyswarm/client/rules.py @@ -39,9 +39,11 @@ def delete(ctx, rule_id): @click.option('--has-new-results', is_flag=True, help='Only rulesets whose stored new-results counter is positive.') @click.option('--sort', type=click.Choice(['active-first']), - help='Order: rulesets with a running live hunt first (as recorded by the ' - "server's live-hunt link, the same one Live Hunt Id renders from), " - 'newest first within each block. Default is newest first.') + help='Order: rulesets carrying a live hunt link first, newest first ' + 'within each block. Default is newest first. The rank is the ' + 'stored link, which is wider than what Live Hunt Id renders ' + 'from: a legacy row whose hunt was stopped without clearing ' + 'the link leads the list with an empty Live Hunt Id.') @click.pass_context def list_rules(ctx, name, status, favorites_only, has_new_results, sort): """List rulesets, optionally filtered. All filters are conjunctive. From bc80446a3190674730d4ad1ad54470e8e4cd1603 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mart=C3=ADnez?= Date: Mon, 14 Sep 2026 19:09:20 -0300 Subject: [PATCH 5/9] fix(rules): dedupe the ruleset walk by id, and stop promising an empty Live Hunt Id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `rules list` walks every page and exposes no limit, which makes it exactly the consumer the SDK puts the dedupe obligation on. Under --sort active-first the ordering key is the live-hunt link, so a ruleset whose hunt stops between two page fetches drops below the cursor and the server serves it again: the run printed it twice and any script counting the output double-counted it. Rows are now emitted at most once per run. The dedupe is unconditional — the id is unique under either order and one set of ids costs nothing next to the rendered rows. The symmetric case cannot be repaired from here and is documented rather than hidden: a hunt STARTED mid-walk moves its row above the cursor and it never reaches this client until the next run. Two doc corrections in the same push. The --sort help promised that a stale-link row "leads the list with an empty Live Hunt Id", but the formatter gates the whole Live Hunt Id pair on a truthy value, so such a row prints no such line at all and reads as idle. And the commands spec called that row "stopped-but-unlinked", the inverse of the state it means: it is stopped and STILL linked, which is why it leads a list ranked on the link being present. --- specs/02-commands.md | 2 +- src/polyswarm/client/rules.py | 24 +++++++++++++++++-- tests/formatter_hunt_fields_test.py | 36 +++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/specs/02-commands.md b/specs/02-commands.md index 873af124..868fa41e 100644 --- a/specs/02-commands.md +++ b/specs/02-commands.md @@ -30,7 +30,7 @@ The top-level command groups, what each is for, and the primary `polyswarm-api` | `tag` (`tags.py`) | Tag CRUD | `tag_{create,delete,get,list}` | | `link` (`links.py`) | Tag/family links on artifacts | `tag_link_multiple`, `tag_link_get`, `tag_link_list` | | `family` (`families.py`) | Malware-family CRUD | `family_{create,update,delete,get,list}` | -| `rules` (`rules.py`) | YARA ruleset CRUD (`list` takes the server-side filters plus `--sort active-first`, the hunt page's order: rulesets carrying a live hunt link first — the rank is that stored link, wider than what Live Hunt Id renders from, so a legacy stopped-but-unlinked row leads the list with an empty Live Hunt Id; ordering is the server's, never a local re-sort of one keyset page) plus `favorite [--unfavorite]` (the star toggle: renders the new state + the server-owned "N of M used" budget, and converts the machine-readable `FAVORITE_LIMIT` refusal into a clean actionable message at exit 2, never 1 — 1 is reserved for no-results/not-found; 2 is the broad bucket `ExceptionHandlingGroup` maps the PolyswarmException hierarchies to. **2 does not identify a server refusal:** click exits 2 for a `UsageError` too, so a scripted caller cannot tell “the favorite budget is full” from “you passed a bad flag” without reading the message). `list` takes the server-side filters `--name` / `--status active` / `--favorites-only` / `--has-new-results` (conjunctive; the list is keyset-paginated, so filtering locally would mean walking every page). `rules favorite` and the `rules list` filters need SDK 4.4.0 and `rules list --sort` needs 4.5.0, which the pin requires (see [05-sdk-contract.md](./05-sdk-contract.md) §Current floor), so they are called directly. The formatters read the hunt-page fields directly: the pin guarantees the SDK parses them, so `None` means the *server* had no answer | `ruleset_{create,delete,update,get,list,favorite}` | +| `rules` (`rules.py`) | YARA ruleset CRUD (`list` takes the server-side filters plus `--sort active-first`, the hunt page's order: rulesets carrying a live hunt link first — the rank is that stored link, wider than what Live Hunt Id renders from, so a legacy row that was stopped while STILL LINKED leads the list and renders no Live Hunt Id at all — read the field, never the position. The command walks every page and so dedupes by id, the obligation the SDK puts on a multi-page consumer of this mutable key (a row whose hunt stops mid-walk is served twice; one started mid-walk is missed until the next run); ordering is the server's, never a local re-sort of one keyset page) plus `favorite [--unfavorite]` (the star toggle: renders the new state + the server-owned "N of M used" budget, and converts the machine-readable `FAVORITE_LIMIT` refusal into a clean actionable message at exit 2, never 1 — 1 is reserved for no-results/not-found; 2 is the broad bucket `ExceptionHandlingGroup` maps the PolyswarmException hierarchies to. **2 does not identify a server refusal:** click exits 2 for a `UsageError` too, so a scripted caller cannot tell “the favorite budget is full” from “you passed a bad flag” without reading the message). `list` takes the server-side filters `--name` / `--status active` / `--favorites-only` / `--has-new-results` (conjunctive; the list is keyset-paginated, so filtering locally would mean walking every page). `rules favorite` and the `rules list` filters need SDK 4.4.0 and `rules list --sort` needs 4.5.0, which the pin requires (see [05-sdk-contract.md](./05-sdk-contract.md) §Current floor), so they are called directly. The formatters read the hunt-page fields directly: the pin guarantees the SDK parses them, so `None` means the *server* had no answer | `ruleset_{create,delete,update,get,list,favorite}` | | `metadata` (`metadata.py`) | Rerun metadata; scan lookup; IP/URL analysis | `rerun_metadata`, `scan_lookup`, `submit_url` | | `activity` (`event.py`) | List account activity/events | `event_list` | | `account` (`account.py`) | Account whois / features | `account_whois`, `account_features` | diff --git a/src/polyswarm/client/rules.py b/src/polyswarm/client/rules.py index ca01f15c..a558b63b 100644 --- a/src/polyswarm/client/rules.py +++ b/src/polyswarm/client/rules.py @@ -41,9 +41,11 @@ def delete(ctx, rule_id): @click.option('--sort', type=click.Choice(['active-first']), help='Order: rulesets carrying a live hunt link first, newest first ' 'within each block. Default is newest first. The rank is the ' - 'stored link, which is wider than what Live Hunt Id renders ' + 'stored link, which is WIDER than what Live Hunt Id renders ' 'from: a legacy row whose hunt was stopped without clearing ' - 'the link leads the list with an empty Live Hunt Id.') + 'the link leads the list while rendering no Live Hunt Id at ' + 'all, indistinguishable from an idle one. Read the field, ' + 'never the position.') @click.pass_context def list_rules(ctx, name, status, favorites_only, has_new_results, sort): """List rulesets, optionally filtered. All filters are conjunctive. @@ -51,6 +53,16 @@ def list_rules(ctx, name, status, favorites_only, has_new_results, sort): Filtering and ordering are applied SERVER-side: the list is keyset-paginated, so a client filtering or sorting locally would have to walk every page to get it right. + + This command walks EVERY page, which makes it the consumer the SDK puts the + dedupe obligation on: under --sort active-first the ordering key is mutable + (it is the live-hunt link the sort ranks on), so a ruleset whose hunt stops + between two page fetches drops below the cursor and the server serves it a + second time. Rows are therefore emitted at most once per run, keyed on id. + + The symmetric case cannot be repaired from here and is not hidden: a hunt + STARTED mid-walk moves its ruleset above the cursor, so that row never + reaches this client at all. A re-run lists it. """ api = ctx.obj['api'] output = ctx.obj['output'] @@ -61,7 +73,15 @@ def list_rules(ctx, name, status, favorites_only, has_new_results, sort): ('has_new_results', has_new_results or None), ('sort', sort.replace('-', '_') if sort else None)) if v is not None} + seen = set() for ruleset in api.ruleset_list(**kwargs): + # Unconditional rather than gated on `sort`: the id is unique either + # way, one set of ids costs nothing next to the rows already rendered, + # and a gate would be a second place to update when another mutable + # order appears. Under the default order this never drops anything. + if ruleset.id in seen: + continue + seen.add(ruleset.id) output.ruleset(ruleset) diff --git a/tests/formatter_hunt_fields_test.py b/tests/formatter_hunt_fields_test.py index 187b8faa..2dfe9d23 100644 --- a/tests/formatter_hunt_fields_test.py +++ b/tests/formatter_hunt_fields_test.py @@ -198,6 +198,42 @@ def test_sort_composes_with_the_filters(self): ruleset_list.assert_called_once_with( mock.ANY, status='active', favorites_only=True, sort='active_first') + def test_a_row_served_twice_by_the_mutable_sort_is_printed_once(self): + """The command walks every page, so it is the consumer the SDK puts the + dedupe obligation on. Under the active-first order a ruleset whose hunt + stops between two page fetches falls below the cursor and the server + serves it again; without the dedupe the run prints it twice and any + script counting the output double-counts it.""" + repeated = [_ruleset(id='5', name='stops-mid-walk'), + _ruleset(id='7', name='other'), + _ruleset(id='5', name='stops-mid-walk')] + with mock.patch('polyswarm_api.api.PolyswarmAPI.ruleset_list', + autospec=True, return_value=iter(repeated)): + result = CliRunner().invoke( + client.polyswarm_cli, + ['-a', '1' * 32, '-u', 'http://ai:9696/v3', '-c', 'gamma', + 'rules', 'list', '--sort', 'active-first'], + catch_exceptions=False) + assert result.exit_code == 0, result.output + assert result.output.count('stops-mid-walk') == 1, result.output + # The row between the duplicates still renders — dedupe, not truncation. + assert 'other' in result.output, result.output + + def test_the_default_order_is_not_narrowed_by_the_dedupe(self): + # The id-desc default cannot repeat a row, so every row it yields must + # still reach the output; the dedupe is unconditional and must be inert + # here rather than dropping a distinct row that happens to look alike. + rows = [_ruleset(id='5', name='first'), _ruleset(id='7', name='second')] + with mock.patch('polyswarm_api.api.PolyswarmAPI.ruleset_list', + autospec=True, return_value=iter(rows)): + result = CliRunner().invoke( + client.polyswarm_cli, + ['-a', '1' * 32, '-u', 'http://ai:9696/v3', '-c', 'gamma', + 'rules', 'list'], + catch_exceptions=False) + assert result.exit_code == 0, result.output + assert 'first' in result.output and 'second' in result.output, result.output + def test_sort_rejects_an_unknown_order(self): # A closed choice on the CLI side too: the server would 400 an unknown # sort, but the CLI should not have to make the round trip to say so. From 68e4f62c461ed652dcf6ef2504b992108fee1807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mart=C3=ADnez?= Date: Mon, 14 Sep 2026 19:36:13 -0300 Subject: [PATCH 6/9] test: pin the dedupe key as the id, not the rendered name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two dedupe tests repeated the same row wholesale and compared rows that differed in every field, so an implementation keyed on the name — or on the rendered block — passed both. Ruleset names are not unique, so that variant would swallow a real row from any inventory listing. The new case is the one that separates them: two distinct rulesets sharing a name must both render. Re-keying the dedupe on the name fails it and nothing else. --- tests/formatter_hunt_fields_test.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/formatter_hunt_fields_test.py b/tests/formatter_hunt_fields_test.py index 2dfe9d23..04372f42 100644 --- a/tests/formatter_hunt_fields_test.py +++ b/tests/formatter_hunt_fields_test.py @@ -219,6 +219,22 @@ def test_a_row_served_twice_by_the_mutable_sort_is_printed_once(self): # The row between the duplicates still renders — dedupe, not truncation. assert 'other' in result.output, result.output + def test_two_rulesets_sharing_a_name_both_render(self): + """The key is the id, and only the id. Ruleset names are not unique, so + an implementation that deduped on the name — or on the whole rendered + block — would swallow a real row from an inventory listing while passing + every other test in this class.""" + rows = [_ruleset(id='5', name='dup'), _ruleset(id='7', name='dup')] + with mock.patch('polyswarm_api.api.PolyswarmAPI.ruleset_list', + autospec=True, return_value=iter(rows)): + result = CliRunner().invoke( + client.polyswarm_cli, + ['-a', '1' * 32, '-u', 'http://ai:9696/v3', '-c', 'gamma', + 'rules', 'list', '--sort', 'active-first'], + catch_exceptions=False) + assert result.exit_code == 0, result.output + assert result.output.count('dup') == 2, result.output + def test_the_default_order_is_not_narrowed_by_the_dedupe(self): # The id-desc default cannot repeat a row, so every row it yields must # still reach the output; the dedupe is unconditional and must be inert From eb1f8102d9245e5f5a1f62b8acf825cfe3e4c64f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mart=C3=ADnez?= Date: Mon, 14 Sep 2026 19:43:01 -0300 Subject: [PATCH 7/9] docs+test: the surviving copy of a moved row is the stale one, and say so MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round two of review. The dedupe keeps the first copy — the only choice a streaming printer has, since that copy is already on stdout when the second arrives — and the first copy carries the values from BEFORE the transition. So the one row the dedupe acts on prints the Live Hunt Id of a hunt that has already stopped. The help said to read the field rather than the position; under a mid-walk move neither is authoritative, and the help, the command docstring and the commands catalogue now say that instead of promising it. The dedupe test repeated a row wholesale, which is not the shape a re-serve takes: the two copies differ in exactly the field the sort ranks on, because that is why the row moved. It uses that shape now, and a new case pins first-wins as a decision — a last-wins rewrite fails it and nothing else. The invariant itself moved to specs/05 §Consuming the SDK correctly, beside the other SDK-consumption rules, where the next command that walks a mutably ordered endpoint will find it; the commands catalogue points at it. --- specs/02-commands.md | 2 +- specs/05-sdk-contract.md | 26 +++++++++++++++++++++++ src/polyswarm/client/rules.py | 13 ++++++++++-- tests/formatter_hunt_fields_test.py | 32 ++++++++++++++++++++++++++--- 4 files changed, 67 insertions(+), 6 deletions(-) diff --git a/specs/02-commands.md b/specs/02-commands.md index 868fa41e..04808eb9 100644 --- a/specs/02-commands.md +++ b/specs/02-commands.md @@ -30,7 +30,7 @@ The top-level command groups, what each is for, and the primary `polyswarm-api` | `tag` (`tags.py`) | Tag CRUD | `tag_{create,delete,get,list}` | | `link` (`links.py`) | Tag/family links on artifacts | `tag_link_multiple`, `tag_link_get`, `tag_link_list` | | `family` (`families.py`) | Malware-family CRUD | `family_{create,update,delete,get,list}` | -| `rules` (`rules.py`) | YARA ruleset CRUD (`list` takes the server-side filters plus `--sort active-first`, the hunt page's order: rulesets carrying a live hunt link first — the rank is that stored link, wider than what Live Hunt Id renders from, so a legacy row that was stopped while STILL LINKED leads the list and renders no Live Hunt Id at all — read the field, never the position. The command walks every page and so dedupes by id, the obligation the SDK puts on a multi-page consumer of this mutable key (a row whose hunt stops mid-walk is served twice; one started mid-walk is missed until the next run); ordering is the server's, never a local re-sort of one keyset page) plus `favorite [--unfavorite]` (the star toggle: renders the new state + the server-owned "N of M used" budget, and converts the machine-readable `FAVORITE_LIMIT` refusal into a clean actionable message at exit 2, never 1 — 1 is reserved for no-results/not-found; 2 is the broad bucket `ExceptionHandlingGroup` maps the PolyswarmException hierarchies to. **2 does not identify a server refusal:** click exits 2 for a `UsageError` too, so a scripted caller cannot tell “the favorite budget is full” from “you passed a bad flag” without reading the message). `list` takes the server-side filters `--name` / `--status active` / `--favorites-only` / `--has-new-results` (conjunctive; the list is keyset-paginated, so filtering locally would mean walking every page). `rules favorite` and the `rules list` filters need SDK 4.4.0 and `rules list --sort` needs 4.5.0, which the pin requires (see [05-sdk-contract.md](./05-sdk-contract.md) §Current floor), so they are called directly. The formatters read the hunt-page fields directly: the pin guarantees the SDK parses them, so `None` means the *server* had no answer | `ruleset_{create,delete,update,get,list,favorite}` | +| `rules` (`rules.py`) | YARA ruleset CRUD (`list` takes the server-side filters plus `--sort active-first`, the hunt page's order: rulesets carrying a live hunt link first — the rank is that stored link, wider than what Live Hunt Id renders from, so a legacy row that was stopped while STILL LINKED leads the list and renders no Live Hunt Id at all — the position is not evidence a hunt is running, and for a row that moved mid-walk neither is the field. The command walks every page and so dedupes by id, keeping the first copy — see [05-sdk-contract.md](./05-sdk-contract.md) §A mutable order makes the walk the caller's problem for what that costs; ordering is the server's, never a local re-sort of one keyset page) plus `favorite [--unfavorite]` (the star toggle: renders the new state + the server-owned "N of M used" budget, and converts the machine-readable `FAVORITE_LIMIT` refusal into a clean actionable message at exit 2, never 1 — 1 is reserved for no-results/not-found; 2 is the broad bucket `ExceptionHandlingGroup` maps the PolyswarmException hierarchies to. **2 does not identify a server refusal:** click exits 2 for a `UsageError` too, so a scripted caller cannot tell “the favorite budget is full” from “you passed a bad flag” without reading the message). `list` takes the server-side filters `--name` / `--status active` / `--favorites-only` / `--has-new-results` (conjunctive; the list is keyset-paginated, so filtering locally would mean walking every page). `rules favorite` and the `rules list` filters need SDK 4.4.0 and `rules list --sort` needs 4.5.0, which the pin requires (see [05-sdk-contract.md](./05-sdk-contract.md) §Current floor), so they are called directly. The formatters read the hunt-page fields directly: the pin guarantees the SDK parses them, so `None` means the *server* had no answer | `ruleset_{create,delete,update,get,list,favorite}` | | `metadata` (`metadata.py`) | Rerun metadata; scan lookup; IP/URL analysis | `rerun_metadata`, `scan_lookup`, `submit_url` | | `activity` (`event.py`) | List account activity/events | `event_list` | | `account` (`account.py`) | Account whois / features | `account_whois`, `account_features` | diff --git a/specs/05-sdk-contract.md b/specs/05-sdk-contract.md index 3eb39c27..ed2ae902 100644 --- a/specs/05-sdk-contract.md +++ b/specs/05-sdk-contract.md @@ -39,6 +39,32 @@ for item in api.iocs_by_hash(type, value): Because the generators are **lazy**, calling one does no I/O and raises nothing until iterated. Code that runs SDK calls through a thread pool must consume the generator **inside the worker** so per-item exception handling fires where it's expected — this is why `utils.parallel_executor_iterable_results` materialises each generator inside the submitted callable (see `01-architecture.md`). +### A mutable order makes the walk the caller's problem + +Most list endpoints are keyset-paginated on an immutable key, so a walk sees every row once. +`ruleset_list(sort='active_first')` is the exception in the current surface: its key is the +live-hunt link, which the hunt itself flips, and the SDK's own docstring puts the consequence +on the caller — *"This generator streams pages and does not dedupe — dedupe by `id` if you +consume more than one page"*. + +A command that walks every page of such an endpoint must therefore: + +- **Dedupe by `id`.** A row whose key changes mid-walk drops below the cursor and is served + again. `rules list` keeps a `seen` set (`client/rules.py`); the dedupe is unconditional, + because the id is unique under either order and a gate is one more thing to update when the + next mutable order appears. +- **Keep the FIRST copy, and know what that costs.** A streaming printer has already written + copy one when copy two arrives, so first-wins is the only option — and copy one carries the + PRE-transition values. A ruleset whose hunt stopped mid-walk prints with its old + `Live Hunt Id`. Under a mutable order neither the position nor the row's own fields are + authoritative for a row that moved; a fresh run shows the settled state. +- **Say both in the command's help**, not only here. The user reading `--help` is the one who + will act on a stale field. + +What cannot be repaired client-side: a row whose key changes so that it moves ABOVE the +cursor is never served at all, so it is missing from that walk entirely. Document it; a +re-run lists it. + ### No-results signalling A search that the server answers `204 No Content` raises `NoResultsException` from the SDK **when the generator is iterated**. The CLI's `ExceptionHandlingGroup` maps that (and the CLI's own aggregate `NoResultsException` from `parallel_executor`) to exit code `1`. Don't swallow it in command code. diff --git a/src/polyswarm/client/rules.py b/src/polyswarm/client/rules.py index a558b63b..51f45cf1 100644 --- a/src/polyswarm/client/rules.py +++ b/src/polyswarm/client/rules.py @@ -44,8 +44,10 @@ def delete(ctx, rule_id): 'stored link, which is WIDER than what Live Hunt Id renders ' 'from: a legacy row whose hunt was stopped without clearing ' 'the link leads the list while rendering no Live Hunt Id at ' - 'all, indistinguishable from an idle one. Read the field, ' - 'never the position.') + 'all, indistinguishable from an idle one — so the position ' + 'is not evidence that a hunt is running. Neither is the ' + 'field for a row that MOVED during the walk: see the note ' + 'on stale copies below.') @click.pass_context def list_rules(ctx, name, status, favorites_only, has_new_results, sort): """List rulesets, optionally filtered. All filters are conjunctive. @@ -60,6 +62,13 @@ def list_rules(ctx, name, status, favorites_only, has_new_results, sort): between two page fetches drops below the cursor and the server serves it a second time. Rows are therefore emitted at most once per run, keyed on id. + The copy that survives is the FIRST one, which is the only choice a + streaming printer has — it wrote that copy before the second arrived — and + it carries the values from BEFORE the transition. So the one row the dedupe + acts on prints its old Live Hunt Id, for a hunt that has since stopped. + Under this order a moved row is authoritative in neither its position nor + its fields; a fresh run shows the settled state. + The symmetric case cannot be repaired from here and is not hidden: a hunt STARTED mid-walk moves its ruleset above the cursor, so that row never reaches this client at all. A re-run lists it. diff --git a/tests/formatter_hunt_fields_test.py b/tests/formatter_hunt_fields_test.py index 04372f42..e191859b 100644 --- a/tests/formatter_hunt_fields_test.py +++ b/tests/formatter_hunt_fields_test.py @@ -203,10 +203,13 @@ def test_a_row_served_twice_by_the_mutable_sort_is_printed_once(self): dedupe obligation on. Under the active-first order a ruleset whose hunt stops between two page fetches falls below the cursor and the server serves it again; without the dedupe the run prints it twice and any - script counting the output double-counts it.""" - repeated = [_ruleset(id='5', name='stops-mid-walk'), + script counting the output double-counts it. + + The two copies differ in the field the sort ranks on — that is WHY the + row moved — so this is the real re-serve shape, not a repeated row.""" + repeated = [_ruleset(id='5', name='stops-mid-walk', livescan_id='77'), _ruleset(id='7', name='other'), - _ruleset(id='5', name='stops-mid-walk')] + _ruleset(id='5', name='stops-mid-walk', livescan_id=None)] with mock.patch('polyswarm_api.api.PolyswarmAPI.ruleset_list', autospec=True, return_value=iter(repeated)): result = CliRunner().invoke( @@ -219,6 +222,29 @@ def test_a_row_served_twice_by_the_mutable_sort_is_printed_once(self): # The row between the duplicates still renders — dedupe, not truncation. assert 'other' in result.output, result.output + def test_the_surviving_copy_is_the_first_one_stale_values_and_all(self): + """First-wins is a decision, not an accident of the loop: a streaming + printer has already written copy one when copy two arrives. The cost is + that the surviving copy carries the PRE-transition values — the row + prints the Live Hunt Id of a hunt that has since stopped — which is why + the help and specs/05 say a moved row is authoritative in neither its + position nor its fields. A last-wins rewrite would flip this.""" + served = [_ruleset(id='5', name='stops-mid-walk', livescan_id='77'), + _ruleset(id='5', name='stops-mid-walk', livescan_id=None)] + with mock.patch('polyswarm_api.api.PolyswarmAPI.ruleset_list', + autospec=True, return_value=iter(served)): + result = CliRunner().invoke( + client.polyswarm_cli, + ['-a', '1' * 32, '-u', 'http://ai:9696/v3', '-c', 'gamma', + 'rules', 'list', '--sort', 'active-first'], + catch_exceptions=False) + assert result.exit_code == 0, result.output + assert result.output.count('stops-mid-walk') == 1, result.output + # The formatter gates the pair on a truthy livescan_id, so the line is + # present iff the copy that survived is the one from before the stop. + assert 'Live Hunt Id' in result.output, result.output + assert '77' in result.output, result.output + def test_two_rulesets_sharing_a_name_both_render(self): """The key is the id, and only the id. Ruleset names are not unique, so an implementation that deduped on the name — or on the whole rendered From 1620f4af5d2eedae91b2d29164dadd08f32fe726 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mart=C3=ADnez?= Date: Mon, 14 Sep 2026 20:04:04 -0300 Subject: [PATCH 8/9] test+docs: give the sort and dedupe pins their own class, and trim the duplicated prose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seven tests about sort forwarding and mid-walk dedupe had accumulated inside a class whose docstring promised only the zero-argument call, so nobody landing on the name would expect the dedupe pins to be there. They are their own class now, with a docstring that states both contracts it holds; the module docstring enumerates the new one alongside the others. The same ~150 words about stale Live Hunt Id values were restated in the help, the commands catalogue and the SDK-contract spec. The help keeps them — the user reading --help is the one who acts on a stale field — and the catalogue cell, the copy most likely to drift, is back to the one-line claim plus its existing pointer. --- specs/02-commands.md | 2 +- tests/formatter_hunt_fields_test.py | 32 ++++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/specs/02-commands.md b/specs/02-commands.md index 04808eb9..71c869ff 100644 --- a/specs/02-commands.md +++ b/specs/02-commands.md @@ -30,7 +30,7 @@ The top-level command groups, what each is for, and the primary `polyswarm-api` | `tag` (`tags.py`) | Tag CRUD | `tag_{create,delete,get,list}` | | `link` (`links.py`) | Tag/family links on artifacts | `tag_link_multiple`, `tag_link_get`, `tag_link_list` | | `family` (`families.py`) | Malware-family CRUD | `family_{create,update,delete,get,list}` | -| `rules` (`rules.py`) | YARA ruleset CRUD (`list` takes the server-side filters plus `--sort active-first`, the hunt page's order: rulesets carrying a live hunt link first — the rank is that stored link, wider than what Live Hunt Id renders from, so a legacy row that was stopped while STILL LINKED leads the list and renders no Live Hunt Id at all — the position is not evidence a hunt is running, and for a row that moved mid-walk neither is the field. The command walks every page and so dedupes by id, keeping the first copy — see [05-sdk-contract.md](./05-sdk-contract.md) §A mutable order makes the walk the caller's problem for what that costs; ordering is the server's, never a local re-sort of one keyset page) plus `favorite [--unfavorite]` (the star toggle: renders the new state + the server-owned "N of M used" budget, and converts the machine-readable `FAVORITE_LIMIT` refusal into a clean actionable message at exit 2, never 1 — 1 is reserved for no-results/not-found; 2 is the broad bucket `ExceptionHandlingGroup` maps the PolyswarmException hierarchies to. **2 does not identify a server refusal:** click exits 2 for a `UsageError` too, so a scripted caller cannot tell “the favorite budget is full” from “you passed a bad flag” without reading the message). `list` takes the server-side filters `--name` / `--status active` / `--favorites-only` / `--has-new-results` (conjunctive; the list is keyset-paginated, so filtering locally would mean walking every page). `rules favorite` and the `rules list` filters need SDK 4.4.0 and `rules list --sort` needs 4.5.0, which the pin requires (see [05-sdk-contract.md](./05-sdk-contract.md) §Current floor), so they are called directly. The formatters read the hunt-page fields directly: the pin guarantees the SDK parses them, so `None` means the *server* had no answer | `ruleset_{create,delete,update,get,list,favorite}` | +| `rules` (`rules.py`) | YARA ruleset CRUD (`list` takes the server-side filters plus `--sort active-first`, the hunt page's order: rulesets carrying a live hunt link first, deduped by id because the key is mutable — neither the position nor a moved row's own fields are evidence a hunt is running, see [05-sdk-contract.md](./05-sdk-contract.md) §A mutable order makes the walk the caller's problem. Ordering is the server's, never a local re-sort of one keyset page) plus `favorite [--unfavorite]` (the star toggle: renders the new state + the server-owned "N of M used" budget, and converts the machine-readable `FAVORITE_LIMIT` refusal into a clean actionable message at exit 2, never 1 — 1 is reserved for no-results/not-found; 2 is the broad bucket `ExceptionHandlingGroup` maps the PolyswarmException hierarchies to. **2 does not identify a server refusal:** click exits 2 for a `UsageError` too, so a scripted caller cannot tell “the favorite budget is full” from “you passed a bad flag” without reading the message). `list` takes the server-side filters `--name` / `--status active` / `--favorites-only` / `--has-new-results` (conjunctive; the list is keyset-paginated, so filtering locally would mean walking every page). `rules favorite` and the `rules list` filters need SDK 4.4.0 and `rules list --sort` needs 4.5.0, which the pin requires (see [05-sdk-contract.md](./05-sdk-contract.md) §Current floor), so they are called directly. The formatters read the hunt-page fields directly: the pin guarantees the SDK parses them, so `None` means the *server* had no answer | `ruleset_{create,delete,update,get,list,favorite}` | | `metadata` (`metadata.py`) | Rerun metadata; scan lookup; IP/URL analysis | `rerun_metadata`, `scan_lookup`, `submit_url` | | `activity` (`event.py`) | List account activity/events | `event_list` | | `account` (`account.py`) | Account whois / features | `account_whois`, `account_features` | diff --git a/tests/formatter_hunt_fields_test.py b/tests/formatter_hunt_fields_test.py index e191859b..52a22921 100644 --- a/tests/formatter_hunt_fields_test.py +++ b/tests/formatter_hunt_fields_test.py @@ -15,7 +15,10 @@ favorite`` renders the toggle response and converts the machine-readable FAVORITE_LIMIT refusal into a clean message. All are asserted through autospec'd mocks, so every call is signature-checked against the SDK the - pin actually installs. + pin actually installs; and +* the active-first order (4.5.0): the ``--sort`` token that reaches the server, + and what walking every page of a MUTABLY ordered list obliges this command to + do — dedupe by id, first copy wins, stale values and all. """ from unittest import TestCase, mock @@ -132,10 +135,13 @@ def test_hunt_unknown_tri_state_prints_nothing(self): class RulesListZeroArgTest(TestCase): - """`rules list` calls a zero-argument ``ruleset_list()`` — a False flag - is not a filter, so an unfiltered list forwards no - behaviour at all. autospec makes the assertion a signature check against - the installed SDK.""" + """`rules list` calls a zero-argument ``ruleset_list()`` — a False flag is + not a filter, so an unfiltered list forwards no behaviour at all, and a + filtered one forwards exactly the filters given. autospec makes each + assertion a signature check against the installed SDK. + + The `--sort` token and the mid-walk dedupe live in + ``RulesListSortAndDedupeTest`` below.""" def test_list_passes_no_kwargs_at_all(self): with mock.patch('polyswarm_api.api.PolyswarmAPI.ruleset_list', @@ -168,6 +174,22 @@ def test_filters_are_forwarded_only_when_given(self): mock.ANY, name='alpha', status='active', favorites_only=True, has_new_results=True) + +class RulesListSortAndDedupeTest(TestCase): + """`rules list --sort active-first` — the token that reaches the server, and + what walking every page of a MUTABLY ordered list obliges this command to do. + + Two separate contracts. The token: the CLI spelling is hyphenated, the + server's is not, the sort is forwarded only when given, and an unknown one + is refused here rather than at the server. The walk: the ordering key is the + live-hunt link the sort ranks on, so a row whose hunt changes mid-walk is + served twice or missed — this command dedupes by id, keeps the FIRST copy, + and therefore renders that row's PRE-transition values + (specs/05-sdk-contract.md §A mutable order makes the walk the caller's + problem). Each dedupe test fails against a different wrong implementation: + keyed on the name, or last-wins. + """ + def test_sort_active_first_is_forwarded_as_the_server_token(self): """`--sort active-first` (CLI spelling, hyphen) reaches the SDK as the server's `sort='active_first'` token — and, like the filters, only From 98b4842639c0e9feda1f7f1bac6e049542bd388b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Mart=C3=ADnez?= Date: Mon, 14 Sep 2026 22:45:59 -0300 Subject: [PATCH 9/9] feat(rules): list --exclude-favorites The inverse of --favorites-only, which the server refuses to combine with it. It exists for a client that renders the favorites as their own list: leaving them in the paginated list too makes a page repeat a row or come back short. A False flag is not a filter here either, so an unflagged invocation sends nothing new. --- specs/02-commands.md | 2 +- src/polyswarm/client/rules.py | 7 ++++++- tests/formatter_hunt_fields_test.py | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/specs/02-commands.md b/specs/02-commands.md index 71c869ff..62a62aae 100644 --- a/specs/02-commands.md +++ b/specs/02-commands.md @@ -30,7 +30,7 @@ The top-level command groups, what each is for, and the primary `polyswarm-api` | `tag` (`tags.py`) | Tag CRUD | `tag_{create,delete,get,list}` | | `link` (`links.py`) | Tag/family links on artifacts | `tag_link_multiple`, `tag_link_get`, `tag_link_list` | | `family` (`families.py`) | Malware-family CRUD | `family_{create,update,delete,get,list}` | -| `rules` (`rules.py`) | YARA ruleset CRUD (`list` takes the server-side filters plus `--sort active-first`, the hunt page's order: rulesets carrying a live hunt link first, deduped by id because the key is mutable — neither the position nor a moved row's own fields are evidence a hunt is running, see [05-sdk-contract.md](./05-sdk-contract.md) §A mutable order makes the walk the caller's problem. Ordering is the server's, never a local re-sort of one keyset page) plus `favorite [--unfavorite]` (the star toggle: renders the new state + the server-owned "N of M used" budget, and converts the machine-readable `FAVORITE_LIMIT` refusal into a clean actionable message at exit 2, never 1 — 1 is reserved for no-results/not-found; 2 is the broad bucket `ExceptionHandlingGroup` maps the PolyswarmException hierarchies to. **2 does not identify a server refusal:** click exits 2 for a `UsageError` too, so a scripted caller cannot tell “the favorite budget is full” from “you passed a bad flag” without reading the message). `list` takes the server-side filters `--name` / `--status active` / `--favorites-only` / `--has-new-results` (conjunctive; the list is keyset-paginated, so filtering locally would mean walking every page). `rules favorite` and the `rules list` filters need SDK 4.4.0 and `rules list --sort` needs 4.5.0, which the pin requires (see [05-sdk-contract.md](./05-sdk-contract.md) §Current floor), so they are called directly. The formatters read the hunt-page fields directly: the pin guarantees the SDK parses them, so `None` means the *server* had no answer | `ruleset_{create,delete,update,get,list,favorite}` | +| `rules` (`rules.py`) | YARA ruleset CRUD (`list` takes the server-side filters plus `--sort active-first`, the hunt page's order: rulesets carrying a live hunt link first, deduped by id because the key is mutable — neither the position nor a moved row's own fields are evidence a hunt is running, see [05-sdk-contract.md](./05-sdk-contract.md) §A mutable order makes the walk the caller's problem. Ordering is the server's, never a local re-sort of one keyset page) plus `favorite [--unfavorite]` (the star toggle: renders the new state + the server-owned "N of M used" budget, and converts the machine-readable `FAVORITE_LIMIT` refusal into a clean actionable message at exit 2, never 1 — 1 is reserved for no-results/not-found; 2 is the broad bucket `ExceptionHandlingGroup` maps the PolyswarmException hierarchies to. **2 does not identify a server refusal:** click exits 2 for a `UsageError` too, so a scripted caller cannot tell “the favorite budget is full” from “you passed a bad flag” without reading the message). `list` takes the server-side filters `--name` / `--status active` / `--favorites-only` / `--exclude-favorites` (its inverse, refused together with it — for a client that lists the favorites separately) / `--has-new-results` (conjunctive; the list is keyset-paginated, so filtering locally would mean walking every page). `rules favorite` and the `rules list` filters need SDK 4.4.0 and `rules list --sort` needs 4.5.0, which the pin requires (see [05-sdk-contract.md](./05-sdk-contract.md) §Current floor), so they are called directly. The formatters read the hunt-page fields directly: the pin guarantees the SDK parses them, so `None` means the *server* had no answer | `ruleset_{create,delete,update,get,list,favorite}` | | `metadata` (`metadata.py`) | Rerun metadata; scan lookup; IP/URL analysis | `rerun_metadata`, `scan_lookup`, `submit_url` | | `activity` (`event.py`) | List account activity/events | `event_list` | | `account` (`account.py`) | Account whois / features | `account_whois`, `account_features` | diff --git a/src/polyswarm/client/rules.py b/src/polyswarm/client/rules.py index 51f45cf1..2b3aa376 100644 --- a/src/polyswarm/client/rules.py +++ b/src/polyswarm/client/rules.py @@ -36,6 +36,10 @@ def delete(ctx, rule_id): @click.option('-s', '--status', type=click.Choice(['active']), help='Only rulesets whose live hunt is currently running.') @click.option('--favorites-only', is_flag=True, help='Only favorited (starred) rulesets.') +@click.option('--exclude-favorites', is_flag=True, + help='Only rulesets that are NOT favorited. The inverse of ' + '--favorites-only, and refused together with it. For a ' + 'client that lists the favorites separately.') @click.option('--has-new-results', is_flag=True, help='Only rulesets whose stored new-results counter is positive.') @click.option('--sort', type=click.Choice(['active-first']), @@ -49,7 +53,7 @@ def delete(ctx, rule_id): 'field for a row that MOVED during the walk: see the note ' 'on stale copies below.') @click.pass_context -def list_rules(ctx, name, status, favorites_only, has_new_results, sort): +def list_rules(ctx, name, status, favorites_only, exclude_favorites, has_new_results, sort): """List rulesets, optionally filtered. All filters are conjunctive. Filtering and ordering are applied SERVER-side: the list is @@ -79,6 +83,7 @@ def list_rules(ctx, name, status, favorites_only, has_new_results, sort): # The CLI spells the sort with a hyphen; the server token is 'active_first'. kwargs = {k: v for k, v in (('name', name), ('status', status), ('favorites_only', favorites_only or None), + ('exclude_favorites', exclude_favorites or None), ('has_new_results', has_new_results or None), ('sort', sort.replace('-', '_') if sort else None)) if v is not None} diff --git a/tests/formatter_hunt_fields_test.py b/tests/formatter_hunt_fields_test.py index 52a22921..05c7afa8 100644 --- a/tests/formatter_hunt_fields_test.py +++ b/tests/formatter_hunt_fields_test.py @@ -298,6 +298,22 @@ def test_the_default_order_is_not_narrowed_by_the_dedupe(self): assert result.exit_code == 0, result.output assert 'first' in result.output and 'second' in result.output, result.output + def test_exclude_favorites_is_forwarded_only_when_given(self): + """The inverse filter the hunt page sends alongside the sort: the + favorites are their own list above the page, so the paginated list asks + for the non-favorites. Like every other flag here, a False one is not a + filter and must not reach the SDK.""" + with mock.patch('polyswarm_api.api.PolyswarmAPI.ruleset_list', + autospec=True, return_value=iter(())) as ruleset_list: + result = CliRunner().invoke( + client.polyswarm_cli, + ['-a', '1' * 32, '-u', 'http://ai:9696/v3', '-c', 'gamma', + 'rules', 'list', '--exclude-favorites', '--sort', 'active-first'], + catch_exceptions=False) + assert result.exit_code == 0, result.output + ruleset_list.assert_called_once_with( + mock.ANY, exclude_favorites=True, sort='active_first') + def test_sort_rejects_an_unknown_order(self): # A closed choice on the CLI side too: the server would 400 an unknown # sort, but the CLI should not have to make the round trip to say so.