diff --git a/pyproject.toml b/pyproject.toml index 682d705c..142d64bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "polyswarm_api" -version = "4.6.0" +version = "4.7.0" description = "Client library to simplify interacting with the PolySwarm consumer API" readme = "README.md" requires-python = ">=3.10,<4" @@ -55,7 +55,7 @@ package-dir = { "" = "src" } where = ["src"] [tool.bumpversion] -current_version = "4.6.0" +current_version = "4.7.0" commit = true tag = false sign_tags = true diff --git a/specs/03-endpoints.md b/specs/03-endpoints.md index 12e4f857..44c2ea66 100644 --- a/specs/03-endpoints.md +++ b/specs/03-endpoints.md @@ -185,7 +185,7 @@ refusal. | `search_scans(hash_)` | `ArtifactInstance.list_scans` | | `search_by_metadata(query, include=None, exclude=None, ips=None, urls=None, domains=None)` | `Metadata.get` | | `iocs_by_hash(hash_type, hash_value, hide_known_good=False, beta=False)` | `IOC.iocs_by_hash` | -| `search_by_ioc(ip=None, domain=None, ttp=None, imphash=None)` | `IOC.ioc_search` | +| `search_by_ioc(ip=None, domain=None, ttp=None, imphash=None, with_artifacts=False)` | `IOC.ioc_search` — the reverse IOC search. By default each item is an `IOC` whose `json` is a bare sha256 string, and the request carries no `with_artifacts`, so it is byte-compatible with the old contract. `with_artifacts=True` (4.7.0) sends `with_artifacts=1` (an int: the session renders a bool as `'True'`, which the server's boolean parser refuses) and yields `Metadata` resources instead — metadata-search rows trimmed to an include set the SERVER owns (today `artifact.*`; `scan.first_seen`, `scan.first_scan.created`, `scan.last_seen`, `scan.detections`, `scan.mimetype`; `scan.latest_scan.{polyscore,artifact_instance_id,created}`; `scan.filename`, `scan.url`; `polyunite.malware_family`; `hash.ssdeep`, `hash.tlsh`). So `first_seen`, `last_scanned`, the detection counts, mimetypes, filenames, `ssdeep` and `tlsh` are populated; attributes read from outside the set (the `strings.*` IOC lists, for instance) parse as `None` or empty. The SDK does not restate the set: a server-side change to it shows up here without an SDK release. Against a server that predates the parameter the flag is ignored upstream and the sha256 strings fail to parse as `Metadata` with a `TypeError` — so this SDK version must not be released ahead of that server. The SDK does not validate the terms. With `with_artifacts=True` and none of ip/domain/ttp/imphash the server answers 400, which surfaces as the usual typed exception; without the flag a bare call behaves exactly as before. | | `check_known_hosts(ips=[], domains=[])` | `IOC.check_known_hosts` | | `live_feed(since=None, …, livescan_id=None, max_results=None)` | `LiveHuntResult.list` — `livescan_id` scopes the feed to one live hunt (the hunt-page per-ruleset feed); `since` is in **SECONDS** (the server converts with `timedelta(seconds=since)`; the 3.x/4.x docstring said minutes and was wrong), and absent-or-`0` means no time filter at all — the server applies it on a truthiness test; `max_results` bounds how many results the generator yields — `None`/`0`/negative means no bound; it does not alter the request | | `historical_list(since=None)` | `HistoricalHunt.list` | diff --git a/specs/04-testing.md b/specs/04-testing.md index 19d7c6b2..6930af7e 100644 --- a/specs/04-testing.md +++ b/specs/04-testing.md @@ -31,6 +31,7 @@ How the test suite is organised. Three layers: pure unit tests (no HTTP at all - `test/malicious` — fixture file for upload tests (`test/eicar.yara` was retired when the rules tests moved to per-test `uid_yara` bodies). - `test/hunt_tracking_builder_test.py` — pure-unit request-shape and parse tests for the hunt-page tracking builders/resources. - `test/refang_test.py` — pure-unit tests for `polyswarm_api.refang` (driven by the shared case table `test/fixtures/refang_cases.json`, kept byte-identical with the other clients that implement the same contract) and request-shape tests for every client method that refangs its IoC inputs, captured at the `_paginate` / `_single` boundary for both transports. Pure-unit because this is client-side input normalization: the server contract is unchanged, so there is no new endpoint behaviour for a cassette to pin. +- `test/ioc_search_test.py` — `search_by_ioc(with_artifacts=)`: pure-unit request shape (the int flag, the unchanged default and bare call), pass-through on both client methods, and a dual-transport (`ClientTestCase`) respx parse of an artifact row — a stand-in until the e2e stack serves the parameter (see `99-open-questions.md`). - `test/ruleset_favorite_respx_test.py` — dual-transport (`ClientTestCase`) respx suite for the favorite toggle: the `FAVORITE_LIMIT` refusal envelope and the query/body split. ## Three test layers diff --git a/specs/05-downstream-contract.md b/specs/05-downstream-contract.md index 271c6db6..360bbe99 100644 --- a/specs/05-downstream-contract.md +++ b/specs/05-downstream-contract.md @@ -13,7 +13,7 @@ This spec describes the **4.0 surface**. The 3.x → 4.0 migration is covered in 3. **Exception class names and the inheritance hierarchy are part of the contract.** Callers catch on specific subclasses (`except NotFoundException:`). 4. **The session classes `PolyswarmSession` / `AsyncPolyswarmSession` are the customization point.** Subclass them, override the methods you want to change, and pass via `PolyswarmAPI(session=...)` / `PolySwarmAsyncAPI(session=...)`. There are no module-level monkey-patch sites. 5. **The `[async]` extras group is preserved.** Downstream consumers pin `polyswarm-api[async]`. The extra is an empty list (since `httpx` is a core dependency) but the name must remain so old pin specs parse. -6. **Version bumps go on the `develop → master` step, not feature PRs.** A PyPI release fires automatically when `pyproject.toml` `version` changes on `master`. +6. **Version bumps go on the `develop → master` step, not feature PRs.** A PyPI release fires automatically when `pyproject.toml` `version` changes on `master`. The one exception is AGENTS.md's standing exception: when a sibling resolves this repo from source by branch name and must raise its `polyswarm_api>=` floor to the version introducing a surface, the feature PR carries the bump, because a floor cannot name a version this repo has not declared. That bump must emit a clean `X.Y.0`: PEP 440 orders a `.devN` suffix below the release, so a suffixed version fails the sibling's floor and sends its CI to PyPI for a version that does not exist yet. ## Files diff --git a/specs/99-open-questions.md b/specs/99-open-questions.md index 4882a14f..5d4dfc1a 100644 --- a/specs/99-open-questions.md +++ b/specs/99-open-questions.md @@ -210,3 +210,19 @@ Producing an over-budget match on the e2e stack means a rule whose matches excee server's per-hunt byte budget across a single artifact — engineering a fixture for that is disproportionate to what it would pin. **Recorded rather than tested, deliberately.** If a stack fixture ever produces one cheaply, assert both claims there and delete this entry. + +## `search_by_ioc(with_artifacts=True)` is not pinned against a live server + +**Status:** gap, blocked on the server leg reaching the e2e stack. + +The artifact-row shape is asserted only against a fabricated respx envelope +(`test/ioc_search_test.py`), cut by hand to the server's current include set (`IOC_ARTIFACT_INCLUDES` +upstream; listed in `03-endpoints.md`), so it drifts silently if the server changes that set. +The default sha256 path is still covered live by `test_search_by_ioc` / +`test_async_search_by_ioc`. That is the "asserts what we *think* the server returns" gap +invariant 1 exists to close. + +**Action:** once the e2e stack serves the parameter, add a `with_artifacts=True` pass to +the live `test_search_by_ioc` pair, record both cassettes against a fresh stack, and delete +the respx `IocSearchWithArtifactsTestCase` along with this entry. The builder and +pass-through tests in that module stay: they pin request shape, which a cassette cannot. diff --git a/src/polyswarm_api/__init__.py b/src/polyswarm_api/__init__.py index e80be0e3..5cf1f295 100644 --- a/src/polyswarm_api/__init__.py +++ b/src/polyswarm_api/__init__.py @@ -1,5 +1,5 @@ # https://www.python.org/dev/peps/pep-0008/#module-level-dunder-names -__version__ = '4.6.0' +__version__ = '4.7.0' __release_url__ = 'https://api.github.com/repos/polyswarm/polyswarm-api/releases/latest' from . import api diff --git a/src/polyswarm_api/aio/api.py b/src/polyswarm_api/aio/api.py index 7cbc1ab9..ff13bf62 100644 --- a/src/polyswarm_api/aio/api.py +++ b/src/polyswarm_api/aio/api.py @@ -410,19 +410,28 @@ async def iocs_by_hash(self, hash_type, hash_value, hide_known_good=False, beta= async for item in self._paginate(resources.IOC.iocs_by_hash(self, hash_value, hash_type, hide_known_good=hide_known_good, beta=beta)): yield item - async def search_by_ioc(self, ip=None, domain=None, ttp=None, imphash=None): + async def search_by_ioc(self, ip=None, domain=None, ttp=None, imphash=None, with_artifacts=False): """ Search artifacts by IOC (ip, domain, ttp, or imphash) - + :param ip: ip address to search by :param domain: domain address to search by :param ttp: ttp to search by :param imphash: ImpHash to search by - :return: Generator of ArtifactInstance resources + :param with_artifacts: True yields a Metadata resource per matching artifact + (a metadata-search row trimmed to a field set the server owns: artifact.*, + the scan summary, ssdeep/tlsh and the malware family) instead of its bare + sha256. With no ip/domain/ttp/imphash the server refuses it with a 400 + (a typed exception); without it a bare call behaves as it always has. + Needs a server that supports the parameter: + an older one ignores it and answers bare sha256 strings, which fail to + parse as Metadata (TypeError). + :return: Generator of IOC resources whose ``json`` is a sha256 string, or of + Metadata resources when ``with_artifacts`` is True """ ip, domain = self._refang(ip), self._refang(domain) - logger.info('Searching by ioc %s', dict(ip=ip, domain=domain, ttp=ttp, imphash=imphash)) - async for item in self._paginate(resources.IOC.ioc_search(self, ip=ip, domain=domain, ttp=ttp, imphash=imphash)): + logger.info('Searching by ioc %s', dict(ip=ip, domain=domain, ttp=ttp, imphash=imphash, with_artifacts=with_artifacts)) + async for item in self._paginate(resources.IOC.ioc_search(self, ip=ip, domain=domain, ttp=ttp, imphash=imphash, with_artifacts=with_artifacts)): yield item async def check_known_hosts(self, ips=[], domains=[]): diff --git a/src/polyswarm_api/api.py b/src/polyswarm_api/api.py index 3ac8beda..4d2a05e4 100644 --- a/src/polyswarm_api/api.py +++ b/src/polyswarm_api/api.py @@ -459,7 +459,9 @@ def iocs_by_hash(self, hash_type, hash_value, hide_known_good=False, beta=False) ): yield item - def search_by_ioc(self, ip=None, domain=None, ttp=None, imphash=None): + def search_by_ioc( + self, ip=None, domain=None, ttp=None, imphash=None, with_artifacts=False + ): """ Search artifacts by IOC (ip, domain, ttp, or imphash) @@ -467,15 +469,36 @@ def search_by_ioc(self, ip=None, domain=None, ttp=None, imphash=None): :param domain: domain address to search by :param ttp: ttp to search by :param imphash: ImpHash to search by - :return: Generator of ArtifactInstance resources + :param with_artifacts: True yields a Metadata resource per matching artifact + (a metadata-search row trimmed to a field set the server owns: artifact.*, + the scan summary, ssdeep/tlsh and the malware family) instead of its bare + sha256. With no ip/domain/ttp/imphash the server refuses it with a 400 + (a typed exception); without it a bare call behaves as it always has. + Needs a server that supports the parameter: + an older one ignores it and answers bare sha256 strings, which fail to + parse as Metadata (TypeError). + :return: Generator of IOC resources whose ``json`` is a sha256 string, or of + Metadata resources when ``with_artifacts`` is True """ ip, domain = self._refang(ip), self._refang(domain) logger.info( - "Searching by ioc %s", dict(ip=ip, domain=domain, ttp=ttp, imphash=imphash) + "Searching by ioc %s", + dict( + ip=ip, + domain=domain, + ttp=ttp, + imphash=imphash, + with_artifacts=with_artifacts, + ), ) for item in self._paginate( resources.IOC.ioc_search( - self, ip=ip, domain=domain, ttp=ttp, imphash=imphash + self, + ip=ip, + domain=domain, + ttp=ttp, + imphash=imphash, + with_artifacts=with_artifacts, ) ): yield item diff --git a/src/polyswarm_api/resources.py b/src/polyswarm_api/resources.py index 19a26e10..d75d9808 100644 --- a/src/polyswarm_api/resources.py +++ b/src/polyswarm_api/resources.py @@ -169,7 +169,7 @@ def iocs_by_hash(cls, api, hash_value, hash_type, hide_known_good=False, beta=Fa ) @classmethod - def ioc_search(cls, api, ip=None, domain=None, ttp=None, imphash=None): + def ioc_search(cls, api, ip=None, domain=None, ttp=None, imphash=None, with_artifacts=False): params = dict(community=api.community) if ip is not None: params['ip'] = ip @@ -179,12 +179,17 @@ def ioc_search(cls, api, ip=None, domain=None, ttp=None, imphash=None): params['ttp'] = ttp if imphash is not None: params['imphash'] = imphash + if with_artifacts: + # An int, not a bool: the session renders bools as 'True', which the + # server's boolean parser refuses (it accepts only 0/1/false/true). + params['with_artifacts'] = 1 return core.PolyswarmRequest( api=api, method='GET', url=f'{api.uri}/ioc/search', params=params, - result_parser=cls, + # Opt-in rows are metadata-search documents; the default is bare sha256s. + result_parser=Metadata if with_artifacts else cls, ) @classmethod diff --git a/test/ioc_search_test.py b/test/ioc_search_test.py new file mode 100644 index 00000000..7208cbab --- /dev/null +++ b/test/ioc_search_test.py @@ -0,0 +1,199 @@ +"""``search_by_ioc(with_artifacts=True)`` — the reverse IOC search answering +artifact rows instead of bare sha256s — and the unchanged default request. + +Three tiers (specs/04-testing.md): + +* pure-unit builder tests for the request shape: the opt-in parameter rides + the query as an int, the default request is byte-compatible with the + pre-change contract, and the parser follows the flag; +* client-method pass-through tests on both transports, which fail if either + signature stops forwarding the keyword; and +* a respx ``ClientTestCase`` body for the parse of a returned row. That one is + a stand-in: the e2e stack does not serve ``with_artifacts`` yet, so a live + VCR lifecycle test cannot be recorded (specs/99-open-questions.md). The + default path stays covered live by ``test_search_by_ioc`` / + ``test_async_search_by_ioc``. +""" +import asyncio + +import pytest + +from polyswarm_api import resources +from polyswarm_api.aio import PolySwarmAsyncAPI +from polyswarm_api.api import PolyswarmAPI +from test._client_harness import BASE_URL, COMMUNITY, ClientTestCase + +_SHA256 = '0285922fdd731d6905d5a6dc51d75e3bd504c5ac418b8c7b431063c6cee8d064' + + +def _artifact_row(): + """A metadata-search ``_source`` row holding exactly the server's include set + for ``with_artifacts`` (``IOC_ARTIFACT_INCLUDES`` upstream).""" + return { + 'artifact': { + 'created': '2026-06-05T19:01:38.104756+00:00', + 'id': '31542742786663251', + 'md5': '5d3bc3c626be6b6f59195afc4ab80fc8', + 'sha1': 'f1d73dcb1286cf42e707b5d7fcf918dd45290411', + 'sha256': _SHA256, + 'size': 89, + }, + 'scan': { + 'first_seen': '2026-06-04T10:00:00+00:00', + 'first_scan': {'created': '2026-06-04T10:00:00+00:00'}, + 'last_seen': '2026-06-05T19:01:38.104756+00:00', + 'detections': {'benign': 0, 'malicious': 1, 'total': 1}, + 'mimetype': {'extended': 'EICAR virus test files', 'mime': 'text/plain'}, + 'latest_scan': { + 'polyscore': 0.97, + 'artifact_instance_id': '88272874449980049', + 'created': '2026-06-05T19:01:38.104756+00:00', + }, + 'filename': ['artifact'], + 'url': [], + }, + 'polyunite': {'malware_family': 'EICAR'}, + 'hash': {'ssdeep': '3:a+JraNvsgzsVqSwHq9:tJuOgzsko', 'tlsh': 'T1A1B2C3'}, + } + + +class _FakeApi: + uri = 'https://api.example.test' + community = 'gamma' + + +class TestIocSearchBuilder: + def test_default_request_is_unchanged(self): + req = resources.IOC.ioc_search(_FakeApi(), ip='9.9.9.9') + assert req.method == 'GET' + assert req.url == f'{_FakeApi.uri}/ioc/search' + assert req.params == {'community': 'gamma', 'ip': '9.9.9.9'} + assert req.result_parser is resources.IOC + + def test_with_artifacts_rides_the_query_as_an_int_and_parses_metadata(self): + req = resources.IOC.ioc_search(_FakeApi(), domain='evil.test', with_artifacts=True) + assert req.params == {'community': 'gamma', 'domain': 'evil.test', 'with_artifacts': 1} + # A bool would be rendered 'True' on the wire, which the server refuses. + assert type(req.params['with_artifacts']) is int + assert req.result_parser is resources.Metadata + + def test_with_artifacts_false_sends_nothing(self): + req = resources.IOC.ioc_search(_FakeApi(), ttp='T1081', with_artifacts=False) + assert 'with_artifacts' not in req.params + assert req.result_parser is resources.IOC + + def test_a_bare_call_is_still_sent_as_before(self): + # The SDK adds no client-side term check: a call with no term builds + # the same request it always did and lets the server answer it. + req = resources.IOC.ioc_search(_FakeApi()) + assert req.method == 'GET' + assert req.url == f'{_FakeApi.uri}/ioc/search' + assert req.params == {'community': 'gamma'} + assert req.result_parser is resources.IOC + + @pytest.mark.parametrize('term', ['ip', 'domain', 'ttp', 'imphash']) + def test_any_single_term_is_enough(self, term): + req = resources.IOC.ioc_search(_FakeApi(), **{term: 'x'}) + assert req.params[term] == 'x' + + +class TestSearchByIocForwardsTheFlag: + """Drive both CLIENT methods: dropping the keyword from either transport's + signature or its pass-through fails here, which the builder tests cannot + see. The sync one is the mirror the CLI calls.""" + + @staticmethod + def _sync_request(**kwargs): + api = PolyswarmAPI.__new__(PolyswarmAPI) + api.uri, api.community = _FakeApi.uri, _FakeApi.community + api.refang_iocs = False # the constructor's default + captured = [] + + def capture(request, *a, **kw): + captured.append(request) + return iter(()) + + api._paginate = capture + list(api.search_by_ioc(**kwargs)) + return captured[0] + + @staticmethod + def _async_request(**kwargs): + api = PolySwarmAsyncAPI.__new__(PolySwarmAsyncAPI) + api.uri, api.community = _FakeApi.uri, _FakeApi.community + api.refang_iocs = False # the constructor's default + captured = [] + + async def paginate(request, *a, **kw): + captured.append(request) + return + yield # pragma: no cover — makes this an async generator + + api._paginate = paginate + + async def run(): + return [item async for item in api.search_by_ioc(**kwargs)] + + asyncio.run(run()) + return captured[0] + + @pytest.mark.parametrize('build', ['_sync_request', '_async_request']) + def test_with_artifacts_reaches_the_request(self, build): + req = getattr(self, build)(imphash='a' * 32, with_artifacts=True) + assert req.params['with_artifacts'] == 1 + assert req.result_parser is resources.Metadata + + @pytest.mark.parametrize('build', ['_sync_request', '_async_request']) + def test_a_bare_call_reaches_the_request_unchanged(self, build): + req = getattr(self, build)() + assert req.params == {'community': 'gamma'} + assert req.result_parser is resources.IOC + + @pytest.mark.parametrize('build', ['_sync_request', '_async_request']) + def test_default_sends_no_flag(self, build): + req = getattr(self, build)(ip='9.9.9.9') + assert 'with_artifacts' not in req.params + assert req.result_parser is resources.IOC + + +class IocSearchWithArtifactsTestCase(ClientTestCase): + """Full pipeline (session -> parse_response -> resource) on both + transports, against a fabricated envelope — see the module docstring for + why this is respx rather than a recorded cassette.""" + + _URL = f'{BASE_URL}/ioc/search' + + def test_rows_parse_as_metadata(self): + self.mock.add('GET', self._URL, json={ + 'status': 'OK', 'result': [_artifact_row()], + 'has_more': False, 'limit': 50, 'offset': None}) + results = list(self.api.search_by_ioc(ip='9.9.9.9', with_artifacts=True)) + assert 'with_artifacts=1' in self.mock.last_request_url + assert f'community={COMMUNITY}' in self.mock.last_request_url + assert len(results) == 1 + row = results[0] + assert isinstance(row, resources.Metadata) + assert row.id == '31542742786663251' + assert row.sha256 == _SHA256 + assert row.md5 == '5d3bc3c626be6b6f59195afc4ab80fc8' + assert row.created is not None + assert row.last_scanned is not None + assert row.malicious == 1 + assert row.mimetype == 'text/plain' + assert row.filenames == ['artifact'] + assert row.json['polyunite']['malware_family'] == 'EICAR' + assert row.first_seen.isoformat() == '2026-06-04T10:00:00+00:00' + assert row.ssdeep == '3:a+JraNvsgzsVqSwHq9:tJuOgzsko' + assert row.tlsh == 'T1A1B2C3' + # strings.* is outside the include set, so its attributes stay unset. + assert row.domains is None + assert row.ipv4 is None + + def test_default_rows_stay_sha256_strings(self): + self.mock.add('GET', self._URL, json={ + 'status': 'OK', 'result': [_SHA256], + 'has_more': False, 'limit': 50, 'offset': None}) + results = list(self.api.search_by_ioc(ip='9.9.9.9')) + assert 'with_artifacts' not in self.mock.last_request_url + assert [r.json for r in results] == [_SHA256] + assert isinstance(results[0], resources.IOC)