diff --git a/VERSION b/VERSION index 3989355..0044d6c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.20.0 +1.20.1 diff --git a/docs/architecture/tools/outbound-request-filtering.md b/docs/architecture/tools/outbound-request-filtering.md index bc837d8..a63fce4 100644 --- a/docs/architecture/tools/outbound-request-filtering.md +++ b/docs/architecture/tools/outbound-request-filtering.md @@ -217,15 +217,19 @@ third-party destinations: targets are provider-echoed and repeat the structural public-URL checks, tracking-parameter removal, public-address vetting, and shared request deadline rather than the agent-input guard. URLs are limited to 200 ASCII - characters, except for exact hostnames in Web Fetch's reviewed `TRUSTED_DOMAINS` - list (initially `scontent-iad3-2.cdninstagram.com`), which allow 4,096. + characters, except for Instagram media hosts `scontent.cdninstagram.com` and + `scontent--.cdninstagram.com`, which allow 4,096. + Web Fetch matches the entire lower-cased parsed hostname against + `scontent(?:-[a-z]{3}[0-9]+-[0-9]+)?\.cdninstagram\.com`. This deliberately + replaces the exact-node list because Instagram rotates serving nodes; it does + not grant an exception to other `cdninstagram.com` subdomains or Meta domains. For these hosts only, the wire URL and decoded components use `allow_longer_text=True` and `allow_machine_tokens=True` so signed CDN paths and query values can pass. Explicit secret, credential, and personal-identifier checks remain enabled. Each redirect destination gets its own hostname/length - check; the source host grants no exception to another destination. Subdomains, - suffix matches, and trailing-dot aliases do not inherit the exception. - Changes to this list require a host release, not agent tool input; + check; the source host grants no exception to another destination. Additional + subdomains, suffix matches, and trailing-dot aliases do not inherit the exception. + Changes to this pattern require a host release, not agent tool input; - generation and speech prompts, and external media URLs supplied to a generation tool that the provider then fetches (Runway image_url / video_url, Seedance image_url); - slugs and hashtags (after their stricter grammar checks); - package-registry names and query values on the agent network path diff --git a/host/tools/web_fetch/__init__.py b/host/tools/web_fetch/__init__.py index 15ba8c3..6069bd3 100644 --- a/host/tools/web_fetch/__init__.py +++ b/host/tools/web_fetch/__init__.py @@ -59,13 +59,16 @@ MAX_PREVIEW_CHARS = 1024 MAX_REDIRECTS = 3 MAX_URL_CHARS = 200 -# Exact hostnames only; additions require a reviewed host change. This grants -# longer signed URLs and opaque provider tokens, not a wildcard CDN exception. -TRUSTED_DOMAINS = frozenset({"scontent-iad3-2.cdninstagram.com"}) +# Instagram rotates media nodes, so trust this narrowly defined hostname family +# instead of enumerating exact nodes. This intentionally grants longer signed +# URLs and opaque provider tokens to these media hosts, not all cdninstagram.com +# subdomains. Changes to the family require a reviewed host release. +_TRUSTED_MEDIA_HOST_RE = re.compile(r"scontent(?:-[a-z]{3}[0-9]+-[0-9]+)?\.cdninstagram\.com") MAX_TRUSTED_URL_CHARS = 4096 _URL_LIMIT_DESCRIPTION = ( - f"Up to {MAX_URL_CHARS} ASCII characters, or {MAX_TRUSTED_URL_CHARS} for domains on the trusted list. " - "Only trusted domains permit opaque provider tokens; " + f"Up to {MAX_URL_CHARS} ASCII characters, or {MAX_TRUSTED_URL_CHARS} for Instagram media hosts: " + "scontent.cdninstagram.com or scontent--.cdninstagram.com. " + "Only these media hosts permit opaque provider tokens; " "secret and personal-identifier checks still apply." ) FETCH_USER_AGENT = "KernWebFetch/1.0 (https://kernai.cloud)" @@ -302,10 +305,11 @@ technical_details=( PARAM_GUARD_TECHNICAL_DETAIL, _URL_LIMIT_DESCRIPTION + " The longer-text and machine-token exceptions apply only to " - "exact hosts in TRUSTED_DOMAINS, on the complete wire URL and every decoded path/query view. " + "this media-host family, matched against the entire lower-cased parsed hostname, " + "on the complete wire URL and every decoded path/query view. " "Other hosts retain the default guard. Each redirect target is independently checked " - "against the hostname list and length limit; permission is not inherited from the source. " - "The list is maintained in the Web Fetch package and cannot be changed by tool input.", + "against the hostname pattern and length limit; permission is not inherited from the source. " + "The pattern is maintained in the Web Fetch package and cannot be changed by tool input.", "The tool resolves each hostname itself, verifies every resolved address is publicly " "routable, and tries the vetted addresses within the shared deadline with TLS verified " "against the hostname, so a DNS entry pointing at a private or link-local address cannot " @@ -383,6 +387,10 @@ class _MediaResponseError(ValueError): """A safe, fixed message for a rejected media response.""" +def _is_trusted_media_host(hostname: str) -> bool: + return _TRUSTED_MEDIA_HOST_RE.fullmatch(hostname.lower()) is not None + + def _structural_page_url(url: str, invalid_message: str) -> str: """Apply the shared structural checks and normalize the URL for fetching. @@ -401,7 +409,7 @@ def _structural_page_url(url: str, invalid_message: str) -> str: raise ValueError(invalid_message) parsed = urllib.parse.urlsplit(url) hostname = parsed.hostname or "" - limit = MAX_TRUSTED_URL_CHARS if hostname in TRUSTED_DOMAINS else MAX_URL_CHARS + limit = MAX_TRUSTED_URL_CHARS if _is_trusted_media_host(hostname) else MAX_URL_CHARS if hostname.endswith(".") or len(url) > limit: raise ValueError(invalid_message) query = _query_without_tracking_parameters(parsed.query) @@ -875,7 +883,7 @@ def execute(self, action: str, tool_input: JSONObject, api: HostAPI) -> ActionRe return ActionFailed("Unsupported Web Fetch action.") try: url = _validated_page_url(tool_input.get("url")) - trusted_domain = urllib.parse.urlsplit(url).hostname in TRUSTED_DOMAINS + trusted_domain = _is_trusted_media_host(urllib.parse.urlsplit(url).hostname or "") guarded_url = guard_url_parameter_string( url, api, allow_longer_text=trusted_domain, allow_machine_tokens=trusted_domain, ) diff --git a/tests/test_tools_web_fetch.py b/tests/test_tools_web_fetch.py index a25ece3..a710907 100644 --- a/tests/test_tools_web_fetch.py +++ b/tests/test_tools_web_fetch.py @@ -38,44 +38,76 @@ def _addrinfo(address: str) -> list[tuple[object, object, int, str, tuple[str, i class WebFetchLongUrlTests(unittest.TestCase): - host = "scontent-iad3-2.cdninstagram.com" + hosts = ( + "scontent-iad3-2.cdninstagram.com", + "scontent-lhr6-1.cdninstagram.com", + "scontent-lhr6-2.cdninstagram.com", + "scontent-lhr11-1.cdninstagram.com", + "scontent-lga3-1.cdninstagram.com", + "scontent-bos5-1.cdninstagram.com", + "scontent.cdninstagram.com", + ) + host = hosts[3] # Synthetic opaque CDN parameters: no live signature or expiring asset. url = f"https://{host}/o1/video.mp4?efg=" + "Ab9_ef-X3" * 160 + "&oh=example" - def test_listed_host_accepts_opaque_urls_in_all_actions(self) -> None: - for action in ("fetch_page", "fetch_page_file", "head_url", "download_media"): - with self.subTest(action=action), patch.object( - web_fetch, "_fetch_once", return_value=_response(), - ) as fetch, patch.object(web_fetch, "_open_media_stream") as stream: - result = BUNDLED_TOOL.execute(action, {"url": self.url}, FakeHostAPI()) - self.assertNotIsInstance(result, ActionFailed) - if action == "download_media": - assert isinstance(result, StreamingAsset) - result.open_stream() - stream.assert_called_once_with(self.url) - else: - self.assertEqual(fetch.call_args.args[0], self.url) - - def test_host_match_is_exact_and_case_insensitive(self) -> None: - self.assertEqual(web_fetch._validated_page_url(self.url.replace(self.host, self.host.upper())), - self.url.replace(self.host, self.host.upper())) - for host in ("example.com", "cdninstagram.com", "scontent-iad3-1.cdninstagram.com", - "sub." + self.host, self.host + ".example.com", self.host + ".", - self.host + "@example.com", "user@" + self.host, self.host + ":8443"): - with self.subTest(host=host), patch.object(web_fetch, "_fetch_once") as fetch: - result = BUNDLED_TOOL.execute("fetch_page", {"url": self.url.replace(self.host, host)}, FakeHostAPI()) - self.assertIsInstance(result, ActionFailed) - fetch.assert_not_called() + def test_media_hosts_accept_opaque_urls_in_all_actions(self) -> None: + for host in (*self.hosts, self.hosts[1].upper(), self.hosts[-1].upper()): + url = self.url.replace(self.host, host) + for action in ("fetch_page", "fetch_page_file", "head_url", "download_media"): + with self.subTest(host=host, action=action), patch.object( + web_fetch, "_fetch_once", return_value=_response(), + ) as fetch, patch.object(web_fetch, "_open_media_stream") as stream: + result = BUNDLED_TOOL.execute(action, {"url": url}, FakeHostAPI()) + self.assertNotIsInstance(result, ActionFailed) + if action == "download_media": + assert isinstance(result, StreamingAsset) + result.open_stream() + stream.assert_called_once_with(url) + else: + self.assertEqual(fetch.call_args.args[0], url) + + def test_other_hosts_do_not_receive_either_exception(self) -> None: + for host in ( + "example.com", "cdninstagram.com", "static.cdninstagram.com", + "cdninstagram.com.example.org", "evil-cdninstagram.com", + "scontent-lhr6-1.cdninstagram.com.evil.tld", "scontent.fbcdn.net", + "scontent-lh6-1.cdninstagram.com", "scontent-lhrr6-1.cdninstagram.com", + "scontent-lhr-1.cdninstagram.com", "scontent-lhr6-.cdninstagram.com", + "scontent-lhr6-1-2.cdninstagram.com", "scontent-1236-1.cdninstagram.com", + "sub." + self.host, self.host + ".example.com", self.host + ".", + ): + # Exceed the 128-character token cap while keeping the URL under 200. + urls = (self.url.replace(self.host, host), f"https://{host}/?q=" + "Ab9_ef-X3" * 15) + self.assertLess(len(urls[1]), 200) + for url in urls: + with self.subTest(host=host, url=url), patch.object(web_fetch, "_fetch_once") as fetch: + result = BUNDLED_TOOL.execute("fetch_page", {"url": url}, FakeHostAPI()) + self.assertIsInstance(result, ActionFailed) + fetch.assert_not_called() + + def test_media_hosts_keep_structural_restrictions(self) -> None: + for host in self.hosts: + for url in ( + self.url.replace(self.host, host + "@example.com"), + self.url.replace(self.host, "user@" + host), + self.url.replace(self.host, host + ":8443"), + self.url.replace(self.host, host).replace("https://", "http://"), + ): + with self.subTest(url=url), patch.object(web_fetch, "_fetch_once") as fetch: + result = BUNDLED_TOOL.execute("fetch_page", {"url": url}, FakeHostAPI()) + self.assertIsInstance(result, ActionFailed) + fetch.assert_not_called() def test_length_boundaries_and_shared_default_unchanged(self) -> None: - for host, limit in ((self.host, 4096), ("example.com", 200)): + for host, limit in (*((host, 4096) for host in self.hosts), ("example.com", 200)): prefix = f"https://{host}/" url = prefix + "a" * (limit - len(prefix)) with self.subTest(host=host): self.assertEqual(web_fetch._validated_page_url(url), url) with self.assertRaises(ValueError): web_fetch._validated_page_url(url + "a") - if host == self.host: + if host in self.hosts: with patch.object(web_fetch, "_fetch_once", return_value=_response()) as fetch: result = BUNDLED_TOOL.execute("fetch_page", {"url": url}, FakeHostAPI()) self.assertIsInstance(result, ActionExecuted) @@ -94,7 +126,7 @@ def test_secrets_identifiers_and_nested_encoding_still_blocked(self) -> None: self.assertIsInstance(result, ActionFailed) fetch.assert_not_called() - def test_unlisted_short_urls_keep_machine_token_guard(self) -> None: + def test_untrusted_short_urls_keep_machine_token_guard(self) -> None: url = "https://example.com/?q=" + "Ab9_ef-X3" * 16 self.assertLess(len(url), 200) with patch.object(web_fetch, "_fetch_once") as fetch: @@ -103,8 +135,12 @@ def test_unlisted_short_urls_keep_machine_token_guard(self) -> None: fetch.assert_not_called() def test_redirects_recompute_length_permission_for_every_hop(self) -> None: - for source, target in (("https://example.com/start", self.url), - (self.url, "https://example.com/short")): + for source, target in ( + ("https://example.com/start", self.url), + (self.url, self.url.replace(self.host, self.hosts[1])), + (self.url, self.url.replace(self.host, self.hosts[-1])), + (self.url, "https://example.com/short"), + ): with self.subTest(source=source), patch.object(web_fetch, "_fetch_once", side_effect=[ _response(status=302, headers={"location": target}), _response(), ]) as fetch: @@ -113,6 +149,7 @@ def test_redirects_recompute_length_permission_for_every_hop(self) -> None: self.assertEqual(fetch.call_args.args[0], target) for target in (self.url.replace(self.host, "example.com"), self.url.replace(self.host, "sub." + self.host), + self.url.replace(self.host, "static.cdninstagram.com"), self.url.replace(self.host, "127.0.0.1"), f"https://{self.host}/" + "a" * 4096): with self.subTest(target=target), patch.object(web_fetch, "_fetch_once", return_value= @@ -122,7 +159,7 @@ def test_redirects_recompute_length_permission_for_every_hop(self) -> None: self.assertIsInstance(result, ActionFailed) self.assertEqual(fetch.call_count, 1) - def test_listed_host_still_rejects_private_dns(self) -> None: + def test_media_host_still_rejects_private_dns(self) -> None: with patch.object(web_fetch.socket, "getaddrinfo", return_value=_addrinfo("10.0.0.8")), patch.object( web_fetch, "_PinnedHTTPSConnection", ) as connection: