fix(benchmark): align VM host services with the native Docker firewall

benchmark/README.md

@@ -161,6 +161,18 @@ This routing repair enables matched experiments under the
 [long-horizon research contract](../docs/architecture/rfcs/long-horizon-harness-benchmark-research-program-v0.md);
 it establishes neither sustained operation nor a solver-quality improvement.
 
+When the runner, native judge and credential gateway run **inside the Docker
+VM**, pass `--docker-host-bridge` to every matched arm. Some VM runtimes map
+Docker's `host-gateway` alias to the desktop host, where that gateway is not
+listening. This explicit option pins `host.docker.internal` to the default
+bridge's inspected IPv4 gateway, matching the native firewall's host-service
+endpoints. A missing or ambiguous IPv4 gateway fails before container creation;
+it never guesses a fixed subnet. All services using that alias must be reachable
+on the bridge. Omit the option when services run on the desktop host or another
+topology; omitted behavior is unchanged. Select it only for new attempts and
+retain the setting in each arm's runtime receipt. It changes routing, not
+credentials, allowed ports, task input or feedback policy.
+
 ## Inert checkpoint capture
 
 This is a developer prerequisite for the [research checkpoint contract](../docs/architecture/rfcs/hierarchical-agent-stride-control-v0.md#77-research-state-branching-and-matched-continuation),

benchmark/edgebench/run.py

@@ -194,6 +194,9 @@ def main(argv=None):
                         help="Native default: 3600s between accepted submissions; other modes retain 120s")
     parser.add_argument("--judge-url", required=True)
     parser.add_argument("--api-proxy-url", help="Operator-owned, OpenAI-only CONNECT proxy")
+    parser.add_argument("--docker-host-bridge", action="store_true",
+                        help="Host services run on Docker's default bridge (e.g. runner inside a VM); "
+                             "pin host.docker.internal there instead of Docker's host-gateway alias")
     args = parser.parse_args(argv)
     api_base_url = os.environ.get("OPENAI_BASE_URL") or None
     api_key = os.environ.get("OPENAI_API_KEY") if api_base_url else None
@@ -396,6 +399,7 @@ def main(argv=None):
     backend = RecordingDockerBackend(log_dir=trial / "collected", logger=logger,
                                      oauth_proxy=bool(args.api_proxy_url or api_base_url),
                                      blind_api_endpoint=blind_endpoint, feedback=feedback,
+                                     docker_host_bridge=args.docker_host_bridge,
                                      **({"checkpoint": checkpoint} if checkpoint is not None else {}))
     for image in (task.work_image_key, task.judge_image_key):
         if not backend.image_exists(image):
@@ -421,6 +425,7 @@ def main(argv=None):
            if agent.task_entry == "loopx-planned" else {}),
         "model": args.model, "effort": args.effort, "timeout_seconds": args.timeout,
         "provider_rate_limit_retry_seconds": args.provider_rate_limit_retry_seconds,
+        **({"docker_host_bridge": True} if args.docker_host_bridge else {}),
         "loopx_commit": pins[0], "runner_commit": pins[1],
         "sforge_source": active_source_receipt(),
         **turn_envelope_receipt(explicit=args.turn_envelope),

benchmark/runtime/sforge_backend.py

@@ -4,6 +4,7 @@
 
 import io
 import inspect
+import ipaddress
 import json
 import tarfile
 from pathlib import Path, PurePosixPath
@@ -17,8 +18,12 @@
 
 class RecordingDockerBackend(DockerBackend):
     def __init__(self, *, log_dir: Path, logger, oauth_proxy: bool = False,
-                 blind_api_endpoint: tuple[str, int] | None = None, feedback=None, checkpoint=None, client=None):
+                 blind_api_endpoint: tuple[str, int] | None = None, feedback=None, checkpoint=None, client=None,
+                 docker_host_bridge: bool = False):
+        if not isinstance(docker_host_bridge, bool):
+            raise ValueError("docker_host_bridge must be an explicit boolean")
         super().__init__(client)
+        self.docker_host_bridge = docker_host_bridge
         self.log_dir, self.logger = log_dir, logger
         self.blind_api_endpoint = blind_api_endpoint
         self.feedback = feedback
@@ -52,6 +57,19 @@ def bind_registration(self, token):
 
     def create_container(self, image, name, **kwargs):
         hosts = dict(kwargs.get("extra_hosts") or {})
+        if self.docker_host_bridge:
+            # Docker Desktop/VM runtimes can resolve `host-gateway` to the
+            # desktop host, while a runner inside the VM serves its gateway
+            # on the Docker bridge. Select that topology explicitly, without
+            # guessing from the runner OS or hardcoding a bridge subnet.
+            config = self.client.networks.get("bridge").attrs["IPAM"]["Config"]
+            addresses = [ipaddress.ip_address(item["Gateway"])
+                         for item in config if item.get("Gateway")]
+            gateways = {str(ip) for ip in addresses if ip.version == 4
+                        and not (ip.is_loopback or ip.is_unspecified or ip.is_multicast)}
+            if len(gateways) != 1:
+                raise RuntimeError("Docker host bridge requires one routable IPv4 gateway")
+            hosts["host.docker.internal"] = gateways.pop()
         if self.auth_ips:
             hosts["auth.openai.com"] = self.auth_ips[0]
         kwargs["extra_hosts"] = hosts

benchmark/tests/test_edgebench_feedback_docker.py

@@ -93,6 +93,62 @@ def test_native_execution_disposition_uses_real_docker_exit(tmp_path, agent_imag
         backend.client.close()
 
 
+def test_host_bridge_pin_reaches_only_the_native_allowed_port(tmp_path, agent_image):
+    from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+    from sforge.harness.network_isolation import build_allowed_endpoints
+    from benchmark.runtime.sforge_backend import RecordingDockerBackend
+
+    backend = RecordingDockerBackend(log_dir=tmp_path / "collected",
+        logger=logging.getLogger("bridge-route-smoke"), oauth_proxy=True,
+        docker_host_bridge=True)
+    handle = backend.create_container(agent_image.id, "host-bridge-" + uuid.uuid4().hex[:10],
+        extra_hosts={"host.docker.internal": "host-gateway"}, cpu_limit=1, mem_limit="256m")
+    backend.start_container(handle)
+    gateway = backend.get_container_gateway_ip(handle)
+
+    class Response(BaseHTTPRequestHandler):
+        def do_GET(self):
+            self.send_response(200)
+            self.end_headers()
+            self.wfile.write(b"synthetic-bridge-service")
+
+        def log_message(self, *args):
+            pass
+
+    servers, policy = [], None
+    try:
+        for _ in range(2):
+            server = ThreadingHTTPServer((gateway, 0), Response)
+            servers.append(server)
+            threading.Thread(target=server.serve_forever, daemon=True).start()
+        ports = [server.server_port for server in servers]
+        handle.raw.reload()
+        assert f"host.docker.internal:{gateway}" in handle.raw.attrs["HostConfig"]["ExtraHosts"]
+
+        def get(port):
+            source = ("import urllib.request; "
+                "opener=urllib.request.build_opener(urllib.request.ProxyHandler({})); "
+                f"print(opener.open('http://host.docker.internal:{port}',timeout=2).read().decode())")
+            return backend.exec_run(handle, ["python3", "-c", source], user="agent")
+
+        assert all(get(port).exit_code == 0 for port in ports)
+        endpoints = build_allowed_endpoints(f"http://host.docker.internal:{ports[0]}",
+                                            None, gateway, backend.logger)
+        policy = backend.create_network_isolation(handle, endpoints, backend.logger)
+        policy.apply()
+        allowed = get(ports[0])
+        assert allowed.exit_code == 0 and allowed.output.strip() == "synthetic-bridge-service"
+        assert get(ports[1]).exit_code != 0, "the routing option must not add an allowed port"
+    finally:
+        if policy is not None:
+            policy.cleanup()
+        backend.cleanup_container(handle)
+        backend.client.close()
+        for server in servers:
+            server.shutdown()
+            server.server_close()
+
+
 def test_sforge_checkpoint_declaration_is_a_private_guest_artifact(tmp_path, agent_image, monkeypatch):
     from sforge.harness.config import SForgeConfig
     from benchmark.runtime.sforge import SForgeWorker, BenchmarkCodex

benchmark/tests/test_sforge_host_bridge.py

@@ -0,0 +1,68 @@
+"""Explicit Docker topology keeps container pins aligned with native isolation."""
+from types import SimpleNamespace
+
+import pytest
+
+
+@pytest.mark.parametrize("enabled", [False, True], ids=["docker-default", "docker-bridge"])
+def test_host_bridge_is_explicit_and_preserves_other_container_options(tmp_path, monkeypatch, enabled):
+    pytest.importorskip("sforge")
+    pytest.importorskip("harbor")
+    from benchmark.runtime.sforge_backend import RecordingDockerBackend, DockerBackend
+    queries, calls = [], []
+
+    def network(name):
+        queries.append(name)
+        return SimpleNamespace(attrs={"IPAM": {"Config": [
+            {"Subnet": "192.0.2.0/24", "Gateway": "192.0.2.1"},
+            {"Subnet": "2001:db8::/64", "Gateway": "2001:db8::1"},
+        ]}})
+
+    backend = RecordingDockerBackend(log_dir=tmp_path, logger=None, oauth_proxy=True,
+        client=SimpleNamespace(networks=SimpleNamespace(get=network)),
+        **({"docker_host_bridge": True} if enabled else {}))
+    backend.auth_ips = ["198.51.100.1"]
+    monkeypatch.setattr(DockerBackend, "create_container",
+                        lambda self, image, name, **kwargs: calls.append(kwargs))
+    hosts = {"host.docker.internal": "host-gateway", "unrelated.invalid": "203.0.113.1"}
+    environment = {"OPENAI_API_KEY": "synthetic-placeholder"}
+    backend.create_container("fixture", "fixture", extra_hosts=hosts, environment=environment,
+                             cap_drop=["NET_RAW"], mem_limit="256m")
+
+    assert hosts["host.docker.internal"] == "host-gateway", "do not mutate caller configuration"
+    assert queries == (["bridge"] if enabled else [])
+    assert calls == [{"extra_hosts": {**hosts, "auth.openai.com": "198.51.100.1",
+        "host.docker.internal": "192.0.2.1" if enabled else "host-gateway"},
+        "environment": environment, "cap_drop": ["NET_RAW"], "mem_limit": "256m"}]
+
+
+@pytest.mark.parametrize("gateways", [[], ["2001:db8::1"], ["0.0.0.0"], ["127.0.0.1"],
+                                    ["224.0.0.1"], ["192.0.2.1", "198.51.100.1"]])
+def test_unusable_or_ambiguous_bridge_fails_before_container_creation(tmp_path, monkeypatch, gateways):
+    pytest.importorskip("sforge")
+    pytest.importorskip("harbor")
+    from benchmark.runtime.sforge_backend import RecordingDockerBackend, DockerBackend
+    network = SimpleNamespace(attrs={"IPAM": {"Config": [{"Gateway": ip} for ip in gateways]}})
+    backend = RecordingDockerBackend(log_dir=tmp_path, logger=None, oauth_proxy=True,
+        docker_host_bridge=True, client=SimpleNamespace(networks=SimpleNamespace(get=lambda name: network)))
+    monkeypatch.setattr(DockerBackend, "create_container", lambda *a, **k: pytest.fail("invalid route"))
+    with pytest.raises(RuntimeError, match="one routable IPv4 gateway"):
+        backend.create_container("fixture", "fixture")
+
+
+def test_host_bridge_does_not_weaken_native_or_blind_endpoint_selection(tmp_path, monkeypatch):
+    pytest.importorskip("sforge")
+    pytest.importorskip("harbor")
+    from benchmark.runtime.sforge_backend import RecordingDockerBackend, DockerBackend
+    from sforge.harness.network_isolation import build_allowed_endpoints
+    import logging
+    backend = RecordingDockerBackend(log_dir=tmp_path, logger=None, oauth_proxy=True,
+                                    docker_host_bridge=True, client=SimpleNamespace())
+    endpoints = build_allowed_endpoints("http://host.docker.internal:8080",
+        "http://host.docker.internal:9090", "192.0.2.1", logging.getLogger(__name__))
+    monkeypatch.setattr(DockerBackend, "create_network_isolation", lambda self, handle, allowed, logger: allowed)
+    assert {(ep.ip, ep.port) for ep in backend.create_network_isolation(None, endpoints, None)} == {
+        ("192.0.2.1", 8080), ("192.0.2.1", 9090)}
+    backend.blind_api_endpoint = ("192.0.2.1", 9090)
+    assert [(ep.ip, ep.port) for ep in backend.create_network_isolation(None, endpoints, None)] == [
+        ("192.0.2.1", 9090)]

benchmark/tests/test_sforge_runtime.py

@@ -1284,7 +1284,9 @@ def test_custom_provider_without_key_fails_before_creating_attempt(tmp_path, mon
 
 
 @pytest.mark.parametrize('custom', [False, True])
-def test_edgebench_cli_merges_sforge_env_with_explicit_runner_config(tmp_path, monkeypatch, custom):
+@pytest.mark.parametrize('bridge', [False, True], ids=['docker-default', 'docker-bridge'])
+@pytest.mark.parametrize('worker', ['official', 'heartbeat-resume'])
+def test_edgebench_cli_merges_sforge_env_with_explicit_runner_config(tmp_path, monkeypatch, custom, bridge, worker):
     pytest.importorskip('sforge')
     pytest.importorskip('harbor')
     from types import SimpleNamespace
@@ -1313,6 +1315,7 @@ def test_edgebench_cli_merges_sforge_env_with_explicit_runner_config(tmp_path, m
         judge=SimpleNamespace(selection='score_first', score_direction='maximize')))
     def backend(**kwargs):
         assert kwargs['oauth_proxy'] is custom
+        assert kwargs.get('docker_host_bridge', False) is bridge
         return SimpleNamespace(image_exists=lambda image: True)
     monkeypatch.setattr(run, 'RecordingDockerBackend', backend)
     def stop_before_solver(**kwargs):
@@ -1326,10 +1329,12 @@ def stop_before_solver(**kwargs):
     monkeypatch.setattr(run, 'run_agent', stop_before_solver)
     with pytest.raises(RuntimeError, match='synthetic launch boundary'):
         run.main(['--task', 'fixture', '--tasks-dir', str(tmp_path), '--log-dir', str(tmp_path),
-                  '--run-id', 'route', '--worker', 'official', '--model', 'fixture',
-                  '--effort', 'xhigh', '--judge-url', 'http://127.0.0.1:9999'])
+                  '--run-id', 'route', '--worker', worker, '--model', 'fixture',
+                  '--effort', 'xhigh', '--judge-url', 'http://127.0.0.1:9999',
+                  *(['--docker-host-bridge'] if bridge else [])])
     receipt = (tmp_path / 'runs/route/fixture/runtime-receipt.json').read_text()
     assert 'synthetic-provider-secret' not in receipt
+    assert json.loads(receipt).get('docker_host_bridge', False) is bridge
 
 
 @pytest.mark.parametrize("capture", [False, True])