diff --git a/benchmark/README.md b/benchmark/README.md index bb61cc6b29..e74fc5ee22 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -161,6 +161,18 @@ This routing repair enables matched experiments under the [long-horizon research contract](../docs/architecture/rfcs/long-horizon-harness-benchmark-research-program-v0.md); it establishes neither sustained operation nor a solver-quality improvement. +When the runner, native judge and credential gateway run **inside the Docker +VM**, pass `--docker-host-bridge` to every matched arm. Some VM runtimes map +Docker's `host-gateway` alias to the desktop host, where that gateway is not +listening. This explicit option pins `host.docker.internal` to the default +bridge's inspected IPv4 gateway, matching the native firewall's host-service +endpoints. A missing or ambiguous IPv4 gateway fails before container creation; +it never guesses a fixed subnet. All services using that alias must be reachable +on the bridge. Omit the option when services run on the desktop host or another +topology; omitted behavior is unchanged. Select it only for new attempts and +retain the setting in each arm's runtime receipt. It changes routing, not +credentials, allowed ports, task input or feedback policy. + ## Inert checkpoint capture This is a developer prerequisite for the [research checkpoint contract](../docs/architecture/rfcs/hierarchical-agent-stride-control-v0.md#77-research-state-branching-and-matched-continuation), diff --git a/benchmark/edgebench/run.py b/benchmark/edgebench/run.py index 2da4d2e56e..d709287555 100644 --- a/benchmark/edgebench/run.py +++ b/benchmark/edgebench/run.py @@ -194,6 +194,9 @@ def main(argv=None): help="Native default: 3600s between accepted submissions; other modes retain 120s") parser.add_argument("--judge-url", required=True) parser.add_argument("--api-proxy-url", help="Operator-owned, OpenAI-only CONNECT proxy") + parser.add_argument("--docker-host-bridge", action="store_true", + help="Host services run on Docker's default bridge (e.g. runner inside a VM); " + "pin host.docker.internal there instead of Docker's host-gateway alias") args = parser.parse_args(argv) api_base_url = os.environ.get("OPENAI_BASE_URL") or None api_key = os.environ.get("OPENAI_API_KEY") if api_base_url else None @@ -396,6 +399,7 @@ def main(argv=None): backend = RecordingDockerBackend(log_dir=trial / "collected", logger=logger, oauth_proxy=bool(args.api_proxy_url or api_base_url), blind_api_endpoint=blind_endpoint, feedback=feedback, + docker_host_bridge=args.docker_host_bridge, **({"checkpoint": checkpoint} if checkpoint is not None else {})) for image in (task.work_image_key, task.judge_image_key): if not backend.image_exists(image): @@ -421,6 +425,7 @@ def main(argv=None): if agent.task_entry == "loopx-planned" else {}), "model": args.model, "effort": args.effort, "timeout_seconds": args.timeout, "provider_rate_limit_retry_seconds": args.provider_rate_limit_retry_seconds, + **({"docker_host_bridge": True} if args.docker_host_bridge else {}), "loopx_commit": pins[0], "runner_commit": pins[1], "sforge_source": active_source_receipt(), **turn_envelope_receipt(explicit=args.turn_envelope), diff --git a/benchmark/runtime/sforge_backend.py b/benchmark/runtime/sforge_backend.py index 1ad170d2b1..78c13e4f73 100644 --- a/benchmark/runtime/sforge_backend.py +++ b/benchmark/runtime/sforge_backend.py @@ -4,6 +4,7 @@ import io import inspect +import ipaddress import json import tarfile from pathlib import Path, PurePosixPath @@ -17,8 +18,12 @@ class RecordingDockerBackend(DockerBackend): def __init__(self, *, log_dir: Path, logger, oauth_proxy: bool = False, - blind_api_endpoint: tuple[str, int] | None = None, feedback=None, checkpoint=None, client=None): + blind_api_endpoint: tuple[str, int] | None = None, feedback=None, checkpoint=None, client=None, + docker_host_bridge: bool = False): + if not isinstance(docker_host_bridge, bool): + raise ValueError("docker_host_bridge must be an explicit boolean") super().__init__(client) + self.docker_host_bridge = docker_host_bridge self.log_dir, self.logger = log_dir, logger self.blind_api_endpoint = blind_api_endpoint self.feedback = feedback @@ -52,6 +57,19 @@ def bind_registration(self, token): def create_container(self, image, name, **kwargs): hosts = dict(kwargs.get("extra_hosts") or {}) + if self.docker_host_bridge: + # Docker Desktop/VM runtimes can resolve `host-gateway` to the + # desktop host, while a runner inside the VM serves its gateway + # on the Docker bridge. Select that topology explicitly, without + # guessing from the runner OS or hardcoding a bridge subnet. + config = self.client.networks.get("bridge").attrs["IPAM"]["Config"] + addresses = [ipaddress.ip_address(item["Gateway"]) + for item in config if item.get("Gateway")] + gateways = {str(ip) for ip in addresses if ip.version == 4 + and not (ip.is_loopback or ip.is_unspecified or ip.is_multicast)} + if len(gateways) != 1: + raise RuntimeError("Docker host bridge requires one routable IPv4 gateway") + hosts["host.docker.internal"] = gateways.pop() if self.auth_ips: hosts["auth.openai.com"] = self.auth_ips[0] kwargs["extra_hosts"] = hosts diff --git a/benchmark/tests/test_edgebench_feedback_docker.py b/benchmark/tests/test_edgebench_feedback_docker.py index 62a3fa9e5e..1e409f4d1e 100644 --- a/benchmark/tests/test_edgebench_feedback_docker.py +++ b/benchmark/tests/test_edgebench_feedback_docker.py @@ -93,6 +93,62 @@ def test_native_execution_disposition_uses_real_docker_exit(tmp_path, agent_imag backend.client.close() +def test_host_bridge_pin_reaches_only_the_native_allowed_port(tmp_path, agent_image): + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + from sforge.harness.network_isolation import build_allowed_endpoints + from benchmark.runtime.sforge_backend import RecordingDockerBackend + + backend = RecordingDockerBackend(log_dir=tmp_path / "collected", + logger=logging.getLogger("bridge-route-smoke"), oauth_proxy=True, + docker_host_bridge=True) + handle = backend.create_container(agent_image.id, "host-bridge-" + uuid.uuid4().hex[:10], + extra_hosts={"host.docker.internal": "host-gateway"}, cpu_limit=1, mem_limit="256m") + backend.start_container(handle) + gateway = backend.get_container_gateway_ip(handle) + + class Response(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.end_headers() + self.wfile.write(b"synthetic-bridge-service") + + def log_message(self, *args): + pass + + servers, policy = [], None + try: + for _ in range(2): + server = ThreadingHTTPServer((gateway, 0), Response) + servers.append(server) + threading.Thread(target=server.serve_forever, daemon=True).start() + ports = [server.server_port for server in servers] + handle.raw.reload() + assert f"host.docker.internal:{gateway}" in handle.raw.attrs["HostConfig"]["ExtraHosts"] + + def get(port): + source = ("import urllib.request; " + "opener=urllib.request.build_opener(urllib.request.ProxyHandler({})); " + f"print(opener.open('http://host.docker.internal:{port}',timeout=2).read().decode())") + return backend.exec_run(handle, ["python3", "-c", source], user="agent") + + assert all(get(port).exit_code == 0 for port in ports) + endpoints = build_allowed_endpoints(f"http://host.docker.internal:{ports[0]}", + None, gateway, backend.logger) + policy = backend.create_network_isolation(handle, endpoints, backend.logger) + policy.apply() + allowed = get(ports[0]) + assert allowed.exit_code == 0 and allowed.output.strip() == "synthetic-bridge-service" + assert get(ports[1]).exit_code != 0, "the routing option must not add an allowed port" + finally: + if policy is not None: + policy.cleanup() + backend.cleanup_container(handle) + backend.client.close() + for server in servers: + server.shutdown() + server.server_close() + + def test_sforge_checkpoint_declaration_is_a_private_guest_artifact(tmp_path, agent_image, monkeypatch): from sforge.harness.config import SForgeConfig from benchmark.runtime.sforge import SForgeWorker, BenchmarkCodex diff --git a/benchmark/tests/test_sforge_host_bridge.py b/benchmark/tests/test_sforge_host_bridge.py new file mode 100644 index 0000000000..6f8ee9d7da --- /dev/null +++ b/benchmark/tests/test_sforge_host_bridge.py @@ -0,0 +1,68 @@ +"""Explicit Docker topology keeps container pins aligned with native isolation.""" +from types import SimpleNamespace + +import pytest + + +@pytest.mark.parametrize("enabled", [False, True], ids=["docker-default", "docker-bridge"]) +def test_host_bridge_is_explicit_and_preserves_other_container_options(tmp_path, monkeypatch, enabled): + pytest.importorskip("sforge") + pytest.importorskip("harbor") + from benchmark.runtime.sforge_backend import RecordingDockerBackend, DockerBackend + queries, calls = [], [] + + def network(name): + queries.append(name) + return SimpleNamespace(attrs={"IPAM": {"Config": [ + {"Subnet": "192.0.2.0/24", "Gateway": "192.0.2.1"}, + {"Subnet": "2001:db8::/64", "Gateway": "2001:db8::1"}, + ]}}) + + backend = RecordingDockerBackend(log_dir=tmp_path, logger=None, oauth_proxy=True, + client=SimpleNamespace(networks=SimpleNamespace(get=network)), + **({"docker_host_bridge": True} if enabled else {})) + backend.auth_ips = ["198.51.100.1"] + monkeypatch.setattr(DockerBackend, "create_container", + lambda self, image, name, **kwargs: calls.append(kwargs)) + hosts = {"host.docker.internal": "host-gateway", "unrelated.invalid": "203.0.113.1"} + environment = {"OPENAI_API_KEY": "synthetic-placeholder"} + backend.create_container("fixture", "fixture", extra_hosts=hosts, environment=environment, + cap_drop=["NET_RAW"], mem_limit="256m") + + assert hosts["host.docker.internal"] == "host-gateway", "do not mutate caller configuration" + assert queries == (["bridge"] if enabled else []) + assert calls == [{"extra_hosts": {**hosts, "auth.openai.com": "198.51.100.1", + "host.docker.internal": "192.0.2.1" if enabled else "host-gateway"}, + "environment": environment, "cap_drop": ["NET_RAW"], "mem_limit": "256m"}] + + +@pytest.mark.parametrize("gateways", [[], ["2001:db8::1"], ["0.0.0.0"], ["127.0.0.1"], + ["224.0.0.1"], ["192.0.2.1", "198.51.100.1"]]) +def test_unusable_or_ambiguous_bridge_fails_before_container_creation(tmp_path, monkeypatch, gateways): + pytest.importorskip("sforge") + pytest.importorskip("harbor") + from benchmark.runtime.sforge_backend import RecordingDockerBackend, DockerBackend + network = SimpleNamespace(attrs={"IPAM": {"Config": [{"Gateway": ip} for ip in gateways]}}) + backend = RecordingDockerBackend(log_dir=tmp_path, logger=None, oauth_proxy=True, + docker_host_bridge=True, client=SimpleNamespace(networks=SimpleNamespace(get=lambda name: network))) + monkeypatch.setattr(DockerBackend, "create_container", lambda *a, **k: pytest.fail("invalid route")) + with pytest.raises(RuntimeError, match="one routable IPv4 gateway"): + backend.create_container("fixture", "fixture") + + +def test_host_bridge_does_not_weaken_native_or_blind_endpoint_selection(tmp_path, monkeypatch): + pytest.importorskip("sforge") + pytest.importorskip("harbor") + from benchmark.runtime.sforge_backend import RecordingDockerBackend, DockerBackend + from sforge.harness.network_isolation import build_allowed_endpoints + import logging + backend = RecordingDockerBackend(log_dir=tmp_path, logger=None, oauth_proxy=True, + docker_host_bridge=True, client=SimpleNamespace()) + endpoints = build_allowed_endpoints("http://host.docker.internal:8080", + "http://host.docker.internal:9090", "192.0.2.1", logging.getLogger(__name__)) + monkeypatch.setattr(DockerBackend, "create_network_isolation", lambda self, handle, allowed, logger: allowed) + assert {(ep.ip, ep.port) for ep in backend.create_network_isolation(None, endpoints, None)} == { + ("192.0.2.1", 8080), ("192.0.2.1", 9090)} + backend.blind_api_endpoint = ("192.0.2.1", 9090) + assert [(ep.ip, ep.port) for ep in backend.create_network_isolation(None, endpoints, None)] == [ + ("192.0.2.1", 9090)] diff --git a/benchmark/tests/test_sforge_runtime.py b/benchmark/tests/test_sforge_runtime.py index e8f1ca0fe5..f890a872cb 100644 --- a/benchmark/tests/test_sforge_runtime.py +++ b/benchmark/tests/test_sforge_runtime.py @@ -1284,7 +1284,9 @@ def test_custom_provider_without_key_fails_before_creating_attempt(tmp_path, mon @pytest.mark.parametrize('custom', [False, True]) -def test_edgebench_cli_merges_sforge_env_with_explicit_runner_config(tmp_path, monkeypatch, custom): +@pytest.mark.parametrize('bridge', [False, True], ids=['docker-default', 'docker-bridge']) +@pytest.mark.parametrize('worker', ['official', 'heartbeat-resume']) +def test_edgebench_cli_merges_sforge_env_with_explicit_runner_config(tmp_path, monkeypatch, custom, bridge, worker): pytest.importorskip('sforge') pytest.importorskip('harbor') from types import SimpleNamespace @@ -1313,6 +1315,7 @@ def test_edgebench_cli_merges_sforge_env_with_explicit_runner_config(tmp_path, m judge=SimpleNamespace(selection='score_first', score_direction='maximize'))) def backend(**kwargs): assert kwargs['oauth_proxy'] is custom + assert kwargs.get('docker_host_bridge', False) is bridge return SimpleNamespace(image_exists=lambda image: True) monkeypatch.setattr(run, 'RecordingDockerBackend', backend) def stop_before_solver(**kwargs): @@ -1326,10 +1329,12 @@ def stop_before_solver(**kwargs): monkeypatch.setattr(run, 'run_agent', stop_before_solver) with pytest.raises(RuntimeError, match='synthetic launch boundary'): run.main(['--task', 'fixture', '--tasks-dir', str(tmp_path), '--log-dir', str(tmp_path), - '--run-id', 'route', '--worker', 'official', '--model', 'fixture', - '--effort', 'xhigh', '--judge-url', 'http://127.0.0.1:9999']) + '--run-id', 'route', '--worker', worker, '--model', 'fixture', + '--effort', 'xhigh', '--judge-url', 'http://127.0.0.1:9999', + *(['--docker-host-bridge'] if bridge else [])]) receipt = (tmp_path / 'runs/route/fixture/runtime-receipt.json').read_text() assert 'synthetic-provider-secret' not in receipt + assert json.loads(receipt).get('docker_host_bridge', False) is bridge @pytest.mark.parametrize("capture", [False, True])