From 4be3eb2b9d9661aae36218df4a2d55e92bab33da Mon Sep 17 00:00:00 2001 From: Bai Li Date: Tue, 21 Jul 2026 12:32:36 -0700 Subject: [PATCH 1/8] fix(antigravity): pin google-antigravity 0.1.7 to load on glibc 2.35 The pinned 0.1.5 localharness binary requires the GLIBC_ABI_DT_RELR ABI (glibc 2.36+), so it fails to load on the eval agents' Ubuntu 22.04 (glibc 2.35), taking down the antigravity tempdir/host path (the activation suite and container-less skills tasks) while docker-driver tasks stay fine. 0.1.7 links without DT_RELR (max required symbol GLIBC_2.26) and loads on 2.35+. The manylinux_2_17 wheel tag alone does not guarantee this, so the pin comment documents verifying ELF load before any future bump. Co-Authored-By: Claude Opus 4.8 (1M context) --- pyproject.toml | 11 ++++++++--- uv.lock | 14 +++++++------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d66c4bf..c1b8fa2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,12 +92,17 @@ codex = [ # (drives the bundled `localharness` binary, authenticated via GEMINI_API_KEY). # Pinned exactly: the harness binary is a dominant non-model driver of eval results, # so it travels with the coder_eval release tag and is bumped deliberately (mirrors -# the claude-code CLI / codex cli-bin pins). The wheel ships platform-specific -# (manylinux x86_64 + aarch64, macOS, Windows), so CI/Linux/Docker installs work. +# the claude-code CLI / codex cli-bin pins). +# glibc note — verify before bumping: the wheel's manylinux_2_17 tag is NOT a +# reliable floor; the bundled `localharness` ELF can require a newer glibc than the +# tag advertises. Eval agents run on Ubuntu 22.04 (glibc 2.35), so the pinned build +# must load there. 0.1.7 links without DT_RELR (max required symbol GLIBC_2.26) and +# loads on 2.35+; confirm any future pin does too (readelf -d | grep RELR must be +# empty / max GLIBC symbol <= the oldest target's glibc). # Without this extra the framework still installs and runs; Antigravity-dependent # code paths fail at start() with a clear hint pointing back here. antigravity = [ - "google-antigravity==0.1.5", + "google-antigravity==0.1.7", ] [project.scripts] diff --git a/uv.lock b/uv.lock index a79a6d0..9447d0a 100644 --- a/uv.lock +++ b/uv.lock @@ -503,7 +503,7 @@ requires-dist = [ { name = "bandit", extras = ["toml"], marker = "extra == 'dev'", specifier = ">=1.9.4" }, { name = "claude-agent-sdk", specifier = ">=0.2.82" }, { name = "click", specifier = ">=8.3.3" }, - { name = "google-antigravity", marker = "extra == 'antigravity'", specifier = "==0.1.5" }, + { name = "google-antigravity", marker = "extra == 'antigravity'", specifier = "==0.1.7" }, { name = "jmespath", specifier = ">=1.1.0" }, { name = "jsonschema", specifier = ">=4.26.0" }, { name = "mcp", marker = "extra == 'dev'", specifier = ">=1.28.1" }, @@ -803,7 +803,7 @@ wheels = [ [[package]] name = "google-antigravity" -version = "0.1.5" +version = "0.1.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "absl-py" }, @@ -815,11 +815,11 @@ dependencies = [ { name = "websockets" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/15/7c/8d6151f12e4544229754575c6f8a2b57b615aab312440f2db64580af42e0/google_antigravity-0.1.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:0a12e30836fb6641bc63beaa3e65d3bcbfc39c01a1eb07c7ca7edba317537660", size = 31404545, upload-time = "2026-06-25T14:53:52.586Z" }, - { url = "https://files.pythonhosted.org/packages/0e/e8/4a85ba4a7be1c35acb18c4f070e5049df6f185547b0c9d6794acbc06a452/google_antigravity-0.1.5-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:a74892f9251e4c2cd9962d8631680aa8266d2e216527dd68cf8ac8de1137c996", size = 33945922, upload-time = "2026-06-25T14:53:55.825Z" }, - { url = "https://files.pythonhosted.org/packages/6e/18/d89abb1dafe906451c2bd13f0c32917e1d35ba95d2436a06907045a69cfd/google_antigravity-0.1.5-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:ec1ebe4044b92e41f6f1632cfac98bb29f03ef0be52446cb5a67576147148363", size = 37161595, upload-time = "2026-06-25T14:53:58.841Z" }, - { url = "https://files.pythonhosted.org/packages/f3/5e/c9b6a06f21aeb3547c8194aec11f5fbeffedc0d7973effbace96beca84b2/google_antigravity-0.1.5-py3-none-win_amd64.whl", hash = "sha256:e8707b7fc64c887edf27176b1e8aa52e6cfe50e51dc2d04ef4f57f7ad26f65ee", size = 36188777, upload-time = "2026-06-25T14:54:02.034Z" }, - { url = "https://files.pythonhosted.org/packages/1b/de/e231b4fec58f9415490fb6223556a6dd0e3278876dd508f5396864e25cb3/google_antigravity-0.1.5-py3-none-win_arm64.whl", hash = "sha256:2830e90139552ef3de7d8775ef0ea8d1a1ec2c0f523a8d1db30132b942deb4a0", size = 32694572, upload-time = "2026-06-25T14:54:05.439Z" }, + { url = "https://files.pythonhosted.org/packages/c6/aa/609ecaca0611519fe49094d7993e046f274c5d8622541d502b4a6b6bc8e5/google_antigravity-0.1.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:2707d823bba1dfc4c3384d7d78aea13eb25fdf8a28a8345944b38c16c1405c8a", size = 31644051, upload-time = "2026-07-16T21:03:58.157Z" }, + { url = "https://files.pythonhosted.org/packages/ef/d9/8936a14195c3d923c49f3ee4810ab9dd0655c9618814a7c35837734e075d/google_antigravity-0.1.7-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:c5070a0f9a6c4406ea460550ae1a543cfb7c9fc185b80921c31ea681aa645ec5", size = 37042570, upload-time = "2026-07-16T21:04:01.126Z" }, + { url = "https://files.pythonhosted.org/packages/41/55/c5e11b0f91c70540a0e247c53de117c0986b167e94e205c131dc8fadcf76/google_antigravity-0.1.7-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:0958f4481dd4700a0a32acb948f959fb1858657038f103c5a0d067727010f71e", size = 39141605, upload-time = "2026-07-16T21:04:04.392Z" }, + { url = "https://files.pythonhosted.org/packages/3f/80/1bd2f9ef2dc20b498186d6e888c09b86ce519748e85188c898d1c7bba77c/google_antigravity-0.1.7-py3-none-win_amd64.whl", hash = "sha256:c2a20d7a22b9ca086c570efce2d36b688ef15e30a371f7fbb93cd05aba49d563", size = 36434279, upload-time = "2026-07-16T21:04:07.246Z" }, + { url = "https://files.pythonhosted.org/packages/5e/3b/e2cf44fb9da8ac407199c1c351fcdc4876c0aece1b61123b2a439a36af66/google_antigravity-0.1.7-py3-none-win_arm64.whl", hash = "sha256:6acaa5ed047cb9bad6ad50992aea71616eff2a6767161b96aab5f7a1cd765ce5", size = 32907283, upload-time = "2026-07-16T21:04:10.104Z" }, ] [[package]] From 7dbccc4ec8b49422b665c9f97d416cead48a0252 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Tue, 21 Jul 2026 12:32:36 -0700 Subject: [PATCH 2/8] fix(codex): use full-access sandbox on coder_eval-managed tempdir path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex's read-only / workspace-write sandboxes shell out to an OS-level helper (Landlock, or a bwrap re-exec) that fails to initialize on managed CI hosts ("bwrap: execvp .../codex: Permission denied" where unprivileged user namespaces or exec mounts are restricted); its writes/execs then fail silently and the task produces no artifacts. The docker path already avoids this via CODER_EVAL_IN_CONTAINER. Thread a `sandbox_managed` signal through Agent.start() so the tempdir driver — where coder_eval's ephemeral per-task dir is the isolation boundary — also falls back to full-access, matching Claude Code (no in-agent OS sandbox on that path). Hard isolation of untrusted actions remains the docker driver's job. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/coder_eval/agent.py | 5 +++ src/coder_eval/agents/antigravity_agent.py | 5 +++ src/coder_eval/agents/claude_code_agent.py | 4 ++ src/coder_eval/agents/codex_agent.py | 41 ++++++++++++++------ src/coder_eval/agents/noop_agent.py | 1 + src/coder_eval/orchestrator.py | 9 +++++ tests/test_codex_agent.py | 45 ++++++++++++++++++++++ 7 files changed, 98 insertions(+), 12 deletions(-) diff --git a/src/coder_eval/agent.py b/src/coder_eval/agent.py index 2b244ce..4af84cc 100644 --- a/src/coder_eval/agent.py +++ b/src/coder_eval/agent.py @@ -167,6 +167,7 @@ async def start( *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None, + sandbox_managed: bool = False, ) -> None: """Initialize and start the agent. @@ -180,6 +181,10 @@ async def start( instead of walking up from CWD. An external ``PLUGIN_TOOLS_DIR`` in the process environment still wins. Implementations that don't shell out may ignore this argument. + sandbox_managed: True when coder_eval itself provides this run's isolation + boundary (the ephemeral host tempdir it created for the task). Agents + that run their own in-process OS sandbox may use this to skip it as + redundant; agents without one may ignore this argument. """ pass diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index 8506154..97df31b 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -290,9 +290,14 @@ async def start( *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None, + sandbox_managed: bool = False, ) -> None: """Initialize and start the Antigravity agent's local harness session. + (``sandbox_managed`` is accepted for interface parity with the base + ``Agent.start`` and ignored: the Antigravity harness has no in-process + OS sandbox to drop.) + Args: working_directory: Path to the sandbox working directory. The primary ``workspace`` so file writes (and run_command) operate there — diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index cd24451..998ce16 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -692,6 +692,7 @@ async def start( *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None, + sandbox_managed: bool = False, ) -> None: """Initialize and start the Claude Code agent. @@ -701,6 +702,9 @@ async def start( subprocess (typically the resolved ``SandboxConfig.mock_path_dirs``). plugin_tools_dir: Canonical ``node_modules/@uipath`` to export as ``PLUGIN_TOOLS_DIR``. An external env-var pin still wins. + sandbox_managed: Accepted for interface parity with the base + ``Agent.start`` and ignored — Claude Code runs the tempdir path + with no in-agent OS sandbox to drop. """ self.working_directory = Path(working_directory) self._env_path_prepend = list(env_path_prepend or []) diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index 35b978f..b190cbe 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -663,6 +663,11 @@ def __init__( self.working_directory: Path | None = None self._env_path_prepend: list[str] = [] self._login_shell_home: Path | None = None + # True when coder_eval itself owns the isolation boundary for this run + # (the ephemeral host tempdir it created). Set by start(); combined with + # the docker-path CODER_EVAL_IN_CONTAINER marker in _build_thread_options + # to drop Codex's redundant in-process OS sandbox (rationale there). + self._sandbox_managed: bool = False # _state / _iteration / _iteration_was_incremented / pending_turn lifecycle # bookkeeping lives on the Agent base class (shared defaults + helpers). self._log = PrefixedAdapter(logger, {"prefix": instance_name}) @@ -678,6 +683,7 @@ async def start( *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None, + sandbox_managed: bool = False, ) -> None: """Initialize and start the Codex agent. @@ -688,9 +694,14 @@ async def start( ``SandboxConfig.mock_path_dirs``), so mock CLIs shadow the real ones — same semantics as the Claude agent. plugin_tools_dir: Optional plugin tools directory (for skills setup) + sandbox_managed: True when coder_eval itself provides the isolation + boundary for this run (the ephemeral host tempdir it created). + Lets ``_build_thread_options`` drop Codex's redundant in-process + OS sandbox, which also can't initialize on some managed CI hosts. """ self.working_directory = Path(working_directory) self._env_path_prepend = list(env_path_prepend or []) + self._sandbox_managed = sandbox_managed self._setup_login_shell_home() self._state = AgentState.WORKING @@ -1263,18 +1274,24 @@ def _build_thread_options(self) -> dict[str, Any]: sandbox_mode_str = _PERMISSION_MODE_TO_SANDBOX.get(permission_mode, "workspace-write") approval_mode_str = _CODEX_APPROVAL_MODE - # Codex's read-only / workspace-write sandboxes are enforced by a Landlock - # helper that can't initialize inside the eval's docker container — its - # writes/execs then fail silently and the agent produces no artifacts - # (FAILURE with score 0, no loud error). When the harness already provides - # container isolation (docker driver, signalled by CODER_EVAL_IN_CONTAINER), - # the in-process sandbox is both redundant and broken, so fall back to - # full-access: the container IS the trust boundary. No-op on the host - # (tempdir/host runs), where Landlock works and the marker is unset. - if sandbox_mode_str != "full-access" and os.getenv("CODER_EVAL_IN_CONTAINER"): - self._log.warning( - f"Codex sandbox '{sandbox_mode_str}' can't initialize inside the eval container " - + "(Landlock unavailable); using 'full-access' — the container provides isolation." + # Codex's read-only / workspace-write sandboxes are enforced by an OS-level + # helper (Landlock, or a bubblewrap re-exec where Landlock is unavailable). + # That helper can outright fail to start on managed hosts — inside the eval + # container (Landlock unavailable) and on constrained CI agents where the + # bwrap re-exec is denied ("bwrap: execvp .../codex: Permission denied" when + # unprivileged user namespaces or exec mounts are restricted). When it fails, + # writes/execs fail silently and the agent produces no artifacts (FAILURE, + # score 0, no loud error). Whenever coder_eval already owns the isolation + # boundary — a docker container (CODER_EVAL_IN_CONTAINER) OR the ephemeral + # host tempdir it created (sandbox_managed) — this in-process sandbox is + # redundant, so fall back to full-access. This also matches every other + # harness (e.g. Claude Code), which runs the tempdir path with no in-agent + # OS sandbox; hard isolation of untrusted actions is the docker driver's job. + if sandbox_mode_str != "full-access" and (os.getenv("CODER_EVAL_IN_CONTAINER") or self._sandbox_managed): + boundary = "eval container" if os.getenv("CODER_EVAL_IN_CONTAINER") else "coder_eval-managed host tempdir" + self._log.debug( + f"Codex sandbox '{sandbox_mode_str}' → 'full-access': the {boundary} is the isolation " + + "boundary (the in-process OS sandbox is redundant here and can fail to init on CI hosts)." ) sandbox_mode_str = "full-access" diff --git a/src/coder_eval/agents/noop_agent.py b/src/coder_eval/agents/noop_agent.py index e39c45d..fcd7406 100644 --- a/src/coder_eval/agents/noop_agent.py +++ b/src/coder_eval/agents/noop_agent.py @@ -54,6 +54,7 @@ async def start( *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None, + sandbox_managed: bool = False, ) -> None: """No-op: there is no agent process to launch.""" self._state = AgentState.WORKING diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index b158147..87c25f7 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -970,12 +970,21 @@ async def _setup_sandbox() -> Any: env_path_prepend = [str(p) for p in self.sandbox.resolved_mock_path_dirs] plugin_tools_dir = self.sandbox.plugin_tools_dir + # On the tempdir driver coder_eval itself owns the isolation boundary (the + # ephemeral per-task tempdir), so an agent's own in-process OS sandbox is + # redundant — and on constrained CI hosts it can fail to initialize. Tell + # the agent so it can skip it. The docker driver signals the equivalent + # from inside the container via CODER_EVAL_IN_CONTAINER. + driver = self.task.sandbox.driver if self.task.sandbox else "tempdir" + sandbox_managed = driver == "tempdir" + async def _start_agent() -> None: assert self.agent is not None await self.agent.start( str(sandbox_dir), env_path_prepend=env_path_prepend, plugin_tools_dir=plugin_tools_dir, + sandbox_managed=sandbox_managed, ) await execute_with_retry( diff --git a/tests/test_codex_agent.py b/tests/test_codex_agent.py index e2dcc0d..7abc573 100644 --- a/tests/test_codex_agent.py +++ b/tests/test_codex_agent.py @@ -152,6 +152,51 @@ def test_full_access_unchanged_in_container(self, monkeypatch): assert agent._build_thread_options()["sandbox"] == Sandbox("full-access") +class TestSandboxManagedHostFallback: + """On the tempdir driver coder_eval owns the isolation boundary (the ephemeral + per-task tempdir), so ``start(sandbox_managed=True)`` makes ``_build_thread_options`` + drop Codex's redundant in-process OS sandbox to full-access — WITHOUT the docker + container marker. This is the host/CI path where the sandbox's bwrap re-exec is + denied; the fallback is what keeps those tasks from silently producing no artifacts.""" + + def test_workspace_write_upgraded_to_full_access_when_managed(self, monkeypatch): + from openai_codex.api import Sandbox # pyright: ignore[reportPrivateImportUsage] + + monkeypatch.delenv("CODER_EVAL_IN_CONTAINER", raising=False) + agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, permission_mode="acceptEdits")) + agent._sandbox_managed = True + assert agent._build_thread_options()["sandbox"] == Sandbox("full-access") + + def test_read_only_upgraded_to_full_access_when_managed(self, monkeypatch): + from openai_codex.api import Sandbox # pyright: ignore[reportPrivateImportUsage] + + monkeypatch.delenv("CODER_EVAL_IN_CONTAINER", raising=False) + agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, permission_mode="plan")) + agent._sandbox_managed = True + assert agent._build_thread_options()["sandbox"] == Sandbox("full-access") + + def test_not_managed_and_no_container_keeps_landlock_sandbox(self, monkeypatch): + from openai_codex.api import Sandbox # pyright: ignore[reportPrivateImportUsage] + + monkeypatch.delenv("CODER_EVAL_IN_CONTAINER", raising=False) + agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, permission_mode="acceptEdits")) + assert agent._sandbox_managed is False # default off + assert agent._build_thread_options()["sandbox"] == Sandbox("workspace-write") + + def test_full_access_unchanged_when_managed(self, monkeypatch): + from openai_codex.api import Sandbox # pyright: ignore[reportPrivateImportUsage] + + monkeypatch.delenv("CODER_EVAL_IN_CONTAINER", raising=False) + agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, permission_mode="bypassPermissions")) + agent._sandbox_managed = True + assert agent._build_thread_options()["sandbox"] == Sandbox("full-access") + + def test_default_sandbox_managed_is_false(self): + """Fresh agent defaults to not-managed; start() opts in per the driver.""" + agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, permission_mode="acceptEdits")) + assert agent._sandbox_managed is False + + class TestCodexEnvironmentConfiguration: """Test _build_codex_env: only CODEX_API_KEY travels via env.""" From 3a4088c78f101abe616c6e48746c5bb65721101c Mon Sep 17 00:00:00 2001 From: Bai Li Date: Tue, 21 Jul 2026 13:27:55 -0700 Subject: [PATCH 3/8] test: accept sandbox_managed kwarg in agent test doubles The orchestrator now threads `sandbox_managed` into `Agent.start()`, and the codebase's convention is that agent test doubles mirror the ABC signature exactly. Add the keyword-only param (accepted and ignored, like `env_path_prepend`) to every double that declares `start()`, so the orchestrator-routed doubles (MockAgent and the inline DummyAgents) stop raising `unexpected keyword argument 'sandbox_managed'`. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/fixtures/mock_agent.py | 9 ++++++++- tests/fixtures/text_stub_agent.py | 7 ++++++- tests/test_orchestrator.py | 5 ++++- tests/test_simulation_integration.py | 7 ++++++- tests/test_user_simulator.py | 2 ++ 5 files changed, 26 insertions(+), 4 deletions(-) diff --git a/tests/fixtures/mock_agent.py b/tests/fixtures/mock_agent.py index 9643f35..2065615 100644 --- a/tests/fixtures/mock_agent.py +++ b/tests/fixtures/mock_agent.py @@ -38,13 +38,20 @@ def __init__(self, task: TaskDefinition, scenario: str = "success"): self._iteration = 0 # Track iteration count for TurnRecord async def start( - self, working_directory: str, *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None + self, + working_directory: str, + *, + env_path_prepend: list[str] | None = None, + plugin_tools_dir: str | None = None, + sandbox_managed: bool = False, ) -> None: """Initialize mock agent with working directory. Args: working_directory: Path to the sandbox working directory env_path_prepend: Ignored. Accepted only to match the Agent ABC signature. + plugin_tools_dir: Ignored. Accepted only to match the Agent ABC signature. + sandbox_managed: Ignored. Accepted only to match the Agent ABC signature. """ self.working_directory = Path(working_directory) self.state = AgentState.WORKING diff --git a/tests/fixtures/text_stub_agent.py b/tests/fixtures/text_stub_agent.py index f469d1f..bd2024e 100644 --- a/tests/fixtures/text_stub_agent.py +++ b/tests/fixtures/text_stub_agent.py @@ -26,7 +26,12 @@ def __init__(self, responses: list[str]) -> None: self.stopped = False async def start( - self, working_directory: str, *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None + self, + working_directory: str, + *, + env_path_prepend: list[str] | None = None, + plugin_tools_dir: str | None = None, + sandbox_managed: bool = False, ) -> None: self.working_directory = Path(working_directory) self._state = AgentState.WORKING diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 6b3f25e..c535e80 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -561,6 +561,7 @@ async def start( *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None, + sandbox_managed: bool = False, ) -> None: self.working_directory = working_directory @@ -625,7 +626,9 @@ async def test_direct_write_warns_on_non_empty_target(tmp_path, monkeypatch, cap from coder_eval.sandbox import Sandbox class DummyAgent: - async def start(self, working_directory, *, env_path_prepend=None, plugin_tools_dir=None): + async def start( + self, working_directory, *, env_path_prepend=None, plugin_tools_dir=None, sandbox_managed=False + ): self.working_directory = working_directory def get_sdk_options(self): diff --git a/tests/test_simulation_integration.py b/tests/test_simulation_integration.py index f914bb9..35d46c2 100644 --- a/tests/test_simulation_integration.py +++ b/tests/test_simulation_integration.py @@ -110,7 +110,12 @@ def __init__(self, message: str = "simulator exploded") -> None: self._state = AgentState.WORKING async def start( - self, working_directory: str, *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None + self, + working_directory: str, + *, + env_path_prepend: list[str] | None = None, + plugin_tools_dir: str | None = None, + sandbox_managed: bool = False, ) -> None: pass diff --git a/tests/test_user_simulator.py b/tests/test_user_simulator.py index 3adb964..ac13dec 100644 --- a/tests/test_user_simulator.py +++ b/tests/test_user_simulator.py @@ -267,6 +267,7 @@ async def start( *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None, + sandbox_managed: bool = False, ) -> None: self.captured_dir = Path(working_directory) raise RuntimeError("boom") @@ -303,6 +304,7 @@ async def start( *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None, + sandbox_managed: bool = False, ) -> None: self.captured_dir = Path(working_directory) raise asyncio.CancelledError() From 89463a2fd7f85f98a862ff164d70ba8021561752 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Tue, 21 Jul 2026 13:27:55 -0700 Subject: [PATCH 4/8] fix(deps): bump pyasn1 to 0.6.4 for GHSA-8ppf-4f7h-5ppj / GHSA-hm4w-wwcw-mr6r MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pyasn1 0.6.3 (transitive via google-auth → pyasn1-modules) carries two High advisories fixed in 0.6.4. Pre-existing on main; bumping the lock clears both osv-scanner and pip-audit. No suppression entry is added because a fix is available (osv-scanner.toml is only for unfixable advisories). Co-Authored-By: Claude Opus 4.8 (1M context) --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 9447d0a..1fd7b57 100644 --- a/uv.lock +++ b/uv.lock @@ -1684,11 +1684,11 @@ wheels = [ [[package]] name = "pyasn1" -version = "0.6.3" +version = "0.6.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, + { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, ] [[package]] From fb93be2336690058213c9f3c776f19d234d2309a Mon Sep 17 00:00:00 2001 From: Bai Li Date: Tue, 21 Jul 2026 13:52:08 -0700 Subject: [PATCH 5/8] chore(deps): upgrade agent SDKs to latest (claude 0.2.124, codex 0.144.4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - claude-agent-sdk 0.2.82 -> 0.2.124 - openai-codex 0.1.0b3 -> 0.144.4, and its pinned cli-bin 0.137.0a4 -> 0.144.4 (the SDK version now tracks the codex CLI line; the 0.144.x jump is a re-numbering, not an API rewrite). The public API surface codex_agent.py depends on — Codex/Thread/turn, the notification method strings, item types, and token-usage fields — stayed source-compatible, so no agent code changed. - google-antigravity already at 0.1.7 (latest; pinned by the tempdir fix). Full suite incl. the live codex integration tests is green on all three. Co-Authored-By: Claude Opus 4.8 (1M context) --- pyproject.toml | 18 ++++++++++-------- uv.lock | 44 ++++++++++++++++++++++---------------------- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c1b8fa2..56ddb69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "rich>=14.3.3", "python-dotenv>=1.2.2", "anthropic>=0.86.0", - "claude-agent-sdk>=0.2.82", + "claude-agent-sdk>=0.2.124", "anyio>=4.13.0", "radon>=6.0.1", "tqdm>=4.67.3", @@ -85,7 +85,7 @@ uipath = [ # Without this extra, the framework still installs and runs; Codex-dependent # code paths fail at dispatch with a clear hint pointing back here. codex = [ - "openai-codex>=0.1.0b3", + "openai-codex>=0.144.4", ] # Optional extra that enables Antigravity agent support: # - AntigravityAgent implementation using Google's official google-antigravity SDK @@ -134,12 +134,14 @@ packages = ["src/coder_eval"] allow-direct-references = true [tool.uv] -# openai-codex 0.1.0b3 hardpins `openai-codex-cli-bin==0.137.0a4`, a -# PRE-RELEASE. Because that pin is transitive (not referenced directly by us), -# uv won't select it without an explicit opt-in. Naming it here as an override -# both enables the pre-release and documents the pinned cli-bin build; 0.137.0a4 -# publishes manylinux wheels (x86_64 + aarch64), so CI/Linux installs work. -override-dependencies = ["openai-codex-cli-bin==0.137.0a4"] +# openai-codex 0.144.4 hardpins `openai-codex-cli-bin==0.144.4` (a stable +# release; the SDK version now tracks the codex CLI version line). Naming it +# here documents and holds the pinned cli-bin build — the harness binary is a +# dominant non-model driver of eval results, so it travels with the coder_eval +# release tag and is bumped deliberately (mirrors the antigravity localharness +# and claude-code CLI pins). 0.144.4 publishes manylinux wheels (x86_64 + +# aarch64), so CI/Linux installs work. +override-dependencies = ["openai-codex-cli-bin==0.144.4"] constraint-dependencies = [ # Fix known CVEs in transitive dependencies diff --git a/uv.lock b/uv.lock index 1fd7b57..8f02bb1 100644 --- a/uv.lock +++ b/uv.lock @@ -20,7 +20,7 @@ constraints = [ { name = "starlette", specifier = ">=1.3.1" }, { name = "urllib3", specifier = ">=2.6.3" }, ] -overrides = [{ name = "openai-codex-cli-bin", specifier = "==0.137.0a4" }] +overrides = [{ name = "openai-codex-cli-bin", specifier = "==0.144.4" }] [[package]] name = "absl-py" @@ -419,20 +419,20 @@ wheels = [ [[package]] name = "claude-agent-sdk" -version = "0.2.82" +version = "0.2.124" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "mcp" }, { name = "sniffio" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/57/3d/f75aaecf476c2b2a903dbba6042171b6683eb91c1f97f3ad894784cec270/claude_agent_sdk-0.2.82.tar.gz", hash = "sha256:3e907b7d2bf52a5917d96a3ce336b8aa5546ea31e29ce826a7f346622cf7f4bf", size = 252053, upload-time = "2026-05-15T03:45:34.251Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7a/5f/55c18111253f66158d8f8dc99b0e8f3398272145fee1984ac67d3050d788/claude_agent_sdk-0.2.124.tar.gz", hash = "sha256:ea2207d55f81b6773d5be8928e1c5e1206f49ee402cafd5eb1cc09d389dd4bae", size = 304107, upload-time = "2026-07-20T22:27:26.957Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/19/bc/27cf3aec2a24f2ed1f60277de795496b808a761d2a7a3fd34602a2fec37d/claude_agent_sdk-0.2.82-py3-none-macosx_11_0_arm64.whl", hash = "sha256:24ad8ccbcee9afe206ae5d621a9e40a5022ca3eb8c2c672b36916d3e70746e42", size = 61473506, upload-time = "2026-05-15T03:45:38.745Z" }, - { url = "https://files.pythonhosted.org/packages/96/91/95a83f018dbc8c113233eb542bccf17c1a3f5f689448700daf950602bf5e/claude_agent_sdk-0.2.82-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:13e54d5163d9d4f899c4e2a3f14df597f4e050d5afa104618ccf7bb37b372ad1", size = 63541975, upload-time = "2026-05-15T03:45:46.005Z" }, - { url = "https://files.pythonhosted.org/packages/be/07/9356fe0e30f988bade6b116ecc602b4a9ae4df34fa055305187a835e36e0/claude_agent_sdk-0.2.82-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:3b0a0e3f0927737f1fc91ee4549185172243a4e8f135d4c1e4f1f1eba91373e1", size = 71212904, upload-time = "2026-05-15T03:45:51.121Z" }, - { url = "https://files.pythonhosted.org/packages/b1/d9/e2920b4b6c75cf79ec87ebfb4cc4447c78a4f26317cb3fed77e79fcc804e/claude_agent_sdk-0.2.82-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:b05873f9df01c5894930b87f6ca9315f0d97f1563bc2e4dc0fafe0d4a1e31997", size = 71381948, upload-time = "2026-05-15T03:45:56.153Z" }, - { url = "https://files.pythonhosted.org/packages/89/80/c3ec5a89c735a96d35fe12b6262517169b396ff366149a3b9f4387f797c1/claude_agent_sdk-0.2.82-py3-none-win_amd64.whl", hash = "sha256:71e85e4f50d04cd95e687898092f03648e74e1cd2537583de93370d2da1c0586", size = 71990462, upload-time = "2026-05-15T03:46:01.646Z" }, + { url = "https://files.pythonhosted.org/packages/0f/27/9412242d21ed1f9d45d78c28b28cdbbf24ba1470005a5adf439f2bb2d2e6/claude_agent_sdk-0.2.124-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b52f600ec0c694ac0f54a4e9a4905a014bca8e9fcdad9db96067e123ee030ec9", size = 73073509, upload-time = "2026-07-20T22:27:30.405Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f7/65a84a50cb5de82b3247be83914acecac65d5c890591b1f93ba3479d2235/claude_agent_sdk-0.2.124-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:33f0a45bc3c1768d35838461d76189d897769d31b7fabd1ba540e67669af8094", size = 78211990, upload-time = "2026-07-20T22:27:34.877Z" }, + { url = "https://files.pythonhosted.org/packages/f6/83/4a545978ab9e90844616ad121c3ccbbdf28f15968d1a75f0376739ffabf3/claude_agent_sdk-0.2.124-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:46d1c4e9abffbf5c801fe362fe21436287e3d3b75714ee908a04f1c440a05ab7", size = 82657033, upload-time = "2026-07-20T22:27:39.01Z" }, + { url = "https://files.pythonhosted.org/packages/3e/e1/ada3be7fa95f8a96ac9f23d27609513bd05bebc6ce8a2730b8ffe96db12c/claude_agent_sdk-0.2.124-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:7eaf8a1a746f9fd10ba534101d89188308688990be2d6f9adea9361a4c08c7b7", size = 83708175, upload-time = "2026-07-20T22:27:42.736Z" }, + { url = "https://files.pythonhosted.org/packages/55/18/bbb0dce553c5efbf646fb535e0f0d26610e93d51f57e92bffa7aca1593e3/claude_agent_sdk-0.2.124-py3-none-win_amd64.whl", hash = "sha256:128c6dcb2058d829697d98a93ab2f6c35d35fac0ecabeef478a051959dbe5fbe", size = 83603754, upload-time = "2026-07-20T22:27:46.553Z" }, ] [[package]] @@ -501,13 +501,13 @@ requires-dist = [ { name = "anyio", specifier = ">=4.13.0" }, { name = "azure-monitor-opentelemetry-exporter", specifier = ">=1.0.0b30,<1.1.0" }, { name = "bandit", extras = ["toml"], marker = "extra == 'dev'", specifier = ">=1.9.4" }, - { name = "claude-agent-sdk", specifier = ">=0.2.82" }, + { name = "claude-agent-sdk", specifier = ">=0.2.124" }, { name = "click", specifier = ">=8.3.3" }, { name = "google-antigravity", marker = "extra == 'antigravity'", specifier = "==0.1.7" }, { name = "jmespath", specifier = ">=1.1.0" }, { name = "jsonschema", specifier = ">=4.26.0" }, { name = "mcp", marker = "extra == 'dev'", specifier = ">=1.28.1" }, - { name = "openai-codex", marker = "extra == 'codex'", specifier = ">=0.1.0b3" }, + { name = "openai-codex", marker = "extra == 'codex'", specifier = ">=0.144.4" }, { name = "opentelemetry-sdk", specifier = ">=1.30.0,<2.0.0" }, { name = "pip-audit", marker = "extra == 'dev'", specifier = ">=2.10.0" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=4.5.1" }, @@ -1323,30 +1323,30 @@ wheels = [ [[package]] name = "openai-codex" -version = "0.1.0b3" +version = "0.144.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "openai-codex-cli-bin" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ae/1c/1e5e8b83ea72164d32b1f4e67fc703c8b83591f498a7aaf96f39d352b453/openai_codex-0.1.0b3.tar.gz", hash = "sha256:b76b7afe97953ac65648e9b8ca116b5ff273de91086549bd7ec88037cdc16cab", size = 58995, upload-time = "2026-06-03T19:17:34.707Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/7d/4b999b0ddd05c22d83cc2e879c95e1e92e3b7808b58ac561c4ea91fca1c4/openai_codex-0.144.4.tar.gz", hash = "sha256:91c63a7cb213441569f130e593386b34657ab9e726ae88af255f0ecb8de08ea5", size = 68324, upload-time = "2026-07-17T23:42:17.013Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/ef/f77037d9ccde80a688a17a06aea5a56813ad9c365d49b3f1c7913422af8b/openai_codex-0.1.0b3-py3-none-any.whl", hash = "sha256:8d1f9d346667aeecb435c6a45d0edb3f016187276ec452cf8094d813896276c4", size = 65639, upload-time = "2026-06-03T19:17:33.208Z" }, + { url = "https://files.pythonhosted.org/packages/17/35/5d2a13e38d91278019f18757ac10426d2649b7ec6f031818c792f64559e9/openai_codex-0.144.4-py3-none-any.whl", hash = "sha256:de1513a6e94b9a8d7728a3b74298bc1469428ade10ba0ef2d5db47dd1cb606f5", size = 76244, upload-time = "2026-07-17T23:42:15.658Z" }, ] [[package]] name = "openai-codex-cli-bin" -version = "0.137.0a4" +version = "0.144.4" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bd/60/af73ef1676cd477fa83ed4b889bf3b57c63c47dd87025b2cc4262793cff6/openai_codex_cli_bin-0.137.0a4-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:b33c3917e0b58d527ee11a11a78ad390f7d8e6aa25577dd21665ab3c8bf5cf9a", size = 94300191, upload-time = "2026-06-03T18:44:36.312Z" }, - { url = "https://files.pythonhosted.org/packages/92/8f/d1a5f8c87176e00ef6a85798794f4530f5eb04e5a1a13468b5b3c3a361f9/openai_codex_cli_bin-0.137.0a4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:3d0f0bc5becc88c61952fbfa9bd792ac9d74fa78b3a6bd40f545b612048b07eb", size = 83924479, upload-time = "2026-06-03T18:44:40.854Z" }, - { url = "https://files.pythonhosted.org/packages/3e/3c/fc00bcdc0c302208317d5eb1d0bfaab3024f351cd0121400f19baa6b19aa/openai_codex_cli_bin-0.137.0a4-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:2f1656339e2736868c4cce59f6d9e5c633879123687169b03b1137d42bf2c11a", size = 83363315, upload-time = "2026-06-03T18:44:44.851Z" }, - { url = "https://files.pythonhosted.org/packages/ec/09/39362e944ebeb12fcbfb86881fbb4dd6e806f77f7541c1f1f993bb9351a0/openai_codex_cli_bin-0.137.0a4-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:6454f838d44c56c1ed07a29b391fa412785e5dd2ffd06db0b62e62478c19bb64", size = 90611239, upload-time = "2026-06-03T18:44:49.338Z" }, - { url = "https://files.pythonhosted.org/packages/fa/38/87b1247fdfe95cddce7f7fe8331d6843cf037e14292c0f5004e23247133b/openai_codex_cli_bin-0.137.0a4-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:f5ae7401d00c65d56a75d9645d7bf87d809566a12d238e4b2a8b328a02f2316e", size = 83363315, upload-time = "2026-06-03T18:44:53.428Z" }, - { url = "https://files.pythonhosted.org/packages/fb/c4/3c693ad07e587f6b3a28128c417f2e831d81a40cdbd85c0e5f0f36aaff82/openai_codex_cli_bin-0.137.0a4-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:3dcec1e649448be498d6e7ec0e1f71dca83efa76063d90890dafb41e987069b7", size = 90611238, upload-time = "2026-06-03T18:44:57.612Z" }, - { url = "https://files.pythonhosted.org/packages/9e/26/81e037066b9b8d312a6f9e09015e452ce17630d5ab88e02a4c1d9503e4e8/openai_codex_cli_bin-0.137.0a4-py3-none-win_amd64.whl", hash = "sha256:9e13bf68e18e36bd3a0efd51213281c83e9f6ec22bdb7a45bd2e0211822733a9", size = 94744969, upload-time = "2026-06-03T18:45:02.23Z" }, - { url = "https://files.pythonhosted.org/packages/0d/a3/952bc2a5d62373a51fea161effe3b338b3417c2f6e65fe467ed91b205e2b/openai_codex_cli_bin-0.137.0a4-py3-none-win_arm64.whl", hash = "sha256:5ec4303ca2dcb5f838e0de3ca7f44050b6bcdd41d281a178c3a1420a985a515d", size = 86963504, upload-time = "2026-06-03T18:45:07.131Z" }, + { url = "https://files.pythonhosted.org/packages/79/30/7e457c007a32aa7333a78438a9f532504c3b77a1ea57e7808855712c2c0f/openai_codex_cli_bin-0.144.4-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:4d587d152d5f0aa25f21ded4d08aac5150da48639b29fc3e57b2a8b413d06903", size = 126851183, upload-time = "2026-07-15T00:14:00.171Z" }, + { url = "https://files.pythonhosted.org/packages/65/eb/64c180514a2cc3e2500e486813f5a8d7f7e349342e9bebd78d99ddd9791a/openai_codex_cli_bin-0.144.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:05db505a9c7f020f58b70837a94e00d32a50086986c267bcc44ea97b573d4a05", size = 116474758, upload-time = "2026-07-15T00:14:08.177Z" }, + { url = "https://files.pythonhosted.org/packages/85/34/921ab692c7ed140941a91e39b84c6deafddb45072793f3a5f6dcbf86f59a/openai_codex_cli_bin-0.144.4-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:cd4bb31b8a477a3adba22139c8c493693fa5292ba21e86eef08ace3e96c41294", size = 119437596, upload-time = "2026-07-15T00:14:17.418Z" }, + { url = "https://files.pythonhosted.org/packages/25/62/39e630cf8b7b2e2444a5a6669235a6cd88004869a25bb7f3f629ed35638c/openai_codex_cli_bin-0.144.4-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:4106229c38f37245c3eea8d904426afd45b8bf19831765715647f5402f757c06", size = 128817757, upload-time = "2026-07-15T00:14:30.539Z" }, + { url = "https://files.pythonhosted.org/packages/23/24/318f91a95baaff845307e529d138d42a7202e6bd0e626556058eab3dead5/openai_codex_cli_bin-0.144.4-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:d2d3fada11731938e3d3e3660819d5324b7f92e825a0ed5aede63100b36ffc9a", size = 119437594, upload-time = "2026-07-15T00:14:39.327Z" }, + { url = "https://files.pythonhosted.org/packages/2e/a0/65e6fd3a6fba52639937801d9ce517b0c48026458723bb9f08eb07a1dd35/openai_codex_cli_bin-0.144.4-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:70fb62ed7755e332dd8b00ed48e22ee16df10f4a977335039e237cd330490df3", size = 128817755, upload-time = "2026-07-15T00:14:47.849Z" }, + { url = "https://files.pythonhosted.org/packages/e3/8a/3ab5fc97352e8f780d472c8dd6d7504d6a670cd7dede106d461ac1171999/openai_codex_cli_bin-0.144.4-py3-none-win_amd64.whl", hash = "sha256:56e142974467332f1f669b89f2636e06b3ada09413512abb2f24c33b4a4f59fc", size = 140595092, upload-time = "2026-07-15T00:14:58.484Z" }, + { url = "https://files.pythonhosted.org/packages/70/1a/3aa52ab8f89e0f596b6eea835e3f3494697da51917d3231f5f48f92e2bcb/openai_codex_cli_bin-0.144.4-py3-none-win_arm64.whl", hash = "sha256:2ad01058db7181323ae7c6217e560702e38c4f107595b99ab1d0abc9f184890a", size = 130665105, upload-time = "2026-07-15T00:15:08.861Z" }, ] [[package]] From c0d4ee25dd0beef7ab24f48c92c2a96d89aaa1bf Mon Sep 17 00:00:00 2001 From: Bai Li Date: Tue, 21 Jul 2026 14:36:40 -0700 Subject: [PATCH 6/8] ci(release): publish a prerelease from a non-main branch Dispatching release.yml from a branch (github.ref != main) now runs a prerelease instead of a real release: stamp a throwaway rcN version into pyproject/__init__, refresh uv.lock, build, publish the wheel to public PyPI (OIDC) and a : GHCR agent image. It does NOT move :latest, tag, commit, or push to main. This lets a branch be dry-run on the ADO nightly infra (pinned via coderEvalVersion) before merge. The main path is unchanged; the bump input is ignored off main. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/release.yml | 142 ++++++++++++++++++++++++++++------ 1 file changed, 119 insertions(+), 23 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 37220ed..ee21141 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,13 @@ name: Release # and push. # (Continuous :latest / :sha- agent images still publish on every main push via # docker-publish.yml -- only the versioned release artifacts gate on this run.) +# +# PRERELEASE mode (dispatched from a NON-main branch): instead of bumping and +# pushing main, stamp a throwaway `rc` version, then build and +# publish the wheel to public PyPI + a `:` GHCR image. It does NOT move +# `:latest`, tag, commit, or push to main. This lets a branch be dry-run on the +# ADO nightly infra before merge (pinned via the pipeline's `coderEvalVersion`). +# The `bump` input is ignored off main. On `main` the behavior is unchanged. on: workflow_dispatch: @@ -47,9 +54,9 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 outputs: - # Exposed so the downstream publish-pypi job can gate on a real release - # having been cut (empty => no version bumped => skip PyPI publish). - version: ${{ steps.release.outputs.version }} + # Exposed so the downstream publish-pypi job gates on a version having been + # produced (real release on main, or a stamped prerelease on a branch). + version: ${{ steps.ver.outputs.version }} env: # The self-hosted `uipath-ubuntu-latest` runners enforce a minimum # package-age safe-chain check on uv installs; on GitHub-hosted runners @@ -58,8 +65,13 @@ jobs: SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: "openai-codex-cli-bin,openai-codex" steps: + # Only a real release (main) needs the app token: semantic-release pushes the + # bump commit + tag to the ruleset-protected main branch, and only this app + # has the bypass. A prerelease from a branch never commits or pushes, so it + # skips the token and checks out with the default GITHUB_TOKEN. - name: Mint release app token id: app-token + if: github.ref == 'refs/heads/main' uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: app-id: ${{ secrets.RELEASE_APP_ID }} @@ -70,8 +82,9 @@ jobs: with: fetch-depth: 0 # semantic-release needs full history for tags + changelog # Persisted in .git config so semantic-release's push to main is - # authenticated as the app (which bypasses the branch ruleset). - token: ${{ steps.app-token.outputs.token }} + # authenticated as the app (which bypasses the branch ruleset). Falls back + # to the default token for a prerelease (no push, read-only checkout). + token: ${{ steps.app-token.outputs.token || github.token }} - name: Set up Python 3.13 uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -83,11 +96,38 @@ jobs: with: enable-cache: true + # Release mode is chosen by the dispatched ref: main => real release (the + # semantic-release path below); any other branch => prerelease (stamp a + # throwaway rc version, publish, never touch main). Keying off the ref (not a + # new input) keeps this dispatchable from a branch — a new input would have to + # exist on the default branch first to be accepted. + - name: Determine release mode + id: mode + env: + REF: ${{ github.ref }} + RUN_NUMBER: ${{ github.run_number }} + run: | + set -euo pipefail + if [ "$REF" = "refs/heads/main" ]; then + echo "prerelease=false" >> "$GITHUB_OUTPUT" + echo "Mode: RELEASE (main)" + else + # Next patch of the current version, suffixed with the run number so + # repeated dispatches never collide on PyPI. An exact `==` pin installs + # it even though pip/uv skip prereleases by default. + NEXT=$(python3 -c 'import re,tomllib; v=tomllib.load(open("pyproject.toml","rb"))["project"]["version"]; m=re.match(r"(\d+)\.(\d+)\.(\d+)",v); print("{}.{}.{}".format(int(m[1]),int(m[2]),int(m[3])+1))') + PRE="${NEXT}rc${RUN_NUMBER}" + echo "prerelease=true" >> "$GITHUB_OUTPUT" + echo "version=${PRE}" >> "$GITHUB_OUTPUT" + echo "Mode: PRERELEASE ${PRE}" + fi + - name: Install build + release tools run: uv tool install python-semantic-release && uv tool install twine - name: Run semantic-release (bump + tag, no push yet) id: release + if: steps.mode.outputs.prerelease != 'true' env: GH_TOKEN: ${{ steps.app-token.outputs.token }} # Passed via env (not interpolated into the script) per GitHub's @@ -105,8 +145,45 @@ jobs: # A dispatch always cuts a release; report the just-published version. echo "version=$($PSR version --print)" >> "$GITHUB_OUTPUT" + # PRERELEASE: stamp the rc version into the two canonical spots (mirrors + # publish-testpypi.yml), then refresh uv.lock so the Dockerfile's + # `uv export --frozen` accepts the bumped project version. No commit is made — + # the working tree is what `uv build` and the image build below consume. + - name: Stamp prerelease version + if: steps.mode.outputs.prerelease == 'true' + env: + PRE_VERSION: ${{ steps.mode.outputs.version }} + run: | + set -euo pipefail + python3 - <<'PY' + import os, re, pathlib + version = os.environ["PRE_VERSION"] + for path, key in (("pyproject.toml", "version"), ("src/coder_eval/__init__.py", "__version__")): + p = pathlib.Path(path) + new, n = re.subn(rf'(?m)^{key}\s*=\s*".+?"$', f'{key} = "{version}"', p.read_text(), count=1) + if n != 1: + raise SystemExit(f"version pattern did not match {path} (matched {n})") + p.write_text(new) + print(f"Stamped prerelease version: {version}") + PY + uv lock + + # Single source of truth for the steps below: the real release version (main) + # or the stamped prerelease version (branch). + - name: Resolve published version + id: ver + env: + REL: ${{ steps.release.outputs.version }} + PRE: ${{ steps.mode.outputs.version }} + run: | + set -euo pipefail + V="${REL:-$PRE}" + if [ -z "$V" ]; then echo "no version resolved" >&2; exit 1; fi + echo "version=$V" >> "$GITHUB_OUTPUT" + echo "Publishing version: $V" + - name: Regenerate uv.lock and amend release commit - if: steps.release.outputs.version != '' + if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != '' run: | uv lock if ! git diff --quiet uv.lock; then @@ -119,43 +196,64 @@ jobs: fi - name: Push release commit and tags - if: steps.release.outputs.version != '' + if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != '' run: git push origin main "v${{ steps.release.outputs.version }}" - name: Build wheel + sdist - if: steps.release.outputs.version != '' + if: steps.ver.outputs.version != '' run: uv build # Hand the exact built artifacts to the publish-pypi job. Publishing to # public PyPI runs in its own environment-gated job (OIDC), so it must # consume these files rather than rebuild them. - name: Upload dist for PyPI publish - if: steps.release.outputs.version != '' + if: steps.ver.outputs.version != '' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: release-dist path: dist/ if-no-files-found: error - # Build + push the agent image HERE, in the same job that cut the release, - # so the `:` tag is built from the BUMPED pyproject (the version - # only exists after the semantic-release step above). docker-publish.yml - # runs on the triggering commit, BEFORE the bump, so it can never tag the - # release version -- this is the authoritative versioned image. It also - # repoints `:latest` to the exact release artifact. + # Build + push the agent image HERE, in the same job that produced the + # version, so the `:` tag is built from the correct pyproject (bumped + # by semantic-release on main, or the stamped rc on a prerelease). + # docker-publish.yml runs on the triggering commit, BEFORE a main bump, so it + # can never tag the release version -- this is the authoritative versioned + # image. A real release also repoints `:latest`; a prerelease publishes only + # its `:` tag (see Compute image tags). - name: Lowercase owner for GHCR - if: steps.release.outputs.version != '' + if: steps.ver.outputs.version != '' id: img run: echo "owner_lc=$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_OUTPUT" + # `:` always; `:latest` only for a real release. A prerelease must + # not move `:latest`, which the nightly treats as tip-of-main. + - name: Compute image tags + if: steps.ver.outputs.version != '' + id: tags + env: + OWNER: ${{ steps.img.outputs.owner_lc }} + VERSION: ${{ steps.ver.outputs.version }} + IS_PRERELEASE: ${{ steps.mode.outputs.prerelease }} + run: | + set -euo pipefail + { + echo "tags<> "$GITHUB_OUTPUT" + - name: Set up Docker Buildx continue-on-error: true # GHCR image is internal/best-effort; don't block PyPI - if: steps.release.outputs.version != '' + if: steps.ver.outputs.version != '' uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Log in to GHCR continue-on-error: true # GHCR image is internal/best-effort; don't block PyPI - if: steps.release.outputs.version != '' + if: steps.ver.outputs.version != '' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ghcr.io @@ -164,17 +262,15 @@ jobs: - name: Build and push versioned agent image continue-on-error: true # GHCR image is internal/best-effort; don't block PyPI - if: steps.release.outputs.version != '' + if: steps.ver.outputs.version != '' uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: . file: docker/Dockerfile push: true - tags: | - ghcr.io/${{ steps.img.outputs.owner_lc }}/coder-eval-agent:${{ steps.release.outputs.version }} - ghcr.io/${{ steps.img.outputs.owner_lc }}/coder-eval-agent:latest + tags: ${{ steps.tags.outputs.tags }} build-args: | - CODER_EVAL_VERSION=${{ steps.release.outputs.version }} + CODER_EVAL_VERSION=${{ steps.ver.outputs.version }} secrets: | "uv_index_username=${{ secrets.UV_INDEX_UIPATH_USERNAME }}" "uv_index_password=${{ secrets.UV_INDEX_UIPATH_PASSWORD }}" From 6b43af543b44966aa4aa7b289fd495ba69d5cbc6 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Tue, 21 Jul 2026 16:13:08 -0700 Subject: [PATCH 7/8] fix(codex): always run full-access; drop the in-process OS sandbox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit coder_eval always owns a run's isolation boundary — a docker container (docker driver) or an ephemeral per-task tempdir it creates and discards (tempdir driver); those are the only two drivers — so Codex's own in-process OS sandbox (Landlock/seatbelt) is always redundant. Worse, it breaks on the paths we rely on: Landlock is unavailable in the container, the bwrap re-exec is denied on constrained CI agents, and Windows has no OS sandbox at all — in each case a read-only/workspace-write run fails its writes/execs silently and scores 0. This is what left the Windows uipath-rpa tasks unable to run `uip` (exit -1) even after the earlier container/tempdir fallback landed. Run full-access unconditionally (approval_mode stays deny_all), matching Claude Code and Antigravity, which run with no in-agent OS sandbox; hard isolation of untrusted actions is the docker driver's job. Removes the now-obsolete sandbox_managed plumbing and the per-mode sandbox mapping. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/coder_eval/agent.py | 5 - src/coder_eval/agents/antigravity_agent.py | 5 - src/coder_eval/agents/claude_code_agent.py | 4 - src/coder_eval/agents/codex_agent.py | 60 +++------ src/coder_eval/agents/noop_agent.py | 1 - src/coder_eval/orchestrator.py | 9 -- tests/fixtures/mock_agent.py | 2 - tests/fixtures/text_stub_agent.py | 1 - tests/test_codex_agent.py | 136 +++++---------------- tests/test_orchestrator.py | 5 +- tests/test_simulation_integration.py | 1 - tests/test_user_simulator.py | 2 - 12 files changed, 47 insertions(+), 184 deletions(-) diff --git a/src/coder_eval/agent.py b/src/coder_eval/agent.py index 4af84cc..2b244ce 100644 --- a/src/coder_eval/agent.py +++ b/src/coder_eval/agent.py @@ -167,7 +167,6 @@ async def start( *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None, - sandbox_managed: bool = False, ) -> None: """Initialize and start the agent. @@ -181,10 +180,6 @@ async def start( instead of walking up from CWD. An external ``PLUGIN_TOOLS_DIR`` in the process environment still wins. Implementations that don't shell out may ignore this argument. - sandbox_managed: True when coder_eval itself provides this run's isolation - boundary (the ephemeral host tempdir it created for the task). Agents - that run their own in-process OS sandbox may use this to skip it as - redundant; agents without one may ignore this argument. """ pass diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index 97df31b..8506154 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -290,14 +290,9 @@ async def start( *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None, - sandbox_managed: bool = False, ) -> None: """Initialize and start the Antigravity agent's local harness session. - (``sandbox_managed`` is accepted for interface parity with the base - ``Agent.start`` and ignored: the Antigravity harness has no in-process - OS sandbox to drop.) - Args: working_directory: Path to the sandbox working directory. The primary ``workspace`` so file writes (and run_command) operate there — diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index 998ce16..cd24451 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -692,7 +692,6 @@ async def start( *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None, - sandbox_managed: bool = False, ) -> None: """Initialize and start the Claude Code agent. @@ -702,9 +701,6 @@ async def start( subprocess (typically the resolved ``SandboxConfig.mock_path_dirs``). plugin_tools_dir: Canonical ``node_modules/@uipath`` to export as ``PLUGIN_TOOLS_DIR``. An external env-var pin still wins. - sandbox_managed: Accepted for interface parity with the base - ``Agent.start`` and ignored — Claude Code runs the tempdir path - with no in-agent OS sandbox to drop. """ self.working_directory = Path(working_directory) self._env_path_prepend = list(env_path_prepend or []) diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index b190cbe..c29bc88 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -67,14 +67,6 @@ "Glob": "shell", } -# Permission mode → sandbox mode mapping -_PERMISSION_MODE_TO_SANDBOX: dict[str, str] = { - "bypassPermissions": "full-access", - "acceptEdits": "workspace-write", - "default": "workspace-write", - "plan": "read-only", -} - # Approval mode — the SAME for every permission mode (no per-mode mapping). # # The Codex SDK exposes exactly two approval modes: @@ -90,9 +82,9 @@ # escalations BEYOND the sandbox are refused. # # An eval harness never wants a reviewer that can flake, so EVERY permission mode -# uses deny_all. The trust boundary is the sandbox (_PERMISSION_MODE_TO_SANDBOX), -# which DOES vary by mode: `plan` stays read-only, `bypassPermissions` is -# full-access (intended for isolated Docker runs). +# uses deny_all. The trust boundary is coder_eval's per-run sandbox — a docker +# container or an ephemeral tempdir — NOT Codex's in-process OS sandbox, which +# _build_thread_options always drops to full-access (rationale there). _CODEX_APPROVAL_MODE = "deny_all" # Provider id registered in thread config when CODEX_BASE_URL routes to a @@ -663,11 +655,6 @@ def __init__( self.working_directory: Path | None = None self._env_path_prepend: list[str] = [] self._login_shell_home: Path | None = None - # True when coder_eval itself owns the isolation boundary for this run - # (the ephemeral host tempdir it created). Set by start(); combined with - # the docker-path CODER_EVAL_IN_CONTAINER marker in _build_thread_options - # to drop Codex's redundant in-process OS sandbox (rationale there). - self._sandbox_managed: bool = False # _state / _iteration / _iteration_was_incremented / pending_turn lifecycle # bookkeeping lives on the Agent base class (shared defaults + helpers). self._log = PrefixedAdapter(logger, {"prefix": instance_name}) @@ -683,7 +670,6 @@ async def start( *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None, - sandbox_managed: bool = False, ) -> None: """Initialize and start the Codex agent. @@ -694,14 +680,9 @@ async def start( ``SandboxConfig.mock_path_dirs``), so mock CLIs shadow the real ones — same semantics as the Claude agent. plugin_tools_dir: Optional plugin tools directory (for skills setup) - sandbox_managed: True when coder_eval itself provides the isolation - boundary for this run (the ephemeral host tempdir it created). - Lets ``_build_thread_options`` drop Codex's redundant in-process - OS sandbox, which also can't initialize on some managed CI hosts. """ self.working_directory = Path(working_directory) self._env_path_prepend = list(env_path_prepend or []) - self._sandbox_managed = sandbox_managed self._setup_login_shell_home() self._state = AgentState.WORKING @@ -1269,31 +1250,22 @@ def _build_thread_options(self) -> dict[str, Any]: options["model"] = effective_model self._log.debug(f"Codex model pinned to {effective_model}") - # Map permission_mode to sandbox and approval_mode permission_mode = self.config.permission_mode.value - sandbox_mode_str = _PERMISSION_MODE_TO_SANDBOX.get(permission_mode, "workspace-write") approval_mode_str = _CODEX_APPROVAL_MODE - # Codex's read-only / workspace-write sandboxes are enforced by an OS-level - # helper (Landlock, or a bubblewrap re-exec where Landlock is unavailable). - # That helper can outright fail to start on managed hosts — inside the eval - # container (Landlock unavailable) and on constrained CI agents where the - # bwrap re-exec is denied ("bwrap: execvp .../codex: Permission denied" when - # unprivileged user namespaces or exec mounts are restricted). When it fails, - # writes/execs fail silently and the agent produces no artifacts (FAILURE, - # score 0, no loud error). Whenever coder_eval already owns the isolation - # boundary — a docker container (CODER_EVAL_IN_CONTAINER) OR the ephemeral - # host tempdir it created (sandbox_managed) — this in-process sandbox is - # redundant, so fall back to full-access. This also matches every other - # harness (e.g. Claude Code), which runs the tempdir path with no in-agent - # OS sandbox; hard isolation of untrusted actions is the docker driver's job. - if sandbox_mode_str != "full-access" and (os.getenv("CODER_EVAL_IN_CONTAINER") or self._sandbox_managed): - boundary = "eval container" if os.getenv("CODER_EVAL_IN_CONTAINER") else "coder_eval-managed host tempdir" - self._log.debug( - f"Codex sandbox '{sandbox_mode_str}' → 'full-access': the {boundary} is the isolation " - + "boundary (the in-process OS sandbox is redundant here and can fail to init on CI hosts)." - ) - sandbox_mode_str = "full-access" + # Codex always runs full-access. coder_eval owns this run's isolation + # boundary either way — a docker container (docker driver) or an ephemeral + # per-task tempdir it creates and discards (tempdir driver); those are the + # only two drivers — so Codex's own in-process OS sandbox (Landlock/seatbelt) + # is always redundant. Worse, it actively breaks on the paths we rely on: + # inside the container Landlock is unavailable, on constrained CI agents the + # bwrap re-exec is denied ("bwrap: execvp .../codex: Permission denied"), and + # on Windows there is no OS sandbox at all — in each case a read-only / + # workspace-write run fails its writes/execs silently and scores 0 with no + # loud error. Dropping to full-access matches Claude Code and Antigravity, + # which run with no in-agent OS sandbox; hard isolation of untrusted actions + # is the docker driver's job, and approval_mode stays deny_all regardless. + sandbox_mode_str = "full-access" # Convert to Codex SDK enum values # Sandbox enum values use hyphens, ApprovalMode uses underscores diff --git a/src/coder_eval/agents/noop_agent.py b/src/coder_eval/agents/noop_agent.py index fcd7406..e39c45d 100644 --- a/src/coder_eval/agents/noop_agent.py +++ b/src/coder_eval/agents/noop_agent.py @@ -54,7 +54,6 @@ async def start( *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None, - sandbox_managed: bool = False, ) -> None: """No-op: there is no agent process to launch.""" self._state = AgentState.WORKING diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 87c25f7..b158147 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -970,21 +970,12 @@ async def _setup_sandbox() -> Any: env_path_prepend = [str(p) for p in self.sandbox.resolved_mock_path_dirs] plugin_tools_dir = self.sandbox.plugin_tools_dir - # On the tempdir driver coder_eval itself owns the isolation boundary (the - # ephemeral per-task tempdir), so an agent's own in-process OS sandbox is - # redundant — and on constrained CI hosts it can fail to initialize. Tell - # the agent so it can skip it. The docker driver signals the equivalent - # from inside the container via CODER_EVAL_IN_CONTAINER. - driver = self.task.sandbox.driver if self.task.sandbox else "tempdir" - sandbox_managed = driver == "tempdir" - async def _start_agent() -> None: assert self.agent is not None await self.agent.start( str(sandbox_dir), env_path_prepend=env_path_prepend, plugin_tools_dir=plugin_tools_dir, - sandbox_managed=sandbox_managed, ) await execute_with_retry( diff --git a/tests/fixtures/mock_agent.py b/tests/fixtures/mock_agent.py index 2065615..14ffbf0 100644 --- a/tests/fixtures/mock_agent.py +++ b/tests/fixtures/mock_agent.py @@ -43,7 +43,6 @@ async def start( *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None, - sandbox_managed: bool = False, ) -> None: """Initialize mock agent with working directory. @@ -51,7 +50,6 @@ async def start( working_directory: Path to the sandbox working directory env_path_prepend: Ignored. Accepted only to match the Agent ABC signature. plugin_tools_dir: Ignored. Accepted only to match the Agent ABC signature. - sandbox_managed: Ignored. Accepted only to match the Agent ABC signature. """ self.working_directory = Path(working_directory) self.state = AgentState.WORKING diff --git a/tests/fixtures/text_stub_agent.py b/tests/fixtures/text_stub_agent.py index bd2024e..e873fde 100644 --- a/tests/fixtures/text_stub_agent.py +++ b/tests/fixtures/text_stub_agent.py @@ -31,7 +31,6 @@ async def start( *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None, - sandbox_managed: bool = False, ) -> None: self.working_directory = Path(working_directory) self._state = AgentState.WORKING diff --git a/tests/test_codex_agent.py b/tests/test_codex_agent.py index 7abc573..95f93c2 100644 --- a/tests/test_codex_agent.py +++ b/tests/test_codex_agent.py @@ -15,7 +15,6 @@ from coder_eval.agent import AgentState from coder_eval.agents.codex_agent import ( _CODEX_APPROVAL_MODE, - _PERMISSION_MODE_TO_SANDBOX, CodexAgent, ) from coder_eval.models import AgentConfig, AgentKind, parse_agent_config @@ -78,34 +77,14 @@ def test_codex_agent_with_instance_name(self): # optional `openai_codex` extra. Keep new pure-logic tests there, not here. -class TestPermissionModeMapping: - """Test permission_mode to sandbox/approval_mode mapping.""" - - def test_accept_edits_sandbox_mode(self): - """acceptEdits maps to workspace-write.""" - assert _PERMISSION_MODE_TO_SANDBOX["acceptEdits"] == "workspace-write" - - def test_plan_sandbox_mode(self): - """plan maps to read-only.""" - assert _PERMISSION_MODE_TO_SANDBOX["plan"] == "read-only" - - def test_bypass_permissions_sandbox_mode(self): - """bypassPermissions maps to full-access.""" - assert _PERMISSION_MODE_TO_SANDBOX["bypassPermissions"] == "full-access" - - def test_default_sandbox_mode(self): - """default maps to workspace-write.""" - assert _PERMISSION_MODE_TO_SANDBOX["default"] == "workspace-write" - - def test_all_modes_have_sandbox_mapping(self): - """All permission modes have sandbox_mode mapping.""" - modes = {"default", "acceptEdits", "plan", "bypassPermissions"} - assert set(_PERMISSION_MODE_TO_SANDBOX.keys()) == modes +class TestApprovalMode: + """Approval is a single constant (deny_all) for every permission mode — the + sandbox (always full-access; see TestSandboxAlwaysFullAccess) is the trust + boundary, not a per-command reviewer.""" def test_approval_mode_constant_is_deny_all(self): - """Approval is a single constant (deny_all) — no per-mode mapping; the - sandbox is the trust boundary. Per-mode ApprovalMode.deny_all resolution - is covered behaviorally in TestBuildThreadOptions.""" + """Approval is a single constant (deny_all) — no per-mode mapping. Per-mode + ApprovalMode.deny_all resolution is covered behaviorally below.""" assert _CODEX_APPROVAL_MODE == "deny_all" def test_every_permission_mode_resolves_to_deny_all(self): @@ -117,85 +96,30 @@ def test_every_permission_mode_resolves_to_deny_all(self): assert agent._build_thread_options()["approval_mode"] == ApprovalMode.deny_all -class TestInContainerSandboxFallback: - """Under the docker driver (``CODER_EVAL_IN_CONTAINER`` set), Codex's - Landlock-backed read-only / workspace-write sandboxes can't initialize, so - ``_build_thread_options`` falls back to full-access — the container is the - trust boundary. No-op on the host (marker unset), where Landlock works.""" - - def test_workspace_write_upgraded_to_full_access_in_container(self, monkeypatch): - from openai_codex.api import Sandbox # pyright: ignore[reportPrivateImportUsage] - - monkeypatch.setenv("CODER_EVAL_IN_CONTAINER", "1") - agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, permission_mode="acceptEdits")) - assert agent._build_thread_options()["sandbox"] == Sandbox("full-access") - - def test_read_only_upgraded_to_full_access_in_container(self, monkeypatch): - from openai_codex.api import Sandbox # pyright: ignore[reportPrivateImportUsage] - - monkeypatch.setenv("CODER_EVAL_IN_CONTAINER", "1") - agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, permission_mode="plan")) - assert agent._build_thread_options()["sandbox"] == Sandbox("full-access") - - def test_no_marker_keeps_landlock_sandbox(self, monkeypatch): - from openai_codex.api import Sandbox # pyright: ignore[reportPrivateImportUsage] - - monkeypatch.delenv("CODER_EVAL_IN_CONTAINER", raising=False) - agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, permission_mode="acceptEdits")) - assert agent._build_thread_options()["sandbox"] == Sandbox("workspace-write") - - def test_full_access_unchanged_in_container(self, monkeypatch): - from openai_codex.api import Sandbox # pyright: ignore[reportPrivateImportUsage] +class TestSandboxAlwaysFullAccess: + """coder_eval always owns the isolation boundary (a docker container or an + ephemeral per-task tempdir), so Codex always runs full-access: for every + permission mode, with or without the container marker, and on both Unix and + Windows (where Codex has no OS sandbox at all). Its own in-process OS sandbox + is redundant and, on constrained CI hosts / Windows, actively breaks — silently + scoring 0. Hard isolation of untrusted actions is the docker driver's job.""" - monkeypatch.setenv("CODER_EVAL_IN_CONTAINER", "1") - agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, permission_mode="bypassPermissions")) - assert agent._build_thread_options()["sandbox"] == Sandbox("full-access") - - -class TestSandboxManagedHostFallback: - """On the tempdir driver coder_eval owns the isolation boundary (the ephemeral - per-task tempdir), so ``start(sandbox_managed=True)`` makes ``_build_thread_options`` - drop Codex's redundant in-process OS sandbox to full-access — WITHOUT the docker - container marker. This is the host/CI path where the sandbox's bwrap re-exec is - denied; the fallback is what keeps those tasks from silently producing no artifacts.""" - - def test_workspace_write_upgraded_to_full_access_when_managed(self, monkeypatch): - from openai_codex.api import Sandbox # pyright: ignore[reportPrivateImportUsage] - - monkeypatch.delenv("CODER_EVAL_IN_CONTAINER", raising=False) - agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, permission_mode="acceptEdits")) - agent._sandbox_managed = True - assert agent._build_thread_options()["sandbox"] == Sandbox("full-access") - - def test_read_only_upgraded_to_full_access_when_managed(self, monkeypatch): - from openai_codex.api import Sandbox # pyright: ignore[reportPrivateImportUsage] - - monkeypatch.delenv("CODER_EVAL_IN_CONTAINER", raising=False) - agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, permission_mode="plan")) - agent._sandbox_managed = True - assert agent._build_thread_options()["sandbox"] == Sandbox("full-access") - - def test_not_managed_and_no_container_keeps_landlock_sandbox(self, monkeypatch): - from openai_codex.api import Sandbox # pyright: ignore[reportPrivateImportUsage] - - monkeypatch.delenv("CODER_EVAL_IN_CONTAINER", raising=False) - agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, permission_mode="acceptEdits")) - assert agent._sandbox_managed is False # default off - assert agent._build_thread_options()["sandbox"] == Sandbox("workspace-write") + @pytest.mark.parametrize("mode", ["default", "acceptEdits", "plan", "bypassPermissions"]) + @pytest.mark.parametrize("in_container", [True, False]) + @pytest.mark.parametrize("os_name", ["posix", "nt"]) + def test_sandbox_is_full_access(self, monkeypatch, mode, in_container, os_name): + import os as _os - def test_full_access_unchanged_when_managed(self, monkeypatch): from openai_codex.api import Sandbox # pyright: ignore[reportPrivateImportUsage] - monkeypatch.delenv("CODER_EVAL_IN_CONTAINER", raising=False) - agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, permission_mode="bypassPermissions")) - agent._sandbox_managed = True + if in_container: + monkeypatch.setenv("CODER_EVAL_IN_CONTAINER", "1") + else: + monkeypatch.delenv("CODER_EVAL_IN_CONTAINER", raising=False) + monkeypatch.setattr(_os, "name", os_name) + agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, permission_mode=mode)) assert agent._build_thread_options()["sandbox"] == Sandbox("full-access") - def test_default_sandbox_managed_is_false(self): - """Fresh agent defaults to not-managed; start() opts in per the driver.""" - agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, permission_mode="acceptEdits")) - assert agent._sandbox_managed is False - class TestCodexEnvironmentConfiguration: """Test _build_codex_env: only CODEX_API_KEY travels via env.""" @@ -425,11 +349,11 @@ def test_build_thread_options_with_accept_edits(self): options = agent._build_thread_options() assert options is not None - assert options["sandbox"] == Sandbox.workspace_write + assert options["sandbox"] == Sandbox.full_access assert options["approval_mode"] == ApprovalMode.deny_all def test_build_thread_options_with_plan(self): - """_build_thread_options builds correct options for plan.""" + """_build_thread_options builds correct options for plan (sandbox still full-access).""" from openai_codex.api import ApprovalMode, Sandbox # pyright: ignore[reportPrivateImportUsage] config = parse_agent_config( @@ -441,7 +365,7 @@ def test_build_thread_options_with_plan(self): options = agent._build_thread_options() assert options is not None - assert options["sandbox"] == Sandbox.read_only + assert options["sandbox"] == Sandbox.full_access assert options["approval_mode"] == ApprovalMode.deny_all def test_build_thread_options_with_allowed_tools(self): @@ -473,7 +397,7 @@ def test_build_thread_options_with_disallowed_tools(self): assert options["config"]["disabled_tools"] == ["apply_patch", "apply_patch", "shell"] def test_build_thread_options_with_no_permission_mode(self): - """_build_thread_options defaults to workspace_write/deny_all when no permission_mode set.""" + """_build_thread_options is full-access/deny_all even without a permission_mode.""" from openai_codex.api import ApprovalMode, Sandbox # pyright: ignore[reportPrivateImportUsage] config = parse_agent_config(type=AgentKind.CODEX) @@ -482,7 +406,7 @@ def test_build_thread_options_with_no_permission_mode(self): options = agent._build_thread_options() # Should have defaults even without explicit permission_mode - assert options["sandbox"] == Sandbox.workspace_write + assert options["sandbox"] == Sandbox.full_access assert options["approval_mode"] == ApprovalMode.deny_all def test_build_thread_options_with_permission_and_tools(self): @@ -499,7 +423,7 @@ def test_build_thread_options_with_permission_and_tools(self): options = agent._build_thread_options() assert options is not None - assert options["sandbox"] == Sandbox.read_only + assert options["sandbox"] == Sandbox.full_access assert options["approval_mode"] == ApprovalMode.deny_all assert options["config"]["enabled_tools"] == ["shell", "shell"] diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index c535e80..6b3f25e 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -561,7 +561,6 @@ async def start( *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None, - sandbox_managed: bool = False, ) -> None: self.working_directory = working_directory @@ -626,9 +625,7 @@ async def test_direct_write_warns_on_non_empty_target(tmp_path, monkeypatch, cap from coder_eval.sandbox import Sandbox class DummyAgent: - async def start( - self, working_directory, *, env_path_prepend=None, plugin_tools_dir=None, sandbox_managed=False - ): + async def start(self, working_directory, *, env_path_prepend=None, plugin_tools_dir=None): self.working_directory = working_directory def get_sdk_options(self): diff --git a/tests/test_simulation_integration.py b/tests/test_simulation_integration.py index 35d46c2..98a6820 100644 --- a/tests/test_simulation_integration.py +++ b/tests/test_simulation_integration.py @@ -115,7 +115,6 @@ async def start( *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None, - sandbox_managed: bool = False, ) -> None: pass diff --git a/tests/test_user_simulator.py b/tests/test_user_simulator.py index ac13dec..3adb964 100644 --- a/tests/test_user_simulator.py +++ b/tests/test_user_simulator.py @@ -267,7 +267,6 @@ async def start( *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None, - sandbox_managed: bool = False, ) -> None: self.captured_dir = Path(working_directory) raise RuntimeError("boom") @@ -304,7 +303,6 @@ async def start( *, env_path_prepend: list[str] | None = None, plugin_tools_dir: str | None = None, - sandbox_managed: bool = False, ) -> None: self.captured_dir = Path(working_directory) raise asyncio.CancelledError() From 8c0d5e09dfef8c800feb7f377053e1c59cb6fbbd Mon Sep 17 00:00:00 2001 From: Bai Li Date: Tue, 21 Jul 2026 18:14:36 -0700 Subject: [PATCH 8/8] refactor(codex): drop dead sandbox branch + honest full-access messaging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses review on PR #33 (always-full-access): - Collapse the sandbox assignment to `Sandbox.full_access` and delete the now-unreachable `workspace-write` network-opener branch (dead since the mode became a constant). - `_log_config_enforcement` now emits the [SECURITY] full-access notice for EVERY permission_mode, not just bypassPermissions — all modes resolve to full-access, so the old per-mode warning misled operators. - Document that permission_mode does not confine Codex and that the docker driver is the OS-level write boundary (code comment + TASK_DEFINITION_GUIDE). - Note the test matrix's os_name/in_container dims are invariance-by-construction. - Disclose in the release.yml `bump` input that it is ignored off-main. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/release.yml | 2 +- docs/TASK_DEFINITION_GUIDE.md | 2 ++ src/coder_eval/agents/codex_agent.py | 38 +++++++++++++++------------- tests/test_codex_agent.py | 4 +++ 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ee21141..ff2a7eb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,7 +25,7 @@ on: workflow_dispatch: inputs: bump: - description: 'Version bump to release' + description: 'Version bump to release (ignored on non-main / prerelease dispatch, which always stamps a next-patch rc)' type: choice default: patch options: diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index e86c3c0..3de0c19 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -130,6 +130,8 @@ an error. - `plan` — Agent proposes changes, waits for approval - `bypassPermissions` — No permission checks (use with caution) +> **Codex note:** `permission_mode` confines the **`claude-code`** agent only. The **`codex`** agent always runs full-access regardless of the mode — its in-process OS sandbox is redundant given coder_eval's docker/tempdir isolation and unusable on our CI hosts (and on Windows). Run adversarial or untrusted Codex evals under the **docker driver**, which is the OS-level write boundary; the tempdir/host driver is a working directory, not a confinement boundary. + **Agent Types:** - `claude-code` (default) — Claude Code SDK agent. Supports `sdk_options`, `claude_settings`, and all permission modes. - `codex` — OpenAI Codex agent (requires `[codex]` extra; set `CODEX_API_KEY` and optional `CODEX_BASE_URL` environment variables). diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index c29bc88..43e63d6 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -1265,11 +1265,15 @@ def _build_thread_options(self) -> dict[str, Any]: # loud error. Dropping to full-access matches Claude Code and Antigravity, # which run with no in-agent OS sandbox; hard isolation of untrusted actions # is the docker driver's job, and approval_mode stays deny_all regardless. - sandbox_mode_str = "full-access" - - # Convert to Codex SDK enum values - # Sandbox enum values use hyphens, ApprovalMode uses underscores - options["sandbox"] = Sandbox(sandbox_mode_str) + # + # Consequence: permission_mode does NOT confine Codex — every mode resolves + # to full-access. The docker driver is the only OS-level write boundary here; + # the tempdir/host driver is a working directory, not a confinement boundary + # (same as Claude Code / Antigravity already run there), so adversarial or + # untrusted evals belong on the docker driver. _log_config_enforcement + # surfaces this. full-access (danger-full-access) also keeps network on, so + # tool installs (the UiPath CLI, npm/pip) work without extra sandbox config. + options["sandbox"] = Sandbox.full_access options["approval_mode"] = ApprovalMode(approval_mode_str) # For logging, use the enum names (which use underscores) @@ -1281,14 +1285,6 @@ def _build_thread_options(self) -> dict[str, Any]: # Build config dict for tool enforcement tool_config: dict[str, Any] = {} - # Codex's workspace-write sandbox disables network by default, so any - # tool the agent needs to install (npm/pip/etc.) fails with - # "fetch failed". Always open network in workspace-write — every task - # we exercise needs the UiPath CLI / package installs. Read-only and - # danger-full-access keep their built-in network defaults. - if sandbox_mode_str == "workspace-write": - tool_config["sandbox_workspace_write"] = {"network_access": True} - if self.config.allowed_tools: enabled_tools = [_CLAUDE_TO_CODEX_TOOL_MAP.get(tool, tool) for tool in self.config.allowed_tools] tool_config["enabled_tools"] = enabled_tools @@ -1352,11 +1348,17 @@ def _log_config_enforcement(self) -> None: self._log.debug(f"Disallowed tools: {', '.join(self.config.disallowed_tools)}") self._log.debug(f"Permission mode: {self.config.permission_mode.value}") - if self.config.permission_mode.value == "bypassPermissions": - self._log.warning( - "[SECURITY] bypassPermissions grants unrestricted sandbox access (full-access). " - + "Only use in fully isolated environments with untrusted code execution disabled." - ) + # Codex always runs full-access regardless of permission_mode (see + # _build_thread_options — its in-process OS sandbox is redundant given + # coder_eval's docker/tempdir boundary and unusable on our CI hosts). The + # notice fires for EVERY mode, not just bypassPermissions, so operators are + # not misled that plan/acceptEdits/default confine Codex — none of them do. + self._log.warning( + "[SECURITY] Codex runs full-access on every permission_mode " + + f"(configured: {self.config.permission_mode.value}); permission_mode does not confine it. " + + "OS-level isolation of untrusted code is the docker driver's job — use it for " + + "adversarial or untrusted evals; the tempdir/host driver is not a confinement boundary." + ) def _format_turn_result(self, turn_result: Any) -> str: """Format a Codex Turn to a readable string — fallback when no text streamed. diff --git a/tests/test_codex_agent.py b/tests/test_codex_agent.py index 95f93c2..a0e4f7b 100644 --- a/tests/test_codex_agent.py +++ b/tests/test_codex_agent.py @@ -104,6 +104,10 @@ class TestSandboxAlwaysFullAccess: is redundant and, on constrained CI hosts / Windows, actively breaks — silently scoring 0. Hard isolation of untrusted actions is the docker driver's job.""" + # The in_container / os_name dimensions are invariance-by-construction guards: + # _build_thread_options reads neither CODER_EVAL_IN_CONTAINER nor os.name after + # the always-full-access change, so these axes assert that a future edit can't + # reintroduce an env/OS-conditional sandbox branch without turning a case red. @pytest.mark.parametrize("mode", ["default", "acceptEdits", "plan", "bypassPermissions"]) @pytest.mark.parametrize("in_container", [True, False]) @pytest.mark.parametrize("os_name", ["posix", "nt"])