diff --git a/README.md b/README.md index a5e91206..3d9ba3c9 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,9 @@ guardrails, memory, tool guidance, subagents, skills, scripts, and plugins**. They control context construction, the tool interface, delegation, and behavior inside the tool loop. +Deployers can supply a [pinned policy](docs/PINNED_POLICY.md) outside the editable +tree; runners inject and verify it at the final execution input. +

Eight harness components grouped by runtime role: system prompt, guardrails, memory, and skills construct context; tool guidance and scripts define the tool interface; subagents orchestrate delegated work; plugins control middleware. diff --git a/dependencies/ASSETS.json b/dependencies/ASSETS.json index 64c838fd..fda7f455 100644 --- a/dependencies/ASSETS.json +++ b/dependencies/ASSETS.json @@ -1,6 +1,6 @@ { "third_party/jb_runner/LICENSE": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4", - "third_party/jb_runner/PROVENANCE.md": "7e16c8f5e43aa272baa5cf39bf245b42cf1f40f885457ac4123c91b7de0b06ef", + "third_party/jb_runner/PROVENANCE.md": "d05d13291dd3bf1ae774573a0e18b55a6eebbd2ec09e95f7339306e0c1b487a8", "third_party/jb_runner/codex_base_harness/AGENTS.md": "b5df5264478dfce7d6af52f8443a1797b17967d1bc4a8c4f8c5fe43933516a89", "third_party/jb_runner/harness_baseline_oc/LongTermMEMORY.md": "c78b55245ef1433c017bc3f6f796094e16c640b98543836f6d930972243d28d2", "third_party/jb_runner/harness_baseline_oc/guardrails.md": "ebd0b0f287bcc9bea5efa063d632677b3c0fdf014c7b67bec088c11253605a53", @@ -10,15 +10,15 @@ "third_party/jb_runner/harness_baseline_oc/sub_agents/README.md": "56c5507e02c30c0d611efe2d7426141e9fc626a91599738afc7e003a07d3adb0", "third_party/jb_runner/harness_baseline_oc/systemprompt.md": "f5fa6862d87e4c51b78a4ce4446dace9438b3d201e0350c91ca02d0c83f552e3", "third_party/jb_runner/harness_baseline_oc/tool_descriptions/tool_guidance.md": "5ae4f052905a31e9573abe4971172bf40b4e1f5d265d1be072ed444232371f41", - "third_party/jb_runner/run_benchmark_codex_cli_adapt.sh": "a94c645d3061da354591dde446d11f8b3e6d08b4005f8f587668aae07c0fc100", + "third_party/jb_runner/run_benchmark_codex_cli_adapt.sh": "06c9e8e1c410094d578b2a3ad59f81db8f0a904076acef1f3a11dd531a512ce3", "third_party/lab_runner/LICENSE": "f92627d2ebe80fc0add3b171b2d7eee5e28a98dd0d0a4a5ee5829314243bb3b9", - "third_party/lab_runner/PROVENANCE.md": "b5b42319000e1522765d2520dffcd9d1058c5472cc49046ec1906ffffa66c53c", - "third_party/lab_runner/adapt_runs_cx/run_codex_lab.py": "567d6a06d8104f853b8d83953af333ea8ef57a9053e0c9788a42692c48196556", + "third_party/lab_runner/PROVENANCE.md": "f3ba3923c0e0cc7f8ef9423e645517b898c201d7cd0b8a468bfbaed161093337", + "third_party/lab_runner/adapt_runs_cx/run_codex_lab.py": "3372a02b834f5a4cbd6063144dddc42fb46de7d48acd6c651f361deea89e1598", "third_party/lab_runner/opencode/README.md": "20f2739eada1a76560c412d39603e040c420d3d36a2bf67a99a814500306f0ae", "third_party/lab_runner/opencode/__init__.py": "df2b298f529bbcba94713aa748b5a04569f38bbb3f96c9219b7fb0d7186066a8", - "third_party/lab_runner/opencode/render.py": "b636a60d039fdfa66d64eb5d5d3c93a99464be2bcbd5b80dc8f91d1f104ea5ee", + "third_party/lab_runner/opencode/render.py": "3aab2f5fa993acec5d79634a2712af25e92040fa4c9cfb05f276851a9cc04dae", "third_party/lab_runner/opencode/render_tree.py": "54d64b461e430905b212b46902512aad38d3585fb5921b7a9982eea0413a4b73", - "third_party/lab_runner/opencode/run_opencode.py": "7e7178583a6acb2e07bffd1fbef78f1b0095975bc2c61f8b0038fcc528090e32", + "third_party/lab_runner/opencode/run_opencode.py": "3fb56d2e8a96094c6603973d5f9d3fcdd3c3c3789b5c9c10da97de78dcd9bec1", "third_party/lab_runner/opencode/scripts/read_doc.py": "6b3c8feb3b90b0b2bbbf4cb491c81f1114b67ada04d2c988fac8cb1af68533b5", "third_party/lab_runner/opencode/session.py": "e804e29a20ac4fd71454c85b9e15432cf303739c7f8d7aacc0df19f732bfcb62", "third_party/lab_runner/tts_lab/__init__.py": "4ae8a9112bc00a182c983a35c8c5db40717007cefbc1b0bed89f04bc384a0106", @@ -32,10 +32,10 @@ "third_party/patches/opencode-compat.diff": "c9a8a0e826d48f3fcfc7d818321d128c28e1f7f7c594954c19453dfca4946dc8", "third_party/patches/workbuddy-bench-runtime.diff": "1d581ee0c23a9a5574ce0f9317d0bda17f6c3a2556898c89b34ccbdbd5b87b3f", "third_party/wb_runner/LICENSE": "b466fc7a63acc5cac4275fc4b0f1e22f332ad188ea600b217076ea3a8e040956", - "third_party/wb_runner/PROVENANCE.md": "9011685c8808fb6ab56d23c823af884e5905caecd00f64323f3d02f22e45524c", + "third_party/wb_runner/PROVENANCE.md": "d7325534e1a57633ad47c7169944586ea4a67d9fa437309f47252370d7ffc403", "third_party/wb_runner/agents/codex_agent.py": "dbdf48c19c51d7a0389caf3f5601c1ff967ef534137d829afe8ca8792db40c9d", - "third_party/wb_runner/agents/codex_harness_agent.py": "fc1d1ecfc2de1debabb0ff84ec7a113b05536873ac76f73806835ff873b2f580", - "third_party/wb_runner/agents/oc_agent.py": "5375a49b9a9e6d6d6212b47156b36dca17934e54728ddeed6218ac3662c67fff", + "third_party/wb_runner/agents/codex_harness_agent.py": "b1a3c0cb1731933f131ac27543785bb1183bdc05f2cb8009f5f7fa9fbc1007aa", + "third_party/wb_runner/agents/oc_agent.py": "7529c047435e096158101f44dade2ae3f6c96121baf1dbd8560a610cd9cb23f2", "third_party/wb_runner/agents/oc_harness_agent.py": "647e31187af1305a5ddc42f7e41740795f825e0c708f63f5d1f29b954625c09c", "third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml": "5dddd5c358eff75ca69f7277ae6ccd58a2e208e19d722b862f04062a3718479e", "third_party/wb_runner/configs/harnesses/codex-harness/versions/0.144.6.yaml": "f6ebdc457786c3869ceb8a11ee0c4a63db53a080961075c15edff2a7640293bf", diff --git a/docs/PINNED_POLICY.md b/docs/PINNED_POLICY.md new file mode 100644 index 00000000..e365bcc2 --- /dev/null +++ b/docs/PINNED_POLICY.md @@ -0,0 +1,69 @@ +# Pinned deployer policy + +Set `BenchSpec.pinned_policy` through the Python adapter API before starting a run: + +```python +from dataclasses import replace +from pathlib import Path + +adapter.spec = replace( + adapter.spec, + pinned_policy=Path("/trusted/config/policy.md").read_text(encoding="utf-8"), +) +``` + +Keep the policy source outside the editable harness directory. This is an API setting; +there is no additional CLI flag. Empty or whitespace-only text disables the feature. +Leading and trailing whitespace is stripped once when constructing the policy block; +internal text is preserved literally, including template syntax and HTML comments. + +## Delivery contract + +1. Materialisation stages the editable tree without inserting the policy into its files + or templates. `Install.extras["pinned_policy"]` carries the trusted policy separately. +2. Each benchmark renders its editable content using its existing rules. +3. The launcher prepends the literal policy block after the final rendering or workspace + installation step, then verifies the execution input before starting the model. + +| Execution path | Final policy delivery | +|---|---| +| JobBench / OpenCode | After template interpolation and comment removal; the copied workspace `AGENTS.md` is checked before each attempt. | +| LAB / OpenCode | After the base or candidate renderer; the final `AGENTS.md` or primary agent prompt is read back. | +| WorkBuddy / OpenCode | After Harbor renders the Jinja instruction template; the resulting string is passed directly to the CLI. | +| Codex | After installing the workspace harness; the final `AGENTS.md` is read back before execution. | + +LAB and WorkBuddy base runs retain their original base rendering and component selection. +They receive the same policy as candidates. A WorkBuddy Codex base run prepends to any +`AGENTS.md` already supplied by the task image instead of replacing its instructions. +The local deterministic demo does not execute an LLM prompt and is not a policy-enforcement +integration; its scheduler can still exercise policy cache invalidation. + +## Limits and caching + +The policy block consumes the editable always-on budget (24,000 characters for OpenCode, +8,000 for Codex). Direct materialisation and final delivery both check the applicable +budget. Codex also checks the 32,768-byte UTF-8 limit. In WorkBuddy's composed instruction, +the task instruction itself is excluded from the always-on character budget. Oversized +content fails before model execution rather than relying on runtime truncation. + +Run markers include the full SHA-256 policy identity. Task summaries record and compare +that identity before resuming, and summaries from different policies cannot merge their +mode results. Adding, changing, or removing a policy invalidates previous execution and +whole-task results. With no policy, legacy execution fingerprints and summary resume +behavior remain compatible. + +## Installing updated runners + +LAB and WorkBuddy use isolated Python environments. Setup copies the standard-library-only +`harness2/core/policy.py` beside their runner overlays so all paths use the same code. +After updating Harness², rerun the relevant setup command (`harness2-setup lab` or +`harness2-setup wb`, with the selected substrate). Configured-policy launches reject +missing or outdated runner overlays and explain which setup command to run. + +For a custom substrate or launcher, carrying the setting in an `Install` is insufficient: +implement the same final-input injection and verification contract. Keep the expected +policy in trusted launcher configuration, not in a candidate-controlled file. + +This protects policy text against editing and delivery-time transformations. It does not +make the model obey that text or prevent a running hook, plugin, or agent from modifying +its environment. Tool restrictions and runtime isolation require separate enforcement. diff --git a/harness2/benchmarks/jb_opencode_runner.py b/harness2/benchmarks/jb_opencode_runner.py index 1b35190d..70b64d33 100644 --- a/harness2/benchmarks/jb_opencode_runner.py +++ b/harness2/benchmarks/jb_opencode_runner.py @@ -18,6 +18,7 @@ import json import os +import runpy import shutil import subprocess import sys @@ -175,6 +176,13 @@ def _attempt( cfg: Settings, workspace: Path, traj_file: Path, log_file: Path, task_name: str, attempt: int ) -> int: """Run OpenCode once; its event stream lands in `traj_file`, everything else in the log.""" + if os.environ.get("HARNESS2_PINNED_POLICY", "").strip(): + runtime = runpy.run_path(os.environ["HARNESS2_POLICY_MODULE"]) + runtime["verify_file"]( + workspace / "AGENTS.md", + os.environ["HARNESS2_PINNED_POLICY"], + max_chars=runtime["OPENCODE_MAX_CHARS"], + ) if cfg.agent_steps and not (workspace / "opencode.json").is_file(): (workspace / "opencode.json").write_text( json.dumps( diff --git a/harness2/benchmarks/jobbench.py b/harness2/benchmarks/jobbench.py index 3d5e1720..a343ef1e 100644 --- a/harness2/benchmarks/jobbench.py +++ b/harness2/benchmarks/jobbench.py @@ -34,6 +34,7 @@ from pathlib import Path from .. import config +from ..core import policy from ..core import runmark as RM from ..core import tree as HT from ..core.spec import BenchPrompts, BenchSpec, resolve_channel_model @@ -454,7 +455,9 @@ def _ensure_mode_subagent(text: str) -> str: return "---\nmode: subagent\ndescription: delegated subagent\n---\n\n" + text -def render_opencode_tree(staged: Path, render_dir: Path, ctx: dict) -> None: +def render_opencode_tree( + staged: Path, render_dir: Path, ctx: dict, *, pinned_policy: str = "" +) -> None: """Materialise the staged tree into OpenCode-loadable artifacts under render_dir: AGENTS.md, .opencode/skills//, .opencode/agent/.md, scripts/, plugin/. Skill directories are copied whole, so skills//scripts/*.py survive.""" @@ -464,7 +467,9 @@ def render_opencode_tree(staged: Path, render_dir: Path, ctx: dict) -> None: (rd / ".opencode" / "agent").mkdir(parents=True) (rd / ".opencode" / "skills").mkdir(parents=True) - body = compose_agent_body(staged, ctx) + body = policy.prepend( + compose_agent_body(staged, ctx), pinned_policy, max_chars=policy.OPENCODE_MAX_CHARS + ) if body.strip(): (rd / "AGENTS.md").write_text(body, encoding="utf-8") @@ -714,6 +719,10 @@ def _ensure_store_task(self, task_id: str) -> Path: raise return dst + @property + def _policy_salt(self) -> str: + return policy.digest(self.spec.pinned_policy) + def execute( self, task_id: str, harness_dir: Path | None, run_id: str, *, timeout: int | None = None ) -> Path: @@ -721,11 +730,13 @@ def execute( label = run_id.replace("/", "__") out = self._out_dir(task_id, label) mark = self._traj_dir(task_id, label) - if RM.is_complete(mark, harness_dir=harness_dir) and self.artifacts_intact(out): + if RM.is_complete( + mark, harness_dir=harness_dir, salt=self._policy_salt + ) and self.artifacts_intact(out): out.mkdir(parents=True, exist_ok=True) return out if out.is_dir() or mark.is_dir(): - reason = RM.why_not(mark, harness_dir=harness_dir) + reason = RM.why_not(mark, harness_dir=harness_dir, salt=self._policy_salt) if reason == "reusable": reason = "output fingerprint missing or changed" print( @@ -752,6 +763,7 @@ def execute( mark, run_id=run_id, harness_dir=harness_dir, + salt=self._policy_salt, extra={"output_fingerprint": self._output_fingerprint(out)}, ) return dirs[0] @@ -786,23 +798,31 @@ def _launch_common(self, req: RunRequest) -> tuple[str, str, int]: def _launch_opencode(self, req: RunRequest, install: Install) -> list[Path]: task_id, label, t = self._launch_common(req) self.substrate.preflight() - render_dir = Path(tempfile.mkdtemp(prefix=f"hrender_{label}_")) - render_opencode_tree(install.staging_dir, render_dir, _render_ctx(self.domain_of(task_id))) - env = { - **os.environ, - "TASK_IDS": task_id, - "LABEL": label, - "HARNESS_RENDER_DIR": str(render_dir), - "AGENT_NAME": "", - "TASKS_BASE_DIR": str(self.store), - "OPENCODE_DIR": str(config.OPENCODE_DIR), - "MODEL_ID": config.SOLVER_MODEL_OC, - "TIMEOUT_PER_TASK": str(t), - "MAX_RETRIES": "2", - "MAX_CONCURRENT": "1", - "OPENCODE_EXPERIMENTAL_OUTPUT_TOKEN_MAX": str(OUTPUT_TOKEN_MAX), - } - self._run_runner(OC_RUNNER, env, wall=2 * t + 600, label=label) + with tempfile.TemporaryDirectory(prefix=f"hrender_{label}_") as temporary: + render_dir = Path(temporary) + render_opencode_tree( + install.staging_dir, + render_dir, + _render_ctx(self.domain_of(task_id)), + pinned_policy=install.extras.get("pinned_policy", ""), + ) + env = { + **os.environ, + "TASK_IDS": task_id, + "LABEL": label, + "HARNESS_RENDER_DIR": str(render_dir), + policy.ENV: install.extras.get("pinned_policy", ""), + "HARNESS2_POLICY_MODULE": str(Path(policy.__file__).resolve()), + "AGENT_NAME": "", + "TASKS_BASE_DIR": str(self.store), + "OPENCODE_DIR": str(config.OPENCODE_DIR), + "MODEL_ID": config.SOLVER_MODEL_OC, + "TIMEOUT_PER_TASK": str(t), + "MAX_RETRIES": "2", + "MAX_CONCURRENT": "1", + "OPENCODE_EXPERIMENTAL_OUTPUT_TOKEN_MAX": str(OUTPUT_TOKEN_MAX), + } + self._run_runner(OC_RUNNER, env, wall=2 * t + 600, label=label) return [self._require_artifacts(task_id, label)] def _launch_codex(self, req: RunRequest, install: Install) -> list[Path]: @@ -819,6 +839,9 @@ def _launch_codex(self, req: RunRequest, install: Install) -> list[Path]: "TASK_IDS": task_id, "LABEL": label, "HARNESS_RENDER_DIR": str(install.staging_dir), + policy.ENV: install.extras.get("pinned_policy", ""), + "HARNESS2_POLICY_MODULE": str(Path(policy.__file__).resolve()), + "HARNESS2_POLICY_PYTHON": sys.executable, "TASKS_BASE_DIR": str(self.store), "BENCHMARK_MODELS": config.SOLVER_MODEL_CX, "MAX_CONCURRENT_PER_MODEL": "1", diff --git a/harness2/benchmarks/lab.py b/harness2/benchmarks/lab.py index 77320b21..2b7adaef 100644 --- a/harness2/benchmarks/lab.py +++ b/harness2/benchmarks/lab.py @@ -32,9 +32,11 @@ from pathlib import Path from .. import config +from ..core import policy from ..core import runmark as RM from ..core import tree as HT from ..core.spec import BenchPrompts, BenchSpec, resolve_channel_model +from ..install import require_policy_runtime from ..substrate.base import Install, RunRequest from ..substrate.codex import CODEX_SURFACE, CodexSubstrate from ..substrate.opencode import OPENCODE_SURFACE, OpencodeSubstrate @@ -589,11 +591,13 @@ def execute( self, task_id: str, harness_dir: Path | None, run_id: str, *, timeout: int | None = None ) -> Path: out = self._run_dir(run_id) - if RM.is_complete(out, harness_dir=harness_dir) and self.artifacts_intact(out): + if RM.is_complete( + out, harness_dir=harness_dir, salt=self._policy_salt + ) and self.artifacts_intact(out): self._check_tool_gate(out) return out if out.is_dir() and any(out.iterdir()): - reason = RM.why_not(out, harness_dir=harness_dir) + reason = RM.why_not(out, harness_dir=harness_dir, salt=self._policy_salt) if reason == "reusable": reason = "output fingerprint missing or changed" print( @@ -619,16 +623,23 @@ def execute( out, run_id=run_id, harness_dir=harness_dir, + salt=self._policy_salt, extra={"output_fingerprint": self._output_fingerprint(out)}, ) return out + @property + def _policy_salt(self) -> str: + return policy.digest(self.spec.pinned_policy) + def _install(self, harness_dir: Path | None) -> Install: """BASE (harness_dir=None) is a value: fingerprint 'base', and the launcher decides what 'base' means on its substrate.""" if harness_dir is None: return Install( - staging_dir=self.base_harness(), fingerprint="base", extras={"harness": "base"} + staging_dir=self.base_harness(), + fingerprint=RM.fingerprint(None, self._policy_salt), + extras={"harness": "base", "pinned_policy": self.spec.pinned_policy}, ) inst = self.substrate.materialise(Path(harness_dir), self.spec, {}) return Install( @@ -638,6 +649,8 @@ def _install(self, harness_dir: Path | None) -> Install: ) def _launch_oc(self, req: RunRequest, install: Install) -> list[Path]: + if policy.block(install.extras.get("pinned_policy", "")): + require_policy_runtime("harvey-labs", "oc") runner = config.lab_repo() / "opencode" / "run_opencode.py" _require_runner(runner) dirs = [] @@ -673,7 +686,7 @@ def _launch_oc(self, req: RunRequest, install: Install) -> list[Path]: run_detached( cmd, cwd=config.lab_repo(), - env=_lab_env(), + env={**_lab_env(), policy.ENV: install.extras.get("pinned_policy", "")}, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, @@ -685,6 +698,8 @@ def _launch_oc(self, req: RunRequest, install: Install) -> list[Path]: return dirs def _launch_cx(self, req: RunRequest, install: Install) -> list[Path]: + if policy.block(install.extras.get("pinned_policy", "")): + require_policy_runtime("harvey-labs", "cx") runner = config.lab_repo() / "external" / "adapt_runs_cx" / "run_codex_lab.py" _require_runner(runner) dirs = [] @@ -709,6 +724,7 @@ def _launch_cx(self, req: RunRequest, install: Install) -> list[Path]: env = { **_lab_env(), + policy.ENV: install.extras.get("pinned_policy", ""), "ADAPT_LABCX_CODEX_BIN": str(Path(config.CODEX_DIST) / "codex"), "ADAPT_LABCX_RESULTS": str(self._results_root), "ADAPT_LABCX_SCRATCH": os.environ.get("ADAPT_LABCX_SCRATCH") diff --git a/harness2/benchmarks/workbuddy.py b/harness2/benchmarks/workbuddy.py index 27a95b5c..00ddd320 100644 --- a/harness2/benchmarks/workbuddy.py +++ b/harness2/benchmarks/workbuddy.py @@ -36,10 +36,12 @@ from pathlib import Path from .. import config +from ..core import policy from ..core import runmark as RM from ..core import tree as HT from ..core.selection import order_graded_first from ..core.spec import BenchPrompts, BenchSpec, resolve_channel_model +from ..install import require_policy_runtime from ..substrate import opencode as OC from ..substrate.base import Install, RunRequest from ..substrate.codex import CODEX_SURFACE, CodexSubstrate @@ -498,6 +500,8 @@ def _job_yaml( ws: Path | None, k: int, jobs_dir_abs: Path, + *, + pinned_policy: str = "", ) -> str: sel = "\n".join(f" - {t}" for t in tasks) if substrate == "oc": @@ -508,6 +512,8 @@ def _job_yaml( composed = ws / "_composed" / "prompt.j2" if composed.is_file(): hp["prompt_template_path"] = str(composed.resolve()) + if policy.block(pinned_policy): + hp[policy.ENV] = pinned_policy hp_lines = "\n".join(f" {key}: {json.dumps(val)}" for key, val in sorted(hp.items())) extra = f"harness_params_override:\n{hp_lines}\n" else: @@ -515,6 +521,8 @@ def _job_yaml( hp = {"CODEX_MODEL": config.SOLVER_MODEL_CX} if ws is not None: hp["WB_HARNESS_WS"] = str(ws.resolve()) + if policy.block(pinned_policy): + hp[policy.ENV] = pinned_policy hp_block = "" if hp: hp_lines = "\n".join(f" {key}: {json.dumps(val)}" for key, val in sorted(hp.items())) @@ -590,6 +598,7 @@ def run_batch( done, need=None, timeout_s: int | None = None, + pinned_policy: str = "", ) -> list[Path]: """Roll `tasks` k times each under harness workspace `ws`, returning the run dirs THIS invocation created. `done(task_id)` is the caller's skip predicate; `need(task_id)` is @@ -613,7 +622,10 @@ def run_batch( yaml_path = config.wb_repo() / "configs" / "jobs" / f"{slug}.yaml" label_dir.mkdir(parents=True, exist_ok=True) yaml_path.write_text( - _job_yaml(subset, substrate, label, todo, ws, k, label_dir), encoding="utf-8" + _job_yaml( + subset, substrate, label, todo, ws, k, label_dir, pinned_policy=pinned_policy + ), + encoding="utf-8", ) timeout_s = timeout_s or int((len(todo) * k / _conc(substrate)) * 3600 * 1.2 + 900) @@ -750,29 +762,37 @@ def _identity_matches(self, trial: Path) -> bool: m = RM.read_mark(Path(trial)) or {} return all(m.get(key) == val for key, val in self._cell_identity().items()) + @property + def _policy_salt(self) -> str: + return policy.digest(self.spec.pinned_policy) + def _reusable(self, trial: Path, harness_dir: Path | None) -> bool: return ( _trial_finished(trial) - and RM.is_complete(trial, harness_dir=harness_dir) + and RM.is_complete(trial, harness_dir=harness_dir, salt=self._policy_salt) and self._identity_matches(trial) ) def _install(self, harness_dir: Path | None) -> Install: if harness_dir is None: return Install( - staging_dir=self.spec.runs_root, fingerprint="base", extras={"install_mode": "none"} + staging_dir=self.spec.runs_root, + fingerprint=RM.fingerprint(None, self._policy_salt), + extras={"install_mode": "none", "pinned_policy": self.spec.pinned_policy}, ) return self.substrate.materialise(Path(harness_dir), self.spec, {}) @staticmethod def _drop_staging(install: Install) -> None: - if install.fingerprint != "base": + if install.extras.get("install_mode") != "none": shutil.rmtree(install.staging_dir, ignore_errors=True) def _launch(self, req: RunRequest, install: Install) -> list[Path]: """The benchmark-supplied launcher body: the finished trials among the run dirs this invocation created, ranked; the adapter trims and marks them.""" - ws = None if install.fingerprint == "base" else install.staging_dir + if policy.block(install.extras.get("pinned_policy", "")): + require_policy_runtime("workbuddy-bench", self.substrate.name) + ws = None if install.extras.get("install_mode") == "none" else install.staging_dir k = len(req.run_ids) label = self._label(req.run_ids[0], k) if self.substrate.name == "cx": @@ -798,6 +818,7 @@ def _have(task_id: str) -> int: done=lambda tid: _have(tid) >= k, need=lambda tid: k - _have(tid), timeout_s=req.timeout or None, + pinned_policy=install.extras.get("pinned_policy", ""), ) fresh = [t for t in _trials_in(new_dirs, req.task_ref, self.subset) if _trial_finished(t)] return self._rank(fresh) @@ -819,7 +840,11 @@ def execute( raise RuntimeError(f"Harbor produced no finished trial for {label} / {task_id}") chosen = fresh[0] RM.mark_complete( - chosen, run_id=run_id, harness_dir=harness_dir, extra=self._cell_identity() + chosen, + run_id=run_id, + harness_dir=harness_dir, + salt=self._policy_salt, + extra=self._cell_identity(), ) return chosen @@ -851,7 +876,11 @@ def _mine() -> list[Path]: for t in fresh: if not RM.read_mark(t): RM.mark_complete( - t, run_id=run_ids[0], harness_dir=harness_dir, extra=self._cell_identity() + t, + run_id=run_ids[0], + harness_dir=harness_dir, + salt=self._policy_salt, + extra=self._cell_identity(), ) mine = _mine() return self._rank(mine)[:k] diff --git a/harness2/core/policy.py b/harness2/core/policy.py new file mode 100644 index 00000000..85fb52c6 --- /dev/null +++ b/harness2/core/policy.py @@ -0,0 +1,118 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Literal deployer policy, verified at the final execution input. + +This module uses only the standard library. Setup copies it beside isolated benchmark +runners so they enforce exactly the same policy without installing harness2 there. +The policy preserves prompt text; it does not enforce model or tool behavior. +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +BEGIN = "PINNED POLICY (fixed by the deployer; do not override)" +END = "END PINNED POLICY" +OPENCODE_MAX_CHARS = 24_000 +CODEX_MAX_CHARS = 8_000 +CODEX_MAX_BYTES = 32_768 +ENV = "HARNESS2_PINNED_POLICY" + + +def block(policy: str) -> str: + body = policy.strip() + return f"{BEGIN}\n{body}\n{END}\n\n" if body else "" + + +def digest(policy: str) -> str: + b = block(policy) + return hashlib.sha256(b.encode("utf-8")).hexdigest() if b else "" + + +def reserved_chars(policy: str) -> int: + return len(block(policy)) + + +def check_size(text: str, *, max_chars: int, max_bytes: int | None = None) -> None: + if len(text) > max_chars: + raise ValueError(f"always-on content is {len(text)} chars, over the {max_chars} cap") + if max_bytes is not None and len(text.encode("utf-8")) > max_bytes: + raise ValueError(f"always-on content exceeds the {max_bytes} UTF-8 byte cap") + + +def verify_prompt(text: str, policy: str, *, max_chars: int, max_bytes: int | None = None) -> None: + """Check the actual launch input, including limits applied by the runtime.""" + b = block(policy) + if not b: + return + if not text.startswith(b): + raise ValueError("pinned policy missing or altered; refusing to run without it") + check_size(text, max_chars=max_chars, max_bytes=max_bytes) + + +def prepend(text: str, policy: str, *, max_chars: int, max_bytes: int | None = None) -> str: + if not block(policy): + return text + result = block(policy) + text + verify_prompt(result, policy, max_chars=max_chars, max_bytes=max_bytes) + return result + + +def inject_file(target: Path, policy: str, *, max_chars: int, max_bytes: int | None = None) -> None: + """Inject only after rendering/installation; read back the bytes handed to the CLI.""" + if not block(policy): + return + existing = target.read_bytes().decode("utf-8") if target.is_file() else "" + result = prepend(existing, policy, max_chars=max_chars, max_bytes=max_bytes) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(result.encode("utf-8")) + verify_file(target, policy, max_chars=max_chars, max_bytes=max_bytes) + + +def verify_file(target: Path, policy: str, *, max_chars: int, max_bytes: int | None = None) -> None: + if not block(policy): + return + text = target.read_bytes().decode("utf-8") if target.is_file() else "" + verify_prompt(text, policy, max_chars=max_chars, max_bytes=max_bytes) + + +def validate_tree(surface, staged: Path, policy: str) -> None: + """Reserve policy space even for base trees and direct materialisation.""" + if not block(policy): + return + text = "\n\n".join( + (staged / rel).read_bytes().decode("utf-8") if (staged / rel).is_file() else "" + for rel in surface.always_on + ) + prepend( + text, + policy, + max_chars=surface.always_on_max, + max_bytes=CODEX_MAX_BYTES if surface.name == "codex" else None, + ) + + +if __name__ == "__main__": + import os + import sys + + # The JobBench shell runner calls this on its installed workspace, before codex. + inject_file( + Path(sys.argv[1]) / "AGENTS.md", + os.environ.get(ENV, ""), + max_chars=CODEX_MAX_CHARS, + max_bytes=CODEX_MAX_BYTES, + ) diff --git a/harness2/core/recursion.py b/harness2/core/recursion.py index dda6b98c..560d4a64 100644 --- a/harness2/core/recursion.py +++ b/harness2/core/recursion.py @@ -27,7 +27,7 @@ from typing import Callable, Sequence from .. import config -from . import context, prompts +from . import context, policy, prompts from . import tree as harness_tree from .ledger import Ledger from .proposer import Proposer, ProposerError @@ -113,7 +113,8 @@ def _accept_tree( label: str, ) -> tuple[bool, str]: """Repair then validate a proposed tree: (kept_base, "") if accepted, else the error.""" - triggered = harness_tree.autotrigger(surface, new) + cap = surface.always_on_max - policy.reserved_chars(adapter.spec.pinned_policy) + triggered = harness_tree.autotrigger(surface, new, always_on_max=cap) if triggered: print(f" [{label}] auto-triggered skill(s): {', '.join(triggered)}", flush=True) changed = harness_tree.diff_summary(surface, fallback, new) @@ -123,6 +124,7 @@ def _accept_tree( surface, new, allow_subagents=adapter.spec.allow_subagents, + always_on_max=cap, reserved_scripts=harness_tree.RESERVED_SCRIPTS | adapter.spec.reserved_scripts, ) adapter.validate_tree(new) diff --git a/harness2/core/runmark.py b/harness2/core/runmark.py index 1de5794b..2192ea77 100644 --- a/harness2/core/runmark.py +++ b/harness2/core/runmark.py @@ -38,16 +38,19 @@ def _ignored(p: Path) -> bool: return "__pycache__" in p.parts or p.suffix == ".pyc" or p.name == ".DS_Store" -def fingerprint(harness_dir: Path | None) -> str: +def fingerprint(harness_dir: Path | None, salt: str = "") -> str: """A stable digest of the harness tree that produced a rollout; `None` means a base - rollout on the inherited harness and digests to the shared value "base". + rollout on the inherited harness and digests to the shared value "base". A non-empty + `salt` (the pinned-policy digest) changes the identity; "" leaves it unchanged. """ if harness_dir is None: - return "base" + return f"base-{salt}" if salt else "base" root = Path(harness_dir) if not root.is_dir(): return "absent" h = hashlib.sha1() + if salt: + h.update(b"policy\0" + salt.encode() + b"\0") for p in sorted(root.rglob("*")): if not p.is_file() or p.name == MARK or _ignored(p): continue @@ -62,7 +65,12 @@ def fingerprint(harness_dir: Path | None) -> str: def mark_complete( - where: Path, *, run_id: str, harness_dir: Path | None = None, extra: dict | None = None + where: Path, + *, + run_id: str, + harness_dir: Path | None = None, + extra: dict | None = None, + salt: str = "", ) -> Path: """Record that `run_id` finished. Call only after the runner returned normally.""" d = Path(where) @@ -71,7 +79,7 @@ def mark_complete( "version": VERSION, "run_id": run_id, "at": time.time(), - "harness": fingerprint(harness_dir), + "harness": fingerprint(harness_dir, salt), **(extra or {}), } tmp = d / f"{MARK}.{os.getpid()}.tmp" @@ -89,7 +97,7 @@ def read_mark(where: Path) -> dict | None: def is_complete( - where: Path, *, harness_dir: Path | None = None, match_harness: bool = True + where: Path, *, harness_dir: Path | None = None, match_harness: bool = True, salt: str = "" ) -> bool: """Whether the artifacts at `where` may be reused: a current marker naming the same harness, unless `match_harness` is off. @@ -99,17 +107,17 @@ def is_complete( return False if not match_harness: return True - return m.get("harness") == fingerprint(harness_dir) + return m.get("harness") == fingerprint(harness_dir, salt) -def why_not(where: Path, *, harness_dir: Path | None = None) -> str: +def why_not(where: Path, *, harness_dir: Path | None = None, salt: str = "") -> str: """A sentence for a log line explaining why `where` is or is not reusable.""" m = read_mark(where) if not m: return f"no {MARK} -- never finished, or finished before completion was recorded" if m.get("version") != VERSION: return f"marker version {m.get('version')} != {VERSION}" - want, got = fingerprint(harness_dir), m.get("harness") + want, got = fingerprint(harness_dir, salt), m.get("harness") if want != got: return f"harness fingerprint {got} != {want} -- produced by a different configuration" return "reusable" diff --git a/harness2/core/schedule.py b/harness2/core/schedule.py index f91f450b..a6838efa 100644 --- a/harness2/core/schedule.py +++ b/harness2/core/schedule.py @@ -33,7 +33,7 @@ from typing import Mapping, Sequence from . import library as library_store -from . import recursion, runmark, selection +from . import policy, recursion, runmark, selection from . import tree as harness_tree from .proposer import Proposer from .spec import BenchmarkAdapter, RunView @@ -242,6 +242,7 @@ class TaskResult: domain: str k: int seed: int | None = None + policy_digest: str = "" status: str = "running" modes: dict = field(default_factory=dict) selections: dict = field(default_factory=dict) @@ -254,6 +255,7 @@ def to_dict(self) -> dict: "domain": self.domain, "k": self.k, "seed": self.seed, + "policy_digest": self.policy_digest, "status": self.status, "base": self.base, "modes": self.modes, @@ -300,6 +302,8 @@ def _write_summary(summary_path: Path, res: TaskResult) -> None: merged = res.to_dict() if summary_path.is_file(): old = _load_old(summary_path) + if old.get("policy_digest", "") != res.policy_digest: + old = {} for key in ("modes", "selections"): prev = _modes_of(old) if key == "modes" else dict(old.get(key) or {}) prev.update(merged.get(key) or {}) @@ -368,6 +372,8 @@ def _resume( flush=True, ) return False + if got.get("policy_digest", "") != policy.digest(adapter.spec.pinned_policy): + return False if got.get("status") != "ok": return False cached_modes = _modes_of(got) @@ -602,7 +608,13 @@ def run_task( bench, sub = adapter.spec.name, adapter.substrate.name domain = adapter.domain_of(task_id) - res = TaskResult(task_id=task_id, domain=domain, k=k, seed=seed) + res = TaskResult( + task_id=task_id, + domain=domain, + k=k, + seed=seed, + policy_digest=policy.digest(adapter.spec.pinned_policy), + ) base_ns = Namespace(bench, sub, "agg", k, seed, tag) work = base_ns.task_dir(runs_root, task_id) work.mkdir(parents=True, exist_ok=True) diff --git a/harness2/core/spec.py b/harness2/core/spec.py index e58d1485..0f4c3ae1 100644 --- a/harness2/core/spec.py +++ b/harness2/core/spec.py @@ -141,6 +141,8 @@ class BenchSpec: domain_word: str = "domain" leak_globs: tuple[str, ...] = () allow_subagents: bool = False + # Literal deployer text, outside editable trees; runners inject it after rendering. + pinned_policy: str = "" reserved_scripts: frozenset[str] = frozenset() proposer_deliverables: str = "contents" diff --git a/harness2/install.py b/harness2/install.py index e126bc42..2c1e2e77 100644 --- a/harness2/install.py +++ b/harness2/install.py @@ -147,6 +147,47 @@ def overlays(name: str, *, dry_run=False): ignore=shutil.ignore_patterns("__pycache__", "*.pyc", "PROVENANCE.md"), ) + policy_dest = { + "harvey-labs": repo / "opencode" / "pinned_policy.py", + "workbuddy-bench": repo / "src" / "workbuddy_bench" / "agents" / "pinned_policy.py", + }.get(name) + if policy_dest is not None: + source = Path(__file__).parent / "core" / "policy.py" + print(f"Copy policy runtime: {source} -> {policy_dest}", flush=True) + if not dry_run: + policy_dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, policy_dest) + + +def require_policy_runtime(name: str, substrate: str) -> None: + """Refuse configured policies when an isolated checkout still has old runners.""" + repo = config.DEPENDENCIES / name + if name == "harvey-labs": + pairs = [(Path(__file__).parent / "core/policy.py", repo / "opencode/pinned_policy.py")] + files = ( + ("opencode/render.py", "opencode/run_opencode.py") + if substrate == "oc" + else ("adapt_runs_cx/run_codex_lab.py",) + ) + for rel in files: + dest = repo / rel if rel.startswith("opencode/") else repo / "external" / rel + pairs.append((config.THIRD_PARTY / "lab_runner" / rel, dest)) + setup = "lab" + else: + agents = repo / "src/workbuddy_bench/agents" + filename = "oc_agent.py" if substrate == "oc" else "codex_harness_agent.py" + pairs = [ + (Path(__file__).parent / "core/policy.py", agents / "pinned_policy.py"), + (config.THIRD_PARTY / "wb_runner/agents" / filename, agents / filename), + ] + setup = "wb" + for source, dest in pairs: + if not dest.is_file() or source.read_bytes() != dest.read_bytes(): + raise RuntimeError( + f"pinned-policy runtime is missing or outdated: {dest}; " + f"rerun harness2-setup {setup} --substrate {substrate}" + ) + def workbuddy_models(*, dry_run=False): """Generate the two Vertex routes from the user's model IDs, never credentials.""" diff --git a/harness2/substrate/base.py b/harness2/substrate/base.py index 19ef1dac..b050cf5a 100644 --- a/harness2/substrate/base.py +++ b/harness2/substrate/base.py @@ -25,7 +25,11 @@ @dataclass(frozen=True) class Install: - """A materialised harness ready for a launcher; `fingerprint` digests the SOURCE tree.""" + """A staged editable harness; fingerprint identifies source plus pinned policy. + + extras["pinned_policy"] is trusted deployer text. Launchers must prepend it after + their final rendering/installation step and verify the actual execution input. + """ staging_dir: Path fingerprint: str diff --git a/harness2/substrate/codex.py b/harness2/substrate/codex.py index fd34eacd..870a4139 100644 --- a/harness2/substrate/codex.py +++ b/harness2/substrate/codex.py @@ -31,6 +31,7 @@ from typing import Mapping from .. import config +from ..core import policy as POLICY from ..core import runmark as RM from ..core import tree as HT from ..core.tree import SurfaceSpec @@ -39,7 +40,7 @@ CODEX_SURFACE = SurfaceSpec( name="codex", always_on=("AGENTS.md",), - always_on_max=8_000, + always_on_max=POLICY.CODEX_MAX_CHARS, always_on_cap_reason=( "AGENTS.md is injected whole into EVERY turn, and codex additionally " "truncates it silently at 32,768 bytes" @@ -129,20 +130,26 @@ def materialise(self, tree_dir: Path, spec, ctx: Mapping[str, str]) -> Install: if not HT.is_tree(CODEX_SURFACE, tree_dir): raise ValueError(f"{tree_dir} is not a codex harness tree (no AGENTS.md)") staged = Path(tempfile.mkdtemp(prefix="adapt-cx-root-")) - for entry in CODEX_ROOT_ENTRIES: - src = tree_dir / entry - if not src.exists(): - continue - if src.is_dir(): - shutil.copytree(src, staged / entry) - else: - shutil.copy2(src, staged / entry) + try: + for entry in CODEX_ROOT_ENTRIES: + src = tree_dir / entry + if not src.exists(): + continue + if src.is_dir(): + shutil.copytree(src, staged / entry) + else: + shutil.copy2(src, staged / entry) + POLICY.validate_tree(CODEX_SURFACE, staged, spec.pinned_policy) + except BaseException: + shutil.rmtree(staged, ignore_errors=True) + raise return Install( staging_dir=staged, - fingerprint=RM.fingerprint(tree_dir), + fingerprint=RM.fingerprint(tree_dir, POLICY.digest(spec.pinned_policy)), extras={ "install_mode": "codex_root", **{k: v for k, v in ctx.items() if not k.startswith("file:")}, + "pinned_policy": spec.pinned_policy, }, ) diff --git a/harness2/substrate/opencode.py b/harness2/substrate/opencode.py index ad0a2ea4..3547823d 100644 --- a/harness2/substrate/opencode.py +++ b/harness2/substrate/opencode.py @@ -34,6 +34,7 @@ from typing import Mapping from .. import config +from ..core import policy as POLICY from ..core import runmark as RM from ..core import tree as HT from ..core.tree import SurfaceSpec @@ -47,7 +48,7 @@ "LongTermMEMORY.md", "tool_descriptions/tool_guidance.md", ), - always_on_max=24_000, + always_on_max=POLICY.OPENCODE_MAX_CHARS, always_on_cap_reason="every token here competes with the agent's own output budget", skill_dir="skills", skill_name_rule="must_match", @@ -196,7 +197,6 @@ def materialise(self, tree_dir: Path, spec, ctx: Mapping[str, str]) -> Install: staged = Path(tempfile.mkdtemp(prefix=f"adapt-oc-{mode}-")) try: shutil.copytree(tree_dir, staged, dirs_exist_ok=True) - for key, body in ctx.items(): if key.startswith("file:"): rel = HT.normalize_rel(key[len("file:") :]) @@ -204,6 +204,7 @@ def materialise(self, tree_dir: Path, spec, ctx: Mapping[str, str]) -> Install: p.parent.mkdir(parents=True, exist_ok=True) p.write_text(body, encoding="utf-8") + POLICY.validate_tree(OPENCODE_SURFACE, staged, spec.pinned_policy) if mode == "home_dir_jinja": composed = compose_prompt(staged) @@ -212,6 +213,11 @@ def materialise(self, tree_dir: Path, spec, ctx: Mapping[str, str]) -> Install: shutil.move(str(tg), staged / "tool_guidance.md") shutil.rmtree(staged / "tool_descriptions", ignore_errors=True) assert_jinja_safe(composed) + POLICY.prepend( + composed.removesuffix(_JINJA_PLACEHOLDER), + spec.pinned_policy, + max_chars=OPENCODE_SURFACE.always_on_max, + ) out = staged / "_composed" / "prompt.j2" out.parent.mkdir(parents=True, exist_ok=True) out.write_text(composed, encoding="utf-8") @@ -220,7 +226,9 @@ def materialise(self, tree_dir: Path, spec, ctx: Mapping[str, str]) -> Install: raise return Install( - staging_dir=staged, fingerprint=RM.fingerprint(tree_dir), extras={"install_mode": mode} + staging_dir=staged, + fingerprint=RM.fingerprint(tree_dir, POLICY.digest(spec.pinned_policy)), + extras={"install_mode": mode, "pinned_policy": spec.pinned_policy}, ) def run_argv( diff --git a/tests/test_pinned_policy.py b/tests/test_pinned_policy.py new file mode 100644 index 00000000..0f206746 --- /dev/null +++ b/tests/test_pinned_policy.py @@ -0,0 +1,119 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A deployer's pinned policy is staged, verified, fingerprinted, and charged to the cap.""" + +import dataclasses +import shutil +import tempfile +import unittest +from pathlib import Path + +from harness2.core import policy, recursion, runmark, tree +from harness2.demo import DemoAdapter +from harness2.substrate.codex import CODEX_SURFACE, CodexSubstrate + +POLICY = "Never delete files outside the workspace." + + +class PinnedPolicyTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.adapter = DemoAdapter(self.root) + self.surface = self.adapter.substrate.surface() + + def stage(self, pinned: str) -> tuple[Path, str]: + spec = dataclasses.replace(self.adapter.spec, pinned_policy=pinned) + install = self.adapter.substrate.materialise(self.adapter.base_harness(), spec, {}) + self.addCleanup(shutil.rmtree, install.staging_dir, ignore_errors=True) + return install.staging_dir, install.fingerprint + + def test_policy_stays_out_of_editable_and_staged_source(self): + source = self.adapter.base_harness() / "systemprompt.md" + before = source.read_text() + staged, _ = self.stage(POLICY) + text = (staged / "systemprompt.md").read_text() + self.assertEqual(text, before) + self.assertEqual(source.read_text(), before) + + def test_no_policy_changes_nothing(self): + staged, fingerprint = self.stage("") + source = self.adapter.base_harness() + self.assertEqual( + (staged / "systemprompt.md").read_text(), (source / "systemprompt.md").read_text() + ) + self.assertEqual(fingerprint, runmark.fingerprint(source)) + self.assertEqual(policy.digest(""), "") + self.assertEqual(policy.reserved_chars(""), 0) + + def test_policy_changes_the_fingerprint(self): + _, none = self.stage("") + _, one = self.stage(POLICY) + _, two = self.stage(POLICY + " Ask before sending email.") + self.assertEqual(len({none, one, two}), 3) + + def test_marker_from_another_policy_is_not_reusable(self): + harness = self.adapter.base_harness() + where = self.root / "mark" + runmark.mark_complete(where, run_id="r", harness_dir=harness, salt=policy.digest(POLICY)) + self.assertTrue(runmark.is_complete(where, harness_dir=harness, salt=policy.digest(POLICY))) + self.assertFalse(runmark.is_complete(where, harness_dir=harness)) + self.assertFalse( + runmark.is_complete(where, harness_dir=harness, salt=policy.digest("other")) + ) + + def test_verify_fails_closed_on_tamper(self): + staged, _ = self.stage(POLICY) + target = staged / "systemprompt.md" + policy.inject_file(target, POLICY, max_chars=self.surface.always_on_max) + target.write_text(target.read_text().replace("Never delete", "Feel free to delete")) + with self.assertRaisesRegex(ValueError, "pinned policy missing or altered"): + policy.verify_file(target, POLICY, max_chars=self.surface.always_on_max) + target.write_text("") + with self.assertRaisesRegex(ValueError, "pinned policy missing or altered"): + policy.verify_file(target, POLICY, max_chars=self.surface.always_on_max) + + def test_codex_delivers_policy_separately_from_staged_source(self): + source = self.root / "cx" + tree.write_tree(CODEX_SURFACE, {"AGENTS.md": "Do the work.\n"}, source, stubs=False) + spec = dataclasses.replace( + self.adapter.spec, install_mode="codex_root", pinned_policy=POLICY + ) + install = CodexSubstrate().materialise(source, spec, {}) + self.addCleanup(shutil.rmtree, install.staging_dir, ignore_errors=True) + self.assertEqual((install.staging_dir / "AGENTS.md").read_text(), "Do the work.\n") + self.assertEqual(install.extras["pinned_policy"], POLICY) + self.assertNotEqual(install.fingerprint, runmark.fingerprint(source)) + + def test_policy_is_charged_to_the_always_on_cap(self): + spec = dataclasses.replace(self.adapter.spec, pinned_policy=POLICY) + adapter = self.adapter + adapter.spec = spec + proposed = {**tree.read_tree(self.surface, self.adapter.base_harness())} + proposed["systemprompt.md"] = "" + room = self.surface.always_on_max - len(tree.always_on_text(self.surface, proposed)) + body = "x" * (room - policy.reserved_chars(POLICY) + 1) + proposed["systemprompt.md"] = body + base = dict(proposed) + _, err = recursion._accept_tree(self.surface, adapter, proposed, base, "t") + self.assertIn("over the", err) + proposed["systemprompt.md"] = body[:-1] + _, err = recursion._accept_tree(self.surface, adapter, proposed, base, "t") + self.assertNotIn("over the", err) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_policy_execution.py b/tests/test_policy_execution.py new file mode 100644 index 00000000..ac8c3e03 --- /dev/null +++ b/tests/test_policy_execution.py @@ -0,0 +1,364 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regressions at render, launch, and whole-task resume boundaries (no model calls).""" + +import asyncio +import dataclasses +import importlib +import json +import os +import shutil +import subprocess +import sys +import tempfile +import types +import unittest +from contextlib import contextmanager +from pathlib import Path +from unittest.mock import AsyncMock, Mock, patch + +from harness2 import config, install +from harness2.benchmarks import jb_opencode_runner, jobbench, lab, workbuddy +from harness2.core import policy, recursion, schedule, tree +from harness2.demo import DemoAdapter, DemoProposer +from harness2.substrate.codex import CODEX_SURFACE, CodexSubstrate + +ROOT = Path(__file__).resolve().parents[1] +LITERAL = "Keep {{ forbidden_host }} secret. {% literal %} {# text #}" + + +def package(name, path=None): + module = types.ModuleType(name) + module.__path__ = [str(path)] if path else [] + return module + + +@contextmanager +def lab_runtime(root): + """Supply only external LAB data; load the shipped renderer and installer unchanged.""" + harness_run = types.ModuleType("harness.run") + harness_run.DEFAULT_SKILLS = [] + harness_run.SKILLS_DIR = root + harness_run.SYSTEM_PROMPT_PREAMBLE = "Stock LAB prompt." + harness_run.load_skills = lambda names: "" + tts_config = types.ModuleType("external.tts_lab.config") + modules = { + "harness": package("harness"), + "harness.run": harness_run, + "opencode": package("opencode", ROOT / "third_party/lab_runner/opencode"), + "opencode.pinned_policy": policy, + "external": package("external"), + "external.tts_lab": package("external.tts_lab"), + "external.tts_lab.config": tts_config, + } + with patch.dict(sys.modules, modules): + renderer = importlib.import_module("opencode.render") + spec = importlib.util.spec_from_file_location( + "policy_lab_codex", ROOT / "third_party/lab_runner/adapt_runs_cx/run_codex_lab.py" + ) + codex = importlib.util.module_from_spec(spec) + spec.loader.exec_module(codex) + codex.READ_DOC = ROOT / "third_party/lab_runner/opencode/scripts/read_doc.py" + yield renderer, codex + + +@contextmanager +def wb_runtime(): + """Load actual agents with Harbor's external process/model interfaces stubbed.""" + modules = {"harbor": package("harbor")} + imports = { + "harbor.agents.installed.base": "BaseInstalledAgent", + "harbor.environments.base": "BaseEnvironment", + "harbor.models.agent.context": "AgentContext", + "harbor.models.trajectories.agent": "Agent", + "harbor.models.trajectories.final_metrics": "FinalMetrics", + "harbor.models.trajectories.metrics": "Metrics", + "harbor.models.trajectories.step": "Step", + "harbor.models.trajectories.tool_call": "ToolCall", + "harbor.models.trajectories.trajectory": "Trajectory", + } + base = type("BaseInstalledAgent", (), {"render_instruction": lambda self, value: value}) + for name, symbol in imports.items(): + for i in range(1, len(name.split(".")) + 1): + parent = ".".join(name.split(".")[:i]) + modules.setdefault(parent, package(parent)) + setattr(modules[name], symbol, base if symbol == "BaseInstalledAgent" else object) + modules["workbuddy_bench"] = package("workbuddy_bench") + modules["workbuddy_bench.agents"] = package( + "workbuddy_bench.agents", ROOT / "third_party/wb_runner/agents" + ) + modules["workbuddy_bench.agents.pinned_policy"] = policy + user = types.ModuleType("workbuddy_bench.agents._agent_user") + user.ensure_agent_user = AsyncMock() + modules[user.__name__] = user + with patch.dict(sys.modules, modules): + oc = importlib.import_module("workbuddy_bench.agents.oc_agent") + cx = importlib.import_module("workbuddy_bench.agents.codex_harness_agent") + yield oc, cx, base + + +class PolicyExecutionTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.adapter = DemoAdapter(self.root) + + def test_jobbench_renders_before_literal_policy_injection(self): + staged = self.adapter.base_harness() + (staged / "systemprompt.md").write_text("Editable {{ date }}. ") + out = self.root / "rendered" + jobbench.render_opencode_tree(staged, out, {"date": "today"}, pinned_policy=LITERAL) + actual = (out / "AGENTS.md").read_text() + self.assertTrue(actual.startswith(policy.block(LITERAL))) + self.assertIn("Editable today.", actual) + self.assertNotIn("", actual) + + def test_jobbench_checks_installed_file_before_launch(self): + workspace = self.root / "ws" + workspace.mkdir() + (workspace / "AGENTS.md").write_text("tampered") + with ( + patch.dict( + os.environ, + { + policy.ENV: LITERAL, + "HARNESS2_POLICY_MODULE": str(Path(policy.__file__).resolve()), + }, + ), + patch.object(jb_opencode_runner.subprocess, "run") as launch, + ): + with self.assertRaisesRegex(ValueError, "pinned policy missing"): + jb_opencode_runner._attempt( + Mock(), workspace, self.root / "traj", self.root / "log", "task", 1 + ) + launch.assert_not_called() + + def test_codex_shell_policy_runtime_preserves_literal_bytes_and_refuses_oversize(self): + ws = self.root / "ws" + ws.mkdir() + target = ws / "AGENTS.md" + target.write_text("Stock body.") + env = {**os.environ, policy.ENV: LITERAL} + result = subprocess.run( + [sys.executable, policy.__file__, str(ws)], env=env, capture_output=True, text=True + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(target.read_text(), policy.block(LITERAL) + "Stock body.") + before = target.read_bytes() + env[policy.ENV] = "x" * 40000 + result = subprocess.run( + [sys.executable, policy.__file__, str(ws)], env=env, capture_output=True, text=True + ) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(target.read_bytes(), before) + + def test_direct_materialisation_and_ctx_overrides_obey_policy_budget(self): + spec = dataclasses.replace(self.adapter.spec, pinned_policy="x" * 24000) + with self.assertRaisesRegex(ValueError, "over the"): + self.adapter.substrate.materialise(self.adapter.base_harness(), spec, {}) + spec = dataclasses.replace(self.adapter.spec, pinned_policy=LITERAL) + with self.assertRaisesRegex(ValueError, "over the"): + self.adapter.substrate.materialise( + self.adapter.base_harness(), spec, {"file:systemprompt.md": "x" * 24000} + ) + source = tree.write_tree(CODEX_SURFACE, {"AGENTS.md": "Work."}, self.root / "cx") + spec = dataclasses.replace(spec, install_mode="codex_root", pinned_policy="x" * 40000) + with self.assertRaisesRegex(ValueError, "over the"): + CodexSubstrate().materialise(source, spec, {}) + + def test_byte_limit_is_checked_separately_and_whitespace_policy_is_noop(self): + with self.assertRaisesRegex(ValueError, "UTF-8 byte"): + policy.prepend("", "😀" * 100, max_chars=1000, max_bytes=300) + self.assertEqual(policy.prepend("body", " \n\t", max_chars=0), "body") + self.assertEqual(policy.digest(" \n\t"), "") + + def test_autotrigger_reserves_policy_budget(self): + adapter = self.adapter + adapter.spec = dataclasses.replace(adapter.spec, pinned_policy=LITERAL) + surface = adapter.substrate.surface() + proposed = tree.read_tree(surface, adapter.base_harness()) + proposed["skills/helper/SKILL.md"] = "---\nname: helper\ndescription: do work\n---\nSteps." + cap = surface.always_on_max - policy.reserved_chars(LITERAL) + proposed["systemprompt.md"] = "x" * (cap - 6) + _, error = recursion._accept_tree(surface, adapter, proposed, dict(proposed), "test") + self.assertNotIn("over the", error) + self.assertNotIn("load the `helper`", proposed["guardrails.md"]) + + def test_lab_base_and_candidate_keep_rendered_body_with_literal_policy(self): + with lab_runtime(self.root) as (renderer, _): + for candidate in (None, self.adapter.base_harness()): + for mode in ("append", "replace"): + with self.subTest(candidate=candidate, mode=mode): + ws = self.root / f"lab-{candidate is None}-{mode}" + original = renderer.render_tree( + ws, candidate, base_skill_names=[], prompt_mode=mode + ) + before = original.prompt + renderer.apply_pinned_policy(ws, original, LITERAL) + self.assertEqual(original.prompt, policy.block(LITERAL) + before) + target = ( + ws / "AGENTS.md" if mode == "append" else ws / ".opencode/agent/lab.md" + ) + self.assertIn(original.prompt, target.read_text()) + + def test_lab_codex_base_and_candidate_inject_after_install(self): + with lab_runtime(self.root) as (_, cx), patch.dict(os.environ, {policy.ENV: LITERAL}): + source = tree.write_tree( + CODEX_SURFACE, {"AGENTS.md": "Existing base."}, self.root / "cx" + ) + for candidate in (None, source): + ws = self.root / f"cx-{candidate is None}" + ws.mkdir() + cx.install_harness(ws, candidate) + expected = "" if candidate is None else "Existing base." + self.assertEqual((ws / "AGENTS.md").read_text(), policy.block(LITERAL) + expected) + + def test_lab_and_workbuddy_base_installs_preserve_base_semantics(self): + for cls in (lab.LabAdapter, workbuddy.WbAdapter): + adapter = object.__new__(cls) + adapter.spec = dataclasses.replace(self.adapter.spec, pinned_policy=LITERAL) + adapter.base_harness = Mock(return_value=self.adapter.base_harness()) + adapter.substrate = Mock() + result = adapter._install(None) + self.assertEqual(result.extras["pinned_policy"], LITERAL) + self.assertTrue(result.fingerprint.startswith("base-")) + adapter.substrate.materialise.assert_not_called() + if cls is workbuddy.WbAdapter: + self.assertEqual(result.extras["install_mode"], "none") + with patch.object(shutil, "rmtree") as remove: + adapter._drop_staging(result) + remove.assert_not_called() + else: + self.assertEqual(result.extras["harness"], "base") + + def test_workbuddy_policy_is_prepended_after_template_render_for_base_and_candidate(self): + with wb_runtime() as (oc, _, base): + agent = object.__new__(oc.OcAgent) + agent._pinned_policy = LITERAL + for rendered in ("Task text.", "Editable instructions.\nTask text."): + with patch.object(base, "render_instruction", return_value=rendered) as render: + actual = agent.render_instruction("Task text.") + render.assert_called_once_with("Task text.") + self.assertEqual(actual, policy.block(LITERAL) + rendered) + agent._pinned_policy = "" + self.assertEqual(agent.render_instruction("Task text."), "Task text.") + + def test_workbuddy_codex_installs_then_injects_without_replacing_base_instructions(self): + with wb_runtime() as (_, cx, _): + source = tree.write_tree(CODEX_SURFACE, {"AGENTS.md": "Candidate."}, self.root / "cx") + for candidate in (None, source): + agent = object.__new__(cx.CodexHarnessAgent) + agent._harness_ws = candidate + agent._pinned_policy = LITERAL + " $(touch should-not-exist) `literal` 'quote'" + ws = self.root / f"container-{candidate is None}" + ws.mkdir() + (ws / "AGENTS.md").write_text("Task image instructions.") + + async def execute(environment, *, command): + result = subprocess.run( + ["bash", "-o", "pipefail", "-c", command.replace("/workspace", str(ws))], + capture_output=True, + text=True, + ) + return types.SimpleNamespace(return_code=result.returncode) + + agent.exec_as_agent = AsyncMock(side_effect=execute) + asyncio.run(agent._materialize_harness(Mock())) + self.assertEqual( + (ws / "AGENTS.md").read_text(), + policy.block(agent._pinned_policy) + + ("Candidate." if candidate else "Task image instructions."), + ) + self.assertFalse((ws / "should-not-exist").exists()) + agent._pinned_policy = "x" * 40000 + with patch.object(cx.CodexAgent, "run", new_callable=AsyncMock) as launch: + with self.assertRaisesRegex(RuntimeError, "verification failed"): + asyncio.run(agent.run("Task", Mock(), Mock())) + launch.assert_not_called() + self.assertEqual((source / "AGENTS.md").read_text(), "Candidate.") + + def test_workbuddy_job_passes_policy_to_base_and_candidate_agents(self): + for substrate in ("oc", "cx"): + for ws in (None, self.adapter.base_harness()): + text = workbuddy._job_yaml( + workbuddy.SUBSETS["office"], + substrate, + "test", + ["task"], + ws, + 1, + self.root, + pinned_policy=LITERAL, + ) + self.assertIn(f"{policy.ENV}: {json.dumps(LITERAL)}", text) + + def test_setup_copies_one_policy_runtime_into_both_isolated_checkouts(self): + with patch.object(config, "DEPENDENCIES", self.root / "deps"): + for name, relative in ( + ("harvey-labs", "opencode/pinned_policy.py"), + ("workbuddy-bench", "src/workbuddy_bench/agents/pinned_policy.py"), + ): + install.overlays(name) + for substrate in ("oc", "cx"): + install.require_policy_runtime(name, substrate) + self.assertEqual( + (config.DEPENDENCIES / name / relative).read_bytes(), + Path(policy.__file__).read_bytes(), + ) + + def test_old_runner_is_rejected_before_a_configured_policy_can_be_ignored(self): + with patch.object(config, "DEPENDENCIES", self.root / "deps"): + with self.assertRaisesRegex(RuntimeError, "rerun harness2-setup"): + install.require_policy_runtime("harvey-labs", "oc") + install.overlays("harvey-labs") + runner = config.DEPENDENCIES / "harvey-labs/opencode/run_opencode.py" + runner.write_text("# old runner that ignores the policy environment") + with self.assertRaisesRegex(RuntimeError, "outdated"): + install.require_policy_runtime("harvey-labs", "oc") + + def test_policy_changes_and_removal_invalidate_whole_task_resume(self): + adapter = self.adapter + proposer = DemoProposer(adapter.spec.channels["proposer"]) + kwargs = dict( + adapter=adapter, + prop=proposer, + runs_root=adapter.spec.runs_root, + library_root=self.root / "library", + ) + for pinned in ("", LITERAL, "Different policy.", ""): + adapter.spec = dataclasses.replace(adapter.spec, pinned_policy=pinned) + result = schedule.run_domain("accounting", ["ledger-1"], 1, **kwargs) + self.assertEqual([row["status"] for row in result["tasks"]], ["ok"]) + before = adapter.execution_count + result = schedule.run_domain("accounting", ["ledger-1"], 1, **kwargs) + self.assertEqual([row["status"] for row in result["tasks"]], ["skipped"]) + self.assertEqual(adapter.execution_count, before) + + def test_summary_never_merges_modes_from_a_different_policy(self): + summary = self.root / "summary.json" + old = schedule.TaskResult("task", "domain", 1, policy_digest=policy.digest("old")) + old.modes = {"seq": {"ok": True}} + old.selections = {"seq": {"ok": True, "chosen_label": "old"}} + schedule._write_summary(summary, old) + new = schedule.TaskResult("task", "domain", 1, policy_digest=policy.digest("new")) + new.modes = {"agg": {"ok": True}} + new.selections = {"agg": {"ok": True, "chosen_label": "new"}} + schedule._write_summary(summary, new) + self.assertEqual(set(json.loads(summary.read_text())["modes"]), {"agg"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/third_party/jb_runner/PROVENANCE.md b/third_party/jb_runner/PROVENANCE.md index 65e16ced..bfb71eab 100644 --- a/third_party/jb_runner/PROVENANCE.md +++ b/third_party/jb_runner/PROVENANCE.md @@ -23,5 +23,8 @@ with upstream. It has been replaced by the first-party Python module which the adapter launches as a subprocess in its place. That module is release code, not a vendored asset, so it is not listed in `ASSETS.json`. +The Codex launcher invokes the first-party policy runtime after workspace +installation to inject and verify the final `AGENTS.md` before execution. + [ASSETS.json](../../../dependencies/ASSETS.json) records the shipped file checksums. Verify them with `python scripts/check_provenance.py`. diff --git a/third_party/jb_runner/run_benchmark_codex_cli_adapt.sh b/third_party/jb_runner/run_benchmark_codex_cli_adapt.sh index 0105da58..7467621f 100644 --- a/third_party/jb_runner/run_benchmark_codex_cli_adapt.sh +++ b/third_party/jb_runner/run_benchmark_codex_cli_adapt.sh @@ -266,6 +266,11 @@ run_task() { echo "Harness render dir: ${HARNESS_RENDER_DIR}" | tee -a "$log_file" fi + # Inject and read back the final installed AGENTS.md before starting the model. + if [[ -n "${HARNESS2_PINNED_POLICY:-}" ]]; then + "${HARNESS2_POLICY_PYTHON:?}" "${HARNESS2_POLICY_MODULE:?}" "$temp_workspace" || return 1 + fi + # Make sure TASK_INSTRUCTIONS.txt exists local instructions_file="${temp_task_folder}/TASK_INSTRUCTIONS.txt" if [[ ! -f "$instructions_file" ]]; then diff --git a/third_party/lab_runner/PROVENANCE.md b/third_party/lab_runner/PROVENANCE.md index 25c29d8f..87441b38 100644 --- a/third_party/lab_runner/PROVENANCE.md +++ b/third_party/lab_runner/PROVENANCE.md @@ -7,5 +7,10 @@ The integration adds editable harness installation and configurable models, paths, and endpoints. Local model-provider names, environment variables, and log paths use the `harness2` namespace. Setup installs these runners into the LAB checkout as overlays. +The policy integration preserves base execution behavior, injects literal deployer +policy after rendering or workspace installation, and verifies the final input. +Setup also copies the first-party `harness2/core/policy.py` module beside these +runners as `pinned_policy.py`; it is not an upstream-derived asset. + [ASSETS.json](../../../dependencies/ASSETS.json) records the shipped file checksums. Verify them with `python scripts/check_provenance.py`. diff --git a/third_party/lab_runner/adapt_runs_cx/run_codex_lab.py b/third_party/lab_runner/adapt_runs_cx/run_codex_lab.py index cb4ed81c..71b0f6c5 100644 --- a/third_party/lab_runner/adapt_runs_cx/run_codex_lab.py +++ b/third_party/lab_runner/adapt_runs_cx/run_codex_lab.py @@ -130,6 +130,10 @@ def install_harness(ws: Path, harness_dir: Path | None) -> list[str]: raise SystemExit(f"runtime document reader missing: {READ_DOC} -- without it every " f"binary document 'read' silently returns noise") shutil.copy2(READ_DOC, ws / "scripts" / "read_doc.py") # LAST: runtime wins + from opencode import pinned_policy as pinned + + pinned.inject_file(ws / "AGENTS.md", os.environ.get(pinned.ENV, ""), + max_chars=pinned.CODEX_MAX_CHARS, max_bytes=pinned.CODEX_MAX_BYTES) return installed diff --git a/third_party/lab_runner/opencode/render.py b/third_party/lab_runner/opencode/render.py index 747c7469..e14a200e 100644 --- a/third_party/lab_runner/opencode/render.py +++ b/third_party/lab_runner/opencode/render.py @@ -460,3 +460,20 @@ def render_tree( plugins=plugins, plugin_tools=[t for t in plugin_tools if not (t in seen or seen.add(t))], ) + + +def apply_pinned_policy(workspace: Path, rendered: RenderResult, policy: str) -> None: + """Apply literal policy after rendering, preserving the baseline's components.""" + from opencode import pinned_policy as pinned + + if not pinned.block(policy): + return + rendered.prompt = pinned.prepend(rendered.prompt, policy, max_chars=pinned.OPENCODE_MAX_CHARS) + _write_prompt(workspace, rendered.prompt, "replace" if rendered.agent else "append", + rendered.agent or AGENT_NAME) + if rendered.agent: + target = workspace / ".opencode" / "agent" / f"{rendered.agent}.md" + actual = target.read_bytes().decode("utf-8").split("---\n\n", 1)[1] + pinned.verify_prompt(actual, policy, max_chars=pinned.OPENCODE_MAX_CHARS) + else: + pinned.verify_file(workspace / "AGENTS.md", policy, max_chars=pinned.OPENCODE_MAX_CHARS) diff --git a/third_party/lab_runner/opencode/run_opencode.py b/third_party/lab_runner/opencode/run_opencode.py index c30cdac4..67f58d0c 100644 --- a/third_party/lab_runner/opencode/run_opencode.py +++ b/third_party/lab_runner/opencode/run_opencode.py @@ -25,7 +25,7 @@ from harness.run import BENCH_ROOT, DEFAULT_SKILLS, _load_env, load_task from harness.tools import get_all_tool_definitions from opencode import session as oc_session -from opencode.render import RenderResult, render_tree +from opencode.render import RenderResult, apply_pinned_policy, render_tree from utils.stdio import force_utf8_stdio DEFAULT_OPENCODE_DIR = Path(os.environ.get("OPENCODE_DIR", "")) @@ -219,6 +219,7 @@ def main(args) -> None: sys.exit(f"harness defines subagents {rendered.subagents} but --allow-subagents was not " f"passed; `task` has no native-harness equivalent, so the run would not be " f"tool-matched to the baselines") + apply_pinned_policy(workspace, rendered, os.environ.get("HARNESS2_PINNED_POLICY", "")) prompt, agent = rendered.prompt, rendered.agent components = harness_inventory(rendered) tools = tool_map(args.tool_mode, args.skill_mode, diff --git a/third_party/wb_runner/PROVENANCE.md b/third_party/wb_runner/PROVENANCE.md index 4d099148..a02d8d18 100644 --- a/third_party/wb_runner/PROVENANCE.md +++ b/third_party/wb_runner/PROVENANCE.md @@ -7,5 +7,10 @@ The integration adds editable harness installation and configurable models, paths, and endpoints. Local model-provider names, environment variables, and log paths use the `harness2` namespace. Setup installs these agents and harness configurations into the WorkBuddy checkout. +The policy integration preserves base execution behavior, injects literal deployer +policy after rendering or workspace installation, and verifies the final input. +Setup also copies the first-party `harness2/core/policy.py` module beside these +runners as `pinned_policy.py`; it is not an upstream-derived asset. + [ASSETS.json](../../../dependencies/ASSETS.json) records the shipped file checksums. Verify them with `python scripts/check_provenance.py`. diff --git a/third_party/wb_runner/agents/codex_harness_agent.py b/third_party/wb_runner/agents/codex_harness_agent.py index 2e1e356b..e7ab05de 100644 --- a/third_party/wb_runner/agents/codex_harness_agent.py +++ b/third_party/wb_runner/agents/codex_harness_agent.py @@ -4,6 +4,7 @@ import base64 import io +import shlex import tarfile from pathlib import Path @@ -11,6 +12,7 @@ from harbor.models.agent.context import AgentContext from workbuddy_bench.agents.codex_agent import CodexAgent +from workbuddy_bench.agents import pinned_policy as pinned _HARNESS_ENTRIES = ("AGENTS.md", ".agents", "scripts", ".codex") @@ -19,6 +21,7 @@ class CodexHarnessAgent(CodexAgent): """Codex CLI + a harness workspace installed (and committed) at /workspace.""" def __init__(self, logs_dir: Path, *args, **kwargs): + self._pinned_policy = kwargs.pop(pinned.ENV, "") ws = kwargs.pop("WB_HARNESS_WS", None) self._harness_ws: Path | None = Path(ws) if ws else None super().__init__(logs_dir, *args, **kwargs) @@ -30,7 +33,7 @@ def name() -> str: async def run( self, instruction: str, environment: BaseEnvironment, context: AgentContext ) -> None: - if self._harness_ws is not None: + if self._harness_ws is not None or pinned.block(self._pinned_policy): await self._materialize_harness(environment) await super().run(instruction, environment, context) @@ -41,6 +44,8 @@ def _harness_payload(self) -> tuple[bytes, int]: n = 0 with tarfile.open(fileobj=payload, mode="w:gz") as tar: for entry in _HARNESS_ENTRIES: + if ws is None: + continue src = ws / entry if src.is_file(): tar.add(src, arcname=entry) @@ -58,16 +63,25 @@ async def _materialize_harness(self, environment: BaseEnvironment) -> None: The commit is conditional on staged changes, so a no-op harness is not a failure. """ ws = self._harness_ws - if ws is None or not ws.is_dir(): - return + if ws is not None and not ws.is_dir(): + raise ValueError(f"harness workspace missing: {ws}") blob_bytes, n = self._harness_payload() - if n == 0: + if n == 0 and not pinned.block(self._pinned_policy): return - blob = base64.b64encode(blob_bytes).decode() - cmd = ( - f"cd /workspace && " - f"echo {blob} | base64 -d | tar -xzf - -C /workspace && " - # Neutralize the install in the diff capture; skip when this is not a git repo. + commands = ["cd /workspace"] + if n: + blob = base64.b64encode(blob_bytes).decode() + commands.append(f"echo {blob} | base64 -d | tar -xzf - -C /workspace") + if pinned.block(self._pinned_policy): + # Run the shared verifier inside the container after installation. For BASE, + # prepend to any AGENTS.md supplied by the task image rather than replacing it. + runtime = base64.b64encode(Path(pinned.__file__).read_bytes()).decode() + commands.append( + f"echo {runtime} | base64 -d | " + f"{pinned.ENV}={shlex.quote(self._pinned_policy)} python3 - /workspace" + ) + # Neutralize the install in diff capture; skip when this is not a git repo. + commands.append( "if git rev-parse --git-dir >/dev/null 2>&1; then " "git add -A && " "{ git diff --cached --quiet || " @@ -75,7 +89,10 @@ async def _materialize_harness(self, environment: BaseEnvironment) -> None: 'commit -q -m "harness install (pre-agent)"; }; ' "fi" ) - await self.exec_as_agent(environment, command=cmd) + cmd = " && ".join(commands) + result = await self.exec_as_agent(environment, command=cmd) + if result.return_code != 0: + raise RuntimeError("Codex harness installation or pinned policy verification failed") log = getattr(self, "logger", None) if log is not None: log.info("materialized + committed %d harness file(s) from %s", n, ws) diff --git a/third_party/wb_runner/agents/oc_agent.py b/third_party/wb_runner/agents/oc_agent.py index eeac9f6b..c6c18655 100644 --- a/third_party/wb_runner/agents/oc_agent.py +++ b/third_party/wb_runner/agents/oc_agent.py @@ -20,6 +20,7 @@ from harbor.models.trajectories.trajectory import Trajectory from workbuddy_bench.agents._agent_user import ensure_agent_user +from workbuddy_bench.agents import pinned_policy as pinned _OUTPUT_FILENAME = "oc-output.txt" @@ -34,6 +35,7 @@ class OcAgent(BaseInstalledAgent): SUPPORTS_ATIF: bool = True def __init__(self, logs_dir: Path, *args, **kwargs): + self._pinned_policy = kwargs.pop(pinned.ENV, "") self._oc_version: str | None = kwargs.pop("OPENCODE_VERSION", None) # OpenCode has no --max-turns; the cap is advisory and recorded only, since the # real bound is the per-trial timeout. @@ -150,6 +152,12 @@ async def install(self, environment: BaseEnvironment) -> None: ) await ensure_agent_user(self, environment) + def render_instruction(self, instruction: str) -> str: + rendered = super().render_instruction(instruction) + # The task instruction is not part of the editable always-on budget. + return pinned.prepend(rendered, self._pinned_policy, + max_chars=pinned.OPENCODE_MAX_CHARS + len(instruction)) + async def run( self, instruction: str, environment: BaseEnvironment, context: AgentContext ) -> None: