Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ guardrails, memory, tool guidance, subagents, skills, scripts, and plugins**.
They control context construction, the tool interface, delegation, and behavior
inside the tool loop.

Deployers can supply a [pinned policy](docs/PINNED_POLICY.md) outside the editable
tree; runners inject and verify it at the final execution input.

<p align="center">
<a href="assets/components.png">
<img src="assets/components.png" alt="Eight harness components grouped by runtime role: system prompt, guardrails, memory, and skills construct context; tool guidance and scripts define the tool interface; subagents orchestrate delegated work; plugins control middleware." width="100%">
Expand Down
18 changes: 9 additions & 9 deletions dependencies/ASSETS.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"third_party/jb_runner/LICENSE": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4",
"third_party/jb_runner/PROVENANCE.md": "7e16c8f5e43aa272baa5cf39bf245b42cf1f40f885457ac4123c91b7de0b06ef",
"third_party/jb_runner/PROVENANCE.md": "d05d13291dd3bf1ae774573a0e18b55a6eebbd2ec09e95f7339306e0c1b487a8",
"third_party/jb_runner/codex_base_harness/AGENTS.md": "b5df5264478dfce7d6af52f8443a1797b17967d1bc4a8c4f8c5fe43933516a89",
"third_party/jb_runner/harness_baseline_oc/LongTermMEMORY.md": "c78b55245ef1433c017bc3f6f796094e16c640b98543836f6d930972243d28d2",
"third_party/jb_runner/harness_baseline_oc/guardrails.md": "ebd0b0f287bcc9bea5efa063d632677b3c0fdf014c7b67bec088c11253605a53",
Expand All @@ -10,15 +10,15 @@
"third_party/jb_runner/harness_baseline_oc/sub_agents/README.md": "56c5507e02c30c0d611efe2d7426141e9fc626a91599738afc7e003a07d3adb0",
"third_party/jb_runner/harness_baseline_oc/systemprompt.md": "f5fa6862d87e4c51b78a4ce4446dace9438b3d201e0350c91ca02d0c83f552e3",
"third_party/jb_runner/harness_baseline_oc/tool_descriptions/tool_guidance.md": "5ae4f052905a31e9573abe4971172bf40b4e1f5d265d1be072ed444232371f41",
"third_party/jb_runner/run_benchmark_codex_cli_adapt.sh": "a94c645d3061da354591dde446d11f8b3e6d08b4005f8f587668aae07c0fc100",
"third_party/jb_runner/run_benchmark_codex_cli_adapt.sh": "06c9e8e1c410094d578b2a3ad59f81db8f0a904076acef1f3a11dd531a512ce3",
"third_party/lab_runner/LICENSE": "f92627d2ebe80fc0add3b171b2d7eee5e28a98dd0d0a4a5ee5829314243bb3b9",
"third_party/lab_runner/PROVENANCE.md": "b5b42319000e1522765d2520dffcd9d1058c5472cc49046ec1906ffffa66c53c",
"third_party/lab_runner/adapt_runs_cx/run_codex_lab.py": "567d6a06d8104f853b8d83953af333ea8ef57a9053e0c9788a42692c48196556",
"third_party/lab_runner/PROVENANCE.md": "f3ba3923c0e0cc7f8ef9423e645517b898c201d7cd0b8a468bfbaed161093337",
"third_party/lab_runner/adapt_runs_cx/run_codex_lab.py": "3372a02b834f5a4cbd6063144dddc42fb46de7d48acd6c651f361deea89e1598",
"third_party/lab_runner/opencode/README.md": "20f2739eada1a76560c412d39603e040c420d3d36a2bf67a99a814500306f0ae",
"third_party/lab_runner/opencode/__init__.py": "df2b298f529bbcba94713aa748b5a04569f38bbb3f96c9219b7fb0d7186066a8",
"third_party/lab_runner/opencode/render.py": "b636a60d039fdfa66d64eb5d5d3c93a99464be2bcbd5b80dc8f91d1f104ea5ee",
"third_party/lab_runner/opencode/render.py": "3aab2f5fa993acec5d79634a2712af25e92040fa4c9cfb05f276851a9cc04dae",
"third_party/lab_runner/opencode/render_tree.py": "54d64b461e430905b212b46902512aad38d3585fb5921b7a9982eea0413a4b73",
"third_party/lab_runner/opencode/run_opencode.py": "7e7178583a6acb2e07bffd1fbef78f1b0095975bc2c61f8b0038fcc528090e32",
"third_party/lab_runner/opencode/run_opencode.py": "3fb56d2e8a96094c6603973d5f9d3fcdd3c3c3789b5c9c10da97de78dcd9bec1",
"third_party/lab_runner/opencode/scripts/read_doc.py": "6b3c8feb3b90b0b2bbbf4cb491c81f1114b67ada04d2c988fac8cb1af68533b5",
"third_party/lab_runner/opencode/session.py": "e804e29a20ac4fd71454c85b9e15432cf303739c7f8d7aacc0df19f732bfcb62",
"third_party/lab_runner/tts_lab/__init__.py": "4ae8a9112bc00a182c983a35c8c5db40717007cefbc1b0bed89f04bc384a0106",
Expand All @@ -32,10 +32,10 @@
"third_party/patches/opencode-compat.diff": "c9a8a0e826d48f3fcfc7d818321d128c28e1f7f7c594954c19453dfca4946dc8",
"third_party/patches/workbuddy-bench-runtime.diff": "1d581ee0c23a9a5574ce0f9317d0bda17f6c3a2556898c89b34ccbdbd5b87b3f",
"third_party/wb_runner/LICENSE": "b466fc7a63acc5cac4275fc4b0f1e22f332ad188ea600b217076ea3a8e040956",
"third_party/wb_runner/PROVENANCE.md": "9011685c8808fb6ab56d23c823af884e5905caecd00f64323f3d02f22e45524c",
"third_party/wb_runner/PROVENANCE.md": "d7325534e1a57633ad47c7169944586ea4a67d9fa437309f47252370d7ffc403",
"third_party/wb_runner/agents/codex_agent.py": "dbdf48c19c51d7a0389caf3f5601c1ff967ef534137d829afe8ca8792db40c9d",
"third_party/wb_runner/agents/codex_harness_agent.py": "fc1d1ecfc2de1debabb0ff84ec7a113b05536873ac76f73806835ff873b2f580",
"third_party/wb_runner/agents/oc_agent.py": "5375a49b9a9e6d6d6212b47156b36dca17934e54728ddeed6218ac3662c67fff",
"third_party/wb_runner/agents/codex_harness_agent.py": "b1a3c0cb1731933f131ac27543785bb1183bdc05f2cb8009f5f7fa9fbc1007aa",
"third_party/wb_runner/agents/oc_agent.py": "7529c047435e096158101f44dade2ae3f6c96121baf1dbd8560a610cd9cb23f2",
"third_party/wb_runner/agents/oc_harness_agent.py": "647e31187af1305a5ddc42f7e41740795f825e0c708f63f5d1f29b954625c09c",
"third_party/wb_runner/configs/harnesses/codex-harness/_defaults.yaml": "5dddd5c358eff75ca69f7277ae6ccd58a2e208e19d722b862f04062a3718479e",
"third_party/wb_runner/configs/harnesses/codex-harness/versions/0.144.6.yaml": "f6ebdc457786c3869ceb8a11ee0c4a63db53a080961075c15edff2a7640293bf",
Expand Down
69 changes: 69 additions & 0 deletions docs/PINNED_POLICY.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
# Pinned deployer policy

Set `BenchSpec.pinned_policy` through the Python adapter API before starting a run:

```python
from dataclasses import replace
from pathlib import Path

adapter.spec = replace(
adapter.spec,
pinned_policy=Path("/trusted/config/policy.md").read_text(encoding="utf-8"),
)
```

Keep the policy source outside the editable harness directory. This is an API setting;
there is no additional CLI flag. Empty or whitespace-only text disables the feature.
Leading and trailing whitespace is stripped once when constructing the policy block;
internal text is preserved literally, including template syntax and HTML comments.

## Delivery contract

1. Materialisation stages the editable tree without inserting the policy into its files
or templates. `Install.extras["pinned_policy"]` carries the trusted policy separately.
2. Each benchmark renders its editable content using its existing rules.
3. The launcher prepends the literal policy block after the final rendering or workspace
installation step, then verifies the execution input before starting the model.

| Execution path | Final policy delivery |
|---|---|
| JobBench / OpenCode | After template interpolation and comment removal; the copied workspace `AGENTS.md` is checked before each attempt. |
| LAB / OpenCode | After the base or candidate renderer; the final `AGENTS.md` or primary agent prompt is read back. |
| WorkBuddy / OpenCode | After Harbor renders the Jinja instruction template; the resulting string is passed directly to the CLI. |
| Codex | After installing the workspace harness; the final `AGENTS.md` is read back before execution. |

LAB and WorkBuddy base runs retain their original base rendering and component selection.
They receive the same policy as candidates. A WorkBuddy Codex base run prepends to any
`AGENTS.md` already supplied by the task image instead of replacing its instructions.
The local deterministic demo does not execute an LLM prompt and is not a policy-enforcement
integration; its scheduler can still exercise policy cache invalidation.

## Limits and caching

The policy block consumes the editable always-on budget (24,000 characters for OpenCode,
8,000 for Codex). Direct materialisation and final delivery both check the applicable
budget. Codex also checks the 32,768-byte UTF-8 limit. In WorkBuddy's composed instruction,
the task instruction itself is excluded from the always-on character budget. Oversized
content fails before model execution rather than relying on runtime truncation.

Run markers include the full SHA-256 policy identity. Task summaries record and compare
that identity before resuming, and summaries from different policies cannot merge their
mode results. Adding, changing, or removing a policy invalidates previous execution and
whole-task results. With no policy, legacy execution fingerprints and summary resume
behavior remain compatible.

## Installing updated runners

LAB and WorkBuddy use isolated Python environments. Setup copies the standard-library-only
`harness2/core/policy.py` beside their runner overlays so all paths use the same code.
After updating Harness², rerun the relevant setup command (`harness2-setup lab` or
`harness2-setup wb`, with the selected substrate). Configured-policy launches reject
missing or outdated runner overlays and explain which setup command to run.

For a custom substrate or launcher, carrying the setting in an `Install` is insufficient:
implement the same final-input injection and verification contract. Keep the expected
policy in trusted launcher configuration, not in a candidate-controlled file.

This protects policy text against editing and delivery-time transformations. It does not
make the model obey that text or prevent a running hook, plugin, or agent from modifying
its environment. Tool restrictions and runtime isolation require separate enforcement.
8 changes: 8 additions & 0 deletions harness2/benchmarks/jb_opencode_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import json
import os
import runpy
import shutil
import subprocess
import sys
Expand Down Expand Up @@ -175,6 +176,13 @@ def _attempt(
cfg: Settings, workspace: Path, traj_file: Path, log_file: Path, task_name: str, attempt: int
) -> int:
"""Run OpenCode once; its event stream lands in `traj_file`, everything else in the log."""
if os.environ.get("HARNESS2_PINNED_POLICY", "").strip():
runtime = runpy.run_path(os.environ["HARNESS2_POLICY_MODULE"])
runtime["verify_file"](
workspace / "AGENTS.md",
os.environ["HARNESS2_PINNED_POLICY"],
max_chars=runtime["OPENCODE_MAX_CHARS"],
)
if cfg.agent_steps and not (workspace / "opencode.json").is_file():
(workspace / "opencode.json").write_text(
json.dumps(
Expand Down
65 changes: 44 additions & 21 deletions harness2/benchmarks/jobbench.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
from pathlib import Path

from .. import config
from ..core import policy
from ..core import runmark as RM
from ..core import tree as HT
from ..core.spec import BenchPrompts, BenchSpec, resolve_channel_model
Expand Down Expand Up @@ -454,7 +455,9 @@ def _ensure_mode_subagent(text: str) -> str:
return "---\nmode: subagent\ndescription: delegated subagent\n---\n\n" + text


def render_opencode_tree(staged: Path, render_dir: Path, ctx: dict) -> None:
def render_opencode_tree(
staged: Path, render_dir: Path, ctx: dict, *, pinned_policy: str = ""
) -> None:
"""Materialise the staged tree into OpenCode-loadable artifacts under render_dir:
AGENTS.md, .opencode/skills/<n>/, .opencode/agent/<sub>.md, scripts/, plugin/.
Skill directories are copied whole, so skills/<n>/scripts/*.py survive."""
Expand All @@ -464,7 +467,9 @@ def render_opencode_tree(staged: Path, render_dir: Path, ctx: dict) -> None:
(rd / ".opencode" / "agent").mkdir(parents=True)
(rd / ".opencode" / "skills").mkdir(parents=True)

body = compose_agent_body(staged, ctx)
body = policy.prepend(
compose_agent_body(staged, ctx), pinned_policy, max_chars=policy.OPENCODE_MAX_CHARS
)
if body.strip():
(rd / "AGENTS.md").write_text(body, encoding="utf-8")

Expand Down Expand Up @@ -714,18 +719,24 @@ def _ensure_store_task(self, task_id: str) -> Path:
raise
return dst

@property
def _policy_salt(self) -> str:
return policy.digest(self.spec.pinned_policy)

def execute(
self, task_id: str, harness_dir: Path | None, run_id: str, *, timeout: int | None = None
) -> Path:
"""One rollout; the label carries the whole namespace (run_id, / rewritten __)."""
label = run_id.replace("/", "__")
out = self._out_dir(task_id, label)
mark = self._traj_dir(task_id, label)
if RM.is_complete(mark, harness_dir=harness_dir) and self.artifacts_intact(out):
if RM.is_complete(
mark, harness_dir=harness_dir, salt=self._policy_salt
) and self.artifacts_intact(out):
out.mkdir(parents=True, exist_ok=True)
return out
if out.is_dir() or mark.is_dir():
reason = RM.why_not(mark, harness_dir=harness_dir)
reason = RM.why_not(mark, harness_dir=harness_dir, salt=self._policy_salt)
if reason == "reusable":
reason = "output fingerprint missing or changed"
print(
Expand All @@ -752,6 +763,7 @@ def execute(
mark,
run_id=run_id,
harness_dir=harness_dir,
salt=self._policy_salt,
extra={"output_fingerprint": self._output_fingerprint(out)},
)
return dirs[0]
Expand Down Expand Up @@ -786,23 +798,31 @@ def _launch_common(self, req: RunRequest) -> tuple[str, str, int]:
def _launch_opencode(self, req: RunRequest, install: Install) -> list[Path]:
task_id, label, t = self._launch_common(req)
self.substrate.preflight()
render_dir = Path(tempfile.mkdtemp(prefix=f"hrender_{label}_"))
render_opencode_tree(install.staging_dir, render_dir, _render_ctx(self.domain_of(task_id)))
env = {
**os.environ,
"TASK_IDS": task_id,
"LABEL": label,
"HARNESS_RENDER_DIR": str(render_dir),
"AGENT_NAME": "",
"TASKS_BASE_DIR": str(self.store),
"OPENCODE_DIR": str(config.OPENCODE_DIR),
"MODEL_ID": config.SOLVER_MODEL_OC,
"TIMEOUT_PER_TASK": str(t),
"MAX_RETRIES": "2",
"MAX_CONCURRENT": "1",
"OPENCODE_EXPERIMENTAL_OUTPUT_TOKEN_MAX": str(OUTPUT_TOKEN_MAX),
}
self._run_runner(OC_RUNNER, env, wall=2 * t + 600, label=label)
with tempfile.TemporaryDirectory(prefix=f"hrender_{label}_") as temporary:
render_dir = Path(temporary)
render_opencode_tree(
install.staging_dir,
render_dir,
_render_ctx(self.domain_of(task_id)),
pinned_policy=install.extras.get("pinned_policy", ""),
)
env = {
**os.environ,
"TASK_IDS": task_id,
"LABEL": label,
"HARNESS_RENDER_DIR": str(render_dir),
policy.ENV: install.extras.get("pinned_policy", ""),
"HARNESS2_POLICY_MODULE": str(Path(policy.__file__).resolve()),
"AGENT_NAME": "",
"TASKS_BASE_DIR": str(self.store),
"OPENCODE_DIR": str(config.OPENCODE_DIR),
"MODEL_ID": config.SOLVER_MODEL_OC,
"TIMEOUT_PER_TASK": str(t),
"MAX_RETRIES": "2",
"MAX_CONCURRENT": "1",
"OPENCODE_EXPERIMENTAL_OUTPUT_TOKEN_MAX": str(OUTPUT_TOKEN_MAX),
}
self._run_runner(OC_RUNNER, env, wall=2 * t + 600, label=label)
return [self._require_artifacts(task_id, label)]

def _launch_codex(self, req: RunRequest, install: Install) -> list[Path]:
Expand All @@ -819,6 +839,9 @@ def _launch_codex(self, req: RunRequest, install: Install) -> list[Path]:
"TASK_IDS": task_id,
"LABEL": label,
"HARNESS_RENDER_DIR": str(install.staging_dir),
policy.ENV: install.extras.get("pinned_policy", ""),
"HARNESS2_POLICY_MODULE": str(Path(policy.__file__).resolve()),
"HARNESS2_POLICY_PYTHON": sys.executable,
"TASKS_BASE_DIR": str(self.store),
"BENCHMARK_MODELS": config.SOLVER_MODEL_CX,
"MAX_CONCURRENT_PER_MODEL": "1",
Expand Down
24 changes: 20 additions & 4 deletions harness2/benchmarks/lab.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,11 @@
from pathlib import Path

from .. import config
from ..core import policy
from ..core import runmark as RM
from ..core import tree as HT
from ..core.spec import BenchPrompts, BenchSpec, resolve_channel_model
from ..install import require_policy_runtime
from ..substrate.base import Install, RunRequest
from ..substrate.codex import CODEX_SURFACE, CodexSubstrate
from ..substrate.opencode import OPENCODE_SURFACE, OpencodeSubstrate
Expand Down Expand Up @@ -589,11 +591,13 @@ def execute(
self, task_id: str, harness_dir: Path | None, run_id: str, *, timeout: int | None = None
) -> Path:
out = self._run_dir(run_id)
if RM.is_complete(out, harness_dir=harness_dir) and self.artifacts_intact(out):
if RM.is_complete(
out, harness_dir=harness_dir, salt=self._policy_salt
) and self.artifacts_intact(out):
self._check_tool_gate(out)
return out
if out.is_dir() and any(out.iterdir()):
reason = RM.why_not(out, harness_dir=harness_dir)
reason = RM.why_not(out, harness_dir=harness_dir, salt=self._policy_salt)
if reason == "reusable":
reason = "output fingerprint missing or changed"
print(
Expand All @@ -619,16 +623,23 @@ def execute(
out,
run_id=run_id,
harness_dir=harness_dir,
salt=self._policy_salt,
extra={"output_fingerprint": self._output_fingerprint(out)},
)
return out

@property
def _policy_salt(self) -> str:
return policy.digest(self.spec.pinned_policy)

def _install(self, harness_dir: Path | None) -> Install:
"""BASE (harness_dir=None) is a value: fingerprint 'base', and the launcher decides
what 'base' means on its substrate."""
if harness_dir is None:
return Install(
staging_dir=self.base_harness(), fingerprint="base", extras={"harness": "base"}
staging_dir=self.base_harness(),
fingerprint=RM.fingerprint(None, self._policy_salt),
extras={"harness": "base", "pinned_policy": self.spec.pinned_policy},
)
inst = self.substrate.materialise(Path(harness_dir), self.spec, {})
return Install(
Expand All @@ -638,6 +649,8 @@ def _install(self, harness_dir: Path | None) -> Install:
)

def _launch_oc(self, req: RunRequest, install: Install) -> list[Path]:
if policy.block(install.extras.get("pinned_policy", "")):
require_policy_runtime("harvey-labs", "oc")
runner = config.lab_repo() / "opencode" / "run_opencode.py"
_require_runner(runner)
dirs = []
Expand Down Expand Up @@ -673,7 +686,7 @@ def _launch_oc(self, req: RunRequest, install: Install) -> list[Path]:
run_detached(
cmd,
cwd=config.lab_repo(),
env=_lab_env(),
env={**_lab_env(), policy.ENV: install.extras.get("pinned_policy", "")},
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
Expand All @@ -685,6 +698,8 @@ def _launch_oc(self, req: RunRequest, install: Install) -> list[Path]:
return dirs

def _launch_cx(self, req: RunRequest, install: Install) -> list[Path]:
if policy.block(install.extras.get("pinned_policy", "")):
require_policy_runtime("harvey-labs", "cx")
runner = config.lab_repo() / "external" / "adapt_runs_cx" / "run_codex_lab.py"
_require_runner(runner)
dirs = []
Expand All @@ -709,6 +724,7 @@ def _launch_cx(self, req: RunRequest, install: Install) -> list[Path]:

env = {
**_lab_env(),
policy.ENV: install.extras.get("pinned_policy", ""),
"ADAPT_LABCX_CODEX_BIN": str(Path(config.CODEX_DIST) / "codex"),
"ADAPT_LABCX_RESULTS": str(self._results_root),
"ADAPT_LABCX_SCRATCH": os.environ.get("ADAPT_LABCX_SCRATCH")
Expand Down
Loading
Loading