From 8445bda8288f25e0b9fd325247b9a04941fae294 Mon Sep 17 00:00:00 2001 From: Trecek Date: Thu, 22 Oct 2026 23:20:34 -0700 Subject: [PATCH 01/12] fix(e2e): retain commissioning evidence for every outcome --- scripts/e2e/README.md | 58 +++-- scripts/e2e/e2e_harness.py | 191 ++++++++++---- tests/infra/test_e2e_harness.py | 447 +++++++++++++++++++++++++++----- 3 files changed, 573 insertions(+), 123 deletions(-) diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md index da923f52bd..cc157b6fa5 100644 --- a/scripts/e2e/README.md +++ b/scripts/e2e/README.md @@ -224,18 +224,36 @@ For model tests, the container's AutoSkillit data directory (`~/.local/share/aut from the runner, so session logs survive a killed container; the harness also reads this log tree after fleet runs. Those files belong to uid 1000 and are often `0600`, so a second container run copies `out/` and `data/logs/` into -`upload/` world-readable with both secrets replaced by `[REDACTED]`, skips symlinks, and -fails if any secret survives. Only `upload/` is uploaded, and only when redaction -succeeded. - -When a test does not pass, the harness preserves plans from each pipeline clone under -`out/pipeline-artifacts///` and Claude Code JSONL under -`out/transcripts/`, retaining relative paths. The transcripts include the orchestrator, -skill sessions and their subagents; the orchestrator's transcript holds each raw -`bind_plan_set` response and its `plan_parts` argument. These files go through the same -redaction and upload steps as the other output artifacts. Collection also runs for an -expected failure or a harness exception. A container killed before or during collection -may leave no evidence or only partial evidence. +`upload/` world-readable, skipping symlinks. The workflow supplies `MINIMAX_API_KEY` and +`E2E_SANDBOX_TOKEN`; the redactor replaces their nonempty literal byte values in file +contents with `[REDACTED]`, re-reads all uploaded file contents, and fails closed if a +supplied literal survives. Encoded or transformed values, unknown credentials and file +names are outside this check. Only `upload/` is uploaded, and only when redaction succeeded. + +Every test copies the stage directories selected by `PIPELINE_EVIDENCE` in +`e2e_harness.py`, on every outcome: pass, failure, expected failure or harness exception. +Sources are the `.autoskillit/temp` of the sandbox, each pipeline clone and each surviving +worktree. They land under `out/pipeline-artifacts///`. +Ledgers, kitchen and guard state, counters and the worktrees sidecar are not copied. +The `pipeline_files` evidence counter records copied stage files. + +Claude Code `*.jsonl` and `*.txt` files under the projects tree land in +`out/transcripts/`, retaining relative paths for orchestrator, skill and subagent sessions. +The text-file selection includes `tool-results/*.txt`: a transcript's `bind_plan_set` +response may be a pointer to one of these retained payloads rather than inline bytes. +All collected files go through the same redaction and upload steps as other artifacts. + +`evidence.unretained` lists existing files under any `.autoskillit/temp` mentioned by a +`sessions.jsonl` row that were not copied. An entry for a commissioning stage means that +stage's bytes are missing. No `evidence` key means nothing was found to collect. +Uploaded names escape `~` and upload-forbidden characters as `~XX`, so `sha256:` becomes +`sha256~3a`; paths quoted inside session logs and transcripts keep their original characters. + +A merged run's worktree temp is deleted by `merge_worktree`. The implement step's binding +digest, id, `sealed` state and coverage status remain in that session's +`verified_plan_set_preflight` in `logs/sessions.jsonl`. Only the final review iteration's +files survive `pre_review_cleanup`. A container killed before or during collection may +leave no evidence or only partial evidence. `clean-install` mounts no home directories and retains evidence under `out/`. Its redactor receives the same required `--secret-env` names with variables unset and skips the absent @@ -381,13 +399,19 @@ After the plan-set changes have merged, perform this check before CI commissioni and the evidence collector. Build and run that revision with the Docker invocation above, selecting `--test implementation`. Use the same revision for commissioning and record the model. -2. If the run reaches implementation, the first bind succeeded. If the test does not - pass, inspect the orchestrator's `bind_plan_set` results in `out/transcripts/`. +2. Inspect the orchestrator's `bind_plan_set` results in `out/transcripts/` regardless of + outcome, including `tool-results/` when a transcript holds only a pointer. Inspect the + sealed records under + `out/pipeline-artifacts/autoskillit-runs//plan-set-authority/`. An observed `success: true` on the first or second bind is the pass condition. A `coverage_failed` result explains the rejection in `coverage_gaps`, and - `out/pipeline-artifacts/` retains the rejected plan. Missing evidence is inconclusive - and does not satisfy this gate: require an observed successful bind. When a run - replans, also confirm the second bind's `plan_parts` names only the replacement plan. + `out/pipeline-artifacts/` retains the rejected plan. Treat an `evidence.unretained` + entry for a commissioning stage as missing evidence. An empty list covers existing + session-referenced temp files only; unmentioned producers depend on `PIPELINE_EVIDENCE`, + so inspect each required commissioning payload directly before signing off. + Missing evidence is inconclusive and does not satisfy this gate: require an observed + successful bind. When a run replans, also confirm the second bind's `plan_parts` names + only the replacement plan. 3. Record the model, plan path and bind result in issue #5259's body using `gh issue edit --body-file`, never as a comment. Only an observed successful bind unblocks `gh workflow run e2e.yml -f tests=implementation`. diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index 15e810040d..1f7271a13f 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -40,6 +40,32 @@ CONFIG_TEMPLATE = Path(__file__).with_name("autoskillit-config.yaml") SANDBOX_CLONE = Path("/workspace/sandbox") PIPELINE_RUNS_DIR = "autoskillit-runs" +PIPELINE_WORKTREES_DIR = "worktrees" +# Select stage artifacts by directory and file name; runtime state stays behind. +PIPELINE_EVIDENCE: tuple[tuple[str, str], ...] = ( + ("make-plan", "*"), + ("rectify", "*"), + ("plan-set-authority", "*"), + ("audit-admission", "*.json"), # ledger.sqlite3 shares this directory + ("audit-disposition", "*.json"), + ("audit-impl", "*"), + ("review-approach", "*"), + ("arch-lens-*", "*"), + ("prepare-pr", "*"), + ("compose-pr", "*"), + ("review-pr", "*"), + ("resolve-review", "*"), + ("diagnose-ci", "*"), + ("resolve-failures", "*"), + ("resolve-merge-conflicts", "*"), + ("retry-worktree", "*"), +) +# Rejected in uploaded paths by actions/upload-artifact v4.6.2 (e2e.yml). +UPLOAD_FORBIDDEN_CHARACTERS = '":<>|*?\r\n' +_UPLOAD_ESCAPES = str.maketrans({c: f"~{ord(c):02x}" for c in "~" + UPLOAD_FORBIDDEN_CHARACTERS}) +_TEMP_PATH = re.compile( + r"(?:/[^\s\"'`\\<>|*?,;()\[\]{}]*)?\.autoskillit/temp/[^\s\"'`\\<>|*?,;()\[\]{}]+" +) _SCRUBBED_ENV = frozenset({"CLAUDE_CODE_OAUTH_TOKEN", *SECRET_ENV}) _STDERR_TAIL_CHARS = 2000 @@ -1105,27 +1131,31 @@ def _run_test( token = env["E2E_SANDBOX_TOKEN"] recipe_env = { **scrubbed, - "AUTOSKILLIT_LOG_DIR": str(home / ".local" / "share" / "autoskillit" / "logs"), + "AUTOSKILLIT_LOG_DIR": str(autoskillit_log_dir(home)), } return run_recipe(test, catalog.sandbox_repository, token, out, recipe_env, runner) -def _copy_failure_files( +def autoskillit_log_dir(home: Path) -> Path: + return home / ".local" / "share" / "autoskillit" / "logs" + + +def _copy_evidence_files( source: Path, dest: Path, boundary: Path, pattern: str, errors: list[str] -) -> int: +) -> list[Path]: relative = source.relative_to(boundary) try: if any( boundary.joinpath(*relative.parts[:depth]).is_symlink() for depth in range(len(relative.parts) + 1) ): - return 0 + return [] if not source.exists(): - return 0 + return [] except OSError as exc: errors.append(f"{source}: {exc}") - return 0 - copied = 0 + return [] + copied = [] for root, dirs, files in os.walk( source, followlinks=False, onerror=lambda exc: errors.append(f"{source}: {exc}") ): @@ -1138,37 +1168,103 @@ def _copy_failure_files( target = dest / path.relative_to(source) target.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(path, target) - copied += 1 + copied.append(path) except OSError as exc: errors.append(f"{path}: {exc}") return copied -def collect_failure_evidence(out: Path, home: Path) -> dict[str, Any]: - """Keep pipeline plans and Claude transcripts for an unsuccessful E2E run.""" - errors: list[str] = [] +def _evidence_roots(errors: list[str]) -> list[Path]: runs = SANDBOX_CLONE.parent / PIPELINE_RUNS_DIR + roots = [SANDBOX_CLONE] + for parent, skip in ((runs, PIPELINE_WORKTREES_DIR), (runs / PIPELINE_WORKTREES_DIR, None)): + try: + linked = runs.is_symlink() or parent.is_symlink() + entries = [] if linked else sorted(parent.iterdir()) + except FileNotFoundError: + entries = [] + except OSError as exc: + errors.append(f"{parent}: {exc}") + entries = [] + roots += [entry for entry in entries if entry.name != skip] + return roots + + +def _collect_stage_files(out: Path, roots: list[Path], errors: list[str]) -> list[Path]: + workspace = SANDBOX_CLONE.parent + copied = [] + for root in roots: + temp = root / ".autoskillit" / "temp" + try: + # iterdir propagates scan errors instead of hiding unreadable directories. + entries = sorted(temp.iterdir()) + except FileNotFoundError: + continue + except OSError as exc: + errors.append(f"{temp}: {exc}") + continue + for entry in entries: + for stage_pattern, file_pattern in PIPELINE_EVIDENCE: + if fnmatch.fnmatch(entry.name, stage_pattern): + copied.extend( + _copy_evidence_files( + entry, + out / "pipeline-artifacts" / root.relative_to(workspace) / entry.name, + workspace, + file_pattern, + errors, + ) + ) + return copied + + +def _recorded_temp_paths(line: str) -> list[Path]: + try: + row = json.loads(line) + except json.JSONDecodeError: + return [] + cwd = row.get("cwd") if isinstance(row, dict) else None + paths = [] + for match in _TEMP_PATH.finditer(line): + path = Path(match.group()) + if path.is_absolute(): + paths.append(path) + elif isinstance(cwd, str): + paths.append(Path(cwd) / path) + return paths + + +def _unretained_references(index: Path, retained: set[str]) -> list[str]: try: - clones = [] if runs.is_symlink() else sorted(runs.iterdir()) + lines = index.read_text(encoding="utf-8").splitlines() except FileNotFoundError: - clones = [] - except OSError as exc: - errors.append(f"{runs}: {exc}") - clones = [] - plan_files = 0 - for clone in clones: - for kind in ("make-plan", "rectify"): - plan_files += _copy_failure_files( - clone / ".autoskillit" / "temp" / kind, - out / "pipeline-artifacts" / clone.name / kind, - clone, - "*", - errors, - ) - transcripts = _copy_failure_files( - home / ".claude" / "projects", out / "transcripts", home, "*.jsonl", errors + return [] + return sorted( + { + str(path) + for line in lines + for path in _recorded_temp_paths(line) + if path.is_file() and os.path.realpath(path) not in retained + } + ) + + +def collect_run_evidence(out: Path, home: Path) -> dict[str, Any]: + """Copy every run's stage artifacts and Claude transcripts into *out*, whatever the verdict.""" + errors: list[str] = [] + copied = _collect_stage_files(out, _evidence_roots(errors), errors) + projects = home / ".claude" / "projects" + transcripts = _copy_evidence_files(projects, out / "transcripts", home, "*.jsonl", errors) + transcripts += _copy_evidence_files(projects, out / "transcripts", home, "*.txt", errors) + unretained = _unretained_references( + autoskillit_log_dir(home) / "sessions.jsonl", {os.path.realpath(p) for p in copied} ) - return {"plan_files": plan_files, "transcripts": transcripts, "errors": errors} + return { + "pipeline_files": len(copied), + "transcripts": len(transcripts), + "unretained": unretained, + "errors": errors, + } def run_test( @@ -1197,18 +1293,18 @@ def run_test( result.update( outcome=outcome, passed=outcome == "passed", expected_findings=matched_findings ) - if not result["passed"]: - try: - evidence = collect_failure_evidence(out, home) - except Exception as exc: - evidence = { - "plan_files": 0, - "transcripts": 0, - "errors": [f"{type(exc).__name__}: {exc}"], - "exception": traceback.format_exc(), - } - if any(evidence.values()): - result["evidence"] = evidence + try: + evidence = collect_run_evidence(out, home) + except Exception as exc: + evidence = { + "pipeline_files": 0, + "transcripts": 0, + "unretained": [], + "errors": [f"{type(exc).__name__}: {exc}"], + "exception": traceback.format_exc(), + } + if any(evidence.values()): + result["evidence"] = evidence (out / "result.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") return failures @@ -1224,16 +1320,23 @@ def _copy_redacted(source: Path, target: Path, needles: Sequence[bytes]) -> None target.chmod(0o644) +def upload_name(name: str) -> str: + """Escape ``~`` and upload-forbidden characters as ``~XX`` without name collisions.""" + return name.translate(_UPLOAD_ESCAPES) + + def _copy_tree_redacted(source: Path, target: Path, needles: Sequence[bytes]) -> None: for root, _dirs, files in os.walk(source, followlinks=False): root_path = Path(root) - target_dir = target / root_path.relative_to(source) + target_dir = target.joinpath( + *(upload_name(part) for part in root_path.relative_to(source).parts) + ) target_dir.mkdir(parents=True, exist_ok=True) target_dir.chmod(0o755) for name in files: path = root_path / name if not path.is_symlink() and path.is_file(): - _copy_redacted(path, target_dir / name, needles) + _copy_redacted(path, target_dir / upload_name(name), needles) def _find_leak(dest: Path, needles: Sequence[bytes]) -> Path | None: @@ -1257,7 +1360,7 @@ def redact_tree(sources: Sequence[Path], dest: Path, secrets: Sequence[str]) -> for source in sources: if source.is_symlink() or not source.exists(): continue - target = dest / source.name + target = dest / upload_name(source.name) if source.is_dir(): _copy_tree_redacted(source, target, needles) else: diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index c2236f0ef3..59d4aaf5fe 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -32,29 +32,129 @@ def _isolated_sandbox(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @pytest.fixture -def failure_sources(tmp_path: Path) -> dict[Path, Path]: - runs = harness.SANDBOX_CLONE.parent / harness.PIPELINE_RUNS_DIR - project = tmp_path / "home" / ".claude" / "projects" / "project" - sources = { - runs / "clone-a/.autoskillit/temp/make-plan/a_plan.md": Path( - "pipeline-artifacts/clone-a/make-plan/a_plan.md" - ), - runs / "clone-b/.autoskillit/temp/rectify/b_plan.md": Path( - "pipeline-artifacts/clone-b/rectify/b_plan.md" - ), - runs / "clone-a/.autoskillit/temp/make-plan/nested/metadata.json": Path( - "pipeline-artifacts/clone-a/make-plan/nested/metadata.json" +def evidence_sources(tmp_path: Path) -> tuple[dict[Path, Path], dict[Path, Path]]: + workspace = harness.SANDBOX_CLONE.parent + runs = workspace / "autoskillit-runs" + kept: dict[Path, Path] = {} + excluded: dict[Path, Path] = {} + + def add_pipeline_files( + root: Path, + artifact_root: Path, + relative_paths: tuple[str, ...], + destination: dict[Path, Path], + ) -> None: + for value in relative_paths: + relative = Path(value) + source = root / ".autoskillit" / "temp" / relative + destination[source] = Path("pipeline-artifacts") / artifact_root / relative + source.parent.mkdir(parents=True, exist_ok=True) + source.write_bytes(f"evidence for {relative}\n".encode()) + + audit_slot = f"sha256:{'a' * 64}" + audit_attempt = "b" * 32 + plan_set_file = "0123456789abcdef01234567.json" + worktree_plan_set_file = "fedcba9876543210fedcba98.json" + clone_a_artifact_root = Path("autoskillit-runs/clone-a") + clone_a = workspace / clone_a_artifact_root + add_pipeline_files( + harness.SANDBOX_CLONE, + Path("sandbox"), + ("audit-disposition/a1/p1/disposition-report.json",), + kept, + ) + add_pipeline_files( + clone_a, + clone_a_artifact_root, + ( + "make-plan/a_plan.md", + "make-plan/nested/metadata.json", + f"plan-set-authority/planset-x/{plan_set_file}", + f"plan-set-authority/planset-x/issue.{plan_set_file.removesuffix('.json')}.md", + f"audit-admission/{audit_slot}/{audit_attempt}/semantic.json", + f"audit-admission/{audit_slot}/{audit_attempt}/inventory.json", + f"audit-admission/{audit_slot}/{audit_attempt}/authority.json", + "arch-lens-concurrency/arch_diag_x.md", + "review-pr/iter_1/local_findings_26.json", + "resolve-review/deferred_observations_26.json", + "prepare-pr/pr_prep_x.md", + "compose-pr/pr_body_x.md", ), - project / "session.jsonl": Path("transcripts/project/session.jsonl"), - project / "session/subagents/agent-one.jsonl": Path( - "transcripts/project/session/subagents/agent-one.jsonl" + kept, + ) + add_pipeline_files( + workspace / "autoskillit-runs/clone-b", + Path("autoskillit-runs/clone-b"), + ("rectify/b_plan.md",), + kept, + ) + add_pipeline_files( + runs / "worktrees/impl-1-x", + Path("autoskillit-runs/worktrees/impl-1-x"), + (f"plan-set-authority/planset-y/{worktree_plan_set_file}",), + kept, + ) + + add_pipeline_files( + clone_a, + clone_a_artifact_root, + ( + "audit-admission/ledger.sqlite3", + "context-admission/ledger.sqlite3", + "kitchen_state/state.json", + "guard_decisions.jsonl", + "eject_count", + "worktrees/impl-1-x/base-branch", ), - } - for source, relative in sources.items(): + excluded, + ) + add_pipeline_files( + harness.SANDBOX_CLONE, + Path("sandbox"), + ("audit-admission/ledger.sqlite3",), + excluded, + ) + + home = tmp_path / "home" + projects = home / ".claude" / "projects" + for relative_value in ( + "project/session.jsonl", + "project/session/subagents/agent-one.jsonl", + "project/session/tool-results/call_1.txt", + ): + relative = Path(relative_value) + source = projects / relative + kept[source] = Path("transcripts") / relative source.parent.mkdir(parents=True, exist_ok=True) source.write_bytes(f"evidence for {relative}\n".encode()) - (project / "ignored.txt").write_text("not a transcript") - return sources + ignored = projects / "project/ignored.md" + excluded[ignored] = Path("transcripts/project/ignored.md") + ignored.parent.mkdir(parents=True, exist_ok=True) + ignored.write_text("not a transcript", encoding="utf-8") + + linked_temp = runs / "worktrees/impl-2-x/.autoskillit/temp" + linked_temp.parent.mkdir(parents=True, exist_ok=True) + linked_temp.symlink_to(clone_a / ".autoskillit" / "temp", target_is_directory=True) + excluded[linked_temp] = Path("pipeline-artifacts/autoskillit-runs/worktrees/impl-2-x") + return kept, excluded + + +def _pipeline_file_count(sources: dict[Path, Path]) -> int: + return sum(relative.parts[0] == "pipeline-artifacts" for relative in sources.values()) + + +def _assert_evidence_sources( + out: Path, kept: dict[Path, Path], excluded: dict[Path, Path] +) -> None: + for source, relative in kept.items(): + assert (out / relative).read_bytes() == source.read_bytes() + for relative in excluded.values(): + assert not (out / relative).exists() + + +def _uploaded_path(upload: Path, relative: Path) -> Path: + escaped = Path(*(harness.upload_name(part) for part in relative.parts)) + return upload / "out" / escaped @pytest.mark.parametrize( @@ -66,9 +166,11 @@ def failure_sources(tmp_path: Path) -> dict[Path, Path]: ("passed", "passed", True), ), ) -def test_run_test_retains_evidence_only_when_it_does_not_pass( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure_sources, case, outcome, passed +def test_run_test_retains_evidence_for_every_outcome( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, evidence_sources, case, outcome, passed ) -> None: + kept, excluded = evidence_sources + def run(*_args, **_kwargs): if case == "exception": raise RuntimeError("harness crashed") @@ -83,22 +185,31 @@ def run(*_args, **_kwargs): assert result["passed"] is passed assert result["outcome"] == outcome assert result["failures"] == failures - if passed: - assert "evidence" not in result - assert not (out / "pipeline-artifacts").exists() - assert not (out / "transcripts").exists() - return - assert result["evidence"] == {"plan_files": 3, "transcripts": 2, "errors": []} - for source, relative in failure_sources.items(): - assert (out / relative).read_bytes() == source.read_bytes() - assert not (out / "transcripts/project/ignored.txt").exists() + assert result["evidence"] == { + "pipeline_files": _pipeline_file_count(kept), + "transcripts": 3, + "unretained": [], + "errors": [], + } + _assert_evidence_sources(out, kept, excluded) @pytest.mark.parametrize("failure_site", ("copy", "walk")) +@pytest.mark.parametrize( + ("stub_failures", "expected_outcome", "expected_passed"), + ((["recipe failed"], "failed", False), ([], "passed", True)), +) def test_evidence_errors_preserve_the_verdict_and_continue_other_sources( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure_sources, failure_site + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + evidence_sources, + failure_site, + stub_failures, + expected_outcome, + expected_passed, ) -> None: - source = next(path for path in failure_sources if path.name == "a_plan.md") + kept, _ = evidence_sources + source = next(path for path in kept if path.name == "a_plan.md") target = source if failure_site == "copy" else source.parent owner = harness.shutil if failure_site == "copy" else harness.os name = "copyfile" if failure_site == "copy" else "scandir" @@ -110,19 +221,28 @@ def unreadable(path, *args, **kwargs): return original(path, *args, **kwargs) monkeypatch.setattr(owner, name, unreadable) - monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (["test failed"], [])) + monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (stub_failures, [])) failures, result = _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) - assert failures == ["test failed"] - assert result["outcome"] == "failed" - assert result["passed"] is False - assert result["evidence"]["transcripts"] == 2 + assert failures == stub_failures + assert result["outcome"] == expected_outcome + assert result["passed"] is expected_passed + assert result["evidence"]["transcripts"] == 3 assert any(str(target) in error for error in result["evidence"]["errors"]) - readable = next(path for path in failure_sources if path.name == "b_plan.md") - assert (tmp_path / "out" / failure_sources[readable]).read_bytes() == readable.read_bytes() + readable = next(path for path in kept if path.name == "b_plan.md") + relative = kept[readable] + assert (tmp_path / "out" / relative).read_bytes() == readable.read_bytes() +@pytest.mark.parametrize( + ("stub_failures", "expected_outcome", "expected_passed"), + ((["recipe failed"], "failed", False), ([], "passed", True)), +) def test_unexpected_collection_error_still_writes_the_result( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + stub_failures, + expected_outcome, + expected_passed, ) -> None: def crash(*_args, **_kwargs): try: @@ -130,28 +250,48 @@ def crash(*_args, **_kwargs): except ValueError as exc: raise RuntimeError("collection crashed") from exc - monkeypatch.setattr(harness, "collect_failure_evidence", crash) - monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (["test failed"], [])) + monkeypatch.setattr(harness, "collect_run_evidence", crash) + monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (stub_failures, [])) failures, result = _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) - assert failures == ["test failed"] - assert result["outcome"] == "failed" - assert result["evidence"]["plan_files"] == result["evidence"]["transcripts"] == 0 + assert failures == stub_failures + assert result["outcome"] == expected_outcome + assert result["passed"] is expected_passed + assert set(result["evidence"]) == { + "pipeline_files", + "transcripts", + "unretained", + "errors", + "exception", + } + assert result["evidence"]["pipeline_files"] == result["evidence"]["transcripts"] == 0 + assert result["evidence"]["unretained"] == [] assert "collection crashed" in result["evidence"]["errors"][0] assert "ValueError: collection root cause" in result["evidence"]["exception"] assert "RuntimeError: collection crashed" in result["evidence"]["exception"] assert 'raise RuntimeError("collection crashed") from exc' in result["evidence"]["exception"] +@pytest.mark.parametrize( + ("stub_failures", "expected_outcome", "expected_passed"), + ((["recipe failed"], "failed", False), ([], "passed", True)), +) def test_nothing_to_collect_adds_no_evidence_key( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + stub_failures, + expected_outcome, + expected_passed, ) -> None: - monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (["test failed"], [])) - _, result = _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) + monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (stub_failures, [])) + failures, result = _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) + assert failures == stub_failures + assert result["outcome"] == expected_outcome + assert result["passed"] is expected_passed assert "evidence" not in result -def test_failure_evidence_skips_symlinks_and_linked_scan_ancestors(tmp_path: Path) -> None: - runs = harness.SANDBOX_CLONE.parent / harness.PIPELINE_RUNS_DIR +def test_evidence_skips_symlinks_and_linked_scan_ancestors(tmp_path: Path) -> None: + runs = harness.SANDBOX_CLONE.parent / "autoskillit-runs" plans = runs / "clone-a/.autoskillit/temp/make-plan" plans.mkdir(parents=True) (plans / "kept.md").write_text("kept") @@ -164,6 +304,7 @@ def test_failure_evidence_skips_symlinks_and_linked_scan_ancestors(tmp_path: Pat ancestor.mkdir(parents=True) (ancestor / "temp").symlink_to(external, target_is_directory=True) (runs / "linked-clone").symlink_to(runs / "clone-a", target_is_directory=True) + (runs / "worktrees").symlink_to(external, target_is_directory=True) home = tmp_path / "home" home.mkdir() (external / "projects").mkdir() @@ -171,37 +312,207 @@ def test_failure_evidence_skips_symlinks_and_linked_scan_ancestors(tmp_path: Pat (home / ".claude").symlink_to(external, target_is_directory=True) out = tmp_path / "out" - result = harness.collect_failure_evidence(out, home) + result = harness.collect_run_evidence(out, home) - assert result == {"plan_files": 1, "transcripts": 0, "errors": []} + assert result == { + "pipeline_files": 1, + "transcripts": 0, + "unretained": [], + "errors": [], + } assert [path.relative_to(out) for path in out.rglob("*") if path.is_file()] == [ - Path("pipeline-artifacts/clone-a/make-plan/kept.md") + Path("pipeline-artifacts/autoskillit-runs/clone-a/make-plan/kept.md") + ] + + +def test_passing_pipeline_recipe_retains_commissioning_evidence( + tmp_path: Path, evidence_sources +) -> None: + kept, excluded = evidence_sources + out = tmp_path / "out" + by_destination = {relative: source for source, relative in kept.items()} + semantic = Path( + "pipeline-artifacts/autoskillit-runs/clone-a/audit-admission/" + f"sha256:{'a' * 64}/{'b' * 32}/semantic.json" + ) + review = Path( + "pipeline-artifacts/autoskillit-runs/clone-a/review-pr/iter_1/local_findings_26.json" + ) + transcript = Path("transcripts/project/session.jsonl") + tool_results = Path("transcripts/project/session/tool-results/call_1.txt") + secrets = (semantic, review, transcript, tool_results) + for relative in secrets: + source = by_destination[relative] + source.write_text(f"{MINIMAX_KEY} and {SANDBOX_TOKEN}", encoding="utf-8") + + clone_a = harness.SANDBOX_CLONE.parent / "autoskillit-runs/clone-a" + plan = by_destination[Path("pipeline-artifacts/autoskillit-runs/clone-a/make-plan/a_plan.md")] + plan_set = by_destination[ + Path( + "pipeline-artifacts/autoskillit-runs/clone-a/plan-set-authority/" + "planset-x/0123456789abcdef01234567.json" + ) ] + def seed(argv: list[str]) -> None: + if argv[:3] != ["autoskillit", "fleet", "run"]: + return None + log_dir = _log_dir(tmp_path) + log_dir.mkdir(parents=True, exist_ok=True) + rows = ( + { + "cwd": str(clone_a), + "outcome_fields": { + "plan_path": str(plan), + "audit_semantic_result_path": str(by_destination[semantic]), + }, + "skill_command": f"read {plan_set}\ncontinue", + }, + { + "cwd": str(clone_a), + "outcome_fields": { + "deferred_observations_path": ( + ".autoskillit/temp/resolve-review/deferred_observations_26.json" + ) + }, + }, + ) + with (log_dir / "sessions.jsonl").open("a", encoding="utf-8") as index: + for row in rows: + index.write(json.dumps(row) + "\n") + return None + + runner = FakeRunner(_recipe_handler(log_dir=_log_dir(tmp_path), extra=seed)) + failures, result = _run_recipe(tmp_path, runner) + + assert failures == [] + assert result["outcome"] == "passed" + assert result["passed"] is True + assert result["evidence"] == { + "pipeline_files": _pipeline_file_count(kept), + "transcripts": 3, + "unretained": [], + "errors": [], + } + _assert_evidence_sources(out, kept, excluded) + upload = tmp_path / "upload" + harness.redact_tree([out], upload, [MINIMAX_KEY, SANDBOX_TOKEN]) + semantic_upload = _uploaded_path(upload, semantic) + assert semantic_upload == ( + upload + / "out" + / "pipeline-artifacts/autoskillit-runs/clone-a/audit-admission" + / f"sha256~3a{'a' * 64}" + / ("b" * 32) + / "semantic.json" + ) + for relative in secrets: + copied = _uploaded_path(upload, relative) + content = copied.read_bytes() + assert harness.REDACTED in content + assert MINIMAX_KEY.encode() not in content + assert SANDBOX_TOKEN.encode() not in content + for artifact in upload.rglob("*"): + assert not any( + character in artifact.relative_to(upload).as_posix() + for character in harness.UPLOAD_FORBIDDEN_CHARACTERS + ) + if artifact.is_file(): + content = artifact.read_bytes() + assert MINIMAX_KEY.encode() not in content + assert SANDBOX_TOKEN.encode() not in content + + +def test_unretained_lists_recorded_files_that_were_not_kept( + tmp_path: Path, evidence_sources +) -> None: + kept, _ = evidence_sources + workspace = harness.SANDBOX_CLONE.parent + clone_a = workspace / "autoskillit-runs/clone-a" + new_producer = clone_a / ".autoskillit/temp/new-producer/verdict.json" + cwd_resolved = clone_a / ".autoskillit/temp/new-producer-2/x.json" + new_producer.parent.mkdir(parents=True, exist_ok=True) + cwd_resolved.parent.mkdir(parents=True, exist_ok=True) + new_producer.write_text("new evidence", encoding="utf-8") + cwd_resolved.write_text("new evidence", encoding="utf-8") + + semantic = next( + source + for source, relative in kept.items() + if relative.name == "semantic.json" and "audit-admission" in relative.parts + ) + plan_set = next( + source + for source, relative in kept.items() + if relative.name == "0123456789abcdef01234567.json" + ) + symlinked_plan_set = ( + workspace + / "autoskillit-runs/worktrees/impl-2-x/.autoskillit/temp" + / plan_set.relative_to(clone_a / ".autoskillit/temp") + ) + missing = ".autoskillit/temp/not-created/missing.json" + rows = ( + {"outcome_fields": {"remediation_path": str(new_producer)}}, + { + "cwd": str(clone_a), + "outcome_fields": {"remediation_path": ".autoskillit/temp/new-producer-2/x.json"}, + }, + {"outcome_fields": {"audit_semantic_result_path": str(semantic)}}, + {"skill_command": f"read {symlinked_plan_set}"}, + {"cwd": str(clone_a), "outcome_fields": {"remediation_path": missing}}, + ) + home = tmp_path / "home" + index = home / ".local/share/autoskillit/logs/sessions.jsonl" + index.parent.mkdir(parents=True, exist_ok=True) + index.write_text( + "".join(json.dumps(row) + "\n" for row in rows) + "{malformed line\n", + encoding="utf-8", + ) + + evidence = harness.collect_run_evidence(tmp_path / "out", home) + + assert evidence["unretained"] == sorted([str(new_producer), str(cwd_resolved)]) + assert evidence["errors"] == [] + + +def test_upload_name_escapes_forbidden_characters_injectively() -> None: + for character in harness.UPLOAD_FORBIDDEN_CHARACTERS + "~": + assert harness.upload_name(character) == f"~{ord(character):02x}" + assert harness.upload_name("sha256:ab") == "sha256~3aab" + names = {harness.upload_name(name) for name in ("a:b", "a~3ab", "a-b")} + assert len(names) == 3 + assert harness.upload_name("plain-name") == "plain-name" + + +@pytest.mark.parametrize("case", ("failed", "passed")) def test_collected_plans_and_transcripts_are_redacted( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure_sources + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, evidence_sources, case ) -> None: - plan = next(path for path in failure_sources if path.name == "a_plan.md") - transcript = next(path for path in failure_sources if path.name == "session.jsonl") + kept, _ = evidence_sources + plan = next(path for path in kept if path.name == "a_plan.md") + transcript = next(path for path in kept if path.name == "session.jsonl") plan.write_bytes(MINIMAX_KEY.encode()) transcript.write_bytes(SANDBOX_TOKEN.encode()) - monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (["test failed"], [])) + stub_failures = ["test failed"] if case == "failed" else [] + monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (stub_failures, [])) _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) upload = tmp_path / "upload" harness.redact_tree([tmp_path / "out"], upload, [MINIMAX_KEY, SANDBOX_TOKEN]) for source, secret in ((plan, MINIMAX_KEY), (transcript, SANDBOX_TOKEN)): - copied = (upload / "out" / failure_sources[source]).read_bytes() + copied = _uploaded_path(upload, kept[source]).read_bytes() assert harness.REDACTED in copied assert secret.encode() not in copied def test_runs_scan_error_keeps_readable_transcripts( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure_sources + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, evidence_sources ) -> None: - runs = harness.SANDBOX_CLONE.parent / harness.PIPELINE_RUNS_DIR + kept, _ = evidence_sources + runs = harness.SANDBOX_CLONE.parent / "autoskillit-runs" original = Path.iterdir def unreadable(path): @@ -214,8 +525,14 @@ def unreadable(path): failures, result = _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) assert failures == ["test failed"] assert result["outcome"] == "failed" - assert result["evidence"]["plan_files"] == 0 - assert result["evidence"]["transcripts"] == 2 + expected_pipeline_files = sum( + relative.parts[1] == "sandbox" + or relative.parts[1:4] == ("autoskillit-runs", "worktrees", "impl-1-x") + for relative in kept.values() + if relative.parts[0] == "pipeline-artifacts" + ) + assert result["evidence"]["pipeline_files"] == expected_pipeline_files + assert result["evidence"]["transcripts"] == 3 assert str(runs) in result["evidence"]["errors"][0] @@ -225,6 +542,12 @@ def test_pipeline_runs_directory_matches_clone_source_of_truth() -> None: assert harness.PIPELINE_RUNS_DIR == RUNS_DIR +def test_worktrees_directory_matches_clone_source_of_truth() -> None: + from autoskillit.workspace.clone._worktree import WORKTREES_DIR + + assert harness.PIPELINE_WORKTREES_DIR == WORKTREES_DIR + + MINIMAX_KEY = "minimax-test-key" SANDBOX_TOKEN = "sandbox-test-token" SANDBOX = "owner/sandbox" From 41ad51478ab1020e57e04d758fa9233621bef0ad Mon Sep 17 00:00:00 2001 From: Trecek Date: Fri, 23 Oct 2026 07:36:54 -0700 Subject: [PATCH 02/12] fix(review): collect transcript formats in one traversal --- scripts/e2e/e2e_harness.py | 13 ++++++++----- tests/infra/test_e2e_harness.py | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index 1f7271a13f..c45adfd69b 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -1141,7 +1141,7 @@ def autoskillit_log_dir(home: Path) -> Path: def _copy_evidence_files( - source: Path, dest: Path, boundary: Path, pattern: str, errors: list[str] + source: Path, dest: Path, boundary: Path, patterns: tuple[str, ...], errors: list[str] ) -> list[Path]: relative = source.relative_to(boundary) try: @@ -1160,7 +1160,9 @@ def _copy_evidence_files( source, followlinks=False, onerror=lambda exc: errors.append(f"{source}: {exc}") ): dirs.sort() - for name in sorted(fnmatch.filter(files, pattern)): + for name in sorted(files): + if not any(fnmatch.fnmatch(name, pattern) for pattern in patterns): + continue path = Path(root) / name try: if path.is_symlink() or not path.is_file(): @@ -1211,7 +1213,7 @@ def _collect_stage_files(out: Path, roots: list[Path], errors: list[str]) -> lis entry, out / "pipeline-artifacts" / root.relative_to(workspace) / entry.name, workspace, - file_pattern, + (file_pattern,), errors, ) ) @@ -1254,8 +1256,9 @@ def collect_run_evidence(out: Path, home: Path) -> dict[str, Any]: errors: list[str] = [] copied = _collect_stage_files(out, _evidence_roots(errors), errors) projects = home / ".claude" / "projects" - transcripts = _copy_evidence_files(projects, out / "transcripts", home, "*.jsonl", errors) - transcripts += _copy_evidence_files(projects, out / "transcripts", home, "*.txt", errors) + transcripts = _copy_evidence_files( + projects, out / "transcripts", home, ("*.jsonl", "*.txt"), errors + ) unretained = _unretained_references( autoskillit_log_dir(home) / "sessions.jsonl", {os.path.realpath(p) for p in copied} ) diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index 59d4aaf5fe..765ffcb756 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -157,6 +157,27 @@ def _uploaded_path(upload: Path, relative: Path) -> Path: return upload / "out" / escaped +def test_transcripts_are_collected_in_one_walk( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, evidence_sources +) -> None: + home = tmp_path / "home" + projects = home / ".claude" / "projects" + walked = [] + original = harness.os.walk + + def walk(source, *args, **kwargs): + walked.append(source) + return original(source, *args, **kwargs) + + monkeypatch.setattr(harness.os, "walk", walk) + + evidence = harness.collect_run_evidence(tmp_path / "out", home) + + assert walked.count(projects) == 1 + assert evidence["transcripts"] == 3 + assert evidence["errors"] == [] + + @pytest.mark.parametrize( ("case", "outcome", "passed"), ( From a8058f5640f2422d15f78ab3c60f80883728de0d Mon Sep 17 00:00:00 2001 From: Trecek Date: Fri, 23 Oct 2026 07:37:49 -0700 Subject: [PATCH 03/12] fix(review): retain evidence counts when the session index is unreadable --- scripts/e2e/e2e_harness.py | 7 +++++-- tests/infra/test_e2e_harness.py | 35 +++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index c45adfd69b..494c7a3f22 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -1236,11 +1236,14 @@ def _recorded_temp_paths(line: str) -> list[Path]: return paths -def _unretained_references(index: Path, retained: set[str]) -> list[str]: +def _unretained_references(index: Path, retained: set[str], errors: list[str]) -> list[str]: try: lines = index.read_text(encoding="utf-8").splitlines() except FileNotFoundError: return [] + except (OSError, UnicodeDecodeError) as exc: + errors.append(f"{index}: {exc}") + return [] return sorted( { str(path) @@ -1260,7 +1263,7 @@ def collect_run_evidence(out: Path, home: Path) -> dict[str, Any]: projects, out / "transcripts", home, ("*.jsonl", "*.txt"), errors ) unretained = _unretained_references( - autoskillit_log_dir(home) / "sessions.jsonl", {os.path.realpath(p) for p in copied} + autoskillit_log_dir(home) / "sessions.jsonl", {os.path.realpath(p) for p in copied}, errors ) return { "pipeline_files": len(copied), diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index 765ffcb756..59f3d712b1 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -498,6 +498,41 @@ def test_unretained_lists_recorded_files_that_were_not_kept( assert evidence["errors"] == [] +@pytest.mark.parametrize( + "error", + ( + PermissionError("cannot read session index"), + UnicodeDecodeError("utf-8", b"\xff", 0, 1, "invalid start byte"), + ), +) +def test_index_read_error_preserves_collected_evidence( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, evidence_sources, error +) -> None: + kept, excluded = evidence_sources + index = harness.autoskillit_log_dir(tmp_path / "home") / "sessions.jsonl" + original = Path.read_text + + def unreadable(path, *args, **kwargs): + if path == index: + raise error + return original(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", unreadable) + monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: ([], [])) + + failures, result = _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) + + assert failures == [] + assert result["outcome"] == "passed" + assert result["evidence"] == { + "pipeline_files": _pipeline_file_count(kept), + "transcripts": 3, + "unretained": [], + "errors": [f"{index}: {error}"], + } + _assert_evidence_sources(tmp_path / "out", kept, excluded) + + def test_upload_name_escapes_forbidden_characters_injectively() -> None: for character in harness.UPLOAD_FORBIDDEN_CHARACTERS + "~": assert harness.upload_name(character) == f"~{ord(character):02x}" From bf1cafb913982854e4ba43bc7056ca29b70f8bc6 Mon Sep 17 00:00:00 2001 From: Trecek Date: Fri, 23 Oct 2026 07:38:59 -0700 Subject: [PATCH 04/12] fix(review): ignore unrelated files among evidence roots --- scripts/e2e/e2e_harness.py | 4 +++- tests/infra/test_e2e_harness.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index 494c7a3f22..c3cdff3624 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -1182,7 +1182,9 @@ def _evidence_roots(errors: list[str]) -> list[Path]: for parent, skip in ((runs, PIPELINE_WORKTREES_DIR), (runs / PIPELINE_WORKTREES_DIR, None)): try: linked = runs.is_symlink() or parent.is_symlink() - entries = [] if linked else sorted(parent.iterdir()) + entries = ( + [] if linked else sorted(entry for entry in parent.iterdir() if entry.is_dir()) + ) except FileNotFoundError: entries = [] except OSError as exc: diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index 59f3d712b1..3f7525dae1 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -311,6 +311,21 @@ def test_nothing_to_collect_adds_no_evidence_key( assert "evidence" not in result +def test_evidence_ignores_files_in_run_root_directories(tmp_path: Path) -> None: + runs = harness.SANDBOX_CLONE.parent / harness.PIPELINE_RUNS_DIR + worktrees = runs / harness.PIPELINE_WORKTREES_DIR + worktrees.mkdir(parents=True) + (runs / "unrelated.txt").write_text("not a clone") + (worktrees / "unrelated.txt").write_text("not a worktree") + + assert harness.collect_run_evidence(tmp_path / "out", tmp_path / "home") == { + "pipeline_files": 0, + "transcripts": 0, + "unretained": [], + "errors": [], + } + + def test_evidence_skips_symlinks_and_linked_scan_ancestors(tmp_path: Path) -> None: runs = harness.SANDBOX_CLONE.parent / "autoskillit-runs" plans = runs / "clone-a/.autoskillit/temp/make-plan" From b08280a21a68485294e78b65d18839fec7de4916 Mon Sep 17 00:00:00 2001 From: Trecek Date: Fri, 23 Oct 2026 07:39:39 -0700 Subject: [PATCH 05/12] test(review): use public clone directory constants in guards --- tests/infra/test_e2e_harness.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index 3f7525dae1..bc6e4f39b3 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -608,13 +608,13 @@ def unreadable(path): def test_pipeline_runs_directory_matches_clone_source_of_truth() -> None: - from autoskillit.workspace.clone._detect import RUNS_DIR + from autoskillit.workspace.clone import RUNS_DIR assert harness.PIPELINE_RUNS_DIR == RUNS_DIR def test_worktrees_directory_matches_clone_source_of_truth() -> None: - from autoskillit.workspace.clone._worktree import WORKTREES_DIR + from autoskillit.workspace.clone import WORKTREES_DIR assert harness.PIPELINE_WORKTREES_DIR == WORKTREES_DIR From 588a509d6ecc9b244d879b868326dfc7892363ee Mon Sep 17 00:00:00 2001 From: Trecek Date: Fri, 23 Oct 2026 07:40:27 -0700 Subject: [PATCH 06/12] test(review): cover escaping top-level upload source names --- tests/infra/test_e2e_harness.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index bc6e4f39b3..a913a6010b 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -557,6 +557,22 @@ def test_upload_name_escapes_forbidden_characters_injectively() -> None: assert harness.upload_name("plain-name") == "plain-name" +@pytest.mark.parametrize("directory", (False, True)) +def test_redaction_escapes_top_level_source_names(tmp_path: Path, directory: bool) -> None: + source = tmp_path / "out:root~3a" + artifact = source / "log.txt" if directory else source + artifact.parent.mkdir(parents=True, exist_ok=True) + artifact.write_bytes(MINIMAX_KEY.encode()) + upload = tmp_path / "upload" + + harness.redact_tree([source], upload, [MINIMAX_KEY]) + + target = upload / "out~3aroot~7e3a" + copied = target / "log.txt" if directory else target + assert copied.read_bytes() == harness.REDACTED + assert not (upload / source.name).exists() + + @pytest.mark.parametrize("case", ("failed", "passed")) def test_collected_plans_and_transcripts_are_redacted( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, evidence_sources, case From fc61d690953cd35627eaf0a4f265bddaafb71a09 Mon Sep 17 00:00:00 2001 From: Trecek Date: Fri, 23 Oct 2026 07:41:04 -0700 Subject: [PATCH 07/12] test(review): reuse the session log directory helper --- tests/infra/test_e2e_harness.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index a913a6010b..189587cd96 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -500,7 +500,7 @@ def test_unretained_lists_recorded_files_that_were_not_kept( {"cwd": str(clone_a), "outcome_fields": {"remediation_path": missing}}, ) home = tmp_path / "home" - index = home / ".local/share/autoskillit/logs/sessions.jsonl" + index = harness.autoskillit_log_dir(home) / "sessions.jsonl" index.parent.mkdir(parents=True, exist_ok=True) index.write_text( "".join(json.dumps(row) + "\n" for row in rows) + "{malformed line\n", From de1ad7bb7c30e6f34e4043a8b17b10fb89a61e92 Mon Sep 17 00:00:00 2001 From: Trecek Date: Fri, 23 Oct 2026 07:41:35 -0700 Subject: [PATCH 08/12] test(review): remove duplicate per-artifact secret assertions --- tests/infra/test_e2e_harness.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index 189587cd96..3e3a3b0a6d 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -447,8 +447,6 @@ def seed(argv: list[str]) -> None: copied = _uploaded_path(upload, relative) content = copied.read_bytes() assert harness.REDACTED in content - assert MINIMAX_KEY.encode() not in content - assert SANDBOX_TOKEN.encode() not in content for artifact in upload.rglob("*"): assert not any( character in artifact.relative_to(upload).as_posix() From 6277463bffe8ac09f90d41116fd244f246d2c7bf Mon Sep 17 00:00:00 2001 From: Trecek Date: Fri, 23 Oct 2026 07:42:18 -0700 Subject: [PATCH 09/12] test(review): require retained-path session seeding to execute --- tests/infra/test_e2e_harness.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index 3e3a3b0a6d..54828680a7 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -390,7 +390,10 @@ def test_passing_pipeline_recipe_retains_commissioning_evidence( ) ] + seeded = False + def seed(argv: list[str]) -> None: + nonlocal seeded if argv[:3] != ["autoskillit", "fleet", "run"]: return None log_dir = _log_dir(tmp_path) @@ -416,11 +419,13 @@ def seed(argv: list[str]) -> None: with (log_dir / "sessions.jsonl").open("a", encoding="utf-8") as index: for row in rows: index.write(json.dumps(row) + "\n") + seeded = True return None runner = FakeRunner(_recipe_handler(log_dir=_log_dir(tmp_path), extra=seed)) failures, result = _run_recipe(tmp_path, runner) + assert seeded, "the recipe runner must seed the retained-path session rows" assert failures == [] assert result["outcome"] == "passed" assert result["passed"] is True From 47faf3130d08d7a28ced544d6ed739f3acf0b78d Mon Sep 17 00:00:00 2001 From: Trecek Date: Fri, 23 Oct 2026 07:43:08 -0700 Subject: [PATCH 10/12] test(review): share audit and plan-set fixture identities --- tests/infra/test_e2e_harness.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index 54828680a7..8185fd6993 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -25,6 +25,10 @@ harness = load_check_script("_autoskillit_e2e_harness", _HARNESS_SCRIPT) e2e_catalog = harness.e2e_catalog +_AUDIT_SLOT = f"sha256:{'a' * 64}" +_AUDIT_ATTEMPT = "b" * 32 +_PLAN_SET_FILE = "0123456789abcdef01234567.json" + @pytest.fixture(autouse=True) def _isolated_sandbox(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -51,9 +55,6 @@ def add_pipeline_files( source.parent.mkdir(parents=True, exist_ok=True) source.write_bytes(f"evidence for {relative}\n".encode()) - audit_slot = f"sha256:{'a' * 64}" - audit_attempt = "b" * 32 - plan_set_file = "0123456789abcdef01234567.json" worktree_plan_set_file = "fedcba9876543210fedcba98.json" clone_a_artifact_root = Path("autoskillit-runs/clone-a") clone_a = workspace / clone_a_artifact_root @@ -69,11 +70,11 @@ def add_pipeline_files( ( "make-plan/a_plan.md", "make-plan/nested/metadata.json", - f"plan-set-authority/planset-x/{plan_set_file}", - f"plan-set-authority/planset-x/issue.{plan_set_file.removesuffix('.json')}.md", - f"audit-admission/{audit_slot}/{audit_attempt}/semantic.json", - f"audit-admission/{audit_slot}/{audit_attempt}/inventory.json", - f"audit-admission/{audit_slot}/{audit_attempt}/authority.json", + f"plan-set-authority/planset-x/{_PLAN_SET_FILE}", + f"plan-set-authority/planset-x/issue.{_PLAN_SET_FILE.removesuffix('.json')}.md", + f"audit-admission/{_AUDIT_SLOT}/{_AUDIT_ATTEMPT}/semantic.json", + f"audit-admission/{_AUDIT_SLOT}/{_AUDIT_ATTEMPT}/inventory.json", + f"audit-admission/{_AUDIT_SLOT}/{_AUDIT_ATTEMPT}/authority.json", "arch-lens-concurrency/arch_diag_x.md", "review-pr/iter_1/local_findings_26.json", "resolve-review/deferred_observations_26.json", @@ -369,7 +370,7 @@ def test_passing_pipeline_recipe_retains_commissioning_evidence( by_destination = {relative: source for source, relative in kept.items()} semantic = Path( "pipeline-artifacts/autoskillit-runs/clone-a/audit-admission/" - f"sha256:{'a' * 64}/{'b' * 32}/semantic.json" + f"{_AUDIT_SLOT}/{_AUDIT_ATTEMPT}/semantic.json" ) review = Path( "pipeline-artifacts/autoskillit-runs/clone-a/review-pr/iter_1/local_findings_26.json" @@ -386,7 +387,7 @@ def test_passing_pipeline_recipe_retains_commissioning_evidence( plan_set = by_destination[ Path( "pipeline-artifacts/autoskillit-runs/clone-a/plan-set-authority/" - "planset-x/0123456789abcdef01234567.json" + f"planset-x/{_PLAN_SET_FILE}" ) ] @@ -444,8 +445,8 @@ def seed(argv: list[str]) -> None: upload / "out" / "pipeline-artifacts/autoskillit-runs/clone-a/audit-admission" - / f"sha256~3a{'a' * 64}" - / ("b" * 32) + / f"sha256~3a{_AUDIT_SLOT.removeprefix('sha256:')}" + / _AUDIT_ATTEMPT / "semantic.json" ) for relative in secrets: @@ -481,11 +482,7 @@ def test_unretained_lists_recorded_files_that_were_not_kept( for source, relative in kept.items() if relative.name == "semantic.json" and "audit-admission" in relative.parts ) - plan_set = next( - source - for source, relative in kept.items() - if relative.name == "0123456789abcdef01234567.json" - ) + plan_set = next(source for source, relative in kept.items() if relative.name == _PLAN_SET_FILE) symlinked_plan_set = ( workspace / "autoskillit-runs/worktrees/impl-2-x/.autoskillit/temp" From f5e77fddca7c501a79ba83953715d130f9978fcd Mon Sep 17 00:00:00 2001 From: Trecek Date: Fri, 23 Oct 2026 07:43:53 -0700 Subject: [PATCH 11/12] test(review): share repeated recipe outcome cases --- tests/infra/test_e2e_harness.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index 8185fd6993..7ecc934604 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -28,6 +28,7 @@ _AUDIT_SLOT = f"sha256:{'a' * 64}" _AUDIT_ATTEMPT = "b" * 32 _PLAN_SET_FILE = "0123456789abcdef01234567.json" +_RECIPE_OUTCOMES = ((["recipe failed"], "failed", False), ([], "passed", True)) @pytest.fixture(autouse=True) @@ -219,7 +220,7 @@ def run(*_args, **_kwargs): @pytest.mark.parametrize("failure_site", ("copy", "walk")) @pytest.mark.parametrize( ("stub_failures", "expected_outcome", "expected_passed"), - ((["recipe failed"], "failed", False), ([], "passed", True)), + _RECIPE_OUTCOMES, ) def test_evidence_errors_preserve_the_verdict_and_continue_other_sources( tmp_path: Path, @@ -257,7 +258,7 @@ def unreadable(path, *args, **kwargs): @pytest.mark.parametrize( ("stub_failures", "expected_outcome", "expected_passed"), - ((["recipe failed"], "failed", False), ([], "passed", True)), + _RECIPE_OUTCOMES, ) def test_unexpected_collection_error_still_writes_the_result( tmp_path: Path, @@ -295,7 +296,7 @@ def crash(*_args, **_kwargs): @pytest.mark.parametrize( ("stub_failures", "expected_outcome", "expected_passed"), - ((["recipe failed"], "failed", False), ([], "passed", True)), + _RECIPE_OUTCOMES, ) def test_nothing_to_collect_adds_no_evidence_key( tmp_path: Path, From 52e000295d223dc4697f3406d15fdc12ae09bf02 Mon Sep 17 00:00:00 2001 From: Trecek Date: Fri, 23 Oct 2026 07:45:13 -0700 Subject: [PATCH 12/12] test(review): derive evidence roots from harness directory constants --- tests/infra/test_e2e_harness.py | 62 ++++++++++++++++----------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index 7ecc934604..64b7cd26d0 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -39,7 +39,7 @@ def _isolated_sandbox(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @pytest.fixture def evidence_sources(tmp_path: Path) -> tuple[dict[Path, Path], dict[Path, Path]]: workspace = harness.SANDBOX_CLONE.parent - runs = workspace / "autoskillit-runs" + runs = workspace / harness.PIPELINE_RUNS_DIR kept: dict[Path, Path] = {} excluded: dict[Path, Path] = {} @@ -57,7 +57,7 @@ def add_pipeline_files( source.write_bytes(f"evidence for {relative}\n".encode()) worktree_plan_set_file = "fedcba9876543210fedcba98.json" - clone_a_artifact_root = Path("autoskillit-runs/clone-a") + clone_a_artifact_root = Path(harness.PIPELINE_RUNS_DIR) / "clone-a" clone_a = workspace / clone_a_artifact_root add_pipeline_files( harness.SANDBOX_CLONE, @@ -85,14 +85,14 @@ def add_pipeline_files( kept, ) add_pipeline_files( - workspace / "autoskillit-runs/clone-b", - Path("autoskillit-runs/clone-b"), + runs / "clone-b", + Path(harness.PIPELINE_RUNS_DIR) / "clone-b", ("rectify/b_plan.md",), kept, ) add_pipeline_files( - runs / "worktrees/impl-1-x", - Path("autoskillit-runs/worktrees/impl-1-x"), + runs / harness.PIPELINE_WORKTREES_DIR / "impl-1-x", + Path(harness.PIPELINE_RUNS_DIR) / harness.PIPELINE_WORKTREES_DIR / "impl-1-x", (f"plan-set-authority/planset-y/{worktree_plan_set_file}",), kept, ) @@ -134,10 +134,15 @@ def add_pipeline_files( ignored.parent.mkdir(parents=True, exist_ok=True) ignored.write_text("not a transcript", encoding="utf-8") - linked_temp = runs / "worktrees/impl-2-x/.autoskillit/temp" + linked_temp = runs / harness.PIPELINE_WORKTREES_DIR / "impl-2-x/.autoskillit/temp" linked_temp.parent.mkdir(parents=True, exist_ok=True) linked_temp.symlink_to(clone_a / ".autoskillit" / "temp", target_is_directory=True) - excluded[linked_temp] = Path("pipeline-artifacts/autoskillit-runs/worktrees/impl-2-x") + excluded[linked_temp] = ( + Path("pipeline-artifacts") + / harness.PIPELINE_RUNS_DIR + / harness.PIPELINE_WORKTREES_DIR + / "impl-2-x" + ) return kept, excluded @@ -329,7 +334,7 @@ def test_evidence_ignores_files_in_run_root_directories(tmp_path: Path) -> None: def test_evidence_skips_symlinks_and_linked_scan_ancestors(tmp_path: Path) -> None: - runs = harness.SANDBOX_CLONE.parent / "autoskillit-runs" + runs = harness.SANDBOX_CLONE.parent / harness.PIPELINE_RUNS_DIR plans = runs / "clone-a/.autoskillit/temp/make-plan" plans.mkdir(parents=True) (plans / "kept.md").write_text("kept") @@ -342,7 +347,7 @@ def test_evidence_skips_symlinks_and_linked_scan_ancestors(tmp_path: Path) -> No ancestor.mkdir(parents=True) (ancestor / "temp").symlink_to(external, target_is_directory=True) (runs / "linked-clone").symlink_to(runs / "clone-a", target_is_directory=True) - (runs / "worktrees").symlink_to(external, target_is_directory=True) + (runs / harness.PIPELINE_WORKTREES_DIR).symlink_to(external, target_is_directory=True) home = tmp_path / "home" home.mkdir() (external / "projects").mkdir() @@ -359,7 +364,7 @@ def test_evidence_skips_symlinks_and_linked_scan_ancestors(tmp_path: Path) -> No "errors": [], } assert [path.relative_to(out) for path in out.rglob("*") if path.is_file()] == [ - Path("pipeline-artifacts/autoskillit-runs/clone-a/make-plan/kept.md") + Path("pipeline-artifacts") / harness.PIPELINE_RUNS_DIR / "clone-a/make-plan/kept.md" ] @@ -369,13 +374,9 @@ def test_passing_pipeline_recipe_retains_commissioning_evidence( kept, excluded = evidence_sources out = tmp_path / "out" by_destination = {relative: source for source, relative in kept.items()} - semantic = Path( - "pipeline-artifacts/autoskillit-runs/clone-a/audit-admission/" - f"{_AUDIT_SLOT}/{_AUDIT_ATTEMPT}/semantic.json" - ) - review = Path( - "pipeline-artifacts/autoskillit-runs/clone-a/review-pr/iter_1/local_findings_26.json" - ) + clone_artifacts = Path("pipeline-artifacts") / harness.PIPELINE_RUNS_DIR / "clone-a" + semantic = clone_artifacts / "audit-admission" / _AUDIT_SLOT / _AUDIT_ATTEMPT / "semantic.json" + review = clone_artifacts / "review-pr/iter_1/local_findings_26.json" transcript = Path("transcripts/project/session.jsonl") tool_results = Path("transcripts/project/session/tool-results/call_1.txt") secrets = (semantic, review, transcript, tool_results) @@ -383,14 +384,9 @@ def test_passing_pipeline_recipe_retains_commissioning_evidence( source = by_destination[relative] source.write_text(f"{MINIMAX_KEY} and {SANDBOX_TOKEN}", encoding="utf-8") - clone_a = harness.SANDBOX_CLONE.parent / "autoskillit-runs/clone-a" - plan = by_destination[Path("pipeline-artifacts/autoskillit-runs/clone-a/make-plan/a_plan.md")] - plan_set = by_destination[ - Path( - "pipeline-artifacts/autoskillit-runs/clone-a/plan-set-authority/" - f"planset-x/{_PLAN_SET_FILE}" - ) - ] + clone_a = harness.SANDBOX_CLONE.parent / harness.PIPELINE_RUNS_DIR / "clone-a" + plan = by_destination[clone_artifacts / "make-plan/a_plan.md"] + plan_set = by_destination[clone_artifacts / "plan-set-authority/planset-x" / _PLAN_SET_FILE] seeded = False @@ -445,7 +441,8 @@ def seed(argv: list[str]) -> None: assert semantic_upload == ( upload / "out" - / "pipeline-artifacts/autoskillit-runs/clone-a/audit-admission" + / clone_artifacts + / "audit-admission" / f"sha256~3a{_AUDIT_SLOT.removeprefix('sha256:')}" / _AUDIT_ATTEMPT / "semantic.json" @@ -470,7 +467,7 @@ def test_unretained_lists_recorded_files_that_were_not_kept( ) -> None: kept, _ = evidence_sources workspace = harness.SANDBOX_CLONE.parent - clone_a = workspace / "autoskillit-runs/clone-a" + clone_a = workspace / harness.PIPELINE_RUNS_DIR / "clone-a" new_producer = clone_a / ".autoskillit/temp/new-producer/verdict.json" cwd_resolved = clone_a / ".autoskillit/temp/new-producer-2/x.json" new_producer.parent.mkdir(parents=True, exist_ok=True) @@ -486,7 +483,9 @@ def test_unretained_lists_recorded_files_that_were_not_kept( plan_set = next(source for source, relative in kept.items() if relative.name == _PLAN_SET_FILE) symlinked_plan_set = ( workspace - / "autoskillit-runs/worktrees/impl-2-x/.autoskillit/temp" + / harness.PIPELINE_RUNS_DIR + / harness.PIPELINE_WORKTREES_DIR + / "impl-2-x/.autoskillit/temp" / plan_set.relative_to(clone_a / ".autoskillit/temp") ) missing = ".autoskillit/temp/not-created/missing.json" @@ -600,7 +599,7 @@ def test_runs_scan_error_keeps_readable_transcripts( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, evidence_sources ) -> None: kept, _ = evidence_sources - runs = harness.SANDBOX_CLONE.parent / "autoskillit-runs" + runs = harness.SANDBOX_CLONE.parent / harness.PIPELINE_RUNS_DIR original = Path.iterdir def unreadable(path): @@ -615,7 +614,8 @@ def unreadable(path): assert result["outcome"] == "failed" expected_pipeline_files = sum( relative.parts[1] == "sandbox" - or relative.parts[1:4] == ("autoskillit-runs", "worktrees", "impl-1-x") + or relative.parts[1:4] + == (harness.PIPELINE_RUNS_DIR, harness.PIPELINE_WORKTREES_DIR, "impl-1-x") for relative in kept.values() if relative.parts[0] == "pipeline-artifacts" )