diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md index da923f52bd..cc157b6fa5 100644 --- a/scripts/e2e/README.md +++ b/scripts/e2e/README.md @@ -224,18 +224,36 @@ For model tests, the container's AutoSkillit data directory (`~/.local/share/aut from the runner, so session logs survive a killed container; the harness also reads this log tree after fleet runs. Those files belong to uid 1000 and are often `0600`, so a second container run copies `out/` and `data/logs/` into -`upload/` world-readable with both secrets replaced by `[REDACTED]`, skips symlinks, and -fails if any secret survives. Only `upload/` is uploaded, and only when redaction -succeeded. - -When a test does not pass, the harness preserves plans from each pipeline clone under -`out/pipeline-artifacts///` and Claude Code JSONL under -`out/transcripts/`, retaining relative paths. The transcripts include the orchestrator, -skill sessions and their subagents; the orchestrator's transcript holds each raw -`bind_plan_set` response and its `plan_parts` argument. These files go through the same -redaction and upload steps as the other output artifacts. Collection also runs for an -expected failure or a harness exception. A container killed before or during collection -may leave no evidence or only partial evidence. +`upload/` world-readable, skipping symlinks. The workflow supplies `MINIMAX_API_KEY` and +`E2E_SANDBOX_TOKEN`; the redactor replaces their nonempty literal byte values in file +contents with `[REDACTED]`, re-reads all uploaded file contents, and fails closed if a +supplied literal survives. Encoded or transformed values, unknown credentials and file +names are outside this check. Only `upload/` is uploaded, and only when redaction succeeded. + +Every test copies the stage directories selected by `PIPELINE_EVIDENCE` in +`e2e_harness.py`, on every outcome: pass, failure, expected failure or harness exception. +Sources are the `.autoskillit/temp` of the sandbox, each pipeline clone and each surviving +worktree. They land under `out/pipeline-artifacts///`. +Ledgers, kitchen and guard state, counters and the worktrees sidecar are not copied. +The `pipeline_files` evidence counter records copied stage files. + +Claude Code `*.jsonl` and `*.txt` files under the projects tree land in +`out/transcripts/`, retaining relative paths for orchestrator, skill and subagent sessions. +The text-file selection includes `tool-results/*.txt`: a transcript's `bind_plan_set` +response may be a pointer to one of these retained payloads rather than inline bytes. +All collected files go through the same redaction and upload steps as other artifacts. + +`evidence.unretained` lists existing files under any `.autoskillit/temp` mentioned by a +`sessions.jsonl` row that were not copied. An entry for a commissioning stage means that +stage's bytes are missing. No `evidence` key means nothing was found to collect. +Uploaded names escape `~` and upload-forbidden characters as `~XX`, so `sha256:` becomes +`sha256~3a`; paths quoted inside session logs and transcripts keep their original characters. + +A merged run's worktree temp is deleted by `merge_worktree`. The implement step's binding +digest, id, `sealed` state and coverage status remain in that session's +`verified_plan_set_preflight` in `logs/sessions.jsonl`. Only the final review iteration's +files survive `pre_review_cleanup`. A container killed before or during collection may +leave no evidence or only partial evidence. `clean-install` mounts no home directories and retains evidence under `out/`. Its redactor receives the same required `--secret-env` names with variables unset and skips the absent @@ -381,13 +399,19 @@ After the plan-set changes have merged, perform this check before CI commissioni and the evidence collector. Build and run that revision with the Docker invocation above, selecting `--test implementation`. Use the same revision for commissioning and record the model. -2. If the run reaches implementation, the first bind succeeded. If the test does not - pass, inspect the orchestrator's `bind_plan_set` results in `out/transcripts/`. +2. Inspect the orchestrator's `bind_plan_set` results in `out/transcripts/` regardless of + outcome, including `tool-results/` when a transcript holds only a pointer. Inspect the + sealed records under + `out/pipeline-artifacts/autoskillit-runs//plan-set-authority/`. An observed `success: true` on the first or second bind is the pass condition. A `coverage_failed` result explains the rejection in `coverage_gaps`, and - `out/pipeline-artifacts/` retains the rejected plan. Missing evidence is inconclusive - and does not satisfy this gate: require an observed successful bind. When a run - replans, also confirm the second bind's `plan_parts` names only the replacement plan. + `out/pipeline-artifacts/` retains the rejected plan. Treat an `evidence.unretained` + entry for a commissioning stage as missing evidence. An empty list covers existing + session-referenced temp files only; unmentioned producers depend on `PIPELINE_EVIDENCE`, + so inspect each required commissioning payload directly before signing off. + Missing evidence is inconclusive and does not satisfy this gate: require an observed + successful bind. When a run replans, also confirm the second bind's `plan_parts` names + only the replacement plan. 3. Record the model, plan path and bind result in issue #5259's body using `gh issue edit --body-file`, never as a comment. Only an observed successful bind unblocks `gh workflow run e2e.yml -f tests=implementation`. diff --git a/scripts/e2e/e2e_harness.py b/scripts/e2e/e2e_harness.py index 15e810040d..c3cdff3624 100644 --- a/scripts/e2e/e2e_harness.py +++ b/scripts/e2e/e2e_harness.py @@ -40,6 +40,32 @@ CONFIG_TEMPLATE = Path(__file__).with_name("autoskillit-config.yaml") SANDBOX_CLONE = Path("/workspace/sandbox") PIPELINE_RUNS_DIR = "autoskillit-runs" +PIPELINE_WORKTREES_DIR = "worktrees" +# Select stage artifacts by directory and file name; runtime state stays behind. +PIPELINE_EVIDENCE: tuple[tuple[str, str], ...] = ( + ("make-plan", "*"), + ("rectify", "*"), + ("plan-set-authority", "*"), + ("audit-admission", "*.json"), # ledger.sqlite3 shares this directory + ("audit-disposition", "*.json"), + ("audit-impl", "*"), + ("review-approach", "*"), + ("arch-lens-*", "*"), + ("prepare-pr", "*"), + ("compose-pr", "*"), + ("review-pr", "*"), + ("resolve-review", "*"), + ("diagnose-ci", "*"), + ("resolve-failures", "*"), + ("resolve-merge-conflicts", "*"), + ("retry-worktree", "*"), +) +# Rejected in uploaded paths by actions/upload-artifact v4.6.2 (e2e.yml). +UPLOAD_FORBIDDEN_CHARACTERS = '":<>|*?\r\n' +_UPLOAD_ESCAPES = str.maketrans({c: f"~{ord(c):02x}" for c in "~" + UPLOAD_FORBIDDEN_CHARACTERS}) +_TEMP_PATH = re.compile( + r"(?:/[^\s\"'`\\<>|*?,;()\[\]{}]*)?\.autoskillit/temp/[^\s\"'`\\<>|*?,;()\[\]{}]+" +) _SCRUBBED_ENV = frozenset({"CLAUDE_CODE_OAUTH_TOKEN", *SECRET_ENV}) _STDERR_TAIL_CHARS = 2000 @@ -1105,32 +1131,38 @@ def _run_test( token = env["E2E_SANDBOX_TOKEN"] recipe_env = { **scrubbed, - "AUTOSKILLIT_LOG_DIR": str(home / ".local" / "share" / "autoskillit" / "logs"), + "AUTOSKILLIT_LOG_DIR": str(autoskillit_log_dir(home)), } return run_recipe(test, catalog.sandbox_repository, token, out, recipe_env, runner) -def _copy_failure_files( - source: Path, dest: Path, boundary: Path, pattern: str, errors: list[str] -) -> int: +def autoskillit_log_dir(home: Path) -> Path: + return home / ".local" / "share" / "autoskillit" / "logs" + + +def _copy_evidence_files( + source: Path, dest: Path, boundary: Path, patterns: tuple[str, ...], errors: list[str] +) -> list[Path]: relative = source.relative_to(boundary) try: if any( boundary.joinpath(*relative.parts[:depth]).is_symlink() for depth in range(len(relative.parts) + 1) ): - return 0 + return [] if not source.exists(): - return 0 + return [] except OSError as exc: errors.append(f"{source}: {exc}") - return 0 - copied = 0 + return [] + copied = [] for root, dirs, files in os.walk( source, followlinks=False, onerror=lambda exc: errors.append(f"{source}: {exc}") ): dirs.sort() - for name in sorted(fnmatch.filter(files, pattern)): + for name in sorted(files): + if not any(fnmatch.fnmatch(name, pattern) for pattern in patterns): + continue path = Path(root) / name try: if path.is_symlink() or not path.is_file(): @@ -1138,37 +1170,109 @@ def _copy_failure_files( target = dest / path.relative_to(source) target.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(path, target) - copied += 1 + copied.append(path) except OSError as exc: errors.append(f"{path}: {exc}") return copied -def collect_failure_evidence(out: Path, home: Path) -> dict[str, Any]: - """Keep pipeline plans and Claude transcripts for an unsuccessful E2E run.""" - errors: list[str] = [] +def _evidence_roots(errors: list[str]) -> list[Path]: runs = SANDBOX_CLONE.parent / PIPELINE_RUNS_DIR + roots = [SANDBOX_CLONE] + for parent, skip in ((runs, PIPELINE_WORKTREES_DIR), (runs / PIPELINE_WORKTREES_DIR, None)): + try: + linked = runs.is_symlink() or parent.is_symlink() + entries = ( + [] if linked else sorted(entry for entry in parent.iterdir() if entry.is_dir()) + ) + except FileNotFoundError: + entries = [] + except OSError as exc: + errors.append(f"{parent}: {exc}") + entries = [] + roots += [entry for entry in entries if entry.name != skip] + return roots + + +def _collect_stage_files(out: Path, roots: list[Path], errors: list[str]) -> list[Path]: + workspace = SANDBOX_CLONE.parent + copied = [] + for root in roots: + temp = root / ".autoskillit" / "temp" + try: + # iterdir propagates scan errors instead of hiding unreadable directories. + entries = sorted(temp.iterdir()) + except FileNotFoundError: + continue + except OSError as exc: + errors.append(f"{temp}: {exc}") + continue + for entry in entries: + for stage_pattern, file_pattern in PIPELINE_EVIDENCE: + if fnmatch.fnmatch(entry.name, stage_pattern): + copied.extend( + _copy_evidence_files( + entry, + out / "pipeline-artifacts" / root.relative_to(workspace) / entry.name, + workspace, + (file_pattern,), + errors, + ) + ) + return copied + + +def _recorded_temp_paths(line: str) -> list[Path]: + try: + row = json.loads(line) + except json.JSONDecodeError: + return [] + cwd = row.get("cwd") if isinstance(row, dict) else None + paths = [] + for match in _TEMP_PATH.finditer(line): + path = Path(match.group()) + if path.is_absolute(): + paths.append(path) + elif isinstance(cwd, str): + paths.append(Path(cwd) / path) + return paths + + +def _unretained_references(index: Path, retained: set[str], errors: list[str]) -> list[str]: try: - clones = [] if runs.is_symlink() else sorted(runs.iterdir()) + lines = index.read_text(encoding="utf-8").splitlines() except FileNotFoundError: - clones = [] - except OSError as exc: - errors.append(f"{runs}: {exc}") - clones = [] - plan_files = 0 - for clone in clones: - for kind in ("make-plan", "rectify"): - plan_files += _copy_failure_files( - clone / ".autoskillit" / "temp" / kind, - out / "pipeline-artifacts" / clone.name / kind, - clone, - "*", - errors, - ) - transcripts = _copy_failure_files( - home / ".claude" / "projects", out / "transcripts", home, "*.jsonl", errors + return [] + except (OSError, UnicodeDecodeError) as exc: + errors.append(f"{index}: {exc}") + return [] + return sorted( + { + str(path) + for line in lines + for path in _recorded_temp_paths(line) + if path.is_file() and os.path.realpath(path) not in retained + } ) - return {"plan_files": plan_files, "transcripts": transcripts, "errors": errors} + + +def collect_run_evidence(out: Path, home: Path) -> dict[str, Any]: + """Copy every run's stage artifacts and Claude transcripts into *out*, whatever the verdict.""" + errors: list[str] = [] + copied = _collect_stage_files(out, _evidence_roots(errors), errors) + projects = home / ".claude" / "projects" + transcripts = _copy_evidence_files( + projects, out / "transcripts", home, ("*.jsonl", "*.txt"), errors + ) + unretained = _unretained_references( + autoskillit_log_dir(home) / "sessions.jsonl", {os.path.realpath(p) for p in copied}, errors + ) + return { + "pipeline_files": len(copied), + "transcripts": len(transcripts), + "unretained": unretained, + "errors": errors, + } def run_test( @@ -1197,18 +1301,18 @@ def run_test( result.update( outcome=outcome, passed=outcome == "passed", expected_findings=matched_findings ) - if not result["passed"]: - try: - evidence = collect_failure_evidence(out, home) - except Exception as exc: - evidence = { - "plan_files": 0, - "transcripts": 0, - "errors": [f"{type(exc).__name__}: {exc}"], - "exception": traceback.format_exc(), - } - if any(evidence.values()): - result["evidence"] = evidence + try: + evidence = collect_run_evidence(out, home) + except Exception as exc: + evidence = { + "pipeline_files": 0, + "transcripts": 0, + "unretained": [], + "errors": [f"{type(exc).__name__}: {exc}"], + "exception": traceback.format_exc(), + } + if any(evidence.values()): + result["evidence"] = evidence (out / "result.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") return failures @@ -1224,16 +1328,23 @@ def _copy_redacted(source: Path, target: Path, needles: Sequence[bytes]) -> None target.chmod(0o644) +def upload_name(name: str) -> str: + """Escape ``~`` and upload-forbidden characters as ``~XX`` without name collisions.""" + return name.translate(_UPLOAD_ESCAPES) + + def _copy_tree_redacted(source: Path, target: Path, needles: Sequence[bytes]) -> None: for root, _dirs, files in os.walk(source, followlinks=False): root_path = Path(root) - target_dir = target / root_path.relative_to(source) + target_dir = target.joinpath( + *(upload_name(part) for part in root_path.relative_to(source).parts) + ) target_dir.mkdir(parents=True, exist_ok=True) target_dir.chmod(0o755) for name in files: path = root_path / name if not path.is_symlink() and path.is_file(): - _copy_redacted(path, target_dir / name, needles) + _copy_redacted(path, target_dir / upload_name(name), needles) def _find_leak(dest: Path, needles: Sequence[bytes]) -> Path | None: @@ -1257,7 +1368,7 @@ def redact_tree(sources: Sequence[Path], dest: Path, secrets: Sequence[str]) -> for source in sources: if source.is_symlink() or not source.exists(): continue - target = dest / source.name + target = dest / upload_name(source.name) if source.is_dir(): _copy_tree_redacted(source, target, needles) else: diff --git a/tests/infra/test_e2e_harness.py b/tests/infra/test_e2e_harness.py index c2236f0ef3..64b7cd26d0 100644 --- a/tests/infra/test_e2e_harness.py +++ b/tests/infra/test_e2e_harness.py @@ -25,6 +25,11 @@ harness = load_check_script("_autoskillit_e2e_harness", _HARNESS_SCRIPT) e2e_catalog = harness.e2e_catalog +_AUDIT_SLOT = f"sha256:{'a' * 64}" +_AUDIT_ATTEMPT = "b" * 32 +_PLAN_SET_FILE = "0123456789abcdef01234567.json" +_RECIPE_OUTCOMES = ((["recipe failed"], "failed", False), ([], "passed", True)) + @pytest.fixture(autouse=True) def _isolated_sandbox(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -32,29 +37,152 @@ def _isolated_sandbox(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @pytest.fixture -def failure_sources(tmp_path: Path) -> dict[Path, Path]: - runs = harness.SANDBOX_CLONE.parent / harness.PIPELINE_RUNS_DIR - project = tmp_path / "home" / ".claude" / "projects" / "project" - sources = { - runs / "clone-a/.autoskillit/temp/make-plan/a_plan.md": Path( - "pipeline-artifacts/clone-a/make-plan/a_plan.md" - ), - runs / "clone-b/.autoskillit/temp/rectify/b_plan.md": Path( - "pipeline-artifacts/clone-b/rectify/b_plan.md" - ), - runs / "clone-a/.autoskillit/temp/make-plan/nested/metadata.json": Path( - "pipeline-artifacts/clone-a/make-plan/nested/metadata.json" +def evidence_sources(tmp_path: Path) -> tuple[dict[Path, Path], dict[Path, Path]]: + workspace = harness.SANDBOX_CLONE.parent + runs = workspace / harness.PIPELINE_RUNS_DIR + kept: dict[Path, Path] = {} + excluded: dict[Path, Path] = {} + + def add_pipeline_files( + root: Path, + artifact_root: Path, + relative_paths: tuple[str, ...], + destination: dict[Path, Path], + ) -> None: + for value in relative_paths: + relative = Path(value) + source = root / ".autoskillit" / "temp" / relative + destination[source] = Path("pipeline-artifacts") / artifact_root / relative + source.parent.mkdir(parents=True, exist_ok=True) + source.write_bytes(f"evidence for {relative}\n".encode()) + + worktree_plan_set_file = "fedcba9876543210fedcba98.json" + clone_a_artifact_root = Path(harness.PIPELINE_RUNS_DIR) / "clone-a" + clone_a = workspace / clone_a_artifact_root + add_pipeline_files( + harness.SANDBOX_CLONE, + Path("sandbox"), + ("audit-disposition/a1/p1/disposition-report.json",), + kept, + ) + add_pipeline_files( + clone_a, + clone_a_artifact_root, + ( + "make-plan/a_plan.md", + "make-plan/nested/metadata.json", + f"plan-set-authority/planset-x/{_PLAN_SET_FILE}", + f"plan-set-authority/planset-x/issue.{_PLAN_SET_FILE.removesuffix('.json')}.md", + f"audit-admission/{_AUDIT_SLOT}/{_AUDIT_ATTEMPT}/semantic.json", + f"audit-admission/{_AUDIT_SLOT}/{_AUDIT_ATTEMPT}/inventory.json", + f"audit-admission/{_AUDIT_SLOT}/{_AUDIT_ATTEMPT}/authority.json", + "arch-lens-concurrency/arch_diag_x.md", + "review-pr/iter_1/local_findings_26.json", + "resolve-review/deferred_observations_26.json", + "prepare-pr/pr_prep_x.md", + "compose-pr/pr_body_x.md", ), - project / "session.jsonl": Path("transcripts/project/session.jsonl"), - project / "session/subagents/agent-one.jsonl": Path( - "transcripts/project/session/subagents/agent-one.jsonl" + kept, + ) + add_pipeline_files( + runs / "clone-b", + Path(harness.PIPELINE_RUNS_DIR) / "clone-b", + ("rectify/b_plan.md",), + kept, + ) + add_pipeline_files( + runs / harness.PIPELINE_WORKTREES_DIR / "impl-1-x", + Path(harness.PIPELINE_RUNS_DIR) / harness.PIPELINE_WORKTREES_DIR / "impl-1-x", + (f"plan-set-authority/planset-y/{worktree_plan_set_file}",), + kept, + ) + + add_pipeline_files( + clone_a, + clone_a_artifact_root, + ( + "audit-admission/ledger.sqlite3", + "context-admission/ledger.sqlite3", + "kitchen_state/state.json", + "guard_decisions.jsonl", + "eject_count", + "worktrees/impl-1-x/base-branch", ), - } - for source, relative in sources.items(): + excluded, + ) + add_pipeline_files( + harness.SANDBOX_CLONE, + Path("sandbox"), + ("audit-admission/ledger.sqlite3",), + excluded, + ) + + home = tmp_path / "home" + projects = home / ".claude" / "projects" + for relative_value in ( + "project/session.jsonl", + "project/session/subagents/agent-one.jsonl", + "project/session/tool-results/call_1.txt", + ): + relative = Path(relative_value) + source = projects / relative + kept[source] = Path("transcripts") / relative source.parent.mkdir(parents=True, exist_ok=True) source.write_bytes(f"evidence for {relative}\n".encode()) - (project / "ignored.txt").write_text("not a transcript") - return sources + ignored = projects / "project/ignored.md" + excluded[ignored] = Path("transcripts/project/ignored.md") + ignored.parent.mkdir(parents=True, exist_ok=True) + ignored.write_text("not a transcript", encoding="utf-8") + + linked_temp = runs / harness.PIPELINE_WORKTREES_DIR / "impl-2-x/.autoskillit/temp" + linked_temp.parent.mkdir(parents=True, exist_ok=True) + linked_temp.symlink_to(clone_a / ".autoskillit" / "temp", target_is_directory=True) + excluded[linked_temp] = ( + Path("pipeline-artifacts") + / harness.PIPELINE_RUNS_DIR + / harness.PIPELINE_WORKTREES_DIR + / "impl-2-x" + ) + return kept, excluded + + +def _pipeline_file_count(sources: dict[Path, Path]) -> int: + return sum(relative.parts[0] == "pipeline-artifacts" for relative in sources.values()) + + +def _assert_evidence_sources( + out: Path, kept: dict[Path, Path], excluded: dict[Path, Path] +) -> None: + for source, relative in kept.items(): + assert (out / relative).read_bytes() == source.read_bytes() + for relative in excluded.values(): + assert not (out / relative).exists() + + +def _uploaded_path(upload: Path, relative: Path) -> Path: + escaped = Path(*(harness.upload_name(part) for part in relative.parts)) + return upload / "out" / escaped + + +def test_transcripts_are_collected_in_one_walk( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, evidence_sources +) -> None: + home = tmp_path / "home" + projects = home / ".claude" / "projects" + walked = [] + original = harness.os.walk + + def walk(source, *args, **kwargs): + walked.append(source) + return original(source, *args, **kwargs) + + monkeypatch.setattr(harness.os, "walk", walk) + + evidence = harness.collect_run_evidence(tmp_path / "out", home) + + assert walked.count(projects) == 1 + assert evidence["transcripts"] == 3 + assert evidence["errors"] == [] @pytest.mark.parametrize( @@ -66,9 +194,11 @@ def failure_sources(tmp_path: Path) -> dict[Path, Path]: ("passed", "passed", True), ), ) -def test_run_test_retains_evidence_only_when_it_does_not_pass( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure_sources, case, outcome, passed +def test_run_test_retains_evidence_for_every_outcome( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, evidence_sources, case, outcome, passed ) -> None: + kept, excluded = evidence_sources + def run(*_args, **_kwargs): if case == "exception": raise RuntimeError("harness crashed") @@ -83,22 +213,31 @@ def run(*_args, **_kwargs): assert result["passed"] is passed assert result["outcome"] == outcome assert result["failures"] == failures - if passed: - assert "evidence" not in result - assert not (out / "pipeline-artifacts").exists() - assert not (out / "transcripts").exists() - return - assert result["evidence"] == {"plan_files": 3, "transcripts": 2, "errors": []} - for source, relative in failure_sources.items(): - assert (out / relative).read_bytes() == source.read_bytes() - assert not (out / "transcripts/project/ignored.txt").exists() + assert result["evidence"] == { + "pipeline_files": _pipeline_file_count(kept), + "transcripts": 3, + "unretained": [], + "errors": [], + } + _assert_evidence_sources(out, kept, excluded) @pytest.mark.parametrize("failure_site", ("copy", "walk")) +@pytest.mark.parametrize( + ("stub_failures", "expected_outcome", "expected_passed"), + _RECIPE_OUTCOMES, +) def test_evidence_errors_preserve_the_verdict_and_continue_other_sources( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure_sources, failure_site + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + evidence_sources, + failure_site, + stub_failures, + expected_outcome, + expected_passed, ) -> None: - source = next(path for path in failure_sources if path.name == "a_plan.md") + kept, _ = evidence_sources + source = next(path for path in kept if path.name == "a_plan.md") target = source if failure_site == "copy" else source.parent owner = harness.shutil if failure_site == "copy" else harness.os name = "copyfile" if failure_site == "copy" else "scandir" @@ -110,19 +249,28 @@ def unreadable(path, *args, **kwargs): return original(path, *args, **kwargs) monkeypatch.setattr(owner, name, unreadable) - monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (["test failed"], [])) + monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (stub_failures, [])) failures, result = _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) - assert failures == ["test failed"] - assert result["outcome"] == "failed" - assert result["passed"] is False - assert result["evidence"]["transcripts"] == 2 + assert failures == stub_failures + assert result["outcome"] == expected_outcome + assert result["passed"] is expected_passed + assert result["evidence"]["transcripts"] == 3 assert any(str(target) in error for error in result["evidence"]["errors"]) - readable = next(path for path in failure_sources if path.name == "b_plan.md") - assert (tmp_path / "out" / failure_sources[readable]).read_bytes() == readable.read_bytes() + readable = next(path for path in kept if path.name == "b_plan.md") + relative = kept[readable] + assert (tmp_path / "out" / relative).read_bytes() == readable.read_bytes() +@pytest.mark.parametrize( + ("stub_failures", "expected_outcome", "expected_passed"), + _RECIPE_OUTCOMES, +) def test_unexpected_collection_error_still_writes_the_result( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + stub_failures, + expected_outcome, + expected_passed, ) -> None: def crash(*_args, **_kwargs): try: @@ -130,27 +278,62 @@ def crash(*_args, **_kwargs): except ValueError as exc: raise RuntimeError("collection crashed") from exc - monkeypatch.setattr(harness, "collect_failure_evidence", crash) - monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (["test failed"], [])) + monkeypatch.setattr(harness, "collect_run_evidence", crash) + monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (stub_failures, [])) failures, result = _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) - assert failures == ["test failed"] - assert result["outcome"] == "failed" - assert result["evidence"]["plan_files"] == result["evidence"]["transcripts"] == 0 + assert failures == stub_failures + assert result["outcome"] == expected_outcome + assert result["passed"] is expected_passed + assert set(result["evidence"]) == { + "pipeline_files", + "transcripts", + "unretained", + "errors", + "exception", + } + assert result["evidence"]["pipeline_files"] == result["evidence"]["transcripts"] == 0 + assert result["evidence"]["unretained"] == [] assert "collection crashed" in result["evidence"]["errors"][0] assert "ValueError: collection root cause" in result["evidence"]["exception"] assert "RuntimeError: collection crashed" in result["evidence"]["exception"] assert 'raise RuntimeError("collection crashed") from exc' in result["evidence"]["exception"] +@pytest.mark.parametrize( + ("stub_failures", "expected_outcome", "expected_passed"), + _RECIPE_OUTCOMES, +) def test_nothing_to_collect_adds_no_evidence_key( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + stub_failures, + expected_outcome, + expected_passed, ) -> None: - monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (["test failed"], [])) - _, result = _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) + monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (stub_failures, [])) + failures, result = _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) + assert failures == stub_failures + assert result["outcome"] == expected_outcome + assert result["passed"] is expected_passed assert "evidence" not in result -def test_failure_evidence_skips_symlinks_and_linked_scan_ancestors(tmp_path: Path) -> None: +def test_evidence_ignores_files_in_run_root_directories(tmp_path: Path) -> None: + runs = harness.SANDBOX_CLONE.parent / harness.PIPELINE_RUNS_DIR + worktrees = runs / harness.PIPELINE_WORKTREES_DIR + worktrees.mkdir(parents=True) + (runs / "unrelated.txt").write_text("not a clone") + (worktrees / "unrelated.txt").write_text("not a worktree") + + assert harness.collect_run_evidence(tmp_path / "out", tmp_path / "home") == { + "pipeline_files": 0, + "transcripts": 0, + "unretained": [], + "errors": [], + } + + +def test_evidence_skips_symlinks_and_linked_scan_ancestors(tmp_path: Path) -> None: runs = harness.SANDBOX_CLONE.parent / harness.PIPELINE_RUNS_DIR plans = runs / "clone-a/.autoskillit/temp/make-plan" plans.mkdir(parents=True) @@ -164,6 +347,7 @@ def test_failure_evidence_skips_symlinks_and_linked_scan_ancestors(tmp_path: Pat ancestor.mkdir(parents=True) (ancestor / "temp").symlink_to(external, target_is_directory=True) (runs / "linked-clone").symlink_to(runs / "clone-a", target_is_directory=True) + (runs / harness.PIPELINE_WORKTREES_DIR).symlink_to(external, target_is_directory=True) home = tmp_path / "home" home.mkdir() (external / "projects").mkdir() @@ -171,36 +355,250 @@ def test_failure_evidence_skips_symlinks_and_linked_scan_ancestors(tmp_path: Pat (home / ".claude").symlink_to(external, target_is_directory=True) out = tmp_path / "out" - result = harness.collect_failure_evidence(out, home) + result = harness.collect_run_evidence(out, home) - assert result == {"plan_files": 1, "transcripts": 0, "errors": []} + assert result == { + "pipeline_files": 1, + "transcripts": 0, + "unretained": [], + "errors": [], + } assert [path.relative_to(out) for path in out.rglob("*") if path.is_file()] == [ - Path("pipeline-artifacts/clone-a/make-plan/kept.md") + Path("pipeline-artifacts") / harness.PIPELINE_RUNS_DIR / "clone-a/make-plan/kept.md" ] +def test_passing_pipeline_recipe_retains_commissioning_evidence( + tmp_path: Path, evidence_sources +) -> None: + kept, excluded = evidence_sources + out = tmp_path / "out" + by_destination = {relative: source for source, relative in kept.items()} + clone_artifacts = Path("pipeline-artifacts") / harness.PIPELINE_RUNS_DIR / "clone-a" + semantic = clone_artifacts / "audit-admission" / _AUDIT_SLOT / _AUDIT_ATTEMPT / "semantic.json" + review = clone_artifacts / "review-pr/iter_1/local_findings_26.json" + transcript = Path("transcripts/project/session.jsonl") + tool_results = Path("transcripts/project/session/tool-results/call_1.txt") + secrets = (semantic, review, transcript, tool_results) + for relative in secrets: + source = by_destination[relative] + source.write_text(f"{MINIMAX_KEY} and {SANDBOX_TOKEN}", encoding="utf-8") + + clone_a = harness.SANDBOX_CLONE.parent / harness.PIPELINE_RUNS_DIR / "clone-a" + plan = by_destination[clone_artifacts / "make-plan/a_plan.md"] + plan_set = by_destination[clone_artifacts / "plan-set-authority/planset-x" / _PLAN_SET_FILE] + + seeded = False + + def seed(argv: list[str]) -> None: + nonlocal seeded + if argv[:3] != ["autoskillit", "fleet", "run"]: + return None + log_dir = _log_dir(tmp_path) + log_dir.mkdir(parents=True, exist_ok=True) + rows = ( + { + "cwd": str(clone_a), + "outcome_fields": { + "plan_path": str(plan), + "audit_semantic_result_path": str(by_destination[semantic]), + }, + "skill_command": f"read {plan_set}\ncontinue", + }, + { + "cwd": str(clone_a), + "outcome_fields": { + "deferred_observations_path": ( + ".autoskillit/temp/resolve-review/deferred_observations_26.json" + ) + }, + }, + ) + with (log_dir / "sessions.jsonl").open("a", encoding="utf-8") as index: + for row in rows: + index.write(json.dumps(row) + "\n") + seeded = True + return None + + runner = FakeRunner(_recipe_handler(log_dir=_log_dir(tmp_path), extra=seed)) + failures, result = _run_recipe(tmp_path, runner) + + assert seeded, "the recipe runner must seed the retained-path session rows" + assert failures == [] + assert result["outcome"] == "passed" + assert result["passed"] is True + assert result["evidence"] == { + "pipeline_files": _pipeline_file_count(kept), + "transcripts": 3, + "unretained": [], + "errors": [], + } + _assert_evidence_sources(out, kept, excluded) + + upload = tmp_path / "upload" + harness.redact_tree([out], upload, [MINIMAX_KEY, SANDBOX_TOKEN]) + semantic_upload = _uploaded_path(upload, semantic) + assert semantic_upload == ( + upload + / "out" + / clone_artifacts + / "audit-admission" + / f"sha256~3a{_AUDIT_SLOT.removeprefix('sha256:')}" + / _AUDIT_ATTEMPT + / "semantic.json" + ) + for relative in secrets: + copied = _uploaded_path(upload, relative) + content = copied.read_bytes() + assert harness.REDACTED in content + for artifact in upload.rglob("*"): + assert not any( + character in artifact.relative_to(upload).as_posix() + for character in harness.UPLOAD_FORBIDDEN_CHARACTERS + ) + if artifact.is_file(): + content = artifact.read_bytes() + assert MINIMAX_KEY.encode() not in content + assert SANDBOX_TOKEN.encode() not in content + + +def test_unretained_lists_recorded_files_that_were_not_kept( + tmp_path: Path, evidence_sources +) -> None: + kept, _ = evidence_sources + workspace = harness.SANDBOX_CLONE.parent + clone_a = workspace / harness.PIPELINE_RUNS_DIR / "clone-a" + new_producer = clone_a / ".autoskillit/temp/new-producer/verdict.json" + cwd_resolved = clone_a / ".autoskillit/temp/new-producer-2/x.json" + new_producer.parent.mkdir(parents=True, exist_ok=True) + cwd_resolved.parent.mkdir(parents=True, exist_ok=True) + new_producer.write_text("new evidence", encoding="utf-8") + cwd_resolved.write_text("new evidence", encoding="utf-8") + + semantic = next( + source + for source, relative in kept.items() + if relative.name == "semantic.json" and "audit-admission" in relative.parts + ) + plan_set = next(source for source, relative in kept.items() if relative.name == _PLAN_SET_FILE) + symlinked_plan_set = ( + workspace + / harness.PIPELINE_RUNS_DIR + / harness.PIPELINE_WORKTREES_DIR + / "impl-2-x/.autoskillit/temp" + / plan_set.relative_to(clone_a / ".autoskillit/temp") + ) + missing = ".autoskillit/temp/not-created/missing.json" + rows = ( + {"outcome_fields": {"remediation_path": str(new_producer)}}, + { + "cwd": str(clone_a), + "outcome_fields": {"remediation_path": ".autoskillit/temp/new-producer-2/x.json"}, + }, + {"outcome_fields": {"audit_semantic_result_path": str(semantic)}}, + {"skill_command": f"read {symlinked_plan_set}"}, + {"cwd": str(clone_a), "outcome_fields": {"remediation_path": missing}}, + ) + home = tmp_path / "home" + index = harness.autoskillit_log_dir(home) / "sessions.jsonl" + index.parent.mkdir(parents=True, exist_ok=True) + index.write_text( + "".join(json.dumps(row) + "\n" for row in rows) + "{malformed line\n", + encoding="utf-8", + ) + + evidence = harness.collect_run_evidence(tmp_path / "out", home) + + assert evidence["unretained"] == sorted([str(new_producer), str(cwd_resolved)]) + assert evidence["errors"] == [] + + +@pytest.mark.parametrize( + "error", + ( + PermissionError("cannot read session index"), + UnicodeDecodeError("utf-8", b"\xff", 0, 1, "invalid start byte"), + ), +) +def test_index_read_error_preserves_collected_evidence( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, evidence_sources, error +) -> None: + kept, excluded = evidence_sources + index = harness.autoskillit_log_dir(tmp_path / "home") / "sessions.jsonl" + original = Path.read_text + + def unreadable(path, *args, **kwargs): + if path == index: + raise error + return original(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", unreadable) + monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: ([], [])) + + failures, result = _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) + + assert failures == [] + assert result["outcome"] == "passed" + assert result["evidence"] == { + "pipeline_files": _pipeline_file_count(kept), + "transcripts": 3, + "unretained": [], + "errors": [f"{index}: {error}"], + } + _assert_evidence_sources(tmp_path / "out", kept, excluded) + + +def test_upload_name_escapes_forbidden_characters_injectively() -> None: + for character in harness.UPLOAD_FORBIDDEN_CHARACTERS + "~": + assert harness.upload_name(character) == f"~{ord(character):02x}" + assert harness.upload_name("sha256:ab") == "sha256~3aab" + names = {harness.upload_name(name) for name in ("a:b", "a~3ab", "a-b")} + assert len(names) == 3 + assert harness.upload_name("plain-name") == "plain-name" + + +@pytest.mark.parametrize("directory", (False, True)) +def test_redaction_escapes_top_level_source_names(tmp_path: Path, directory: bool) -> None: + source = tmp_path / "out:root~3a" + artifact = source / "log.txt" if directory else source + artifact.parent.mkdir(parents=True, exist_ok=True) + artifact.write_bytes(MINIMAX_KEY.encode()) + upload = tmp_path / "upload" + + harness.redact_tree([source], upload, [MINIMAX_KEY]) + + target = upload / "out~3aroot~7e3a" + copied = target / "log.txt" if directory else target + assert copied.read_bytes() == harness.REDACTED + assert not (upload / source.name).exists() + + +@pytest.mark.parametrize("case", ("failed", "passed")) def test_collected_plans_and_transcripts_are_redacted( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure_sources + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, evidence_sources, case ) -> None: - plan = next(path for path in failure_sources if path.name == "a_plan.md") - transcript = next(path for path in failure_sources if path.name == "session.jsonl") + kept, _ = evidence_sources + plan = next(path for path in kept if path.name == "a_plan.md") + transcript = next(path for path in kept if path.name == "session.jsonl") plan.write_bytes(MINIMAX_KEY.encode()) transcript.write_bytes(SANDBOX_TOKEN.encode()) - monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (["test failed"], [])) + stub_failures = ["test failed"] if case == "failed" else [] + monkeypatch.setattr(harness, "_run_test", lambda *_args, **_kwargs: (stub_failures, [])) _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) upload = tmp_path / "upload" harness.redact_tree([tmp_path / "out"], upload, [MINIMAX_KEY, SANDBOX_TOKEN]) for source, secret in ((plan, MINIMAX_KEY), (transcript, SANDBOX_TOKEN)): - copied = (upload / "out" / failure_sources[source]).read_bytes() + copied = _uploaded_path(upload, kept[source]).read_bytes() assert harness.REDACTED in copied assert secret.encode() not in copied def test_runs_scan_error_keeps_readable_transcripts( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure_sources + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, evidence_sources ) -> None: + kept, _ = evidence_sources runs = harness.SANDBOX_CLONE.parent / harness.PIPELINE_RUNS_DIR original = Path.iterdir @@ -214,17 +612,30 @@ def unreadable(path): failures, result = _run_recipe(tmp_path, FakeRunner(lambda _argv: None)) assert failures == ["test failed"] assert result["outcome"] == "failed" - assert result["evidence"]["plan_files"] == 0 - assert result["evidence"]["transcripts"] == 2 + expected_pipeline_files = sum( + relative.parts[1] == "sandbox" + or relative.parts[1:4] + == (harness.PIPELINE_RUNS_DIR, harness.PIPELINE_WORKTREES_DIR, "impl-1-x") + for relative in kept.values() + if relative.parts[0] == "pipeline-artifacts" + ) + assert result["evidence"]["pipeline_files"] == expected_pipeline_files + assert result["evidence"]["transcripts"] == 3 assert str(runs) in result["evidence"]["errors"][0] def test_pipeline_runs_directory_matches_clone_source_of_truth() -> None: - from autoskillit.workspace.clone._detect import RUNS_DIR + from autoskillit.workspace.clone import RUNS_DIR assert harness.PIPELINE_RUNS_DIR == RUNS_DIR +def test_worktrees_directory_matches_clone_source_of_truth() -> None: + from autoskillit.workspace.clone import WORKTREES_DIR + + assert harness.PIPELINE_WORKTREES_DIR == WORKTREES_DIR + + MINIMAX_KEY = "minimax-test-key" SANDBOX_TOKEN = "sandbox-test-token" SANDBOX = "owner/sandbox"