diff --git a/Detection/benchmark/benchmark_pack.py b/Detection/benchmark/benchmark_pack.py index 0eff4037..85da0096 100644 --- a/Detection/benchmark/benchmark_pack.py +++ b/Detection/benchmark/benchmark_pack.py @@ -83,10 +83,11 @@ def cmd_deflate(benchmark_dir: Path, tasks_file_arg: str | None, output: Path | else: print("Warning: ground_truth.json not found; manifest will omit ground truth", file=sys.stderr) - # Discover task directories (task_NNN) + # Discover task directories (task_NNN), skipping raw AgentDojo output + # directories (task_NNN_agentdojo_run) that hold no conversation task_dirs = sorted( d for d in benchmark_dir.iterdir() - if d.is_dir() and d.name.startswith("task_") + if d.is_dir() and d.name.startswith("task_") and not d.name.endswith("_agentdojo_run") ) task_count = len(task_dirs) diff --git a/Detection/tests/test_benchmark_pack.py b/Detection/tests/test_benchmark_pack.py index 79e38d67..63e58184 100644 --- a/Detection/tests/test_benchmark_pack.py +++ b/Detection/tests/test_benchmark_pack.py @@ -35,6 +35,23 @@ def test_deflate_and_inflate_round_trip(self, tmp_path: Path): assert (out_dir / "task_001" / "result.json").exists() + def test_deflate_skips_raw_agentdojo_run_dirs(self, tmp_path: Path): + bench_dir = tmp_path / "agentdojo_20260101_000000" + bench_dir.mkdir() + workspace = bench_dir / "task_001" / "workspace" + workspace.mkdir(parents=True) + with open(workspace / "claude_conversation.json", "w", encoding="utf-8") as f: + json.dump([{"role": "user", "content": "hi"}], f) + # Raw AgentDojo output written next to the converted task directory. + (bench_dir / "task_001_agentdojo_run" / "slack").mkdir(parents=True) + + jsonl_path = tmp_path / "out.jsonl" + cmd_deflate(bench_dir, tasks_file_arg=None, output=jsonl_path) + + records = [json.loads(line) for line in jsonl_path.read_text().splitlines()] + assert records[0]["task_count"] == 1 + assert [r["task_id"] for r in records[1:]] == ["task_001"] + class TestBenchmarkPackSecurity: def test_rejects_path_traversal_task_id(self, tmp_path: Path, sample_jsonl: Path, capsys): malicious = sample_jsonl.read_text().splitlines()