Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 133 additions & 0 deletions .github/aicodingflow-tests/test_codex_model_provider_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

import unittest

import yaml

from script_imports import ROOT


Expand All @@ -14,8 +16,10 @@
".github/workflows/update-dedupe.yml",
".github/workflows/update-pr-review.yml",
".github/workflows/update-triage.yml",
".github/workflows/nightly-codebase-review.yml",
)
PINNED_REVIEW_ACTION = "uses: openai/codex-action@52fe01ec70a42f454c9d2ebd47598f9fd6893d56"
PINNED_CLAUDE_ACTION = "anthropics/claude-code-action@c0949c50a78c1ec8ce2647db36629b870e715d0c"


class CodexModelProviderConfigTest(unittest.TestCase):
Expand All @@ -37,6 +41,135 @@ def test_all_codex_workflows_use_shared_provider_configuration(self) -> None:
self.assertIn("*/responses", contents)
self.assertIn("$endpoint/responses", contents)
self.assertIn("CODEX_API_ENDPOINT is not set", contents)
self.assertIn(f"uses: {PINNED_CLAUDE_ACTION}", contents)
self.assertIn("anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}", contents)
self.assertIn("claude_args:", contents)
self.assertIn("prompt: &agent_prompt |", contents)
self.assertIn("prompt: *agent_prompt", contents)
self.assertIn("vars.AI_PROVIDER == 'claude'", contents)
self.assertIn("vars.AI_PROVIDER == '' || vars.AI_PROVIDER == 'codex'", contents)
self.assertIn(".github/scripts/validate_ai_provider.py", contents)

def test_nightly_review_normalizes_both_provider_outputs(self) -> None:
contents = (ROOT / ".github/workflows/nightly-codebase-review.yml").read_text(encoding="utf-8")
self.assertIn("steps.codex.outputs.final-message || steps.claude.outputs.structured_output", contents)
self.assertIn("--json-schema", contents)
self.assertIn("--allowedTools Read,Glob,Grep", contents)

def test_both_actions_receive_the_same_prompt_and_exclusive_gates(self) -> None:
for path in CODEX_WORKFLOWS:
with self.subTest(path=path):
workflow = yaml.safe_load((ROOT / path).read_text(encoding="utf-8"))
all_steps = [step for job in workflow["jobs"].values() for step in job.get("steps", [])]
claude = next(step for step in all_steps if step.get("uses") == PINNED_CLAUDE_ACTION)
for job in workflow["jobs"].values():
steps = job.get("steps", [])
codex = next((step for step in steps if str(step.get("uses", "")).startswith("openai/codex-action@")), None)
if codex is None:
continue
self.assertEqual(codex["with"]["prompt"], claude["with"]["prompt"])
self.assertIn("AI_PROVIDER == 'codex'", codex["if"])
self.assertIn("AI_PROVIDER == 'claude'", claude["if"])

def test_pr_comment_claude_uses_read_only_job_and_validated_publisher(self) -> None:
workflow = yaml.safe_load((ROOT / ".github/workflows/respond-to-pr-comment.yml").read_text(encoding="utf-8"))
reader = workflow["jobs"]["respond_claude"]
publisher = workflow["jobs"]["publish_claude"]
self.assertEqual(reader["permissions"], {"contents": "read", "issues": "read", "pull-requests": "read"})
self.assertEqual(publisher["permissions"]["contents"], "write")
self.assertEqual(publisher["needs"], ["preflight", "respond_claude"])
claude = next(step for step in reader["steps"] if step.get("uses") == PINNED_CLAUDE_ACTION)
self.assertIn("--disallowedTools Bash,WebFetch,WebSearch", claude["with"]["claude_args"])
self.assertNotIn("gh auth setup-git", str(reader))
self.assertIn("git -C pr-worktree apply --check", str(publisher))
self.assertIn("validate_pr_comment_result.py", str(publisher))
self.assertNotIn(PINNED_CLAUDE_ACTION, str(publisher))

def test_implementation_claude_uses_read_only_job_and_validated_publisher(self) -> None:
workflow = yaml.safe_load((ROOT / ".github/workflows/create-implementation-from-issue.yml").read_text(encoding="utf-8"))
reader = workflow["jobs"]["implement_claude"]
publisher = workflow["jobs"]["publish_claude_implementation"]
self.assertEqual(reader["permissions"], {"contents": "read", "issues": "read", "pull-requests": "read"})
self.assertEqual(publisher["permissions"]["contents"], "write")
self.assertEqual(publisher["needs"], "implement_claude")
claude = next(step for step in reader["steps"] if step.get("uses") == PINNED_CLAUDE_ACTION)
self.assertIn("--disallowedTools Bash,WebFetch,WebSearch", claude["with"]["claude_args"])
self.assertIn("git diff --cached --binary", str(reader))
self.assertIn("git apply --check", str(publisher))
preserve = next(step for step in publisher["steps"] if step.get("name") == "preserve-trusted-implementation-scripts")
apply = next(step for step in publisher["steps"] if step.get("name") == "apply-implementation-output")
validate = next(step for step in publisher["steps"] if step.get("name") == "validate-implementation-output")
self.assertLess(publisher["steps"].index(preserve), publisher["steps"].index(apply))
self.assertIn('$RUNNER_TEMP/trusted-scripts/apply_implementation_output.py', apply["run"])
self.assertIn('$RUNNER_TEMP/trusted-scripts/validate_implementation_output.py', validate["run"])
for job in (workflow["jobs"]["create-implementation"], publisher):
checkout_branch = next(step for step in job["steps"] if step.get("name") == "checkout-implementation-target-branch")
commit = next(step for step in job["steps"] if step.get("name") == "commit-and-push-implementation-branch")
self.assertEqual(checkout_branch["env"]["GH_TOKEN"], "${{ github.token }}")
self.assertIn('GIT_CONFIG_VALUE_0="!gh auth git-credential"', checkout_branch["run"])
self.assertEqual(commit["env"]["GITHUB_TOKEN"], "${{ github.token }}")
self.assertNotIn(PINNED_CLAUDE_ACTION, str(publisher))

def test_every_claude_action_runs_without_repository_write_permissions(self) -> None:
for path in CODEX_WORKFLOWS:
with self.subTest(path=path):
workflow = yaml.safe_load((ROOT / path).read_text(encoding="utf-8"))
agents = [
(job, step)
for job in workflow["jobs"].values()
for step in job.get("steps", [])
if step.get("uses") == PINNED_CLAUDE_ACTION
]
self.assertEqual(len(agents), 1)
job, step = agents[0]
permissions = job.get("permissions", workflow.get("permissions", {}))
self.assertFalse(any(value == "write" for value in permissions.values()))
self.assertIn("Bash", step["with"]["claude_args"].split("--disallowedTools ", 1)[1])
checkouts = [
item for item in job["steps"]
if str(item.get("uses", "")).startswith("actions/checkout@")
]
self.assertTrue(checkouts)
for checkout in checkouts:
self.assertIs(checkout.get("with", {}).get("persist-credentials"), False)

def test_guidance_update_claude_jobs_publish_validated_artifacts(self) -> None:
for name in ("update-dedupe", "update-pr-review", "update-triage"):
with self.subTest(name=name):
workflow = yaml.safe_load((ROOT / ".github/workflows" / f"{name}.yml").read_text(encoding="utf-8"))
reader = workflow["jobs"]["update_claude"]
publisher = workflow["jobs"]["publish_claude"]
checkout = reader["steps"][0]
self.assertIs(checkout["with"]["persist-credentials"], False)
self.assertEqual(publisher["needs"], "update_claude")
self.assertIn("apply_guidance_output.py", str(publisher))
self.assertIn("validate_write_surface.py", str(publisher))
self.assertNotIn(PINNED_CLAUDE_ACTION, str(publisher))

def test_spec_claude_uses_read_only_job_and_validated_publisher(self) -> None:
workflow = yaml.safe_load((ROOT / ".github/workflows/create-spec-from-issue.yml").read_text(encoding="utf-8"))
reader = workflow["jobs"]["create_spec_claude"]
publisher = workflow["jobs"]["publish_claude_spec"]
self.assertEqual(reader["permissions"], {"contents": "read", "issues": "read", "pull-requests": "read"})
self.assertEqual(publisher["permissions"]["contents"], "write")
self.assertIs(reader["steps"][0]["with"]["persist-credentials"], False)
self.assertIn("git add -A -- specs", str(reader))
self.assertIn("git apply --check", str(publisher))
self.assertIn("validate_spec_output.py", str(publisher))
self.assertNotIn(PINNED_CLAUDE_ACTION, str(publisher))

def test_triage_publisher_verifies_context_from_before_agent_run(self) -> None:
workflow = yaml.safe_load((ROOT / ".github/workflows/triage-issue.yml").read_text(encoding="utf-8"))
generator = workflow["jobs"]["generate"]
publisher = workflow["jobs"]["apply"]
self.assertEqual(generator["outputs"]["context_sha256"], "${{ steps.context_snapshot.outputs.sha256 }}")
snapshot = next(step for step in generator["steps"] if step.get("name") == "snapshot-triage-context")
agent = next(step for step in generator["steps"] if step.get("uses") == PINNED_CLAUDE_ACTION)
verify = next(step for step in publisher["steps"] if step.get("name") == "verify-triage-context")
apply = next(step for step in publisher["steps"] if step.get("name") == "apply-triage-result")
self.assertLess(generator["steps"].index(snapshot), generator["steps"].index(agent))
self.assertLess(publisher["steps"].index(verify), publisher["steps"].index(apply))
self.assertIn("sha256sum --check", verify["run"])

def test_workflows_do_not_read_the_legacy_endpoint_environment_directly(self) -> None:
for path in CODEX_WORKFLOWS:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,7 @@ def test_workflow_schedules_beijing_night_and_publishes_outside_agent(self) -> N
workflow = (ROOT / ".github/workflows/nightly-codebase-review.yml").read_text(encoding="utf-8")
self.assertTrue(workflow.startswith("name: nightly-codebase-review\n"))
step_names = re.findall(r"^ - name: (.+)$", workflow, flags=re.MULTILINE)
self.assertEqual(len(step_names), 14)
self.assertEqual(len(step_names), 16)
for step_name in step_names:
with self.subTest(step_name=step_name):
self.assertRegex(step_name, r"^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$")
Expand All @@ -333,7 +333,7 @@ def test_workflow_schedules_beijing_night_and_publishes_outside_agent(self) -> N
self.assertIn("publish_nightly_findings.py", workflow)
self.assertIn("actions: write", workflow)
self.assertIn("sandbox: read-only", workflow)
self.assertIn("findings_json: ${{ steps.codex.outputs.final-message }}", workflow)
self.assertIn("findings_json: ${{ steps.codex.outputs.final-message || steps.claude.outputs.structured_output }}", workflow)
self.assertLess(workflow.index(" prepare:"), workflow.index(" review:"))
self.assertLess(workflow.index(" review:"), workflow.index(" publish:"))
prepare_job = workflow.split(" prepare:\n", 1)[1].split(" review:\n", 1)[0]
Expand All @@ -346,7 +346,8 @@ def test_workflow_schedules_beijing_night_and_publishes_outside_agent(self) -> N
self.assertNotIn("actions: write", review_job)
self.assertNotIn("GH_TOKEN:", review_job)
self.assertEqual(review_job.count("uses: openai/codex-action@v1"), 1)
self.assertTrue(review_job.rstrip().endswith("issues, or post comments. The publisher validates your final message."))
self.assertEqual(review_job.count("uses: anthropics/claude-code-action@c0949c50a78c1ec8ce2647db36629b870e715d0c"), 1)
self.assertIn("issues, or post comments. The publisher validates your final message.", review_job)
self.assertIn("ref: ${{ needs.prepare.outputs.commit }}", publish_job)
self.assertIn("name: nightly-codebase-review-input-${{ github.run_id }}", publish_job)
self.assertIn("GH_TOKEN: ${{ github.token }}", publish_job)
Expand Down
27 changes: 19 additions & 8 deletions .github/aicodingflow-tests/test_review_workflow_dispatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,10 +236,10 @@ def test_review_workflow_resolves_pr_before_checkout_and_uses_normalized_event(s
self.assertIn("--output \"$RUNNER_TEMP/pr_event.json\"", resolve_step["run"])
self.assertIn("--agent-login \"${{ vars.AGENT_LOGIN }}\"", resolve_step["run"])

pending_status_step = next(step for step in review_steps if step.get("name") == "mark-pr-review-status-pending")
pending_status_step = next(step for step in steps(data, "preflight") if step.get("name") == "mark-pr-review-status-pending")
self.assertEqual(
pending_status_step["if"],
"steps.pr.outputs.head_sha != '' && steps.pr.outputs.head_repo == github.repository",
"steps.pr.outputs.reviewable == 'true' && steps.seen.outputs.already_reviewed != 'true' && steps.pr.outputs.head_sha != '' && steps.pr.outputs.head_repo == github.repository",
)
self.assertEqual(pending_status_step["env"]["HEAD_SHA"], "${{ steps.pr.outputs.head_sha }}")
self.assertIn("repos/${{ github.repository }}/statuses/$HEAD_SHA", pending_status_step["run"])
Expand All @@ -256,7 +256,7 @@ def test_review_workflow_resolves_pr_before_checkout_and_uses_normalized_event(s
description_step = next(step for step in review_steps if step.get("name") == "snapshot-pr-description")
spec_context_step = next(step for step in review_steps if step.get("name") == "snapshot-spec-context")
discussion_context_step = next(step for step in review_steps if step.get("name") == "snapshot-review-discussion-context")
post_step = next(step for step in review_steps if step.get("name") == "post-pr-review")
post_step = next(step for step in steps(data, "publish") if step.get("name") == "post-pr-review")
self.assertEqual(description_step["env"]["PR_EVENT_PATH"], "${{ steps.pr.outputs.event_path }}")
self.assertIn("--output pr-worktree/pr_description.txt", description_step["run"])
self.assertEqual(spec_context_step["env"]["PR_EVENT_PATH"], "${{ steps.pr.outputs.event_path }}")
Expand All @@ -267,7 +267,7 @@ def test_review_workflow_resolves_pr_before_checkout_and_uses_normalized_event(s
self.assertIn(".github/scripts/prepare_review_discussion_context.py", discussion_context_step["run"])
self.assertIn("--pr-number \"${{ steps.pr.outputs.number }}\"", discussion_context_step["run"])
self.assertIn("--output pr-worktree/review_discussion_context.json", discussion_context_step["run"])
self.assertEqual(post_step["env"]["PR_EVENT_PATH"], "${{ steps.pr.outputs.event_path }}")
self.assertEqual(post_step["env"]["PR_EVENT_PATH"], "${{ runner.temp }}/pr_event.json")
self.assertEqual(post_step["env"]["REVIEW_BOT_LOGIN"], "${{ vars.REVIEW_BOT_LOGIN }}")
self.assertIn("--review pr-worktree/review.json", post_step["run"])
self.assertIn("--diff pr-worktree/pr_diff.txt", post_step["run"])
Expand Down Expand Up @@ -308,7 +308,7 @@ def test_review_workflow_resolves_pr_before_checkout_and_uses_normalized_event(s
)
self.assertEqual(ai_step["timeout-minutes"], 20)
self.assertIs(ai_step["with"]["allow-bots"], True)
self.assertIn("First change directory to pr-worktree", ai_step["with"]["prompt"])
self.assertIn("Inspect files under pr-worktree", ai_step["with"]["prompt"])
self.assertIn("Read .agents/contracts/review.md", ai_step["with"]["prompt"])
self.assertIn("shared review contract", ai_step["with"]["prompt"])
self.assertIn("must not override the contract", ai_step["with"]["prompt"])
Expand All @@ -324,14 +324,25 @@ def test_review_workflow_resolves_pr_before_checkout_and_uses_normalized_event(s
validate_step = next(step for step in review_steps if step.get("name") == "validate-review-output")
self.assertIn("pr-worktree/pr_diff.txt pr-worktree/review.json", validate_step["run"])

complete_status_step = next(step for step in review_steps if step.get("name") == "mark-pr-review-status-complete")
complete_status_step = next(step for step in steps(data, "publish") if step.get("name") == "mark-pr-review-status-complete")
self.assertEqual(
complete_status_step["if"],
"always() && steps.pr.outputs.head_sha != '' && steps.pr.outputs.head_repo == github.repository",
"always() && needs.preflight.outputs.head_sha != '' && needs.preflight.outputs.head_repo == github.repository",
)
self.assertEqual(complete_status_step["env"]["STATE"], "${{ job.status == 'success' && 'success' || 'failure' }}")
self.assertEqual(complete_status_step["env"]["STATE"], "${{ needs.review.result == 'success' && job.status == 'success' && 'success' || 'failure' }}")
self.assertIn("repos/${{ github.repository }}/statuses/$HEAD_SHA", complete_status_step["run"])

def test_review_agent_cannot_publish_with_write_token(self) -> None:
data = workflow(".github/workflows/review-pr.yml")
review = data["jobs"]["review"]
publish = data["jobs"]["publish"]
self.assertEqual(review["permissions"], {"contents": "read", "pull-requests": "read"})
self.assertEqual(review["outputs"]["diff_sha256"], "${{ steps.diff_snapshot.outputs.sha256 }}")
self.assertEqual(publish["permissions"]["pull-requests"], "write")
self.assertEqual(publish["permissions"]["statuses"], "write")
self.assertIn("sha256sum --check", next(step for step in publish["steps"] if step.get("name") == "verify-review-snapshot")["run"])
self.assertFalse(any(step.get("name") == "post-pr-review" for step in review["steps"]))

def test_triage_workflow_allows_regular_issue_authors_through_preflight(self) -> None:
data = workflow(".github/workflows/triage-issue.yml")
triggers = data[True]
Expand Down
36 changes: 36 additions & 0 deletions .github/aicodingflow-tests/test_validate_ai_provider.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
from __future__ import annotations

import unittest

from script_imports import import_script


provider = import_script(".github/scripts/validate_ai_provider.py", "validate_ai_provider")


class ValidateAIProviderTest(unittest.TestCase):
def test_default_codex_and_explicit_codex(self) -> None:
self.assertEqual(provider.validate("", "key", "", ""), "codex")
self.assertEqual(provider.validate("codex", "key", "", ""), "codex")

def test_claude_uses_only_its_own_credentials(self) -> None:
self.assertEqual(provider.validate("claude", "", "key", "claude-sonnet-4-5"), "claude")

def test_missing_selected_credential_fails(self) -> None:
with self.assertRaisesRegex(ValueError, "CODEX_API_KEY"):
provider.validate("", "", "key", "claude-sonnet-4-5")
with self.assertRaisesRegex(ValueError, "ANTHROPIC_API_KEY"):
provider.validate("claude", "key", "", "claude-sonnet-4-5")

def test_claude_model_must_be_safe_for_action_arguments(self) -> None:
for model in ("", "model name", 'model" --foo', "model\n--foo"):
with self.subTest(model=model), self.assertRaisesRegex(ValueError, "CLAUDE_MODEL"):
provider.validate("claude", "", "key", model)

def test_unknown_provider_fails(self) -> None:
with self.assertRaisesRegex(ValueError, "AI_PROVIDER"):
provider.validate("other", "key", "key", "claude-sonnet-4-5")


if __name__ == "__main__":
unittest.main()
Loading
Loading