diff --git a/src/autoskillit/_probe_canary.py b/src/autoskillit/_probe_canary.py index 2bbf8d492d..bee790128f 100644 --- a/src/autoskillit/_probe_canary.py +++ b/src/autoskillit/_probe_canary.py @@ -1,6 +1,6 @@ """Reusable canary state machine and GitHub issue updater for live probes and E2E runs. -IL-1 module: imports only stdlib and `autoskillit.core`. Provides the persistence + +IL-1 module: imports only stdlib, `regex` and `autoskillit.core`. Provides the persistence + flake-guard primitives live probes build on and the E2E workflow's failure issue. """ @@ -14,6 +14,8 @@ from enum import StrEnum, unique from pathlib import Path +import regex as re + from autoskillit.core import atomic_write, get_logger, run_gh logger = get_logger(__name__) @@ -148,19 +150,19 @@ def _create(self, title: str, body: str) -> int: f"{self._owner}/{self._repo}", "--title", title, - "--json", - "number", ], body, ) if result.returncode != 0: msg = f"gh issue create failed: {result.stderr}" raise RuntimeError(msg) - try: - issue_number = json.loads(result.stdout)["number"] - except (json.JSONDecodeError, KeyError) as exc: + match = re.fullmatch( + r"https?://[^/\s]+/[^/\s]+/[^/\s]+/issues/([0-9]+)", result.stdout.strip() + ) + issue_number = int(match.group(1)) if match else 0 + if issue_number < 1: msg = f"gh issue create returned unexpected output: {result.stdout!r}" - raise RuntimeError(msg) from exc + raise RuntimeError(msg) return issue_number def _current_body(self, number: int) -> str: diff --git a/tests/execution/backends/test_probe_canary.py b/tests/execution/backends/test_probe_canary.py index 06ef2f2389..bb7082786e 100644 --- a/tests/execution/backends/test_probe_canary.py +++ b/tests/execution/backends/test_probe_canary.py @@ -1,8 +1,9 @@ from __future__ import annotations import json +import shutil from pathlib import Path -from subprocess import CompletedProcess +from subprocess import CompletedProcess, run import pytest import structlog.testing @@ -17,7 +18,7 @@ _cli_main, ) -pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] class TestCanaryStateLoad: @@ -127,27 +128,105 @@ def test_values(self) -> None: class TestIssueUpdater: - def test_ensure_issue_creates_new(self, monkeypatch: pytest.MonkeyPatch) -> None: + @pytest.mark.parametrize( + "stdout", + [ + "https://github.com/test-org/test-repo/issues/123\n", + "\n http://ghe.example.com:8080/test-org/test-repo/issues/123 \n", + ], + ids=["https", "http-enterprise-host"], + ) + def test_ensure_issue_creates_new(self, monkeypatch: pytest.MonkeyPatch, stdout: str) -> None: def mock_run_gh(args, **kwargs): if args[0:2] == ["issue", "list"]: return CompletedProcess(args=args, returncode=0, stdout="[]", stderr="") if args[0:2] == ["issue", "create"]: assert "--body-file" in args - return CompletedProcess( - args=args, - returncode=0, - stdout=json.dumps({"number": 123}), - stderr="", - ) + return CompletedProcess(args=args, returncode=0, stdout=stdout, stderr="") return CompletedProcess(args=args, returncode=1, stdout="", stderr="") monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", mock_run_gh) updater = IssueUpdater(owner="test-org", repo="test-repo") state = CanaryState() num = updater.ensure_issue(state, "Probe failure", "Details here") + assert type(num) is int assert num == 123 + assert type(state.last_issue_number) is int assert state.last_issue_number == 123 + @pytest.mark.parametrize( + "stdout", + [ + json.dumps({"number": 123}), + "https://github.com/test-org/test-repo/issues/abc\n", + "https://github.com/test-org/test-repo/issues/0\n", + ], + ids=["legacy-json", "nonnumeric-issue", "zero-issue"], + ) + def test_ensure_issue_rejects_unexpected_create_output( + self, monkeypatch: pytest.MonkeyPatch, stdout: str + ) -> None: + body_files: list[Path] = [] + + def mock_run_gh(args, **kwargs): + if args[0:2] == ["issue", "list"]: + return CompletedProcess(args=args, returncode=0, stdout="[]", stderr="") + if args[0:2] == ["issue", "create"]: + body_files.append(Path(args[args.index("--body-file") + 1])) + return CompletedProcess(args=args, returncode=0, stdout=stdout, stderr="") + raise AssertionError(f"Unexpected gh call: {args}") + + monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", mock_run_gh) + updater = IssueUpdater(owner="test-org", repo="test-repo") + state = CanaryState(last_issue_number=5) + with pytest.raises(RuntimeError, match="gh issue create returned unexpected output"): + updater.ensure_issue(state, "Probe failure", "Details") + assert state.last_issue_number == 5 + assert len(body_files) == 1 + assert not body_files[0].exists() + + def test_issue_create_flags_are_accepted_by_installed_gh( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Run the production create argv with `--help` appended so nothing is created. + + Only flag validity is checked: `gh issue create` has no dry-run, so its stdout shape + is covered by the URL fixtures above, not by the real binary. + """ + gh_executable = shutil.which("gh") + if gh_executable is None: + pytest.skip("gh binary not available in this environment") + body = "Contract body" + body_files: list[Path] = [] + + def run_help_instead_of_create(args, **kwargs): + assert "--body-file" in args + body_file = Path(args[args.index("--body-file") + 1]) + body_files.append(body_file) + assert body_file.read_text() == body + help_result = run( + [gh_executable, *args, "--help"], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert help_result.returncode == 0, help_result.stderr + assert "--json" not in args + return CompletedProcess( + args=args, + returncode=0, + stdout="https://github.com/test-org/test-repo/issues/123\n", + stderr="", + ) + + monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", run_help_instead_of_create) + updater = IssueUpdater(owner="test-org", repo="test-repo") + number = updater._create("Contract title", body) + assert number == 123 + assert len(body_files) == 1 + assert not body_files[0].exists() + def test_ensure_issue_updates_existing(self, monkeypatch: pytest.MonkeyPatch) -> None: def mock_run_gh(args, **kwargs): if args[0:2] == ["issue", "list"]: @@ -174,7 +253,12 @@ def mock_run_gh(args, **kwargs): if args[0:2] == ["issue", "list"]: return CompletedProcess(args=args, returncode=0, stdout="[]", stderr="") if args[0:2] == ["issue", "create"]: - return CompletedProcess(args=args, returncode=1, stdout="", stderr="auth error") + return CompletedProcess( + args=args, + returncode=1, + stdout="https://github.com/test-org/test-repo/issues/5\n", + stderr="auth error", + ) raise AssertionError(f"Unexpected gh call: {args}") monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", mock_run_gh) @@ -182,6 +266,7 @@ def mock_run_gh(args, **kwargs): state = CanaryState() with pytest.raises(RuntimeError, match="gh issue create failed"): updater.ensure_issue(state, "Probe failure", "Details") + assert state.last_issue_number is None def test_ensure_issue_logs_on_edit_failure(self, monkeypatch: pytest.MonkeyPatch) -> None: def mock_run_gh(args, **kwargs): @@ -243,7 +328,7 @@ def __call__(self, args, **kwargs): stdout = json.dumps({"body": self.body}) return CompletedProcess(args=args, returncode=0, stdout=stdout, stderr="") if command == "create": - stdout = json.dumps({"number": 99}) + stdout = "https://github.com/test-org/test-repo/issues/99\n" return CompletedProcess(args=args, returncode=0, stdout=stdout, stderr="") return CompletedProcess(args=args, returncode=0, stdout="", stderr="") @@ -441,7 +526,7 @@ def mock_run_gh(args, **kwargs): return CompletedProcess( args=args, returncode=0, - stdout=json.dumps({"number": 7}), + stdout="https://github.com/test-org/test-repo/issues/7\n", stderr="", ) return CompletedProcess(args=args, returncode=1, stdout="", stderr="")