From 8539eaf72009719287d7158f7f2374cdc2cd7ae6 Mon Sep 17 00:00:00 2001 From: Trecek Date: Wed, 4 Nov 2026 08:11:21 -0800 Subject: [PATCH 1/5] fix: parse issue URL from gh issue create instead of requesting JSON (#5249) IssueUpdater._create passed `--json number` to `gh issue create`, which the CLI does not support, and parsed JSON from stdout. Permissive mocks returned JSON, so no test noticed. Drop the flag and parse the documented issue URL (HTTP(S), any host) into a positive int, keeping the nonzero-exit check ahead of parsing and the existing diagnostics. Correct the creation fixtures to URL output, add unexpected-output cases, and add a contract test that runs the production arguments against the installed `gh` with `--help` appended. The module now spawns a subprocess, so it is classified medium. Co-Authored-By: Claude Sonnet 5.5 --- src/autoskillit/_probe_canary.py | 16 +-- tests/execution/backends/test_probe_canary.py | 101 ++++++++++++++++-- 2 files changed, 99 insertions(+), 18 deletions(-) diff --git a/src/autoskillit/_probe_canary.py b/src/autoskillit/_probe_canary.py index 2bbf8d492d..bee790128f 100644 --- a/src/autoskillit/_probe_canary.py +++ b/src/autoskillit/_probe_canary.py @@ -1,6 +1,6 @@ """Reusable canary state machine and GitHub issue updater for live probes and E2E runs. -IL-1 module: imports only stdlib and `autoskillit.core`. Provides the persistence + +IL-1 module: imports only stdlib, `regex` and `autoskillit.core`. Provides the persistence + flake-guard primitives live probes build on and the E2E workflow's failure issue. """ @@ -14,6 +14,8 @@ from enum import StrEnum, unique from pathlib import Path +import regex as re + from autoskillit.core import atomic_write, get_logger, run_gh logger = get_logger(__name__) @@ -148,19 +150,19 @@ def _create(self, title: str, body: str) -> int: f"{self._owner}/{self._repo}", "--title", title, - "--json", - "number", ], body, ) if result.returncode != 0: msg = f"gh issue create failed: {result.stderr}" raise RuntimeError(msg) - try: - issue_number = json.loads(result.stdout)["number"] - except (json.JSONDecodeError, KeyError) as exc: + match = re.fullmatch( + r"https?://[^/\s]+/[^/\s]+/[^/\s]+/issues/([0-9]+)", result.stdout.strip() + ) + issue_number = int(match.group(1)) if match else 0 + if issue_number < 1: msg = f"gh issue create returned unexpected output: {result.stdout!r}" - raise RuntimeError(msg) from exc + raise RuntimeError(msg) return issue_number def _current_body(self, number: int) -> str: diff --git a/tests/execution/backends/test_probe_canary.py b/tests/execution/backends/test_probe_canary.py index 06ef2f2389..24f50a3463 100644 --- a/tests/execution/backends/test_probe_canary.py +++ b/tests/execution/backends/test_probe_canary.py @@ -1,6 +1,8 @@ from __future__ import annotations import json +import shutil +import subprocess from pathlib import Path from subprocess import CompletedProcess @@ -17,7 +19,7 @@ _cli_main, ) -pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] +pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium] class TestCanaryStateLoad: @@ -127,27 +129,98 @@ def test_values(self) -> None: class TestIssueUpdater: - def test_ensure_issue_creates_new(self, monkeypatch: pytest.MonkeyPatch) -> None: + @pytest.mark.parametrize( + "stdout", + [ + "https://github.com/test-org/test-repo/issues/123\n", + "\n http://ghe.example.com:8080/test-org/test-repo/issues/123 \n", + ], + ids=["https", "http-enterprise-host"], + ) + def test_ensure_issue_creates_new(self, monkeypatch: pytest.MonkeyPatch, stdout: str) -> None: def mock_run_gh(args, **kwargs): if args[0:2] == ["issue", "list"]: return CompletedProcess(args=args, returncode=0, stdout="[]", stderr="") if args[0:2] == ["issue", "create"]: assert "--body-file" in args - return CompletedProcess( - args=args, - returncode=0, - stdout=json.dumps({"number": 123}), - stderr="", - ) + return CompletedProcess(args=args, returncode=0, stdout=stdout, stderr="") return CompletedProcess(args=args, returncode=1, stdout="", stderr="") monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", mock_run_gh) updater = IssueUpdater(owner="test-org", repo="test-repo") state = CanaryState() num = updater.ensure_issue(state, "Probe failure", "Details here") + assert type(num) is int assert num == 123 + assert type(state.last_issue_number) is int assert state.last_issue_number == 123 + @pytest.mark.parametrize( + "stdout", + [ + json.dumps({"number": 123}), + "https://github.com/test-org/test-repo/issues/abc\n", + "https://github.com/test-org/test-repo/issues/0\n", + ], + ids=["legacy-json", "nonnumeric-issue", "zero-issue"], + ) + def test_ensure_issue_rejects_unexpected_create_output( + self, monkeypatch: pytest.MonkeyPatch, stdout: str + ) -> None: + body_files: list[Path] = [] + + def mock_run_gh(args, **kwargs): + if args[0:2] == ["issue", "list"]: + return CompletedProcess(args=args, returncode=0, stdout="[]", stderr="") + if args[0:2] == ["issue", "create"]: + body_files.append(Path(args[args.index("--body-file") + 1])) + return CompletedProcess(args=args, returncode=0, stdout=stdout, stderr="") + raise AssertionError(f"Unexpected gh call: {args}") + + monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", mock_run_gh) + updater = IssueUpdater(owner="test-org", repo="test-repo") + state = CanaryState(last_issue_number=5) + with pytest.raises(RuntimeError, match="gh issue create returned unexpected output"): + updater.ensure_issue(state, "Probe failure", "Details") + assert state.last_issue_number == 5 + assert len(body_files) == 1 + assert not body_files[0].exists() + + def test_issue_create_arguments_are_supported_by_installed_gh( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + gh_executable = shutil.which("gh") + assert gh_executable is not None, "the gh CLI must be installed for this contract test" + body = "Contract body" + body_files: list[Path] = [] + + def run_help_instead_of_create(args, **kwargs): + body_file = Path(args[args.index("--body-file") + 1]) + body_files.append(body_file) + assert body_file.read_text() == body + help_result = subprocess.run( + [gh_executable, *args, "--help"], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert help_result.returncode == 0, help_result.stderr + assert "--json" not in args + return CompletedProcess( + args=args, + returncode=0, + stdout="https://github.com/test-org/test-repo/issues/123\n", + stderr="", + ) + + monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", run_help_instead_of_create) + updater = IssueUpdater(owner="test-org", repo="test-repo") + number = updater._create("Contract title", body) + assert number == 123 + assert len(body_files) == 1 + assert not body_files[0].exists() + def test_ensure_issue_updates_existing(self, monkeypatch: pytest.MonkeyPatch) -> None: def mock_run_gh(args, **kwargs): if args[0:2] == ["issue", "list"]: @@ -174,7 +247,12 @@ def mock_run_gh(args, **kwargs): if args[0:2] == ["issue", "list"]: return CompletedProcess(args=args, returncode=0, stdout="[]", stderr="") if args[0:2] == ["issue", "create"]: - return CompletedProcess(args=args, returncode=1, stdout="", stderr="auth error") + return CompletedProcess( + args=args, + returncode=1, + stdout="https://github.com/test-org/test-repo/issues/5\n", + stderr="auth error", + ) raise AssertionError(f"Unexpected gh call: {args}") monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", mock_run_gh) @@ -182,6 +260,7 @@ def mock_run_gh(args, **kwargs): state = CanaryState() with pytest.raises(RuntimeError, match="gh issue create failed"): updater.ensure_issue(state, "Probe failure", "Details") + assert state.last_issue_number is None def test_ensure_issue_logs_on_edit_failure(self, monkeypatch: pytest.MonkeyPatch) -> None: def mock_run_gh(args, **kwargs): @@ -243,7 +322,7 @@ def __call__(self, args, **kwargs): stdout = json.dumps({"body": self.body}) return CompletedProcess(args=args, returncode=0, stdout=stdout, stderr="") if command == "create": - stdout = json.dumps({"number": 99}) + stdout = "https://github.com/test-org/test-repo/issues/99\n" return CompletedProcess(args=args, returncode=0, stdout=stdout, stderr="") return CompletedProcess(args=args, returncode=0, stdout="", stderr="") @@ -441,7 +520,7 @@ def mock_run_gh(args, **kwargs): return CompletedProcess( args=args, returncode=0, - stdout=json.dumps({"number": 7}), + stdout="https://github.com/test-org/test-repo/issues/7\n", stderr="", ) return CompletedProcess(args=args, returncode=1, stdout="", stderr="") From e78abbca3be1e1ee8c9c31df8cb130061dcb7184 Mon Sep 17 00:00:00 2001 From: Trecek Date: Wed, 4 Nov 2026 12:06:26 -0800 Subject: [PATCH 2/5] fix(review): narrow gh contract test name to flag validity The test only runs `gh issue create ... --help`, which validates flags but cannot observe the real stdout shape (gh issue create has no dry-run). Rename it to say what it checks and document where the stdout contract is pinned. --- tests/execution/backends/test_probe_canary.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/execution/backends/test_probe_canary.py b/tests/execution/backends/test_probe_canary.py index 24f50a3463..c41e2ccbee 100644 --- a/tests/execution/backends/test_probe_canary.py +++ b/tests/execution/backends/test_probe_canary.py @@ -186,9 +186,14 @@ def mock_run_gh(args, **kwargs): assert len(body_files) == 1 assert not body_files[0].exists() - def test_issue_create_arguments_are_supported_by_installed_gh( + def test_issue_create_flags_are_accepted_by_installed_gh( self, monkeypatch: pytest.MonkeyPatch ) -> None: + """Run the production create argv with `--help` appended so nothing is created. + + Only flag validity is checked: `gh issue create` has no dry-run, so its stdout shape + is covered by the URL fixtures above, not by the real binary. + """ gh_executable = shutil.which("gh") assert gh_executable is not None, "the gh CLI must be installed for this contract test" body = "Contract body" From 43f39544a1b02a8644dc620470acf90f552dd6d5 Mon Sep 17 00:00:00 2001 From: Trecek Date: Wed, 4 Nov 2026 12:06:43 -0800 Subject: [PATCH 3/5] fix(review): assert --body-file is passed before indexing it If production stopped passing --body-file, the gh contract test failed with a bare ValueError from list.index. Assert the flag first, matching the style of the neighbouring tests, so the failure names the contract violation. --- tests/execution/backends/test_probe_canary.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/execution/backends/test_probe_canary.py b/tests/execution/backends/test_probe_canary.py index c41e2ccbee..6c5d587b58 100644 --- a/tests/execution/backends/test_probe_canary.py +++ b/tests/execution/backends/test_probe_canary.py @@ -200,6 +200,7 @@ def test_issue_create_flags_are_accepted_by_installed_gh( body_files: list[Path] = [] def run_help_instead_of_create(args, **kwargs): + assert "--body-file" in args body_file = Path(args[args.index("--body-file") + 1]) body_files.append(body_file) assert body_file.read_text() == body From d60a708c635463949ac619a36288a02e229761a9 Mon Sep 17 00:00:00 2001 From: Trecek Date: Wed, 4 Nov 2026 12:06:53 -0800 Subject: [PATCH 4/5] fix(review): skip the gh contract test when gh is unavailable The live-CLI contract test hard-asserted that gh is installed, so on a host without the CLI it errored instead of skipping. Follow the repo's live-CLI contract convention (tests/AGENTS.md, CLI Flag-Spec Coverage) and skip only when the binary is absent. --- tests/execution/backends/test_probe_canary.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/execution/backends/test_probe_canary.py b/tests/execution/backends/test_probe_canary.py index 6c5d587b58..8f61b8e611 100644 --- a/tests/execution/backends/test_probe_canary.py +++ b/tests/execution/backends/test_probe_canary.py @@ -195,7 +195,8 @@ def test_issue_create_flags_are_accepted_by_installed_gh( is covered by the URL fixtures above, not by the real binary. """ gh_executable = shutil.which("gh") - assert gh_executable is not None, "the gh CLI must be installed for this contract test" + if gh_executable is None: + pytest.skip("gh binary not available in this environment") body = "Contract body" body_files: list[Path] = [] From 8351cc42d22e61a34ca4ed14d7670be57422f19d Mon Sep 17 00:00:00 2001 From: Trecek Date: Wed, 4 Nov 2026 12:07:07 -0800 Subject: [PATCH 5/5] fix(review): use one import style for subprocess in test_probe_canary The module imported both `subprocess` and `from subprocess import CompletedProcess`. Consolidate on the existing from-import convention. --- tests/execution/backends/test_probe_canary.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/execution/backends/test_probe_canary.py b/tests/execution/backends/test_probe_canary.py index 8f61b8e611..bb7082786e 100644 --- a/tests/execution/backends/test_probe_canary.py +++ b/tests/execution/backends/test_probe_canary.py @@ -2,9 +2,8 @@ import json import shutil -import subprocess from pathlib import Path -from subprocess import CompletedProcess +from subprocess import CompletedProcess, run import pytest import structlog.testing @@ -205,7 +204,7 @@ def run_help_instead_of_create(args, **kwargs): body_file = Path(args[args.index("--body-file") + 1]) body_files.append(body_file) assert body_file.read_text() == body - help_result = subprocess.run( + help_result = run( [gh_executable, *args, "--help"], capture_output=True, text=True,