Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions src/autoskillit/_probe_canary.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""Reusable canary state machine and GitHub issue updater for live probes and E2E runs.

IL-1 module: imports only stdlib and `autoskillit.core`. Provides the persistence +
IL-1 module: imports only stdlib, `regex` and `autoskillit.core`. Provides the persistence +
flake-guard primitives live probes build on and the E2E workflow's failure issue.
"""

Expand All @@ -14,6 +14,8 @@
from enum import StrEnum, unique
from pathlib import Path

import regex as re

from autoskillit.core import atomic_write, get_logger, run_gh

logger = get_logger(__name__)
Expand Down Expand Up @@ -148,19 +150,19 @@ def _create(self, title: str, body: str) -> int:
f"{self._owner}/{self._repo}",
"--title",
title,
"--json",
"number",
],
body,
)
if result.returncode != 0:
msg = f"gh issue create failed: {result.stderr}"
raise RuntimeError(msg)
try:
issue_number = json.loads(result.stdout)["number"]
except (json.JSONDecodeError, KeyError) as exc:
match = re.fullmatch(
r"https?://[^/\s]+/[^/\s]+/[^/\s]+/issues/([0-9]+)", result.stdout.strip()
)
issue_number = int(match.group(1)) if match else 0
if issue_number < 1:
msg = f"gh issue create returned unexpected output: {result.stdout!r}"
raise RuntimeError(msg) from exc
raise RuntimeError(msg)
return issue_number

def _current_body(self, number: int) -> str:
Expand Down
109 changes: 97 additions & 12 deletions tests/execution/backends/test_probe_canary.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
from __future__ import annotations

import json
import shutil
from pathlib import Path
from subprocess import CompletedProcess
from subprocess import CompletedProcess, run

import pytest
import structlog.testing
Expand All @@ -17,7 +18,7 @@
_cli_main,
)

pytestmark = [pytest.mark.layer("execution"), pytest.mark.small]
pytestmark = [pytest.mark.layer("execution"), pytest.mark.medium]


class TestCanaryStateLoad:
Expand Down Expand Up @@ -127,27 +128,105 @@ def test_values(self) -> None:


class TestIssueUpdater:
def test_ensure_issue_creates_new(self, monkeypatch: pytest.MonkeyPatch) -> None:
@pytest.mark.parametrize(
"stdout",
[
"https://github.com/test-org/test-repo/issues/123\n",
"\n http://ghe.example.com:8080/test-org/test-repo/issues/123 \n",
],
ids=["https", "http-enterprise-host"],
)
def test_ensure_issue_creates_new(self, monkeypatch: pytest.MonkeyPatch, stdout: str) -> None:
def mock_run_gh(args, **kwargs):
if args[0:2] == ["issue", "list"]:
return CompletedProcess(args=args, returncode=0, stdout="[]", stderr="")
if args[0:2] == ["issue", "create"]:
assert "--body-file" in args
return CompletedProcess(
args=args,
returncode=0,
stdout=json.dumps({"number": 123}),
stderr="",
)
return CompletedProcess(args=args, returncode=0, stdout=stdout, stderr="")
return CompletedProcess(args=args, returncode=1, stdout="", stderr="")

monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", mock_run_gh)
updater = IssueUpdater(owner="test-org", repo="test-repo")
state = CanaryState()
num = updater.ensure_issue(state, "Probe failure", "Details here")
assert type(num) is int
assert num == 123
assert type(state.last_issue_number) is int
assert state.last_issue_number == 123

@pytest.mark.parametrize(
"stdout",
[
json.dumps({"number": 123}),
"https://github.com/test-org/test-repo/issues/abc\n",
"https://github.com/test-org/test-repo/issues/0\n",
],
ids=["legacy-json", "nonnumeric-issue", "zero-issue"],
)
def test_ensure_issue_rejects_unexpected_create_output(
self, monkeypatch: pytest.MonkeyPatch, stdout: str
) -> None:
body_files: list[Path] = []

def mock_run_gh(args, **kwargs):
if args[0:2] == ["issue", "list"]:
return CompletedProcess(args=args, returncode=0, stdout="[]", stderr="")
if args[0:2] == ["issue", "create"]:
body_files.append(Path(args[args.index("--body-file") + 1]))
return CompletedProcess(args=args, returncode=0, stdout=stdout, stderr="")
raise AssertionError(f"Unexpected gh call: {args}")

monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", mock_run_gh)
updater = IssueUpdater(owner="test-org", repo="test-repo")
state = CanaryState(last_issue_number=5)
with pytest.raises(RuntimeError, match="gh issue create returned unexpected output"):
updater.ensure_issue(state, "Probe failure", "Details")
assert state.last_issue_number == 5
assert len(body_files) == 1
assert not body_files[0].exists()

def test_issue_create_flags_are_accepted_by_installed_gh(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Run the production create argv with `--help` appended so nothing is created.

Only flag validity is checked: `gh issue create` has no dry-run, so its stdout shape
is covered by the URL fixtures above, not by the real binary.
"""
gh_executable = shutil.which("gh")
if gh_executable is None:
pytest.skip("gh binary not available in this environment")
body = "Contract body"
body_files: list[Path] = []

def run_help_instead_of_create(args, **kwargs):
assert "--body-file" in args
body_file = Path(args[args.index("--body-file") + 1])
body_files.append(body_file)
assert body_file.read_text() == body
help_result = run(
[gh_executable, *args, "--help"],
capture_output=True,
text=True,
timeout=10,
check=False,
)
assert help_result.returncode == 0, help_result.stderr
assert "--json" not in args
return CompletedProcess(
args=args,
returncode=0,
stdout="https://github.com/test-org/test-repo/issues/123\n",
stderr="",
)

monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", run_help_instead_of_create)
updater = IssueUpdater(owner="test-org", repo="test-repo")
number = updater._create("Contract title", body)
assert number == 123
assert len(body_files) == 1
assert not body_files[0].exists()

def test_ensure_issue_updates_existing(self, monkeypatch: pytest.MonkeyPatch) -> None:
def mock_run_gh(args, **kwargs):
if args[0:2] == ["issue", "list"]:
Expand All @@ -174,14 +253,20 @@ def mock_run_gh(args, **kwargs):
if args[0:2] == ["issue", "list"]:
return CompletedProcess(args=args, returncode=0, stdout="[]", stderr="")
if args[0:2] == ["issue", "create"]:
return CompletedProcess(args=args, returncode=1, stdout="", stderr="auth error")
return CompletedProcess(
args=args,
returncode=1,
stdout="https://github.com/test-org/test-repo/issues/5\n",
stderr="auth error",
)
raise AssertionError(f"Unexpected gh call: {args}")

monkeypatch.setattr(_patch_autoskillit__probe_canary, "run_gh", mock_run_gh)
updater = IssueUpdater(owner="test-org", repo="test-repo")
state = CanaryState()
with pytest.raises(RuntimeError, match="gh issue create failed"):
updater.ensure_issue(state, "Probe failure", "Details")
assert state.last_issue_number is None

def test_ensure_issue_logs_on_edit_failure(self, monkeypatch: pytest.MonkeyPatch) -> None:
def mock_run_gh(args, **kwargs):
Expand Down Expand Up @@ -243,7 +328,7 @@ def __call__(self, args, **kwargs):
stdout = json.dumps({"body": self.body})
return CompletedProcess(args=args, returncode=0, stdout=stdout, stderr="")
if command == "create":
stdout = json.dumps({"number": 99})
stdout = "https://github.com/test-org/test-repo/issues/99\n"
return CompletedProcess(args=args, returncode=0, stdout=stdout, stderr="")
return CompletedProcess(args=args, returncode=0, stdout="", stderr="")

Expand Down Expand Up @@ -441,7 +526,7 @@ def mock_run_gh(args, **kwargs):
return CompletedProcess(
args=args,
returncode=0,
stdout=json.dumps({"number": 7}),
stdout="https://github.com/test-org/test-repo/issues/7\n",
stderr="",
)
return CompletedProcess(args=args, returncode=1, stdout="", stderr="")
Expand Down
Loading