From e5bfacd5a5ada7d0b2ace616c6586f64b341c505 Mon Sep 17 00:00:00 2001 From: Jared Wein Date: Mon, 29 Jun 2026 16:04:22 -0400 Subject: [PATCH] test_runner: add --prompt mode driving the Firefox DevTools MCP Add an agent-driven verdict mode, the natural-language equivalent of --command (like `git bisect run`). `--prompt ""` shells out to the `claude` CLI in headless mode, pointed at the @mozilla/firefox-devtools-mcp server via a generated --mcp-config, to inspect each build and decide good/bad. The MCP launches the build itself, so AgentTestRunner installs the build to obtain the binary path (passed via --firefox-path) without starting it. Verdicts are parsed as GOOD/BAD from the agent output. Gating: the Firefox DevTools MCP supports Firefox 100+, so ranges that predate it are rejected. This happens both up front (resolved good/bad range in cli.validate) and per build (mozversion application_version), configurable via --prompt-min-version (default 100). A pre-run check (Application.check_prerequisites) fails fast before bisecting if `claude`/`npx` are missing or if the instruction is not usable for a good/bad determination. --prompt is mutually exclusive with --command and --launch. Adds --prompt-headless and --prompt-model. New UnsupportedVersionError. --- mozregression/cli.py | 108 ++++++++++++- mozregression/errors.py | 15 ++ mozregression/main.py | 18 ++- mozregression/test_runner.py | 268 ++++++++++++++++++++++++++++++++- tests/unit/test_cli.py | 48 ++++++ tests/unit/test_main.py | 34 ++++- tests/unit/test_test_runner.py | 177 ++++++++++++++++++++++ 7 files changed, 663 insertions(+), 5 deletions(-) diff --git a/mozregression/cli.py b/mozregression/cli.py index 835ec623d..3a15c8889 100644 --- a/mozregression/cli.py +++ b/mozregression/cli.py @@ -21,7 +21,12 @@ from mozregression.branches import get_name from mozregression.config import DEFAULT_CONF_FNAME, get_config, write_config from mozregression.dates import is_date_or_datetime, parse_date, to_datetime -from mozregression.errors import DateFormatError, MozRegressionError, UnavailableRelease +from mozregression.errors import ( + DateFormatError, + MozRegressionError, + UnavailableRelease, + UnsupportedVersionError, +) from mozregression.fetch_configs import REGISTRY as FC_REGISTRY from mozregression.fetch_configs import create_config from mozregression.log import colorize, init_logger @@ -301,6 +306,79 @@ def create_parser(defaults): ), ) + parser.add_argument( + "--prompt", + help=( + "Evaluate builds automatically with an LLM agent driving the" + " Firefox DevTools MCP. Give a natural language instruction" + " describing what to check, e.g." + ' --prompt "open example.com and tell me if the search box is' + " present\". The build is judged good or bad based on the agent's" + " findings. Mutually exclusive with --command. Requires the" + " `claude` CLI (installed and authenticated) and Node/`npx` on the" + " PATH. Note: the Firefox DevTools MCP only supports recent Firefox" + " versions, so older regression ranges are rejected (see" + " --prompt-min-version)." + ), + ) + + parser.add_argument( + "--prompt-min-version", + type=int, + default=100, + help=( + "Minimum Firefox major version the Firefox DevTools MCP supports," + " used to gate the --prompt option. Builds (and regression ranges)" + " older than this are rejected. Defaults to %(default)s." + ), + ) + + parser.add_argument( + "--prompt-headless", + action="store_true", + help="Run the --prompt Firefox build in headless mode.", + ) + + parser.add_argument( + "--prompt-model", + default=None, + help=( + "Model passed to the `claude` CLI for the per-build --prompt agent" + " that drives the browser. Defaults to the `claude` CLI's configured" + " model." + ), + ) + + parser.add_argument( + "--prompt-recheck-mcp", + action="store_true", + help=( + "Re-check the npm registry for the latest Firefox DevTools MCP" + " version on every build. By default the version npx already has" + " cached is reused (no per-build registry round-trip)." + ), + ) + + parser.add_argument( + "--prompt-allow-other-mcp", + action="store_true", + help=( + "Also load your other configured MCP servers when running the" + " --prompt agent. By default only the Firefox DevTools MCP is used" + " (claude is run with --strict-mcp-config)." + ), + ) + + parser.add_argument( + "--max-budget-usd", + type=float, + default=10.0, + help=( + "Maximum dollar amount the --prompt agent may spend per build," + " passed to the `claude` CLI. Defaults to %(default)s." + ), + ) + parser.add_argument( "--persist", default=defaults["persist"], @@ -554,6 +632,26 @@ def _convert_to_bisect_arg(self, value): self.logger.info("%s is not a release, assuming it's a hash..." % value) return value + def _check_prompt_min_version(self, options): + """ + Up-front gate for the --prompt option: reject regression ranges that + predate the minimum Firefox version supported by the Firefox DevTools + MCP. + + Only date-based endpoints (dates, buildids, release numbers) can be + checked here without extra network requests; raw changeset endpoints are + left to the authoritative per-build version check in AgentTestRunner. + """ + try: + cutoff = to_datetime(parse_date(date_of_release(options.prompt_min_version))) + except (UnavailableRelease, DateFormatError): + # we can't resolve a cutoff date for this version; rely on the + # per-build check instead. + return + for endpoint in (options.good, options.bad): + if is_date_or_datetime(endpoint) and to_datetime(endpoint) < cutoff: + raise UnsupportedVersionError(endpoint, options.prompt_min_version) + def validate(self): """ Validate the options, define the `action` and `fetch_config` that @@ -561,6 +659,12 @@ def validate(self): """ options = self.options + if options.prompt is not None: + if options.command is not None: + raise MozRegressionError("--prompt can not be used together with --command.") + if options.launch: + raise MozRegressionError("--prompt can not be used together with --launch.") + arch_options = { "firefox": [ "aarch64", @@ -703,6 +807,8 @@ def validate(self): ) if fetch_config.should_use_archive(): self.action = "bisect_nightlies" + if options.prompt is not None: + self._check_prompt_min_version(options) if ( self.action in ("launch_integration", "bisect_integration") and not fetch_config.is_integration() diff --git a/mozregression/errors.py b/mozregression/errors.py index b4fb017dc..e0032df22 100644 --- a/mozregression/errors.py +++ b/mozregression/errors.py @@ -16,6 +16,21 @@ def __init__(self): MozRegressionError.__init__(self, "Can't run Windows builds before" " 2010-03-18") +class UnsupportedVersionError(MozRegressionError): + """ + Raised when a build's Firefox version is too old to be driven by the + Firefox DevTools MCP used by the ``--prompt`` option. + """ + + def __init__(self, build, min_version): + MozRegressionError.__init__( + self, + "Build %s is too old for --prompt: the Firefox DevTools MCP requires" + " Firefox %s or later. Narrow the regression range, or lower" + " --prompt-min-version if you know it works on older builds." % (build, min_version), + ) + + class DateFormatError(MozRegressionError): """ Raised when a date can not be parsed from a string. diff --git a/mozregression/main.py b/mozregression/main.py index 4b63ce9dd..34c89a6f3 100644 --- a/mozregression/main.py +++ b/mozregression/main.py @@ -30,7 +30,7 @@ from mozregression.persist_limit import PersistLimit from mozregression.telemetry import UsageMetrics, get_system_info, send_telemetry_ping_oop from mozregression.tempdir import safe_mkdtemp -from mozregression.test_runner import CommandTestRunner, ManualTestRunner +from mozregression.test_runner import AgentTestRunner, CommandTestRunner, ManualTestRunner LOG = get_proxy_logger("main") @@ -81,7 +81,17 @@ def clear(self): @property def test_runner(self): if self._test_runner is None: - if self.options.command is None: + if self.options.prompt is not None: + self._test_runner = AgentTestRunner( + self.options.prompt, + min_version=self.options.prompt_min_version, + headless=self.options.prompt_headless, + model=self.options.prompt_model, + recheck_mcp=self.options.prompt_recheck_mcp, + allow_other_mcp=self.options.prompt_allow_other_mcp, + max_budget_usd=self.options.max_budget_usd, + ) + elif self.options.command is None: self._test_runner = ManualTestRunner( launcher_kwargs=dict( addons=self.options.addons, @@ -330,6 +340,10 @@ def main( set_http_session(get_defaults={"timeout": config.options.http_timeout}) app = Application(config.fetch_config, config.options) + if config.options.prompt is not None: + # fail fast before bisecting: ensure the agent can run and that the + # prompt is able to yield a good/bad verdict. + app.test_runner.check_prerequisites() send_telemetry_ping_oop( UsageMetrics( variant=mozregression_variant, diff --git a/mozregression/test_runner.py b/mozregression/test_runner.py index 5ca0d22d5..2c09bb41e 100644 --- a/mozregression/test_runner.py +++ b/mozregression/test_runner.py @@ -6,15 +6,19 @@ from __future__ import absolute_import, print_function import datetime +import json import os +import re import shlex +import shutil import subprocess import sys +import tempfile from abc import ABCMeta, abstractmethod from mozlog import get_proxy_logger -from mozregression.errors import LauncherError, TestCommandError +from mozregression.errors import LauncherError, TestCommandError, UnsupportedVersionError from mozregression.launchers import create_launcher as mozlauncher LOG = get_proxy_logger("Test Runner") @@ -229,3 +233,265 @@ def evaluate(self, build_info, allow_back=False): def run_once(self, build_info): return 0 if self.evaluate(build_info) == "g" else 1 + + +# Verdict tokens the agent is instructed to emit, matched as standalone words. +_VERDICT_RE = re.compile(r"\b(GOOD|BAD)\b") + + +def _major_version(version): + """ + Return the integer major version from a version string like "128.0.1", + or None if it can not be parsed. + """ + if not version: + return None + match = re.match(r"\s*(\d+)", str(version)) + return int(match.group(1)) if match else None + + +class AgentTestRunner(TestRunner): + """ + A TestRunner subclass that evaluates builds with an LLM agent driving the + Firefox DevTools MCP (https://github.com/mozilla/firefox-devtools-mcp). + + Given a natural language instruction, the agent inspects the running build + via the MCP and decides whether it is good or bad. This is the higher level + equivalent of :class:`CommandTestRunner` (similar to ``git bisect run``). + + The agent is run by shelling out to the ``claude`` CLI in headless mode, + pointed at the MCP through a generated ``--mcp-config``. The MCP launches + the build itself, so this runner installs the build (to obtain the binary + path) but does not start it. + + Requires the ``claude`` CLI (installed and authenticated) and Node/``npx`` + on the PATH. + """ + + #: Name used for the MCP server in the generated config, also the prefix of + #: the tool names exposed to the agent (``mcp__firefox-devtools__*``). + MCP_SERVER_NAME = "firefox-devtools" + + #: npm package providing the Firefox DevTools MCP server. + MCP_PACKAGE = "@mozilla/firefox-devtools-mcp" + + #: Model/effort for the up-front prompt validation only. That is a trivial + #: text yes/no check (it does not drive the MCP), so a fast, cheap model at + #: low effort is plenty. The per-build verdict agent, which actually drives + #: the browser, uses the `claude` CLI's default model unless overridden with + #: ``--prompt-model`` -- a weaker model there misreads multi-step + #: instructions and navigates to the wrong place. + VALIDATION_MODEL = "haiku" + VALIDATION_EFFORT = "low" + + def __init__( + self, + instruction, + min_version=100, + headless=False, + model=None, + recheck_mcp=False, + allow_other_mcp=False, + max_budget_usd=10.0, + ): + TestRunner.__init__(self) + self.instruction = instruction + self.min_version = min_version + self.headless = headless + self.model = model + self.recheck_mcp = recheck_mcp + self.allow_other_mcp = allow_other_mcp + self.max_budget_usd = max_budget_usd + + def check_prerequisites(self): + """ + Fail fast, before any bisection happens, if the agent can not run or if + the instruction is not usable. Checks that the required executables are + available and that the prompt is able to yield a good/bad verdict. + """ + for executable in ("claude", "npx"): + if shutil.which(executable) is None: + raise TestCommandError( + "`%s` is required for --prompt but was not found on the" + " PATH. Install it (and run `claude` once to authenticate)" + " before using --prompt." % executable + ) + self._validate_prompt() + + def _validate_prompt(self): + """ + Ask the agent whether the instruction can produce a clear good/bad + determination, and raise :class:`TestCommandError` if it can not. + """ + meta_prompt = ( + "You are validating an instruction that will be used to judge" + " whether a Firefox build is GOOD or BAD during a regression" + " bisection. A usable instruction describes something observable in" + " the browser that maps to a clear good-or-bad outcome.\n\n" + 'Instruction: "%s"\n\n' + "If the instruction is usable, reply with exactly: VALID\n" + "Otherwise reply with: INVALID: " % self.instruction + ) + command = ( + ["claude", "-p", meta_prompt, "--output-format", "json"] + + ["--model", self.VALIDATION_MODEL, "--effort", self.VALIDATION_EFFORT] + + self._budget_flags() + ) + LOG.info("Validating --prompt instruction with the agent...") + proc = self._invoke_claude(command) + if proc.returncode != 0: + _raise_command_error( + "claude exited with code %d: %s" % (proc.returncode, proc.stderr.strip()) + ) + text = self._result_text(proc.stdout) + if "VALID" not in text.upper() or "INVALID" in text.upper(): + raise TestCommandError( + "the --prompt instruction does not look usable for a good/bad" + " verdict: %s" % text.strip() + ) + LOG.info("--prompt instruction validated.") + + def _budget_flags(self): + """ + Per-call spending cap shared by the validation and verdict claude calls. + """ + if self.max_budget_usd is not None: + return ["--max-budget-usd", str(self.max_budget_usd)] + return [] + + def _invoke_claude(self, command): + try: + return subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + except OSError as exc: + _raise_command_error(exc, " (claude not found or not executable)") + + def _check_version(self, app_info): + version = app_info.get("application_version") + major = _major_version(version) + if major is not None and major < self.min_version: + raise UnsupportedVersionError(version, self.min_version) + + def _mcp_config(self, binary): + # By default reuse whatever npx already has cached and stay offline, so + # each build in the bisection does not pay a registry round-trip (and + # uses a consistent version). --prompt-recheck-mcp forces npx to fetch + # the latest published version instead. + if self.recheck_mcp: + args = ["-y", "--prefer-online", self.MCP_PACKAGE + "@latest"] + else: + args = ["-y", "--prefer-offline", self.MCP_PACKAGE] + args += ["--firefox-path", binary] + if self.headless: + args.append("--headless") + return { + "mcpServers": { + self.MCP_SERVER_NAME: { + "command": "npx", + "args": args, + } + } + } + + def _build_prompt(self): + return ( + "You are evaluating a Firefox build during a regression bisection." + " Use the Firefox DevTools MCP tools to investigate the running" + " build, then decide whether the build is GOOD or BAD according to" + " this instruction:\n\n" + "%s\n\n" + "When you are done investigating, reply with exactly one word on" + " the final line: GOOD if the build behaves as expected, or BAD if" + " it exhibits the problem." % self.instruction + ) + + @staticmethod + def _result_text(stdout): + """ + Return the agent's final answer text from the ``claude`` output. With + ``--output-format json`` the answer is wrapped in a "result" field; + otherwise the raw stdout is returned. + """ + try: + payload = json.loads(stdout) + except ValueError: + return stdout + if isinstance(payload, dict): + return payload.get("result") or "" + return stdout + + @classmethod + def _parse_verdict(cls, stdout): + """ + Extract a 'g'/'b' verdict from the ``claude`` output, or return None if + no verdict could be determined. + """ + matches = _VERDICT_RE.findall(cls._result_text(stdout)) + if not matches: + return None + # the verdict is the last standalone GOOD/BAD token emitted. + return "g" if matches[-1] == "GOOD" else "b" + + def _run_agent(self, binary, build_info): + config = self._mcp_config(binary) + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", prefix="mozregression-mcp-", delete=False + ) as fp: + json.dump(config, fp) + config_path = fp.name + try: + command = [ + "claude", + "-p", + self._build_prompt(), + "--mcp-config", + config_path, + "--allowedTools", + "mcp__%s" % self.MCP_SERVER_NAME, + "--output-format", + "json", + "--permission-mode", + "bypassPermissions", + ] + if not self.allow_other_mcp: + # only load the Firefox DevTools MCP from our generated config, + # ignoring any other MCP servers the user has configured. + command.append("--strict-mcp-config") + # the verdict agent drives the browser, so it keeps the claude CLI's + # default model/effort unless --prompt-model overrides it. + if self.model: + command += ["--model", self.model] + command += self._budget_flags() + LOG.info("Running agent with instruction: %r" % self.instruction) + proc = self._invoke_claude(command) + finally: + try: + os.unlink(config_path) + except OSError: + pass + + if proc.returncode != 0: + _raise_command_error( + "claude exited with code %d: %s" % (proc.returncode, proc.stderr.strip()) + ) + verdict = self._parse_verdict(proc.stdout) + if verdict is None: + _raise_command_error( + "could not find a GOOD/BAD verdict in the agent output:" " %s" % proc.stdout.strip() + ) + LOG.info("Agent verdict: build is %s" % ("good" if verdict == "g" else "bad")) + return verdict + + def evaluate(self, build_info, allow_back=False): + with create_launcher(build_info) as launcher: + app_info = launcher.get_app_info() + build_info.update_from_app_info(app_info) + self._check_version(app_info) + return self._run_agent(launcher.binary, build_info) + + def run_once(self, build_info): + return 0 if self.evaluate(build_info) == "g" else 1 diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index edf919f48..6ca1d4064 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -151,6 +151,54 @@ def test_no_args(): assert config.enable_telemetry +def test_prompt_recent_range(): + # the default good/bad range is the last year, well after Firefox 100, + # so the up-front gate should accept it and --prompt should be recorded. + config = do_cli("--prompt", "check the page") + assert config.options.prompt == "check the page" + assert config.options.prompt_min_version == 100 + assert config.action == "bisect_nightlies" + + +def test_prompt_efficiency_defaults(): + # the per-build agent reuses the cached MCP, stays strict, and is capped. + config = do_cli("--prompt", "check the page") + assert config.options.prompt_recheck_mcp is False + assert config.options.prompt_allow_other_mcp is False + assert config.options.max_budget_usd == 10.0 + + +def test_prompt_with_command_is_rejected(): + with pytest.raises(errors.MozRegressionError): + do_cli("--prompt", "check", "--command", "true") + + +def test_prompt_with_launch_is_rejected(): + with pytest.raises(errors.MozRegressionError): + do_cli("--prompt", "check", "--launch", "2025-01-01") + + +def test_prompt_range_too_old(): + # a range predating Firefox 100 (released in 2022) must be rejected up front. + with pytest.raises(errors.UnsupportedVersionError): + do_cli("--prompt", "check", "--good", "2017-01-01", "--bad", "2017-06-01") + + +def test_prompt_min_version_override_allows_old_range(): + # lowering the minimum version below the range's era should let it through. + config = do_cli( + "--prompt", + "check", + "--good", + "2017-01-01", + "--bad", + "2017-06-01", + "--prompt-min-version", + "50", + ) + assert config.options.prompt_min_version == 50 + + TODAY = datetime.date.today() SOME_DATE = TODAY + datetime.timedelta(days=-20) SOME_OLDER_DATE = TODAY + datetime.timedelta(days=-10) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 4df780f7b..1c2ac108f 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -12,7 +12,7 @@ from mozregression.bisector import Bisection, Bisector, IntegrationHandler, NightlyHandler from mozregression.download_manager import BuildDownloadManager from mozregression.telemetry import UsageMetrics, get_system_info -from mozregression.test_runner import CommandTestRunner, ManualTestRunner +from mozregression.test_runner import AgentTestRunner, CommandTestRunner, ManualTestRunner class AppCreator(object): @@ -66,6 +66,38 @@ def test_app_get_command_test_runner(create_app): assert app.test_runner.command == "echo {binary}" +def test_app_get_agent_test_runner(create_app): + app = create_app( + ["--prompt", "check the page", "--prompt-headless", "--prompt-model", "claude-x"] + ) + assert isinstance(app.test_runner, AgentTestRunner) + assert app.test_runner.instruction == "check the page" + assert app.test_runner.min_version == 100 + assert app.test_runner.headless is True + assert app.test_runner.model == "claude-x" + # new options keep their defaults unless overridden + assert app.test_runner.recheck_mcp is False + assert app.test_runner.allow_other_mcp is False + assert app.test_runner.max_budget_usd == 10.0 + + +def test_app_get_agent_test_runner_options_forwarded(create_app): + app = create_app( + [ + "--prompt", + "check the page", + "--prompt-recheck-mcp", + "--prompt-allow-other-mcp", + "--max-budget-usd", + "3.5", + ] + ) + assert isinstance(app.test_runner, AgentTestRunner) + assert app.test_runner.recheck_mcp is True + assert app.test_runner.allow_other_mcp is True + assert app.test_runner.max_budget_usd == 3.5 + + @pytest.mark.parametrize( "argv,background_dl_policy,size_limit", [ diff --git a/tests/unit/test_test_runner.py b/tests/unit/test_test_runner.py index d2a2f793d..4f0d6afd3 100644 --- a/tests/unit/test_test_runner.py +++ b/tests/unit/test_test_runner.py @@ -266,6 +266,183 @@ def test_run_once(self): self.runner.evaluate.assert_called_once_with(build_info) +class TestAgentTestRunner(unittest.TestCase): + def setUp(self): + self.runner = test_runner.AgentTestRunner("check the page", min_version=100) + self.launcher = Mock(binary="/path/to/firefox") + self.launcher.get_app_info.return_value = {"application_version": "128.0"} + + @patch("mozregression.test_runner.create_launcher") + @patch("mozregression.test_runner.subprocess.run") + def evaluate(self, run, create_launcher, stdout=None, returncode=0, run_effect=None): + create_launcher.return_value = Launcher(self.launcher) + proc = Mock(returncode=returncode, stdout=stdout or "", stderr="") + run.return_value = proc + if run_effect: + run.side_effect = run_effect + self.subprocess_run = run + return self.runner.evaluate(mockinfo(to_dict=lambda: {})) + + def test_create(self): + self.assertEqual(self.runner.instruction, "check the page") + self.assertEqual(self.runner.min_version, 100) + + def test_evaluate_good(self): + verdict = self.evaluate(stdout='{"result": "Looks fine. GOOD"}') + self.assertEqual("g", verdict) + + def test_evaluate_bad(self): + verdict = self.evaluate(stdout='{"result": "The box is missing. BAD"}') + self.assertEqual("b", verdict) + + def test_evaluate_plain_text_output(self): + # output that is not JSON is scanned directly for the verdict + verdict = self.evaluate(stdout="some logs\nBAD\n") + self.assertEqual("b", verdict) + + def test_evaluate_last_verdict_wins(self): + verdict = self.evaluate(stdout='{"result": "first I thought BAD but it is GOOD"}') + self.assertEqual("g", verdict) + + def test_command_built(self): + self.evaluate(stdout='{"result": "GOOD"}') + command = self.subprocess_run.mock_calls[0][1][0] + self.assertEqual(command[0], "claude") + self.assertIn("--mcp-config", command) + self.assertIn("mcp__firefox-devtools", command) + + def test_verdict_uses_cli_default_model_and_budget(self): + # the verdict agent drives the browser, so it keeps the claude CLI's + # default model/effort (no --model/--effort forced) but stays capped. + self.evaluate(stdout='{"result": "GOOD"}') + command = self.subprocess_run.mock_calls[0][1][0] + self.assertNotIn("--model", command) + self.assertNotIn("--effort", command) + self.assertEqual(command[command.index("--max-budget-usd") + 1], "10.0") + + def test_strict_mcp_config_by_default(self): + self.evaluate(stdout='{"result": "GOOD"}') + command = self.subprocess_run.mock_calls[0][1][0] + self.assertIn("--strict-mcp-config", command) + + def test_allow_other_mcp_drops_strict_flag(self): + self.runner = test_runner.AgentTestRunner("check", min_version=100, allow_other_mcp=True) + self.evaluate(stdout='{"result": "GOOD"}') + command = self.subprocess_run.mock_calls[0][1][0] + self.assertNotIn("--strict-mcp-config", command) + + def test_max_budget_override(self): + self.runner = test_runner.AgentTestRunner("check", min_version=100, max_budget_usd=2.5) + self.evaluate(stdout='{"result": "GOOD"}') + command = self.subprocess_run.mock_calls[0][1][0] + self.assertEqual(command[command.index("--max-budget-usd") + 1], "2.5") + + def test_mcp_config_reuses_cached_package_by_default(self): + with patch("mozregression.test_runner.json.dump") as dump: + self.evaluate(stdout='{"result": "GOOD"}') + config = dump.mock_calls[0][1][0] + args = config["mcpServers"]["firefox-devtools"]["args"] + self.assertIn("--prefer-offline", args) + self.assertIn(test_runner.AgentTestRunner.MCP_PACKAGE, args) + self.assertNotIn(test_runner.AgentTestRunner.MCP_PACKAGE + "@latest", args) + + def test_mcp_config_recheck_fetches_latest(self): + self.runner = test_runner.AgentTestRunner("check", min_version=100, recheck_mcp=True) + with patch("mozregression.test_runner.json.dump") as dump: + self.evaluate(stdout='{"result": "GOOD"}') + config = dump.mock_calls[0][1][0] + args = config["mcpServers"]["firefox-devtools"]["args"] + self.assertIn("--prefer-online", args) + self.assertIn(test_runner.AgentTestRunner.MCP_PACKAGE + "@latest", args) + + def test_headless_and_model_forwarded(self): + self.runner = test_runner.AgentTestRunner( + "check", min_version=100, headless=True, model="claude-x" + ) + with patch("mozregression.test_runner.json.dump") as dump: + self.evaluate(stdout='{"result": "GOOD"}') + config = dump.mock_calls[0][1][0] + args = config["mcpServers"]["firefox-devtools"]["args"] + self.assertIn("--firefox-path", args) + self.assertIn("/path/to/firefox", args) + self.assertIn("--headless", args) + command = self.subprocess_run.mock_calls[0][1][0] + self.assertIn("--model", command) + self.assertIn("claude-x", command) + + def test_unsupported_version(self): + self.launcher.get_app_info.return_value = {"application_version": "96.0"} + self.assertRaises(errors.UnsupportedVersionError, self.evaluate) + + def test_unknown_version_passes_to_agent(self): + # if mozversion can't report a version, defer to the per-build agent + self.launcher.get_app_info.return_value = {} + verdict = self.evaluate(stdout='{"result": "GOOD"}') + self.assertEqual("g", verdict) + + def test_no_verdict_in_output(self): + self.assertRaisesRegex( + errors.TestCommandError, "verdict", self.evaluate, stdout='{"result": "no idea"}' + ) + + def test_nonzero_returncode(self): + self.assertRaisesRegex(errors.TestCommandError, "exited", self.evaluate, returncode=1) + + def test_claude_missing(self): + self.assertRaisesRegex( + errors.TestCommandError, "not found", self.evaluate, run_effect=OSError + ) + + def test_run_once(self): + self.runner.evaluate = Mock(return_value="g") + build_info = Mock() + self.assertEqual(self.runner.run_once(build_info), 0) + self.runner.evaluate.assert_called_once_with(build_info) + + @patch("mozregression.test_runner.shutil.which") + @patch("mozregression.test_runner.subprocess.run") + def check_prerequisites(self, run, which, missing=(), validation='{"result": "VALID"}'): + which.side_effect = lambda exe: None if exe in missing else "/usr/bin/" + exe + run.return_value = Mock(returncode=0, stdout=validation, stderr="") + self.subprocess_run = run + return self.runner.check_prerequisites() + + def test_check_prerequisites_ok(self): + # claude + npx present, prompt validated: no error + self.check_prerequisites() + # the validation call does not enable any MCP/tools + command = self.subprocess_run.mock_calls[0][1][0] + self.assertEqual(command[0], "claude") + self.assertNotIn("--mcp-config", command) + + def test_validation_uses_fast_model(self): + # the validation step is a trivial text check, so it always uses the + # fast model at low effort regardless of --prompt-model. + self.runner = test_runner.AgentTestRunner("check", min_version=100, model="sonnet") + self.check_prerequisites() + command = self.subprocess_run.mock_calls[0][1][0] + self.assertEqual(command[command.index("--model") + 1], "haiku") + self.assertEqual(command[command.index("--effort") + 1], "low") + + def test_check_prerequisites_claude_missing(self): + self.assertRaisesRegex( + errors.TestCommandError, "claude", self.check_prerequisites, missing=("claude",) + ) + + def test_check_prerequisites_npx_missing(self): + self.assertRaisesRegex( + errors.TestCommandError, "npx", self.check_prerequisites, missing=("npx",) + ) + + def test_check_prerequisites_invalid_prompt(self): + self.assertRaisesRegex( + errors.TestCommandError, + "usable", + self.check_prerequisites, + validation='{"result": "INVALID: too vague"}', + ) + + @pytest.mark.parametrize( "brange,input,allowed_range,result", [ # noqa