test_runner: add --prompt mode driving the Firefox DevTools MCP

mozregression/cli.py

@@ -21,7 +21,12 @@
 from mozregression.branches import get_name
 from mozregression.config import DEFAULT_CONF_FNAME, get_config, write_config
 from mozregression.dates import is_date_or_datetime, parse_date, to_datetime
-from mozregression.errors import DateFormatError, MozRegressionError, UnavailableRelease
+from mozregression.errors import (
+    DateFormatError,
+    MozRegressionError,
+    UnavailableRelease,
+    UnsupportedVersionError,
+)
 from mozregression.fetch_configs import REGISTRY as FC_REGISTRY
 from mozregression.fetch_configs import create_config
 from mozregression.log import colorize, init_logger
@@ -301,6 +306,79 @@ def create_parser(defaults):
         ),
     )
 
+    parser.add_argument(
+        "--prompt",
+        help=(
+            "Evaluate builds automatically with an LLM agent driving the"
+            " Firefox DevTools MCP. Give a natural language instruction"
+            " describing what to check, e.g."
+            ' --prompt "open example.com and tell me if the search box is'
+            " present\". The build is judged good or bad based on the agent's"
+            " findings. Mutually exclusive with --command. Requires the"
+            " `claude` CLI (installed and authenticated) and Node/`npx` on the"
+            " PATH. Note: the Firefox DevTools MCP only supports recent Firefox"
+            " versions, so older regression ranges are rejected (see"
+            " --prompt-min-version)."
+        ),
+    )
+
+    parser.add_argument(
+        "--prompt-min-version",
+        type=int,
+        default=100,
+        help=(
+            "Minimum Firefox major version the Firefox DevTools MCP supports,"
+            " used to gate the --prompt option. Builds (and regression ranges)"
+            " older than this are rejected. Defaults to %(default)s."
+        ),
+    )
+
+    parser.add_argument(
+        "--prompt-headless",
+        action="store_true",
+        help="Run the --prompt Firefox build in headless mode.",
+    )
+
+    parser.add_argument(
+        "--prompt-model",
+        default=None,
+        help=(
+            "Model passed to the `claude` CLI for the per-build --prompt agent"
+            " that drives the browser. Defaults to the `claude` CLI's configured"
+            " model."
+        ),
+    )
+
+    parser.add_argument(
+        "--prompt-recheck-mcp",
+        action="store_true",
+        help=(
+            "Re-check the npm registry for the latest Firefox DevTools MCP"
+            " version on every build. By default the version npx already has"
+            " cached is reused (no per-build registry round-trip)."
+        ),
+    )
+
+    parser.add_argument(
+        "--prompt-allow-other-mcp",
+        action="store_true",
+        help=(
+            "Also load your other configured MCP servers when running the"
+            " --prompt agent. By default only the Firefox DevTools MCP is used"
+            " (claude is run with --strict-mcp-config)."
+        ),
+    )
+
+    parser.add_argument(
+        "--max-budget-usd",
+        type=float,
+        default=10.0,
+        help=(
+            "Maximum dollar amount the --prompt agent may spend per build,"
+            " passed to the `claude` CLI. Defaults to %(default)s."
+        ),
+    )
+
     parser.add_argument(
         "--persist",
         default=defaults["persist"],
@@ -554,13 +632,39 @@ def _convert_to_bisect_arg(self, value):
                 self.logger.info("%s is not a release, assuming it's a hash..." % value)
         return value
 
+    def _check_prompt_min_version(self, options):
+        """
+        Up-front gate for the --prompt option: reject regression ranges that
+        predate the minimum Firefox version supported by the Firefox DevTools
+        MCP.
+
+        Only date-based endpoints (dates, buildids, release numbers) can be
+        checked here without extra network requests; raw changeset endpoints are
+        left to the authoritative per-build version check in AgentTestRunner.
+        """
+        try:
+            cutoff = to_datetime(parse_date(date_of_release(options.prompt_min_version)))
+        except (UnavailableRelease, DateFormatError):
+            # we can't resolve a cutoff date for this version; rely on the
+            # per-build check instead.
+            return
+        for endpoint in (options.good, options.bad):
+            if is_date_or_datetime(endpoint) and to_datetime(endpoint) < cutoff:
+                raise UnsupportedVersionError(endpoint, options.prompt_min_version)
+
     def validate(self):
         """
         Validate the options, define the `action` and `fetch_config` that
         should be used to run the application.
         """
         options = self.options
 
+        if options.prompt is not None:
+            if options.command is not None:
+                raise MozRegressionError("--prompt can not be used together with --command.")
+            if options.launch:
+                raise MozRegressionError("--prompt can not be used together with --launch.")
+
         arch_options = {
             "firefox": [
                 "aarch64",
@@ -703,6 +807,8 @@ def validate(self):
                     )
                 if fetch_config.should_use_archive():
                     self.action = "bisect_nightlies"
+            if options.prompt is not None:
+                self._check_prompt_min_version(options)
         if (
             self.action in ("launch_integration", "bisect_integration")
             and not fetch_config.is_integration()

mozregression/errors.py

@@ -16,6 +16,21 @@ def __init__(self):
         MozRegressionError.__init__(self, "Can't run Windows builds before" " 2010-03-18")
 
 
+class UnsupportedVersionError(MozRegressionError):
+    """
+    Raised when a build's Firefox version is too old to be driven by the
+    Firefox DevTools MCP used by the ``--prompt`` option.
+    """
+
+    def __init__(self, build, min_version):
+        MozRegressionError.__init__(
+            self,
+            "Build %s is too old for --prompt: the Firefox DevTools MCP requires"
+            " Firefox %s or later. Narrow the regression range, or lower"
+            " --prompt-min-version if you know it works on older builds." % (build, min_version),
+        )
+
+
 class DateFormatError(MozRegressionError):
     """
     Raised when a date can not be parsed from a string.

mozregression/main.py

@@ -30,7 +30,7 @@
 from mozregression.persist_limit import PersistLimit
 from mozregression.telemetry import UsageMetrics, get_system_info, send_telemetry_ping_oop
 from mozregression.tempdir import safe_mkdtemp
-from mozregression.test_runner import CommandTestRunner, ManualTestRunner
+from mozregression.test_runner import AgentTestRunner, CommandTestRunner, ManualTestRunner
 
 LOG = get_proxy_logger("main")
 
@@ -81,7 +81,17 @@ def clear(self):
     @property
     def test_runner(self):
         if self._test_runner is None:
-            if self.options.command is None:
+            if self.options.prompt is not None:
+                self._test_runner = AgentTestRunner(
+                    self.options.prompt,
+                    min_version=self.options.prompt_min_version,
+                    headless=self.options.prompt_headless,
+                    model=self.options.prompt_model,
+                    recheck_mcp=self.options.prompt_recheck_mcp,
+                    allow_other_mcp=self.options.prompt_allow_other_mcp,
+                    max_budget_usd=self.options.max_budget_usd,
+                )
+            elif self.options.command is None:
                 self._test_runner = ManualTestRunner(
                     launcher_kwargs=dict(
                         addons=self.options.addons,
@@ -330,6 +340,10 @@ def main(
         set_http_session(get_defaults={"timeout": config.options.http_timeout})
 
         app = Application(config.fetch_config, config.options)
+        if config.options.prompt is not None:
+            # fail fast before bisecting: ensure the agent can run and that the
+            # prompt is able to yield a good/bad verdict.
+            app.test_runner.check_prerequisites()
         send_telemetry_ping_oop(
             UsageMetrics(
                 variant=mozregression_variant,