diff --git a/deps/cloudxr/webxr_client/tests/mock/cloudxr-mock-alias.ts b/deps/cloudxr/webxr_client/tests/mock/cloudxr-mock-alias.ts index 4b1cfd5cca..0ec9edb4ae 100644 --- a/deps/cloudxr/webxr_client/tests/mock/cloudxr-mock-alias.ts +++ b/deps/cloudxr/webxr_client/tests/mock/cloudxr-mock-alias.ts @@ -58,13 +58,20 @@ export function createSession( declare global { interface Window { - __mockCloudXRFail?: (message?: string) => void; + __mockCloudXRFail?: (message?: string, code?: number) => void; __mockCloudXRConnectDelayMs?: number; } } if (typeof window !== 'undefined') { - window.__mockCloudXRFail = (message = 'Mock transient server/tunnel outage') => { - activeSession?.triggerFailure({ name: 'StreamingError', message }); + // code is optional and omitted by default (a recoverable failure, matching the SDK's + // own default): CloudXRComponent.tsx's onStreamStopped auto-reconnects on a recoverable + // error without ever showing #errorMessageBox, exactly as it would for a real transient + // failure. Pass a code in the 0xc0f22300-0xc0f223ff (server-disconnect) range - see + // helpers/streamingErrorClassification.ts's isRecoverable() - for a genuine terminal + // error that surfaces on the DOM, matching CloudXRComponentTest.tsx's own + // NON_RETRYABLE_CODE steps. + window.__mockCloudXRFail = (message = 'Mock transient server/tunnel outage', code) => { + activeSession?.triggerFailure({ name: 'StreamingError', message, code }); }; } diff --git a/deps/cloudxr/webxr_client/webpack.app-mock.js b/deps/cloudxr/webxr_client/webpack.app-mock.js index 7cfaaa943f..9d5b96b6c9 100644 --- a/deps/cloudxr/webxr_client/webpack.app-mock.js +++ b/deps/cloudxr/webxr_client/webpack.app-mock.js @@ -29,8 +29,17 @@ const { merge } = require('webpack-merge'); const common = require('./webpack.common.js'); module.exports = merge(common, { - mode: 'development', - devtool: 'eval-source-map', + // production, not development: src/index.tsx wraps the app in + // , which double-invokes effects in a dev build - that calls + // CloudXRComponent.tsx's establishSession() (and so CloudXR.createSession()) + // twice, leaving tests/mock/cloudxr-mock-alias.ts's activeSession pointing at + // whichever MockCloudXR instance was created last while the mounted component + // may be running against the other one. window.__mockCloudXRFail() then + // silently no-ops against a session nothing is actually using. A production + // build's NODE_ENV=production makes React skip StrictMode's double-invoke + // entirely, so this is the actual fix, not a workaround. + mode: 'production', + devtool: 'source-map', resolve: { alias: { // Exact match ($) only: the shim itself (and MockCloudXR.ts) reach the real SDK via the diff --git a/oob-real-browser-integration-test-plan.md b/oob-real-browser-integration-test-plan.md new file mode 100644 index 0000000000..6aade285dd --- /dev/null +++ b/oob-real-browser-integration-test-plan.md @@ -0,0 +1,103 @@ + + +# OOB real-browser integration test + +Exercises `run_oob_connect()`'s real orchestration logic against a real +Chromium instance running the real teleop web client (real IWER-emulated +WebXR, real CDP, real client JS), as additional, slower coverage alongside +the fast, fully-scripted `mock_adb()` / fake-CDP-server unit tests already in +`test_oob_teleop_adb.py`. + +## What's real, what's faked + +- **ADB**: no physical device exists in CI/dev-sandbox, so this stays + synthetic via `RealBrowserAdb` (`conftest.py`) — but `am start` and `adb + forward` have real side effects (navigating a real local Chromium, aliasing + the local TCP port straight to Chromium's own `--remote-debugging-port`) + instead of returning canned output. Error-injection kwargs + (`device_state`, `am_start_rc`, `devtools_socket`, `forward_rc`) work the + same as on `FakeAdb`. +- **CDP**: fully real — a real Chromium (`real_chrome()`, matching + `playwright.config.js`'s launch args: SwiftShader software WebGL, + `--disable-features=WebXR` so IWER provides `navigator.xr`) reachable over + its own `/json`, `/json/close/`, and debugger websocket. +- **OOB control-hub** (`/oob/v1/ws`): the real `OOBControlHub`, via a real + `wss.py` (`real_wss_proxy()`), run for real — pure Python, no GPU + dependency. +- **CloudXR signaling**: never a real or mocked runtime backend. + `wss.py` already treats "nothing listening on `backend_port`" as an + anticipated configuration (`_is_backend_connection_refused()`), not an + error to work around, so the client's own `#errorMessageBox` on a refused + signaling connection is a genuine, free signal — not something this test + infra needs to fake. +- **The web client**: always `MockCloudXR` (`npm run build:app-mock`), never + the real `@nvidia/cloudxr` SDK. The real SDK genuinely tries to stream + against whatever's on `backend_port`, which is unpredictable and + uncontrollable to assert against. `MockCloudXR` mounts the identical + `App.tsx`/`CloudXRComponent.tsx` UI (same `#startButton`/`#errorMessageBox` + DOM) with a fully deterministic, externally-controllable session + underneath — `window.__mockCloudXRFail(message?, code?)`, callable over CDP + via `cdp_evaluate()` — and opens no socket of its own. + +## Two bugs found building this infra (worth knowing if this code is touched again) + +- **Webpack HMR module duplication.** A dev-server's hot-reload re-executes a + module's top-level code on live-reload, creating a *second* instance of it + side by side with the one React already mounted against. For + `cloudxr-mock-alias.ts` this meant `window.__mockCloudXRFail()` silently + bound to a fresh, never-used `activeSession` while the real, + already-running session lived on untouched — callable, no error, just no + effect. Fixed by never using a dev-server: `static_webxr_build()` + (`conftest.py`) always does a real `npm run build:app-mock` production + build and serves the static output, which has no live-reload runtime at + all. +- **`` double-invoking effects in a dev-mode build.** + Independent of HMR: `webpack.app-mock.js` used `mode: 'development'`, and + `src/index.tsx` wraps the app in `StrictMode`, which deliberately + double-invokes effects in dev builds — calling `CloudXRComponent.tsx`'s + `establishSession()` (and so `CloudXR.createSession()`) twice. Fixed by + switching `webpack.app-mock.js` to `mode: 'production'`, which sets + `NODE_ENV=production` and makes React skip StrictMode's double-invoke + entirely. +- A third, unrelated trap hit while diagnosing the above: `webpack.common.js`'s + persistent filesystem cache (`cache: {type: 'filesystem'}`) is keyed only + on the webpack config file, not on every source file it compiles — a stale + cache entry silently served an old compiled bundle that didn't include a + real source edit. `static_webxr_build()` now deletes + `node_modules/.cache/webpack` before every build. + +## ADB/CDP command inventory (what `oob_teleop_adb.py` actually talks to) + +| adb command | Function | Used for | +|---|---|---| +| `adb get-state` | `adb_device_state()` → `assert_adb_device_online()` | Preflight gate before nearly every other call; drives offline→`adb reconnect` retry | +| `adb shell cat /proc/net/unix` | `_discover_devtools_socket()` | Regex-scans for `@..._devtools_remote[_pid]` | +| `adb shell am start -a android.intent.action.VIEW -d ` | `open_url_on_headset()` → `run_adb_headset_bookmark()` / `run_oob_connect()` | Opens the browser to the teleop URL; retried in a poll loop until a matching tab appears over CDP | +| `adb forward tcp: localabstract:` | `_adb_forward_cdp()` | Maps the discovered DevTools abstract socket to a local TCP port — what makes CDP reachable at all | +| `adb forward --remove tcp:` | `_adb_forward_remove()` | Cleanup | + +CDP itself (`_cdp_list_tabs()`, `_cdp_session_click_connect()`, +`_monitor_teleop_error_banner()`) is reached over the `adb forward`'d port +and never touches application/CloudXR traffic directly. + +## Tests + +Three tests in `test_oob_teleop_adb.py`, all slow (real browser + a real +webpack production build + real `wss.py`), requiring a system Chrome +(`PLAYWRIGHT_CHROME_PATH` override; defaults to `/usr/bin/google-chrome`) and +Node/npm on `PATH` — run manually for now, not yet wired into CI: + +- `test_run_oob_connect_real_browser_end_to_end` — the full real + adb→CDP→click path against `MockCloudXR`, proving `run_oob_connect()`'s own + orchestration for real. +- `test_run_oob_connect_real_mock_cxr_crash_surfaces_error_banner` — proves + the crash-trigger mechanism itself: a real, deterministic, DOM-visible + error banner on demand. +- `test_run_oob_connect_real_mock_cxr_crash_triggers_real_relaunch` — + exercises the repair-capable `_monitor_teleop_error_banner` added by + `gmorgan/oob-error-relaunch` (#1146): a genuine crash recovered by genuine + ADB+CDP automation (a second real `am start` and a second real tab), not a + scripted stand-in for either. diff --git a/tests/python/core/cloudxr/conftest.py b/tests/python/core/cloudxr/conftest.py index 6d6daec014..eaebe2643f 100644 --- a/tests/python/core/cloudxr/conftest.py +++ b/tests/python/core/cloudxr/conftest.py @@ -11,15 +11,22 @@ from __future__ import annotations +import asyncio import contextlib import importlib.util +import json import os import shlex +import shutil import socket import subprocess import sys +import time import types -from contextlib import contextmanager +import urllib.error +import urllib.request +from contextlib import asynccontextmanager, contextmanager +from dataclasses import dataclass from pathlib import Path from unittest.mock import MagicMock, patch @@ -336,3 +343,328 @@ def mock_adb(**kwargs): fake = FakeAdb(**kwargs) with patch("subprocess.run", side_effect=fake): yield fake + + +# ============================================================================ +# Real-browser OOB integration test-infra (see /oob-real-browser-integration- +# test-plan.md at the repo root): a real Chromium running the real webxr +# client, driven by run_oob_connect() through a RealBrowserAdb whose +# success-path commands have real side effects instead of canned output. +# ============================================================================ + +_CHROME_EXECUTABLE = os.environ.get("PLAYWRIGHT_CHROME_PATH", "/usr/bin/google-chrome") + +# Matches deps/cloudxr/webxr_client/playwright.config.js's CHROMIUM_ARGS: +# SwiftShader software WebGL (no GPU in this sandbox) and --disable-features=WebXR +# so native/real XR is off and IWER (loaded by the page itself) provides navigator.xr. +_CHROME_LAUNCH_ARGS = [ + "--no-sandbox", + "--disable-setuid-sandbox", + "--ignore-gpu-blocklist", + "--enable-webgl", + "--use-angle=swiftshader", + "--use-gl=angle", + "--disable-features=WebXR", +] + + +def _free_tcp_port() -> int: + """Bind an ephemeral port and immediately release it for a subprocess to reuse.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _wait_for_http(url: str, *, timeout: float) -> None: + deadline = time.monotonic() + timeout + last_exc: Exception | None = None + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(url, timeout=1): + return + except (urllib.error.URLError, ConnectionError, OSError) as exc: + last_exc = exc + time.sleep(0.2) + raise RuntimeError(f"{url} did not respond within {timeout:.0f}s") from last_exc + + +class RealChrome: + """A real, locally-launched Chromium instance reachable over its own CDP port.""" + + def __init__(self, cdp_port: int) -> None: + self.cdp_port = cdp_port + + def open_url(self, url: str) -> dict: + """Open *url* in a new real tab via CDP's HTTP endpoint (``PUT /json/new``).""" + req = urllib.request.Request( + f"http://localhost:{self.cdp_port}/json/new?{url}", method="PUT" + ) + with urllib.request.urlopen(req, timeout=10) as resp: + return json.loads(resp.read()) + + def list_tabs(self) -> list[dict]: + """Real ``GET /json`` — every open tab, with its ``webSocketDebuggerUrl``.""" + with urllib.request.urlopen( + f"http://localhost:{self.cdp_port}/json", timeout=5 + ) as resp: + return json.loads(resp.read()) + + +@contextmanager +def real_chrome(cdp_port: int, *, user_data_dir: Path): + """Launch a real local Chromium reachable over CDP on *cdp_port*, for the block's duration.""" + args = [ + _CHROME_EXECUTABLE, + f"--remote-debugging-port={cdp_port}", + f"--user-data-dir={user_data_dir}", + "--no-first-run", + "--no-default-browser-check", + *_CHROME_LAUNCH_ARGS, + "about:blank", + ] + proc = subprocess.Popen(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + _wait_for_http(f"http://localhost:{cdp_port}/json/version", timeout=15.0) + yield RealChrome(cdp_port) + finally: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + + +@contextmanager +def static_webxr_build(port: int): + """Build webxr_client once via a real ``npm run build:app-mock`` production build + (no dev-server, no HMR), then serve the resulting static directory with a plain + HTTP server for the block's duration. + + Always the mock build (``@nvidia/cloudxr`` aliased to ``MockCloudXR``, same + ``App.tsx``/``CloudXRComponent.tsx`` UI): the real SDK genuinely tries to stream + against whatever's on ``backend_port``, which is unpredictable and uncontrollable + to assert against. ``MockCloudXR`` exposes ``window.__mockCloudXRFail(message?, + code?)`` globally for a deterministic, controllable mid-session failure — see + :func:`cdp_evaluate` — and opens no socket of its own. + + Deliberately never a dev-server: webpack's hot-module-replacement re-executes a + module's top-level code on live-reload, creating a *second* instance of it + side by side with the one React already mounted against — for + ``tests/mock/cloudxr-mock-alias.ts`` this means ``window.__mockCloudXRFail()`` + silently binds to a fresh, never-used ``activeSession`` while the real, + already-running session lives on in the original instance untouched. A static + build has no live-reload runtime at all, so this class of bug is structurally + impossible, not just avoided by luck. + + Always rebuilds (mirrors ``cloudxr-js``'s ``global-setup.js``: a stale bundle + from a previous branch must never silently pass) rather than reusing whatever + happens to be on disk or already listening on *port*. + """ + npm_build_script = "build:app-mock" + build_dir_name = "build-app-mock" + webxr_client_dir = repo_root() / "deps" / "cloudxr" / "webxr_client" + # webpack.common.js turns on persistent filesystem caching + # (cache: {type: 'filesystem'}), keyed only on the config file + # (buildDependencies.config), not on every source file it compiles. A stale + # entry there can silently serve an old compiled module for a source file + # that *did* change - hit for real while building this fixture: an edit to + # tests/mock/cloudxr-mock-alias.ts was invisible in the output bundle + # (`grep` for the new symbol found nothing) until this cache was cleared. + # Correctness matters far more than incremental-build speed for a test + # fixture, so always start from a clean cache rather than trust it. + shutil.rmtree( + webxr_client_dir / "node_modules" / ".cache" / "webpack", ignore_errors=True + ) + subprocess.run( + ["npm", "run", npm_build_script], + cwd=webxr_client_dir, + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + build_dir = webxr_client_dir / build_dir_name + if not (build_dir / "index.html").is_file(): + raise RuntimeError( + f"{build_dir} has no index.html after `npm run {npm_build_script}`" + ) + + from functools import partial + from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler + from threading import Thread + + handler = partial(SimpleHTTPRequestHandler, directory=str(build_dir)) + httpd = ThreadingHTTPServer(("localhost", port), handler) + thread = Thread(target=httpd.serve_forever, daemon=True) + thread.start() + try: + _wait_for_http(f"http://localhost:{port}/", timeout=15.0) + yield + finally: + httpd.shutdown() + httpd.server_close() + thread.join(timeout=5) + + +@dataclass +class RealBrowserTestEnv: + """Ports/dirs shared by every real-browser ``run_oob_connect()`` test — see + :func:`real_browser_env`.""" + + client_port: int + wss_port: int + backend_port: int + user_data_dir: Path + install_dir: Path + + +@contextmanager +def real_browser_env(tmp_path: Path, monkeypatch): + """Allocate the ports/dirs every real-browser ``run_oob_connect()`` test needs + (client static-build port, wss proxy port, unused CloudXR backend port, a fresh + Chrome profile dir, a fresh install dir) and point ``TELEOP_WEB_CLIENT_BASE`` at + the client port, for the block's duration.""" + client_port = _free_tcp_port() + monkeypatch.setenv("TELEOP_WEB_CLIENT_BASE", f"http://localhost:{client_port}") + monkeypatch.delenv("CONTROL_TOKEN", raising=False) + + user_data_dir = tmp_path / "chrome-profile" + user_data_dir.mkdir() + + yield RealBrowserTestEnv( + client_port=client_port, + wss_port=_free_tcp_port(), + backend_port=_free_tcp_port(), + user_data_dir=user_data_dir, + install_dir=tmp_path / "cxr-install", + ) + + +async def cdp_evaluate(ws_url: str, expression: str) -> dict: + """Open a one-shot CDP session to *ws_url* and evaluate *expression*. + + Mirrors the ``send()`` helper duplicated in ``_cdp_session_click_connect``/ + ``_monitor_teleop_error_banner`` (``oob_teleop_adb.py``), for a test that + needs to poke a real tab from outside those functions — e.g. calling + ``window.__mockCloudXRFail()`` to force a deterministic mid-session error. + """ + from websockets.asyncio.client import connect as ws_connect # noqa: PLC0415 + + async with ws_connect(ws_url) as ws: + await ws.send( + json.dumps( + { + "id": 1, + "method": "Runtime.evaluate", + "params": {"expression": expression, "returnByValue": True}, + } + ) + ) + while True: + msg = json.loads(await asyncio.wait_for(ws.recv(), timeout=10.0)) + if msg.get("id") == 1: + return msg.get("result", {}) + + +@asynccontextmanager +async def real_wss_proxy(install_dir: Path, *, proxy_port: int, backend_port: int): + """Run the real ``wss.py`` proxy (TLS + OOB hub) for the block's duration. + + ``TELEOP_OOB_HUB_ONLY=1`` makes ``setup_oob=True`` wire up the real + ``OOBControlHub`` without also having ``wss.py`` itself call + ``run_oob_connect()`` on our behalf — the test drives that call directly, + against the real proxy this starts. ``usb_local=False`` here (a ``wss.run()`` + param, independent of the ``usb_local`` a test passes to its own + ``run_oob_connect()`` call) skips ``wss.py``'s own adb-reverse/coturn setup, + which has no real device to run against in this test. + """ + from cloudxr_py_test_ns.wss import run as wss_run # noqa: PLC0415 + + prev_install_dir = os.environ.get("CXR_INSTALL_DIR") + os.environ["CXR_INSTALL_DIR"] = str(install_dir) + prev_hub_only = os.environ.get("TELEOP_OOB_HUB_ONLY") + os.environ["TELEOP_OOB_HUB_ONLY"] = "1" + + stop_future: asyncio.Future = asyncio.get_running_loop().create_future() + listening = asyncio.Event() + + task = asyncio.create_task( + wss_run( + None, + stop_future, + backend_host="localhost", + backend_port=backend_port, + proxy_port=proxy_port, + setup_oob=True, + usb_local=False, + host_client=False, + on_listening=listening.set, + ) + ) + try: + await asyncio.wait_for(listening.wait(), timeout=15.0) + yield + finally: + if not stop_future.done(): + stop_future.set_result(None) + await asyncio.wait_for(task, timeout=10.0) + if prev_hub_only is None: + os.environ.pop("TELEOP_OOB_HUB_ONLY", None) + else: + os.environ["TELEOP_OOB_HUB_ONLY"] = prev_hub_only + if prev_install_dir is None: + os.environ.pop("CXR_INSTALL_DIR", None) + else: + os.environ["CXR_INSTALL_DIR"] = prev_install_dir + + +class RealBrowserAdb(FakeAdb): + """Like :class:`FakeAdb`, but ``am start`` and ``adb forward`` have real side + effects against a real :class:`RealChrome` instead of returning canned output. + + Error-injection kwargs (``device_state``, ``am_start_rc``, ``devtools_socket``, + ``forward_rc``) work exactly as on :class:`FakeAdb`: the fake side of each still + gates whether the real side effect happens at all, so tests can still + deliberately drive the same failure paths ``FakeAdb``-based tests do. + + ``adb forward`` is a real no-op (not a real TCP forward) because *chrome* + is launched with ``--remote-debugging-port`` equal to + ``oob_teleop_adb._CDP_LOCAL_PORT`` already — the port ``run_oob_connect()`` + forwards to is already the real CDP port, so there is nothing to actually + forward. + """ + + def __init__(self, chrome: RealChrome, **kwargs) -> None: + super().__init__(**kwargs) + self._chrome = chrome + + def __call__(self, args: list[str], **kwargs) -> subprocess.CompletedProcess: + def result( + rc: int, stdout: str = "", stderr: str = "" + ) -> subprocess.CompletedProcess: + self.calls.append(list(args)) + completed = subprocess.CompletedProcess(args, rc, stdout, stderr) + if kwargs.get("check", False): + completed.check_returncode() + return completed + + if args[:2] == ["adb", "shell"] and any("am start" in a for a in args): + if self.am_start_rc != 0: + return result(self.am_start_rc, stderr="am start failed") + shell_cmd = args[2] + tokens = shlex.split(shell_cmd) + url = tokens[tokens.index("-d") + 1] + self._chrome.open_url(url) + return result(0) + if args[:2] == ["adb", "forward"] and "--remove" not in args: + return result(self.forward_rc) + return super().__call__(args, **kwargs) + + +@contextmanager +def real_browser_adb(chrome: RealChrome, **kwargs): + """Patches ``subprocess.run`` with a :class:`RealBrowserAdb` for the block's + duration; yields the instance so a test can inspect ``.calls``.""" + fake = RealBrowserAdb(chrome, **kwargs) + with patch("subprocess.run", side_effect=fake): + yield fake diff --git a/tests/python/core/cloudxr/pyproject.toml b/tests/python/core/cloudxr/pyproject.toml index d9b1072301..d03fbcaef1 100644 --- a/tests/python/core/cloudxr/pyproject.toml +++ b/tests/python/core/cloudxr/pyproject.toml @@ -13,12 +13,14 @@ dev = [ "pytest-asyncio>=1.3.0", "numpy", "websockets>=14", + "cryptography>=3.0", ] test = [ "pytest", "pytest-asyncio>=1.3.0", "numpy", "websockets>=14", + "cryptography>=3.0", ] [tool.pytest.ini_options] diff --git a/tests/python/core/cloudxr/test_oob_teleop_adb.py b/tests/python/core/cloudxr/test_oob_teleop_adb.py index efb131b5af..8059dcd6ba 100644 --- a/tests/python/core/cloudxr/test_oob_teleop_adb.py +++ b/tests/python/core/cloudxr/test_oob_teleop_adb.py @@ -1711,3 +1711,253 @@ async def test_monitor_without_callback_still_logs_and_keeps_watching() -> None: task.cancel() with pytest.raises(asyncio.CancelledError): await task + + +# ============================================================================ +# run_oob_connect(): real-browser integration test (see +# /oob-real-browser-integration-test-plan.md at the repo root). A real +# Chromium runs the real webxr_client (real IWER-emulated WebXR, real CDP), +# served from a real static production build (never a dev-server — see +# static_webxr_build()'s docstring for why HMR is structurally unsafe for +# this), behind a real wss.py proxy (real OOBControlHub). Only adb itself is +# faked (no physical Android device), via RealBrowserAdb: its `am start` and +# `adb forward` drive the real Chromium instead of returning canned output. +# +# ALWAYS the MockCloudXR build (`build:app-mock`), never the real +# `@nvidia/cloudxr` SDK: the real SDK genuinely tries to stream against +# whatever's on `backend_port`, which is unpredictable and uncontrollable in +# a test (see the plan's §0/§3 history — this was tried and abandoned). +# MockCloudXR gives the same real App.tsx/CloudXRComponent UI (same +# #startButton/#errorMessageBox DOM) with a fully deterministic, externally +# controllable session underneath (window.__mockCloudXRFail(), see +# cdp_evaluate()), and never opens a socket of its own — no +# minimal_mock_runtime() stand-in needed for CloudXR signaling at all. +# +# This replaces the fully-synthetic mock_adb()/fake-CDP-server +# run_oob_connect() tests from PR #1148 — those proved run_oob_connect()'s +# own orchestration logic against scripted responses; this proves the same +# code path against the real client it actually launches. +# +# Slow (real browser + a real webpack production build + real wss.py) and +# requires a system Chrome (PLAYWRIGHT_CHROME_PATH override; defaults to +# /usr/bin/google-chrome) and Node/npm on PATH — run manually for now; not +# yet wired into CI. +# ============================================================================ + +import contextlib # noqa: E402 +from pathlib import Path # noqa: E402 + +from conftest import ( # noqa: E402 + cdp_evaluate, + real_browser_adb, + real_browser_env, + real_chrome, + real_wss_proxy, + static_webxr_build, +) + + +async def test_run_oob_connect_real_browser_end_to_end( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The real run_oob_connect(), driven against a real Chromium running the real + webxr client (MockCloudXR underneath — see module docstring): device online -> + stale-tab scan -> am start opens a real tab -> real devtools socket already + reachable (Chrome's own --remote-debugging-port equals _CDP_LOCAL_PORT, so adb + forward is a real no-op) -> real CDP tab discovery finds the real tab -> real + _cdp_session_click_connect polls the real client's readiness state machine + (real IWER capability checks) and clicks the real CONNECT button. + """ + with ( + real_browser_env(tmp_path, monkeypatch) as env, + static_webxr_build(env.client_port), + real_chrome(_CDP_LOCAL_PORT, user_data_dir=env.user_data_dir) as chrome, + real_browser_adb(chrome) as fake_adb, + ): + async with real_wss_proxy( + env.install_dir, proxy_port=env.wss_port, backend_port=env.backend_port + ): + task = await run_oob_connect( + resolved_port=env.wss_port, timeout=30.0, usb_local=True + ) + + # A real task, not None: the click genuinely succeeded and + # _monitor_teleop_error_banner is genuinely running against the real + # client's real tab, not swallowed as "click phase ok, monitor + # failed to spawn" (see run_oob_connect()'s own return-value contract). + assert isinstance(task, asyncio.Task) + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + + assert any(c[:2] == ["adb", "get-state"] for c in fake_adb.calls) + assert any("am start" in " ".join(c) for c in fake_adb.calls) + + +async def test_run_oob_connect_real_mock_cxr_crash_surfaces_error_banner( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The real run_oob_connect() driven against the real App.tsx/CloudXRComponent + UI with MockCloudXR swapped in for @nvidia/cloudxr (``npm run + build:app-mock``, ``tests/mock/cloudxr-mock-alias.ts``, static build, no + dev-server — see :func:`static_webxr_build`): identical + ``#startButton``/``#errorMessageBox`` DOM to production, but + ``window.__mockCloudXRFail()`` lets this test force a deterministic + mid-session failure instead of guessing at how a real backend failure + would surface over the wire. + + MockCloudXR never opens a socket of its own (zero ``WebSocket`` usage in + ``tests/mock/MockCloudXR.ts``), so nothing needs to stand in for a CloudXR + runtime here; ``wss.py`` + ``OOBControlHub`` still run for real so the URL + ``run_oob_connect()`` builds is a genuine ``wss://.../oob/v1/ws`` endpoint, + matching what a real launch produces. + + Proves the crash-trigger mechanism itself — a real, real-DOM error banner + appears on demand — independent of whether anything recovers from it; see + :func:`test_run_oob_connect_real_mock_cxr_crash_triggers_real_relaunch` + for the recovery assertion. + """ + with ( + real_browser_env(tmp_path, monkeypatch) as env, + static_webxr_build(env.client_port), + real_chrome(_CDP_LOCAL_PORT, user_data_dir=env.user_data_dir) as chrome, + real_browser_adb(chrome), + ): + async with real_wss_proxy( + env.install_dir, proxy_port=env.wss_port, backend_port=env.backend_port + ): + task = await run_oob_connect( + resolved_port=env.wss_port, timeout=30.0, usb_local=True + ) + assert isinstance(task, asyncio.Task) + + teleop_tabs = [ + t for t in chrome.list_tabs() if "oobEnable=" in (t.get("url") or "") + ] + assert len(teleop_tabs) == 1 + ws_url = teleop_tabs[0]["webSocketDebuggerUrl"] + + # code in the 0xc0f22300-0xc0f223ff (server-disconnect) range: a + # code-less failure is "recoverable" per + # helpers/streamingErrorClassification.ts's isRecoverable(), and + # CloudXRComponent.tsx's onStreamStopped auto-reconnects on those + # without ever showing #errorMessageBox at all. + # + # Retried inside the poll loop, not fired once beforehand: + # triggerFailure() is a no-op unless MockCloudXR's session is + # already Connecting/Connected (see MockCloudXR.ts), and + # run_oob_connect() returns as soon as the button text changes - + # which happens on WebXR session entry, near-instantly, likely + # before CloudXR.createSession()'s own async chain + # (applyTargetFrameRate/requestReferenceSpace) has finished. A + # call before the session exists silently does nothing, so retry + # until the banner actually appears rather than firing once and + # hoping the timing lines up. Idempotent once it does land: a + # session already in Error state fails the same Connecting/ + # Connected guard, so later retries no-op harmlessly. + # 5s is ample margin above the ~1s this actually takes once the + # session exists (observed) - MockCloudXR's own transitions are + # near-instant, unlike the relaunch test below which waits on + # real am start + tab discovery. + banner = "" + deadline = asyncio.get_running_loop().time() + 5.0 + while asyncio.get_running_loop().time() < deadline: + await cdp_evaluate( + ws_url, + "window.__mockCloudXRFail('simulated crash', 0xc0f22300)", + ) + r = await cdp_evaluate( + ws_url, + """(function() { + const box = document.getElementById('errorMessageBox'); + if (!box || !box.classList.contains('show')) return ''; + if (box.classList.contains('success') || + box.classList.contains('info')) return ''; + return document.getElementById('errorMessageText') + ?.textContent?.trim() || ''; + })()""", + ) + banner = (r.get("result") or {}).get("value") or "" + if banner: + break + await asyncio.sleep(0.5) + + assert "simulated crash" in banner + + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + + +async def test_run_oob_connect_real_mock_cxr_crash_triggers_real_relaunch( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A real client-side terminal error should trigger a real auto-relaunch: + a second real ``am start``, a second real tab, clicked back to CONNECT-ready. + + Exercises the repair-capable ``_monitor_teleop_error_banner`` added by + gmorgan/oob-error-relaunch (#1146): a genuine crash recovered by genuine + ADB+CDP automation, not a scripted stand-in for either. Asserts on + black-box observables only (a second ``am start``/tab), not on any + private function, so it stays correct across internal refactors. + """ + with ( + real_browser_env(tmp_path, monkeypatch) as env, + static_webxr_build(env.client_port), + real_chrome(_CDP_LOCAL_PORT, user_data_dir=env.user_data_dir) as chrome, + real_browser_adb(chrome) as fake_adb, + ): + async with real_wss_proxy( + env.install_dir, proxy_port=env.wss_port, backend_port=env.backend_port + ): + task = await run_oob_connect( + resolved_port=env.wss_port, timeout=30.0, usb_local=True + ) + assert isinstance(task, asyncio.Task) + + teleop_tabs_before = [ + t for t in chrome.list_tabs() if "oobEnable=" in (t.get("url") or "") + ] + assert len(teleop_tabs_before) == 1 + ws_url = teleop_tabs_before[0]["webSocketDebuggerUrl"] + am_start_calls_before = sum( + 1 for c in fake_adb.calls if "am start" in " ".join(c) + ) + + # Same retry-until-it-lands approach as the crash-banner test above: + # triggerFailure() no-ops until MockCloudXR's session actually exists. + # 15s: a relaunch has no cold-launch devtools-socket wait (browser's + # already running) - just am start + tab discovery + + # click, which the initial launch above already does in ~5s. + relaunched = False + deadline = asyncio.get_running_loop().time() + 15.0 + while asyncio.get_running_loop().time() < deadline: + await cdp_evaluate( + ws_url, + "window.__mockCloudXRFail('simulated crash', 0xc0f22300)", + ) + am_start_calls_now = sum( + 1 for c in fake_adb.calls if "am start" in " ".join(c) + ) + teleop_tabs_now = [ + t + for t in chrome.list_tabs() + if "oobEnable=" in (t.get("url") or "") + ] + if ( + am_start_calls_now > am_start_calls_before + and len(teleop_tabs_now) >= 2 + ): + relaunched = True + break + await asyncio.sleep(0.5) + + assert relaunched, ( + "expected a second real `am start` and a second real tab after " + "a terminal client error - the monitor should relaunch, not just " + "log the banner" + ) + + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task